From 8d82ee362c7c5079acc95cf28d6128ce9f6bcffd Mon Sep 17 00:00:00 2001 From: Dnomd343 Date: Mon, 16 Jan 2023 17:18:31 +0800 Subject: [PATCH] update: apply new fast short codec --- src/main.cc | 38 ++-- src/short_code/convert.cc | 187 +++++++++++++++----- src/short_code/data_loader.cc | 100 +---------- src/short_code/offset/all_cases_offset.h | 2 +- src/short_code/offset/basic_ranges_offset.h | 2 +- src/short_code/offset/range_prefix_offset.h | 2 +- src/short_code/short_code.h | 22 ++- src/short_code/short_code_mark.h | 96 +++++----- 8 files changed, 247 insertions(+), 202 deletions(-) diff --git a/src/main.cc b/src/main.cc index b5da5fd..3869f8d 100644 --- a/src/main.cc +++ b/src/main.cc @@ -174,8 +174,15 @@ int main() { // } // } -// ShortCode::speed_up(ShortCode::FAST); -// AllCases::build(); + ShortCode::speed_up(ShortCode::FAST); + AllCases::build(); + + std::vector all_cases; + for (uint64_t head = 0; head < 16; ++head) { + for (const auto &range : BasicRanges::fetch()) { + all_cases.emplace_back(head << 32 | range); + } + } // std::cout << "start benchmark" << std::endl; auto start_time = clock(); @@ -254,21 +261,24 @@ int main() { // printf("%08X\n", range); // } - AllCases::build(); - for (auto head = 0; head < 16; ++head) { - uint64_t prefix = (uint64_t)head << 32; - for (const auto &range : AllCases::fetch()[head]) { - printf("%09lX\n", prefix | range); - } - } +// AllCases::build(); +// for (auto head = 0; head < 16; ++head) { +// uint64_t prefix = (uint64_t)head << 32; +// for (const auto &range : AllCases::fetch()[head]) { +// printf("%09lX\n", prefix | range); +// } +// } -// for (uint32_t i = 0; i < 29334498; ++i) { + for (uint32_t i = 0; i < 29334498; ++i) { // if (ShortCode::fast_encode(ShortCode::fast_decode(i)) != i) { // std::cout << "error" << std::endl; // } -// } +// ShortCode::fast_encode_legacy(i); +// ShortCode::fast_decode(i); + ShortCode::fast_encode(all_cases[i]); + } -// printf("%09lX\n", ShortCode::tiny_decode_demo(14323231)); +// printf("%09lX\n", ShortCode::tiny_decode(14323231)); // std::cout << ShortCode::tiny_encode_demo(0x6EC0F8800) << std::endl; // auto br = BasicRanges::fetch(); @@ -302,8 +312,8 @@ int main() { // printf("%09lX\n", ShortCode::fast_decode(14323231)); // std::cout << ShortCode::fast_encode(0x6EC0F8800) << std::endl; - std::cerr << (clock() - start_time) * 1000 / CLOCKS_PER_SEC << "ms" << std::endl; -// std::cerr << (clock() - start_time) * 1000000 / CLOCKS_PER_SEC << "us" << std::endl; +// std::cerr << (clock() - start_time) * 1000 / CLOCKS_PER_SEC << "ms" << std::endl; + std::cerr << (clock() - start_time) * 1000000 / CLOCKS_PER_SEC << "us" << std::endl; // std::cout << "complete benchmark" << std::endl; // pause(); diff --git a/src/short_code/convert.cc b/src/short_code/convert.cc index 2f9aa1b..eb931a8 100644 --- a/src/short_code/convert.cc +++ b/src/short_code/convert.cc @@ -1,67 +1,174 @@ +#include #include "common.h" #include "short_code.h" #include "basic_ranges.h" -#include "short_code_mark.h" + +//#include "short_code_mark.h" CommonCode ShortCode::to_common_code() const { // convert to common code if (ShortCode::check_mode() == ShortCode::NORMAL) { - return CommonCode::unsafe_create(tiny_decode(code)); // using normal mode + return CommonCode::unsafe_create(tiny_decode(code)); // normal mode } - return CommonCode::unsafe_create(all_cases_list[code]); // using fast mode + return CommonCode::unsafe_create(fast_decode(code)); // fast mode } ShortCode::ShortCode(const CommonCode &common_code) { // convert from common code if (ShortCode::check_mode() == ShortCode::NORMAL) { - code = tiny_encode(common_code.unwrap()); // using normal mode + code = tiny_encode(common_code.unwrap()); // normal mode } else { - code = all_cases_dict[common_code.unwrap()]; // using fast mode + code = fast_encode(common_code.unwrap()); // fast mode } } -/// ensure that input common code is valid -uint32_t ShortCode::tiny_encode(uint64_t common_code) { // common_code --low-memory--> short_code - uint32_t offset = 0; - uint32_t head = common_code >> 32; // common code head - uint32_t prefix = (common_code >> 24) & 0xFF; // common code range prefix - auto target = Common::range_reverse((uint32_t)common_code); // target range +#include "all_cases.h" +#include "all_cases_offset.h" +#include "basic_ranges_offset.h" +#include "range_prefix_offset.h" - for (int index = 0; index < BASIC_RANGES_INDEX[prefix]; ++index) { // traverse basic ranges - uint32_t range = BasicRanges::fetch()[index + BASIC_RANGES_OFFSET[prefix]]; - if (range == target) { - break; // found target range - } - if (Common::check_case(head, range)) { // search for valid cases - ++offset; // record sub offset - } - } - return ALL_CASES_OFFSET[head] + RANGE_PREFIX_OFFSET[head][prefix] + offset; +uint64_t ShortCode::fast_decode(uint32_t short_code) { // short code => common code + auto offset = std::upper_bound( // using binary search + ALL_CASES_OFFSET, ALL_CASES_OFFSET + 16, short_code + ) - 1; + uint64_t head = offset - ALL_CASES_OFFSET; // head index + return (head << 32) | AllCases::fetch()[head][short_code - *offset]; // release common code +} + +uint32_t ShortCode::fast_encode(uint64_t common_code) { // common code => short code + auto head = common_code >> 32; // head index + const auto &ranges = AllCases::fetch()[head]; // available ranges + auto offset = std::lower_bound(ranges.begin(), ranges.end(), (uint32_t)common_code) - ranges.begin(); + return ALL_CASES_OFFSET[head] + offset; // release short code } +/// ensure that input common code is valid +//uint32_t ShortCode::tiny_encode(uint64_t common_code) { // common_code --low-memory--> short_code +// uint32_t offset = 0; +// uint32_t head = common_code >> 32; // common code head +// uint32_t prefix = (common_code >> 24) & 0xFF; // common code range prefix +// auto target = Common::range_reverse((uint32_t)common_code); // target range +// +// for (int index = 0; index < BASIC_RANGES_INDEX[prefix]; ++index) { // traverse basic ranges +// uint32_t range = BasicRanges::fetch()[index + BASIC_RANGES_OFFSET[prefix]]; +// if (range == target) { +// break; // found target range +// } +// if (Common::check_case(head, range)) { // search for valid cases +// ++offset; // record sub offset +// } +// } +// return ALL_CASES_OFFSET[head] + RANGE_PREFIX_OFFSET[head][prefix] + offset; +//} + /// ensure that input short code is valid -uint64_t ShortCode::tiny_decode(uint32_t short_code) { // short_code --low-memory--> common_code - uint32_t head = 0, prefix = 0; - for (; head < 16; ++head) { - if (short_code < ALL_CASES_INDEX[head]) { // match head - break; - } - short_code -= ALL_CASES_INDEX[head]; // short code approximate - } - for (; prefix < 256; ++prefix) { - if (short_code < RANGE_PREFIX_INDEX[head][prefix]) { // match range prefix - break; +//uint64_t ShortCode::tiny_decode(uint32_t short_code) { // short_code --low-memory--> common_code +// uint32_t head = 0, prefix = 0; +// for (; head < 16; ++head) { +// if (short_code < ALL_CASES_INDEX[head]) { // match head +// break; +// } +// short_code -= ALL_CASES_INDEX[head]; // short code approximate +// } +// for (; prefix < 256; ++prefix) { +// if (short_code < RANGE_PREFIX_INDEX[head][prefix]) { // match range prefix +// break; +// } +// short_code -= RANGE_PREFIX_INDEX[head][prefix]; // short code approximate +// } +// +// uint32_t range; +// for (int index = 0; index < BASIC_RANGES_INDEX[prefix]; ++index) { // traverse basic ranges +// range = BasicRanges::fetch()[index + BASIC_RANGES_OFFSET[prefix]]; +// if (Common::check_case(head, range)) { // search for valid cases +// if (short_code == 0) { +// break; // found target range +// } +// --short_code; // short code approximate +// } +// } +// return (uint64_t)head << 32 | Common::range_reverse(range); // release common code +//} + +// TODO: try to using 10-bits range prefix -> less static data + +uint64_t ShortCode::tiny_decode(uint32_t short_code) { + /// match head index + auto offset = std::upper_bound( // binary search + ALL_CASES_OFFSET, ALL_CASES_OFFSET + 16, short_code + ) - 1; + uint64_t head = offset - ALL_CASES_OFFSET; // head index + short_code -= *offset; + + /// match range prefix + offset = std::upper_bound( // binary search + RANGE_PREFIX_OFFSET[head], RANGE_PREFIX_OFFSET[head] + 4096, short_code + ) - 1; + uint32_t prefix = offset - RANGE_PREFIX_OFFSET[head]; // range prefix + short_code -= *offset; + + /// search target range + const auto &basic_ranges = BasicRanges::fetch(); + for (auto index = BASIC_RANGES_OFFSET[prefix]; index < basic_ranges.size(); ++index) { + + uint32_t range = basic_ranges[index]; // traverse basic ranges + + uint32_t broken = Common::check_range(head, basic_ranges[index]); // check and get broken address + + auto range_rev = Common::range_reverse(basic_ranges[index]); // reversed range + + if (broken) { // invalid case + auto delta = (uint32_t)1 << (32 - broken * 2); // this --delta--> next possible range + auto next_min = (range_rev & ~(delta - 1)) + delta; + while (Common::range_reverse(basic_ranges[++index]) < next_min); // located next range + --index; + } else { + + if (!short_code--) { // short code approximate + + /// found target range + return head << 32 | range_rev; + + } + } - short_code -= RANGE_PREFIX_INDEX[head][prefix]; // short code approximate + } - uint32_t range; - for (int index = 0; index < BASIC_RANGES_INDEX[prefix]; ++index) { // traverse basic ranges - range = BasicRanges::fetch()[index + BASIC_RANGES_OFFSET[prefix]]; + return 0; // never reach when input valid +} + +uint32_t ShortCode::tiny_encode(uint64_t common_code) { + +// printf("%09lX\n", common_code); + + uint32_t head = common_code >> 32; + uint32_t prefix = (common_code >> 20) & 0xFFF; + +// printf("head = %d\n", head); +// printf("prefix = %X\n", prefix); + + uint32_t offset = 0; + auto target = Common::range_reverse((uint32_t)common_code); // target range + + const auto &basic_ranges = BasicRanges::fetch(); + + for (auto index = BASIC_RANGES_OFFSET[prefix]; index < basic_ranges.size(); ++index) { // traverse basic ranges + + uint32_t range = basic_ranges[index]; + if (Common::check_case(head, range)) { // search for valid cases - if (short_code == 0) { - break; // found target range + + if (range == target) { // found target range + return ALL_CASES_OFFSET[head] + RANGE_PREFIX_OFFSET[head][prefix] + offset; + } - --short_code; // short code approximate + + ++offset; // record sub offset + } + } - return (uint64_t)head << 32 | Common::range_reverse(range); // release common code + + + return 0; } + diff --git a/src/short_code/data_loader.cc b/src/short_code/data_loader.cc index 6b0d9da..6970e2b 100644 --- a/src/short_code/data_loader.cc +++ b/src/short_code/data_loader.cc @@ -16,6 +16,10 @@ void ShortCode::speed_up(ShortCode::Mode mode) { } if (mode == ShortCode::FAST) { // build fast mode data build_mappings(); + + // TODO: confirm AllCases data available + AllCases::build(); + } else if (mode == ShortCode::NORMAL && !normal_mode_available) { // build normal mode data BasicRanges::build(); // blocking function normal_mode_available = true; @@ -52,20 +56,11 @@ void ShortCode::build_mappings() { // build fast search mappings map_building.unlock(); } -#include - -uint64_t ShortCode::fast_decode(uint32_t short_code) { - auto offset = std::upper_bound(ALL_CASES_OFFSET, ALL_CASES_OFFSET + 16, short_code) - 1; // binary search - uint64_t head = offset - ALL_CASES_OFFSET; // head index - return (head << 32) | AllCases::fetch()[head][short_code - *offset]; // release common code +uint32_t ShortCode::fast_encode_legacy(uint64_t common_code) { + return all_cases_dict[common_code]; } -uint32_t ShortCode::fast_encode(uint64_t common_code) { - auto head = common_code >> 32; // head index - const auto &ranges = AllCases::fetch()[head]; // available ranges - auto offset = std::lower_bound(ranges.begin(), ranges.end(), (uint32_t)common_code) - ranges.begin(); - return ALL_CASES_OFFSET[head] + offset; // release short code -} +#include #include "basic_ranges_offset.h" #include "range_prefix_offset.h" @@ -73,84 +68,3 @@ uint32_t ShortCode::fast_encode(uint64_t common_code) { #include #include "common.h" -uint64_t ShortCode::tiny_decode_demo(uint32_t short_code) { - /// match head index - auto offset = std::upper_bound( // binary search - ALL_CASES_OFFSET, ALL_CASES_OFFSET + 16, short_code - ) - 1; - uint64_t head = offset - ALL_CASES_OFFSET; // head index - short_code -= *offset; - - /// match range prefix - offset = std::upper_bound( // binary search - RANGE_PREFIX_OFFSET_[head], RANGE_PREFIX_OFFSET_[head] + 4096, short_code - ) - 1; - uint32_t prefix = offset - RANGE_PREFIX_OFFSET_[head]; // range prefix - short_code -= *offset; - - /// search target range - const auto &basic_ranges = BasicRanges::fetch(); - for (auto index = BASIC_RANGES_OFFSET_[prefix]; index < basic_ranges.size(); ++index) { - - uint32_t range = basic_ranges[index]; // traverse basic ranges - - uint32_t broken = Common::check_range(head, basic_ranges[index]); // check and get broken address - - auto range_rev = Common::range_reverse(basic_ranges[index]); // reversed range - - if (broken) { // invalid case - auto delta = (uint32_t)1 << (32 - broken * 2); // this --delta--> next possible range - auto next_min = (range_rev & ~(delta - 1)) + delta; - while (Common::range_reverse(basic_ranges[++index]) < next_min); // located next range - --index; - } else { - - if (!short_code--) { // short code approximate - - /// found target range - return head << 32 | range_rev; - - } - - } - - } - - return 0; // never reach when input valid -} - -uint32_t ShortCode::tiny_encode_demo(uint64_t common_code) { - -// printf("%09lX\n", common_code); - - uint32_t head = common_code >> 32; - uint32_t prefix = (common_code >> 20) & 0xFFF; - -// printf("head = %d\n", head); -// printf("prefix = %X\n", prefix); - - uint32_t offset = 0; - auto target = Common::range_reverse((uint32_t)common_code); // target range - - const auto &basic_ranges = BasicRanges::fetch(); - - for (auto index = BASIC_RANGES_OFFSET_[prefix]; index < basic_ranges.size(); ++index) { // traverse basic ranges - - uint32_t range = basic_ranges[index]; - - if (Common::check_case(head, range)) { // search for valid cases - - if (range == target) { // found target range - return ALL_CASES_OFFSET[head] + RANGE_PREFIX_OFFSET_[head][prefix] + offset; - - } - - ++offset; // record sub offset - - } - - } - - - return 0; -} diff --git a/src/short_code/offset/all_cases_offset.h b/src/short_code/offset/all_cases_offset.h index 442e692..ac7feb4 100644 --- a/src/short_code/offset/all_cases_offset.h +++ b/src/short_code/offset/all_cases_offset.h @@ -2,7 +2,7 @@ #include -const uint32_t ALL_CASES_OFFSET_[16] = { +const uint32_t ALL_CASES_OFFSET[16] = { 0, 2942906, 5203298, 8146204, 8146204, 10468254, 12345199, 14667249, 14667249, 16989299, 18866244, 21188294, diff --git a/src/short_code/offset/basic_ranges_offset.h b/src/short_code/offset/basic_ranges_offset.h index bb711b8..cc75586 100644 --- a/src/short_code/offset/basic_ranges_offset.h +++ b/src/short_code/offset/basic_ranges_offset.h @@ -2,7 +2,7 @@ #include -const uint32_t BASIC_RANGES_OFFSET_[4096] = { +const uint32_t BASIC_RANGES_OFFSET[4096] = { 0, 18272, 24960, 31648, 49920, 56608, 59056, 61504, 68192, 74880, 77328, 79776, 86464, 104736, 111424, 118112, 136384, 143072, 145520, 147968, 154656, 157104, 158000, 158896, diff --git a/src/short_code/offset/range_prefix_offset.h b/src/short_code/offset/range_prefix_offset.h index e9f3183..d45e17d 100644 --- a/src/short_code/offset/range_prefix_offset.h +++ b/src/short_code/offset/range_prefix_offset.h @@ -2,7 +2,7 @@ #include -const uint32_t RANGE_PREFIX_OFFSET_[16][4096] = { +const uint32_t RANGE_PREFIX_OFFSET[16][4096] = { /// --------------------------------- 0x0 --------------------------------- 0, 9527, 12991, 16067, 25594, 29058, 30527, 31657, 35121, 38525, 39766, 41041, 44445, 53972, 57436, 60512, diff --git a/src/short_code/short_code.h b/src/short_code/short_code.h index 5e5c92f..3b70d6a 100644 --- a/src/short_code/short_code.h +++ b/src/short_code/short_code.h @@ -21,6 +21,8 @@ public: CommonCode to_common_code() const; static bool check(uint32_t short_code); + // TODO: std::cout << ShortCode(...) << std::endl; + explicit ShortCode(uint32_t short_code); explicit ShortCode(const CommonCode &common_code); explicit ShortCode(const std::string &short_code_str); @@ -34,22 +36,34 @@ public: speed_up(mode); } + // TODO: mark as private after inner test static uint64_t fast_decode(uint32_t short_code); static uint32_t fast_encode(uint64_t common_code); - static uint64_t tiny_decode_demo(uint32_t short_code); - static uint32_t tiny_encode_demo(uint64_t common_code); + static uint64_t tiny_decode(uint32_t short_code); + static uint32_t tiny_encode(uint64_t common_code); + + // TODO: ShortCode::create() / ShortCode::from_str(...) / ShortCode::from_common_code(...) + + static uint32_t fast_encode_legacy(uint64_t common_code); private: uint32_t code; + static std::mutex map_building; static bool fast_mode_available; static bool normal_mode_available; + static const uint32_t SHORT_CODE_LIMIT = 29334498; + + /// NOTE: using binary search instead of global vector and unordered_map + /// for some test, the new function only using < 170MB memory, while the legacy using > 1.5GB + /// but the legacy one is more easy and a little fast. + /// BTW, the new one init less than 1.2s, legacy one need about 15s static std::vector all_cases_list; // short_code -> common_code static std::unordered_map all_cases_dict; // common_code -> short_code static void build_mappings(); - static uint64_t tiny_decode(uint32_t short_code); - static uint32_t tiny_encode(uint64_t common_code); +// static uint64_t tiny_decode(uint32_t short_code); +// static uint32_t tiny_encode(uint64_t common_code); }; diff --git a/src/short_code/short_code_mark.h b/src/short_code/short_code_mark.h index 7070e25..fd51b07 100644 --- a/src/short_code/short_code_mark.h +++ b/src/short_code/short_code_mark.h @@ -2,56 +2,56 @@ #include -const uint32_t ALL_CASES_INDEX[16] = { - 2942906, 2260392, 2942906, 0, - 2322050, 1876945, 2322050, 0, - 2322050, 1876945, 2322050, 0, - 2942906, 2260392, 2942906, 0, -}; +//const uint32_t ALL_CASES_INDEX[16] = { +// 2942906, 2260392, 2942906, 0, +// 2322050, 1876945, 2322050, 0, +// 2322050, 1876945, 2322050, 0, +// 2942906, 2260392, 2942906, 0, +//}; -const uint32_t ALL_CASES_OFFSET[16] = { - 0, 2942906, 5203298, 8146204, - 8146204, 10468254, 12345199, 14667249, - 14667249, 16989299, 18866244, 21188294, - 21188294, 24131200, 26391592, 29334498, -}; +//const uint32_t ALL_CASES_OFFSET[16] = { +// 0, 2942906, 5203298, 8146204, +// 8146204, 10468254, 12345199, 14667249, +// 14667249, 16989299, 18866244, 21188294, +// 21188294, 24131200, 26391592, 29334498, +//}; -const uint32_t BASIC_RANGES_INDEX[256] = { - 136384, 49920, 49920, 136384, 49920, 18272, 18272, 49920, - 49920, 18272, 18272, 49920, 136384, 49920, 49920, 136384, - 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, - 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, - 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, - 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, - 136384, 49920, 49920, 136384, 49920, 17589, 17589, 48555, - 49920, 17589, 17589, 48555, 136384, 48555, 48555, 133653, - 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, - 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, - 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, - 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, - 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, - 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, - 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, - 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, - 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, - 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, - 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, - 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, - 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, - 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, - 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, - 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, - 136384, 49920, 49920, 136384, 49920, 17589, 17589, 48555, - 49920, 17589, 17589, 48555, 136384, 48555, 48555, 133653, - 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, - 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, - 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, - 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, - 136384, 48555, 48555, 133653, 48555, 15163, 15163, 43244, - 48555, 15163, 15163, 43244, 133653, 43244, 43244, 122125, -}; +//const uint32_t BASIC_RANGES_INDEX[256] = { +// 136384, 49920, 49920, 136384, 49920, 18272, 18272, 49920, +// 49920, 18272, 18272, 49920, 136384, 49920, 49920, 136384, +// 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, +// 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, +// 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, +// 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, +// 136384, 49920, 49920, 136384, 49920, 17589, 17589, 48555, +// 49920, 17589, 17589, 48555, 136384, 48555, 48555, 133653, +// 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, +// 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, +// 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, +// 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, +// 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, +// 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, +// 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, +// 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, +// 49920, 18272, 18272, 49920, 18272, 6347, 6347, 17589, +// 18272, 6347, 6347, 17589, 49920, 17589, 17589, 48555, +// 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, +// 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, +// 18272, 6347, 6347, 17589, 6347, 1785, 1785, 5246, +// 6347, 1785, 1785, 5246, 17589, 5246, 5246, 15163, +// 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, +// 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, +// 136384, 49920, 49920, 136384, 49920, 17589, 17589, 48555, +// 49920, 17589, 17589, 48555, 136384, 48555, 48555, 133653, +// 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, +// 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, +// 49920, 17589, 17589, 48555, 17589, 5246, 5246, 15163, +// 17589, 5246, 5246, 15163, 48555, 15163, 15163, 43244, +// 136384, 48555, 48555, 133653, 48555, 15163, 15163, 43244, +// 48555, 15163, 15163, 43244, 133653, 43244, 43244, 122125, +//}; -const uint32_t BASIC_RANGES_OFFSET[256] = { +const uint32_t BASIC_RANGES_OFFSET_8bits[256] = { 0, 136384, 186304, 236224, 372608, 422528, 440800, 459072, 508992, 558912, 577184, 595456, 645376, 781760, 831680, 881600, 1017984, 1067904, 1086176, 1104448, 1154368, 1172640, 1178987, 1185334, @@ -633,7 +633,7 @@ const uint32_t RANGE_PREFIX_INDEX[16][256] = { }, }; -const uint32_t RANGE_PREFIX_OFFSET[16][256] = { +const uint32_t RANGE_PREFIX_OFFSET_8bits[16][256] = { { 0, 70039, 70039, 95633, 165672, 191266, 200793, 210117, 235711, 258169, 258169, 267696, 290154, 360193, 360193, 385787,