Browse Source

feat: try lambda in ShortCode tiny codec

master
Dnomd343 2 years ago
parent
commit
48a6d17fa8
  1. 77
      src/main.cc
  2. 163
      src/short_code/convert.cc
  3. 2
      src/short_code/data_loader.cc
  4. 1181
      src/short_code/short_code_mark.h

77
src/main.cc

@ -178,13 +178,13 @@ int main() {
// AllCases::build();
BasicRanges::build();
std::vector<uint64_t> all_cases;
for (uint64_t head = 0; head < 16; ++head) {
for (const auto &range : AllCases::fetch()[head]) {
all_cases.emplace_back(head << 32 | range);
}
}
std::cout << "test data size: " << all_cases.size() << std::endl;
// std::vector<uint64_t> all_cases;
// for (uint64_t head = 0; head < 16; ++head) {
// for (const auto &range : AllCases::fetch()[head]) {
// all_cases.emplace_back(head << 32 | range);
// }
// }
// std::cout << "test data size: " << all_cases.size() << std::endl;
// std::cout << "start benchmark" << std::endl;
auto start_time = clock();
@ -271,67 +271,12 @@ int main() {
// }
// }
// for (uint32_t i = 0; i < 29334498; ++i) {
// if (ShortCode::fast_encode(ShortCode::fast_decode(i)) != i) {
// std::cout << "error" << std::endl;
// }
// ShortCode::fast_encode_legacy(i);
// ShortCode::fast_decode(i);
// ShortCode::fast_encode(all_cases[i]);
// }
// for (uint32_t i = 1000000; i < 1100000; ++i) {
// ShortCode::tiny_decode(i);
// ShortCode::tiny_decode_10b(i);
// ShortCode::tiny_encode(all_cases[i]);
// ShortCode::tiny_encode_10b(all_cases[i]);
// }
for (uint32_t i = 1000000; i < 1010000; ++i) {
ShortCode::tiny_decode(i);
}
// printf("%09lX\n", ShortCode::tiny_decode(14323231));
// std::cout << ShortCode::tiny_encode_demo(0x6EC0F8800) << std::endl;
// printf("%09lX\n", ShortCode::tiny_decode_10b(14323231));
// std::cout << ShortCode::tiny_encode_10b(0x6EC0F8800) << std::endl;
// auto br = BasicRanges::fetch();
// for (auto &range : br) {
// range = Common::range_reverse(range);
// }
/// 1017983: 0FFFFFFF
/// 1017984: 10000000
/// 1017985: 1000000C
// int sum = 0;
// printf(" ");
// for (uint32_t prefix = 0; prefix < 0x400; ++prefix) {
// printf("%7td, ", std::lower_bound(br.begin(), br.end(), prefix << 22) - br.begin());
// if (sum++ % 8 == 7) {
// printf("\n ");
// }
// }
// const uint32_t TEST[3][4] = {
// {1,1,1,1}, {2,2,}, {3,3,3,}
// };
// for (auto &x : TEST) {
// for (auto &y: x) {
// std::cout << y << std::endl;
// }
// }
// int sum = 0;
// for (int head = 0; head < 16; ++head) {
// auto &a = AllCases::fetch()[head];
// printf("\n/// --------------------------------- 0x%X ---------------------------------\n ", head);
// printf("\n/// --------------------------------------------------------------------- 0x%X ---------------------------------------------------------------------\n ", head);
// for (uint32_t prefix = 0; prefix < 0x1000; ++prefix) {
// printf("%7td, ", std::lower_bound(a.begin(), a.end(), prefix << 20) - a.begin());
// if (sum++ % 16 == 15 and prefix != 0xFFF) {
// printf("\n ");
// }
// }
// }
// printf("\n");
// std::cout << ShortCode::tiny_encode(0x6EC0F8800) << std::endl;
// printf("%09lX\n", ShortCode::fast_decode(14323231));
// std::cout << ShortCode::fast_encode(0x6EC0F8800) << std::endl;

163
src/short_code/convert.cc

@ -1,9 +1,13 @@
#include <algorithm>
#include <functional>
#include "common.h"
#include "short_code.h"
#include "basic_ranges.h"
//#include "short_code_mark.h"
#include "all_cases.h"
#include "all_cases_offset.h"
#include "basic_ranges_offset.h"
#include "range_prefix_offset.h"
CommonCode ShortCode::to_common_code() const { // convert to common code
if (ShortCode::check_mode() == ShortCode::NORMAL) {
@ -20,12 +24,7 @@ ShortCode::ShortCode(const CommonCode &common_code) { // convert from common cod
}
}
#include "all_cases.h"
#include "all_cases_offset.h"
#include "basic_ranges_offset.h"
#include "range_prefix_offset.h"
uint64_t ShortCode::fast_decode(uint32_t short_code) { // short code => common code
uint64_t ShortCode::fast_decode(uint32_t short_code) { // short code --> common code
auto offset = std::upper_bound( // using binary search
ALL_CASES_OFFSET, ALL_CASES_OFFSET + 16, short_code
) - 1;
@ -33,114 +32,120 @@ uint64_t ShortCode::fast_decode(uint32_t short_code) { // short code => common c
return (head << 32) | AllCases::fetch()[head][short_code - *offset]; // release common code
}
uint32_t ShortCode::fast_encode(uint64_t common_code) { // common code => short code
uint32_t ShortCode::fast_encode(uint64_t common_code) { // common code --> short code
auto head = common_code >> 32; // head index
const auto &ranges = AllCases::fetch()[head]; // available ranges
auto offset = std::lower_bound(ranges.begin(), ranges.end(), (uint32_t)common_code) - ranges.begin();
return ALL_CASES_OFFSET[head] + offset; // release short code
}
/// ensure that input common code is valid
//uint32_t ShortCode::tiny_encode(uint64_t common_code) { // common_code --low-memory--> short_code
// uint32_t offset = 0;
// uint32_t head = common_code >> 32; // common code head
// uint32_t prefix = (common_code >> 24) & 0xFF; // common code range prefix
// auto target = Common::range_reverse((uint32_t)common_code); // target range
//
// for (int index = 0; index < BASIC_RANGES_INDEX[prefix]; ++index) { // traverse basic ranges
// uint32_t range = BasicRanges::fetch()[index + BASIC_RANGES_OFFSET[prefix]];
// if (range == target) {
// break; // found target range
// }
// if (Common::check_case(head, range)) { // search for valid cases
// ++offset; // record sub offset
// }
// }
// return ALL_CASES_OFFSET[head] + RANGE_PREFIX_OFFSET[head][prefix] + offset;
//}
/// ensure that input short code is valid
//uint64_t ShortCode::tiny_decode(uint32_t short_code) { // short_code --low-memory--> common_code
// uint32_t head = 0, prefix = 0;
// for (; head < 16; ++head) {
// if (short_code < ALL_CASES_INDEX[head]) { // match head
// break;
// }
// short_code -= ALL_CASES_INDEX[head]; // short code approximate
// }
// for (; prefix < 256; ++prefix) {
// if (short_code < RANGE_PREFIX_INDEX[head][prefix]) { // match range prefix
// break;
// }
// short_code -= RANGE_PREFIX_INDEX[head][prefix]; // short code approximate
// }
//
// uint32_t range;
// for (int index = 0; index < BASIC_RANGES_INDEX[prefix]; ++index) { // traverse basic ranges
// range = BasicRanges::fetch()[index + BASIC_RANGES_OFFSET[prefix]];
// if (Common::check_case(head, range)) { // search for valid cases
// if (short_code == 0) {
// break; // found target range
// }
// --short_code; // short code approximate
// }
// }
// return (uint64_t)head << 32 | Common::range_reverse(range); // release common code
//}
void run(uint32_t head, uint32_t prefix, const std::function<bool(uint32_t)>& f) {
const auto &basic_ranges = BasicRanges::fetch();
for (auto index = BASIC_RANGES_OFFSET[prefix]; index < basic_ranges.size(); ++index) {
auto broken_offset = Common::check_range(head, basic_ranges[index]);
if (broken_offset) { // case invalid
auto delta = (uint32_t)1 << (32 - broken_offset * 2); // delta to next possible range
/// !! -> broken
/// ( xx xx xx ) xx xx xx ... (reversed range)
/// +1 00 00 00 ... (delta)
auto next_min = (Common::range_reverse(basic_ranges[index]) & ~(delta - 1)) + delta;
while (Common::range_reverse(basic_ranges[++index]) < next_min); // located next range
--index;
} else {
if (f(basic_ranges[index])) {
// printf("ret -> %09lX\n", (uint64_t)head << 32 | Common::range_reverse(basic_ranges[index]));
// TODO: try to using 10-bits range prefix -> less static data
return;
}
}
}
}
#include <iostream>
uint64_t ShortCode::tiny_decode(uint32_t short_code) {
/// NOTE: ensure that input short code is valid
uint64_t ShortCode::tiny_decode(uint32_t short_code) { // short code --> common code
/// match head index
auto offset = std::upper_bound( // binary search
ALL_CASES_OFFSET, ALL_CASES_OFFSET + 16, short_code
ALL_CASES_OFFSET, ALL_CASES_OFFSET + 16, short_code
) - 1;
uint64_t head = offset - ALL_CASES_OFFSET; // head index
short_code -= *offset;
/// match range prefix
offset = std::upper_bound( // binary search
RANGE_PREFIX_OFFSET[head], RANGE_PREFIX_OFFSET[head] + 4096, short_code
RANGE_PREFIX_OFFSET[head], RANGE_PREFIX_OFFSET[head] + 4096, short_code
) - 1;
uint32_t prefix = offset - RANGE_PREFIX_OFFSET[head]; // range prefix
short_code -= *offset;
/// search target range
const auto &basic_ranges = BasicRanges::fetch();
for (auto index = BASIC_RANGES_OFFSET[prefix]; index < basic_ranges.size(); ++index) {
uint32_t range = basic_ranges[index]; // traverse basic ranges
// const auto &basic_ranges = BasicRanges::fetch();
// for (auto index = BASIC_RANGES_OFFSET[prefix]; index < basic_ranges.size(); ++index) {
//
// auto broken_offset = Common::check_range(head, basic_ranges[index]);
// if (broken_offset) { // case invalid
// auto delta = (uint32_t)1 << (32 - broken_offset * 2); // delta to next possible range
// /// !! -> broken
// /// ( xx xx xx ) xx xx xx ... (reversed range)
// /// +1 00 00 00 ... (delta)
// auto next_min = (Common::range_reverse(basic_ranges[index]) & ~(delta - 1)) + delta;
// while (Common::range_reverse(basic_ranges[++index]) < next_min); // located next range
// --index;
// } else {
//
// if (!short_code--) { // short code approximate
//
// /// found target range
// return head << 32 | Common::range_reverse(basic_ranges[index]);
//
// }
//
// }
//
//
// }
uint32_t broken = Common::check_range(head, basic_ranges[index]); // check and get broken address
// std::cout << "short code = " << short_code << std::endl;
// return 0;
auto range_rev = Common::range_reverse(basic_ranges[index]); // reversed range
uint32_t range;
if (broken) { // invalid case
auto delta = (uint32_t)1 << (32 - broken * 2); // this --delta--> next possible range
auto next_min = (range_rev & ~(delta - 1)) + delta;
while (Common::range_reverse(basic_ranges[++index]) < next_min); // located next range
--index;
} else {
auto lambda = [&short_code, &range](uint32_t dat) -> bool {
if (!short_code--) { // short code approximate
// return (!short_code--);
// std::cout << "short code = " << short_code << std::endl;
if (!short_code--) { // short code approximate
// std::cout << "short code reach 0" << std::endl;
range = dat;
return true;
/// found target range
return head << 32 | range_rev;
}
// return head << 32 | Common::range_reverse(basic_ranges[index]);
}
return false;
};
run(head, prefix, lambda);
return (uint64_t)head << 32 | Common::range_reverse(range);
printf("ret -> %09lX\n", (uint64_t)head << 32 | Common::range_reverse(range));
}
printf("error\n");
return 0; // never reach when input valid
}
#include <iostream>
uint32_t ShortCode::tiny_encode(uint64_t common_code) {
/// NOTE: ensure that input common code is valid
uint32_t ShortCode::tiny_encode(uint64_t common_code) { // common code --> short code

2
src/short_code/data_loader.cc

@ -1,7 +1,7 @@
#include "all_cases.h"
#include "short_code.h"
#include "basic_ranges.h"
#include "short_code_mark.h"
//#include "short_code_mark.h"
//std::mutex ShortCode::map_building;
bool ShortCode::fast_mode_available = false;

1181
src/short_code/short_code_mark.h

File diff suppressed because it is too large
Loading…
Cancel
Save