25 alignas(64)
constexpr auto B64_ENC_SHUFFLE = []() {
26 std::array<uint8_t, 64> idx = {};
27 for(
size_t g = 0; g != 16; ++g) {
28 idx[4 * g + 0] =
static_cast<uint8_t
>(3 * g + 1);
29 idx[4 * g + 1] =
static_cast<uint8_t
>(3 * g + 0);
30 idx[4 * g + 2] =
static_cast<uint8_t
>(3 * g + 2);
31 idx[4 * g + 3] =
static_cast<uint8_t
>(3 * g + 1);
36 alignas(64)
constexpr char B64_ENC_TABLE[65] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
37 constexpr __mmask64 LOW_48 = 0x0000FFFFFFFFFFFF;
39 const __m512i shuffle = _mm512_load_si512(B64_ENC_SHUFFLE.data());
40 const __m512i lookup = _mm512_load_si512(B64_ENC_TABLE);
42 const __m512i multishift = _mm512_set1_epi64(0x3036242A1016040A);
46 while(length - consumed >= 48) {
47 const __m512i x = _mm512_maskz_loadu_epi8(LOW_48, in + consumed);
49 const __m512i expanded = _mm512_permutexvar_epi8(shuffle, x);
51 const __m512i indices = _mm512_multishift_epi64_epi8(multishift, expanded);
52 const __m512i chars = _mm512_permutexvar_epi8(indices, lookup);
54 _mm512_storeu_si512(out, chars);
68 const size_t todo = std::min<size_t>(64, length - pos);
69 const __mmask64 valid = (todo == 64) ? ~__mmask64(0) : ((__mmask64(1) << todo) - 1);
71 const __m512i x = _mm512_maskz_loadu_epi8(valid, in + pos);
74 _mm512_cmpeq_epi8_mask(x, _mm512_set1_epi8(
' ')) | _mm512_cmpeq_epi8_mask(x, _mm512_set1_epi8(
'\n')) |
75 _mm512_cmpeq_epi8_mask(x, _mm512_set1_epi8(
'\t')) | _mm512_cmpeq_epi8_mask(x, _mm512_set1_epi8(
'\r'));
77 const __mmask64 keep = valid & ~ws;
78 _mm512_mask_compressstoreu_epi8(out + written, keep, x);
79 written +=
static_cast<size_t>(_mm_popcnt_u64(keep));
89 alignas(64)
constexpr auto B64_DEC_TABLE = []() {
90 std::array<uint8_t, 128> tbl = {};
94 for(
size_t i = 0; i != 26; ++i) {
95 tbl[
'A' + i] =
static_cast<uint8_t
>(i);
96 tbl[
'a' + i] =
static_cast<uint8_t
>(26 + i);
98 for(
size_t i = 0; i != 10; ++i) {
99 tbl[
'0' + i] =
static_cast<uint8_t
>(52 + i);
108 alignas(64)
constexpr auto B64_DEC_PACK = []() {
109 std::array<uint8_t, 64> idx = {};
110 for(
size_t g = 0; g != 16; ++g) {
111 idx[3 * g + 0] =
static_cast<uint8_t
>(4 * g + 2);
112 idx[3 * g + 1] =
static_cast<uint8_t
>(4 * g + 1);
113 idx[3 * g + 2] =
static_cast<uint8_t
>(4 * g + 0);
118 constexpr __mmask64 LOW_48 = 0x0000FFFFFFFFFFFF;
120 const __m512i lut_lo = _mm512_load_si512(B64_DEC_TABLE.data());
121 const __m512i lut_hi = _mm512_load_si512(&B64_DEC_TABLE[64]);
122 const __m512i pack_shuffle = _mm512_load_si512(B64_DEC_PACK.data());
126 const __m512i pack_mul1 = _mm512_set1_epi16(0x0140);
127 const __m512i pack_mul2 = _mm512_set1_epi32(0x00011000);
131 while(length - consumed > 64) {
132 const __m512i x = _mm512_loadu_si512(in + consumed);
134 const __m512i values = _mm512_permutex2var_epi8(lut_lo, x, lut_hi);
137 if(_mm512_movepi8_mask(_mm512_or_si512(x, values)) != 0) {
141 const __m512i merged16 = _mm512_maddubs_epi16(values, pack_mul1);
142 const __m512i merged32 = _mm512_madd_epi16(merged16, pack_mul2);
143 const __m512i packed = _mm512_permutexvar_epi8(pack_shuffle, merged32);
145 _mm512_mask_storeu_epi8(out, LOW_48, packed);