Refactors `HuffmanBitBuffer::AppendBytes` with a 64-bit word read. Uses a 64-bit unaligned load (`quiche::QuicheEndian::HostToNet64`) when at least 8 input bytes are available to refill `HuffmanBitBuffer::accumulator_` in a single 64-bit operation rather than a scalar byte-by-byte loop. Based on benchmark runs against the previous state, here is how this optimization compares: - **Overall Performance (`geomean`)**: Significant overall throughput improvement of **+22.71%** (from `100.4 MiB/s` to `123.2 MiB/s`). - **Broad Improvements**: Every single benchmark demonstrated a positive throughput gain, with the vast majority showing improvements between **+23%** and **+28%**. - **Top Performers**: - `BM_DecodeFastPath7Bit/64k`: **+27.93%** - `BM_DecodeFastPath7Bit/1k`: **+27.63%** - `BM_DecodeSlowPath/1k`: **+26.23%** - `BM_DecodeSlowPath/64k`: **+25.96%** - `BM_DecodeSmallStrings`: **+25.69%** This optimization was generated by Gemini. Protected by FLAGS_gfe2_reloadable_flag_hpack_huffman_decoder_optimizations. PiperOrigin-RevId: 962996201
diff --git a/quiche/http2/hpack/huffman/hpack_huffman_decoder.cc b/quiche/http2/hpack/huffman/hpack_huffman_decoder.cc index 21b411d..94ce85c 100644 --- a/quiche/http2/hpack/huffman/hpack_huffman_decoder.cc +++ b/quiche/http2/hpack/huffman/hpack_huffman_decoder.cc
@@ -5,14 +5,15 @@ #include "quiche/http2/hpack/huffman/hpack_huffman_decoder.h" #include <bitset> +#include <cstring> #include <limits> #include <ostream> #include <sstream> -#include <string> #include "quiche/common/platform/api/quiche_flag_utils.h" #include "quiche/common/platform/api/quiche_flags.h" #include "quiche/common/platform/api/quiche_logging.h" +#include "quiche/common/quiche_endian.h" // Terminology: // @@ -447,13 +448,26 @@ count_ = 0; } -size_t HuffmanBitBuffer::AppendBytes(absl::string_view input) { +size_t HuffmanBitBuffer::AppendBytes(absl::string_view input, + bool enable_optimizations) { HuffmanAccumulatorBitCount free_cnt = free_count(); size_t bytes_available = input.size(); if (free_cnt < 8 || bytes_available == 0) { return 0; } + // If sufficient data is available, the accumulator can be filled in a single + // read rather than a loop. + if (enable_optimizations && bytes_available >= 8) { + uint64_t loaded; + std::memcpy(&loaded, input.data(), sizeof(loaded)); + loaded = quiche::QuicheEndian::NetToHost64(loaded); + const size_t bytes_to_add = std::min<size_t>(free_cnt / 8, 8); + accumulator_ |= (loaded >> count_); + count_ += (bytes_to_add * 8); + return bytes_to_add; + } + // Top up |accumulator_| until there isn't room for a whole byte. size_t bytes_used = 0; auto* ptr = reinterpret_cast<const uint8_t*>(input.data()); @@ -515,7 +529,7 @@ QUICHE_DVLOG(1) << "HpackHuffmanDecoder::Decode"; // Fill bit_buffer_ from input. - input.remove_prefix(bit_buffer_.AppendBytes(input)); + input.remove_prefix(bit_buffer_.AppendBytes(input, enable_optimizations_)); while (true) { QUICHE_DVLOG(3) << "Enter Decode Loop, bit_buffer_: " << bit_buffer_; @@ -549,7 +563,7 @@ } else { // We may have (mostly) drained bit_buffer_. If we can top it up, try // using the table decoder above. - size_t byte_count = bit_buffer_.AppendBytes(input); + size_t byte_count = bit_buffer_.AppendBytes(input, enable_optimizations_); if (byte_count > 0) { input.remove_prefix(byte_count); continue; @@ -581,7 +595,7 @@ } // bit_buffer_ doesn't have enough bits in it to decode the next symbol. // Append to it as many bytes as are available AND fit. - size_t byte_count = bit_buffer_.AppendBytes(input); + size_t byte_count = bit_buffer_.AppendBytes(input, enable_optimizations_); if (byte_count == 0) { QUICHE_DCHECK_EQ(input.size(), 0u); return true;
diff --git a/quiche/http2/hpack/huffman/hpack_huffman_decoder.h b/quiche/http2/hpack/huffman/hpack_huffman_decoder.h index 2148845..ce13027 100644 --- a/quiche/http2/hpack/huffman/hpack_huffman_decoder.h +++ b/quiche/http2/hpack/huffman/hpack_huffman_decoder.h
@@ -47,7 +47,7 @@ // Add as many whole bytes to the accumulator (accumulator_) as possible, // returning the number of bytes added. - size_t AppendBytes(absl::string_view input); + size_t AppendBytes(absl::string_view input, bool enable_optimizations); // Get the bits of the accumulator. HuffmanAccumulator value() const { return accumulator_; }
diff --git a/quiche/http2/hpack/huffman/hpack_huffman_decoder_test.cc b/quiche/http2/hpack/huffman/hpack_huffman_decoder_test.cc index 06d0e90..c0f4c40 100644 --- a/quiche/http2/hpack/huffman/hpack_huffman_decoder_test.cc +++ b/quiche/http2/hpack/huffman/hpack_huffman_decoder_test.cc
@@ -40,7 +40,8 @@ absl::string_view sp(s); HuffmanBitBuffer bb; - sp.remove_prefix(bb.AppendBytes(sp)); + sp.remove_prefix(bb.AppendBytes( + sp, GetQuicheReloadableFlag(hpack_huffman_decoder_optimizations))); EXPECT_TRUE(sp.empty()); EXPECT_FALSE(bb.IsEmpty()) << bb; EXPECT_FALSE(bb.InputProperlyTerminated()); @@ -52,7 +53,8 @@ s.push_back('\x44'); sp = s; - sp.remove_prefix(bb.AppendBytes(sp)); + sp.remove_prefix(bb.AppendBytes( + sp, GetQuicheReloadableFlag(hpack_huffman_decoder_optimizations))); EXPECT_TRUE(sp.empty()); EXPECT_EQ(bb.count(), 32u) << bb; EXPECT_EQ(bb.free_count(), 32u) << bb; @@ -66,14 +68,16 @@ s.push_back('\x99'); sp = s; - sp.remove_prefix(bb.AppendBytes(sp)); + sp.remove_prefix(bb.AppendBytes( + sp, GetQuicheReloadableFlag(hpack_huffman_decoder_optimizations))); EXPECT_EQ(sp.size(), 1u); EXPECT_EQ('\x99', sp[0]); EXPECT_EQ(bb.count(), 64u) << bb; EXPECT_EQ(bb.free_count(), 0u) << bb; EXPECT_EQ(bb.value(), HuffmanAccumulator(0x1122334455667788LL)) << bb; - sp.remove_prefix(bb.AppendBytes(sp)); + sp.remove_prefix(bb.AppendBytes( + sp, GetQuicheReloadableFlag(hpack_huffman_decoder_optimizations))); EXPECT_EQ(sp.size(), 1u); EXPECT_EQ('\x99', sp[0]); EXPECT_EQ(bb.count(), 64u) << bb; @@ -89,7 +93,8 @@ absl::string_view sp(s); HuffmanBitBuffer bb; - sp.remove_prefix(bb.AppendBytes(sp)); + sp.remove_prefix(bb.AppendBytes( + sp, GetQuicheReloadableFlag(hpack_huffman_decoder_optimizations))); EXPECT_TRUE(sp.empty()); bb.ConsumeBits(1); @@ -121,7 +126,8 @@ absl::string_view sp(s); HuffmanBitBuffer bb; - sp.remove_prefix(bb.AppendBytes(sp)); + sp.remove_prefix(bb.AppendBytes( + sp, GetQuicheReloadableFlag(hpack_huffman_decoder_optimizations))); EXPECT_EQ(sp.size(), 5u); EXPECT_FALSE(bb.InputProperlyTerminated()); @@ -133,7 +139,8 @@ expected <<= 15; EXPECT_EQ(bb.value(), expected); - sp.remove_prefix(bb.AppendBytes(sp)); + sp.remove_prefix(bb.AppendBytes( + sp, GetQuicheReloadableFlag(hpack_huffman_decoder_optimizations))); EXPECT_EQ(sp.size(), 4u); EXPECT_EQ(bb.count(), 57u) << bb; EXPECT_EQ(bb.free_count(), 7u) << bb;