Transitions `QuicheLinkedHashMap` to use `StableBlockList` as its list type and implements flag protection. This CL wraps the underlying list in `QuicheLinkedHashMap` in a `std::variant` to support both `std::list` and `StableBlockList` implementations, guarded by the reloadable flag `gfe2_reloadable_flag_quiche_linked_hash_map_use_stable_block_list`. Tested: - `//third_party/quiche/common:quiche_linked_hash_map_test` - `//gfe/gfe2:quiche_feature_flags_test` - `//third_party/quiche/common/...` Protected by quic_reloadable_flag_quiche_linked_hash_map_use_stable_block_list. PiperOrigin-RevId: 966366233
diff --git a/quiche/common/quiche_feature_flags_list.h b/quiche/common/quiche_feature_flags_list.h index c523aed..057425e 100755 --- a/quiche/common/quiche_feature_flags_list.h +++ b/quiche/common/quiche_feature_flags_list.h
@@ -63,6 +63,7 @@ QUICHE_FLAG(bool, quiche_reloadable_flag_quic_testonly_default_false, false, false, "A testonly reloadable flag that will always default to false.") QUICHE_FLAG(bool, quiche_reloadable_flag_quic_testonly_default_true, true, true, "A testonly reloadable flag that will always default to true.") QUICHE_FLAG(bool, quiche_reloadable_flag_quic_use_received_client_addresses_cache, true, true, "If true, use a LRU cache to record client addresses of packets received on server's original address.") +QUICHE_FLAG(bool, quiche_reloadable_flag_quiche_linked_hash_map_use_stable_block_list, false, false, "If true, use StableBlockList in QuicheLinkedHashMap.") QUICHE_FLAG(bool, quiche_restart_flag_quic_client_cert_support, false, true, "If true, enables dynamic client certs in QUIC.") QUICHE_FLAG(bool, quiche_restart_flag_quic_dispatcher_close_connection_on_invalid_ack, false, false, "An invalid ack is an ack that the peer sent for a packet that was not sent by the dispatcher. If true, the dispatcher will close the connection if it receives an invalid ack.") QUICHE_FLAG(bool, quiche_restart_flag_quic_support_release_time_for_gso, false, false, "If true, QuicGsoBatchWriter will support release time if it is available and the process has the permission to do so.")
diff --git a/quiche/common/quiche_linked_hash_map.h b/quiche/common/quiche_linked_hash_map.h index 4d4e630..ddd328f 100644 --- a/quiche/common/quiche_linked_hash_map.h +++ b/quiche/common/quiche_linked_hash_map.h
@@ -21,11 +21,15 @@ #include <tuple> #include <type_traits> #include <utility> +#include <variant> #include "absl/container/flat_hash_map.h" #include "absl/hash/hash.h" #include "quiche/common/platform/api/quiche_export.h" +#include "quiche/common/platform/api/quiche_flag_utils.h" +#include "quiche/common/platform/api/quiche_flags.h" #include "quiche/common/platform/api/quiche_logging.h" +#include "quiche/common/stable_block_list.h" namespace quiche { @@ -40,24 +44,136 @@ template <class Key, // QUICHE_NO_EXPORT class Value, // QUICHE_NO_EXPORT class Hash = absl::Hash<Key>, // QUICHE_NO_EXPORT - class Eq = std::equal_to<Key>> // QUICHE_NO_EXPORT + class Eq = std::equal_to<Key>, // QUICHE_NO_EXPORT + size_t BlockSize = 16> // QUICHE_NO_EXPORT class QuicheLinkedHashMap { // QUICHE_NO_EXPORT private: - typedef std::list<std::pair<Key, Value>> ListType; - typedef absl::flat_hash_map<Key, typename ListType::iterator, Hash, Eq> - MapType; + using StdList = std::list<std::pair<Key, Value>>; + using BlockList = quiche::StableBlockList<std::pair<Key, Value>, BlockSize>; + using ListType = std::variant<StdList, BlockList>; public: - typedef typename ListType::iterator iterator; - typedef typename ListType::reverse_iterator reverse_iterator; - typedef typename ListType::const_iterator const_iterator; - typedef typename ListType::const_reverse_iterator const_reverse_iterator; - typedef typename MapType::key_type key_type; - typedef typename ListType::value_type value_type; - typedef typename ListType::size_type size_type; + class const_iterator; - QuicheLinkedHashMap() = default; - explicit QuicheLinkedHashMap(size_type bucket_count) : map_(bucket_count) {} + class iterator { + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = std::pair<Key, Value>; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + + iterator() = default; + iterator(typename StdList::iterator it) : it_(it) {} + iterator(typename BlockList::iterator it) : it_(it) {} + + reference operator*() const { + return std::visit([](auto&& it) -> reference { return *it; }, it_); + } + pointer operator->() const { + return std::visit([](auto&& it) -> pointer { return &*it; }, it_); + } + iterator& operator++() { + std::visit([](auto&& it) { ++it; }, it_); + return *this; + } + iterator operator++(int) { + iterator tmp = *this; + ++(*this); + return tmp; + } + iterator& operator--() { + std::visit([](auto&& it) { --it; }, it_); + return *this; + } + iterator operator--(int) { + iterator tmp = *this; + --(*this); + return tmp; + } + bool operator==(const iterator& other) const { return it_ == other.it_; } + bool operator!=(const iterator& other) const { return !(*this == other); } + + private: + std::variant<typename StdList::iterator, typename BlockList::iterator> it_; + friend class QuicheLinkedHashMap; + friend class const_iterator; + }; + + class const_iterator { + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = std::pair<Key, Value>; + using difference_type = std::ptrdiff_t; + using pointer = const value_type*; + using reference = const value_type&; + + const_iterator() = default; + const_iterator(typename StdList::const_iterator it) : it_(it) {} + const_iterator(typename BlockList::const_iterator it) : it_(it) {} + const_iterator(const iterator& other) { + std::visit([this](auto&& it) { it_ = it; }, other.it_); + } + + reference operator*() const { + return std::visit([](auto&& it) -> reference { return *it; }, it_); + } + pointer operator->() const { + return std::visit([](auto&& it) -> pointer { return &*it; }, it_); + } + const_iterator& operator++() { + std::visit([](auto&& it) { ++it; }, it_); + return *this; + } + const_iterator operator++(int) { + const_iterator tmp = *this; + ++(*this); + return tmp; + } + const_iterator& operator--() { + std::visit([](auto&& it) { --it; }, it_); + return *this; + } + const_iterator operator--(int) { + const_iterator tmp = *this; + --(*this); + return tmp; + } + bool operator==(const const_iterator& other) const { + return it_ == other.it_; + } + bool operator!=(const const_iterator& other) const { + return !(*this == other); + } + + private: + std::variant<typename StdList::const_iterator, + typename BlockList::const_iterator> + it_; + friend class QuicheLinkedHashMap; + }; + + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + using key_type = Key; + using value_type = std::pair<Key, Value>; + using size_type = size_t; + + QuicheLinkedHashMap() { + if (GetQuicheReloadableFlag(quiche_linked_hash_map_use_stable_block_list)) { + list_.template emplace<BlockList>(); + QUICHE_RELOADABLE_FLAG_COUNT( + quiche_linked_hash_map_use_stable_block_list); + } + } + + explicit QuicheLinkedHashMap(size_type bucket_count) : map_(bucket_count) { + if (GetQuicheReloadableFlag(quiche_linked_hash_map_use_stable_block_list)) { + list_.template emplace<BlockList>(); + QUICHE_RELOADABLE_FLAG_COUNT( + quiche_linked_hash_map_use_stable_block_list); + } + } QuicheLinkedHashMap(const QuicheLinkedHashMap& other) = delete; QuicheLinkedHashMap& operator=(const QuicheLinkedHashMap& other) = delete; @@ -66,44 +182,71 @@ // Returns an iterator to the first (insertion-ordered) element. Like a map, // this can be dereferenced to a pair<Key, Value>. - iterator begin() { return list_.begin(); } - const_iterator begin() const { return list_.begin(); } + iterator begin() { + return std::visit([](auto& list) -> iterator { return list.begin(); }, + list_); + } + const_iterator begin() const { + return std::visit([](auto& list) -> const_iterator { return list.begin(); }, + list_); + } // Returns an iterator beyond the last element. - iterator end() { return list_.end(); } - const_iterator end() const { return list_.end(); } + iterator end() { + return std::visit([](auto& list) -> iterator { return list.end(); }, list_); + } + const_iterator end() const { + return std::visit([](auto& list) -> const_iterator { return list.end(); }, + list_); + } // Returns an iterator to the last (insertion-ordered) element. Like a map, // this can be dereferenced to a pair<Key, Value>. - reverse_iterator rbegin() { return list_.rbegin(); } - const_reverse_iterator rbegin() const { return list_.rbegin(); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } // Returns an iterator beyond the first element. - reverse_iterator rend() { return list_.rend(); } - const_reverse_iterator rend() const { return list_.rend(); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } // Front and back accessors common to many stl containers. // Returns the earliest-inserted element - const value_type& front() const { return list_.front(); } - + const value_type& front() const { + return std::visit( + [](auto& list) -> const value_type& { return list.front(); }, list_); + } // Returns the earliest-inserted element. - value_type& front() { return list_.front(); } + value_type& front() { + return std::visit([](auto& list) -> value_type& { return list.front(); }, + list_); + } // Returns the most-recently-inserted element. - const value_type& back() const { return list_.back(); } - + const value_type& back() const { + return std::visit( + [](auto& list) -> const value_type& { return list.back(); }, list_); + } // Returns the most-recently-inserted element. - value_type& back() { return list_.back(); } + value_type& back() { + return std::visit([](auto& list) -> value_type& { return list.back(); }, + list_); + } // Clears the map of all values. void clear() { map_.clear(); - list_.clear(); + std::visit([](auto& list) { list.clear(); }, list_); } // Returns true iff the map is empty. - bool empty() const { return list_.empty(); } + bool empty() const { + return std::visit([](auto& list) { return list.empty(); }, list_); + } // Removes the first element from the list. void pop_front() { erase(begin()); } @@ -116,7 +259,12 @@ return 0; } - list_.erase(found->second); + if (auto* list = std::get_if<BlockList>(&list_)) { + list->erase(std::get<typename BlockList::iterator>(found->second.it_)); + } else { + std::get<StdList>(list_).erase( + std::get<typename StdList::iterator>(found->second.it_)); + } map_.erase(found); return 1; @@ -134,7 +282,12 @@ "invalid."; map_.erase(found); - return list_.erase(position); + if (auto* list = std::get_if<BlockList>(&list_)) { + return list->erase(std::get<typename BlockList::iterator>(position.it_)); + } else { + return std::get<StdList>(list_).erase( + std::get<typename StdList::iterator>(position.it_)); + } } // Erases all the items in the range [first, last). Returns an iterator that @@ -163,7 +316,7 @@ if (found == map_.end()) { return end(); } - return found->second; + return const_iterator(found->second); } bool contains(const Key& key) const { return find(key) != end(); } @@ -197,6 +350,8 @@ return TryEmplaceInternal(std::move(key), std::forward<Args>(args)...); } + // TODO(b/532261946): add back `emplace()` if needed + void swap(QuicheLinkedHashMap& other) { map_.swap(other.map_); list_.swap(other.list_); @@ -215,7 +370,11 @@ } // Otherwise, insert into the list, and set value in map. - auto list_iter = list_.insert(list_.end(), std::forward<U>(pair)); + iterator list_iter = std::visit( + [&pair](auto& list) -> iterator { + return iterator(list.insert(list.end(), std::forward<U>(pair))); + }, + list_); map_iter->second = list_iter; return {list_iter, true}; @@ -229,20 +388,24 @@ return {insert_result.first->second, false}; } - auto list_iter = - list_.emplace(list_.end(), std::piecewise_construct, - std::forward_as_tuple(insert_result.first->first), - std::forward_as_tuple(std::forward<Args>(args)...)); + iterator list_iter = std::visit( + [&insert_result, &args...](auto& list) -> iterator { + return iterator( + list.emplace(list.end(), std::piecewise_construct, + std::forward_as_tuple(insert_result.first->first), + std::forward_as_tuple(std::forward<Args>(args)...))); + }, + list_); insert_result.first->second = list_iter; return {list_iter, true}; } - // The map component, used for speedy lookups - MapType map_; - // The list component, used for maintaining insertion order ListType list_; + using MapType = absl::flat_hash_map<Key, iterator, Hash, Eq>; + // The map component, used for speedy lookups + MapType map_; }; } // namespace quiche
diff --git a/quiche/common/quiche_linked_hash_map_test.cc b/quiche/common/quiche_linked_hash_map_test.cc index 4bdb23e..a84c3d7 100644 --- a/quiche/common/quiche_linked_hash_map_test.cc +++ b/quiche/common/quiche_linked_hash_map_test.cc
@@ -11,8 +11,10 @@ #include <tuple> #include <utility> +#include "quiche/common/platform/api/quiche_flags.h" #include "quiche/common/platform/api/quiche_test.h" +using testing::ElementsAre; using testing::Pair; using testing::Pointee; using testing::UnorderedElementsAre; @@ -20,8 +22,19 @@ namespace quiche { namespace test { +class QuicheLinkedHashMapTest : public QuicheTestWithParam<bool> { + protected: + void SetUp() override { + SetQuicheReloadableFlag(quiche_linked_hash_map_use_stable_block_list, + GetParam()); + } +}; + +INSTANTIATE_TEST_SUITE_P(QuicheLinkedHashMapTests, QuicheLinkedHashMapTest, + ::testing::Bool()); + // Tests that move constructor works. -TEST(LinkedHashMapTest, Move) { +TEST_P(QuicheLinkedHashMapTest, Move) { // Use unique_ptr as an example of a non-copyable type. QuicheLinkedHashMap<int, std::unique_ptr<int>> m; m[2] = std::make_unique<int>(12); @@ -31,7 +44,19 @@ UnorderedElementsAre(Pair(2, Pointee(12)), Pair(3, Pointee(13)))); } -TEST(LinkedHashMapTest, CanTryEmplaceMoveOnly) { +TEST_P(QuicheLinkedHashMapTest, ConstructorWithBucketCount) { + QuicheLinkedHashMap<int, int> m(100); + EXPECT_EQ(0u, m.size()); + EXPECT_TRUE(m.empty()); + + m[1] = 10; + m[2] = 20; + EXPECT_EQ(10, m[1]); + EXPECT_EQ(20, m[2]); + EXPECT_EQ(2u, m.size()); +} + +TEST_P(QuicheLinkedHashMapTest, CanTryEmplaceMoveOnly) { QuicheLinkedHashMap<int, std::unique_ptr<int>> m; struct Data { int k, v; @@ -55,7 +80,7 @@ int x; }; -TEST(LinkedHashMapTest, CanTryEmplaceNoMoveNoCopy) { +TEST_P(QuicheLinkedHashMapTest, CanTryEmplaceNoMoveNoCopy) { QuicheLinkedHashMap<int, NoCopy> m; struct Data { int k, v; @@ -70,7 +95,7 @@ EXPECT_EQ(234, found->second.x); } -TEST(LinkedHashMapTest, TryEmplaceRvalueKey) { +TEST_P(QuicheLinkedHashMapTest, TryEmplaceRvalueKey) { QuicheLinkedHashMap<std::string, int> m; std::string key = "hello"; auto result = m.try_emplace(std::move(key), 42); @@ -80,7 +105,7 @@ EXPECT_EQ(m.begin()->first, "hello"); } -TEST(LinkedHashMapTest, ConstKeys) { +TEST_P(QuicheLinkedHashMapTest, ConstKeys) { QuicheLinkedHashMap<int, int> m; m.insert(std::make_pair(1, 2)); // Test that keys are const in iteration. @@ -89,7 +114,7 @@ } // Tests that iteration from begin() to end() works -TEST(LinkedHashMapTest, Iteration) { +TEST_P(QuicheLinkedHashMapTest, Iteration) { QuicheLinkedHashMap<int, int> m; EXPECT_TRUE(m.begin() == m.end()); @@ -118,7 +143,7 @@ } // Tests that reverse iteration from rbegin() to rend() works -TEST(LinkedHashMapTest, ReverseIteration) { +TEST_P(QuicheLinkedHashMapTest, ReverseIteration) { QuicheLinkedHashMap<int, int> m; EXPECT_TRUE(m.rbegin() == m.rend()); @@ -147,7 +172,7 @@ } // Tests that clear() works -TEST(LinkedHashMapTest, Clear) { +TEST_P(QuicheLinkedHashMapTest, Clear) { QuicheLinkedHashMap<int, int> m; m.insert(std::make_pair(2, 12)); m.insert(std::make_pair(1, 11)); @@ -165,7 +190,7 @@ } // Tests that size() works. -TEST(LinkedHashMapTest, Size) { +TEST_P(QuicheLinkedHashMapTest, Size) { QuicheLinkedHashMap<int, int> m; EXPECT_EQ(0u, m.size()); m.insert(std::make_pair(2, 12)); @@ -179,7 +204,7 @@ } // Tests empty() -TEST(LinkedHashMapTest, Empty) { +TEST_P(QuicheLinkedHashMapTest, Empty) { QuicheLinkedHashMap<int, int> m; ASSERT_TRUE(m.empty()); m.insert(std::make_pair(2, 12)); @@ -188,7 +213,7 @@ ASSERT_TRUE(m.empty()); } -TEST(LinkedHashMapTest, Erase) { +TEST_P(QuicheLinkedHashMapTest, Erase) { QuicheLinkedHashMap<int, int> m; ASSERT_EQ(0u, m.size()); EXPECT_EQ(0u, m.erase(2)); // Nothing to erase yet @@ -202,7 +227,7 @@ EXPECT_EQ(0u, m.size()); } -TEST(LinkedHashMapTest, Erase2) { +TEST_P(QuicheLinkedHashMapTest, Erase2) { QuicheLinkedHashMap<int, int> m; ASSERT_EQ(0u, m.size()); EXPECT_EQ(0u, m.erase(2)); // Nothing to erase yet @@ -241,7 +266,7 @@ } // Test that erase(iter,iter) and erase(iter) compile and work. -TEST(LinkedHashMapTest, Erase3) { +TEST_P(QuicheLinkedHashMapTest, Erase3) { QuicheLinkedHashMap<int, int> m; m.insert(std::make_pair(1, 11)); @@ -276,7 +301,7 @@ ASSERT_TRUE(it == m.end()); } -TEST(LinkedHashMapTest, Insertion) { +TEST_P(QuicheLinkedHashMapTest, Insertion) { QuicheLinkedHashMap<int, int> m; ASSERT_EQ(0u, m.size()); std::pair<QuicheLinkedHashMap<int, int>::iterator, bool> result; @@ -310,7 +335,7 @@ static std::pair<int, int> Pair(int i, int j) { return {i, j}; } // Test front accessors. -TEST(LinkedHashMapTest, Front) { +TEST_P(QuicheLinkedHashMapTest, Front) { QuicheLinkedHashMap<int, int> m; m.insert(std::make_pair(2, 12)); @@ -329,7 +354,7 @@ EXPECT_TRUE(m.empty()); } -TEST(LinkedHashMapTest, Find) { +TEST_P(QuicheLinkedHashMapTest, Find) { QuicheLinkedHashMap<int, int> m; EXPECT_TRUE(m.end() == m.find(1)) @@ -358,7 +383,7 @@ << "We shouldn't find anything in a map that we've cleared."; } -TEST(LinkedHashMapTest, Contains) { +TEST_P(QuicheLinkedHashMapTest, Contains) { QuicheLinkedHashMap<int, int> m; EXPECT_FALSE(m.contains(1)) << "An empty map shouldn't contain anything."; @@ -376,7 +401,7 @@ << "A map that we've cleared shouldn't contain anything."; } -TEST(LinkedHashMapTest, Swap) { +TEST_P(QuicheLinkedHashMapTest, Swap) { QuicheLinkedHashMap<int, int> m1; QuicheLinkedHashMap<int, int> m2; m1.insert(std::make_pair(1, 1)); @@ -389,7 +414,7 @@ ASSERT_EQ(2u, m2.size()); } -TEST(LinkedHashMapTest, CustomHashAndEquality) { +TEST_P(QuicheLinkedHashMapTest, CustomHashAndEquality) { struct CustomIntHash { size_t operator()(int x) const { return x; } }; @@ -399,5 +424,24 @@ EXPECT_EQ(1, m[1]); } +TEST_P(QuicheLinkedHashMapTest, CustomBlockSize) { + // Use a small block size (2) to force multiple blocks. + QuicheLinkedHashMap<int, int, absl::Hash<int>, std::equal_to<int>, 2> m; + m.insert(std::make_pair(1, 10)); + m.insert(std::make_pair(2, 20)); + m.insert(std::make_pair(3, 30)); + m.insert(std::make_pair(4, 40)); + + EXPECT_EQ(4u, m.size()); + EXPECT_EQ(10, m[1]); + EXPECT_EQ(20, m[2]); + EXPECT_EQ(30, m[3]); + EXPECT_EQ(40, m[4]); + + std::vector<std::pair<int, int>> elements(m.begin(), m.end()); + EXPECT_THAT(elements, + ElementsAre(Pair(1, 10), Pair(2, 20), Pair(3, 30), Pair(4, 40))); +} + } // namespace test } // namespace quiche
diff --git a/quiche/common/stable_block_list.h b/quiche/common/stable_block_list.h index f46646d..6c9ef67 100644 --- a/quiche/common/stable_block_list.h +++ b/quiche/common/stable_block_list.h
@@ -444,9 +444,6 @@ } void DeallocateBlock(Block* block) { - BlockAllocator block_alloc(allocator_); - std::allocator_traits<BlockAllocator>::destroy(block_alloc, block); - // Link to free blocks LIFO list using 'next' pointer block->next = control_block_->free_blocks; control_block_->free_blocks = block;
diff --git a/quiche/common/stable_block_list_test.cc b/quiche/common/stable_block_list_test.cc index e1bc942..f91fa6c 100644 --- a/quiche/common/stable_block_list_test.cc +++ b/quiche/common/stable_block_list_test.cc
@@ -382,5 +382,56 @@ EXPECT_EQ(stats->deallocs, 3); } +TEST(StableBlockListTest, FreeListCorruptionRegressionTest) { + // Uses a small block capacity (2) to easily force multi-block behavior. + StableBlockList<int, 2> list; + + // 1. Initial state: block1 [10, 20] -> block2 [30, _] + list.push_back(10); + list.push_back(20); + list.push_back(30); + + // 2. Erases elements in block1 to deallocate it. + // block1 becomes empty and is moved to the free list. + // If the compiler optimized away the write to block1->next during destroy, + // block1->next will still point to block2 (which is active). + auto it = list.begin(); + list.erase(it); // erases 10 + it = list.begin(); + list.erase(it); // erases 20 + + // Now list is: block2 [30, _] + // free_blocks -> block1 + + // 3. Pushes 40. Goes to block2 (has space). + // List: block2 [30, 40] + list.push_back(40); + + // 4. Pushes 50. block2 is full. Needs a new block. + // Reuses block1 from free_blocks. + // If corrupted, free_blocks is set to block1->next (which points to block2). + // List: block2 [30, 40] -> block1 [50, _] + list.push_back(50); + + // 5. Pushes 60. Goes to block1 (has space). + // List: block2 [30, 40] -> block1 [50, 60] + list.push_back(60); + + // 6. Pushes 70. block1 is full. Needs a new block. + // If free_blocks was corrupted to point to block2 (which is active), + // it will reuse block2 and overwrite it, corrupting the list structure. + list.push_back(70); + + // Verifies size and elements. + // If corruption occurred, the list will be truncated or contain garbage. + EXPECT_EQ(list.size(), 5); + + std::vector<int> elements; + for (int x : list) { + elements.push_back(x); + } + EXPECT_THAT(elements, ::testing::ElementsAre(30, 40, 50, 60, 70)); +} + } // namespace } // namespace quiche