Internal change PiperOrigin-RevId: 432953313
diff --git a/common/quiche_buffer_allocator.cc b/common/quiche_buffer_allocator.cc new file mode 100644 index 0000000..0801173 --- /dev/null +++ b/common/quiche_buffer_allocator.cc
@@ -0,0 +1,76 @@ +// Copyright 2022 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "common/quiche_buffer_allocator.h" + +#include <cstring> + +#include "common/platform/api/quiche_bug_tracker.h" +#include "common/platform/api/quiche_logging.h" +#include "common/platform/api/quiche_prefetch.h" + +namespace quiche { + +QuicheBuffer QuicheBuffer::CopyFromIovec(QuicheBufferAllocator* allocator, + const struct iovec* iov, int iov_count, + size_t iov_offset, + size_t buffer_length) { + if (buffer_length == 0) { + return {}; + } + + int iovnum = 0; + while (iovnum < iov_count && iov_offset >= iov[iovnum].iov_len) { + iov_offset -= iov[iovnum].iov_len; + ++iovnum; + } + QUICHE_DCHECK_LE(iovnum, iov_count); + if (iovnum >= iov_count) { + QUICHE_BUG(quiche_bug_10839_1) + << "iov_offset larger than iovec total size."; + return {}; + } + QUICHE_DCHECK_LE(iov_offset, iov[iovnum].iov_len); + + // Unroll the first iteration that handles iov_offset. + const size_t iov_available = iov[iovnum].iov_len - iov_offset; + size_t copy_len = std::min(buffer_length, iov_available); + + // Try to prefetch the next iov if there is at least one more after the + // current. Otherwise, it looks like an irregular access that the hardware + // prefetcher won't speculatively prefetch. Only prefetch one iov because + // generally, the iov_offset is not 0, input iov consists of 2K buffers and + // the output buffer is ~1.4K. + if (copy_len == iov_available && iovnum + 1 < iov_count) { + char* next_base = static_cast<char*>(iov[iovnum + 1].iov_base); + // Prefetch 2 cachelines worth of data to get the prefetcher started; leave + // it to the hardware prefetcher after that. + quiche::QuichePrefetchT0(next_base); + if (iov[iovnum + 1].iov_len >= 64) { + quiche::QuichePrefetchT0(next_base + ABSL_CACHELINE_SIZE); + } + } + + QuicheBuffer buffer(allocator, buffer_length); + + const char* src = static_cast<char*>(iov[iovnum].iov_base) + iov_offset; + char* dst = buffer.data(); + while (true) { + memcpy(dst, src, copy_len); + buffer_length -= copy_len; + dst += copy_len; + if (buffer_length == 0 || ++iovnum >= iov_count) { + break; + } + src = static_cast<char*>(iov[iovnum].iov_base); + copy_len = std::min(buffer_length, iov[iovnum].iov_len); + } + + QUICHE_BUG_IF(quiche_bug_10839_2, buffer_length > 0) + << "iov_offset + buffer_length larger than iovec total size."; + + return buffer; +} + +} // namespace quiche
diff --git a/common/quiche_buffer_allocator.h b/common/quiche_buffer_allocator.h index 40140ac..b0c233b 100644 --- a/common/quiche_buffer_allocator.h +++ b/common/quiche_buffer_allocator.h
@@ -10,6 +10,7 @@ #include <memory> #include "absl/strings/string_view.h" +#include "quic/platform/api/quic_iovec.h" #include "common/platform/api/quiche_export.h" namespace quiche { @@ -88,11 +89,19 @@ // Factory method to create a QuicheBuffer that holds a copy of `data`. static QuicheBuffer Copy(QuicheBufferAllocator* allocator, absl::string_view data) { - QuicheBuffer result(allocator, data.size()); - memcpy(result.data(), data.data(), data.size()); - return result; + QuicheBuffer buffer(allocator, data.size()); + memcpy(buffer.data(), data.data(), data.size()); + return buffer; } + // Factory method to create a QuicheBuffer of length `buffer_length` that + // holds a copy of `buffer_length` bytes from `iov` starting at offset + // `iov_offset`. `iov` must be at least `iov_offset + buffer_length` total + // length. + static QuicheBuffer CopyFromIovec(QuicheBufferAllocator* allocator, + const struct iovec* iov, int iov_count, + size_t iov_offset, size_t buffer_length); + const char* data() const { return buffer_.get(); } char* data() { return buffer_.get(); } size_t size() const { return size_; }
diff --git a/common/quiche_buffer_allocator_test.cc b/common/quiche_buffer_allocator_test.cc new file mode 100644 index 0000000..7d08cf1 --- /dev/null +++ b/common/quiche_buffer_allocator_test.cc
@@ -0,0 +1,141 @@ +// Copyright 2022 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "common/quiche_buffer_allocator.h" + +#include "absl/strings/string_view.h" +#include "common/platform/api/quiche_test.h" +#include "common/platform/api/quiche_test_helpers.h" +#include "common/simple_buffer_allocator.h" +#include "common/test_tools/quiche_test_utils.h" + +namespace quiche { +namespace test { +namespace { + +TEST(QuicheBuffer, CopyFromEmpty) { + SimpleBufferAllocator allocator; + QuicheBuffer buffer = QuicheBuffer::Copy(&allocator, ""); + EXPECT_TRUE(buffer.empty()); +} + +TEST(QuicheBuffer, Copy) { + SimpleBufferAllocator allocator; + QuicheBuffer buffer = QuicheBuffer::Copy(&allocator, "foobar"); + EXPECT_EQ("foobar", buffer.AsStringView()); +} + +TEST(QuicheBuffer, CopyFromIovecZeroBytes) { + const int buffer_length = 0; + + SimpleBufferAllocator allocator; + QuicheBuffer buffer = QuicheBuffer::CopyFromIovec( + &allocator, nullptr, + /* iov_count = */ 0, /* iov_offset = */ 0, buffer_length); + EXPECT_TRUE(buffer.empty()); + + constexpr absl::string_view kData("foobar"); + iovec iov = MakeIOVector(kData); + + buffer = QuicheBuffer::CopyFromIovec(&allocator, &iov, + /* iov_count = */ 1, + /* iov_offset = */ 0, buffer_length); + EXPECT_TRUE(buffer.empty()); + + buffer = QuicheBuffer::CopyFromIovec(&allocator, &iov, + /* iov_count = */ 1, + /* iov_offset = */ 3, buffer_length); + EXPECT_TRUE(buffer.empty()); +} + +TEST(QuicheBuffer, CopyFromIovecSimple) { + constexpr absl::string_view kData("foobar"); + iovec iov = MakeIOVector(kData); + + SimpleBufferAllocator allocator; + QuicheBuffer buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov, + /* iov_count = */ 1, /* iov_offset = */ 0, + /* buffer_length = */ 6); + EXPECT_EQ("foobar", buffer.AsStringView()); + + buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov, + /* iov_count = */ 1, /* iov_offset = */ 0, + /* buffer_length = */ 3); + EXPECT_EQ("foo", buffer.AsStringView()); + + buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov, + /* iov_count = */ 1, /* iov_offset = */ 3, + /* buffer_length = */ 3); + EXPECT_EQ("bar", buffer.AsStringView()); + + buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov, + /* iov_count = */ 1, /* iov_offset = */ 1, + /* buffer_length = */ 4); + EXPECT_EQ("ooba", buffer.AsStringView()); +} + +TEST(QuicheBuffer, CopyFromIovecMultiple) { + constexpr absl::string_view kData1("foo"); + constexpr absl::string_view kData2("bar"); + iovec iov[] = {MakeIOVector(kData1), MakeIOVector(kData2)}; + + SimpleBufferAllocator allocator; + QuicheBuffer buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov[0], + /* iov_count = */ 2, /* iov_offset = */ 0, + /* buffer_length = */ 6); + EXPECT_EQ("foobar", buffer.AsStringView()); + + buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov[0], + /* iov_count = */ 2, /* iov_offset = */ 0, + /* buffer_length = */ 3); + EXPECT_EQ("foo", buffer.AsStringView()); + + buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov[0], + /* iov_count = */ 2, /* iov_offset = */ 3, + /* buffer_length = */ 3); + EXPECT_EQ("bar", buffer.AsStringView()); + + buffer = + QuicheBuffer::CopyFromIovec(&allocator, &iov[0], + /* iov_count = */ 2, /* iov_offset = */ 1, + /* buffer_length = */ 4); + EXPECT_EQ("ooba", buffer.AsStringView()); +} + +TEST(QuicheBuffer, CopyFromIovecOffsetTooLarge) { + constexpr absl::string_view kData1("foo"); + constexpr absl::string_view kData2("bar"); + iovec iov[] = {MakeIOVector(kData1), MakeIOVector(kData2)}; + + SimpleBufferAllocator allocator; + EXPECT_QUICHE_BUG( + QuicheBuffer::CopyFromIovec(&allocator, &iov[0], + /* iov_count = */ 2, /* iov_offset = */ 10, + /* buffer_length = */ 6), + "iov_offset larger than iovec total size"); +} + +TEST(QuicheBuffer, CopyFromIovecTooManyBytesRequested) { + constexpr absl::string_view kData1("foo"); + constexpr absl::string_view kData2("bar"); + iovec iov[] = {MakeIOVector(kData1), MakeIOVector(kData2)}; + + SimpleBufferAllocator allocator; + EXPECT_QUICHE_BUG( + QuicheBuffer::CopyFromIovec(&allocator, &iov[0], + /* iov_count = */ 2, /* iov_offset = */ 2, + /* buffer_length = */ 10), + "iov_offset [+] buffer_length larger than iovec total size"); +} + +} // anonymous namespace +} // namespace test +} // namespace quiche
diff --git a/common/quiche_mem_slice_storage.cc b/common/quiche_mem_slice_storage.cc index 2b1b5e1..d57cf76 100644 --- a/common/quiche_mem_slice_storage.cc +++ b/common/quiche_mem_slice_storage.cc
@@ -23,9 +23,8 @@ size_t io_offset = 0; while (write_len > 0) { size_t slice_len = std::min(write_len, max_slice_len); - QuicheBuffer buffer(allocator, slice_len); - quic::QuicUtils::CopyToBuffer(iov, iov_count, io_offset, slice_len, - buffer.data()); + QuicheBuffer buffer = QuicheBuffer::CopyFromIovec(allocator, iov, iov_count, + io_offset, slice_len); storage_.push_back(QuicheMemSlice(std::move(buffer))); write_len -= slice_len; io_offset += slice_len;
diff --git a/common/test_tools/quiche_test_utils.cc b/common/test_tools/quiche_test_utils.cc index 9a3bb21..5ebe527 100644 --- a/common/test_tools/quiche_test_utils.cc +++ b/common/test_tools/quiche_test_utils.cc
@@ -87,5 +87,9 @@ << HexDumpWithMarks(actual, actual_len, marks.get(), max_len); } +iovec MakeIOVector(absl::string_view str) { + return iovec{const_cast<char*>(str.data()), static_cast<size_t>(str.size())}; +} + } // namespace test } // namespace quiche
diff --git a/common/test_tools/quiche_test_utils.h b/common/test_tools/quiche_test_utils.h index 45a526b..becfea0 100644 --- a/common/test_tools/quiche_test_utils.h +++ b/common/test_tools/quiche_test_utils.h
@@ -7,6 +7,9 @@ #include <string> +#include "absl/strings/string_view.h" +#include "quic/platform/api/quic_iovec.h" + namespace quiche { namespace test { @@ -16,6 +19,9 @@ const char* expected, const int expected_len); +// Create iovec that points to that data that `str` points to. +iovec MakeIOVector(absl::string_view str); + } // namespace test } // namespace quiche
diff --git a/quic/core/quic_stream_send_buffer.h b/quic/core/quic_stream_send_buffer.h index cf3fa13..e972ac0 100644 --- a/quic/core/quic_stream_send_buffer.h +++ b/quic/core/quic_stream_send_buffer.h
@@ -10,7 +10,6 @@ #include "quic/core/quic_interval_deque.h" #include "quic/core/quic_interval_set.h" #include "quic/core/quic_types.h" -#include "quic/platform/api/quic_iovec.h" #include "common/platform/api/quiche_mem_slice.h" #include "common/quiche_circular_deque.h"
diff --git a/quic/core/quic_utils.cc b/quic/core/quic_utils.cc index aa8f59e..94d35eb 100644 --- a/quic/core/quic_utils.cc +++ b/quic/core/quic_utils.cc
@@ -24,7 +24,6 @@ #include "quic/platform/api/quic_flags.h" #include "common/platform/api/quiche_logging.h" #include "common/platform/api/quiche_mem_slice.h" -#include "common/platform/api/quiche_prefetch.h" #include "common/quiche_endian.h" namespace quic { @@ -239,57 +238,6 @@ } // static -void QuicUtils::CopyToBuffer(const struct iovec* iov, - int iov_count, - size_t iov_offset, - size_t buffer_length, - char* buffer) { - int iovnum = 0; - while (iovnum < iov_count && iov_offset >= iov[iovnum].iov_len) { - iov_offset -= iov[iovnum].iov_len; - ++iovnum; - } - QUICHE_DCHECK_LE(iovnum, iov_count); - QUICHE_DCHECK_LE(iov_offset, iov[iovnum].iov_len); - if (iovnum >= iov_count || buffer_length == 0) { - return; - } - - // Unroll the first iteration that handles iov_offset. - const size_t iov_available = iov[iovnum].iov_len - iov_offset; - size_t copy_len = std::min(buffer_length, iov_available); - - // Try to prefetch the next iov if there is at least one more after the - // current. Otherwise, it looks like an irregular access that the hardware - // prefetcher won't speculatively prefetch. Only prefetch one iov because - // generally, the iov_offset is not 0, input iov consists of 2K buffers and - // the output buffer is ~1.4K. - if (copy_len == iov_available && iovnum + 1 < iov_count) { - char* next_base = static_cast<char*>(iov[iovnum + 1].iov_base); - // Prefetch 2 cachelines worth of data to get the prefetcher started; leave - // it to the hardware prefetcher after that. - quiche::QuichePrefetchT0(next_base); - if (iov[iovnum + 1].iov_len >= 64) { - quiche::QuichePrefetchT0(next_base + ABSL_CACHELINE_SIZE); - } - } - - const char* src = static_cast<char*>(iov[iovnum].iov_base) + iov_offset; - while (true) { - memcpy(buffer, src, copy_len); - buffer_length -= copy_len; - buffer += copy_len; - if (buffer_length == 0 || ++iovnum >= iov_count) { - break; - } - src = static_cast<char*>(iov[iovnum].iov_base); - copy_len = std::min(buffer_length, iov[iovnum].iov_len); - } - QUIC_BUG_IF(quic_bug_10839_1, buffer_length > 0) - << "Failed to copy entire length to buffer."; -} - -// static bool QuicUtils::IsAckable(SentPacketState state) { return state != NEVER_SENT && state != ACKED && state != UNACKABLE; }
diff --git a/quic/core/quic_utils.h b/quic/core/quic_utils.h index 085f532..ea03236 100644 --- a/quic/core/quic_utils.h +++ b/quic/core/quic_utils.h
@@ -21,7 +21,6 @@ #include "quic/core/quic_types.h" #include "quic/core/quic_versions.h" #include "quic/platform/api/quic_export.h" -#include "quic/platform/api/quic_iovec.h" #include "quic/platform/api/quic_socket_address.h" #include "common/platform/api/quiche_mem_slice.h" @@ -72,15 +71,6 @@ const QuicSocketAddress& old_address, const QuicSocketAddress& new_address); - // Copies |buffer_length| bytes from iov starting at offset |iov_offset| into - // buffer. |iov| must be at least iov_offset+length total length and buffer - // must be at least |length| long. - static void CopyToBuffer(const struct iovec* iov, - int iov_count, - size_t iov_offset, - size_t buffer_length, - char* buffer); - // Returns the opposite Perspective of the |perspective| passed in. static constexpr Perspective InvertPerspective(Perspective perspective) { return perspective == Perspective::IS_CLIENT ? Perspective::IS_SERVER
diff --git a/quic/test_tools/quic_test_utils.h b/quic/test_tools/quic_test_utils.h index 869e416..d9bc541 100644 --- a/quic/test_tools/quic_test_utils.h +++ b/quic/test_tools/quic_test_utils.h
@@ -1510,12 +1510,6 @@ return l; } -// Utility function that stores |str|'s data in |iov|. -inline void MakeIOVector(absl::string_view str, struct iovec* iov) { - iov->iov_base = const_cast<char*>(str.data()); - iov->iov_len = static_cast<size_t>(str.size()); -} - // Helper functions for stream ids, to allow test logic to abstract over the // HTTP stream numbering scheme (i.e. whether one or two QUIC streams are used // per HTTP transaction).