Implement a function that gathers data from bunch of string_views into a single buffer. This is similar to CopyFromIovec, notable distinctions being: 1. It takes string_views instead of iovecs. 2. It does not allocate anything. 3. The next string_view is always prefetched (CopyFromIovec only prefetches the second one at the beginning). The hope is to use it to speed up QuicStreamSendBuffer::WriteStreamData at some point in the future. PiperOrigin-RevId: 698508368
diff --git a/build/source_list.bzl b/build/source_list.bzl index a27b05a..5b95fcf 100644 --- a/build/source_list.bzl +++ b/build/source_list.bzl
@@ -58,6 +58,7 @@ "common/quiche_text_utils.h", "common/simple_buffer_allocator.h", "common/structured_headers.h", + "common/vectorized_io_utils.h", "common/wire_serialization.h", "http2/adapter/chunked_buffer.h", "http2/adapter/data_source.h", @@ -422,6 +423,7 @@ "common/quiche_text_utils.cc", "common/simple_buffer_allocator.cc", "common/structured_headers.cc", + "common/vectorized_io_utils.cc", "http2/adapter/chunked_buffer.cc", "http2/adapter/event_forwarder.cc", "http2/adapter/header_validator.cc", @@ -1102,6 +1104,7 @@ "common/structured_headers_test.cc", "common/test_tools/mock_streams_test.cc", "common/test_tools/quiche_test_utils_test.cc", + "common/vectorized_io_utils_test.cc", "common/wire_serialization_test.cc", "http2/adapter/chunked_buffer_test.cc", "http2/adapter/event_forwarder_test.cc",
diff --git a/build/source_list.gni b/build/source_list.gni index 17315d7..dc5104f 100644 --- a/build/source_list.gni +++ b/build/source_list.gni
@@ -58,6 +58,7 @@ "src/quiche/common/quiche_text_utils.h", "src/quiche/common/simple_buffer_allocator.h", "src/quiche/common/structured_headers.h", + "src/quiche/common/vectorized_io_utils.h", "src/quiche/common/wire_serialization.h", "src/quiche/http2/adapter/chunked_buffer.h", "src/quiche/http2/adapter/data_source.h", @@ -422,6 +423,7 @@ "src/quiche/common/quiche_text_utils.cc", "src/quiche/common/simple_buffer_allocator.cc", "src/quiche/common/structured_headers.cc", + "src/quiche/common/vectorized_io_utils.cc", "src/quiche/http2/adapter/chunked_buffer.cc", "src/quiche/http2/adapter/event_forwarder.cc", "src/quiche/http2/adapter/header_validator.cc", @@ -1103,6 +1105,7 @@ "src/quiche/common/structured_headers_test.cc", "src/quiche/common/test_tools/mock_streams_test.cc", "src/quiche/common/test_tools/quiche_test_utils_test.cc", + "src/quiche/common/vectorized_io_utils_test.cc", "src/quiche/common/wire_serialization_test.cc", "src/quiche/http2/adapter/chunked_buffer_test.cc", "src/quiche/http2/adapter/event_forwarder_test.cc",
diff --git a/build/source_list.json b/build/source_list.json index 5ea661c..5b746d0 100644 --- a/build/source_list.json +++ b/build/source_list.json
@@ -57,6 +57,7 @@ "quiche/common/quiche_text_utils.h", "quiche/common/simple_buffer_allocator.h", "quiche/common/structured_headers.h", + "quiche/common/vectorized_io_utils.h", "quiche/common/wire_serialization.h", "quiche/http2/adapter/chunked_buffer.h", "quiche/http2/adapter/data_source.h", @@ -421,6 +422,7 @@ "quiche/common/quiche_text_utils.cc", "quiche/common/simple_buffer_allocator.cc", "quiche/common/structured_headers.cc", + "quiche/common/vectorized_io_utils.cc", "quiche/http2/adapter/chunked_buffer.cc", "quiche/http2/adapter/event_forwarder.cc", "quiche/http2/adapter/header_validator.cc", @@ -1102,6 +1104,7 @@ "quiche/common/structured_headers_test.cc", "quiche/common/test_tools/mock_streams_test.cc", "quiche/common/test_tools/quiche_test_utils_test.cc", + "quiche/common/vectorized_io_utils_test.cc", "quiche/common/wire_serialization_test.cc", "quiche/http2/adapter/chunked_buffer_test.cc", "quiche/http2/adapter/event_forwarder_test.cc",
diff --git a/quiche/common/quiche_buffer_allocator.h b/quiche/common/quiche_buffer_allocator.h index 0f36842..4fb1bd4 100644 --- a/quiche/common/quiche_buffer_allocator.h +++ b/quiche/common/quiche_buffer_allocator.h
@@ -10,6 +10,7 @@ #include <memory> #include "absl/strings/string_view.h" +#include "absl/types/span.h" #include "quiche/common/platform/api/quiche_export.h" #include "quiche/common/platform/api/quiche_iovec.h" @@ -109,6 +110,7 @@ absl::string_view AsStringView() const { return absl::string_view(data(), size()); } + absl::Span<char> AsSpan() { return absl::Span<char>(data(), size()); } // Releases the ownership of the underlying buffer. QuicheUniqueBufferPtr Release() {
diff --git a/quiche/common/quiche_stream.h b/quiche/common/quiche_stream.h index 187e362..5a8a1f7 100644 --- a/quiche/common/quiche_stream.h +++ b/quiche/common/quiche_stream.h
@@ -199,15 +199,6 @@ return stream.Writev(absl::Span<const absl::string_view>(), options); } -inline size_t TotalStringViewSpanSize( - absl::Span<const absl::string_view> span) { - size_t total = 0; - for (absl::string_view view : span) { - total += view.size(); - } - return total; -} - } // namespace quiche #endif // QUICHE_COMMON_QUICHE_STREAM_H_
diff --git a/quiche/common/vectorized_io_utils.cc b/quiche/common/vectorized_io_utils.cc new file mode 100644 index 0000000..1ce421a --- /dev/null +++ b/quiche/common/vectorized_io_utils.cc
@@ -0,0 +1,56 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "quiche/common/vectorized_io_utils.h" + +#include <algorithm> +#include <cstddef> +#include <cstring> + +#include "absl/base/optimization.h" +#include "absl/base/prefetch.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace quiche { + +size_t TotalStringViewSpanSize(absl::Span<const absl::string_view> span) { + size_t total = 0; + for (absl::string_view view : span) { + total += view.size(); + } + return total; +} + +size_t GatherStringViewSpan(absl::Span<const absl::string_view> inputs, + absl::Span<char> output) { + size_t bytes_copied = 0; + for (size_t i = 0; i < inputs.size(); ++i) { + if (inputs[i].empty()) { + continue; + } + const size_t bytes_to_copy = std::min(inputs[i].size(), output.size()); + if (bytes_to_copy == 0) { + break; + } + const absl::Span<char> next_output = output.subspan(bytes_to_copy); + + // Prefetch the first two lines of the next input; the hardware prefetcher + // is expected to take care of the rest. + if (!next_output.empty() && (i + 1) < inputs.size() && + !inputs[i + 1].empty()) { + absl::PrefetchToLocalCache(&inputs[i + 1][0]); + if (inputs[i + 1].size() > ABSL_CACHELINE_SIZE) { + absl::PrefetchToLocalCache(&inputs[i + 1][ABSL_CACHELINE_SIZE]); + } + } + + memcpy(output.data(), inputs[i].data(), bytes_to_copy); + bytes_copied += bytes_to_copy; + output = next_output; + } + return bytes_copied; +} + +} // namespace quiche
diff --git a/quiche/common/vectorized_io_utils.h b/quiche/common/vectorized_io_utils.h new file mode 100644 index 0000000..2ac0002 --- /dev/null +++ b/quiche/common/vectorized_io_utils.h
@@ -0,0 +1,26 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef QUICHE_COMMON_VECTORIZED_IO_UTILS_H_ +#define QUICHE_COMMON_VECTORIZED_IO_UTILS_H_ + +#include <cstddef> + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace quiche { + +// Computes the total size of all strings in the provided span. +size_t TotalStringViewSpanSize(absl::Span<const absl::string_view> span); + +// Copies data contained in `inputs` into `output`, up until either the `output` +// is full or the `inputs` are copied fully; returns the actual number of bytes +// copied. +size_t GatherStringViewSpan(absl::Span<const absl::string_view> inputs, + absl::Span<char> output); + +} // namespace quiche + +#endif // QUICHE_COMMON_VECTORIZED_IO_UTILS_H_
diff --git a/quiche/common/vectorized_io_utils_test.cc b/quiche/common/vectorized_io_utils_test.cc new file mode 100644 index 0000000..dfa5e9c --- /dev/null +++ b/quiche/common/vectorized_io_utils_test.cc
@@ -0,0 +1,79 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "quiche/common/vectorized_io_utils.h" + +#include <cstddef> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "quiche/common/platform/api/quiche_test.h" + +namespace quiche { +namespace { + +using ::testing::ElementsAre; + +TEST(VectorizedIoUtils, GatherStringViewSpanEmpty) { + std::vector<absl::string_view> views = {"a", "b", "c"}; + size_t bytes_copied = GatherStringViewSpan(views, absl::Span<char>()); + EXPECT_EQ(bytes_copied, 0); +} + +TEST(VectorizedIoUtils, GatherStringViewSpanSingle) { + std::vector<absl::string_view> views = {"test"}; + char buffer_small[2]; + size_t bytes_copied = + GatherStringViewSpan(views, absl::MakeSpan(buffer_small)); + EXPECT_EQ(bytes_copied, 2); + EXPECT_THAT(buffer_small, ElementsAre('t', 'e')); + + char buffer_exact[4]; + bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer_exact)); + EXPECT_EQ(bytes_copied, 4); + EXPECT_THAT(buffer_exact, ElementsAre('t', 'e', 's', 't')); + + char buffer_large[6] = {'\0'}; + bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer_large)); + EXPECT_EQ(bytes_copied, 4); + EXPECT_THAT(buffer_large, ElementsAre('t', 'e', 's', 't', '\0', '\0')); +} + +TEST(VectorizedIoUtils, GatherStringViewSpanMultiple) { + const std::vector<absl::string_view> views = {"foo", ",", "bar"}; + constexpr absl::string_view kViewsJoined = "foo,bar"; + char buffer[kViewsJoined.size()]; + for (size_t i = 0; i < sizeof(buffer); ++i) { + size_t bytes_copied = + GatherStringViewSpan(views, absl::Span<char>(buffer, i)); + absl::string_view result(buffer, bytes_copied); + EXPECT_EQ(result, kViewsJoined.substr(0, i)); + } +} + +TEST(VectorizedIoUtils, GatherStringViewSpanEmptyElement) { + const std::vector<absl::string_view> views = {"foo", "", "bar"}; + constexpr absl::string_view kViewsJoined = "foobar"; + char buffer[kViewsJoined.size()]; + size_t bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer)); + absl::string_view result(buffer, bytes_copied); + EXPECT_EQ(result, kViewsJoined); +} + +TEST(VectorizedIoUtils, GatherStringViewSpanLarge) { + const std::string a(8192, 'a'); + const std::string b(8192, 'b'); + const std::vector<absl::string_view> views = {a, b}; + const std::string joined = a + b; + char buffer[8192 * 2]; + size_t bytes_copied = GatherStringViewSpan(views, absl::MakeSpan(buffer)); + EXPECT_EQ(bytes_copied, 8192 * 2); + absl::string_view result(buffer, bytes_copied); + EXPECT_EQ(result, joined); +} + +} // namespace +} // namespace quiche
diff --git a/quiche/quic/core/http/web_transport_stream_adapter.cc b/quiche/quic/core/http/web_transport_stream_adapter.cc index 5f84081..0900323 100644 --- a/quiche/quic/core/http/web_transport_stream_adapter.cc +++ b/quiche/quic/core/http/web_transport_stream_adapter.cc
@@ -8,7 +8,7 @@ #include <limits> #include <optional> #include <string> -#include <vector> +#include <utility> #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -25,8 +25,11 @@ #include "quiche/quic/platform/api/quic_bug_tracker.h" #include "quiche/quic/platform/api/quic_flags.h" #include "quiche/quic/platform/api/quic_logging.h" -#include "quiche/common/quiche_mem_slice_storage.h" +#include "quiche/common/platform/api/quiche_logging.h" +#include "quiche/common/platform/api/quiche_mem_slice.h" +#include "quiche/common/quiche_buffer_allocator.h" #include "quiche/common/quiche_stream.h" +#include "quiche/common/vectorized_io_utils.h" #include "quiche/web_transport/web_transport.h" namespace quic { @@ -79,21 +82,20 @@ return initial_check_status; } - std::vector<iovec> iovecs; - size_t total_size = 0; - iovecs.resize(data.size()); - for (size_t i = 0; i < data.size(); i++) { - // QuicheMemSliceStorage only reads iovec, thus this is safe. - iovecs[i].iov_base = const_cast<char*>(data[i].data()); - iovecs[i].iov_len = data[i].size(); - total_size += data[i].size(); + size_t total_size = quiche::TotalStringViewSpanSize(data); + quiche::QuicheMemSlice slice; + if (total_size > 0) { + quiche::QuicheBuffer buffer( + session_->connection()->helper()->GetStreamSendBufferAllocator(), + total_size); + size_t bytes_copied = quiche::GatherStringViewSpan(data, buffer.AsSpan()); + QUICHE_DCHECK_EQ(total_size, bytes_copied); + slice = quiche::QuicheMemSlice(std::move(buffer)); } - quiche::QuicheMemSliceStorage storage( - iovecs.data(), iovecs.size(), - session_->connection()->helper()->GetStreamSendBufferAllocator(), - GetQuicFlag(quic_send_buffer_max_data_slice_size)); QuicConsumedData consumed = stream_->WriteMemSlices( - storage.ToSpan(), /*fin=*/options.send_fin(), + slice.empty() ? absl::Span<quiche::QuicheMemSlice>() + : absl::MakeSpan(&slice, 1), + /*fin=*/options.send_fin(), /*buffer_uncondtionally=*/options.buffer_unconditionally()); if (consumed.bytes_consumed == total_size) {
diff --git a/quiche/web_transport/encapsulated/encapsulated_web_transport.cc b/quiche/web_transport/encapsulated/encapsulated_web_transport.cc index b6380e8..30ab1d4 100644 --- a/quiche/web_transport/encapsulated/encapsulated_web_transport.cc +++ b/quiche/web_transport/encapsulated/encapsulated_web_transport.cc
@@ -36,6 +36,7 @@ #include "quiche/common/quiche_circular_deque.h" #include "quiche/common/quiche_status_utils.h" #include "quiche/common/quiche_stream.h" +#include "quiche/common/vectorized_io_utils.h" #include "quiche/web_transport/web_transport.h" namespace webtransport {