gfe-relnote: For QUIC BBR tests, add the ability to save test results and compare with saved results. Test only, not protected. Note for Chromium/Envoy merge: quic_test_output_impl.(h|cc) needs to implement two functions: 1) QuicSaveTestOutputImpl: Can be implemented as a call to QuicRecordTestOutputToFile. 2) QuicLoadTestOutputImpl: Can be implemented using platform-specific file read apis. For example: use base::ReadFileToString in Chromium. Usage: $ blaze build third_party/quic/core/congestion_control:bbr_sender_test // Save test results(one file per test) to <some directory> $ QUIC_TEST_OUTPUT_DIR=<some directory> blaze-bin/third_party/quic/core/congestion_control/bbr_sender_test --quic_bbr_test_regression_mode=record // Compare with test results in <some directory>, test fails if e.g. a test takes longer than the duration recorded in test results. $ QUIC_TEST_OUTPUT_DIR=<some directory> blaze-bin/third_party/quic/core/congestion_control/bbr_sender_test --quic_bbr_test_regression_mode=regress PiperOrigin-RevId: 284566985 Change-Id: I9610766cde7d014d2ddcdb629cd83626362af8d6
diff --git a/quic/core/congestion_control/bbr2_simulator_test.cc b/quic/core/congestion_control/bbr2_simulator_test.cc index 005e11d..811161d 100644 --- a/quic/core/congestion_control/bbr2_simulator_test.cc +++ b/quic/core/congestion_control/bbr2_simulator_test.cc
@@ -18,6 +18,8 @@ #include "net/third_party/quiche/src/quic/test_tools/quic_connection_peer.h" #include "net/third_party/quiche/src/quic/test_tools/quic_sent_packet_manager_peer.h" #include "net/third_party/quiche/src/quic/test_tools/quic_test_utils.h" +#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_result.pb.h" +#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_utils.h" #include "net/third_party/quiche/src/quic/test_tools/simulator/link.h" #include "net/third_party/quiche/src/quic/test_tools/simulator/quic_endpoint.h" #include "net/third_party/quiche/src/quic/test_tools/simulator/simulator.h" @@ -28,6 +30,14 @@ using testing::Ge; using testing::Le; +DEFINE_QUIC_COMMAND_LINE_FLAG( + std::string, + quic_bbr2_test_regression_mode, + "", + "One of a) 'record' to record test result (one file per test), or " + "b) 'regress' to regress against recorded results, or " + "c) <anything else> for non-regression mode."); + namespace quic { using CyclePhase = Bbr2ProbeBwMode::CyclePhase; @@ -112,6 +122,37 @@ // srtt. Individual test can enable it via QuicConnectionPeer::SetAckMode(). SetQuicReloadableFlag(quic_enable_ack_decimation, false); } + + void SetUp() override { + if (GetQuicFlag(FLAGS_quic_bbr2_test_regression_mode) == "regress") { + SendAlgorithmTestResult expected; + ASSERT_TRUE(LoadSendAlgorithmTestResult(&expected)); + random_seed_ = expected.random_seed(); + } else { + random_seed_ = QuicRandom::GetInstance()->RandUint64(); + } + random_.set_seed(random_seed_); + QUIC_LOG(INFO) << "Using random seed: " << random_seed_; + } + + ~Bbr2SimulatorTest() override { + const std::string regression_mode = + GetQuicFlag(FLAGS_quic_bbr2_test_regression_mode); + const QuicTime::Delta simulated_duration = + SimulatedNow() - QuicTime::Zero(); + if (regression_mode == "record") { + RecordSendAlgorithmTestResult(random_seed_, + simulated_duration.ToMicroseconds()); + } else if (regression_mode == "regress") { + CompareSendAlgorithmTestResult(simulated_duration.ToMicroseconds()); + } + } + + QuicTime SimulatedNow() const { return simulator_.GetClock()->Now(); } + + simulator::Simulator simulator_; + uint64_t random_seed_; + SimpleRandom random_; }; class Bbr2DefaultTopologyTest : public Bbr2SimulatorTest { @@ -129,10 +170,6 @@ TestConnectionId(42)) { sender_ = SetupBbr2Sender(&sender_endpoint_); - uint64_t seed = QuicRandom::GetInstance()->RandUint64(); - random_.set_seed(seed); - QUIC_LOG(INFO) << "Bbr2DefaultTopologyTest set up. Seed: " << seed; - simulator_.set_random_generator(&random_); } @@ -274,8 +311,6 @@ return false; } - QuicTime SimulatedNow() const { return simulator_.GetClock()->Now(); } - const RttStats* rtt_stats() { return sender_endpoint_.connection()->sent_packet_manager().GetRttStats(); } @@ -291,11 +326,9 @@ sender_connection_stats().packets_sent; } - simulator::Simulator simulator_; simulator::QuicEndpoint sender_endpoint_; simulator::QuicEndpoint receiver_endpoint_; Bbr2Sender* sender_; - SimpleRandom random_; std::unique_ptr<simulator::Switch> switch_; std::unique_ptr<simulator::TrafficPolicer> sender_policer_; @@ -786,10 +819,6 @@ "Receiver multiplexer", receiver_endpoint_pointers); sender_1_ = SetupBbr2Sender(sender_endpoints_[0].get()); - uint64_t seed = QuicRandom::GetInstance()->RandUint64(); - random_.set_seed(seed); - QUIC_LOG(INFO) << "Bbr2MultiSenderTest set up. Seed: " << seed; - simulator_.set_random_generator(&random_); } @@ -873,8 +902,6 @@ } } - QuicTime SimulatedNow() const { return simulator_.GetClock()->Now(); } - QuicConnection* sender_connection(size_t which) { return sender_endpoints_[which]->connection(); } @@ -888,12 +915,10 @@ sender_connection_stats(which).packets_sent; } - simulator::Simulator simulator_; std::vector<std::unique_ptr<simulator::QuicEndpoint>> sender_endpoints_; std::vector<std::unique_ptr<simulator::QuicEndpoint>> receiver_endpoints_; std::unique_ptr<simulator::QuicEndpointMultiplexer> receiver_multiplexer_; Bbr2Sender* sender_1_; - SimpleRandom random_; std::unique_ptr<simulator::Switch> switch_; std::vector<std::unique_ptr<simulator::SymmetricLink>> network_links_;
diff --git a/quic/core/congestion_control/bbr_sender_test.cc b/quic/core/congestion_control/bbr_sender_test.cc index 6acfe80..1f45089 100644 --- a/quic/core/congestion_control/bbr_sender_test.cc +++ b/quic/core/congestion_control/bbr_sender_test.cc
@@ -20,6 +20,8 @@ #include "net/third_party/quiche/src/quic/test_tools/quic_connection_peer.h" #include "net/third_party/quiche/src/quic/test_tools/quic_sent_packet_manager_peer.h" #include "net/third_party/quiche/src/quic/test_tools/quic_test_utils.h" +#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_result.pb.h" +#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_utils.h" #include "net/third_party/quiche/src/quic/test_tools/simulator/quic_endpoint.h" #include "net/third_party/quiche/src/quic/test_tools/simulator/simulator.h" #include "net/third_party/quiche/src/quic/test_tools/simulator/switch.h" @@ -28,6 +30,14 @@ using testing::Ge; using testing::Le; +DEFINE_QUIC_COMMAND_LINE_FLAG( + std::string, + quic_bbr_test_regression_mode, + "", + "One of a) 'record' to record test result (one file per test), or " + "b) 'regress' to regress against recorded results, or " + "c) <anything else> for non-regression mode."); + namespace quic { namespace test { @@ -101,10 +111,30 @@ clock_ = simulator_.GetClock(); simulator_.set_random_generator(&random_); + } - uint64_t seed = QuicRandom::GetInstance()->RandUint64(); - random_.set_seed(seed); - QUIC_LOG(INFO) << "BbrSenderTest simulator set up. Seed: " << seed; + void SetUp() override { + if (GetQuicFlag(FLAGS_quic_bbr_test_regression_mode) == "regress") { + SendAlgorithmTestResult expected; + ASSERT_TRUE(LoadSendAlgorithmTestResult(&expected)); + random_seed_ = expected.random_seed(); + } else { + random_seed_ = QuicRandom::GetInstance()->RandUint64(); + } + random_.set_seed(random_seed_); + QUIC_LOG(INFO) << "BbrSenderTest simulator set up. Seed: " << random_seed_; + } + + ~BbrSenderTest() { + const std::string regression_mode = + GetQuicFlag(FLAGS_quic_bbr_test_regression_mode); + const QuicTime::Delta simulated_duration = clock_->Now() - QuicTime::Zero(); + if (regression_mode == "record") { + RecordSendAlgorithmTestResult(random_seed_, + simulated_duration.ToMicroseconds()); + } else if (regression_mode == "regress") { + CompareSendAlgorithmTestResult(simulated_duration.ToMicroseconds()); + } } simulator::Simulator simulator_; @@ -118,6 +148,7 @@ std::unique_ptr<simulator::SymmetricLink> competing_sender_link_; std::unique_ptr<simulator::SymmetricLink> receiver_link_; + uint64_t random_seed_; SimpleRandom random_; // Owned by different components of the connection.
diff --git a/quic/platform/api/quic_test_output.h b/quic/platform/api/quic_test_output.h index dfdb865..f6bb443 100644 --- a/quic/platform/api/quic_test_output.h +++ b/quic/platform/api/quic_test_output.h
@@ -10,6 +10,18 @@ namespace quic { +// Save |data| into ${QUIC_TEST_OUTPUT_DIR}/filename. If a file with the same +// path already exists, overwrite it. +inline void QuicSaveTestOutput(QuicStringPiece filename, QuicStringPiece data) { + QuicSaveTestOutputImpl(filename, data); +} + +// Load the content of ${QUIC_TEST_OUTPUT_DIR}/filename into |*data|. +// Return whether it is successfully loaded. +inline bool QuicLoadTestOutput(QuicStringPiece filename, std::string* data) { + return QuicLoadTestOutputImpl(filename, data); +} + // Records a QUIC trace file(.qtr) into a directory specified by the // QUIC_TEST_OUTPUT_DIR environment variable. Assumes that it's called from a // unit test.
diff --git a/quic/test_tools/send_algorithm_test_result.proto b/quic/test_tools/send_algorithm_test_result.proto new file mode 100644 index 0000000..a836c47 --- /dev/null +++ b/quic/test_tools/send_algorithm_test_result.proto
@@ -0,0 +1,15 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package quic; + +message SendAlgorithmTestResult { + optional string test_name = 1; + optional uint64 random_seed = 2; + optional int64 simulated_duration_micros = 3; +}
diff --git a/quic/test_tools/send_algorithm_test_utils.cc b/quic/test_tools/send_algorithm_test_utils.cc new file mode 100644 index 0000000..122d6e5 --- /dev/null +++ b/quic/test_tools/send_algorithm_test_utils.cc
@@ -0,0 +1,61 @@ +// Copyright (c) 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_utils.h" + +#include "net/third_party/quiche/src/quic/platform/api/quic_logging.h" +#include "net/third_party/quiche/src/quic/platform/api/quic_str_cat.h" +#include "net/third_party/quiche/src/quic/platform/api/quic_test.h" +#include "net/third_party/quiche/src/quic/platform/api/quic_test_output.h" + +namespace quic { +namespace test { + +bool LoadSendAlgorithmTestResult(SendAlgorithmTestResult* result) { + std::string test_result_file_content; + if (!QuicLoadTestOutput(GetSendAlgorithmTestResultFilename(), + &test_result_file_content)) { + return false; + } + return result->ParseFromString(test_result_file_content); +} + +void RecordSendAlgorithmTestResult(uint64_t random_seed, + int64_t simulated_duration_micros) { + SendAlgorithmTestResult result; + result.set_test_name(GetFullSendAlgorithmTestName()); + result.set_random_seed(random_seed); + result.set_simulated_duration_micros(simulated_duration_micros); + + QuicSaveTestOutput(GetSendAlgorithmTestResultFilename(), + result.SerializeAsString()); +} + +void CompareSendAlgorithmTestResult(int64_t actual_simulated_duration_micros) { + SendAlgorithmTestResult expected; + ASSERT_TRUE(LoadSendAlgorithmTestResult(&expected)); + QUIC_LOG(INFO) << "Loaded expected test result: " + << expected.ShortDebugString(); + + EXPECT_GE(expected.simulated_duration_micros(), + actual_simulated_duration_micros); +} + +std::string GetFullSendAlgorithmTestName() { + const auto* test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + const std::string type_param = + test_info->type_param() ? test_info->type_param() : ""; + const std::string value_param = + test_info->value_param() ? test_info->value_param() : ""; + return QuicStrCat(test_info->test_suite_name(), ".", test_info->name(), "_", + type_param, "_", value_param); +} + +std::string GetSendAlgorithmTestResultFilename() { + return GetFullSendAlgorithmTestName() + ".test_result"; +} + +} // namespace test +} // namespace quic
diff --git a/quic/test_tools/send_algorithm_test_utils.h b/quic/test_tools/send_algorithm_test_utils.h new file mode 100644 index 0000000..c36218d --- /dev/null +++ b/quic/test_tools/send_algorithm_test_utils.h
@@ -0,0 +1,29 @@ +// Copyright (c) 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef QUICHE_QUIC_TEST_TOOLS_SEND_ALGORITHM_TEST_UTILS_H_ +#define QUICHE_QUIC_TEST_TOOLS_SEND_ALGORITHM_TEST_UTILS_H_ + +#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_result.pb.h" + +namespace quic { +namespace test { + +bool LoadSendAlgorithmTestResult(SendAlgorithmTestResult* result); + +void RecordSendAlgorithmTestResult(uint64_t random_seed, + int64_t simulated_duration_micros); + +// Load the expected test result with LoadSendAlgorithmTestResult(), and compare +// it with the actual results provided in the arguments. +void CompareSendAlgorithmTestResult(int64_t actual_simulated_duration_micros); + +std::string GetFullSendAlgorithmTestName(); + +std::string GetSendAlgorithmTestResultFilename(); + +} // namespace test +} // namespace quic + +#endif // QUICHE_QUIC_TEST_TOOLS_SEND_ALGORITHM_TEST_UTILS_H_