gfe-relnote: For QUIC BBR tests, add the ability to save test results and compare with saved results. Test only, not protected.
Note for Chromium/Envoy merge: quic_test_output_impl.(h|cc) needs to implement two functions:
1) QuicSaveTestOutputImpl: Can be implemented as a call to QuicRecordTestOutputToFile.
2) QuicLoadTestOutputImpl: Can be implemented using platform-specific file read apis. For example: use base::ReadFileToString in Chromium.
Usage:
$ blaze build third_party/quic/core/congestion_control:bbr_sender_test
// Save test results(one file per test) to <some directory>
$ QUIC_TEST_OUTPUT_DIR=<some directory> blaze-bin/third_party/quic/core/congestion_control/bbr_sender_test --quic_bbr_test_regression_mode=record
// Compare with test results in <some directory>, test fails if e.g. a test takes longer than the duration recorded in test results.
$ QUIC_TEST_OUTPUT_DIR=<some directory> blaze-bin/third_party/quic/core/congestion_control/bbr_sender_test --quic_bbr_test_regression_mode=regress
PiperOrigin-RevId: 284566985
Change-Id: I9610766cde7d014d2ddcdb629cd83626362af8d6
diff --git a/quic/core/congestion_control/bbr2_simulator_test.cc b/quic/core/congestion_control/bbr2_simulator_test.cc
index 005e11d..811161d 100644
--- a/quic/core/congestion_control/bbr2_simulator_test.cc
+++ b/quic/core/congestion_control/bbr2_simulator_test.cc
@@ -18,6 +18,8 @@
#include "net/third_party/quiche/src/quic/test_tools/quic_connection_peer.h"
#include "net/third_party/quiche/src/quic/test_tools/quic_sent_packet_manager_peer.h"
#include "net/third_party/quiche/src/quic/test_tools/quic_test_utils.h"
+#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_result.pb.h"
+#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_utils.h"
#include "net/third_party/quiche/src/quic/test_tools/simulator/link.h"
#include "net/third_party/quiche/src/quic/test_tools/simulator/quic_endpoint.h"
#include "net/third_party/quiche/src/quic/test_tools/simulator/simulator.h"
@@ -28,6 +30,14 @@
using testing::Ge;
using testing::Le;
+DEFINE_QUIC_COMMAND_LINE_FLAG(
+ std::string,
+ quic_bbr2_test_regression_mode,
+ "",
+ "One of a) 'record' to record test result (one file per test), or "
+ "b) 'regress' to regress against recorded results, or "
+ "c) <anything else> for non-regression mode.");
+
namespace quic {
using CyclePhase = Bbr2ProbeBwMode::CyclePhase;
@@ -112,6 +122,37 @@
// srtt. Individual test can enable it via QuicConnectionPeer::SetAckMode().
SetQuicReloadableFlag(quic_enable_ack_decimation, false);
}
+
+ void SetUp() override {
+ if (GetQuicFlag(FLAGS_quic_bbr2_test_regression_mode) == "regress") {
+ SendAlgorithmTestResult expected;
+ ASSERT_TRUE(LoadSendAlgorithmTestResult(&expected));
+ random_seed_ = expected.random_seed();
+ } else {
+ random_seed_ = QuicRandom::GetInstance()->RandUint64();
+ }
+ random_.set_seed(random_seed_);
+ QUIC_LOG(INFO) << "Using random seed: " << random_seed_;
+ }
+
+ ~Bbr2SimulatorTest() override {
+ const std::string regression_mode =
+ GetQuicFlag(FLAGS_quic_bbr2_test_regression_mode);
+ const QuicTime::Delta simulated_duration =
+ SimulatedNow() - QuicTime::Zero();
+ if (regression_mode == "record") {
+ RecordSendAlgorithmTestResult(random_seed_,
+ simulated_duration.ToMicroseconds());
+ } else if (regression_mode == "regress") {
+ CompareSendAlgorithmTestResult(simulated_duration.ToMicroseconds());
+ }
+ }
+
+ QuicTime SimulatedNow() const { return simulator_.GetClock()->Now(); }
+
+ simulator::Simulator simulator_;
+ uint64_t random_seed_;
+ SimpleRandom random_;
};
class Bbr2DefaultTopologyTest : public Bbr2SimulatorTest {
@@ -129,10 +170,6 @@
TestConnectionId(42)) {
sender_ = SetupBbr2Sender(&sender_endpoint_);
- uint64_t seed = QuicRandom::GetInstance()->RandUint64();
- random_.set_seed(seed);
- QUIC_LOG(INFO) << "Bbr2DefaultTopologyTest set up. Seed: " << seed;
-
simulator_.set_random_generator(&random_);
}
@@ -274,8 +311,6 @@
return false;
}
- QuicTime SimulatedNow() const { return simulator_.GetClock()->Now(); }
-
const RttStats* rtt_stats() {
return sender_endpoint_.connection()->sent_packet_manager().GetRttStats();
}
@@ -291,11 +326,9 @@
sender_connection_stats().packets_sent;
}
- simulator::Simulator simulator_;
simulator::QuicEndpoint sender_endpoint_;
simulator::QuicEndpoint receiver_endpoint_;
Bbr2Sender* sender_;
- SimpleRandom random_;
std::unique_ptr<simulator::Switch> switch_;
std::unique_ptr<simulator::TrafficPolicer> sender_policer_;
@@ -786,10 +819,6 @@
"Receiver multiplexer", receiver_endpoint_pointers);
sender_1_ = SetupBbr2Sender(sender_endpoints_[0].get());
- uint64_t seed = QuicRandom::GetInstance()->RandUint64();
- random_.set_seed(seed);
- QUIC_LOG(INFO) << "Bbr2MultiSenderTest set up. Seed: " << seed;
-
simulator_.set_random_generator(&random_);
}
@@ -873,8 +902,6 @@
}
}
- QuicTime SimulatedNow() const { return simulator_.GetClock()->Now(); }
-
QuicConnection* sender_connection(size_t which) {
return sender_endpoints_[which]->connection();
}
@@ -888,12 +915,10 @@
sender_connection_stats(which).packets_sent;
}
- simulator::Simulator simulator_;
std::vector<std::unique_ptr<simulator::QuicEndpoint>> sender_endpoints_;
std::vector<std::unique_ptr<simulator::QuicEndpoint>> receiver_endpoints_;
std::unique_ptr<simulator::QuicEndpointMultiplexer> receiver_multiplexer_;
Bbr2Sender* sender_1_;
- SimpleRandom random_;
std::unique_ptr<simulator::Switch> switch_;
std::vector<std::unique_ptr<simulator::SymmetricLink>> network_links_;
diff --git a/quic/core/congestion_control/bbr_sender_test.cc b/quic/core/congestion_control/bbr_sender_test.cc
index 6acfe80..1f45089 100644
--- a/quic/core/congestion_control/bbr_sender_test.cc
+++ b/quic/core/congestion_control/bbr_sender_test.cc
@@ -20,6 +20,8 @@
#include "net/third_party/quiche/src/quic/test_tools/quic_connection_peer.h"
#include "net/third_party/quiche/src/quic/test_tools/quic_sent_packet_manager_peer.h"
#include "net/third_party/quiche/src/quic/test_tools/quic_test_utils.h"
+#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_result.pb.h"
+#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_utils.h"
#include "net/third_party/quiche/src/quic/test_tools/simulator/quic_endpoint.h"
#include "net/third_party/quiche/src/quic/test_tools/simulator/simulator.h"
#include "net/third_party/quiche/src/quic/test_tools/simulator/switch.h"
@@ -28,6 +30,14 @@
using testing::Ge;
using testing::Le;
+DEFINE_QUIC_COMMAND_LINE_FLAG(
+ std::string,
+ quic_bbr_test_regression_mode,
+ "",
+ "One of a) 'record' to record test result (one file per test), or "
+ "b) 'regress' to regress against recorded results, or "
+ "c) <anything else> for non-regression mode.");
+
namespace quic {
namespace test {
@@ -101,10 +111,30 @@
clock_ = simulator_.GetClock();
simulator_.set_random_generator(&random_);
+ }
- uint64_t seed = QuicRandom::GetInstance()->RandUint64();
- random_.set_seed(seed);
- QUIC_LOG(INFO) << "BbrSenderTest simulator set up. Seed: " << seed;
+ void SetUp() override {
+ if (GetQuicFlag(FLAGS_quic_bbr_test_regression_mode) == "regress") {
+ SendAlgorithmTestResult expected;
+ ASSERT_TRUE(LoadSendAlgorithmTestResult(&expected));
+ random_seed_ = expected.random_seed();
+ } else {
+ random_seed_ = QuicRandom::GetInstance()->RandUint64();
+ }
+ random_.set_seed(random_seed_);
+ QUIC_LOG(INFO) << "BbrSenderTest simulator set up. Seed: " << random_seed_;
+ }
+
+ ~BbrSenderTest() {
+ const std::string regression_mode =
+ GetQuicFlag(FLAGS_quic_bbr_test_regression_mode);
+ const QuicTime::Delta simulated_duration = clock_->Now() - QuicTime::Zero();
+ if (regression_mode == "record") {
+ RecordSendAlgorithmTestResult(random_seed_,
+ simulated_duration.ToMicroseconds());
+ } else if (regression_mode == "regress") {
+ CompareSendAlgorithmTestResult(simulated_duration.ToMicroseconds());
+ }
}
simulator::Simulator simulator_;
@@ -118,6 +148,7 @@
std::unique_ptr<simulator::SymmetricLink> competing_sender_link_;
std::unique_ptr<simulator::SymmetricLink> receiver_link_;
+ uint64_t random_seed_;
SimpleRandom random_;
// Owned by different components of the connection.
diff --git a/quic/platform/api/quic_test_output.h b/quic/platform/api/quic_test_output.h
index dfdb865..f6bb443 100644
--- a/quic/platform/api/quic_test_output.h
+++ b/quic/platform/api/quic_test_output.h
@@ -10,6 +10,18 @@
namespace quic {
+// Save |data| into ${QUIC_TEST_OUTPUT_DIR}/filename. If a file with the same
+// path already exists, overwrite it.
+inline void QuicSaveTestOutput(QuicStringPiece filename, QuicStringPiece data) {
+ QuicSaveTestOutputImpl(filename, data);
+}
+
+// Load the content of ${QUIC_TEST_OUTPUT_DIR}/filename into |*data|.
+// Return whether it is successfully loaded.
+inline bool QuicLoadTestOutput(QuicStringPiece filename, std::string* data) {
+ return QuicLoadTestOutputImpl(filename, data);
+}
+
// Records a QUIC trace file(.qtr) into a directory specified by the
// QUIC_TEST_OUTPUT_DIR environment variable. Assumes that it's called from a
// unit test.
diff --git a/quic/test_tools/send_algorithm_test_result.proto b/quic/test_tools/send_algorithm_test_result.proto
new file mode 100644
index 0000000..a836c47
--- /dev/null
+++ b/quic/test_tools/send_algorithm_test_result.proto
@@ -0,0 +1,15 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+syntax = "proto2";
+
+option optimize_for = LITE_RUNTIME;
+
+package quic;
+
+message SendAlgorithmTestResult {
+ optional string test_name = 1;
+ optional uint64 random_seed = 2;
+ optional int64 simulated_duration_micros = 3;
+}
diff --git a/quic/test_tools/send_algorithm_test_utils.cc b/quic/test_tools/send_algorithm_test_utils.cc
new file mode 100644
index 0000000..122d6e5
--- /dev/null
+++ b/quic/test_tools/send_algorithm_test_utils.cc
@@ -0,0 +1,61 @@
+// Copyright (c) 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_utils.h"
+
+#include "net/third_party/quiche/src/quic/platform/api/quic_logging.h"
+#include "net/third_party/quiche/src/quic/platform/api/quic_str_cat.h"
+#include "net/third_party/quiche/src/quic/platform/api/quic_test.h"
+#include "net/third_party/quiche/src/quic/platform/api/quic_test_output.h"
+
+namespace quic {
+namespace test {
+
+bool LoadSendAlgorithmTestResult(SendAlgorithmTestResult* result) {
+ std::string test_result_file_content;
+ if (!QuicLoadTestOutput(GetSendAlgorithmTestResultFilename(),
+ &test_result_file_content)) {
+ return false;
+ }
+ return result->ParseFromString(test_result_file_content);
+}
+
+void RecordSendAlgorithmTestResult(uint64_t random_seed,
+ int64_t simulated_duration_micros) {
+ SendAlgorithmTestResult result;
+ result.set_test_name(GetFullSendAlgorithmTestName());
+ result.set_random_seed(random_seed);
+ result.set_simulated_duration_micros(simulated_duration_micros);
+
+ QuicSaveTestOutput(GetSendAlgorithmTestResultFilename(),
+ result.SerializeAsString());
+}
+
+void CompareSendAlgorithmTestResult(int64_t actual_simulated_duration_micros) {
+ SendAlgorithmTestResult expected;
+ ASSERT_TRUE(LoadSendAlgorithmTestResult(&expected));
+ QUIC_LOG(INFO) << "Loaded expected test result: "
+ << expected.ShortDebugString();
+
+ EXPECT_GE(expected.simulated_duration_micros(),
+ actual_simulated_duration_micros);
+}
+
+std::string GetFullSendAlgorithmTestName() {
+ const auto* test_info =
+ ::testing::UnitTest::GetInstance()->current_test_info();
+ const std::string type_param =
+ test_info->type_param() ? test_info->type_param() : "";
+ const std::string value_param =
+ test_info->value_param() ? test_info->value_param() : "";
+ return QuicStrCat(test_info->test_suite_name(), ".", test_info->name(), "_",
+ type_param, "_", value_param);
+}
+
+std::string GetSendAlgorithmTestResultFilename() {
+ return GetFullSendAlgorithmTestName() + ".test_result";
+}
+
+} // namespace test
+} // namespace quic
diff --git a/quic/test_tools/send_algorithm_test_utils.h b/quic/test_tools/send_algorithm_test_utils.h
new file mode 100644
index 0000000..c36218d
--- /dev/null
+++ b/quic/test_tools/send_algorithm_test_utils.h
@@ -0,0 +1,29 @@
+// Copyright (c) 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef QUICHE_QUIC_TEST_TOOLS_SEND_ALGORITHM_TEST_UTILS_H_
+#define QUICHE_QUIC_TEST_TOOLS_SEND_ALGORITHM_TEST_UTILS_H_
+
+#include "net/third_party/quiche/src/quic/test_tools/send_algorithm_test_result.pb.h"
+
+namespace quic {
+namespace test {
+
+bool LoadSendAlgorithmTestResult(SendAlgorithmTestResult* result);
+
+void RecordSendAlgorithmTestResult(uint64_t random_seed,
+ int64_t simulated_duration_micros);
+
+// Load the expected test result with LoadSendAlgorithmTestResult(), and compare
+// it with the actual results provided in the arguments.
+void CompareSendAlgorithmTestResult(int64_t actual_simulated_duration_micros);
+
+std::string GetFullSendAlgorithmTestName();
+
+std::string GetSendAlgorithmTestResultFilename();
+
+} // namespace test
+} // namespace quic
+
+#endif // QUICHE_QUIC_TEST_TOOLS_SEND_ALGORITHM_TEST_UTILS_H_
