Add multiqueue support to TunTapDevice. This change refactors TunTapDevice to support multiple queue file descriptors instead of a single file descriptor. PiperOrigin-RevId: 974008389

diff --git a/quiche/quic/qbone/bonnet/mock_tun_device.h b/quiche/quic/qbone/bonnet/mock_tun_device.h
index bc5b462..5e086ee 100644
--- a/quiche/quic/qbone/bonnet/mock_tun_device.h
+++ b/quiche/quic/qbone/bonnet/mock_tun_device.h

@@ -22,6 +22,8 @@
 
   MOCK_METHOD(int, GetReadFileDescriptor, (), (const, override));
   MOCK_METHOD(int, GetWriteFileDescriptor, (), (const, override));
+
+  MOCK_METHOD(int, OpenQueue, (), (override));
 };
 
 }  // namespace quic

diff --git a/quiche/quic/qbone/bonnet/tun_device.cc b/quiche/quic/qbone/bonnet/tun_device.cc
index b989467..64a2c3d 100644
--- a/quiche/quic/qbone/bonnet/tun_device.cc
+++ b/quiche/quic/qbone/bonnet/tun_device.cc

@@ -5,16 +5,16 @@
 #include "quiche/quic/qbone/bonnet/tun_device.h"
 
 #include <fcntl.h>
-#include <linux/filter.h>
 #include <linux/if_tun.h>
 #include <net/if.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 
+#include <cstring>
 #include <ios>
 #include <string>
+#include <vector>
 
-#include "absl/cleanup/cleanup.h"
 #include "absl/flags/flag.h"
 #include "quiche/quic/platform/api/quic_bug_tracker.h"
 #include "quiche/quic/platform/api/quic_logging.h"
@@ -35,7 +35,6 @@
       persist_(persist),
       setup_tun_(setup_tun),
       is_tap_(is_tap),
-      file_descriptor_(kInvalidFd),
       kernel_(*kernel) {}
 
 TunTapDevice::~TunTapDevice() {
@@ -95,6 +94,51 @@
   return NetdeviceIoctl(SIOCSIFFLAGS, reinterpret_cast<void*>(&if_request));
 }
 
+int TunTapDevice::GetReadFileDescriptor() const {
+  if (file_descriptors_.empty()) {
+    return kInvalidFd;
+  }
+  return file_descriptors_[0];
+}
+
+int TunTapDevice::GetWriteFileDescriptor() const {
+  if (file_descriptors_.empty()) {
+    return kInvalidFd;
+  }
+  return file_descriptors_[0];
+}
+
+int TunTapDevice::OpenQueue() {
+  struct ifreq if_request;
+  memset(&if_request, 0, sizeof(if_request));
+  interface_name_.copy(if_request.ifr_name, IFNAMSIZ);
+
+  if_request.ifr_flags = IFF_MULTI_QUEUE | IFF_NO_PI;
+  if (is_tap_) {
+    if_request.ifr_flags |= IFF_TAP;
+  } else {
+    if_request.ifr_flags |= IFF_TUN;
+  }
+
+  int new_fd = kInvalidFd;
+  if (!OpenFileDescriptor(kernel_,
+                          absl::GetFlag(FLAGS_qbone_client_tun_device_path),
+                          if_request, O_NONBLOCK | O_RDWR, persist_, &new_fd)) {
+    if (new_fd != kInvalidFd) {
+      kernel_.close(new_fd);
+    }
+    return kInvalidFd;
+  }
+
+  if (new_fd == kInvalidFd) {
+    QUIC_LOG(ERROR) << "OpenFileDescriptor succeeded with an invalid FD.";
+    return kInvalidFd;
+  }
+
+  file_descriptors_.push_back(new_fd);
+  return new_fd;
+}
+
 bool TunTapDevice::CheckFeatures(KernelInterface& kernel, int tun_device_fd) {
   unsigned int actual_features;
   if (kernel.ioctl(tun_device_fd, TUNGETFEATURES, &actual_features) != 0) {
@@ -141,47 +185,8 @@
 }
 
 bool TunTapDevice::OpenDevice() {
-  if (file_descriptor_ != kInvalidFd) {
-    CloseDevice();
-  }
-
-  struct ifreq if_request;
-  memset(&if_request, 0, sizeof(if_request));
-  // copy does not zero-terminate the result string, but we've memset the entire
-  // struct.
-  interface_name_.copy(if_request.ifr_name, IFNAMSIZ);
-
-  // Always set IFF_MULTI_QUEUE since a persistent device does not allow this
-  // flag to be flipped when re-opening it. The only way to flip this flag is to
-  // destroy the device and create a new one, but that deletes any existing
-  // routing associated with the interface, which makes the meaning of the
-  // 'persist' bit ambiguous.
-  if_request.ifr_flags = IFF_MULTI_QUEUE | IFF_NO_PI;
-  if (is_tap_) {
-    if_request.ifr_flags |= IFF_TAP;
-  } else {
-    if_request.ifr_flags |= IFF_TUN;
-  }
-
-  // When the device is running with IFF_MULTI_QUEUE set, each call to open will
-  // create a queue which can be used to read/write packets from/to the device.
-  bool successfully_opened = false;
-  auto cleanup = absl::MakeCleanup([this, &successfully_opened]() {
-    if (!successfully_opened) {
-      CloseDevice();
-    }
-  });
-
-  // We set O_NONBLOCK for good measure, but, from observation, all write()
-  // syscalls to the device seem to be synchronous regardless.
-  if (!OpenFileDescriptor(
-          kernel_, absl::GetFlag(FLAGS_qbone_client_tun_device_path),
-          if_request, O_NONBLOCK | O_RDWR, persist_, &file_descriptor_)) {
-    return false;
-  }
-
-  successfully_opened = true;
-  return successfully_opened;
+  CloseDevice();
+  return OpenQueue() != kInvalidFd;
 }
 
 // TODO(pengg): might be better to use netlink socket, once we have a library to
@@ -223,10 +228,10 @@
 }
 
 void TunTapDevice::CloseDevice() {
-  if (file_descriptor_ != kInvalidFd) {
-    kernel_.close(file_descriptor_);
-    file_descriptor_ = kInvalidFd;
+  for (int fd : file_descriptors_) {
+    kernel_.close(fd);
   }
+  file_descriptors_.clear();
 }
 
 }  // namespace quic

diff --git a/quiche/quic/qbone/bonnet/tun_device.h b/quiche/quic/qbone/bonnet/tun_device.h
index 8eb1615..20ee320 100644
--- a/quiche/quic/qbone/bonnet/tun_device.h
+++ b/quiche/quic/qbone/bonnet/tun_device.h

@@ -5,6 +5,9 @@
 #ifndef QUICHE_QUIC_QBONE_BONNET_TUN_DEVICE_H_
 #define QUICHE_QUIC_QBONE_BONNET_TUN_DEVICE_H_
 
+#include <net/if.h>
+
+#include <cstddef>
 #include <string>
 #include <vector>
 
@@ -46,15 +49,20 @@
   // Marks the interface down to stop receiving packets.
   bool Down() override;
 
-  // Closes the open file descriptor for the TUN device (if one exists).
+  // Closes all open queue file descriptors for the TUN device.
   // It is safe to reinitialize and reuse this TunTapDevice after calling
   // CloseDevice.
   void CloseDevice() override;
 
-  // Get the file descriptors that can be used to send/receive packets.
+  // Get the file descriptors that can be used to send/receive packets for
+  // the primary queue (queue 0).
   // These return -1 when the TUN device is in an invalid state.
-  int GetReadFileDescriptor() const override { return file_descriptor_; }
-  int GetWriteFileDescriptor() const override { return file_descriptor_; }
+  int GetReadFileDescriptor() const override;
+  int GetWriteFileDescriptor() const override;
+
+  // Opens the next sequential queue file descriptor for this multiqueue device.
+  // Returns the open queue FD on success, or -1 on failure.
+  int OpenQueue() override;
 
   static bool CheckFeatures(KernelInterface& kernel, int tun_device_fd);
   static bool OpenFileDescriptor(KernelInterface& kernel,
@@ -62,7 +70,8 @@
                                  int flags, bool persist, int* return_fd);
 
  private:
-  // Creates or reopens the tun device.
+  // Resets the device by closing all open queues and opening a fresh primary
+  // queue (queue 0).
   bool OpenDevice();
 
   // Configure the interface.
@@ -76,8 +85,9 @@
   const bool persist_;
   const bool setup_tun_;
   const bool is_tap_;
-  int file_descriptor_;
   KernelInterface& kernel_;
+
+  std::vector<int> file_descriptors_;
 };
 
 }  // namespace quic

diff --git a/quiche/quic/qbone/bonnet/tun_device_interface.h b/quiche/quic/qbone/bonnet/tun_device_interface.h
index 1bec7e9..57444c6 100644
--- a/quiche/quic/qbone/bonnet/tun_device_interface.h
+++ b/quiche/quic/qbone/bonnet/tun_device_interface.h

@@ -5,8 +5,6 @@
 #ifndef QUICHE_QUIC_QBONE_BONNET_TUN_DEVICE_INTERFACE_H_
 #define QUICHE_QUIC_QBONE_BONNET_TUN_DEVICE_INTERFACE_H_
 
-#include <vector>
-
 namespace quic {
 
 // An interface with methods for interacting with a TUN device.
@@ -32,6 +30,10 @@
   // These return -1 when the TUN device is in an invalid state.
   virtual int GetReadFileDescriptor() const = 0;
   virtual int GetWriteFileDescriptor() const = 0;
+
+  // Opens the next sequential queue file descriptor for this multiqueue device.
+  // Returns the open queue FD on success, or -1 on failure.
+  virtual int OpenQueue() = 0;
 };
 
 }  // namespace quic

diff --git a/quiche/quic/qbone/bonnet/tun_device_test.cc b/quiche/quic/qbone/bonnet/tun_device_test.cc
index 1e3b9de..ccbab2d 100644
--- a/quiche/quic/qbone/bonnet/tun_device_test.cc
+++ b/quiche/quic/qbone/bonnet/tun_device_test.cc

@@ -41,7 +41,7 @@
   }
 
   // Set the expectations for calling Init().
-  void SetInitExpectations(int mtu, bool persist) {
+  void SetInitExpectations(int mtu, bool persist, bool is_tap = false) {
     EXPECT_CALL(mock_kernel_, open(StrEq("/dev/net/tun"), _))
         .Times(AnyNumber())
         .WillRepeatedly([this](Unused, Unused) {
@@ -57,9 +57,11 @@
         });
     EXPECT_CALL(mock_kernel_, ioctl(_, TUNSETIFF, _))
         .Times(AnyNumber())
-        .WillRepeatedly([](Unused, Unused, void* argp) {
+        .WillRepeatedly([is_tap](Unused, Unused, void* argp) {
           auto* ifr = reinterpret_cast<struct ifreq*>(argp);
-          EXPECT_EQ(IFF_TUN | IFF_MULTI_QUEUE | IFF_NO_PI, ifr->ifr_flags);
+          int expected_flags =
+              (is_tap ? IFF_TAP : IFF_TUN) | IFF_MULTI_QUEUE | IFF_NO_PI;
+          EXPECT_EQ(expected_flags, ifr->ifr_flags);
           EXPECT_THAT(ifr->ifr_name, StrEq(kDeviceName));
           return 0;
         });
@@ -215,5 +217,81 @@
   EXPECT_FALSE(tun_device.Up());
 }
 
+TEST_F(TunDeviceTest, OpensMultipleQueuesDynamically) {
+  SetInitExpectations(/* mtu = */ 1500, /* persist = */ false);
+  TunTapDevice tun_device(kDeviceName, 1500, false, true, false, &mock_kernel_);
+  ASSERT_TRUE(tun_device.Init());
+
+  int fd_0 = tun_device.GetReadFileDescriptor();
+  EXPECT_GT(fd_0, -1);
+  EXPECT_EQ(tun_device.GetWriteFileDescriptor(), fd_0);
+
+  int fd_1 = tun_device.OpenQueue();
+  EXPECT_GT(fd_1, -1);
+  EXPECT_NE(fd_0, fd_1);
+
+  int fd_2 = tun_device.OpenQueue();
+  EXPECT_GT(fd_2, -1);
+  EXPECT_NE(fd_1, fd_2);
+
+  tun_device.CloseDevice();
+  EXPECT_EQ(tun_device.GetReadFileDescriptor(), -1);
+  EXPECT_EQ(tun_device.GetWriteFileDescriptor(), -1);
+
+  // Calling CloseDevice() again is idempotent.
+  tun_device.CloseDevice();
+  EXPECT_EQ(tun_device.GetReadFileDescriptor(), -1);
+  EXPECT_EQ(tun_device.GetWriteFileDescriptor(), -1);
+
+  ExpectDown(/* fail = */ false);
+}
+
+TEST_F(TunDeviceTest, SecondaryQueueOpenFailsCleanly) {
+  SetInitExpectations(/* mtu = */ 1500, /* persist = */ false);
+  TunTapDevice tun_device(kDeviceName, 1500, false, true, false, &mock_kernel_);
+  ASSERT_TRUE(tun_device.Init());
+
+  // Expect opening queue 1 to fail during TUNSETIFF.
+  EXPECT_CALL(mock_kernel_, ioctl(_, TUNSETIFF, _)).WillOnce(Return(-1));
+
+  EXPECT_EQ(tun_device.OpenQueue(), -1);
+  EXPECT_GT(tun_device.GetReadFileDescriptor(), -1);
+
+  ExpectDown(/* fail = */ false);
+}
+
+TEST_F(TunDeviceTest, CallingInitWhileQueuesAreOpenResetsToPrimaryQueue) {
+  SetInitExpectations(/* mtu = */ 1500, /* persist = */ false);
+  TunTapDevice tun_device(kDeviceName, 1500, false, true, false, &mock_kernel_);
+  ASSERT_TRUE(tun_device.Init());
+  int fd_0_orig = tun_device.GetReadFileDescriptor();
+  int fd_1_orig = tun_device.OpenQueue();
+  int fd_2_orig = tun_device.OpenQueue();
+  ASSERT_GT(fd_1_orig, -1);
+  ASSERT_GT(fd_2_orig, -1);
+
+  // Calling Init() again must close all existing queues and reset the device
+  // to only have a single primary queue.
+  ASSERT_TRUE(tun_device.Init());
+  int fd_0_new = tun_device.GetReadFileDescriptor();
+  EXPECT_GT(fd_0_new, -1);
+  EXPECT_NE(fd_0_new, fd_0_orig);
+
+  ExpectDown(/* fail = */ false);
+}
+
+TEST_F(TunDeviceTest, TapDeviceSetsTapFlag) {
+  SetInitExpectations(/* mtu = */ 1500, /* persist = */ false,
+                      /* is_tap = */ true);
+  TunTapDevice tap_device(kDeviceName, 1500, false, true, /* is_tap = */ true,
+                          &mock_kernel_);
+  EXPECT_TRUE(tap_device.Init());
+
+  int fd_1 = tap_device.OpenQueue();
+  EXPECT_GT(fd_1, -1);
+
+  ExpectDown(/* fail = */ false);
+}
+
 }  // namespace
 }  // namespace quic::test