diff --git a/quic/core/quic_connection.cc b/quic/core/quic_connection.cc
index 4e3b40b..3e8f834 100644
--- a/quic/core/quic_connection.cc
+++ b/quic/core/quic_connection.cc
@@ -778,9 +778,7 @@
   // confirmed.
   if (level == ENCRYPTION_FORWARD_SECURE &&
       perspective_ == Perspective::IS_SERVER) {
-    sent_packet_manager_.SetHandshakeConfirmed();
-    // This may have changed the retransmission timer, so re-arm it.
-    SetRetransmissionAlarm();
+    OnHandshakeComplete();
   }
 }
 
@@ -3716,6 +3714,7 @@
 void QuicConnection::SendAllPendingAcks() {
   DCHECK(SupportsMultiplePacketNumberSpaces());
   QUIC_DVLOG(1) << ENDPOINT << "Trying to send all pending ACKs";
+  ack_alarm_->Cancel();
   // Latches current encryption level.
   const EncryptionLevel current_encryption_level = encryption_level_;
   for (int8_t i = INITIAL_DATA; i <= APPLICATION_DATA; ++i) {
diff --git a/quic/core/quic_connection.h b/quic/core/quic_connection.h
index c20a3ef..6a5062a 100644
--- a/quic/core/quic_connection.h
+++ b/quic/core/quic_connection.h
@@ -533,9 +533,10 @@
   // ack_frame().
   const QuicFrame GetUpdatedAckFrame();
 
-  // Called by the crypto stream when the handshake completes. In the server's
-  // case this is when the SHLO has been ACKed. Clients call this on receipt of
-  // the SHLO.
+  // Called when the handshake completes. On the client side, handshake
+  // completes on receipt of SHLO. On the server side, handshake completes when
+  // SHLO gets ACKed (or a forward secure packet gets decrypted successfully).
+  // TODO(fayang): Add a guard that this only gets called once.
   void OnHandshakeComplete();
 
   // Accessors
diff --git a/quic/core/tls_client_handshaker.cc b/quic/core/tls_client_handshaker.cc
index 3623901..f362f1a 100644
--- a/quic/core/tls_client_handshaker.cc
+++ b/quic/core/tls_client_handshaker.cc
@@ -303,6 +303,7 @@
   handshake_confirmed_ = true;
   session()->OnCryptoHandshakeEvent(QuicSession::ENCRYPTION_ESTABLISHED);
   session()->OnCryptoHandshakeEvent(QuicSession::HANDSHAKE_CONFIRMED);
+  session()->connection()->OnHandshakeComplete();
 }
 
 enum ssl_verify_result_t TlsClientHandshaker::VerifyCert(uint8_t* out_alert) {
diff --git a/quic/core/tls_handshaker_test.cc b/quic/core/tls_handshaker_test.cc
index 35efa9f..00c62d6 100644
--- a/quic/core/tls_handshaker_test.cc
+++ b/quic/core/tls_handshaker_test.cc
@@ -317,6 +317,9 @@
 };
 
 TEST_F(TlsHandshakerTest, CryptoHandshake) {
+  EXPECT_FALSE(client_conn_->IsHandshakeConfirmed());
+  EXPECT_FALSE(server_conn_->IsHandshakeConfirmed());
+
   EXPECT_CALL(*client_conn_, CloseConnection(_, _, _)).Times(0);
   EXPECT_CALL(*server_conn_, CloseConnection(_, _, _)).Times(0);
   EXPECT_CALL(client_session_,
@@ -332,6 +335,8 @@
   EXPECT_TRUE(client_stream_->encryption_established());
   EXPECT_TRUE(server_stream_->handshake_confirmed());
   EXPECT_TRUE(server_stream_->encryption_established());
+  EXPECT_TRUE(client_conn_->IsHandshakeConfirmed());
+  EXPECT_FALSE(server_conn_->IsHandshakeConfirmed());
 }
 
 TEST_F(TlsHandshakerTest, HandshakeWithAsyncProofSource) {