fix(api): prevent idle TCP connections from blocking the accept loop
Move per-connection protocol detection (TLS handshake, reader.Peek) out of the accept loop and into a per-connection goroutine. An idle TCP connection that never sends bytes would previously block Peek(1) indefinitely, preventing all subsequent connections from being accepted and making the management/API server unresponsive. Closes #3267
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
@@ -48,68 +49,88 @@ func (s *Server) acceptMuxConnections(listener net.Listener, httpListener *muxLi
|
||||
continue
|
||||
}
|
||||
|
||||
tlsConn, ok := conn.(*tls.Conn)
|
||||
if ok {
|
||||
if errHandshake := tlsConn.Handshake(); errHandshake != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after TLS handshake error: %v", errClose)
|
||||
}
|
||||
continue
|
||||
}
|
||||
proto := strings.TrimSpace(tlsConn.ConnectionState().NegotiatedProtocol)
|
||||
if proto == "h2" || proto == "http/1.1" {
|
||||
if httpListener == nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection: %v", errClose)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if errPut := httpListener.Put(tlsConn); errPut != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after HTTP routing failure: %v", errClose)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
reader := bufio.NewReader(conn)
|
||||
prefix, errPeek := reader.Peek(1)
|
||||
if errPeek != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after protocol peek failure: %v", errClose)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if isRedisRESPPrefix(prefix[0]) {
|
||||
if s.cfg != nil && s.cfg.Home.Enabled {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close redis connection while home mode is enabled: %v", errClose)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !s.managementRoutesEnabled.Load() {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close redis connection while management is disabled: %v", errClose)
|
||||
}
|
||||
continue
|
||||
}
|
||||
go s.handleRedisConnection(conn, reader)
|
||||
continue
|
||||
}
|
||||
|
||||
if httpListener == nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection without HTTP listener: %v", errClose)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if errPut := httpListener.Put(&bufferedConn{Conn: conn, reader: reader}); errPut != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after HTTP routing failure: %v", errClose)
|
||||
}
|
||||
}
|
||||
// Dispatch each connection to a goroutine so that slow/idle clients
|
||||
// cannot block the accept loop. Previously, TLS handshake and
|
||||
// reader.Peek(1) were performed inline; an idle TCP connection that
|
||||
// never sent bytes would block Peek indefinitely, preventing all
|
||||
// subsequent connections from being accepted (issue #3267).
|
||||
go s.routeMuxConnection(conn, httpListener)
|
||||
}
|
||||
}
|
||||
|
||||
// routeMuxConnection performs per-connection protocol detection and routing.
|
||||
func (s *Server) routeMuxConnection(conn net.Conn, httpListener *muxListener) {
|
||||
// Set a read deadline so that idle connections that never send bytes do not
|
||||
// leak goroutines and file descriptors. The deadline is cleared once the
|
||||
// connection is successfully routed to its handler.
|
||||
const muxSniffDeadline = 10 * time.Second
|
||||
_ = conn.SetReadDeadline(time.Now().Add(muxSniffDeadline))
|
||||
|
||||
tlsConn, ok := conn.(*tls.Conn)
|
||||
if ok {
|
||||
if errHandshake := tlsConn.Handshake(); errHandshake != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after TLS handshake error: %v", errClose)
|
||||
}
|
||||
return
|
||||
}
|
||||
proto := strings.TrimSpace(tlsConn.ConnectionState().NegotiatedProtocol)
|
||||
if proto == "h2" || proto == "http/1.1" {
|
||||
if httpListener == nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection: %v", errClose)
|
||||
}
|
||||
return
|
||||
}
|
||||
if errPut := httpListener.Put(tlsConn); errPut != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after HTTP routing failure: %v", errClose)
|
||||
}
|
||||
} else {
|
||||
_ = conn.SetReadDeadline(time.Time{})
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
reader := bufio.NewReader(conn)
|
||||
prefix, errPeek := reader.Peek(1)
|
||||
if errPeek != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after protocol peek failure: %v", errClose)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if isRedisRESPPrefix(prefix[0]) {
|
||||
if s.cfg != nil && s.cfg.Home.Enabled {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close redis connection while home mode is enabled: %v", errClose)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !s.managementRoutesEnabled.Load() {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close redis connection while management is disabled: %v", errClose)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.handleRedisConnection(conn, reader)
|
||||
return
|
||||
}
|
||||
|
||||
if httpListener == nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection without HTTP listener: %v", errClose)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if errPut := httpListener.Put(&bufferedConn{Conn: conn, reader: reader}); errPut != nil {
|
||||
if errClose := conn.Close(); errClose != nil {
|
||||
log.Errorf("failed to close connection after HTTP routing failure: %v", errClose)
|
||||
}
|
||||
} else {
|
||||
_ = conn.SetReadDeadline(time.Time{})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user