feat(redis): implement Pub/Sub support for usage tracking
- Added Redis Pub/Sub capability to broadcast usage updates to subscribed clients. - Enhanced `redisqueue` with subscriber management and message broadcasting. - Updated tests to validate Pub/Sub message handling, subscription behavior, and fallback to the queue after unsubscribing. - Integrated `project_id` parsing into auth-files logic to include project identifiers in metadata.
This commit is contained in:
Luis Pater
@@ -3,10 +3,13 @@ package api
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -171,6 +174,105 @@ func readRESPArrayOfBulkStrings(r *bufio.Reader) ([][]byte, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readTestRESPInteger(r *bufio.Reader) (int, error) {
|
||||
prefix, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if prefix != ':' {
|
||||
return 0, fmt.Errorf("expected integer prefix ':', got %q", prefix)
|
||||
}
|
||||
|
||||
line, err := readTestRESPLine(r)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value, err := strconv.Atoi(line)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid integer %q: %v", line, err)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func readTestRESPArrayHeader(r *bufio.Reader) (int, error) {
|
||||
prefix, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if prefix != '*' {
|
||||
return 0, fmt.Errorf("expected array prefix '*', got %q", prefix)
|
||||
}
|
||||
|
||||
line, err := readTestRESPLine(r)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count, err := strconv.Atoi(line)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid array length %q: %v", line, err)
|
||||
}
|
||||
if count < 0 {
|
||||
return 0, fmt.Errorf("invalid array length %d", count)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func readTestRESPPubSubSubscribe(r *bufio.Reader) (string, int, error) {
|
||||
count, err := readTestRESPArrayHeader(r)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if count != 3 {
|
||||
return "", 0, fmt.Errorf("subscribe array length = %d, want 3", count)
|
||||
}
|
||||
|
||||
kind, err := readTestRESPBulkString(r)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if string(kind) != "subscribe" {
|
||||
return "", 0, fmt.Errorf("pubsub kind = %q, want subscribe", string(kind))
|
||||
}
|
||||
|
||||
channel, err := readTestRESPBulkString(r)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
subscriptions, err := readTestRESPInteger(r)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return string(channel), subscriptions, nil
|
||||
}
|
||||
|
||||
func readTestRESPPubSubMessage(r *bufio.Reader) (string, []byte, error) {
|
||||
count, err := readTestRESPArrayHeader(r)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if count != 3 {
|
||||
return "", nil, fmt.Errorf("message array length = %d, want 3", count)
|
||||
}
|
||||
|
||||
kind, err := readTestRESPBulkString(r)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if string(kind) != "message" {
|
||||
return "", nil, fmt.Errorf("pubsub kind = %q, want message", string(kind))
|
||||
}
|
||||
|
||||
channel, err := readTestRESPBulkString(r)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
payload, err := readTestRESPBulkString(r)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return string(channel), payload, nil
|
||||
}
|
||||
|
||||
func TestRedisProtocol_ManagementDisabled_RejectsConnection(t *testing.T) {
|
||||
t.Setenv("MANAGEMENT_PASSWORD", "")
|
||||
redisqueue.SetEnabled(false)
|
||||
@@ -352,6 +454,127 @@ func TestRedisProtocol_AUTH_And_PopContracts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisProtocol_SubscribeUsageBroadcastsAndSkipsQueue(t *testing.T) {
|
||||
const managementPassword = "test-management-password"
|
||||
|
||||
t.Setenv("MANAGEMENT_PASSWORD", managementPassword)
|
||||
redisqueue.SetEnabled(false)
|
||||
t.Cleanup(func() { redisqueue.SetEnabled(false) })
|
||||
|
||||
server := newTestServer(t)
|
||||
if !server.managementRoutesEnabled.Load() {
|
||||
t.Fatalf("expected managementRoutesEnabled to be true")
|
||||
}
|
||||
|
||||
addr, stop := startRedisMuxListener(t, server)
|
||||
t.Cleanup(stop)
|
||||
|
||||
firstConn, errDialFirst := net.DialTimeout("tcp", addr, time.Second)
|
||||
if errDialFirst != nil {
|
||||
t.Fatalf("failed to dial first redis listener: %v", errDialFirst)
|
||||
}
|
||||
t.Cleanup(func() { _ = firstConn.Close() })
|
||||
firstReader := bufio.NewReader(firstConn)
|
||||
_ = firstConn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
|
||||
if errWrite := writeTestRESPCommand(firstConn, "AUTH", managementPassword); errWrite != nil {
|
||||
t.Fatalf("failed to write first AUTH command: %v", errWrite)
|
||||
}
|
||||
if msg, err := readTestRESPSimpleString(firstReader); err != nil {
|
||||
t.Fatalf("failed to read first AUTH response: %v", err)
|
||||
} else if msg != "OK" {
|
||||
t.Fatalf("unexpected first AUTH response: %q", msg)
|
||||
}
|
||||
if errWrite := writeTestRESPCommand(firstConn, "SUBSCRIBE", "usage"); errWrite != nil {
|
||||
t.Fatalf("failed to write first SUBSCRIBE command: %v", errWrite)
|
||||
}
|
||||
if channel, count, err := readTestRESPPubSubSubscribe(firstReader); err != nil {
|
||||
t.Fatalf("failed to read first SUBSCRIBE response: %v", err)
|
||||
} else if channel != "usage" || count != 1 {
|
||||
t.Fatalf("unexpected first SUBSCRIBE response channel=%q count=%d", channel, count)
|
||||
}
|
||||
|
||||
secondConn, errDialSecond := net.DialTimeout("tcp", addr, time.Second)
|
||||
if errDialSecond != nil {
|
||||
t.Fatalf("failed to dial second redis listener: %v", errDialSecond)
|
||||
}
|
||||
t.Cleanup(func() { _ = secondConn.Close() })
|
||||
secondReader := bufio.NewReader(secondConn)
|
||||
_ = secondConn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
|
||||
if errWrite := writeTestRESPCommand(secondConn, "AUTH", managementPassword); errWrite != nil {
|
||||
t.Fatalf("failed to write second AUTH command: %v", errWrite)
|
||||
}
|
||||
if msg, err := readTestRESPSimpleString(secondReader); err != nil {
|
||||
t.Fatalf("failed to read second AUTH response: %v", err)
|
||||
} else if msg != "OK" {
|
||||
t.Fatalf("unexpected second AUTH response: %q", msg)
|
||||
}
|
||||
if errWrite := writeTestRESPCommand(secondConn, "SUBSCRIBE", "usage"); errWrite != nil {
|
||||
t.Fatalf("failed to write second SUBSCRIBE command: %v", errWrite)
|
||||
}
|
||||
if channel, count, err := readTestRESPPubSubSubscribe(secondReader); err != nil {
|
||||
t.Fatalf("failed to read second SUBSCRIBE response: %v", err)
|
||||
} else if channel != "usage" || count != 1 {
|
||||
t.Fatalf("unexpected second SUBSCRIBE response channel=%q count=%d", channel, count)
|
||||
}
|
||||
|
||||
redisqueue.Enqueue([]byte(`{"id":1}`))
|
||||
|
||||
if channel, payload, err := readTestRESPPubSubMessage(firstReader); err != nil {
|
||||
t.Fatalf("failed to read first pubsub message: %v", err)
|
||||
} else if channel != "usage" || string(payload) != `{"id":1}` {
|
||||
t.Fatalf("unexpected first pubsub message channel=%q payload=%q", channel, string(payload))
|
||||
}
|
||||
if channel, payload, err := readTestRESPPubSubMessage(secondReader); err != nil {
|
||||
t.Fatalf("failed to read second pubsub message: %v", err)
|
||||
} else if channel != "usage" || string(payload) != `{"id":1}` {
|
||||
t.Fatalf("unexpected second pubsub message channel=%q payload=%q", channel, string(payload))
|
||||
}
|
||||
|
||||
popConn, errDialPop := net.DialTimeout("tcp", addr, time.Second)
|
||||
if errDialPop != nil {
|
||||
t.Fatalf("failed to dial pop redis listener: %v", errDialPop)
|
||||
}
|
||||
t.Cleanup(func() { _ = popConn.Close() })
|
||||
popReader := bufio.NewReader(popConn)
|
||||
_ = popConn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
|
||||
if errWrite := writeTestRESPCommand(popConn, "AUTH", managementPassword); errWrite != nil {
|
||||
t.Fatalf("failed to write pop AUTH command: %v", errWrite)
|
||||
}
|
||||
if msg, err := readTestRESPSimpleString(popReader); err != nil {
|
||||
t.Fatalf("failed to read pop AUTH response: %v", err)
|
||||
} else if msg != "OK" {
|
||||
t.Fatalf("unexpected pop AUTH response: %q", msg)
|
||||
}
|
||||
if errWrite := writeTestRESPCommand(popConn, "LPOP", "usage"); errWrite != nil {
|
||||
t.Fatalf("failed to write pop LPOP command: %v", errWrite)
|
||||
}
|
||||
item, errItem := readTestRESPBulkString(popReader)
|
||||
if errItem != nil {
|
||||
t.Fatalf("failed to read pop LPOP response: %v", errItem)
|
||||
}
|
||||
if item != nil {
|
||||
t.Fatalf("expected subscribed usage to skip queue, got %q", string(item))
|
||||
}
|
||||
|
||||
managementReq := httptest.NewRequest(http.MethodGet, "/v0/management/usage-queue?count=1", nil)
|
||||
managementReq.Header.Set("Authorization", "Bearer "+managementPassword)
|
||||
managementRR := httptest.NewRecorder()
|
||||
server.engine.ServeHTTP(managementRR, managementReq)
|
||||
if managementRR.Code != http.StatusOK {
|
||||
t.Fatalf("management usage status = %d, want %d body=%s", managementRR.Code, http.StatusOK, managementRR.Body.String())
|
||||
}
|
||||
var managementPayload []json.RawMessage
|
||||
if errUnmarshal := json.Unmarshal(managementRR.Body.Bytes(), &managementPayload); errUnmarshal != nil {
|
||||
t.Fatalf("unmarshal management usage response: %v", errUnmarshal)
|
||||
}
|
||||
if len(managementPayload) != 0 {
|
||||
t.Fatalf("expected management usage queue to be empty, got %s", managementRR.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisProtocol_IPBan_MirrorsManagementPolicy(t *testing.T) {
|
||||
const managementPassword = "test-management-password"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user