fix: preserve original JSON bytes in normalizeCacheControlTTL when no TTL change needed
normalizeCacheControlTTL unconditionally re-serializes the entire request body through json.Unmarshal/json.Marshal even when no TTL normalization is needed. Go's json.Marshal randomizes map key order and HTML-escapes <, >, & characters (to \u003c, \u003e, \u0026), producing different raw bytes on every call. Anthropic's prompt caching uses byte-prefix matching, so any byte-level difference causes a cache miss. This means the ~119K system prompt and tools are re-processed on every request when routed through CPA. The fix adds a bool return to normalizeTTLForBlock to indicate whether it actually modified anything, and skips the marshal step in normalizeCacheControlTTL when no blocks were changed.
This commit is contained in:
@@ -1485,25 +1485,27 @@ func countCacheControlsMap(root map[string]any) int {
|
|||||||
return count
|
return count
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeTTLForBlock(obj map[string]any, seen5m *bool) {
|
func normalizeTTLForBlock(obj map[string]any, seen5m *bool) bool {
|
||||||
ccRaw, exists := obj["cache_control"]
|
ccRaw, exists := obj["cache_control"]
|
||||||
if !exists {
|
if !exists {
|
||||||
return
|
return false
|
||||||
}
|
}
|
||||||
cc, ok := asObject(ccRaw)
|
cc, ok := asObject(ccRaw)
|
||||||
if !ok {
|
if !ok {
|
||||||
*seen5m = true
|
*seen5m = true
|
||||||
return
|
return false
|
||||||
}
|
}
|
||||||
ttlRaw, ttlExists := cc["ttl"]
|
ttlRaw, ttlExists := cc["ttl"]
|
||||||
ttl, ttlIsString := ttlRaw.(string)
|
ttl, ttlIsString := ttlRaw.(string)
|
||||||
if !ttlExists || !ttlIsString || ttl != "1h" {
|
if !ttlExists || !ttlIsString || ttl != "1h" {
|
||||||
*seen5m = true
|
*seen5m = true
|
||||||
return
|
return false
|
||||||
}
|
}
|
||||||
if *seen5m {
|
if *seen5m {
|
||||||
delete(cc, "ttl")
|
delete(cc, "ttl")
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func findLastCacheControlIndex(arr []any) int {
|
func findLastCacheControlIndex(arr []any) int {
|
||||||
@@ -1599,11 +1601,14 @@ func normalizeCacheControlTTL(payload []byte) []byte {
|
|||||||
}
|
}
|
||||||
|
|
||||||
seen5m := false
|
seen5m := false
|
||||||
|
modified := false
|
||||||
|
|
||||||
if tools, ok := asArray(root["tools"]); ok {
|
if tools, ok := asArray(root["tools"]); ok {
|
||||||
for _, tool := range tools {
|
for _, tool := range tools {
|
||||||
if obj, ok := asObject(tool); ok {
|
if obj, ok := asObject(tool); ok {
|
||||||
normalizeTTLForBlock(obj, &seen5m)
|
if normalizeTTLForBlock(obj, &seen5m) {
|
||||||
|
modified = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1611,7 +1616,9 @@ func normalizeCacheControlTTL(payload []byte) []byte {
|
|||||||
if system, ok := asArray(root["system"]); ok {
|
if system, ok := asArray(root["system"]); ok {
|
||||||
for _, item := range system {
|
for _, item := range system {
|
||||||
if obj, ok := asObject(item); ok {
|
if obj, ok := asObject(item); ok {
|
||||||
normalizeTTLForBlock(obj, &seen5m)
|
if normalizeTTLForBlock(obj, &seen5m) {
|
||||||
|
modified = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1628,12 +1635,17 @@ func normalizeCacheControlTTL(payload []byte) []byte {
|
|||||||
}
|
}
|
||||||
for _, item := range content {
|
for _, item := range content {
|
||||||
if obj, ok := asObject(item); ok {
|
if obj, ok := asObject(item); ok {
|
||||||
normalizeTTLForBlock(obj, &seen5m)
|
if normalizeTTLForBlock(obj, &seen5m) {
|
||||||
|
modified = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !modified {
|
||||||
|
return payload
|
||||||
|
}
|
||||||
return marshalPayloadObject(payload, root)
|
return marshalPayloadObject(payload, root)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -369,6 +369,19 @@ func TestNormalizeCacheControlTTL_DowngradesLaterOneHourBlocks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNormalizeCacheControlTTL_PreservesOriginalBytesWhenNoChange(t *testing.T) {
|
||||||
|
// Payload where no TTL normalization is needed (all blocks use 1h with no
|
||||||
|
// preceding 5m block). The text intentionally contains HTML chars (<, >, &)
|
||||||
|
// that json.Marshal would escape to \u003c etc., altering byte identity.
|
||||||
|
payload := []byte(`{"tools":[{"name":"t1","cache_control":{"type":"ephemeral","ttl":"1h"}}],"system":[{"type":"text","text":"<system-reminder>foo & bar</system-reminder>","cache_control":{"type":"ephemeral","ttl":"1h"}}],"messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}`)
|
||||||
|
|
||||||
|
out := normalizeCacheControlTTL(payload)
|
||||||
|
|
||||||
|
if !bytes.Equal(out, payload) {
|
||||||
|
t.Fatalf("normalizeCacheControlTTL altered bytes when no change was needed.\noriginal: %s\ngot: %s", payload, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestEnforceCacheControlLimit_StripsNonLastToolBeforeMessages(t *testing.T) {
|
func TestEnforceCacheControlLimit_StripsNonLastToolBeforeMessages(t *testing.T) {
|
||||||
payload := []byte(`{
|
payload := []byte(`{
|
||||||
"tools": [
|
"tools": [
|
||||||
|
|||||||
Reference in New Issue
Block a user