feat(antigravity): add AI credits quota fallback

2026-03-30 23:58:12 +08:00
parent 6570692291
commit 88dd9c715d
6 changed files with 670 additions and 51 deletions
@@ -47,12 +47,41 @@ const (
 	defaultAntigravityAgent        = "antigravity/1.19.6 darwin/arm64"
 	antigravityAuthType            = "antigravity"
 	refreshSkew                    = 3000 * time.Second
+	antigravityCreditsRetryTTL     = 5 * time.Hour
 	// systemInstruction              = "You are Antigravity, a powerful agentic AI coding assistant designed by the Google Deepmind team working on Advanced Agentic Coding.You are pair programming with a USER to solve their coding task. The task may require creating a new codebase, modifying or debugging an existing codebase, or simply answering a question.**Absolute paths only****Proactiveness**"
 )

+type antigravity429Category string
+
+const (
+	antigravity429Unknown        antigravity429Category = "unknown"
+	antigravity429RateLimited    antigravity429Category = "rate_limited"
+	antigravity429QuotaExhausted antigravity429Category = "quota_exhausted"
+)
+
 var (
-	randSource      = rand.New(rand.NewSource(time.Now().UnixNano()))
-	randSourceMutex sync.Mutex
+	randSource                        = rand.New(rand.NewSource(time.Now().UnixNano()))
+	randSourceMutex                   sync.Mutex
+	antigravityCreditsExhaustedByAuth sync.Map
+	antigravityPreferCreditsByModel   sync.Map
+	antigravityQuotaExhaustedKeywords = []string{
+		"quota_exhausted",
+		"quota exhausted",
+	}
+	antigravityCreditsExhaustedKeywords = []string{
+		"google_one_ai",
+		"insufficient credit",
+		"insufficient credits",
+		"not enough credit",
+		"not enough credits",
+		"credit exhausted",
+		"credits exhausted",
+		"credit balance",
+		"minimumcreditamountforusage",
+		"minimum credit amount for usage",
+		"minimum credit",
+		"resource has been exhausted",
+	}
 )

 // AntigravityExecutor proxies requests to the antigravity upstream.
@@ -183,6 +212,231 @@ func (e *AntigravityExecutor) HttpRequest(ctx context.Context, auth *cliproxyaut
 	return httpClient.Do(httpReq)
 }

+func injectEnabledCreditTypes(payload []byte) []byte {
+	if len(payload) == 0 {
+		return nil
+	}
+	if !gjson.ValidBytes(payload) {
+		return nil
+	}
+	updated, err := sjson.SetRawBytes(payload, "enabledCreditTypes", []byte(`["GOOGLE_ONE_AI"]`))
+	if err != nil {
+		return nil
+	}
+	return updated
+}
+
+func classifyAntigravity429(body []byte) antigravity429Category {
+	if len(body) == 0 {
+		return antigravity429Unknown
+	}
+	lowerBody := strings.ToLower(string(body))
+	for _, keyword := range antigravityQuotaExhaustedKeywords {
+		if strings.Contains(lowerBody, keyword) {
+			return antigravity429QuotaExhausted
+		}
+	}
+	status := strings.TrimSpace(gjson.GetBytes(body, "error.status").String())
+	if !strings.EqualFold(status, "RESOURCE_EXHAUSTED") {
+		return antigravity429Unknown
+	}
+	details := gjson.GetBytes(body, "error.details")
+	if !details.Exists() || !details.IsArray() {
+		return antigravity429Unknown
+	}
+	for _, detail := range details.Array() {
+		if detail.Get("@type").String() != "type.googleapis.com/google.rpc.ErrorInfo" {
+			continue
+		}
+		reason := strings.TrimSpace(detail.Get("reason").String())
+		if strings.EqualFold(reason, "QUOTA_EXHAUSTED") {
+			return antigravity429QuotaExhausted
+		}
+		if strings.EqualFold(reason, "RATE_LIMIT_EXCEEDED") {
+			return antigravity429RateLimited
+		}
+	}
+	return antigravity429Unknown
+}
+
+func antigravityCreditsRetryEnabled(cfg *config.Config) bool {
+	return cfg != nil && cfg.QuotaExceeded.AntigravityCredits
+}
+
+func antigravityCreditsExhausted(auth *cliproxyauth.Auth, now time.Time) bool {
+	if auth == nil || strings.TrimSpace(auth.ID) == "" {
+		return false
+	}
+	value, ok := antigravityCreditsExhaustedByAuth.Load(auth.ID)
+	if !ok {
+		return false
+	}
+	until, ok := value.(time.Time)
+	if !ok || until.IsZero() {
+		antigravityCreditsExhaustedByAuth.Delete(auth.ID)
+		return false
+	}
+	if !until.After(now) {
+		antigravityCreditsExhaustedByAuth.Delete(auth.ID)
+		return false
+	}
+	return true
+}
+
+func markAntigravityCreditsExhausted(auth *cliproxyauth.Auth, now time.Time) {
+	if auth == nil || strings.TrimSpace(auth.ID) == "" {
+		return
+	}
+	antigravityCreditsExhaustedByAuth.Store(auth.ID, now.Add(antigravityCreditsRetryTTL))
+}
+
+func clearAntigravityCreditsExhausted(auth *cliproxyauth.Auth) {
+	if auth == nil || strings.TrimSpace(auth.ID) == "" {
+		return
+	}
+	antigravityCreditsExhaustedByAuth.Delete(auth.ID)
+}
+
+func antigravityPreferCreditsKey(auth *cliproxyauth.Auth, modelName string) string {
+	if auth == nil {
+		return ""
+	}
+	authID := strings.TrimSpace(auth.ID)
+	modelName = strings.TrimSpace(modelName)
+	if authID == "" || modelName == "" {
+		return ""
+	}
+	return authID + "|" + modelName
+}
+
+func antigravityShouldPreferCredits(auth *cliproxyauth.Auth, modelName string, now time.Time) bool {
+	key := antigravityPreferCreditsKey(auth, modelName)
+	if key == "" {
+		return false
+	}
+	value, ok := antigravityPreferCreditsByModel.Load(key)
+	if !ok {
+		return false
+	}
+	until, ok := value.(time.Time)
+	if !ok || until.IsZero() {
+		antigravityPreferCreditsByModel.Delete(key)
+		return false
+	}
+	if !until.After(now) {
+		antigravityPreferCreditsByModel.Delete(key)
+		return false
+	}
+	return true
+}
+
+func markAntigravityPreferCredits(auth *cliproxyauth.Auth, modelName string, now time.Time, retryAfter *time.Duration) {
+	key := antigravityPreferCreditsKey(auth, modelName)
+	if key == "" {
+		return
+	}
+	until := now.Add(antigravityCreditsRetryTTL)
+	if retryAfter != nil && *retryAfter > 0 {
+		until = now.Add(*retryAfter)
+	}
+	antigravityPreferCreditsByModel.Store(key, until)
+}
+
+func clearAntigravityPreferCredits(auth *cliproxyauth.Auth, modelName string) {
+	key := antigravityPreferCreditsKey(auth, modelName)
+	if key == "" {
+		return
+	}
+	antigravityPreferCreditsByModel.Delete(key)
+}
+
+func shouldMarkAntigravityCreditsExhausted(statusCode int, body []byte, reqErr error) bool {
+	if reqErr != nil || statusCode == 0 {
+		return false
+	}
+	if statusCode >= http.StatusInternalServerError || statusCode == http.StatusRequestTimeout {
+		return false
+	}
+	lowerBody := strings.ToLower(string(body))
+	for _, keyword := range antigravityCreditsExhaustedKeywords {
+		if strings.Contains(lowerBody, keyword) {
+			return true
+		}
+	}
+	return false
+}
+
+func newAntigravityStatusErr(statusCode int, body []byte) statusErr {
+	err := statusErr{code: statusCode, msg: string(body)}
+	if statusCode == http.StatusTooManyRequests {
+		if retryAfter, parseErr := parseRetryDelay(body); parseErr == nil && retryAfter != nil {
+			err.retryAfter = retryAfter
+		}
+	}
+	return err
+}
+
+func (e *AntigravityExecutor) attemptCreditsFallback(
+	ctx context.Context,
+	auth *cliproxyauth.Auth,
+	httpClient *http.Client,
+	token string,
+	modelName string,
+	payload []byte,
+	stream bool,
+	alt string,
+	baseURL string,
+	originalBody []byte,
+) (*http.Response, bool) {
+	if !antigravityCreditsRetryEnabled(e.cfg) {
+		return nil, false
+	}
+	if classifyAntigravity429(originalBody) != antigravity429QuotaExhausted {
+		return nil, false
+	}
+	now := time.Now()
+	if antigravityCreditsExhausted(auth, now) {
+		return nil, false
+	}
+	creditsPayload := injectEnabledCreditTypes(payload)
+	if len(creditsPayload) == 0 {
+		return nil, false
+	}
+
+	httpReq, errReq := e.buildRequest(ctx, auth, token, modelName, creditsPayload, stream, alt, baseURL)
+	if errReq != nil {
+		recordAPIResponseError(ctx, e.cfg, errReq)
+		return nil, true
+	}
+	httpResp, errDo := httpClient.Do(httpReq)
+	if errDo != nil {
+		recordAPIResponseError(ctx, e.cfg, errDo)
+		return nil, true
+	}
+	if httpResp.StatusCode >= http.StatusOK && httpResp.StatusCode < http.StatusMultipleChoices {
+		retryAfter, _ := parseRetryDelay(originalBody)
+		markAntigravityPreferCredits(auth, modelName, now, retryAfter)
+		clearAntigravityCreditsExhausted(auth)
+		return httpResp, true
+	}
+
+	recordAPIResponseMetadata(ctx, e.cfg, httpResp.StatusCode, httpResp.Header.Clone())
+	bodyBytes, errRead := io.ReadAll(httpResp.Body)
+	if errClose := httpResp.Body.Close(); errClose != nil {
+		log.Errorf("antigravity executor: close credits fallback response body error: %v", errClose)
+	}
+	if errRead != nil {
+		recordAPIResponseError(ctx, e.cfg, errRead)
+		return nil, true
+	}
+	appendAPIResponseChunk(ctx, e.cfg, bodyBytes)
+	if shouldMarkAntigravityCreditsExhausted(httpResp.StatusCode, bodyBytes, nil) {
+		clearAntigravityPreferCredits(auth, modelName)
+		markAntigravityCreditsExhausted(auth, now)
+	}
+	return nil, true
+}
+
 // Execute performs a non-streaming request to the Antigravity API.
 func (e *AntigravityExecutor) Execute(ctx context.Context, auth *cliproxyauth.Auth, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (resp cliproxyexecutor.Response, err error) {
 	if opts.Alt == "responses/compact" {
@@ -237,7 +491,15 @@ attemptLoop:
 		var lastErr error

 		for idx, baseURL := range baseURLs {
-			httpReq, errReq := e.buildRequest(ctx, auth, token, baseModel, translated, false, opts.Alt, baseURL)
+			requestPayload := translated
+			usedCreditsDirect := false
+			if antigravityShouldPreferCredits(auth, baseModel, time.Now()) {
+				if creditsPayload := injectEnabledCreditTypes(translated); len(creditsPayload) > 0 {
+					requestPayload = creditsPayload
+					usedCreditsDirect = true
+				}
+			}
+			httpReq, errReq := e.buildRequest(ctx, auth, token, baseModel, requestPayload, false, opts.Alt, baseURL)
 			if errReq != nil {
 				err = errReq
 				return resp, err
@@ -272,6 +534,40 @@ attemptLoop:
 			}
 			appendAPIResponseChunk(ctx, e.cfg, bodyBytes)

+			if httpResp.StatusCode == http.StatusTooManyRequests {
+				if usedCreditsDirect {
+					if shouldMarkAntigravityCreditsExhausted(httpResp.StatusCode, bodyBytes, nil) {
+						clearAntigravityPreferCredits(auth, baseModel)
+						markAntigravityCreditsExhausted(auth, time.Now())
+					}
+				} else {
+					creditsResp, attemptedCredits := e.attemptCreditsFallback(ctx, auth, httpClient, token, baseModel, translated, false, opts.Alt, baseURL, bodyBytes)
+					if creditsResp != nil {
+						recordAPIResponseMetadata(ctx, e.cfg, creditsResp.StatusCode, creditsResp.Header.Clone())
+						creditsBody, errCreditsRead := io.ReadAll(creditsResp.Body)
+						if errClose := creditsResp.Body.Close(); errClose != nil {
+							log.Errorf("antigravity executor: close credits success response body error: %v", errClose)
+						}
+						if errCreditsRead != nil {
+							recordAPIResponseError(ctx, e.cfg, errCreditsRead)
+							err = errCreditsRead
+							return resp, err
+						}
+						appendAPIResponseChunk(ctx, e.cfg, creditsBody)
+						reporter.publish(ctx, parseAntigravityUsage(creditsBody))
+						var param any
+						converted := sdktranslator.TranslateNonStream(ctx, to, from, req.Model, opts.OriginalRequest, translated, creditsBody, &param)
+						resp = cliproxyexecutor.Response{Payload: converted, Headers: creditsResp.Header.Clone()}
+						reporter.ensurePublished(ctx)
+						return resp, nil
+					}
+					if attemptedCredits {
+						err = newAntigravityStatusErr(httpResp.StatusCode, bodyBytes)
+						return resp, err
+					}
+				}
+			}
+
 			if httpResp.StatusCode < http.StatusOK || httpResp.StatusCode >= http.StatusMultipleChoices {
 				log.Debugf("antigravity executor: upstream error status: %d, body: %s", httpResp.StatusCode, summarizeErrorBody(httpResp.Header.Get("Content-Type"), bodyBytes))
 				lastStatus = httpResp.StatusCode
@@ -295,13 +591,7 @@ attemptLoop:
 						continue attemptLoop
 					}
 				}
-				sErr := statusErr{code: httpResp.StatusCode, msg: string(bodyBytes)}
-				if httpResp.StatusCode == http.StatusTooManyRequests {
-					if retryAfter, parseErr := parseRetryDelay(bodyBytes); parseErr == nil && retryAfter != nil {
-						sErr.retryAfter = retryAfter
-					}
-				}
-				err = sErr
+				err = newAntigravityStatusErr(httpResp.StatusCode, bodyBytes)
 				return resp, err
 			}

@@ -315,13 +605,7 @@ attemptLoop:

 		switch {
 		case lastStatus != 0:
-			sErr := statusErr{code: lastStatus, msg: string(lastBody)}
-			if lastStatus == http.StatusTooManyRequests {
-				if retryAfter, parseErr := parseRetryDelay(lastBody); parseErr == nil && retryAfter != nil {
-					sErr.retryAfter = retryAfter
-				}
-			}
-			err = sErr
+			err = newAntigravityStatusErr(lastStatus, lastBody)
 		case lastErr != nil:
 			err = lastErr
 		default:
@@ -379,7 +663,15 @@ attemptLoop:
 		var lastErr error

 		for idx, baseURL := range baseURLs {
-			httpReq, errReq := e.buildRequest(ctx, auth, token, baseModel, translated, true, opts.Alt, baseURL)
+			requestPayload := translated
+			usedCreditsDirect := false
+			if antigravityShouldPreferCredits(auth, baseModel, time.Now()) {
+				if creditsPayload := injectEnabledCreditTypes(translated); len(creditsPayload) > 0 {
+					requestPayload = creditsPayload
+					usedCreditsDirect = true
+				}
+			}
+			httpReq, errReq := e.buildRequest(ctx, auth, token, baseModel, requestPayload, true, opts.Alt, baseURL)
 			if errReq != nil {
 				err = errReq
 				return resp, err
@@ -428,6 +720,26 @@ attemptLoop:
 					return resp, err
 				}
 				appendAPIResponseChunk(ctx, e.cfg, bodyBytes)
+				if httpResp.StatusCode == http.StatusTooManyRequests {
+					if usedCreditsDirect {
+						if shouldMarkAntigravityCreditsExhausted(httpResp.StatusCode, bodyBytes, nil) {
+							clearAntigravityPreferCredits(auth, baseModel)
+							markAntigravityCreditsExhausted(auth, time.Now())
+						}
+					} else {
+						creditsResp, attemptedCredits := e.attemptCreditsFallback(ctx, auth, httpClient, token, baseModel, translated, true, opts.Alt, baseURL, bodyBytes)
+						if creditsResp != nil {
+							httpResp = creditsResp
+							recordAPIResponseMetadata(ctx, e.cfg, httpResp.StatusCode, httpResp.Header.Clone())
+						} else if attemptedCredits {
+							err = newAntigravityStatusErr(http.StatusTooManyRequests, bodyBytes)
+							return resp, err
+						}
+					}
+				}
+				if httpResp.StatusCode >= http.StatusOK && httpResp.StatusCode < http.StatusMultipleChoices {
+					goto streamSuccessClaudeNonStream
+				}
 				lastStatus = httpResp.StatusCode
 				lastBody = append([]byte(nil), bodyBytes...)
 				lastErr = nil
@@ -449,16 +761,11 @@ attemptLoop:
 						continue attemptLoop
 					}
 				}
-				sErr := statusErr{code: httpResp.StatusCode, msg: string(bodyBytes)}
-				if httpResp.StatusCode == http.StatusTooManyRequests {
-					if retryAfter, parseErr := parseRetryDelay(bodyBytes); parseErr == nil && retryAfter != nil {
-						sErr.retryAfter = retryAfter
-					}
-				}
-				err = sErr
+				err = newAntigravityStatusErr(httpResp.StatusCode, bodyBytes)
 				return resp, err
 			}

+		streamSuccessClaudeNonStream:
 			out := make(chan cliproxyexecutor.StreamChunk)
 			go func(resp *http.Response) {
 				defer close(out)
@@ -520,13 +827,7 @@ attemptLoop:

 		switch {
 		case lastStatus != 0:
-			sErr := statusErr{code: lastStatus, msg: string(lastBody)}
-			if lastStatus == http.StatusTooManyRequests {
-				if retryAfter, parseErr := parseRetryDelay(lastBody); parseErr == nil && retryAfter != nil {
-					sErr.retryAfter = retryAfter
-				}
-			}
-			err = sErr
+			err = newAntigravityStatusErr(lastStatus, lastBody)
 		case lastErr != nil:
 			err = lastErr
 		default:
@@ -782,7 +1083,15 @@ attemptLoop:
 		var lastErr error

 		for idx, baseURL := range baseURLs {
-			httpReq, errReq := e.buildRequest(ctx, auth, token, baseModel, translated, true, opts.Alt, baseURL)
+			requestPayload := translated
+			usedCreditsDirect := false
+			if antigravityShouldPreferCredits(auth, baseModel, time.Now()) {
+				if creditsPayload := injectEnabledCreditTypes(translated); len(creditsPayload) > 0 {
+					requestPayload = creditsPayload
+					usedCreditsDirect = true
+				}
+			}
+			httpReq, errReq := e.buildRequest(ctx, auth, token, baseModel, requestPayload, true, opts.Alt, baseURL)
 			if errReq != nil {
 				err = errReq
 				return nil, err
@@ -830,6 +1139,26 @@ attemptLoop:
 					return nil, err
 				}
 				appendAPIResponseChunk(ctx, e.cfg, bodyBytes)
+				if httpResp.StatusCode == http.StatusTooManyRequests {
+					if usedCreditsDirect {
+						if shouldMarkAntigravityCreditsExhausted(httpResp.StatusCode, bodyBytes, nil) {
+							clearAntigravityPreferCredits(auth, baseModel)
+							markAntigravityCreditsExhausted(auth, time.Now())
+						}
+					} else {
+						creditsResp, attemptedCredits := e.attemptCreditsFallback(ctx, auth, httpClient, token, baseModel, translated, true, opts.Alt, baseURL, bodyBytes)
+						if creditsResp != nil {
+							httpResp = creditsResp
+							recordAPIResponseMetadata(ctx, e.cfg, httpResp.StatusCode, httpResp.Header.Clone())
+						} else if attemptedCredits {
+							err = newAntigravityStatusErr(http.StatusTooManyRequests, bodyBytes)
+							return nil, err
+						}
+					}
+				}
+				if httpResp.StatusCode >= http.StatusOK && httpResp.StatusCode < http.StatusMultipleChoices {
+					goto streamSuccessExecuteStream
+				}
 				lastStatus = httpResp.StatusCode
 				lastBody = append([]byte(nil), bodyBytes...)
 				lastErr = nil
@@ -851,16 +1180,11 @@ attemptLoop:
 						continue attemptLoop
 					}
 				}
-				sErr := statusErr{code: httpResp.StatusCode, msg: string(bodyBytes)}
-				if httpResp.StatusCode == http.StatusTooManyRequests {
-					if retryAfter, parseErr := parseRetryDelay(bodyBytes); parseErr == nil && retryAfter != nil {
-						sErr.retryAfter = retryAfter
-					}
-				}
-				err = sErr
+				err = newAntigravityStatusErr(httpResp.StatusCode, bodyBytes)
 				return nil, err
 			}

+		streamSuccessExecuteStream:
 			out := make(chan cliproxyexecutor.StreamChunk)
 			go func(resp *http.Response) {
 				defer close(out)
@@ -911,13 +1235,7 @@ attemptLoop:

 		switch {
 		case lastStatus != 0:
-			sErr := statusErr{code: lastStatus, msg: string(lastBody)}
-			if lastStatus == http.StatusTooManyRequests {
-				if retryAfter, parseErr := parseRetryDelay(lastBody); parseErr == nil && retryAfter != nil {
-					sErr.retryAfter = retryAfter
-				}
-			}
-			err = sErr
+			err = newAntigravityStatusErr(lastStatus, lastBody)
 		case lastErr != nil:
 			err = lastErr
 		default: