airkjw/claude-mem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: stop spinner from spinning forever (#1440)

Browse Source 
* fix: stop spinner from spinning forever due to orphaned DB messages

The activity spinner never stopped because isAnySessionProcessing() queried
ALL pending/processing messages in the database, including orphaned messages
from dead sessions that no generator would ever process.

Root cause: isAnySessionProcessing() used hasAnyPendingWork() which is a
global DB scan. Changed it to use getTotalQueueDepth() which only checks
sessions in the active in-memory Map.

Additional fixes:
- Add terminateSession() to enforce restart-or-terminate invariant
- Fix 3 zombie paths in .finally() handler that left sessions alive
- Clean up idle sessions from memory on successful completion
- Remove redundant bare isProcessing:true broadcast
- Replace inline require() with proper accessor
- Add 8 regression tests for session termination invariant

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: address review findings — idle-timeout race, double broadcast, query amplification

- Move pendingCount check before idle-timeout termination to prevent
  abandoning fresh messages that arrive between idle abort and .finally()
- Move broadcastProcessingStatus() inside restart branch only — the else
  branch already broadcasts via removeSessionImmediate callback
- Compute queueDepth once in broadcastProcessingStatus() and derive
  isProcessing from it, eliminating redundant double iteration

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alex Newman
2026-03-21 14:13:10 -07:00
committed by GitHub
parent 9f529a30f5
commit 4d7bec4d05
7 changed files with 1043 additions and 68213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/worker-service.ts
+44 -23
View File
@@ -653,30 +653,26 @@ export class WorkerService {
// Do NOT restart after unrecoverable errors - prevents infinite loops
if (hadUnrecoverableError) {
logger.warn('SYSTEM', 'Skipping restart due to unrecoverable error', {
sessionId: session.sessionDbId
});
this.broadcastProcessingStatus();
this.terminateSession(session.sessionDbId, 'unrecoverable_error');
return;
}
// Store for pending-count check below
const { PendingMessageStore } = require('./sqlite/PendingMessageStore.js');
const pendingStore = new PendingMessageStore(this.dbManager.getSessionStore().db, 3);
// Idle timeout means no new work arrived for 3 minutes - don't restart
// No need to reset stale processing messages here — claimNextMessage() self-heals
if (session.idleTimedOut) {
logger.info('SYSTEM', 'Generator exited due to idle timeout, not restarting', {
sessionId: session.sessionDbId
});
session.idleTimedOut = false; // Reset flag
this.broadcastProcessingStatus();
return;
}
const pendingStore = this.sessionManager.getPendingMessageStore();
// Check if there's pending work that needs processing with a fresh AbortController
const pendingCount = pendingStore.getPendingCount(session.sessionDbId);
// Idle timeout means no new work arrived for 3 minutes - don't restart
// But check pendingCount first: a message may have arrived between idle
// abort and .finally(), and we must not abandon it
if (session.idleTimedOut) {
session.idleTimedOut = false; // Reset flag
if (pendingCount === 0) {
this.terminateSession(session.sessionDbId, 'idle_timeout');
return;
}
// Fall through to pending-work restart below
}
const MAX_PENDING_RESTARTS = 3;
if (pendingCount > 0) {
@@ -690,7 +686,7 @@ export class WorkerService {
consecutiveRestarts: session.consecutiveRestarts
});
session.consecutiveRestarts = 0;
this.broadcastProcessingStatus();
this.terminateSession(session.sessionDbId, 'max_restarts_exceeded');
return;
}
@@ -703,12 +699,13 @@ export class WorkerService {
session.abortController = new AbortController();
// Restart processor
this.startSessionProcessor(session, 'pending-work-restart');
this.broadcastProcessingStatus();
} else {
// Successful completion with no pending work — reset counter
// Successful completion with no pending work — clean up session
// removeSessionImmediate fires onSessionDeletedCallback → broadcastProcessingStatus()
session.consecutiveRestarts = 0;
this.sessionManager.removeSessionImmediate(session.sessionDbId);
}
this.broadcastProcessingStatus();
});
}
@@ -784,6 +781,30 @@ export class WorkerService {
this.sessionEventBroadcaster.broadcastSessionCompleted(sessionDbId);
}
/**
* Terminate a session that will not restart.
* Enforces the restart-or-terminate invariant: every generator exit
* must either call startSessionProcessor() or terminateSession().
* No zombie sessions allowed.
*
* GENERATOR EXIT INVARIANT:
* .finally() → restart? → startSessionProcessor()
* no? → terminateSession()
*/
private terminateSession(sessionDbId: number, reason: string): void {
const pendingStore = this.sessionManager.getPendingMessageStore();
const abandoned = pendingStore.markAllSessionMessagesAbandoned(sessionDbId);
logger.info('SYSTEM', 'Session terminated', {
sessionId: sessionDbId,
reason,
abandonedMessages: abandoned
});
// removeSessionImmediate fires onSessionDeletedCallback → broadcastProcessingStatus()
this.sessionManager.removeSessionImmediate(sessionDbId);
}
/**
* Process pending session queues
*/
@@ -907,8 +928,8 @@ export class WorkerService {
* Broadcast processing status change to SSE clients
*/
broadcastProcessingStatus(): void {
const isProcessing = this.sessionManager.isAnySessionProcessing();
const queueDepth = this.sessionManager.getTotalActiveWork();
const isProcessing = queueDepth > 0;
const activeSessions = this.sessionManager.getActiveSessionCount();
logger.info('WORKER', 'Broadcasting processing status', {
src/services/worker/SessionManager.ts
+8 -7
View File
@@ -350,7 +350,7 @@ export class SessionManager {
this.sessions.delete(sessionDbId);
this.sessionQueues.delete(sessionDbId);
logger.info('SESSION', 'Session removed (orphaned after SDK termination)', {
logger.info('SESSION', 'Session removed from active sessions', {
sessionId: sessionDbId,
project: session.project
});
@@ -402,10 +402,11 @@ export class SessionManager {
}
/**
* Check if any session has pending messages (for spinner tracking)
* Check if any active session has pending messages (for spinner tracking).
* Scoped to in-memory sessions only.
*/
hasPendingMessages(): boolean {
return this.getPendingStore().hasAnyPendingWork();
return this.getTotalQueueDepth() > 0;
}
/**
@@ -437,12 +438,12 @@ export class SessionManager {
}
/**
* Check if any session is actively processing (has pending messages OR active generator)
* Used for activity indicator to prevent spinner from stopping while SDK is processing
* Check if any active session has pending work.
* Scoped to in-memory sessions only — orphaned DB messages from dead
* sessions must not keep the spinner spinning forever.
*/
isAnySessionProcessing(): boolean {
// hasAnyPendingWork checks for 'pending' OR 'processing'
return this.getPendingStore().hasAnyPendingWork();
return this.getTotalQueueDepth() > 0;
}
/**
src/services/worker/events/SessionEventBroadcaster.ts
-6
View File
@@ -33,12 +33,6 @@ export class SessionEventBroadcaster {
prompt
});
// Start activity indicator (work is about to begin)
this.sseBroadcaster.broadcast({
type: 'processing_status',
isProcessing: true
});
// Update processing status based on queue depth
this.workerService.broadcastProcessingStatus();
}