fix: self-healing claimNextMessage prevents stuck processing messages (#1159)

* fix: self-healing claimNextMessage prevents stuck processing messages

claimAndDelete → claimNextMessage with atomic self-healing: resets stale
processing messages (>60s) back to pending before claiming. Eliminates
stuck messages from generator crashes without external timers. Removes
redundant idle-timeout reset in worker-service.ts. Adds QUEUE to logger
Component type.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: update stale comments in SessionQueueProcessor to reflect claim-confirm pattern

Comments still referenced the old claim-and-delete pattern after the
claimNextMessage rename. Updated to accurately describe the current
lifecycle where messages are marked as processing and stay in DB until
confirmProcessed() is called.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: move Date.now() inside transaction and extract stale threshold constant

- Move Date.now() inside claimNextMessage transaction closure so timestamp
  is fresh if WAL contention causes retry
- Extract STALE_PROCESSING_THRESHOLD_MS to module-level constant
- Add comment clarifying strict < boundary semantics

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Alex Newman
2026-02-17 23:15:46 -05:00
committed by GitHub
parent b2e3a7e668
commit b88251bc8b
9 changed files with 298 additions and 93 deletions
@@ -5,11 +5,11 @@ import type { PendingMessageStore, PersistentPendingMessage } from '../../../src
/**
* Mock PendingMessageStore that returns null (empty queue) by default.
* Individual tests can override claimAndDelete behavior.
* Individual tests can override claimNextMessage behavior.
*/
function createMockStore(): PendingMessageStore {
return {
claimAndDelete: mock(() => null),
claimNextMessage: mock(() => null),
toPendingMessage: mock((msg: PersistentPendingMessage) => ({
type: msg.message_type,
tool_name: msg.tool_name || undefined,
@@ -140,7 +140,7 @@ describe('SessionQueueProcessor', () => {
let callCount = 0;
// Return a message on first call, then null
(store.claimAndDelete as any) = mock(() => {
(store.claimNextMessage as any) = mock(() => {
callCount++;
if (callCount === 1) {
return createMockMessage({ id: 1 });
@@ -170,7 +170,7 @@ describe('SessionQueueProcessor', () => {
expect(results).toHaveLength(1);
expect(results[0]._persistentId).toBe(1);
// Store's claimAndDelete should have been called at least twice
// Store's claimNextMessage should have been called at least twice
// (once returning message, once returning null)
expect(callCount).toBeGreaterThanOrEqual(1);
});
@@ -206,7 +206,7 @@ describe('SessionQueueProcessor', () => {
const onIdleTimeout = mock(() => {});
// Return null to trigger wait
(store.claimAndDelete as any) = mock(() => null);
(store.claimNextMessage as any) = mock(() => null);
const options: CreateIteratorOptions = {
sessionDbId: 123,
@@ -242,7 +242,7 @@ describe('SessionQueueProcessor', () => {
// First call: return null (queue empty)
// After message event: return message
// Then return null again
(store.claimAndDelete as any) = mock(() => {
(store.claimNextMessage as any) = mock(() => {
callCount++;
if (callCount === 1) {
// First check - queue empty, will wait
@@ -312,7 +312,7 @@ describe('SessionQueueProcessor', () => {
it('should clean up event listeners when message received', async () => {
// Return a message immediately
(store.claimAndDelete as any) = mock(() => createMockMessage({ id: 1 }));
(store.claimNextMessage as any) = mock(() => createMockMessage({ id: 1 }));
const options: CreateIteratorOptions = {
sessionDbId: 123,
@@ -344,7 +344,7 @@ describe('SessionQueueProcessor', () => {
it('should continue after store error with backoff', async () => {
let callCount = 0;
(store.claimAndDelete as any) = mock(() => {
(store.claimNextMessage as any) = mock(() => {
callCount++;
if (callCount === 1) {
throw new Error('Database error');
@@ -377,7 +377,7 @@ describe('SessionQueueProcessor', () => {
});
it('should exit cleanly if aborted during error backoff', async () => {
(store.claimAndDelete as any) = mock(() => {
(store.claimNextMessage as any) = mock(() => {
throw new Error('Database error');
});
@@ -413,7 +413,7 @@ describe('SessionQueueProcessor', () => {
created_at_epoch: 1704067200000
});
(store.claimAndDelete as any) = mock(() => mockPersistentMessage);
(store.claimNextMessage as any) = mock(() => mockPersistentMessage);
const options: CreateIteratorOptions = {
sessionDbId: 123,