MAESTRO: fix(db): prevent FK constraint failures on worker restart
Cherry-picked source changes from PR #889 by @Et9797. Fixes #846. Key changes: - Add ensureMemorySessionIdRegistered() guard in SessionStore.ts - Add ON UPDATE CASCADE migration (schema v21) for observations and session_summaries FK constraints - Change message queue from claim-and-delete to claim-confirm pattern (PendingMessageStore.ts) - Add spawn deduplication and unrecoverable error detection in SessionRoutes.ts and worker-service.ts - Add forceInit flag to SDKAgent for stale session recovery Build artifacts skipped (pre-existing dompurify dep issue). Path fixes (HealthMonitor.ts, worker-utils.ts) already merged via PR #634. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Alex Newman
@@ -47,6 +47,7 @@ export class SessionStore {
|
||||
this.renameSessionIdColumns();
|
||||
this.repairSessionIdColumnRename();
|
||||
this.addFailedAtEpochColumn();
|
||||
this.addOnUpdateCascadeToForeignKeys();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -101,7 +102,7 @@ export class SessionStore {
|
||||
type TEXT NOT NULL CHECK(type IN ('decision', 'bugfix', 'feature', 'refactor', 'discovery')),
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id) ON DELETE CASCADE
|
||||
FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_observations_sdk_session ON observations(memory_session_id);
|
||||
@@ -123,7 +124,7 @@ export class SessionStore {
|
||||
notes TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id) ON DELETE CASCADE
|
||||
FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_session_summaries_sdk_session ON session_summaries(memory_session_id);
|
||||
@@ -645,11 +646,187 @@ export class SessionStore {
|
||||
this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(20, new Date().toISOString());
|
||||
}
|
||||
|
||||
/**
|
||||
* Add ON UPDATE CASCADE to FK constraints on observations and session_summaries (migration 21)
|
||||
*
|
||||
* Both tables have FK(memory_session_id) -> sdk_sessions(memory_session_id) with ON DELETE CASCADE
|
||||
* but missing ON UPDATE CASCADE. This causes FK constraint violations when code updates
|
||||
* sdk_sessions.memory_session_id while child rows still reference the old value.
|
||||
*
|
||||
* SQLite doesn't support ALTER TABLE for FK changes, so we recreate both tables.
|
||||
*/
|
||||
private addOnUpdateCascadeToForeignKeys(): void {
|
||||
const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(21) as SchemaVersion | undefined;
|
||||
if (applied) return;
|
||||
|
||||
logger.debug('DB', 'Adding ON UPDATE CASCADE to FK constraints on observations and session_summaries');
|
||||
|
||||
this.db.run('BEGIN TRANSACTION');
|
||||
|
||||
try {
|
||||
// ==========================================
|
||||
// 1. Recreate observations table
|
||||
// ==========================================
|
||||
|
||||
// Drop FTS triggers first (they reference the observations table)
|
||||
this.db.run('DROP TRIGGER IF EXISTS observations_ai');
|
||||
this.db.run('DROP TRIGGER IF EXISTS observations_ad');
|
||||
this.db.run('DROP TRIGGER IF EXISTS observations_au');
|
||||
|
||||
this.db.run(`
|
||||
CREATE TABLE observations_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
memory_session_id TEXT NOT NULL,
|
||||
project TEXT NOT NULL,
|
||||
text TEXT,
|
||||
type TEXT NOT NULL CHECK(type IN ('decision', 'bugfix', 'feature', 'refactor', 'discovery', 'change')),
|
||||
title TEXT,
|
||||
subtitle TEXT,
|
||||
facts TEXT,
|
||||
narrative TEXT,
|
||||
concepts TEXT,
|
||||
files_read TEXT,
|
||||
files_modified TEXT,
|
||||
prompt_number INTEGER,
|
||||
discovery_tokens INTEGER DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
)
|
||||
`);
|
||||
|
||||
this.db.run(`
|
||||
INSERT INTO observations_new
|
||||
SELECT id, memory_session_id, project, text, type, title, subtitle, facts,
|
||||
narrative, concepts, files_read, files_modified, prompt_number,
|
||||
discovery_tokens, created_at, created_at_epoch
|
||||
FROM observations
|
||||
`);
|
||||
|
||||
this.db.run('DROP TABLE observations');
|
||||
this.db.run('ALTER TABLE observations_new RENAME TO observations');
|
||||
|
||||
// Recreate indexes
|
||||
this.db.run(`
|
||||
CREATE INDEX idx_observations_sdk_session ON observations(memory_session_id);
|
||||
CREATE INDEX idx_observations_project ON observations(project);
|
||||
CREATE INDEX idx_observations_type ON observations(type);
|
||||
CREATE INDEX idx_observations_created ON observations(created_at_epoch DESC);
|
||||
`);
|
||||
|
||||
// Recreate FTS triggers only if observations_fts exists
|
||||
// (SessionSearch.ensureFTSTables creates it on first use with IF NOT EXISTS)
|
||||
const hasFTS = (this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='observations_fts'").all() as { name: string }[]).length > 0;
|
||||
if (hasFTS) {
|
||||
this.db.run(`
|
||||
CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative, text, facts, concepts)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative, new.text, new.facts, new.concepts);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative, text, facts, concepts)
|
||||
VALUES('delete', old.id, old.title, old.subtitle, old.narrative, old.text, old.facts, old.concepts);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
|
||||
INSERT INTO observations_fts(observations_fts, rowid, title, subtitle, narrative, text, facts, concepts)
|
||||
VALUES('delete', old.id, old.title, old.subtitle, old.narrative, old.text, old.facts, old.concepts);
|
||||
INSERT INTO observations_fts(rowid, title, subtitle, narrative, text, facts, concepts)
|
||||
VALUES (new.id, new.title, new.subtitle, new.narrative, new.text, new.facts, new.concepts);
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
// ==========================================
|
||||
// 2. Recreate session_summaries table
|
||||
// ==========================================
|
||||
|
||||
this.db.run(`
|
||||
CREATE TABLE session_summaries_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
memory_session_id TEXT NOT NULL,
|
||||
project TEXT NOT NULL,
|
||||
request TEXT,
|
||||
investigated TEXT,
|
||||
learned TEXT,
|
||||
completed TEXT,
|
||||
next_steps TEXT,
|
||||
files_read TEXT,
|
||||
files_edited TEXT,
|
||||
notes TEXT,
|
||||
prompt_number INTEGER,
|
||||
discovery_tokens INTEGER DEFAULT 0,
|
||||
created_at TEXT NOT NULL,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
)
|
||||
`);
|
||||
|
||||
this.db.run(`
|
||||
INSERT INTO session_summaries_new
|
||||
SELECT id, memory_session_id, project, request, investigated, learned,
|
||||
completed, next_steps, files_read, files_edited, notes,
|
||||
prompt_number, discovery_tokens, created_at, created_at_epoch
|
||||
FROM session_summaries
|
||||
`);
|
||||
|
||||
// Drop session_summaries FTS triggers before dropping the table
|
||||
this.db.run('DROP TRIGGER IF EXISTS session_summaries_ai');
|
||||
this.db.run('DROP TRIGGER IF EXISTS session_summaries_ad');
|
||||
this.db.run('DROP TRIGGER IF EXISTS session_summaries_au');
|
||||
|
||||
this.db.run('DROP TABLE session_summaries');
|
||||
this.db.run('ALTER TABLE session_summaries_new RENAME TO session_summaries');
|
||||
|
||||
// Recreate indexes
|
||||
this.db.run(`
|
||||
CREATE INDEX idx_session_summaries_sdk_session ON session_summaries(memory_session_id);
|
||||
CREATE INDEX idx_session_summaries_project ON session_summaries(project);
|
||||
CREATE INDEX idx_session_summaries_created ON session_summaries(created_at_epoch DESC);
|
||||
`);
|
||||
|
||||
// Recreate session_summaries FTS triggers if FTS table exists
|
||||
const hasSummariesFTS = (this.db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='session_summaries_fts'").all() as { name: string }[]).length > 0;
|
||||
if (hasSummariesFTS) {
|
||||
this.db.run(`
|
||||
CREATE TRIGGER IF NOT EXISTS session_summaries_ai AFTER INSERT ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS session_summaries_ad AFTER DELETE ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS session_summaries_au AFTER UPDATE ON session_summaries BEGIN
|
||||
INSERT INTO session_summaries_fts(session_summaries_fts, rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES('delete', old.id, old.request, old.investigated, old.learned, old.completed, old.next_steps, old.notes);
|
||||
INSERT INTO session_summaries_fts(rowid, request, investigated, learned, completed, next_steps, notes)
|
||||
VALUES (new.id, new.request, new.investigated, new.learned, new.completed, new.next_steps, new.notes);
|
||||
END;
|
||||
`);
|
||||
}
|
||||
|
||||
// Record migration
|
||||
this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(21, new Date().toISOString());
|
||||
|
||||
this.db.run('COMMIT');
|
||||
|
||||
logger.debug('DB', 'Successfully added ON UPDATE CASCADE to FK constraints');
|
||||
} catch (error) {
|
||||
this.db.run('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the memory session ID for a session
|
||||
* Called by SDKAgent when it captures the session ID from the first SDK message
|
||||
* Also used to RESET to null on stale resume failures (worker-service.ts)
|
||||
*/
|
||||
updateMemorySessionId(sessionDbId: number, memorySessionId: string): void {
|
||||
updateMemorySessionId(sessionDbId: number, memorySessionId: string | null): void {
|
||||
this.db.prepare(`
|
||||
UPDATE sdk_sessions
|
||||
SET memory_session_id = ?
|
||||
@@ -657,6 +834,37 @@ export class SessionStore {
|
||||
`).run(memorySessionId, sessionDbId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures memory_session_id is registered in sdk_sessions before FK-constrained INSERT.
|
||||
* This fixes Issue #846 where observations fail after worker restart because the
|
||||
* SDK generates a new memory_session_id but it's not registered in the parent table
|
||||
* before child records try to reference it.
|
||||
*
|
||||
* @param sessionDbId - The database ID of the session
|
||||
* @param memorySessionId - The memory session ID to ensure is registered
|
||||
*/
|
||||
ensureMemorySessionIdRegistered(sessionDbId: number, memorySessionId: string): void {
|
||||
const session = this.db.prepare(`
|
||||
SELECT id, memory_session_id FROM sdk_sessions WHERE id = ?
|
||||
`).get(sessionDbId) as { id: number; memory_session_id: string | null } | undefined;
|
||||
|
||||
if (!session) {
|
||||
throw new Error(`Session ${sessionDbId} not found in sdk_sessions`);
|
||||
}
|
||||
|
||||
if (session.memory_session_id !== memorySessionId) {
|
||||
this.db.prepare(`
|
||||
UPDATE sdk_sessions SET memory_session_id = ? WHERE id = ?
|
||||
`).run(memorySessionId, sessionDbId);
|
||||
|
||||
logger.info('DB', 'Registered memory_session_id before storage (FK fix)', {
|
||||
sessionDbId,
|
||||
oldId: session.memory_session_id,
|
||||
newId: memorySessionId
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent session summaries for a project
|
||||
*/
|
||||
@@ -1151,39 +1359,40 @@ export class SessionStore {
|
||||
* - Prompt #2+: session_id exists → INSERT ignored, fetch existing ID
|
||||
* - Result: Same database ID returned for all prompts in conversation
|
||||
*
|
||||
* WHY THIS MATTERS:
|
||||
* - NO "does session exist?" checks needed anywhere
|
||||
* - NO risk of creating duplicate sessions
|
||||
* - ALL hooks automatically connected via session_id
|
||||
* - SAVE hook observations go to correct session (same session_id)
|
||||
* - SDKAgent continuation prompt has correct context (same session_id)
|
||||
*
|
||||
* This is KISS in action: Trust the database UNIQUE constraint and
|
||||
* INSERT OR IGNORE to handle both creation and lookup elegantly.
|
||||
* Pure get-or-create: never modifies memory_session_id.
|
||||
* Multi-terminal isolation is handled by ON UPDATE CASCADE at the schema level.
|
||||
*/
|
||||
createSDKSession(contentSessionId: string, project: string, userPrompt: string): number {
|
||||
const now = new Date();
|
||||
const nowEpoch = now.getTime();
|
||||
|
||||
// INSERT OR IGNORE to create session, then backfill project if it was created empty
|
||||
// Session reuse: Return existing session ID if already created for this contentSessionId.
|
||||
const existing = this.db.prepare(`
|
||||
SELECT id FROM sdk_sessions WHERE content_session_id = ?
|
||||
`).get(contentSessionId) as { id: number } | undefined;
|
||||
|
||||
if (existing) {
|
||||
// Backfill project if session was created by another hook with empty project
|
||||
if (project) {
|
||||
this.db.prepare(`
|
||||
UPDATE sdk_sessions SET project = ?
|
||||
WHERE content_session_id = ? AND (project IS NULL OR project = '')
|
||||
`).run(project, contentSessionId);
|
||||
}
|
||||
return existing.id;
|
||||
}
|
||||
|
||||
// New session - insert fresh row
|
||||
// NOTE: memory_session_id starts as NULL. It is captured by SDKAgent from the first SDK
|
||||
// response and stored via updateMemorySessionId(). CRITICAL: memory_session_id must NEVER
|
||||
// equal contentSessionId - that would inject memory messages into the user's transcript!
|
||||
// response and stored via ensureMemorySessionIdRegistered(). CRITICAL: memory_session_id
|
||||
// must NEVER equal contentSessionId - that would inject memory messages into the user's transcript!
|
||||
this.db.prepare(`
|
||||
INSERT OR IGNORE INTO sdk_sessions
|
||||
INSERT INTO sdk_sessions
|
||||
(content_session_id, memory_session_id, project, user_prompt, started_at, started_at_epoch, status)
|
||||
VALUES (?, NULL, ?, ?, ?, ?, 'active')
|
||||
`).run(contentSessionId, project, userPrompt, now.toISOString(), nowEpoch);
|
||||
|
||||
// Backfill project if session was created by another hook with empty project
|
||||
if (project) {
|
||||
this.db.prepare(`
|
||||
UPDATE sdk_sessions SET project = ?
|
||||
WHERE content_session_id = ? AND (project IS NULL OR project = '')
|
||||
`).run(project, contentSessionId);
|
||||
}
|
||||
|
||||
// Return existing or new ID
|
||||
// Return new ID
|
||||
const row = this.db.prepare('SELECT id FROM sdk_sessions WHERE content_session_id = ?')
|
||||
.get(contentSessionId) as { id: number };
|
||||
return row.id;
|
||||
|
||||
Reference in New Issue
Block a user