fix: make migration 17 fully idempotent for databases in intermediate states (#481)

* fix: make migration 17 idempotent and standardize column names

Migration 17 renamed columns from sdk_session_id to memory_session_id,
but the migration wasn't fully idempotent - it could fail on databases
in intermediate states. This caused errors like:
- "no such column: sdk_session_id" (when columns already renamed)
- "table observations has no column named memory_session_id" (when not renamed)

Changes:
- Rewrite renameSessionIdColumns() to check each table individually
- Use safeRenameColumn() helper that handles all edge cases gracefully
- Deprecate migration 19 (repair migration) since 17 is now idempotent
- Update maintenance scripts to use memory_session_id column name
- Update test files to use new column names

Fixes column mismatch bug in v8.2.6+

* Merge origin/main into column-mismatch

---------

Co-authored-by: Alex Newman <thedotmack@gmail.com>

scripts/export-memories.ts

@@ -12,7 +12,7 @@ import { SettingsDefaultsManager } from '../src/shared/SettingsDefaultsManager';
 
 interface ObservationRecord {
   id: number;
-  sdk_session_id: string;
+  memory_session_id: string;
   project: string;
   text: string | null;
   type: string;
@@ -31,8 +31,8 @@ interface ObservationRecord {
 
 interface SdkSessionRecord {
   id: number;
-  claude_session_id: string;
-  sdk_session_id: string;
+  content_session_id: string;
+  memory_session_id: string;
   project: string;
   user_prompt: string;
   started_at: string;
@@ -44,7 +44,7 @@ interface SdkSessionRecord {
 
 interface SessionSummaryRecord {
   id: number;
-  sdk_session_id: string;
+  memory_session_id: string;
   project: string;
   request: string | null;
   investigated: string | null;
@@ -62,7 +62,7 @@ interface SessionSummaryRecord {
 
 interface UserPromptRecord {
   id: number;
-  claude_session_id: string;
+  content_session_id: string;
   prompt_number: number;
   prompt_text: string;
   created_at: string;
@@ -117,23 +117,23 @@ async function exportMemories(query: string, outputFile: string, project?: strin
     console.log(`✅ Found ${summaries.length} session summaries`);
     console.log(`✅ Found ${prompts.length} user prompts`);
 
-    // Get unique SDK session IDs from observations and summaries
-    const sdkSessionIds = new Set<string>();
+    // Get unique memory session IDs from observations and summaries
+    const memorySessionIds = new Set<string>();
     observations.forEach((o) => {
-      if (o.sdk_session_id) sdkSessionIds.add(o.sdk_session_id);
+      if (o.memory_session_id) memorySessionIds.add(o.memory_session_id);
     });
     summaries.forEach((s) => {
-      if (s.sdk_session_id) sdkSessionIds.add(s.sdk_session_id);
+      if (s.memory_session_id) memorySessionIds.add(s.memory_session_id);
     });
 
     // Get SDK sessions metadata via API
     console.log('📡 Fetching SDK sessions metadata...');
     let sessions: SdkSessionRecord[] = [];
-    if (sdkSessionIds.size > 0) {
+    if (memorySessionIds.size > 0) {
       const sessionsResponse = await fetch(`${baseUrl}/api/sdk-sessions/batch`, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ sdkSessionIds: Array.from(sdkSessionIds) })
+        body: JSON.stringify({ sdkSessionIds: Array.from(memorySessionIds) })
       });
       if (sessionsResponse.ok) {
         sessions = await sessionsResponse.json();

scripts/fix-all-timestamps.ts

@@ -18,7 +18,7 @@ interface CorruptedObservation {
   obs_created: number;
   session_started: number;
   session_completed: number | null;
-  sdk_session_id: string;
+  memory_session_id: string;
 }
 
 function formatTimestamp(epoch: number): string {
@@ -54,9 +54,9 @@ function main() {
         o.created_at_epoch as obs_created,
         s.started_at_epoch as session_started,
         s.completed_at_epoch as session_completed,
-        s.sdk_session_id
+        s.memory_session_id
       FROM observations o
-      JOIN sdk_sessions s ON o.sdk_session_id = s.sdk_session_id
+      JOIN sdk_sessions s ON o.memory_session_id = s.memory_session_id
       WHERE o.created_at_epoch < s.started_at_epoch  -- Observation older than session
          OR (s.completed_at_epoch IS NOT NULL
              AND o.created_at_epoch > (s.completed_at_epoch + 3600000))  -- More than 1hr after session

scripts/fix-corrupted-timestamps.ts

@@ -20,7 +20,7 @@ const BAD_WINDOW_END = 1766626260000;   // Dec 24 20:31 PST
 
 interface AffectedObservation {
   id: number;
-  sdk_session_id: string;
+  memory_session_id: string;
   created_at_epoch: number;
   title: string;
 }
@@ -35,7 +35,7 @@ interface ProcessedMessage {
 
 interface SessionMapping {
   session_db_id: number;
-  sdk_session_id: string;
+  memory_session_id: string;
 }
 
 interface TimestampFix {
@@ -75,7 +75,7 @@ function main() {
     // Step 1: Find affected observations
     console.log('Step 1: Finding observations created during bad window...');
     const affectedObs = db.query<AffectedObservation, []>(`
-      SELECT id, sdk_session_id, created_at_epoch, title
+      SELECT id, memory_session_id, created_at_epoch, title
       FROM observations
       WHERE created_at_epoch >= ${BAD_WINDOW_START}
         AND created_at_epoch <= ${BAD_WINDOW_END}
@@ -111,7 +111,7 @@ function main() {
       obs_title: string;
       obs_created: number;
       session_started: number;
-      sdk_session_id: string;
+      memory_session_id: string;
     }
 
     const obsWithSessions = db.query<ObsWithSession, []>(`
@@ -120,9 +120,9 @@ function main() {
         o.title as obs_title,
         o.created_at_epoch as obs_created,
         s.started_at_epoch as session_started,
-        s.sdk_session_id
+        s.memory_session_id
       FROM observations o
-      JOIN sdk_sessions s ON o.sdk_session_id = s.sdk_session_id
+      JOIN sdk_sessions s ON o.memory_session_id = s.memory_session_id
       WHERE o.created_at_epoch >= ${BAD_WINDOW_START}
         AND o.created_at_epoch <= ${BAD_WINDOW_END}
         AND s.started_at_epoch < ${BAD_WINDOW_START}

scripts/investigate-timestamps.ts

@@ -36,7 +36,7 @@ function main() {
     const dec24End = 1735113600000;   // Dec 25 00:00 PST
 
     const dec24Obs = db.query(`
-      SELECT id, sdk_session_id, created_at_epoch, title
+      SELECT id, memory_session_id, created_at_epoch, title
       FROM observations
       WHERE created_at_epoch >= ${dec24Start}
         AND created_at_epoch < ${dec24End}
@@ -59,7 +59,7 @@ function main() {
     const dec21Start = 1734768000000; // Dec 21 00:00 PST
 
     const oldObs = db.query(`
-      SELECT id, sdk_session_id, created_at_epoch, title
+      SELECT id, memory_session_id, created_at_epoch, title
       FROM observations
       WHERE created_at_epoch >= ${dec17Start}
         AND created_at_epoch < ${dec21Start}

scripts/validate-timestamp-logic.ts

@@ -59,7 +59,7 @@ function main() {
         pm.tool_name,
         pm.created_at_epoch as msg_created,
         pm.status,
-        s.sdk_session_id,
+        s.memory_session_id,
         s.started_at_epoch as session_started,
         s.project
       FROM pending_messages pm

scripts/verify-timestamp-fix.ts

@@ -22,7 +22,7 @@ const ORIGINAL_WINDOW_END = 1766613600000;   // Dec 23 23:59 PST
 
 interface Observation {
   id: number;
-  sdk_session_id: string;
+  memory_session_id: string;
   created_at_epoch: number;
   created_at: string;
   title: string;
@@ -49,7 +49,7 @@ function main() {
     // Check 1: Observations still in bad window
     console.log('Check 1: Looking for observations still in bad window (Dec 24 19:45-20:31)...');
     const badWindowObs = db.query<Observation, []>(`
-      SELECT id, sdk_session_id, created_at_epoch, created_at, title
+      SELECT id, memory_session_id, created_at_epoch, created_at, title
       FROM observations
       WHERE created_at_epoch >= ${BAD_WINDOW_START}
         AND created_at_epoch <= ${BAD_WINDOW_END}
@@ -63,7 +63,7 @@ function main() {
       for (const obs of badWindowObs) {
         console.log(`  Observation #${obs.id}: ${obs.title || '(no title)'}`);
         console.log(`    Timestamp: ${formatTimestamp(obs.created_at_epoch)}`);
-        console.log(`    Session: ${obs.sdk_session_id}\n`);
+        console.log(`    Session: ${obs.memory_session_id}\n`);
       }
     }
 
@@ -81,19 +81,19 @@ function main() {
 
     // Check 3: Session distribution
     console.log('Check 3: Session distribution of corrected observations...');
-    const sessionDist = db.query<{ sdk_session_id: string; count: number }, []>(`
-      SELECT sdk_session_id, COUNT(*) as count
+    const sessionDist = db.query<{ memory_session_id: string; count: number }, []>(`
+      SELECT memory_session_id, COUNT(*) as count
       FROM observations
       WHERE created_at_epoch >= ${ORIGINAL_WINDOW_START}
         AND created_at_epoch <= ${ORIGINAL_WINDOW_END}
-      GROUP BY sdk_session_id
+      GROUP BY memory_session_id
       ORDER BY count DESC
     `).all();
 
     if (sessionDist.length > 0) {
       console.log(`Observations distributed across ${sessionDist.length} sessions:\n`);
       for (const dist of sessionDist.slice(0, 10)) {
-        console.log(`  ${dist.sdk_session_id}: ${dist.count} observations`);
+        console.log(`  ${dist.memory_session_id}: ${dist.count} observations`);
       }
       if (sessionDist.length > 10) {
         console.log(`  ... and ${sessionDist.length - 10} more sessions`);

src/services/sqlite/SessionStore.ts

@@ -582,126 +582,85 @@ export class SessionStore {
    * Rename session ID columns for semantic clarity (migration 17)
    * - claude_session_id → content_session_id (user's observed session)
    * - sdk_session_id → memory_session_id (memory agent's session for resume)
+   *
+   * IDEMPOTENT: Checks each table individually before renaming.
+   * This handles databases in any intermediate state (partial migration, fresh install, etc.)
    */
   private renameSessionIdColumns(): void {
     const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(17) as SchemaVersion | undefined;
     if (applied) return;
 
-    // Check if columns are already renamed (idempotent check)
-    const sessionsInfo = this.db.query('PRAGMA table_info(sdk_sessions)').all() as TableColumnInfo[];
-    const hasContentSessionId = sessionsInfo.some(col => col.name === 'content_session_id');
+    logger.info('DB', 'Checking session ID columns for semantic clarity rename');
 
-    if (hasContentSessionId) {
-      // Already renamed, just record migration
-      this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(17, new Date().toISOString());
-      return;
-    }
+    let renamesPerformed = 0;
 
-    logger.info('DB', 'Renaming session ID columns for semantic clarity');
+    // Helper to safely rename a column if it exists
+    const safeRenameColumn = (table: string, oldCol: string, newCol: string): boolean => {
+      try {
+        const tableInfo = this.db.query(`PRAGMA table_info(${table})`).all() as TableColumnInfo[];
+        const hasOldCol = tableInfo.some(col => col.name === oldCol);
+        const hasNewCol = tableInfo.some(col => col.name === newCol);
 
-    // Begin transaction for atomic rename
-    this.db.run('BEGIN TRANSACTION');
+        if (hasNewCol) {
+          // Already renamed, nothing to do
+          return false;
+        }
 
-    try {
-      // SQLite 3.25+ supports ALTER TABLE RENAME COLUMN
-      // Rename in sdk_sessions table
-      this.db.run('ALTER TABLE sdk_sessions RENAME COLUMN claude_session_id TO content_session_id');
-      this.db.run('ALTER TABLE sdk_sessions RENAME COLUMN sdk_session_id TO memory_session_id');
+        if (hasOldCol) {
+          // SQLite 3.25+ supports ALTER TABLE RENAME COLUMN
+          this.db.run(`ALTER TABLE ${table} RENAME COLUMN ${oldCol} TO ${newCol}`);
+          logger.info('DB', `Renamed ${table}.${oldCol} to ${newCol}`);
+          return true;
+        }
 
-      // Rename in pending_messages table
-      this.db.run('ALTER TABLE pending_messages RENAME COLUMN claude_session_id TO content_session_id');
+        // Neither column exists - table might not exist or has different schema
+        logger.warn('DB', `Column ${oldCol} not found in ${table}, skipping rename`);
+        return false;
+      } catch (error: any) {
+        // Table might not exist yet, which is fine
+        logger.warn('DB', `Could not rename ${table}.${oldCol}: ${error.message}`);
+        return false;
+      }
+    };
 
-      // Rename in observations table
-      this.db.run('ALTER TABLE observations RENAME COLUMN sdk_session_id TO memory_session_id');
+    // Rename in sdk_sessions table
+    if (safeRenameColumn('sdk_sessions', 'claude_session_id', 'content_session_id')) renamesPerformed++;
+    if (safeRenameColumn('sdk_sessions', 'sdk_session_id', 'memory_session_id')) renamesPerformed++;
 
-      // Rename in session_summaries table
-      this.db.run('ALTER TABLE session_summaries RENAME COLUMN sdk_session_id TO memory_session_id');
+    // Rename in pending_messages table
+    if (safeRenameColumn('pending_messages', 'claude_session_id', 'content_session_id')) renamesPerformed++;
 
-      // Rename in user_prompts table
-      this.db.run('ALTER TABLE user_prompts RENAME COLUMN claude_session_id TO content_session_id');
+    // Rename in observations table
+    if (safeRenameColumn('observations', 'sdk_session_id', 'memory_session_id')) renamesPerformed++;
 
-      // Commit transaction
-      this.db.run('COMMIT');
+    // Rename in session_summaries table
+    if (safeRenameColumn('session_summaries', 'sdk_session_id', 'memory_session_id')) renamesPerformed++;
 
-      // Record migration
-      this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(17, new Date().toISOString());
+    // Rename in user_prompts table
+    if (safeRenameColumn('user_prompts', 'claude_session_id', 'content_session_id')) renamesPerformed++;
 
-      logger.info('DB', 'Successfully renamed session ID columns');
-    } catch (error: any) {
-      // Rollback on error
-      this.db.run('ROLLBACK');
-      logger.error('DB', 'Session ID column rename migration error', undefined, error);
-      throw error;
+    // Record migration
+    this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(17, new Date().toISOString());
+
+    if (renamesPerformed > 0) {
+      logger.info('DB', `Successfully renamed ${renamesPerformed} session ID columns`);
+    } else {
+      logger.info('DB', 'No session ID column renames needed (already up to date)');
     }
   }
 
   /**
    * Repair session ID column renames (migration 19)
-   * Migration 17 may have been recorded but failed to actually rename columns.
-   * This migration checks each table and renames if needed (idempotent).
+   * DEPRECATED: Migration 17 is now fully idempotent and handles all cases.
+   * This migration is kept for backwards compatibility but does nothing.
    */
   private repairSessionIdColumnRename(): void {
-    try {
-      const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(19) as SchemaVersion | undefined;
-      if (applied) return;
+    const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(19) as SchemaVersion | undefined;
+    if (applied) return;
 
-      logger.info('DB', 'Checking session ID column renames (repair migration)');
-
-      let repairsNeeded = false;
-
-      // Check and fix sdk_sessions
-      const sessionsInfo = this.db.query('PRAGMA table_info(sdk_sessions)').all() as TableColumnInfo[];
-      if (sessionsInfo.some(col => col.name === 'claude_session_id')) {
-        logger.info('DB', 'Repairing sdk_sessions columns');
-        this.db.run('ALTER TABLE sdk_sessions RENAME COLUMN claude_session_id TO content_session_id');
-        this.db.run('ALTER TABLE sdk_sessions RENAME COLUMN sdk_session_id TO memory_session_id');
-        repairsNeeded = true;
-      }
-
-      // Check and fix pending_messages
-      const pendingInfo = this.db.query('PRAGMA table_info(pending_messages)').all() as TableColumnInfo[];
-      if (pendingInfo.some(col => col.name === 'claude_session_id')) {
-        logger.info('DB', 'Repairing pending_messages columns');
-        this.db.run('ALTER TABLE pending_messages RENAME COLUMN claude_session_id TO content_session_id');
-        repairsNeeded = true;
-      }
-
-      // Check and fix observations
-      const obsInfo = this.db.query('PRAGMA table_info(observations)').all() as TableColumnInfo[];
-      if (obsInfo.some(col => col.name === 'sdk_session_id')) {
-        logger.info('DB', 'Repairing observations columns');
-        this.db.run('ALTER TABLE observations RENAME COLUMN sdk_session_id TO memory_session_id');
-        repairsNeeded = true;
-      }
-
-      // Check and fix session_summaries
-      const summariesInfo = this.db.query('PRAGMA table_info(session_summaries)').all() as TableColumnInfo[];
-      if (summariesInfo.some(col => col.name === 'sdk_session_id')) {
-        logger.info('DB', 'Repairing session_summaries columns');
-        this.db.run('ALTER TABLE session_summaries RENAME COLUMN sdk_session_id TO memory_session_id');
-        repairsNeeded = true;
-      }
-
-      // Check and fix user_prompts
-      const promptsInfo = this.db.query('PRAGMA table_info(user_prompts)').all() as TableColumnInfo[];
-      if (promptsInfo.some(col => col.name === 'claude_session_id')) {
-        logger.info('DB', 'Repairing user_prompts columns');
-        this.db.run('ALTER TABLE user_prompts RENAME COLUMN claude_session_id TO content_session_id');
-        repairsNeeded = true;
-      }
-
-      // Record migration
-      this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(19, new Date().toISOString());
-
-      if (repairsNeeded) {
-        logger.info('DB', 'Session ID column rename repairs completed');
-      } else {
-        logger.info('DB', 'No session ID column repairs needed');
-      }
-    } catch (error: any) {
-      logger.error('DB', 'Session ID column rename repair error', undefined, error);
-      throw error;
-    }
+    // Migration 17 now handles all column rename cases idempotently.
+    // Just record this migration as applied.
+    this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(19, new Date().toISOString());
   }
 
   /**

tests/session_store.test.ts

@@ -37,9 +37,9 @@ describe('SessionStore', () => {
     const claudeId = 'claude-sess-obs';
     const sdkId = store.createSDKSession(claudeId, 'test-project', 'initial prompt');
     
-    // Get the sdk_session_id string (createSDKSession returns number ID, need string for FK)
-    // Wait, createSDKSession inserts using sdk_session_id = claude_session_id in the current implementation
-    // "VALUES (?, ?, ?, ?, ?, ?, 'active')" -> claudeSessionId, claudeSessionId, ...
+    // Get the memory_session_id string (createSDKSession returns number ID, need string for FK)
+    // createSDKSession inserts using memory_session_id = content_session_id in the current implementation
+    // "VALUES (?, ?, ?, ?, ?, ?, 'active')" -> contentSessionId, contentSessionId, ...
     
     const obs = {
       type: 'discovery',

tests/validate_sql_update.test.ts

@@ -7,11 +7,12 @@ describe('Refactor Validation: SQL Updates', () => {
   beforeEach(() => {
     db = new Database(':memory:');
     // Minimal schema for sdk_sessions based on SessionStore.ts migration004
+    // Uses new column names: content_session_id and memory_session_id
     db.run(`
       CREATE TABLE sdk_sessions (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
-        claude_session_id TEXT UNIQUE NOT NULL,
-        sdk_session_id TEXT UNIQUE,
+        content_session_id TEXT UNIQUE NOT NULL,
+        memory_session_id TEXT UNIQUE,
         project TEXT NOT NULL,
         user_prompt TEXT,
         started_at TEXT,
@@ -27,28 +28,26 @@ describe('Refactor Validation: SQL Updates', () => {
     db.close();
   });
 
-  it('should update sdk_session_id using direct SQL (replacing updateSDKSessionId)', () => {
-    // Setup initial state: A session without an sdk_session_id
-    const claudeId = 'claude-session-123';
-    const syntheticId = 'sdk-session-456';
-    
+  it('should update memory_session_id using direct SQL (replacing updateSDKSessionId)', () => {
+    // Setup initial state: A session without a memory_session_id
+    const contentId = 'content-session-123';
+    const memoryId = 'memory-session-456';
+
     db.prepare(`
-      INSERT INTO sdk_sessions (claude_session_id, project, started_at, started_at_epoch)
+      INSERT INTO sdk_sessions (content_session_id, project, started_at, started_at_epoch)
       VALUES (?, ?, ?, ?)
-    `).run(claudeId, 'test-project', '2025-01-01T00:00:00Z', 1735689600000);
+    `).run(contentId, 'test-project', '2025-01-01T00:00:00Z', 1735689600000);
 
     // Verify initial state
-    const before = db.prepare('SELECT sdk_session_id FROM sdk_sessions WHERE claude_session_id = ?').get(claudeId) as any;
-    expect(before.sdk_session_id).toBeNull();
+    const before = db.prepare('SELECT memory_session_id FROM sdk_sessions WHERE content_session_id = ?').get(contentId) as any;
+    expect(before.memory_session_id).toBeNull();
 
-    // EXECUTE: The exact SQL statement from the refactor in import-xml-observations.ts
-    // Original code: db['db'].prepare('UPDATE sdk_sessions SET sdk_session_id = ? WHERE claude_session_id = ?').run(syntheticSdkSessionId, sessionMeta.sessionId);
-    
-    const stmt = db.prepare('UPDATE sdk_sessions SET sdk_session_id = ? WHERE claude_session_id = ?');
-    stmt.run(syntheticId, claudeId);
+    // EXECUTE: The exact SQL statement from the refactor
+    const stmt = db.prepare('UPDATE sdk_sessions SET memory_session_id = ? WHERE content_session_id = ?');
+    stmt.run(memoryId, contentId);
 
     // VERIFY: The update happened
-    const after = db.prepare('SELECT sdk_session_id FROM sdk_sessions WHERE claude_session_id = ?').get(claudeId) as any;
-    expect(after.sdk_session_id).toBe(syntheticId);
+    const after = db.prepare('SELECT memory_session_id FROM sdk_sessions WHERE content_session_id = ?').get(contentId) as any;
+    expect(after.memory_session_id).toBe(memoryId);
   });
 });