Add TypeScript Agent SDK reference documentation

- Introduced comprehensive API reference for the TypeScript Agent SDK.
- Documented installation instructions for the SDK.
- Detailed the main functions: `query()`, `tool()`, and `createSdkMcpServer()`.
- Defined various types including `Options`, `Query`, `AgentDefinition`, and more.
- Included message types and their structures, such as `SDKMessage`, `SDKAssistantMessage`, and `SDKUserMessage`.
- Explained hook types and their usage within the SDK.
- Provided detailed documentation for tool input and output types.
- Added sections on permission types and other relevant types for better clarity.

src/services/worker-service-LEGACY.ts

@@ -569,6 +569,7 @@ class WorkerService {
           ss.id,
           s.claude_session_id as session_id,
           ss.request,
+          ss.investigated,
           ss.learned,
           ss.completed,
           ss.next_steps,

src/services/worker-service.ts

@@ -43,7 +43,6 @@ export class WorkerService {
 
   // Processing status tracking for viewer UI spinner
   private isProcessing: boolean = false;
-  private spinnerStopTimer: NodeJS.Timeout | null = null;
 
   constructor() {
     this.app = express();
@@ -96,6 +95,7 @@ export class WorkerService {
     this.app.get('/api/prompts', this.handleGetPrompts.bind(this));
     this.app.get('/api/stats', this.handleGetStats.bind(this));
     this.app.get('/api/processing-status', this.handleGetProcessingStatus.bind(this));
+    this.app.post('/api/processing', this.handleSetProcessing.bind(this));
 
     // Settings
     this.app.get('/api/settings', this.handleGetSettings.bind(this));
@@ -266,6 +266,7 @@ export class WorkerService {
 
   /**
    * Queue observations for processing
+   * CRITICAL: Ensures SDK agent is running to process the queue (ALWAYS SAVE EVERYTHING)
    */
   private handleObservations(req: Request, res: Response): void {
     try {
@@ -279,6 +280,14 @@ export class WorkerService {
         prompt_number
       });
 
+      // CRITICAL: Ensure SDK agent is running to consume the queue
+      const session = this.sessionManager.getSession(sessionDbId);
+      if (session && !session.generatorPromise) {
+        session.generatorPromise = this.sdkAgent.startSession(session, this).catch(err => {
+          logger.failure('WORKER', 'SDK agent error', { sessionId: sessionDbId }, err);
+        });
+      }
+
       // Broadcast SSE event
       this.sseBroadcaster.broadcast({
         type: 'observation_queued',
@@ -294,12 +303,21 @@ export class WorkerService {
 
   /**
    * Queue summarize request
+   * CRITICAL: Ensures SDK agent is running to process the queue (ALWAYS SAVE EVERYTHING)
    */
   private handleSummarize(req: Request, res: Response): void {
     try {
       const sessionDbId = parseInt(req.params.sessionDbId, 10);
       this.sessionManager.queueSummarize(sessionDbId);
 
+      // CRITICAL: Ensure SDK agent is running to consume the queue
+      const session = this.sessionManager.getSession(sessionDbId);
+      if (session && !session.generatorPromise) {
+        session.generatorPromise = this.sdkAgent.startSession(session, this).catch(err => {
+          logger.failure('WORKER', 'SDK agent error', { sessionId: sessionDbId }, err);
+        });
+      }
+
       res.json({ status: 'queued' });
     } catch (error) {
       logger.failure('WORKER', 'Summarize queuing failed', {}, error as Error);
@@ -604,25 +622,24 @@ export class WorkerService {
   }
 
   /**
-   * Check if all sessions have empty queues and stop spinner after debounce (1.5s)
+   * Set processing status (called by hooks)
    */
-  checkAndStopSpinner(): void {
-    // Clear any existing timer
-    if (this.spinnerStopTimer) {
-      clearTimeout(this.spinnerStopTimer);
-      this.spinnerStopTimer = null;
-    }
+  private handleSetProcessing(req: Request, res: Response): void {
+    try {
+      const { isProcessing } = req.body;
 
-    // Check if any session has pending messages
-    if (!this.sessionManager.hasPendingMessages()) {
-      // Debounce: wait 1.5s and check again
-      this.spinnerStopTimer = setTimeout(() => {
-        if (!this.sessionManager.hasPendingMessages()) {
-          logger.debug('WORKER', 'All queues empty - stopping spinner');
-          this.broadcastProcessingStatus(false);
-        }
-        this.spinnerStopTimer = null;
-      }, 1500);
+      if (typeof isProcessing !== 'boolean') {
+        res.status(400).json({ error: 'isProcessing must be a boolean' });
+        return;
+      }
+
+      this.broadcastProcessingStatus(isProcessing);
+      logger.debug('WORKER', 'Processing status updated', { isProcessing });
+
+      res.json({ status: 'ok', isProcessing });
+    } catch (error) {
+      logger.failure('WORKER', 'Failed to set processing status', {}, error as Error);
+      res.status(500).json({ error: (error as Error).message });
     }
   }
 }

src/services/worker/PaginationHelper.ts

@@ -17,17 +17,73 @@ export class PaginationHelper {
     this.dbManager = dbManager;
   }
 
+  /**
+   * Strip project path from file paths using heuristic
+   * Converts "/Users/user/project/src/file.ts" -> "src/file.ts"
+   * Uses first occurrence of project name from left (project root)
+   */
+  private stripProjectPath(filePath: string, projectName: string): string {
+    const marker = `/${projectName}/`;
+    const index = filePath.indexOf(marker);
+
+    if (index !== -1) {
+      // Strip everything before and including the project name
+      return filePath.substring(index + marker.length);
+    }
+
+    // Fallback: return original path if project name not found
+    return filePath;
+  }
+
+  /**
+   * Strip project path from JSON array of file paths
+   */
+  private stripProjectPaths(filePathsStr: string | null, projectName: string): string | null {
+    if (!filePathsStr) return filePathsStr;
+
+    try {
+      // Parse JSON array
+      const paths = JSON.parse(filePathsStr) as string[];
+
+      // Strip project path from each file
+      const strippedPaths = paths.map(p => this.stripProjectPath(p, projectName));
+
+      // Return as JSON string
+      return JSON.stringify(strippedPaths);
+    } catch (error) {
+      // If parsing fails, return original string
+      return filePathsStr;
+    }
+  }
+
+  /**
+   * Sanitize observation by stripping project paths from files
+   */
+  private sanitizeObservation(obs: Observation): Observation {
+    return {
+      ...obs,
+      files_read: this.stripProjectPaths(obs.files_read, obs.project),
+      files_modified: this.stripProjectPaths(obs.files_modified, obs.project)
+    };
+  }
+
   /**
    * Get paginated observations
    */
   getObservations(offset: number, limit: number, project?: string): PaginatedResult<Observation> {
-    return this.paginate<Observation>(
+    const result = this.paginate<Observation>(
       'observations',
       'id, sdk_session_id, project, type, title, subtitle, narrative, text, facts, concepts, files_read, files_modified, prompt_number, created_at, created_at_epoch',
       offset,
       limit,
       project
     );
+
+    // Strip project paths from file paths before returning
+    return {
+      ...result,
+      items: result.items.map(obs => this.sanitizeObservation(obs))
+    };
   }
 
   /**
@@ -41,6 +97,7 @@ export class PaginationHelper {
         ss.id,
         s.claude_session_id as session_id,
         ss.request,
+        ss.investigated,
         ss.learned,
         ss.completed,
         ss.next_steps,