feat: Add ROI tracking with discovery_tokens for observations and session summaries (#111)

* feat: Add discovery_tokens for ROI tracking in observations and session summaries

- Introduced `discovery_tokens` column in `observations` and `session_summaries` tables to track token costs associated with discovering and creating each observation and summary.
- Updated relevant services and hooks to calculate and display ROI metrics based on discovery tokens.
- Enhanced context economics reporting to include savings from reusing previous observations.
- Implemented migration to ensure the new column is added to existing tables.
- Adjusted data models and sync processes to accommodate the new `discovery_tokens` field.

* refactor: streamline context hook by removing unused functions and updating terminology

- Removed the estimateTokens and getObservations helper functions as they were not utilized.
- Updated the legend and output messages to replace "discovery" with "work" for clarity.
- Changed the emoji representation for different observation types to better reflect their purpose.
- Enhanced output formatting for improved readability and understanding of token usage.

* Refactor user-message-hook and context-hook for improved clarity and functionality

- Updated user-message-hook.js to enhance error messaging and improve variable naming for clarity.
- Modified context-hook.ts to include a new column key section, improved context index instructions, and added emoji icons for observation types.
- Adjusted footer messages in context-hook.ts to emphasize token savings and access to past research.
- Changed user-message-hook.ts to update the feedback and support message for clarity.

* fix: Critical ROI tracking fixes from PR review

Addresses critical findings from PR #111 review:

1. **Fixed incorrect discovery token calculation** (src/services/worker/SDKAgent.ts)
   - Changed from passing cumulative total to per-response delta
   - Now correctly tracks token cost for each observation/summary
   - Captures token state before/after response processing
   - Prevents all observations getting inflated cumulative values

2. **Fixed schema version mismatch** (src/services/sqlite/SessionStore.ts)
   - Changed ensureDiscoveryTokensColumn() from version 11 to version 7
   - Now matches migration007 definition in migrations.ts
   - Ensures consistent version tracking across migration system

These fixes ensure ROI metrics accurately reflect token costs.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

---------

Co-authored-by: Claude <noreply@anthropic.com>

src/services/worker/SDKAgent.ts

@@ -85,6 +85,34 @@ export class SDKAgent {
 
           const responseSize = textContent.length;
 
+          // Capture token state BEFORE updating (for delta calculation)
+          const tokensBeforeResponse = session.cumulativeInputTokens + session.cumulativeOutputTokens;
+
+          // Extract and track token usage
+          const usage = message.message.usage;
+          if (usage) {
+            session.cumulativeInputTokens += usage.input_tokens || 0;
+            session.cumulativeOutputTokens += usage.output_tokens || 0;
+
+            // Cache creation counts as discovery, cache read doesn't
+            if (usage.cache_creation_input_tokens) {
+              session.cumulativeInputTokens += usage.cache_creation_input_tokens;
+            }
+
+            logger.debug('SDK', 'Token usage captured', {
+              sessionId: session.sessionDbId,
+              inputTokens: usage.input_tokens,
+              outputTokens: usage.output_tokens,
+              cacheCreation: usage.cache_creation_input_tokens || 0,
+              cacheRead: usage.cache_read_input_tokens || 0,
+              cumulativeInput: session.cumulativeInputTokens,
+              cumulativeOutput: session.cumulativeOutputTokens
+            });
+          }
+
+          // Calculate discovery tokens (delta for this response only)
+          const discoveryTokens = (session.cumulativeInputTokens + session.cumulativeOutputTokens) - tokensBeforeResponse;
+
           // Only log non-empty responses (filter out noise)
           if (responseSize > 0) {
             const truncatedResponse = responseSize > 100
@@ -95,8 +123,8 @@ export class SDKAgent {
               promptNumber: session.lastPromptNumber
             }, truncatedResponse);
 
-            // Parse and process response
-            await this.processSDKResponse(session, textContent, worker);
+            // Parse and process response with discovery token delta
+            await this.processSDKResponse(session, textContent, worker, discoveryTokens);
           }
         }
 
@@ -218,8 +246,9 @@ export class SDKAgent {
 
   /**
    * Process SDK response text (parse XML, save to database, sync to Chroma)
+   * @param discoveryTokens - Token cost for discovering this response (delta, not cumulative)
    */
-  private async processSDKResponse(session: ActiveSession, text: string, worker?: any): Promise<void> {
+  private async processSDKResponse(session: ActiveSession, text: string, worker: any | undefined, discoveryTokens: number): Promise<void> {
     // Parse observations
     const observations = parseObservations(text, session.claudeSessionId);
 
@@ -229,7 +258,8 @@ export class SDKAgent {
         session.claudeSessionId,
         session.project,
         obs,
-        session.lastPromptNumber
+        session.lastPromptNumber,
+        discoveryTokens
       );
 
       // Log observation details
@@ -253,7 +283,8 @@ export class SDKAgent {
         session.project,
         obs,
         session.lastPromptNumber,
-        createdAtEpoch
+        createdAtEpoch,
+        discoveryTokens
       ).then(() => {
         const chromaDuration = Date.now() - chromaStart;
         logger.debug('CHROMA', 'Observation synced', {
@@ -305,7 +336,8 @@ export class SDKAgent {
         session.claudeSessionId,
         session.project,
         summary,
-        session.lastPromptNumber
+        session.lastPromptNumber,
+        discoveryTokens
       );
 
       // Log summary details
@@ -326,7 +358,8 @@ export class SDKAgent {
         session.project,
         summary,
         session.lastPromptNumber,
-        createdAtEpoch
+        createdAtEpoch,
+        discoveryTokens
       ).then(() => {
         const chromaDuration = Date.now() - chromaStart;
         logger.debug('CHROMA', 'Summary synced', {

src/services/worker/SessionManager.ts

@@ -89,7 +89,9 @@ export class SessionManager {
       abortController: new AbortController(),
       generatorPromise: null,
       lastPromptNumber: promptNumber || this.dbManager.getSessionStore().getPromptCounter(sessionDbId),
-      startTime: Date.now()
+      startTime: Date.now(),
+      cumulativeInputTokens: 0,
+      cumulativeOutputTokens: 0
     };
 
     this.sessions.set(sessionDbId, session);