feat: Add ROI tracking with discovery_tokens for observations and session summaries (#111)
* feat: Add discovery_tokens for ROI tracking in observations and session summaries - Introduced `discovery_tokens` column in `observations` and `session_summaries` tables to track token costs associated with discovering and creating each observation and summary. - Updated relevant services and hooks to calculate and display ROI metrics based on discovery tokens. - Enhanced context economics reporting to include savings from reusing previous observations. - Implemented migration to ensure the new column is added to existing tables. - Adjusted data models and sync processes to accommodate the new `discovery_tokens` field. * refactor: streamline context hook by removing unused functions and updating terminology - Removed the estimateTokens and getObservations helper functions as they were not utilized. - Updated the legend and output messages to replace "discovery" with "work" for clarity. - Changed the emoji representation for different observation types to better reflect their purpose. - Enhanced output formatting for improved readability and understanding of token usage. * Refactor user-message-hook and context-hook for improved clarity and functionality - Updated user-message-hook.js to enhance error messaging and improve variable naming for clarity. - Modified context-hook.ts to include a new column key section, improved context index instructions, and added emoji icons for observation types. - Adjusted footer messages in context-hook.ts to emphasize token savings and access to past research. - Changed user-message-hook.ts to update the feedback and support message for clarity. * fix: Critical ROI tracking fixes from PR review Addresses critical findings from PR #111 review: 1. **Fixed incorrect discovery token calculation** (src/services/worker/SDKAgent.ts) - Changed from passing cumulative total to per-response delta - Now correctly tracks token cost for each observation/summary - Captures token state before/after response processing - Prevents all observations getting inflated cumulative values 2. **Fixed schema version mismatch** (src/services/sqlite/SessionStore.ts) - Changed ensureDiscoveryTokensColumn() from version 11 to version 7 - Now matches migration007 definition in migrations.ts - Ensures consistent version tracking across migration system These fixes ensure ROI metrics accurately reflect token costs. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Alex Newman
@@ -28,6 +28,7 @@ export class SessionStore {
|
||||
this.addObservationHierarchicalFields();
|
||||
this.makeObservationsTextNullable();
|
||||
this.createUserPromptsTable();
|
||||
this.ensureDiscoveryTokensColumn();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -492,6 +493,40 @@ export class SessionStore {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure discovery_tokens column exists (migration 7)
|
||||
*/
|
||||
private ensureDiscoveryTokensColumn(): void {
|
||||
try {
|
||||
// Check if migration already applied
|
||||
const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(7) as {version: number} | undefined;
|
||||
if (applied) return;
|
||||
|
||||
// Check if discovery_tokens column exists in observations table
|
||||
const observationsInfo = this.db.pragma('table_info(observations)');
|
||||
const obsHasDiscoveryTokens = (observationsInfo as any[]).some((col: any) => col.name === 'discovery_tokens');
|
||||
|
||||
if (!obsHasDiscoveryTokens) {
|
||||
this.db.exec('ALTER TABLE observations ADD COLUMN discovery_tokens INTEGER DEFAULT 0');
|
||||
console.error('[SessionStore] Added discovery_tokens column to observations table');
|
||||
}
|
||||
|
||||
// Check if discovery_tokens column exists in session_summaries table
|
||||
const summariesInfo = this.db.pragma('table_info(session_summaries)');
|
||||
const sumHasDiscoveryTokens = (summariesInfo as any[]).some((col: any) => col.name === 'discovery_tokens');
|
||||
|
||||
if (!sumHasDiscoveryTokens) {
|
||||
this.db.exec('ALTER TABLE session_summaries ADD COLUMN discovery_tokens INTEGER DEFAULT 0');
|
||||
console.error('[SessionStore] Added discovery_tokens column to session_summaries table');
|
||||
}
|
||||
|
||||
// Record migration
|
||||
this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(7, new Date().toISOString());
|
||||
} catch (error: any) {
|
||||
console.error('[SessionStore] Discovery tokens migration error:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent session summaries for a project
|
||||
*/
|
||||
@@ -1074,7 +1109,8 @@ export class SessionStore {
|
||||
files_read: string[];
|
||||
files_modified: string[];
|
||||
},
|
||||
promptNumber?: number
|
||||
promptNumber?: number,
|
||||
discoveryTokens: number = 0
|
||||
): { id: number; createdAtEpoch: number } {
|
||||
const now = new Date();
|
||||
const nowEpoch = now.getTime();
|
||||
@@ -1105,8 +1141,8 @@ export class SessionStore {
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO observations
|
||||
(sdk_session_id, project, type, title, subtitle, facts, narrative, concepts,
|
||||
files_read, files_modified, prompt_number, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
files_read, files_modified, prompt_number, discovery_tokens, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const result = stmt.run(
|
||||
@@ -1121,6 +1157,7 @@ export class SessionStore {
|
||||
JSON.stringify(observation.files_read),
|
||||
JSON.stringify(observation.files_modified),
|
||||
promptNumber || null,
|
||||
discoveryTokens,
|
||||
now.toISOString(),
|
||||
nowEpoch
|
||||
);
|
||||
@@ -1146,7 +1183,8 @@ export class SessionStore {
|
||||
next_steps: string;
|
||||
notes: string | null;
|
||||
},
|
||||
promptNumber?: number
|
||||
promptNumber?: number,
|
||||
discoveryTokens: number = 0
|
||||
): { id: number; createdAtEpoch: number } {
|
||||
const now = new Date();
|
||||
const nowEpoch = now.getTime();
|
||||
@@ -1177,8 +1215,8 @@ export class SessionStore {
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO session_summaries
|
||||
(sdk_session_id, project, request, investigated, learned, completed,
|
||||
next_steps, notes, prompt_number, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
next_steps, notes, prompt_number, discovery_tokens, created_at, created_at_epoch)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
const result = stmt.run(
|
||||
@@ -1191,6 +1229,7 @@ export class SessionStore {
|
||||
summary.next_steps,
|
||||
summary.notes,
|
||||
promptNumber || null,
|
||||
discoveryTokens,
|
||||
now.toISOString(),
|
||||
nowEpoch
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user