feat: Implement full-text search for user prompts

- Added functionality to save raw user prompts for full-text search in the newHook function.
- Introduced new search endpoint 'search_user_prompts' to retrieve user prompts using FTS5.
- Created UserPromptRow and UserPromptSearchResult types for handling user prompt data.
- Implemented searchUserPrompts method in SessionSearch class to perform FTS5 queries.
- Created user_prompts table with FTS5 support and necessary triggers for data synchronization.
- Updated SessionStore to include methods for saving user prompts and managing the new table.
This commit is contained in:
Alex Newman
2025-10-21 22:02:06 -04:00
parent a62887a6e0
commit 726f167ebf
12 changed files with 781 additions and 153 deletions
+98 -1
View File
@@ -3,10 +3,12 @@ import { DATA_DIR, DB_PATH, ensureDir } from '../../shared/paths.js';
import {
ObservationSearchResult,
SessionSummarySearchResult,
UserPromptSearchResult,
SearchOptions,
SearchFilters,
DateRange,
ObservationRow
ObservationRow,
UserPromptRow
} from './types.js';
/**
@@ -473,6 +475,101 @@ export class SessionSearch {
return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
}
/**
* Search user prompts with full-text search
*/
searchUserPrompts(query: string, options: SearchOptions = {}): UserPromptSearchResult[] {
const params: any[] = [];
const { limit = 20, offset = 0, orderBy = 'relevance', ...filters } = options;
// Build FTS5 match query
const ftsQuery = this.escapeFTS5(query);
params.push(ftsQuery);
// Build filter conditions (join with sdk_sessions for project filtering)
const baseConditions: string[] = [];
if (filters.project) {
baseConditions.push('s.project = ?');
params.push(filters.project);
}
if (filters.dateRange) {
const { start, end } = filters.dateRange;
if (start) {
const startEpoch = typeof start === 'number' ? start : new Date(start).getTime();
baseConditions.push('up.created_at_epoch >= ?');
params.push(startEpoch);
}
if (end) {
const endEpoch = typeof end === 'number' ? end : new Date(end).getTime();
baseConditions.push('up.created_at_epoch <= ?');
params.push(endEpoch);
}
}
const whereClause = baseConditions.length > 0 ? `AND ${baseConditions.join(' AND ')}` : '';
// Build ORDER BY
const orderClause = orderBy === 'relevance'
? 'ORDER BY user_prompts_fts.rank ASC'
: orderBy === 'date_asc'
? 'ORDER BY up.created_at_epoch ASC'
: 'ORDER BY up.created_at_epoch DESC';
// Main query with FTS5 (join sdk_sessions for project filtering)
const sql = `
SELECT
up.*,
user_prompts_fts.rank as rank
FROM user_prompts up
JOIN user_prompts_fts ON up.id = user_prompts_fts.rowid
JOIN sdk_sessions s ON up.claude_session_id = s.claude_session_id
WHERE user_prompts_fts MATCH ?
${whereClause}
${orderClause}
LIMIT ? OFFSET ?
`;
params.push(limit, offset);
const results = this.db.prepare(sql).all(...params) as UserPromptSearchResult[];
// Normalize rank to score
if (results.length > 0) {
const minRank = Math.min(...results.map(r => r.rank || 0));
const maxRank = Math.max(...results.map(r => r.rank || 0));
const range = maxRank - minRank || 1;
results.forEach(r => {
if (r.rank !== undefined) {
r.score = 1 - ((r.rank - minRank) / range);
}
});
}
return results;
}
/**
* Get all prompts for a session by claude_session_id
*/
getUserPromptsBySession(claudeSessionId: string): UserPromptRow[] {
const stmt = this.db.prepare(`
SELECT
id,
claude_session_id,
prompt_number,
prompt_text,
created_at,
created_at_epoch
FROM user_prompts
WHERE claude_session_id = ?
ORDER BY prompt_number ASC
`);
return stmt.all(claudeSessionId) as UserPromptRow[];
}
/**
* Close the database connection
*/
+104
View File
@@ -27,6 +27,7 @@ export class SessionStore {
this.removeSessionSummariesUniqueConstraint();
this.addObservationHierarchicalFields();
this.makeObservationsTextNullable();
this.createUserPromptsTable();
}
/**
@@ -405,6 +406,92 @@ export class SessionStore {
}
}
/**
* Create user_prompts table with FTS5 support (migration 10)
*/
private createUserPromptsTable(): void {
try {
// Check if migration already applied
const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(10) as {version: number} | undefined;
if (applied) return;
// Check if table already exists
const tableInfo = this.db.pragma('table_info(user_prompts)');
if ((tableInfo as any[]).length > 0) {
// Already migrated
this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(10, new Date().toISOString());
return;
}
console.error('[SessionStore] Creating user_prompts table with FTS5 support...');
// Begin transaction
this.db.exec('BEGIN TRANSACTION');
try {
// Create main table (using claude_session_id since sdk_session_id is set asynchronously by worker)
this.db.exec(`
CREATE TABLE user_prompts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
claude_session_id TEXT NOT NULL,
prompt_number INTEGER NOT NULL,
prompt_text TEXT NOT NULL,
created_at TEXT NOT NULL,
created_at_epoch INTEGER NOT NULL,
FOREIGN KEY(claude_session_id) REFERENCES sdk_sessions(claude_session_id) ON DELETE CASCADE
);
CREATE INDEX idx_user_prompts_claude_session ON user_prompts(claude_session_id);
CREATE INDEX idx_user_prompts_created ON user_prompts(created_at_epoch DESC);
CREATE INDEX idx_user_prompts_prompt_number ON user_prompts(prompt_number);
`);
// Create FTS5 virtual table
this.db.exec(`
CREATE VIRTUAL TABLE user_prompts_fts USING fts5(
prompt_text,
content='user_prompts',
content_rowid='id'
);
`);
// Create triggers to sync FTS5
this.db.exec(`
CREATE TRIGGER user_prompts_ai AFTER INSERT ON user_prompts BEGIN
INSERT INTO user_prompts_fts(rowid, prompt_text)
VALUES (new.id, new.prompt_text);
END;
CREATE TRIGGER user_prompts_ad AFTER DELETE ON user_prompts BEGIN
INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text)
VALUES('delete', old.id, old.prompt_text);
END;
CREATE TRIGGER user_prompts_au AFTER UPDATE ON user_prompts BEGIN
INSERT INTO user_prompts_fts(user_prompts_fts, rowid, prompt_text)
VALUES('delete', old.id, old.prompt_text);
INSERT INTO user_prompts_fts(rowid, prompt_text)
VALUES (new.id, new.prompt_text);
END;
`);
// Commit transaction
this.db.exec('COMMIT');
// Record migration
this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(10, new Date().toISOString());
console.error('[SessionStore] Successfully created user_prompts table with FTS5 support');
} catch (error: any) {
// Rollback on error
this.db.exec('ROLLBACK');
throw error;
}
} catch (error: any) {
console.error('[SessionStore] Migration error (create user_prompts table):', error.message);
}
}
/**
* Get recent session summaries for a project
*/
@@ -788,6 +875,23 @@ export class SessionStore {
return result?.worker_port || null;
}
/**
* Save a user prompt
*/
saveUserPrompt(claudeSessionId: string, promptNumber: number, promptText: string): number {
const now = new Date();
const nowEpoch = now.getTime();
const stmt = this.db.prepare(`
INSERT INTO user_prompts
(claude_session_id, prompt_number, prompt_text, created_at, created_at_epoch)
VALUES (?, ?, ?, ?, ?)
`);
const result = stmt.run(claudeSessionId, promptNumber, promptText, now.toISOString(), nowEpoch);
return result.lastInsertRowid as number;
}
/**
* Store an observation (from SDK parsing)
*/
+14
View File
@@ -236,6 +236,15 @@ export interface SessionSummaryRow {
created_at_epoch: number;
}
export interface UserPromptRow {
id: number;
claude_session_id: string;
prompt_number: number;
prompt_text: string;
created_at: string;
created_at_epoch: number;
}
/**
* Search and Filter Types
*/
@@ -267,3 +276,8 @@ export interface SessionSummarySearchResult extends SessionSummaryRow {
rank?: number; // FTS5 relevance score (lower is better)
score?: number; // Normalized score (higher is better, 0-1)
}
export interface UserPromptSearchResult extends UserPromptRow {
rank?: number; // FTS5 relevance score (lower is better)
score?: number; // Normalized score (higher is better, 0-1)
}