diff --git a/docs/public/docs.json b/docs/public/docs.json index 96ad53c1..a9c3efcd 100644 --- a/docs/public/docs.json +++ b/docs/public/docs.json @@ -38,6 +38,7 @@ "usage/getting-started", "usage/search-tools", "usage/private-tags", + "usage/export-import", "beta-features" ] }, diff --git a/docs/public/usage/export-import.mdx b/docs/public/usage/export-import.mdx new file mode 100644 index 00000000..d8c5b3c4 --- /dev/null +++ b/docs/public/usage/export-import.mdx @@ -0,0 +1,282 @@ +--- +title: "Memory Export/Import" +description: "Share knowledge across claude-mem installations with duplicate prevention" +--- + +# Memory Export/Import Scripts + +Share your claude-mem knowledge with other users! These scripts allow you to export specific memories (observations, sessions, summaries, and prompts) and import them into another claude-mem installation. + +## Use Cases + +- **Share Windows compatibility knowledge** with Windows users +- **Share bug fix patterns** with contributors +- **Share project-specific learnings** across teams +- **Backup specific memory sets** for safekeeping + +## How It Works + +### Export Script + +Searches the database using FTS5 full-text search and exports all matching: +- **Observations** - Individual learnings and discoveries +- **Sessions** - Session metadata +- **Summaries** - Session summaries +- **Prompts** - User prompts that led to the work + +Output is a portable JSON file that can be shared. + +### Import Script + +Imports memories with **duplicate prevention**: +- Checks if each record already exists before inserting +- Skips duplicates automatically +- Maintains data integrity with transactional imports +- Reports what was imported vs. skipped + +**Duplicate Detection Strategy:** +- **Sessions**: By `claude_session_id` (unique) +- **Summaries**: By `sdk_session_id` (unique) +- **Observations**: By `sdk_session_id` + `title` + `created_at_epoch` (composite) +- **Prompts**: By `claude_session_id` + `prompt_number` (composite) + +## Usage + +### Export Memories + +```bash +# Export all Windows-related memories +npx tsx scripts/export-memories.ts "windows" windows-memories.json + +# Export bug fixes +npx tsx scripts/export-memories.ts "bugfix" bugfixes.json + +# Export specific feature work +npx tsx scripts/export-memories.ts "progressive disclosure" progressive-disclosure.json +``` + +**Parameters:** +1. `` - Search query (uses FTS5 full-text search) +2. `` - Output JSON file path + +**Example Output:** +``` +šŸ” Searching for: "windows" +āœ… Found 54 observations +āœ… Found 12 sessions +āœ… Found 12 summaries +āœ… Found 7 prompts + +šŸ“¦ Export complete! +šŸ“„ Output: windows-memories.json +šŸ“Š Stats: + • 54 observations + • 12 sessions + • 12 summaries + • 7 prompts +``` + +### Import Memories + +```bash +# Import from an export file +npx tsx scripts/import-memories.ts windows-memories.json +``` + +**Parameters:** +1. `` - Input JSON file (from export script) + +**Example Output:** +``` +šŸ“¦ Import file: windows-memories.json +šŸ“… Exported: 2025-12-10T23:45:00.000Z +šŸ” Query: "windows" +šŸ“Š Contains: + • 54 observations + • 12 sessions + • 12 summaries + • 7 prompts + +šŸ”„ Importing sessions... + āœ… Imported: 12, Skipped: 0 +šŸ”„ Importing summaries... + āœ… Imported: 12, Skipped: 0 +šŸ”„ Importing observations... + āœ… Imported: 54, Skipped: 0 +šŸ”„ Importing prompts... + āœ… Imported: 7, Skipped: 0 + +āœ… Import complete! +šŸ“Š Summary: + Sessions: 12 imported, 0 skipped + Summaries: 12 imported, 0 skipped + Observations: 54 imported, 0 skipped + Prompts: 7 imported, 0 skipped +``` + +### Re-importing (Duplicate Prevention) + +If you run the import again on the same file, duplicates are automatically skipped: + +``` +šŸ”„ Importing sessions... + āœ… Imported: 0, Skipped: 12 ← All skipped (already exist) +šŸ”„ Importing summaries... + āœ… Imported: 0, Skipped: 12 +šŸ”„ Importing observations... + āœ… Imported: 0, Skipped: 54 +šŸ”„ Importing prompts... + āœ… Imported: 0, Skipped: 7 +``` + +## Sharing Memories + +### For Export Authors + +1. **Export your memories:** + ```bash + npx tsx scripts/export-memories.ts "windows" windows-memories.json + ``` + +2. **Share the JSON file** via: + - GitHub gist + - Project repository (`shared-memories/`) + - Direct file transfer + - Package in releases + +3. **Document what's included:** + - What query was used + - What knowledge is contained + - Who might benefit from it + +### For Import Users + +1. **Download the export file** to your local machine + +2. **Review what's in it** (optional): + ```bash + cat windows-memories.json | jq '.totalObservations, .totalSessions' + ``` + +3. **Import into your database:** + ```bash + npx tsx scripts/import-memories.ts windows-memories.json + ``` + +4. **Verify import** by searching: + ```bash + curl "http://localhost:37777/api/search?query=windows&format=index&limit=10" + ``` + +## JSON Export Format + +```json +{ + "exportedAt": "2025-12-10T23:45:00.000Z", + "exportedAtEpoch": 1733876700000, + "query": "windows", + "totalObservations": 54, + "totalSessions": 12, + "totalSummaries": 12, + "totalPrompts": 7, + "observations": [ /* array of observation objects */ ], + "sessions": [ /* array of session objects */ ], + "summaries": [ /* array of summary objects */ ], + "prompts": [ /* array of prompt objects */ ] +} +``` + +## Safety Features + +āœ… **Duplicate Prevention** - Won't re-import existing records +āœ… **Transactional** - All-or-nothing imports (database stays consistent) +āœ… **Read-only Export** - Export script opens database in read-only mode +āœ… **Dependency Ordering** - Sessions imported before observations/summaries +āœ… **Validation** - Checks database exists before starting + +## Advanced Usage + +### Export by Type + +```bash +# Export only discoveries +npx tsx scripts/export-memories.ts "type:discovery" discoveries.json + +# Export only bug fixes +npx tsx scripts/export-memories.ts "type:bugfix" bugfixes.json +``` + +### Export by Date Range + +FTS5 doesn't support date filtering directly, but you can filter the export after: + +```bash +# Export all memories, then filter manually with jq +npx tsx scripts/export-memories.ts "" all-memories.json +cat all-memories.json | jq '.observations |= map(select(.created_at_epoch > 1700000000000))' > recent-memories.json +``` + +### Combine Multiple Exports + +```bash +# Export different topics +npx tsx scripts/export-memories.ts "windows" windows.json +npx tsx scripts/export-memories.ts "linux" linux.json + +# Import both +npx tsx scripts/import-memories.ts windows.json +npx tsx scripts/import-memories.ts linux.json +``` + +## Troubleshooting + +### Database Not Found + +``` +āŒ Database not found at: /Users/you/.claude-mem/claude-mem.db +``` + +**Solution:** Make sure claude-mem is installed and has been run at least once. + +### Import File Not Found + +``` +āŒ Input file not found: windows-memories.json +``` + +**Solution:** Check the file path. Use absolute paths if needed. + +### Partial Import + +If import fails mid-way, the transaction is rolled back - your database remains unchanged. Fix the issue and try again. + +## Contributing Memory Sets + +If you've exported valuable knowledge that others might benefit from: + +1. Create a PR to the `shared-memories/` directory +2. Include a README describing what's in the export +3. Tag with relevant keywords (windows, linux, bugfix, etc.) +4. Community members can then import your knowledge! + +## Examples of Useful Exports + +**Windows Compatibility Knowledge:** +```bash +npx tsx scripts/export-memories.ts "windows compatibility installation" windows-fixes.json +``` + +**Progressive Disclosure Architecture:** +```bash +npx tsx scripts/export-memories.ts "progressive disclosure architecture token" pd-patterns.json +``` + +**Bug Fix Patterns:** +```bash +npx tsx scripts/export-memories.ts "bugfix error handling" bugfix-patterns.json +``` + +**Performance Optimization:** +```bash +npx tsx scripts/export-memories.ts "performance optimization caching" perf-tips.json +``` diff --git a/scripts/export-memories.ts b/scripts/export-memories.ts new file mode 100644 index 00000000..85dcedfd --- /dev/null +++ b/scripts/export-memories.ts @@ -0,0 +1,133 @@ +#!/usr/bin/env node +/** + * Export memories matching a search query to a portable JSON format + * Usage: npx tsx scripts/export-memories.ts + * Example: npx tsx scripts/export-memories.ts "windows" windows-memories.json + */ + +import Database from 'better-sqlite3'; +import { existsSync, writeFileSync } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; + +interface ExportData { + exportedAt: string; + exportedAtEpoch: number; + query: string; + totalObservations: number; + totalSessions: number; + totalSummaries: number; + totalPrompts: number; + observations: any[]; + sessions: any[]; + summaries: any[]; + prompts: any[]; +} + +function exportMemories(query: string, outputFile: string) { + const dbPath = join(homedir(), '.claude-mem', 'claude-mem.db'); + + if (!existsSync(dbPath)) { + console.error(`āŒ Database not found at: ${dbPath}`); + process.exit(1); + } + + const db = new Database(dbPath, { readonly: true }); + + try { + console.log(`šŸ” Searching for: "${query}"`); + + // Build FTS5 query (escape special characters) + const ftsQuery = query.replace(/[^a-zA-Z0-9\s]/g, ' ').trim() + '*'; + + // Get all observations matching the query + const observations = db.prepare(` + SELECT o.* + FROM observations o + INNER JOIN observations_fts fts ON o.id = fts.rowid + WHERE observations_fts MATCH ? + ORDER BY o.created_at_epoch DESC + `).all(ftsQuery); + + console.log(`āœ… Found ${observations.length} observations`); + + // Get unique SDK session IDs from observations + const sdkSessionIds = [...new Set(observations.map((o: any) => o.sdk_session_id))]; + + // Get all sessions for these SDK sessions + const sessions = sdkSessionIds.length > 0 + ? db.prepare(` + SELECT * FROM sdk_sessions + WHERE sdk_session_id IN (${sdkSessionIds.map(() => '?').join(',')}) + ORDER BY started_at_epoch DESC + `).all(...sdkSessionIds) + : []; + + console.log(`āœ… Found ${sessions.length} sessions`); + + // Get all summaries for these SDK sessions + const summaries = sdkSessionIds.length > 0 + ? db.prepare(` + SELECT * FROM session_summaries + WHERE sdk_session_id IN (${sdkSessionIds.map(() => '?').join(',')}) + ORDER BY created_at_epoch DESC + `).all(...sdkSessionIds) + : []; + + console.log(`āœ… Found ${summaries.length} summaries`); + + // Get unique Claude session IDs + const claudeSessionIds = [...new Set(sessions.map((s: any) => s.claude_session_id))]; + + // Get all prompts for these Claude sessions + const prompts = claudeSessionIds.length > 0 + ? db.prepare(` + SELECT * FROM user_prompts + WHERE claude_session_id IN (${claudeSessionIds.map(() => '?').join(',')}) + ORDER BY created_at_epoch DESC + `).all(...claudeSessionIds) + : []; + + console.log(`āœ… Found ${prompts.length} prompts`); + + // Create export data + const exportData: ExportData = { + exportedAt: new Date().toISOString(), + exportedAtEpoch: Date.now(), + query, + totalObservations: observations.length, + totalSessions: sessions.length, + totalSummaries: summaries.length, + totalPrompts: prompts.length, + observations, + sessions, + summaries, + prompts + }; + + // Write to file + writeFileSync(outputFile, JSON.stringify(exportData, null, 2)); + + console.log(`\nšŸ“¦ Export complete!`); + console.log(`šŸ“„ Output: ${outputFile}`); + console.log(`šŸ“Š Stats:`); + console.log(` • ${exportData.totalObservations} observations`); + console.log(` • ${exportData.totalSessions} sessions`); + console.log(` • ${exportData.totalSummaries} summaries`); + console.log(` • ${exportData.totalPrompts} prompts`); + + } finally { + db.close(); + } +} + +// CLI interface +const args = process.argv.slice(2); +if (args.length < 2) { + console.error('Usage: npx tsx scripts/export-memories.ts '); + console.error('Example: npx tsx scripts/export-memories.ts "windows" windows-memories.json'); + process.exit(1); +} + +const [query, outputFile] = args; +exportMemories(query, outputFile); diff --git a/scripts/import-memories.ts b/scripts/import-memories.ts new file mode 100644 index 00000000..9d60c81a --- /dev/null +++ b/scripts/import-memories.ts @@ -0,0 +1,247 @@ +#!/usr/bin/env node +/** + * Import memories from a JSON export file with duplicate prevention + * Usage: npx tsx scripts/import-memories.ts + * Example: npx tsx scripts/import-memories.ts windows-memories.json + */ + +import Database from 'better-sqlite3'; +import { existsSync, readFileSync } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; + +interface ImportStats { + sessionsImported: number; + sessionsSkipped: number; + summariesImported: number; + summariesSkipped: number; + observationsImported: number; + observationsSkipped: number; + promptsImported: number; + promptsSkipped: number; +} + +function importMemories(inputFile: string) { + if (!existsSync(inputFile)) { + console.error(`āŒ Input file not found: ${inputFile}`); + process.exit(1); + } + + const dbPath = join(homedir(), '.claude-mem', 'claude-mem.db'); + + if (!existsSync(dbPath)) { + console.error(`āŒ Database not found at: ${dbPath}`); + process.exit(1); + } + + // Read and parse export file + const exportData = JSON.parse(readFileSync(inputFile, 'utf-8')); + + console.log(`šŸ“¦ Import file: ${inputFile}`); + console.log(`šŸ“… Exported: ${exportData.exportedAt}`); + console.log(`šŸ” Query: "${exportData.query}"`); + console.log(`šŸ“Š Contains:`); + console.log(` • ${exportData.totalObservations} observations`); + console.log(` • ${exportData.totalSessions} sessions`); + console.log(` • ${exportData.totalSummaries} summaries`); + console.log(` • ${exportData.totalPrompts} prompts`); + console.log(''); + + const db = new Database(dbPath); + const stats: ImportStats = { + sessionsImported: 0, + sessionsSkipped: 0, + summariesImported: 0, + summariesSkipped: 0, + observationsImported: 0, + observationsSkipped: 0, + promptsImported: 0, + promptsSkipped: 0 + }; + + try { + // Prepare statements for duplicate checking + const checkSession = db.prepare('SELECT id FROM sdk_sessions WHERE claude_session_id = ?'); + const checkSummary = db.prepare('SELECT id FROM session_summaries WHERE sdk_session_id = ?'); + const checkObservation = db.prepare(` + SELECT id FROM observations + WHERE sdk_session_id = ? + AND title = ? + AND created_at_epoch = ? + `); + const checkPrompt = db.prepare(` + SELECT id FROM user_prompts + WHERE claude_session_id = ? + AND prompt_number = ? + `); + + // Prepare insert statements + const insertSession = db.prepare(` + INSERT INTO sdk_sessions ( + claude_session_id, sdk_session_id, project, user_prompt, + started_at, started_at_epoch, completed_at, completed_at_epoch, + status, worker_port, prompt_counter + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + + const insertSummary = db.prepare(` + INSERT INTO session_summaries ( + sdk_session_id, project, request, investigated, learned, + completed, next_steps, files_read, files_edited, notes, + prompt_number, discovery_tokens, created_at, created_at_epoch + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + + const insertObservation = db.prepare(` + INSERT INTO observations ( + sdk_session_id, project, text, type, title, subtitle, + facts, narrative, concepts, files_read, files_modified, + prompt_number, discovery_tokens, created_at, created_at_epoch + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + + const insertPrompt = db.prepare(` + INSERT INTO user_prompts ( + claude_session_id, prompt_number, prompt_text, + created_at, created_at_epoch + ) VALUES (?, ?, ?, ?, ?) + `); + + // Import in transaction + db.transaction(() => { + // 1. Import sessions first (dependency for everything else) + console.log('šŸ”„ Importing sessions...'); + for (const session of exportData.sessions) { + const exists = checkSession.get(session.claude_session_id); + if (exists) { + stats.sessionsSkipped++; + continue; + } + + insertSession.run( + session.claude_session_id, + session.sdk_session_id, + session.project, + session.user_prompt, + session.started_at, + session.started_at_epoch, + session.completed_at, + session.completed_at_epoch, + session.status, + session.worker_port || null, + session.prompt_counter || null + ); + stats.sessionsImported++; + } + console.log(` āœ… Imported: ${stats.sessionsImported}, Skipped: ${stats.sessionsSkipped}`); + + // 2. Import summaries (depends on sessions) + console.log('šŸ”„ Importing summaries...'); + for (const summary of exportData.summaries) { + const exists = checkSummary.get(summary.sdk_session_id); + if (exists) { + stats.summariesSkipped++; + continue; + } + + insertSummary.run( + summary.sdk_session_id, + summary.project, + summary.request, + summary.investigated, + summary.learned, + summary.completed, + summary.next_steps, + summary.files_read, + summary.files_edited, + summary.notes, + summary.prompt_number, + summary.discovery_tokens || 0, + summary.created_at, + summary.created_at_epoch + ); + stats.summariesImported++; + } + console.log(` āœ… Imported: ${stats.summariesImported}, Skipped: ${stats.summariesSkipped}`); + + // 3. Import observations (depends on sessions) + console.log('šŸ”„ Importing observations...'); + for (const obs of exportData.observations) { + const exists = checkObservation.get( + obs.sdk_session_id, + obs.title, + obs.created_at_epoch + ); + if (exists) { + stats.observationsSkipped++; + continue; + } + + insertObservation.run( + obs.sdk_session_id, + obs.project, + obs.text, + obs.type, + obs.title, + obs.subtitle, + obs.facts, + obs.narrative, + obs.concepts, + obs.files_read, + obs.files_modified, + obs.prompt_number, + obs.discovery_tokens || 0, + obs.created_at, + obs.created_at_epoch + ); + stats.observationsImported++; + } + console.log(` āœ… Imported: ${stats.observationsImported}, Skipped: ${stats.observationsSkipped}`); + + // 4. Import prompts (depends on sessions) + console.log('šŸ”„ Importing prompts...'); + for (const prompt of exportData.prompts) { + const exists = checkPrompt.get( + prompt.claude_session_id, + prompt.prompt_number + ); + if (exists) { + stats.promptsSkipped++; + continue; + } + + insertPrompt.run( + prompt.claude_session_id, + prompt.prompt_number, + prompt.prompt_text, + prompt.created_at, + prompt.created_at_epoch + ); + stats.promptsImported++; + } + console.log(` āœ… Imported: ${stats.promptsImported}, Skipped: ${stats.promptsSkipped}`); + + })(); + + console.log('\nāœ… Import complete!'); + console.log('šŸ“Š Summary:'); + console.log(` Sessions: ${stats.sessionsImported} imported, ${stats.sessionsSkipped} skipped`); + console.log(` Summaries: ${stats.summariesImported} imported, ${stats.summariesSkipped} skipped`); + console.log(` Observations: ${stats.observationsImported} imported, ${stats.observationsSkipped} skipped`); + console.log(` Prompts: ${stats.promptsImported} imported, ${stats.promptsSkipped} skipped`); + + } finally { + db.close(); + } +} + +// CLI interface +const args = process.argv.slice(2); +if (args.length < 1) { + console.error('Usage: npx tsx scripts/import-memories.ts '); + console.error('Example: npx tsx scripts/import-memories.ts windows-memories.json'); + process.exit(1); +} + +const [inputFile] = args; +importMemories(inputFile);