Release v3.7.0

Published from npm package build
Source: https://github.com/thedotmack/claude-mem-source
This commit is contained in:
Alex Newman
2025-09-17 20:19:19 -04:00
parent 35b7aab174
commit b0032c1745
18 changed files with 2855 additions and 350 deletions
+15 -20
View File
@@ -6,6 +6,7 @@ import os from 'os';
import chalk from 'chalk';
import { TranscriptCompressor } from '../core/compression/TranscriptCompressor.js';
import { TitleGenerator, TitleGenerationRequest } from '../core/titles/TitleGenerator.js';
import { getStorageProvider, needsMigration } from '../shared/storage.js';
interface ConversationMetadata {
sessionId: string;
@@ -100,27 +101,21 @@ function extractFirstUserMessage(filePath: string): string {
}
async function loadImportedSessions(): Promise<Set<string>> {
const importedIds = new Set<string>();
const indexPath = path.join(os.homedir(), '.claude-mem', 'claude-mem-index.jsonl');
if (!fs.existsSync(indexPath)) return importedIds;
const content = fs.readFileSync(indexPath, 'utf-8');
const lines = content.trim().split('\n').filter(Boolean);
for (const line of lines) {
try {
const entry = JSON.parse(line);
// Check both session_id (from index) and sessionId (legacy)
if (entry.session_id) {
importedIds.add(entry.session_id);
} else if (entry.sessionId) {
importedIds.add(entry.sessionId);
}
} catch {}
try {
// Check if migration is needed and warn the user
if (await needsMigration()) {
console.warn('⚠️ JSONL to SQLite migration recommended. Run: claude-mem migrate-index');
}
const storage = await getStorageProvider();
// Use storage provider to get all session IDs efficiently
return await storage.getAllSessionIds();
} catch (error) {
console.warn('Failed to load imported sessions, proceeding with empty set:', error);
return new Set<string>();
}
return importedIds;
}
async function scanConversations(): Promise<{ conversations: ConversationItem[]; skippedCount: number }> {
+201 -73
View File
@@ -9,6 +9,8 @@ import {
formatTimeAgo,
outputSessionStartContent
} from '../prompts/templates/context/ContextTemplates.js';
import { getStorageProvider, needsMigration } from '../shared/storage.js';
import { MemoryRow, OverviewRow, SessionRow } from '../services/sqlite/types.js';
interface TrashStatus {
folderCount: number;
@@ -66,82 +68,84 @@ function getTrashStatus(): TrashStatus {
}
export async function loadContext(options: OptionValues = {}): Promise<void> {
const pathDiscovery = PathDiscovery.getInstance();
const indexPath = pathDiscovery.getIndexPath();
try {
// Check if index file exists
if (!fs.existsSync(indexPath)) {
// Check if migration is needed and warn the user
if (await needsMigration()) {
console.warn('⚠️ JSONL to SQLite migration recommended. Run: claude-mem migrate-index');
}
const storage = await getStorageProvider();
// If using JSONL fallback, use original implementation
if (storage.backend === 'jsonl') {
return await loadContextFromJSONL(options);
}
// SQLite implementation - fetch data using storage provider
let recentMemories: MemoryRow[] = [];
let recentOverviews: OverviewRow[] = [];
let recentSessions: SessionRow[] = [];
// Auto-detect current project for session-start format if no project specified
let projectToUse = options.project;
if (!projectToUse && options.format === 'session-start') {
projectToUse = PathDiscovery.getCurrentProjectName();
}
if (projectToUse) {
recentMemories = await storage.getRecentMemoriesForProject(projectToUse, 10);
recentOverviews = await storage.getRecentOverviewsForProject(projectToUse, options.format === 'session-start' ? 5 : 3);
recentSessions = await storage.getRecentSessionsForProject(projectToUse, 5);
} else {
recentMemories = await storage.getRecentMemories(10);
recentOverviews = await storage.getRecentOverviews(options.format === 'session-start' ? 5 : 3);
recentSessions = await storage.getRecentSessions(5);
}
// Convert SQLite rows to JSONL format for compatibility with existing output functions
const memoriesAsJSON = recentMemories.map(row => ({
type: 'memory',
text: row.text,
document_id: row.document_id,
keywords: row.keywords,
session_id: row.session_id,
project: row.project,
timestamp: row.created_at,
archive: row.archive_basename
}));
const overviewsAsJSON = recentOverviews.map(row => ({
type: 'overview',
content: row.content,
session_id: row.session_id,
project: row.project,
timestamp: row.created_at
}));
const sessionsAsJSON = recentSessions.map(row => ({
type: 'session',
session_id: row.session_id,
project: row.project,
timestamp: row.created_at
}));
// If no data found, show appropriate messages
if (memoriesAsJSON.length === 0 && overviewsAsJSON.length === 0 && sessionsAsJSON.length === 0) {
if (options.format === 'session-start') {
console.log(createContextualError('NO_MEMORIES', options.project || 'this project'));
console.log(createContextualError('NO_MEMORIES', projectToUse || 'this project'));
}
return;
}
const content = fs.readFileSync(indexPath, 'utf-8');
const lines = content.trim().split('\n').filter(line => line.trim());
if (lines.length === 0) {
if (options.format === 'session-start') {
console.log(createContextualError('NO_MEMORIES', options.project || 'this project'));
}
return;
}
// Parse JSONL format - each line is a JSON object
const jsonObjects: any[] = [];
for (const line of lines) {
try {
// Skip lines that don't look like JSON (could be legacy format)
if (!line.trim().startsWith('{')) {
continue;
}
const obj = JSON.parse(line);
jsonObjects.push(obj);
} catch (e) {
// Skip malformed JSON lines
continue;
}
}
if (jsonObjects.length === 0) {
if (options.format === 'session-start') {
console.log(createContextualError('NO_MEMORIES', options.project || 'this project'));
}
return;
}
// Separate memories, overviews, and other types
const memories = jsonObjects.filter(obj => obj.type === 'memory');
const overviews = jsonObjects.filter(obj => obj.type === 'overview');
const sessions = jsonObjects.filter(obj => obj.type === 'session');
// Filter each type by project if specified
// Handle both hyphen and underscore formats since index has mixed entries
let filteredMemories = memories;
let filteredOverviews = overviews;
let filteredSessions = sessions;
if (options.project) {
const matchesProject = buildProjectMatcher(options.project);
filteredMemories = memories.filter(obj => matchesProject(obj.project));
filteredOverviews = overviews.filter(obj => matchesProject(obj.project));
filteredSessions = sessions.filter(obj => matchesProject(obj.project));
}
// Use the same output logic as the original implementation
if (options.format === 'session-start') {
// Get last 10 memories and last 5 overviews for session-start
const recentMemories = filteredMemories.slice(-10);
const recentOverviews = filteredOverviews.slice(-5);
const recentSessions = filteredSessions.slice(-5);
// Combine them for the display
const recentObjects = [...recentSessions, ...recentMemories, ...recentOverviews];
const recentObjects = [...sessionsAsJSON, ...memoriesAsJSON, ...overviewsAsJSON];
// Find most recent timestamp for last session info
let lastSessionTime = 'recently';
const timestamps = recentObjects
.map(obj => {
// Get timestamp from JSON object
return obj.timestamp ? new Date(obj.timestamp) : null;
})
.filter(date => date !== null)
@@ -153,33 +157,29 @@ export async function loadContext(options: OptionValues = {}): Promise<void> {
// Use dual-stream output for session start formatting
outputSessionStartContent({
projectName: options.project || 'your project',
memoryCount: recentMemories.length,
projectName: projectToUse || 'your project',
memoryCount: memoriesAsJSON.length,
lastSessionTime,
recentObjects
});
} else if (options.format === 'json') {
// For JSON format, combine last 10 of each type
const recentMemories = filteredMemories.slice(-10);
const recentOverviews = filteredOverviews.slice(-3);
const recentObjects = [...recentMemories, ...recentOverviews];
const recentObjects = [...memoriesAsJSON, ...overviewsAsJSON];
console.log(JSON.stringify(recentObjects));
} else {
// Default format - show last 10 memories and last 3 overviews
const recentMemories = filteredMemories.slice(-10);
const recentOverviews = filteredOverviews.slice(-3);
const totalCount = recentMemories.length + recentOverviews.length;
const totalCount = memoriesAsJSON.length + overviewsAsJSON.length;
console.log(createCompletionMessage('Context loading', totalCount, 'recent entries found'));
// Show memories first
recentMemories.forEach((obj) => {
memoriesAsJSON.forEach((obj) => {
console.log(`${obj.text} | ${obj.document_id} | ${obj.keywords}`);
});
// Then show overviews
recentOverviews.forEach((obj) => {
overviewsAsJSON.forEach((obj) => {
console.log(`**Overview:** ${obj.content}`);
});
}
@@ -203,3 +203,131 @@ export async function loadContext(options: OptionValues = {}): Promise<void> {
}
}
}
/**
* Original JSONL-based implementation for fallback compatibility
*/
async function loadContextFromJSONL(options: OptionValues = {}): Promise<void> {
const pathDiscovery = PathDiscovery.getInstance();
const indexPath = pathDiscovery.getIndexPath();
// Auto-detect current project for session-start format if no project specified
let projectToUse = options.project;
if (!projectToUse && options.format === 'session-start') {
projectToUse = PathDiscovery.getCurrentProjectName();
}
// Check if index file exists
if (!fs.existsSync(indexPath)) {
if (options.format === 'session-start') {
console.log(createContextualError('NO_MEMORIES', projectToUse || 'this project'));
}
return;
}
const content = fs.readFileSync(indexPath, 'utf-8');
const lines = content.trim().split('\n').filter(line => line.trim());
if (lines.length === 0) {
if (options.format === 'session-start') {
console.log(createContextualError('NO_MEMORIES', projectToUse || 'this project'));
}
return;
}
// Parse JSONL format - each line is a JSON object
const jsonObjects: any[] = [];
for (const line of lines) {
try {
// Skip lines that don't look like JSON (could be legacy format)
if (!line.trim().startsWith('{')) {
continue;
}
const obj = JSON.parse(line);
jsonObjects.push(obj);
} catch (e) {
// Skip malformed JSON lines
continue;
}
}
if (jsonObjects.length === 0) {
if (options.format === 'session-start') {
console.log(createContextualError('NO_MEMORIES', projectToUse || 'this project'));
}
return;
}
// Separate memories, overviews, and other types
const memories = jsonObjects.filter(obj => obj.type === 'memory');
const overviews = jsonObjects.filter(obj => obj.type === 'overview');
const sessions = jsonObjects.filter(obj => obj.type === 'session');
// Filter each type by project if specified
// Handle both hyphen and underscore formats since index has mixed entries
let filteredMemories = memories;
let filteredOverviews = overviews;
let filteredSessions = sessions;
if (projectToUse) {
const matchesProject = buildProjectMatcher(projectToUse);
filteredMemories = memories.filter(obj => matchesProject(obj.project));
filteredOverviews = overviews.filter(obj => matchesProject(obj.project));
filteredSessions = sessions.filter(obj => matchesProject(obj.project));
}
if (options.format === 'session-start') {
// Get last 10 memories and last 5 overviews for session-start
const recentMemories = filteredMemories.slice(-10);
const recentOverviews = filteredOverviews.slice(-5);
const recentSessions = filteredSessions.slice(-5);
// Combine them for the display
const recentObjects = [...recentSessions, ...recentMemories, ...recentOverviews];
// Find most recent timestamp for last session info
let lastSessionTime = 'recently';
const timestamps = recentObjects
.map(obj => {
// Get timestamp from JSON object
return obj.timestamp ? new Date(obj.timestamp) : null;
})
.filter(date => date !== null)
.sort((a, b) => b.getTime() - a.getTime());
if (timestamps.length > 0) {
lastSessionTime = formatTimeAgo(timestamps[0]);
}
// Use dual-stream output for session start formatting
outputSessionStartContent({
projectName: projectToUse || 'your project',
memoryCount: recentMemories.length,
lastSessionTime,
recentObjects
});
} else if (options.format === 'json') {
// For JSON format, combine last 10 of each type
const recentMemories = filteredMemories.slice(-10);
const recentOverviews = filteredOverviews.slice(-3);
const recentObjects = [...recentMemories, ...recentOverviews];
console.log(JSON.stringify(recentObjects));
} else {
// Default format - show last 10 memories and last 3 overviews
const recentMemories = filteredMemories.slice(-10);
const recentOverviews = filteredOverviews.slice(-3);
const totalCount = recentMemories.length + recentOverviews.length;
console.log(createCompletionMessage('Context loading', totalCount, 'recent entries found'));
// Show memories first
recentMemories.forEach((obj) => {
console.log(`${obj.text} | ${obj.document_id} | ${obj.keywords}`);
});
// Then show overviews
recentOverviews.forEach((obj) => {
console.log(`**Overview:** ${obj.content}`);
});
}
}
+300
View File
@@ -0,0 +1,300 @@
import { OptionValues } from 'commander';
import fs from 'fs';
import path from 'path';
import { PathDiscovery } from '../services/path-discovery.js';
import {
createStores,
SessionInput,
MemoryInput,
OverviewInput,
DiagnosticInput,
normalizeTimestamp
} from '../services/sqlite/index.js';
interface MigrationStats {
totalLines: number;
skippedLines: number;
invalidJson: number;
sessionsCreated: number;
memoriesCreated: number;
overviewsCreated: number;
diagnosticsCreated: number;
orphanedOverviews: number;
orphanedMemories: number;
}
/**
* Migrate claude-mem index from JSONL to SQLite
*/
export async function migrateIndex(options: OptionValues = {}): Promise<void> {
const pathDiscovery = PathDiscovery.getInstance();
const indexPath = pathDiscovery.getIndexPath();
const backupPath = `${indexPath}.backup-${Date.now()}`;
console.log('🔄 Starting JSONL to SQLite migration...');
console.log(`📁 Index file: ${indexPath}`);
// Check if JSONL file exists
if (!fs.existsSync(indexPath)) {
console.log('️ No JSONL index file found - nothing to migrate');
return;
}
try {
// Initialize SQLite database and stores
console.log('🏗️ Initializing SQLite database...');
const stores = await createStores();
// Check if we already have data in SQLite
const existingSessions = stores.sessions.count();
if (existingSessions > 0 && !options.force) {
console.log(`⚠️ SQLite database already contains ${existingSessions} sessions.`);
console.log(' Use --force to migrate anyway (will skip duplicates)');
return;
}
// Create backup of JSONL file
if (!options.keepJsonl) {
console.log(`💾 Creating backup: ${path.basename(backupPath)}`);
fs.copyFileSync(indexPath, backupPath);
}
// Read and parse JSONL file
console.log('📖 Reading JSONL index file...');
const content = fs.readFileSync(indexPath, 'utf-8');
const lines = content.trim().split('\n').filter(line => line.trim());
const stats: MigrationStats = {
totalLines: lines.length,
skippedLines: 0,
invalidJson: 0,
sessionsCreated: 0,
memoriesCreated: 0,
overviewsCreated: 0,
diagnosticsCreated: 0,
orphanedOverviews: 0,
orphanedMemories: 0
};
console.log(`📝 Processing ${stats.totalLines} lines...`);
// Parse all lines first
const records: any[] = [];
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
try {
// Skip lines that don't look like JSON
if (!line.trim().startsWith('{')) {
stats.skippedLines++;
continue;
}
const record = JSON.parse(line);
if (record && typeof record === 'object') {
records.push({ ...record, _lineNumber: i + 1 });
} else {
stats.skippedLines++;
}
} catch (error) {
stats.invalidJson++;
console.warn(`⚠️ Invalid JSON at line ${i + 1}: ${line.substring(0, 50)}...`);
}
}
console.log(`✅ Parsed ${records.length} valid records`);
// Group records by type
const sessions = records.filter(r => r.type === 'session');
const memories = records.filter(r => r.type === 'memory');
const overviews = records.filter(r => r.type === 'overview');
const diagnostics = records.filter(r => r.type === 'diagnostic');
const unknown = records.filter(r => !['session', 'memory', 'overview', 'diagnostic'].includes(r.type));
if (unknown.length > 0) {
console.log(`⚠️ Found ${unknown.length} records with unknown types - will skip`);
stats.skippedLines += unknown.length;
}
// Create session tracking
const sessionIds = new Set(sessions.map(s => s.session_id));
const orphanedSessionIds = new Set();
// Migrate sessions first
console.log('💾 Migrating sessions...');
for (const sessionData of sessions) {
try {
const { isoString } = normalizeTimestamp(sessionData.timestamp);
const sessionInput: SessionInput = {
session_id: sessionData.session_id,
project: sessionData.project || 'unknown',
created_at: isoString,
source: 'legacy-jsonl'
};
// Skip if session already exists (when using --force)
if (!stores.sessions.has(sessionInput.session_id)) {
stores.sessions.create(sessionInput);
stats.sessionsCreated++;
}
} catch (error) {
console.warn(`⚠️ Failed to migrate session ${sessionData.session_id}: ${error}`);
}
}
// Migrate memories
console.log('🧠 Migrating memories...');
for (const memoryData of memories) {
try {
const { isoString } = normalizeTimestamp(memoryData.timestamp);
// Check if session exists, create orphaned session if needed
if (!sessionIds.has(memoryData.session_id)) {
if (!orphanedSessionIds.has(memoryData.session_id)) {
orphanedSessionIds.add(memoryData.session_id);
const orphanedSession: SessionInput = {
session_id: memoryData.session_id,
project: memoryData.project || 'unknown',
created_at: isoString,
source: 'legacy-jsonl'
};
if (!stores.sessions.has(orphanedSession.session_id)) {
stores.sessions.create(orphanedSession);
stats.sessionsCreated++;
stats.orphanedMemories++;
}
}
}
const memoryInput: MemoryInput = {
session_id: memoryData.session_id,
text: memoryData.text || '',
document_id: memoryData.document_id,
keywords: memoryData.keywords,
created_at: isoString,
project: memoryData.project || 'unknown',
archive_basename: memoryData.archive,
origin: 'transcript'
};
// Skip duplicate document_ids
if (!memoryInput.document_id || !stores.memories.hasDocumentId(memoryInput.document_id)) {
stores.memories.create(memoryInput);
stats.memoriesCreated++;
}
} catch (error) {
console.warn(`⚠️ Failed to migrate memory ${memoryData.document_id}: ${error}`);
}
}
// Migrate overviews
console.log('📋 Migrating overviews...');
for (const overviewData of overviews) {
try {
const { isoString } = normalizeTimestamp(overviewData.timestamp);
// Check if session exists, create orphaned session if needed
if (!sessionIds.has(overviewData.session_id)) {
if (!orphanedSessionIds.has(overviewData.session_id)) {
orphanedSessionIds.add(overviewData.session_id);
const orphanedSession: SessionInput = {
session_id: overviewData.session_id,
project: overviewData.project || 'unknown',
created_at: isoString,
source: 'legacy-jsonl'
};
if (!stores.sessions.has(orphanedSession.session_id)) {
stores.sessions.create(orphanedSession);
stats.sessionsCreated++;
stats.orphanedOverviews++;
}
}
}
const overviewInput: OverviewInput = {
session_id: overviewData.session_id,
content: overviewData.content || '',
created_at: isoString,
project: overviewData.project || 'unknown',
origin: 'claude'
};
stores.overviews.upsert(overviewInput);
stats.overviewsCreated++;
} catch (error) {
console.warn(`⚠️ Failed to migrate overview ${overviewData.session_id}: ${error}`);
}
}
// Migrate diagnostics
console.log('🩺 Migrating diagnostics...');
for (const diagnosticData of diagnostics) {
try {
const { isoString } = normalizeTimestamp(diagnosticData.timestamp);
const diagnosticInput: DiagnosticInput = {
session_id: diagnosticData.session_id,
message: diagnosticData.message || '',
severity: 'warn',
created_at: isoString,
project: diagnosticData.project || 'unknown',
origin: 'compressor'
};
stores.diagnostics.create(diagnosticInput);
stats.diagnosticsCreated++;
} catch (error) {
console.warn(`⚠️ Failed to migrate diagnostic: ${error}`);
}
}
// Print migration summary
console.log('\n✅ Migration completed successfully!');
console.log('\n📊 Migration Summary:');
console.log(` Total lines processed: ${stats.totalLines}`);
console.log(` Skipped lines: ${stats.skippedLines}`);
console.log(` Invalid JSON lines: ${stats.invalidJson}`);
console.log(` Sessions created: ${stats.sessionsCreated}`);
console.log(` Memories created: ${stats.memoriesCreated}`);
console.log(` Overviews created: ${stats.overviewsCreated}`);
console.log(` Diagnostics created: ${stats.diagnosticsCreated}`);
if (stats.orphanedOverviews > 0 || stats.orphanedMemories > 0) {
console.log(` Orphaned records (sessions synthesized): ${stats.orphanedOverviews + stats.orphanedMemories}`);
}
// Archive or keep JSONL file
if (options.keepJsonl) {
console.log(`\n💾 Original JSONL file preserved: ${indexPath}`);
console.log(` SQLite database is now the primary index`);
} else {
const archiveDir = path.join(pathDiscovery.getDataDirectory(), 'archive', 'legacy');
fs.mkdirSync(archiveDir, { recursive: true });
const archivedPath = path.join(archiveDir, `claude-mem-index-${Date.now()}.jsonl`);
fs.renameSync(indexPath, archivedPath);
console.log(`\n📦 Original JSONL file archived: ${path.basename(archivedPath)}`);
console.log(` Backup available at: ${path.basename(backupPath)}`);
}
console.log('\n🎉 Migration complete! You can now use claude-mem with SQLite backend.');
console.log(' Run `claude-mem load-context` to verify the migration worked.');
} catch (error) {
console.error('\n❌ Migration failed:', error);
// Restore backup if we created one
if (fs.existsSync(backupPath) && !fs.existsSync(indexPath)) {
console.log('🔄 Restoring backup...');
fs.renameSync(backupPath, indexPath);
}
process.exit(1);
}
}
+43 -28
View File
@@ -1,6 +1,7 @@
import { OptionValues } from 'commander';
import { appendFileSync } from 'fs';
import { PathDiscovery } from '../services/path-discovery.js';
import { getStorageProvider, needsMigration } from '../shared/storage.js';
/**
* Generates a descriptive session ID from the message content
@@ -25,7 +26,7 @@ function generateSessionId(message: string): string {
}
/**
* Save command - stores a message to both Chroma collection and JSONL index
* Save command - stores a message using the configured storage provider
*/
export async function save(message: string, options: OptionValues = {}): Promise<void> {
// Debug: Log what we receive
@@ -38,38 +39,52 @@ export async function save(message: string, options: OptionValues = {}): Promise
process.exit(1);
}
const pathDiscovery = PathDiscovery.getInstance();
const timestamp = new Date().toISOString();
const projectName = PathDiscovery.getCurrentProjectName();
const sessionId = generateSessionId(message);
const documentId = `${projectName}_${sessionId}_overview`;
// 1. Save to Chroma collection (skip for now - MCP tools only available in Claude Code context)
// TODO: Add Chroma integration when called from Claude Code with MCP server running
try {
// Check if migration is needed
if (await needsMigration()) {
console.warn('⚠️ JSONL to SQLite migration recommended. Run: claude-mem migrate-index');
}
// 2. Append to JSONL index file
const indexPath = pathDiscovery.getIndexPath();
const indexEntry = {
type: "overview",
content: message,
session_id: sessionId,
project: projectName,
timestamp: timestamp
};
// Get storage provider (SQLite preferred, JSONL fallback)
const storage = await getStorageProvider();
// Ensure the directory exists
pathDiscovery.ensureDirectory(pathDiscovery.getDataDirectory());
// Append to JSONL file
appendFileSync(indexPath, JSON.stringify(indexEntry) + '\n', 'utf8');
// Ensure session exists or create it
if (!await storage.hasSession(sessionId)) {
await storage.createSession({
session_id: sessionId,
project: projectName,
created_at: timestamp,
source: 'save'
});
}
// 3. Return JSON response for hook compatibility
console.log(JSON.stringify({
success: true,
document_id: documentId,
session_id: sessionId,
project: projectName,
timestamp: timestamp,
suppressOutput: true
}));
}
// Upsert the overview
await storage.upsertOverview({
session_id: sessionId,
content: message,
created_at: timestamp,
project: projectName,
origin: 'manual'
});
// Return JSON response for hook compatibility
console.log(JSON.stringify({
success: true,
document_id: documentId,
session_id: sessionId,
project: projectName,
timestamp: timestamp,
backend: storage.backend,
suppressOutput: true
}));
} catch (error) {
console.error('Error saving message:', error);
process.exit(1);
}
}