Alex Newman
656a7f7c0f
- Updated SQL queries in cleanup-duplicates.ts and context-generator.ts to use memory_session_id. - Modified interfaces in context-generator.ts to reflect the new session ID naming. - Implemented a repair migration in SessionStore.ts to rename columns in existing tables. - Adjusted FormattingService.ts and SDKAgent.ts to utilize memory_session_id for session handling. - Ensured SearchManager.ts retrieves summaries and observations using the updated memory_session_id.
99 lines
3.1 KiB
JavaScript
99 lines
3.1 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Cleanup duplicate observations and summaries from the database
|
|
* Keeps the earliest entry (MIN(id)) for each duplicate group
|
|
*/
|
|
|
|
import { SessionStore } from '../services/sqlite/SessionStore.js';
|
|
|
|
function main() {
|
|
console.log('Starting duplicate cleanup...\n');
|
|
|
|
const db = new SessionStore();
|
|
|
|
// Find and delete duplicate observations
|
|
console.log('Finding duplicate observations...');
|
|
|
|
const duplicateObsQuery = db['db'].prepare(`
|
|
SELECT memory_session_id, title, subtitle, type, COUNT(*) as count, GROUP_CONCAT(id) as ids
|
|
FROM observations
|
|
GROUP BY memory_session_id, title, subtitle, type
|
|
HAVING count > 1
|
|
`);
|
|
|
|
const duplicateObs = duplicateObsQuery.all() as Array<{
|
|
memory_session_id: string;
|
|
title: string;
|
|
subtitle: string;
|
|
type: string;
|
|
count: number;
|
|
ids: string;
|
|
}>;
|
|
|
|
console.log(`Found ${duplicateObs.length} duplicate observation groups\n`);
|
|
|
|
let deletedObs = 0;
|
|
for (const dup of duplicateObs) {
|
|
const ids = dup.ids.split(',').map(id => parseInt(id, 10));
|
|
const keepId = Math.min(...ids);
|
|
const deleteIds = ids.filter(id => id !== keepId);
|
|
|
|
console.log(`Observation "${dup.title.substring(0, 60)}..."`);
|
|
console.log(` Found ${dup.count} copies, keeping ID ${keepId}, deleting ${deleteIds.length} duplicates`);
|
|
|
|
const deleteStmt = db['db'].prepare(`DELETE FROM observations WHERE id IN (${deleteIds.join(',')})`);
|
|
deleteStmt.run();
|
|
deletedObs += deleteIds.length;
|
|
}
|
|
|
|
// Find and delete duplicate summaries
|
|
console.log('\n\nFinding duplicate summaries...');
|
|
|
|
const duplicateSumQuery = db['db'].prepare(`
|
|
SELECT memory_session_id, request, completed, learned, COUNT(*) as count, GROUP_CONCAT(id) as ids
|
|
FROM session_summaries
|
|
GROUP BY memory_session_id, request, completed, learned
|
|
HAVING count > 1
|
|
`);
|
|
|
|
const duplicateSum = duplicateSumQuery.all() as Array<{
|
|
memory_session_id: string;
|
|
request: string;
|
|
completed: string;
|
|
learned: string;
|
|
count: number;
|
|
ids: string;
|
|
}>;
|
|
|
|
console.log(`Found ${duplicateSum.length} duplicate summary groups\n`);
|
|
|
|
let deletedSum = 0;
|
|
for (const dup of duplicateSum) {
|
|
const ids = dup.ids.split(',').map(id => parseInt(id, 10));
|
|
const keepId = Math.min(...ids);
|
|
const deleteIds = ids.filter(id => id !== keepId);
|
|
|
|
console.log(`Summary "${dup.request.substring(0, 60)}..."`);
|
|
console.log(` Found ${dup.count} copies, keeping ID ${keepId}, deleting ${deleteIds.length} duplicates`);
|
|
|
|
const deleteStmt = db['db'].prepare(`DELETE FROM session_summaries WHERE id IN (${deleteIds.join(',')})`);
|
|
deleteStmt.run();
|
|
deletedSum += deleteIds.length;
|
|
}
|
|
|
|
db.close();
|
|
|
|
console.log('\n' + '='.repeat(60));
|
|
console.log('Cleanup Complete!');
|
|
console.log('='.repeat(60));
|
|
console.log(`🗑️ Deleted: ${deletedObs} duplicate observations`);
|
|
console.log(`🗑️ Deleted: ${deletedSum} duplicate summaries`);
|
|
console.log(`🗑️ Total: ${deletedObs + deletedSum} duplicates removed`);
|
|
console.log('='.repeat(60));
|
|
}
|
|
|
|
// Run if executed directly
|
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
main();
|
|
}
|