From 5b338ba34e5f6ff6972b4739b5cf3fe599d4925b Mon Sep 17 00:00:00 2001 From: Alex Newman Date: Wed, 10 Dec 2025 20:38:21 -0500 Subject: [PATCH] Fix project filter and update export/import docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical bug fix: - Pass project filter to getSessionSummariesByIds() and getUserPromptsByIds() in SearchManager - Previously only observations were filtered by project, sessions and prompts leaked from other projects Documentation improvements: - Update "FTS5 search" to "hybrid search" (accurate terminology) - Add privacy warning about sensitive data in exports - Document --project parameter for filtered exports - Add "Export by Project" examples to advanced usage Verified with test export using --project=claude-mem filter. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- docs/public/usage/export-import.mdx | 19 ++++++++++++++++--- plugin/scripts/worker-service.cjs | 2 +- src/services/worker/SearchManager.ts | 4 ++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/public/usage/export-import.mdx b/docs/public/usage/export-import.mdx index d8c5b3c4..59eca515 100644 --- a/docs/public/usage/export-import.mdx +++ b/docs/public/usage/export-import.mdx @@ -18,7 +18,7 @@ Share your claude-mem knowledge with other users! These scripts allow you to exp ### Export Script -Searches the database using FTS5 full-text search and exports all matching: +Searches the database using **hybrid search** (combines ChromaDB vector embeddings with FTS5 full-text search) and exports all matching: - **Observations** - Individual learnings and discoveries - **Sessions** - Session metadata - **Summaries** - Session summaries @@ -26,6 +26,8 @@ Searches the database using FTS5 full-text search and exports all matching: Output is a portable JSON file that can be shared. +> **Privacy Note:** Export files contain all matching memory data in plain text. Review exports before sharing to ensure no sensitive information (API keys, passwords, private paths) is included. + ### Import Script Imports memories with **duplicate prevention**: @@ -56,8 +58,9 @@ npx tsx scripts/export-memories.ts "progressive disclosure" progressive-disclosu ``` **Parameters:** -1. `` - Search query (uses FTS5 full-text search) +1. `` - Search query (uses hybrid semantic + full-text search) 2. `` - Output JSON file path +3. `--project=name` - Optional: filter results to a specific project **Example Output:** ``` @@ -196,6 +199,16 @@ If you run the import again on the same file, duplicates are automatically skipp ## Advanced Usage +### Export by Project + +```bash +# Export only claude-mem project memories +npx tsx scripts/export-memories.ts "bugfix" bugfixes.json --project=claude-mem + +# Export all memories for a specific project +npx tsx scripts/export-memories.ts "" all-project.json --project=my-app +``` + ### Export by Type ```bash @@ -208,7 +221,7 @@ npx tsx scripts/export-memories.ts "type:bugfix" bugfixes.json ### Export by Date Range -FTS5 doesn't support date filtering directly, but you can filter the export after: +You can filter the export after exporting: ```bash # Export all memories, then filter manually with jq diff --git a/plugin/scripts/worker-service.cjs b/plugin/scripts/worker-service.cjs index c72f5ee2..278e31a0 100755 --- a/plugin/scripts/worker-service.cjs +++ b/plugin/scripts/worker-service.cjs @@ -920,7 +920,7 @@ MEMORY PROCESSING CONTINUED `,n=[];t&&(i+=" WHERE s.project = ?",n.push(t)),i+=" ORDER BY up.created_at_epoch DESC LIMIT ? OFFSET ?",n.push(r+1,e);let l=s.prepare(i).all(...n);return{items:l.slice(0,r),hasMore:l.length>r,offset:e,limit:r}}paginate(e,r,t,s,i){let n=this.dbManager.getSessionStore().db,o=`SELECT ${r} FROM ${e}`,l=[];i&&(o+=" WHERE project = ?",l.push(i)),o+=" ORDER BY created_at_epoch DESC LIMIT ? OFFSET ?",l.push(s+1,t);let u=n.prepare(o).all(...l);return{items:u.slice(0,s),hasMore:u.length>s,offset:t,limit:s}}};xt();var Rl=class{dbManager;defaultSettings={sidebarOpen:!0,selectedProject:null,theme:"system"};constructor(e){this.dbManager=e}getSettings(){let e=this.dbManager.getSessionStore().db;try{let t=e.prepare("SELECT key, value FROM viewer_settings").all(),s={...this.defaultSettings};for(let i of t){let n=i.key;n in s&&(s[n]=JSON.parse(i.value))}return s}catch(r){return V.debug("WORKER","Failed to load settings, using defaults",{},r),{...this.defaultSettings}}}updateSettings(e){let t=this.dbManager.getSessionStore().db.prepare(` INSERT OR REPLACE INTO viewer_settings (key, value) VALUES (?, ?) - `);for(let[s,i]of Object.entries(e))t.run(s,JSON.stringify(i));return this.getSettings()}};var Uw=require("path");var Pl=class{constructor(e,r,t,s,i){this.sessionSearch=e;this.sessionStore=r;this.chromaSync=t;this.formatter=s;this.timelineService=i}async queryChroma(e,r,t){return await this.chromaSync.queryChroma(e,r,t)}normalizeParams(e){let r={...e};return r.concepts&&typeof r.concepts=="string"&&(r.concepts=r.concepts.split(",").map(t=>t.trim()).filter(Boolean)),r.files&&typeof r.files=="string"&&(r.files=r.files.split(",").map(t=>t.trim()).filter(Boolean)),r.obs_type&&typeof r.obs_type=="string"&&(r.obs_type=r.obs_type.split(",").map(t=>t.trim()).filter(Boolean)),r.type&&typeof r.type=="string"&&r.type.includes(",")&&(r.type=r.type.split(",").map(t=>t.trim()).filter(Boolean)),(r.dateStart||r.dateEnd)&&(r.dateRange={start:r.dateStart,end:r.dateEnd},delete r.dateStart,delete r.dateEnd),r}async search(e){try{let r=this.normalizeParams(e),{query:t,format:s="index",type:i,obs_type:n,concepts:o,files:l,...c}=r,u=[],p=[],m=[],d=!i||i==="observations",v=!i||i==="sessions",h=!i||i==="prompts";if(t)if(this.chromaSync){let w=!1;try{ae(`[mcp-server] Using ChromaDB semantic search (type filter: ${i||"all"})`);let P;i==="observations"?P={doc_type:"observation"}:i==="sessions"?P={doc_type:"session_summary"}:i==="prompts"&&(P={doc_type:"user_prompt"});let T=await this.queryChroma(t,100,P);if(w=!0,ae(`[mcp-server] ChromaDB returned ${T.ids.length} semantic matches`),T.ids.length>0){let k=Date.now()-7776e6,j=T.metadatas.map((C,O)=>({id:T.ids[O],meta:C,isRecent:C&&C.created_at_epoch>k})).filter(C=>C.isRecent);ae(`[mcp-server] ${j.length} results within 90-day window`);let I=[],$=[],N=[];for(let C of j){let O=C.meta?.doc_type;O==="observation"&&d?I.push(C.id):O==="session_summary"&&v?$.push(C.id):O==="user_prompt"&&h&&N.push(C.id)}if(ae(`[mcp-server] Categorized: ${I.length} obs, ${$.length} sessions, ${N.length} prompts`),I.length>0){let C={...c,type:n,concepts:o,files:l};u=this.sessionStore.getObservationsByIds(I,C)}$.length>0&&(p=this.sessionStore.getSessionSummariesByIds($,{orderBy:"date_desc",limit:c.limit})),N.length>0&&(m=this.sessionStore.getUserPromptsByIds(N,{orderBy:"date_desc",limit:c.limit})),ae(`[mcp-server] Hydrated ${u.length} obs, ${p.length} sessions, ${m.length} prompts from SQLite`)}else ae("[mcp-server] ChromaDB found no matches (this is final - NOT falling back to FTS5)")}catch(P){ae("[mcp-server] ChromaDB failed - returning empty results (FTS5 fallback removed):",P.message),ae("[mcp-server] Install UVX/Python to enable vector search: https://docs.astral.sh/uv/getting-started/installation/"),u=[],p=[],m=[]}}else ae("[mcp-server] ChromaDB not initialized - returning empty results (FTS5 fallback removed)"),ae("[mcp-server] Install UVX/Python to enable vector search: https://docs.astral.sh/uv/getting-started/installation/"),u=[],p=[],m=[];else{ae("[mcp-server] Filter-only query (no query text), using direct SQLite filtering (enables date filters)");let w={...c,type:n,concepts:o,files:l};d&&(u=this.sessionSearch.searchObservations(void 0,w)),v&&(p=this.sessionSearch.searchSessions(void 0,c)),h&&(m=this.sessionSearch.searchUserPrompts(void 0,c))}let f=u.length+p.length+m.length;if(f===0)return s==="json"?{observations:[],sessions:[],prompts:[]}:{content:[{type:"text",text:`No results found matching "${t}"`}]};let y=[...u.map(w=>({type:"observation",data:w,epoch:w.created_at_epoch})),...p.map(w=>({type:"session",data:w,epoch:w.created_at_epoch})),...m.map(w=>({type:"prompt",data:w,epoch:w.created_at_epoch}))];c.orderBy==="date_desc"?y.sort((w,P)=>P.epoch-w.epoch):c.orderBy==="date_asc"&&y.sort((w,P)=>w.epoch-P.epoch);let g=y.slice(0,c.limit||20);if(s==="json")return{observations:u,sessions:p,prompts:m};let b;if(s==="index"){let w=`Found ${f} result(s) matching "${t}" (${u.length} obs, ${p.length} sessions, ${m.length} prompts): + `);for(let[s,i]of Object.entries(e))t.run(s,JSON.stringify(i));return this.getSettings()}};var Uw=require("path");var Pl=class{constructor(e,r,t,s,i){this.sessionSearch=e;this.sessionStore=r;this.chromaSync=t;this.formatter=s;this.timelineService=i}async queryChroma(e,r,t){return await this.chromaSync.queryChroma(e,r,t)}normalizeParams(e){let r={...e};return r.concepts&&typeof r.concepts=="string"&&(r.concepts=r.concepts.split(",").map(t=>t.trim()).filter(Boolean)),r.files&&typeof r.files=="string"&&(r.files=r.files.split(",").map(t=>t.trim()).filter(Boolean)),r.obs_type&&typeof r.obs_type=="string"&&(r.obs_type=r.obs_type.split(",").map(t=>t.trim()).filter(Boolean)),r.type&&typeof r.type=="string"&&r.type.includes(",")&&(r.type=r.type.split(",").map(t=>t.trim()).filter(Boolean)),(r.dateStart||r.dateEnd)&&(r.dateRange={start:r.dateStart,end:r.dateEnd},delete r.dateStart,delete r.dateEnd),r}async search(e){try{let r=this.normalizeParams(e),{query:t,format:s="index",type:i,obs_type:n,concepts:o,files:l,...c}=r,u=[],p=[],m=[],d=!i||i==="observations",v=!i||i==="sessions",h=!i||i==="prompts";if(t)if(this.chromaSync){let w=!1;try{ae(`[mcp-server] Using ChromaDB semantic search (type filter: ${i||"all"})`);let P;i==="observations"?P={doc_type:"observation"}:i==="sessions"?P={doc_type:"session_summary"}:i==="prompts"&&(P={doc_type:"user_prompt"});let T=await this.queryChroma(t,100,P);if(w=!0,ae(`[mcp-server] ChromaDB returned ${T.ids.length} semantic matches`),T.ids.length>0){let k=Date.now()-7776e6,j=T.metadatas.map((C,O)=>({id:T.ids[O],meta:C,isRecent:C&&C.created_at_epoch>k})).filter(C=>C.isRecent);ae(`[mcp-server] ${j.length} results within 90-day window`);let I=[],$=[],N=[];for(let C of j){let O=C.meta?.doc_type;O==="observation"&&d?I.push(C.id):O==="session_summary"&&v?$.push(C.id):O==="user_prompt"&&h&&N.push(C.id)}if(ae(`[mcp-server] Categorized: ${I.length} obs, ${$.length} sessions, ${N.length} prompts`),I.length>0){let C={...c,type:n,concepts:o,files:l};u=this.sessionStore.getObservationsByIds(I,C)}$.length>0&&(p=this.sessionStore.getSessionSummariesByIds($,{orderBy:"date_desc",limit:c.limit,project:c.project})),N.length>0&&(m=this.sessionStore.getUserPromptsByIds(N,{orderBy:"date_desc",limit:c.limit,project:c.project})),ae(`[mcp-server] Hydrated ${u.length} obs, ${p.length} sessions, ${m.length} prompts from SQLite`)}else ae("[mcp-server] ChromaDB found no matches (this is final - NOT falling back to FTS5)")}catch(P){ae("[mcp-server] ChromaDB failed - returning empty results (FTS5 fallback removed):",P.message),ae("[mcp-server] Install UVX/Python to enable vector search: https://docs.astral.sh/uv/getting-started/installation/"),u=[],p=[],m=[]}}else ae("[mcp-server] ChromaDB not initialized - returning empty results (FTS5 fallback removed)"),ae("[mcp-server] Install UVX/Python to enable vector search: https://docs.astral.sh/uv/getting-started/installation/"),u=[],p=[],m=[];else{ae("[mcp-server] Filter-only query (no query text), using direct SQLite filtering (enables date filters)");let w={...c,type:n,concepts:o,files:l};d&&(u=this.sessionSearch.searchObservations(void 0,w)),v&&(p=this.sessionSearch.searchSessions(void 0,c)),h&&(m=this.sessionSearch.searchUserPrompts(void 0,c))}let f=u.length+p.length+m.length;if(f===0)return s==="json"?{observations:[],sessions:[],prompts:[]}:{content:[{type:"text",text:`No results found matching "${t}"`}]};let y=[...u.map(w=>({type:"observation",data:w,epoch:w.created_at_epoch})),...p.map(w=>({type:"session",data:w,epoch:w.created_at_epoch})),...m.map(w=>({type:"prompt",data:w,epoch:w.created_at_epoch}))];c.orderBy==="date_desc"?y.sort((w,P)=>P.epoch-w.epoch):c.orderBy==="date_asc"&&y.sort((w,P)=>w.epoch-P.epoch);let g=y.slice(0,c.limit||20);if(s==="json")return{observations:u,sessions:p,prompts:m};let b;if(s==="index"){let w=`Found ${f} result(s) matching "${t}" (${u.length} obs, ${p.length} sessions, ${m.length} prompts): `,P=g.map((T,k)=>T.type==="observation"?this.formatter.formatObservationIndex(T.data,k):T.type==="session"?this.formatter.formatSessionIndex(T.data,k):this.formatter.formatUserPromptIndex(T.data,k));b=w+P.join(` diff --git a/src/services/worker/SearchManager.ts b/src/services/worker/SearchManager.ts index df5b28ab..af150a18 100644 --- a/src/services/worker/SearchManager.ts +++ b/src/services/worker/SearchManager.ts @@ -166,10 +166,10 @@ export class SearchManager { observations = this.sessionStore.getObservationsByIds(obsIds, obsOptions); } if (sessionIds.length > 0) { - sessions = this.sessionStore.getSessionSummariesByIds(sessionIds, { orderBy: 'date_desc', limit: options.limit }); + sessions = this.sessionStore.getSessionSummariesByIds(sessionIds, { orderBy: 'date_desc', limit: options.limit, project: options.project }); } if (promptIds.length > 0) { - prompts = this.sessionStore.getUserPromptsByIds(promptIds, { orderBy: 'date_desc', limit: options.limit }); + prompts = this.sessionStore.getUserPromptsByIds(promptIds, { orderBy: 'date_desc', limit: options.limit, project: options.project }); } happy_path_error__with_fallback(`[mcp-server] Hydrated ${observations.length} obs, ${sessions.length} sessions, ${prompts.length} prompts from SQLite`);