Enhance observation parsing and querying functionality

- Filter out observation type from concepts array in parseObservations function to ensure types and concepts are treated as separate dimensions. Added logging for removed types.
- Update prompts documentation to clarify that the observation type must not be included in the concepts array.
- Modify search-server to provide clearer guidance on result limits, emphasizing starting with smaller limits to avoid exceeding token limits.
- Refactor SessionSearch methods to accept options for limit, offset, and orderBy parameters, improving flexibility in querying observations by concept and type.

src/sdk/parser.ts

@@ -68,13 +68,25 @@ export function parseObservations(text: string, correlationId?: string): ParsedO
       continue;
     }
 
+    // Filter out type from concepts array (types and concepts are separate dimensions)
+    const cleanedConcepts = concepts.filter(c => c !== type.trim());
+
+    if (cleanedConcepts.length !== concepts.length) {
+      logger.warn('PARSER', 'Removed observation type from concepts array', {
+        correlationId,
+        type: type.trim(),
+        originalConcepts: concepts,
+        cleanedConcepts
+      });
+    }
+
     observations.push({
       type: type.trim(),
       title,
       subtitle,
       facts,
       narrative,
-      concepts,
+      concepts: cleanedConcepts,
       files_read,
       files_modified
     });

src/sdk/prompts.ts

@@ -55,7 +55,7 @@ Output observations using this XML structure:
 <observation>
   <type>[ change | discovery | decision ]</type>
   <!--
-    **type**: One of:
+    **type**: MUST be EXACTLY one of these 3 options (no other values allowed):
       - change: modifications to code, config, or documentation
       - discovery: learning about existing system
       - decision: choosing an approach and why it was chosen
@@ -79,7 +79,7 @@ Output observations using this XML structure:
     <concept>[knowledge-type-category]</concept>
   </concepts>
   <!--
-    **concepts**: 2-5 knowledge-type categories:
+    **concepts**: 2-5 knowledge-type categories. MUST use ONLY these exact keywords:
       - how-it-works: understanding mechanisms
       - why-it-exists: purpose or rationale
       - what-changed: modifications made
@@ -87,6 +87,9 @@ Output observations using this XML structure:
       - gotcha: traps or edge cases
       - pattern: reusable approach
       - trade-off: pros/cons of a decision
+
+    IMPORTANT: Do NOT include the observation type (change/discovery/decision) as a concept.
+    Types and concepts are separate dimensions.
   -->
   <files_read>
     <file>[path/to/file]</file>

src/servers/search-server.ts

@@ -361,7 +361,7 @@ const tools = [
         start: z.union([z.string(), z.number()]).optional(),
         end: z.union([z.string(), z.number()]).optional()
       }).optional().describe('Filter by date range'),
-      limit: z.number().min(1).max(100).default(20).describe('Maximum number of results'),
+      limit: z.number().min(1).max(100).default(20).describe('Maximum results. IMPORTANT: Start with 3-5 to avoid exceeding MCP token limits, even in index mode.'),
       offset: z.number().min(0).default(0).describe('Number of results to skip'),
       orderBy: z.enum(['relevance', 'date_desc', 'date_asc']).default('date_desc').describe('Sort order')
     }),
@@ -418,7 +418,7 @@ const tools = [
         start: z.union([z.string(), z.number()]).optional(),
         end: z.union([z.string(), z.number()]).optional()
       }).optional().describe('Filter by date range'),
-      limit: z.number().min(1).max(100).default(20).describe('Maximum number of results'),
+      limit: z.number().min(1).max(100).default(20).describe('Maximum results. IMPORTANT: Start with 3-5 to avoid exceeding MCP token limits, even in index mode.'),
       offset: z.number().min(0).default(0).describe('Number of results to skip'),
       orderBy: z.enum(['relevance', 'date_desc', 'date_asc']).default('date_desc').describe('Sort order')
     }),
@@ -501,7 +501,7 @@ const tools = [
         start: z.union([z.string(), z.number()]).optional(),
         end: z.union([z.string(), z.number()]).optional()
       }).optional().describe('Filter by date range'),
-      limit: z.number().min(1).max(100).default(20).describe('Maximum number of results'),
+      limit: z.number().min(1).max(100).default(20).describe('Maximum results. IMPORTANT: Start with 3-5 to avoid exceeding MCP token limits, even in index mode.'),
       offset: z.number().min(0).default(0).describe('Number of results to skip'),
       orderBy: z.enum(['relevance', 'date_desc', 'date_asc']).default('date_desc').describe('Sort order')
     }),

src/services/sqlite/SessionSearch.ts

@@ -349,43 +349,53 @@ export class SessionSearch {
   /**
    * Find observations by concept tag
    */
-  findByConcept(concept: string, filters: SearchFilters = {}): ObservationSearchResult[] {
+  findByConcept(concept: string, options: SearchOptions = {}): ObservationSearchResult[] {
     const params: any[] = [];
+    const { limit = 50, offset = 0, orderBy = 'date_desc', ...filters } = options;
 
     // Add concept to filters
     const conceptFilters = { ...filters, concepts: concept };
     const filterClause = this.buildFilterClause(conceptFilters, params, 'o');
+    const orderClause = this.buildOrderClause(orderBy, false);
 
     const sql = `
       SELECT o.*
       FROM observations o
       WHERE ${filterClause}
-      ORDER BY o.created_at_epoch DESC
+      ${orderClause}
+      LIMIT ? OFFSET ?
     `;
 
+    params.push(limit, offset);
+
     return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
   }
 
   /**
    * Find observations and summaries by file path
    */
-  findByFile(filePath: string, filters: SearchFilters = {}): {
+  findByFile(filePath: string, options: SearchOptions = {}): {
     observations: ObservationSearchResult[];
     sessions: SessionSummarySearchResult[];
   } {
     const params: any[] = [];
+    const { limit = 50, offset = 0, orderBy = 'date_desc', ...filters } = options;
 
     // Add file to filters
     const fileFilters = { ...filters, files: filePath };
     const filterClause = this.buildFilterClause(fileFilters, params, 'o');
+    const orderClause = this.buildOrderClause(orderBy, false);
 
     const observationsSql = `
       SELECT o.*
       FROM observations o
       WHERE ${filterClause}
-      ORDER BY o.created_at_epoch DESC
+      ${orderClause}
+      LIMIT ? OFFSET ?
     `;
 
+    params.push(limit, offset);
+
     const observations = this.db.prepare(observationsSql).all(...params) as ObservationSearchResult[];
 
     // For session summaries, search files_read and files_edited
@@ -425,8 +435,11 @@ export class SessionSearch {
       FROM session_summaries s
       WHERE ${baseConditions.join(' AND ')}
       ORDER BY s.created_at_epoch DESC
+      LIMIT ? OFFSET ?
     `;
 
+    sessionParams.push(limit, offset);
+
     const sessions = this.db.prepare(sessionsSql).all(...sessionParams) as SessionSummarySearchResult[];
 
     return { observations, sessions };
@@ -437,21 +450,26 @@ export class SessionSearch {
    */
   findByType(
     type: ObservationRow['type'] | ObservationRow['type'][],
-    filters: SearchFilters = {}
+    options: SearchOptions = {}
   ): ObservationSearchResult[] {
     const params: any[] = [];
+    const { limit = 50, offset = 0, orderBy = 'date_desc', ...filters } = options;
 
     // Add type to filters
     const typeFilters = { ...filters, type };
     const filterClause = this.buildFilterClause(typeFilters, params, 'o');
+    const orderClause = this.buildOrderClause(orderBy, false);
 
     const sql = `
       SELECT o.*
       FROM observations o
       WHERE ${filterClause}
-      ORDER BY o.created_at_epoch DESC
+      ${orderClause}
+      LIMIT ? OFFSET ?
     `;
 
+    params.push(limit, offset);
+
     return this.db.prepare(sql).all(...params) as ObservationSearchResult[];
   }