feat: Add dual-tag system for meta-observation control (#153)
* feat: Add dual-tag system for meta-observation control Implements <private> and <claude-mem-context> tag stripping at hook layer to give users fine-grained control over what gets persisted in observations and enable future real-time context injection without recursive storage. **Features:** - stripMemoryTags() function in save-hook.ts - Strips both <private> and <claude-mem-context> tags before sending to worker - Always active (no configuration needed) - Comprehensive test suite (19 tests, all passing) - User documentation for <private> tag - Technical architecture documentation **Architecture:** - Edge processing pattern (filter at hook, not worker) - Defensive type handling with silentDebug - Supports multiline, nested, and multiple tags - Enables strategic orchestration for internal tools **User-Facing:** - <private> tag for manual privacy control (documented) - Prevents sensitive data from persisting in observations **Infrastructure:** - <claude-mem-context> tag ready for real-time context feature - Prevents recursive storage when context injection ships **Files:** - src/hooks/save-hook.ts: Core implementation - tests/strip-memory-tags.test.ts: Test suite (19/19 passing) - docs/public/usage/private-tags.mdx: User guide - docs/public/docs.json: Navigation update - docs/context/dual-tag-system-architecture.md: Technical docs - plugin/scripts/save-hook.js: Built hook 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix: Strip private tags from user prompts and skip memory ops for fully private prompts Fixes critical privacy bug where <private> tags were not being stripped from user prompts before storage in user_prompts table, making private content searchable via mem-search. Changes: 1. new-hook.ts: Skip memory operations for fully private prompts - If cleaned prompt is empty after stripping tags, skip saveUserPrompt - Skip worker init to avoid wasting resources on empty prompts - Logs: "(fully private - skipped)" 2. save-hook.ts: Skip observations for fully private prompts - Check if user prompt was entirely private before creating observations - Respects user intent: fully private prompt = no observations at all - Prevents "thoughts pop up" issue where private prompts create public observations 3. SessionStore.ts: Add getUserPrompt() method - Retrieves prompt text by session_id and prompt_number - Used by save-hook to check if prompt was private 4. Tests: Added 4 new tests for fully private prompt detection (16 total, all passing) 5. Docs: Updated private-tags.mdx to reflect correct behavior - User prompts ARE now filtered before storage - Private content never reaches database or search indices Privacy Protection: - Fully private prompts: No user_prompt saved, no worker init, no observations - Partially private prompts: Tags stripped, content sanitized before storage - Zero leaks: Private content never indexed or searchable Addresses reviewer feedback on PR #153 about user prompt filtering. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * feat: Enhance memory tag handling and indexing in user prompts - Added a new index `idx_user_prompts_lookup` on `user_prompts` for improved query performance based on `claude_session_id` and `prompt_number`. - Refactored memory tag stripping functionality into dedicated utility functions: `stripMemoryTagsFromJson` and `stripMemoryTagsFromPrompt` for better separation of concerns and reusability. - Updated hooks (`new-hook.ts` and `save-hook.ts`) to utilize the new tag stripping functions, ensuring private content is not stored or searchable. - Removed redundant inline tag stripping functions from hooks to streamline code. - Added tests for the new tag stripping utilities to ensure functionality and prevent regressions. --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Alex Newman
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* Tests for stripMemoryTags function
|
||||
* Verifies tag stripping and type safety for dual-tag system
|
||||
*/
|
||||
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert';
|
||||
import { stripMemoryTagsFromJson } from '../dist/utils/tag-stripping.js';
|
||||
|
||||
// Alias for clarity in tests (this tests the JSON context version)
|
||||
const stripMemoryTags = stripMemoryTagsFromJson;
|
||||
|
||||
describe('stripMemoryTags', () => {
|
||||
// Basic functionality tests - <claude-mem-context>
|
||||
it('should strip <claude-mem-context> tags', () => {
|
||||
const input = 'before <claude-mem-context>injected content</claude-mem-context> after';
|
||||
const expected = 'before after';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
// Basic functionality tests - <private>
|
||||
it('should strip <private> tags', () => {
|
||||
const input = 'before <private>sensitive data</private> after';
|
||||
const expected = 'before after';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should strip both tag types in one string', () => {
|
||||
const input = '<claude-mem-context>context</claude-mem-context> middle <private>private</private>';
|
||||
const expected = 'middle';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle nested tags', () => {
|
||||
const input = '<claude-mem-context>outer <private>inner</private> outer</claude-mem-context>';
|
||||
const expected = '';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle multiline content in tags', () => {
|
||||
const input = `before
|
||||
<claude-mem-context>
|
||||
line 1
|
||||
line 2
|
||||
line 3
|
||||
</claude-mem-context>
|
||||
after`;
|
||||
const expected = 'before\n\nafter';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle multiple tags of same type', () => {
|
||||
const input = '<private>first</private> middle <private>second</private>';
|
||||
const expected = 'middle';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should return empty string for content that is only tags', () => {
|
||||
const input = '<claude-mem-context>only this</claude-mem-context>';
|
||||
const expected = '';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle strings without tags', () => {
|
||||
const input = 'no tags here';
|
||||
const expected = 'no tags here';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle empty string', () => {
|
||||
const input = '';
|
||||
const expected = '';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should trim whitespace after stripping', () => {
|
||||
const input = ' <claude-mem-context>content</claude-mem-context> ';
|
||||
const expected = '';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle malformed tags (unclosed)', () => {
|
||||
const input = '<claude-mem-context>unclosed tag content';
|
||||
const expected = '<claude-mem-context>unclosed tag content';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle tag-like strings that are not actual tags', () => {
|
||||
const input = 'This is not a <tag> but looks like one';
|
||||
const expected = 'This is not a <tag> but looks like one';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
// Type safety tests
|
||||
it('should handle non-string input safely (number)', () => {
|
||||
const input = 123 as any;
|
||||
const expected = '{}';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle non-string input safely (null)', () => {
|
||||
const input = null as any;
|
||||
const expected = '{}';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle non-string input safely (undefined)', () => {
|
||||
const input = undefined as any;
|
||||
const expected = '{}';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle non-string input safely (object)', () => {
|
||||
const input = { foo: 'bar' } as any;
|
||||
const expected = '{}';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
it('should handle non-string input safely (array)', () => {
|
||||
const input = ['test'] as any;
|
||||
const expected = '{}';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
|
||||
// Real-world JSON scenarios
|
||||
it('should strip tags from JSON.stringify output', () => {
|
||||
const obj = {
|
||||
message: 'hello',
|
||||
context: '<claude-mem-context>past observation</claude-mem-context>',
|
||||
private: '<private>sensitive</private>'
|
||||
};
|
||||
const jsonStr = JSON.stringify(obj);
|
||||
const result = stripMemoryTags(jsonStr);
|
||||
|
||||
// Tags should be stripped from the JSON string
|
||||
assert.ok(!result.includes('<claude-mem-context>'));
|
||||
assert.ok(!result.includes('</claude-mem-context>'));
|
||||
assert.ok(!result.includes('<private>'));
|
||||
assert.ok(!result.includes('</private>'));
|
||||
});
|
||||
|
||||
it('should handle very large content efficiently', () => {
|
||||
const largeContent = 'x'.repeat(10000);
|
||||
const input = `<claude-mem-context>${largeContent}</claude-mem-context>`;
|
||||
const expected = '';
|
||||
assert.strictEqual(stripMemoryTags(input), expected);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Integration tests for user prompt tag stripping
|
||||
* Verifies that <private> and <claude-mem-context> tags are stripped
|
||||
* from user prompts before storage in the user_prompts table.
|
||||
*/
|
||||
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert';
|
||||
import { stripMemoryTagsFromPrompt } from '../dist/utils/tag-stripping.js';
|
||||
|
||||
// Alias for clarity in tests (this tests the prompt context version)
|
||||
const stripMemoryTags = stripMemoryTagsFromPrompt;
|
||||
|
||||
describe('User Prompt Tag Stripping', () => {
|
||||
it('should strip <private> tags from user prompts', () => {
|
||||
const userPrompt = 'Please analyze this: <private>API_KEY=secret123</private>';
|
||||
const expected = 'Please analyze this:';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should strip <claude-mem-context> tags from user prompts', () => {
|
||||
const userPrompt = '<claude-mem-context>Past observations...</claude-mem-context> Continue working';
|
||||
const expected = 'Continue working';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should handle prompts with multiple <private> sections', () => {
|
||||
const userPrompt = '<private>secret1</private> public text <private>secret2</private>';
|
||||
const expected = 'public text';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should handle prompts that are entirely private', () => {
|
||||
const userPrompt = '<private>This entire prompt should not be stored</private>';
|
||||
const expected = '';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should preserve prompts without tags', () => {
|
||||
const userPrompt = 'This is a normal prompt without any tags';
|
||||
const expected = 'This is a normal prompt without any tags';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should handle multiline private content in prompts', () => {
|
||||
const userPrompt = `Before
|
||||
<private>
|
||||
Line 1 of secret
|
||||
Line 2 of secret
|
||||
Line 3 of secret
|
||||
</private>
|
||||
After`;
|
||||
const expected = 'Before\n\nAfter';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should handle mixed tags in user prompts', () => {
|
||||
const userPrompt = '<claude-mem-context>Context</claude-mem-context> middle <private>private</private> end';
|
||||
const expected = 'middle end';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should handle real-world example: API credentials', () => {
|
||||
const userPrompt = `<private>
|
||||
OPENAI_API_KEY=sk-proj-abc123
|
||||
DATABASE_URL=postgresql://user:pass@host/db
|
||||
</private>
|
||||
|
||||
Please help me connect to this database and run a query`;
|
||||
|
||||
const result = stripMemoryTags(userPrompt);
|
||||
assert.ok(!result.includes('OPENAI_API_KEY'), 'API key should be stripped');
|
||||
assert.ok(!result.includes('DATABASE_URL'), 'Database URL should be stripped');
|
||||
assert.ok(!result.includes('<private>'), 'Private tags should be stripped');
|
||||
assert.ok(result.includes('Please help me connect'), 'Non-private content should remain');
|
||||
});
|
||||
|
||||
it('should handle real-world example: debugging context', () => {
|
||||
const userPrompt = `I'm getting an error in the authentication flow.
|
||||
|
||||
<private>
|
||||
Internal debugging notes:
|
||||
- This is for the Smith project
|
||||
- Deadline is tomorrow
|
||||
- Using staging environment
|
||||
</private>
|
||||
|
||||
Can you help me fix the token validation?`;
|
||||
|
||||
const result = stripMemoryTags(userPrompt);
|
||||
assert.ok(!result.includes('Smith project'), 'Debug notes should be stripped');
|
||||
assert.ok(!result.includes('Deadline'), 'Private context should be stripped');
|
||||
assert.ok(result.includes('authentication flow'), 'Problem description should remain');
|
||||
assert.ok(result.includes('token validation'), 'Question should remain');
|
||||
});
|
||||
|
||||
it('should handle edge case: only whitespace after tag removal', () => {
|
||||
const userPrompt = ' <private>everything</private> ';
|
||||
const expected = '';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should handle edge case: unclosed tags (no stripping)', () => {
|
||||
const userPrompt = 'Text <private>unclosed tag';
|
||||
const expected = 'Text <private>unclosed tag';
|
||||
assert.strictEqual(stripMemoryTags(userPrompt), expected);
|
||||
});
|
||||
|
||||
it('should handle non-string input gracefully', () => {
|
||||
// @ts-expect-error Testing runtime type safety
|
||||
const result = stripMemoryTags(null);
|
||||
assert.strictEqual(result, '');
|
||||
});
|
||||
|
||||
// Tests for fully private prompt behavior
|
||||
it('should return empty string for fully private prompts', () => {
|
||||
const fullyPrivate = '<private>Everything is private here</private>';
|
||||
const result = stripMemoryTags(fullyPrivate);
|
||||
assert.strictEqual(result, '');
|
||||
});
|
||||
|
||||
it('should return empty string for multiple private sections covering entire prompt', () => {
|
||||
const fullyPrivate = '<private>Part 1</private> <private>Part 2</private> <private>Part 3</private>';
|
||||
const result = stripMemoryTags(fullyPrivate);
|
||||
assert.strictEqual(result, '');
|
||||
});
|
||||
|
||||
it('should detect fully private prompts with only whitespace outside tags', () => {
|
||||
const fullyPrivate = ' <private>Content</private> ';
|
||||
const result = stripMemoryTags(fullyPrivate);
|
||||
assert.strictEqual(result, '');
|
||||
});
|
||||
|
||||
it('should not return empty for partially private prompts', () => {
|
||||
const partiallyPrivate = '<private>Secret</private> Public content here';
|
||||
const result = stripMemoryTags(partiallyPrivate);
|
||||
assert.ok(result.trim().length > 0, 'Should have non-empty content');
|
||||
assert.ok(result.includes('Public'), 'Should contain public content');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user