feat: Add beta channel for experimental features and update documentation
This commit is contained in:
@@ -17,7 +17,7 @@
|
|||||||
<img src="https://img.shields.io/badge/License-AGPL%203.0-blue.svg" alt="License">
|
<img src="https://img.shields.io/badge/License-AGPL%203.0-blue.svg" alt="License">
|
||||||
</a>
|
</a>
|
||||||
<a href="package.json">
|
<a href="package.json">
|
||||||
<img src="https://img.shields.io/badge/version-6.0.0-green.svg" alt="Version">
|
<img src="https://img.shields.io/badge/version-6.3.0-green.svg" alt="Version">
|
||||||
</a>
|
</a>
|
||||||
<a href="package.json">
|
<a href="package.json">
|
||||||
<img src="https://img.shields.io/badge/node-%3E%3D18.0.0-brightgreen.svg" alt="Node">
|
<img src="https://img.shields.io/badge/node-%3E%3D18.0.0-brightgreen.svg" alt="Node">
|
||||||
@@ -73,6 +73,7 @@ Restart Claude Code. Context from previous sessions will automatically appear in
|
|||||||
- 🖥️ **Web Viewer UI** - Real-time memory stream at http://localhost:37777
|
- 🖥️ **Web Viewer UI** - Real-time memory stream at http://localhost:37777
|
||||||
- 🤖 **Automatic Operation** - No manual intervention required
|
- 🤖 **Automatic Operation** - No manual intervention required
|
||||||
- 🔗 **Citations** - Reference past decisions with `claude-mem://` URIs
|
- 🔗 **Citations** - Reference past decisions with `claude-mem://` URIs
|
||||||
|
- 🧪 **Beta Channel** - Try experimental features like Endless Mode via version switching
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -92,6 +93,7 @@ npx mintlify dev
|
|||||||
- **[Installation Guide](https://docs.claude-mem.ai/installation)** - Quick start & advanced installation
|
- **[Installation Guide](https://docs.claude-mem.ai/installation)** - Quick start & advanced installation
|
||||||
- **[Usage Guide](https://docs.claude-mem.ai/usage/getting-started)** - How Claude-Mem works automatically
|
- **[Usage Guide](https://docs.claude-mem.ai/usage/getting-started)** - How Claude-Mem works automatically
|
||||||
- **[Search Tools](https://docs.claude-mem.ai/usage/search-tools)** - Query your project history with natural language
|
- **[Search Tools](https://docs.claude-mem.ai/usage/search-tools)** - Query your project history with natural language
|
||||||
|
- **[Beta Features](https://docs.claude-mem.ai/beta-features)** - Try experimental features like Endless Mode
|
||||||
|
|
||||||
### Best Practices
|
### Best Practices
|
||||||
|
|
||||||
@@ -189,6 +191,44 @@ See [Search Tools Guide](https://docs.claude-mem.ai/usage/search-tools) for deta
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Beta Features & Endless Mode
|
||||||
|
|
||||||
|
Claude-Mem offers a **beta channel** with experimental features. Switch between stable and beta versions directly from the web viewer UI.
|
||||||
|
|
||||||
|
### How to Try Beta
|
||||||
|
|
||||||
|
1. Open http://localhost:37777
|
||||||
|
2. Click Settings (gear icon)
|
||||||
|
3. In **Version Channel**, click "Try Beta (Endless Mode)"
|
||||||
|
4. Wait for the worker to restart
|
||||||
|
|
||||||
|
Your memory data is preserved when switching versions.
|
||||||
|
|
||||||
|
### Endless Mode (Beta)
|
||||||
|
|
||||||
|
The flagship beta feature is **Endless Mode** - a biomimetic memory architecture that dramatically extends session length:
|
||||||
|
|
||||||
|
**The Problem**: Standard Claude Code sessions hit context limits after ~50 tool uses. Each tool adds 1-10k+ tokens, and Claude re-synthesizes all previous outputs on every response (O(N²) complexity).
|
||||||
|
|
||||||
|
**The Solution**: Endless Mode compresses tool outputs into ~500-token observations and transforms the transcript in real-time:
|
||||||
|
|
||||||
|
```
|
||||||
|
Working Memory (Context): Compressed observations (~500 tokens each)
|
||||||
|
Archive Memory (Disk): Full tool outputs preserved for recall
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected Results**:
|
||||||
|
- ~95% token reduction in context window
|
||||||
|
- ~20x more tool uses before context exhaustion
|
||||||
|
- Linear O(N) scaling instead of quadratic O(N²)
|
||||||
|
- Full transcripts preserved for perfect recall
|
||||||
|
|
||||||
|
**Caveats**: Adds latency (60-90s per tool for observation generation), still experimental.
|
||||||
|
|
||||||
|
See [Beta Features Documentation](https://docs.claude-mem.ai/beta-features) for details.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## What's New in v6.0.0
|
## What's New in v6.0.0
|
||||||
|
|
||||||
**🚀 Major Session Management & Transcript Processing Improvements:**
|
**🚀 Major Session Management & Transcript Processing Improvements:**
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
---
|
||||||
|
title: "Beta Features"
|
||||||
|
description: "Try experimental features like Endless Mode before they're released"
|
||||||
|
---
|
||||||
|
|
||||||
|
# Beta Features
|
||||||
|
|
||||||
|
Claude-Mem offers a beta channel for users who want to try experimental features before they're released to the stable channel.
|
||||||
|
|
||||||
|
## Version Channel Switching
|
||||||
|
|
||||||
|
You can switch between stable and beta versions directly from the web viewer UI at http://localhost:37777.
|
||||||
|
|
||||||
|
### How to Access
|
||||||
|
|
||||||
|
1. Open the Claude-Mem viewer at http://localhost:37777
|
||||||
|
2. Click the **Settings** gear icon in the top-right
|
||||||
|
3. Find the **Version Channel** section
|
||||||
|
4. Click **Try Beta (Endless Mode)** to switch to beta, or **Switch to Stable** to return
|
||||||
|
|
||||||
|
### What Happens When You Switch
|
||||||
|
|
||||||
|
When switching versions:
|
||||||
|
|
||||||
|
1. **Local changes are discarded** - Any modifications in the plugin directory are reset
|
||||||
|
2. **Git fetch and checkout** - The installed plugin switches to the target branch
|
||||||
|
3. **Dependencies reinstall** - `npm install` runs to ensure correct dependencies
|
||||||
|
4. **Worker restarts automatically** - The background service restarts with the new version
|
||||||
|
|
||||||
|
**Your memory data is always preserved.** The database at `~/.claude-mem/claude-mem.db` is not affected by version switching. All your observations, sessions, and summaries remain intact.
|
||||||
|
|
||||||
|
### Version Indicators
|
||||||
|
|
||||||
|
The Version Channel section shows your current status:
|
||||||
|
|
||||||
|
- **Stable** (green badge) - You're running the production release
|
||||||
|
- **Beta** (orange badge) - You're running the beta with experimental features
|
||||||
|
|
||||||
|
You'll also see the exact branch name (e.g., `main` for stable, `beta/7.0` for beta).
|
||||||
|
|
||||||
|
## Endless Mode (Beta)
|
||||||
|
|
||||||
|
The flagship experimental feature in beta is **Endless Mode** - a biomimetic memory architecture that dramatically extends how long Claude can maintain context in a session.
|
||||||
|
|
||||||
|
### The Problem Endless Mode Solves
|
||||||
|
|
||||||
|
In standard Claude Code sessions:
|
||||||
|
|
||||||
|
- Tool outputs (file reads, bash output, search results) accumulate in the context window
|
||||||
|
- Each tool can add 1-10k+ tokens to the context
|
||||||
|
- After ~50 tool uses, the context window fills up (~200k tokens)
|
||||||
|
- You're forced to start a new session, losing conversational continuity
|
||||||
|
|
||||||
|
Worse, Claude **re-synthesizes all previous tool outputs** on every response. This is O(N²) complexity - quadratically growing both in tokens and compute.
|
||||||
|
|
||||||
|
### How Endless Mode Works
|
||||||
|
|
||||||
|
Endless Mode applies a biomimetic memory architecture inspired by how human memory works:
|
||||||
|
|
||||||
|
**Two-Tier Memory System:**
|
||||||
|
|
||||||
|
```
|
||||||
|
Working Memory (Context Window):
|
||||||
|
→ Compressed observations only (~500 tokens each)
|
||||||
|
→ Fast, efficient, manageable
|
||||||
|
|
||||||
|
Archive Memory (Transcript File):
|
||||||
|
→ Full tool outputs preserved on disk
|
||||||
|
→ Perfect recall, searchable
|
||||||
|
```
|
||||||
|
|
||||||
|
**The Key Innovation**: After each tool use, Endless Mode:
|
||||||
|
1. Waits for the worker to generate a compressed observation (blocking)
|
||||||
|
2. Transforms the transcript file on disk
|
||||||
|
3. Replaces the full tool output with the compressed observation
|
||||||
|
4. Claude resumes with the compressed context
|
||||||
|
|
||||||
|
This transforms O(N²) scaling into O(N) - linear instead of quadratic.
|
||||||
|
|
||||||
|
### Expected Results
|
||||||
|
|
||||||
|
Based on analysis of real sessions:
|
||||||
|
|
||||||
|
- **Token savings**: ~95% reduction in context window usage
|
||||||
|
- **Efficiency gain**: ~20x more tool uses before context exhaustion
|
||||||
|
- **Quality preservation**: Observations cache the synthesis result, so no information is lost
|
||||||
|
|
||||||
|
### Caveats
|
||||||
|
|
||||||
|
Endless Mode is experimental:
|
||||||
|
|
||||||
|
- **Adds latency** - Blocking hooks wait for observation generation (60-90s per tool use)
|
||||||
|
- **Requires working database** - Observations must save successfully for transformation
|
||||||
|
- **New architecture** - Less battle-tested than standard mode
|
||||||
|
|
||||||
|
### When to Use Beta
|
||||||
|
|
||||||
|
Consider switching to beta if you:
|
||||||
|
|
||||||
|
- Frequently hit context window limits
|
||||||
|
- Work on long, complex sessions with many tool uses
|
||||||
|
- Want to help test and provide feedback on new features
|
||||||
|
- Are comfortable with experimental software
|
||||||
|
|
||||||
|
### When to Stay on Stable
|
||||||
|
|
||||||
|
Stay on stable if you:
|
||||||
|
|
||||||
|
- Need maximum reliability for critical work
|
||||||
|
- Prefer battle-tested, production-ready features
|
||||||
|
- Don't frequently hit context limits
|
||||||
|
- Want the smoothest, fastest experience
|
||||||
|
|
||||||
|
## Checking for Updates
|
||||||
|
|
||||||
|
While on beta (or stable), you can check for updates:
|
||||||
|
|
||||||
|
1. Open Settings in the viewer
|
||||||
|
2. In the Version Channel section, click **Check for Updates**
|
||||||
|
3. The plugin will pull the latest changes and restart
|
||||||
|
|
||||||
|
## Switching Back
|
||||||
|
|
||||||
|
If you encounter issues on beta:
|
||||||
|
|
||||||
|
1. Open Settings in the viewer
|
||||||
|
2. Click **Switch to Stable**
|
||||||
|
3. Wait for the worker to restart
|
||||||
|
|
||||||
|
Your memory data is preserved, and you'll be back on the stable release.
|
||||||
|
|
||||||
|
## Providing Feedback
|
||||||
|
|
||||||
|
If you encounter bugs or have feedback about beta features:
|
||||||
|
|
||||||
|
- Open an issue at [GitHub Issues](https://github.com/thedotmack/claude-mem/issues)
|
||||||
|
- Include your branch (`beta/7.0` etc.) in the report
|
||||||
|
- Describe what you expected vs. what happened
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
- [Configuration](configuration) - Customize other Claude-Mem settings
|
||||||
|
- [Troubleshooting](troubleshooting) - Common issues and solutions
|
||||||
|
- [Architecture Overview](architecture/overview) - Understand how Claude-Mem works
|
||||||
@@ -151,6 +151,26 @@ Search operations are now provided via:
|
|||||||
- **HTTP API**: 10 endpoints on worker service port 37777
|
- **HTTP API**: 10 endpoints on worker service port 37777
|
||||||
- **Progressive Disclosure**: Full instructions loaded on-demand only when needed
|
- **Progressive Disclosure**: Full instructions loaded on-demand only when needed
|
||||||
|
|
||||||
|
## Version Channel
|
||||||
|
|
||||||
|
Claude-Mem supports switching between stable and beta versions via the web viewer UI.
|
||||||
|
|
||||||
|
### Accessing Version Channel
|
||||||
|
|
||||||
|
1. Open the viewer at http://localhost:37777
|
||||||
|
2. Click the Settings gear icon
|
||||||
|
3. Find the **Version Channel** section
|
||||||
|
|
||||||
|
### Switching Versions
|
||||||
|
|
||||||
|
- **Try Beta**: Click "Try Beta (Endless Mode)" to switch to the beta branch with experimental features
|
||||||
|
- **Switch to Stable**: Click "Switch to Stable" to return to the production release
|
||||||
|
- **Check for Updates**: Pull the latest changes for your current branch
|
||||||
|
|
||||||
|
**Your memory data is preserved** when switching versions. Only the plugin code changes.
|
||||||
|
|
||||||
|
See [Beta Features](beta-features) for details on what's available in beta.
|
||||||
|
|
||||||
## PM2 Configuration
|
## PM2 Configuration
|
||||||
|
|
||||||
Worker service is managed by PM2 via `ecosystem.config.cjs`:
|
Worker service is managed by PM2 via `ecosystem.config.cjs`:
|
||||||
|
|||||||
@@ -36,7 +36,8 @@
|
|||||||
"introduction",
|
"introduction",
|
||||||
"installation",
|
"installation",
|
||||||
"usage/getting-started",
|
"usage/getting-started",
|
||||||
"usage/search-tools"
|
"usage/search-tools",
|
||||||
|
"beta-features"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,410 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
import fs from 'fs';
|
||||||
|
import Database from 'better-sqlite3';
|
||||||
|
import readline from 'readline';
|
||||||
|
import path from 'path';
|
||||||
|
import { homedir } from 'os';
|
||||||
|
import { globSync } from 'glob';
|
||||||
|
|
||||||
|
// =============================================================================
|
||||||
|
// TOOL REPLACEMENT DECISION TABLE
|
||||||
|
// =============================================================================
|
||||||
|
//
|
||||||
|
// KEY INSIGHT: Observations are the SEMANTIC SYNTHESIS of tool results.
|
||||||
|
// They contain what Claude LEARNED, which is what future Claude needs.
|
||||||
|
//
|
||||||
|
// Tool | Replace OUTPUT? | Reason
|
||||||
|
// ------------------|-----------------|----------------------------------------
|
||||||
|
// Read | ✅ YES | Observation = what was learned from file
|
||||||
|
// Bash | ✅ YES | Observation = what command revealed
|
||||||
|
// Grep | ✅ YES | Observation = what search found
|
||||||
|
// Task | ✅ YES | Observation = what agent discovered
|
||||||
|
// WebFetch | ✅ YES | Observation = what page contained
|
||||||
|
// Glob | ⚠️ MAYBE | File lists are often small already
|
||||||
|
// WebSearch | ⚠️ MAYBE | Results are moderate size
|
||||||
|
// Edit | ❌ NO | OUTPUT is tiny ("success"), INPUT is ground truth
|
||||||
|
// Write | ❌ NO | OUTPUT is tiny, INPUT is the file content
|
||||||
|
// NotebookEdit | ❌ NO | OUTPUT is tiny, INPUT is the code
|
||||||
|
// TodoWrite | ❌ NO | Both tiny
|
||||||
|
// AskUserQuestion | ❌ NO | Both small, user input matters
|
||||||
|
// mcp__* | ⚠️ MAYBE | Varies by tool
|
||||||
|
//
|
||||||
|
// NEVER REPLACE INPUT - it contains the action (diff, command, query, path)
|
||||||
|
// ONLY REPLACE OUTPUT - swap raw results for semantic synthesis (observation)
|
||||||
|
//
|
||||||
|
// REPLACEMENT FORMAT:
|
||||||
|
// Original output gets replaced with:
|
||||||
|
// "[Strategically Omitted by Claude-Mem to save tokens]
|
||||||
|
//
|
||||||
|
// [Observation: Title here]
|
||||||
|
// Facts: ...
|
||||||
|
// Concepts: ..."
|
||||||
|
// =============================================================================
|
||||||
|
|
||||||
|
// Configuration
|
||||||
|
const DB_PATH = path.join(homedir(), '.claude-mem', 'claude-mem.db');
|
||||||
|
const MAX_TRANSCRIPTS = parseInt(process.env.MAX_TRANSCRIPTS || '500', 10);
|
||||||
|
|
||||||
|
// Find transcript files (most recent first)
|
||||||
|
const TRANSCRIPT_DIR = path.join(homedir(), '.claude/projects/-Users-alexnewman-Scripts-claude-mem');
|
||||||
|
const allTranscriptFiles = globSync(path.join(TRANSCRIPT_DIR, '*.jsonl'));
|
||||||
|
|
||||||
|
// Sort by modification time (most recent first), take MAX_TRANSCRIPTS
|
||||||
|
const transcriptFiles = allTranscriptFiles
|
||||||
|
.map(f => ({ path: f, mtime: fs.statSync(f).mtime }))
|
||||||
|
.sort((a, b) => b.mtime - a.mtime)
|
||||||
|
.slice(0, MAX_TRANSCRIPTS)
|
||||||
|
.map(f => f.path);
|
||||||
|
|
||||||
|
console.log(`Config: MAX_TRANSCRIPTS=${MAX_TRANSCRIPTS}`);
|
||||||
|
console.log(`Using ${transcriptFiles.length} most recent transcript files (of ${allTranscriptFiles.length} total)\n`);
|
||||||
|
|
||||||
|
// Map to store original content from transcript (both inputs and outputs)
|
||||||
|
const originalContent = new Map();
|
||||||
|
|
||||||
|
// Track contaminated (already transformed) transcripts
|
||||||
|
let skippedTranscripts = 0;
|
||||||
|
|
||||||
|
// Marker for already-transformed content (endless mode replacement format)
|
||||||
|
const TRANSFORMATION_MARKER = '**Key Facts:**';
|
||||||
|
|
||||||
|
// Auto-discover agent transcripts linked to main session
|
||||||
|
async function discoverAgentFiles(mainTranscriptPath) {
|
||||||
|
console.log('Discovering linked agent transcripts...');
|
||||||
|
|
||||||
|
const agentIds = new Set();
|
||||||
|
const fileStream = fs.createReadStream(mainTranscriptPath);
|
||||||
|
const rl = readline.createInterface({
|
||||||
|
input: fileStream,
|
||||||
|
crlfDelay: Infinity
|
||||||
|
});
|
||||||
|
|
||||||
|
for await (const line of rl) {
|
||||||
|
if (!line.includes('agentId')) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const obj = JSON.parse(line);
|
||||||
|
|
||||||
|
// Check for agentId in toolUseResult
|
||||||
|
if (obj.toolUseResult?.agentId) {
|
||||||
|
agentIds.add(obj.toolUseResult.agentId);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// Skip malformed lines
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build agent file paths
|
||||||
|
const directory = path.dirname(mainTranscriptPath);
|
||||||
|
const agentFiles = Array.from(agentIds).map(id =>
|
||||||
|
path.join(directory, `agent-${id}.jsonl`)
|
||||||
|
).filter(filePath => fs.existsSync(filePath));
|
||||||
|
|
||||||
|
console.log(` → Found ${agentIds.size} agent IDs`);
|
||||||
|
console.log(` → ${agentFiles.length} agent files exist on disk\n`);
|
||||||
|
|
||||||
|
return agentFiles;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse transcript to get BOTH tool_use (inputs) and tool_result (outputs) content
|
||||||
|
// Returns true if transcript is clean, false if contaminated (already transformed)
|
||||||
|
async function loadOriginalContentFromFile(filePath, fileLabel) {
|
||||||
|
const fileStream = fs.createReadStream(filePath);
|
||||||
|
const rl = readline.createInterface({
|
||||||
|
input: fileStream,
|
||||||
|
crlfDelay: Infinity
|
||||||
|
});
|
||||||
|
|
||||||
|
let count = 0;
|
||||||
|
let isContaminated = false;
|
||||||
|
const toolUseIdsFromThisFile = new Set();
|
||||||
|
|
||||||
|
for await (const line of rl) {
|
||||||
|
if (!line.includes('toolu_')) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const obj = JSON.parse(line);
|
||||||
|
|
||||||
|
if (obj.message?.content) {
|
||||||
|
for (const item of obj.message.content) {
|
||||||
|
// Capture tool_use (inputs)
|
||||||
|
if (item.type === 'tool_use' && item.id) {
|
||||||
|
const existing = originalContent.get(item.id) || { input: '', output: '', name: '' };
|
||||||
|
existing.input = JSON.stringify(item.input || {});
|
||||||
|
existing.name = item.name;
|
||||||
|
originalContent.set(item.id, existing);
|
||||||
|
toolUseIdsFromThisFile.add(item.id);
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture tool_result (outputs)
|
||||||
|
if (item.type === 'tool_result' && item.tool_use_id) {
|
||||||
|
const content = typeof item.content === 'string' ? item.content : JSON.stringify(item.content);
|
||||||
|
|
||||||
|
// Check for transformation marker - if found, transcript is contaminated
|
||||||
|
if (content.includes(TRANSFORMATION_MARKER)) {
|
||||||
|
isContaminated = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = originalContent.get(item.tool_use_id) || { input: '', output: '', name: '' };
|
||||||
|
existing.output = content;
|
||||||
|
originalContent.set(item.tool_use_id, existing);
|
||||||
|
toolUseIdsFromThisFile.add(item.tool_use_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// Skip malformed lines
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If contaminated, remove all data from this file and report
|
||||||
|
if (isContaminated) {
|
||||||
|
for (const id of toolUseIdsFromThisFile) {
|
||||||
|
originalContent.delete(id);
|
||||||
|
}
|
||||||
|
console.log(` ⚠️ Skipped ${fileLabel} (already transformed)`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (count > 0) {
|
||||||
|
console.log(` → Found ${count} tool uses in ${fileLabel}`);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadOriginalContent() {
|
||||||
|
console.log('Loading original content from transcripts...');
|
||||||
|
console.log(` → Scanning ${transcriptFiles.length} transcript files...\n`);
|
||||||
|
|
||||||
|
let cleanTranscripts = 0;
|
||||||
|
|
||||||
|
// Load from all transcript files
|
||||||
|
for (const transcriptFile of transcriptFiles) {
|
||||||
|
const filename = path.basename(transcriptFile);
|
||||||
|
const isClean = await loadOriginalContentFromFile(transcriptFile, filename);
|
||||||
|
if (isClean) {
|
||||||
|
cleanTranscripts++;
|
||||||
|
} else {
|
||||||
|
skippedTranscripts++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check for any agent files not already included
|
||||||
|
for (const transcriptFile of transcriptFiles) {
|
||||||
|
if (transcriptFile.includes('agent-')) continue; // Already an agent file
|
||||||
|
const agentFiles = await discoverAgentFiles(transcriptFile);
|
||||||
|
for (const agentFile of agentFiles) {
|
||||||
|
if (transcriptFiles.includes(agentFile)) continue; // Already processed
|
||||||
|
const filename = path.basename(agentFile);
|
||||||
|
const isClean = await loadOriginalContentFromFile(agentFile, `agent transcript (${filename})`);
|
||||||
|
if (!isClean) {
|
||||||
|
skippedTranscripts++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\nTotal: Loaded original content for ${originalContent.size} tool uses (inputs + outputs)`);
|
||||||
|
if (skippedTranscripts > 0) {
|
||||||
|
console.log(`⚠️ Skipped ${skippedTranscripts} transcripts (already transformed with endless mode)`);
|
||||||
|
}
|
||||||
|
console.log();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Strip __N suffix from tool_use_id to get base ID
|
||||||
|
function getBaseToolUseId(id) {
|
||||||
|
return id ? id.replace(/__\d+$/, '') : id;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query observations from database using tool_use_ids found in transcripts
|
||||||
|
// Handles suffixed IDs like toolu_abc__1, toolu_abc__2 matching transcript's toolu_abc
|
||||||
|
function queryObservations() {
|
||||||
|
// Get tool_use_ids from the loaded transcript content
|
||||||
|
const toolUseIds = Array.from(originalContent.keys());
|
||||||
|
|
||||||
|
if (toolUseIds.length === 0) {
|
||||||
|
console.log('No tool use IDs found in transcripts\n');
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Querying observations for ${toolUseIds.length} tool use IDs from transcripts...`);
|
||||||
|
|
||||||
|
const db = new Database(DB_PATH, { readonly: true });
|
||||||
|
|
||||||
|
// Build LIKE clauses to match both exact IDs and suffixed variants (toolu_abc, toolu_abc__1, etc)
|
||||||
|
const likeConditions = toolUseIds.map(() => 'tool_use_id LIKE ?').join(' OR ');
|
||||||
|
const likeParams = toolUseIds.map(id => `${id}%`);
|
||||||
|
|
||||||
|
const query = `
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
tool_use_id,
|
||||||
|
type,
|
||||||
|
narrative,
|
||||||
|
title,
|
||||||
|
facts,
|
||||||
|
concepts,
|
||||||
|
LENGTH(COALESCE(facts,'')) as facts_len,
|
||||||
|
LENGTH(COALESCE(title,'')) + LENGTH(COALESCE(facts,'')) as title_facts_len,
|
||||||
|
LENGTH(COALESCE(title,'')) + LENGTH(COALESCE(facts,'')) + LENGTH(COALESCE(concepts,'')) as compact_len,
|
||||||
|
LENGTH(COALESCE(narrative,'')) as narrative_len,
|
||||||
|
LENGTH(COALESCE(title,'')) + LENGTH(COALESCE(narrative,'')) + LENGTH(COALESCE(facts,'')) + LENGTH(COALESCE(concepts,'')) as full_obs_len
|
||||||
|
FROM observations
|
||||||
|
WHERE ${likeConditions}
|
||||||
|
ORDER BY created_at DESC
|
||||||
|
`;
|
||||||
|
|
||||||
|
const observations = db.prepare(query).all(...likeParams);
|
||||||
|
db.close();
|
||||||
|
|
||||||
|
console.log(`Found ${observations.length} observations matching tool use IDs (including suffixed variants)\n`);
|
||||||
|
|
||||||
|
return observations;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tools eligible for OUTPUT replacement (observation = semantic synthesis of result)
|
||||||
|
const REPLACEABLE_TOOLS = new Set(['Read', 'Bash', 'Grep', 'Task', 'WebFetch', 'Glob', 'WebSearch']);
|
||||||
|
|
||||||
|
// Analyze OUTPUT-only replacement for eligible tools
|
||||||
|
function analyzeTransformations(observations) {
|
||||||
|
console.log('='.repeat(110));
|
||||||
|
console.log('OUTPUT REPLACEMENT ANALYSIS (Eligible Tools Only)');
|
||||||
|
console.log('='.repeat(110));
|
||||||
|
console.log();
|
||||||
|
console.log('Eligible tools:', Array.from(REPLACEABLE_TOOLS).join(', '));
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// Group observations by BASE tool_use_id (strip __N suffix)
|
||||||
|
// This groups toolu_abc, toolu_abc__1, toolu_abc__2 together
|
||||||
|
const obsByToolId = new Map();
|
||||||
|
observations.forEach(obs => {
|
||||||
|
const baseId = getBaseToolUseId(obs.tool_use_id);
|
||||||
|
if (!obsByToolId.has(baseId)) {
|
||||||
|
obsByToolId.set(baseId, []);
|
||||||
|
}
|
||||||
|
obsByToolId.get(baseId).push(obs);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Define strategies to test
|
||||||
|
const strategies = [
|
||||||
|
{ name: 'facts_only', field: 'facts_len', desc: 'Facts only (~400 chars)' },
|
||||||
|
{ name: 'title_facts', field: 'title_facts_len', desc: 'Title + Facts (~450 chars)' },
|
||||||
|
{ name: 'compact', field: 'compact_len', desc: 'Title + Facts + Concepts (~500 chars)' },
|
||||||
|
{ name: 'narrative', field: 'narrative_len', desc: 'Narrative only (~700 chars)' },
|
||||||
|
{ name: 'full', field: 'full_obs_len', desc: 'Full observation (~1200 chars)' }
|
||||||
|
];
|
||||||
|
|
||||||
|
// Track results per strategy
|
||||||
|
const results = {};
|
||||||
|
strategies.forEach(s => {
|
||||||
|
results[s.name] = {
|
||||||
|
transforms: 0,
|
||||||
|
noTransform: 0,
|
||||||
|
saved: 0,
|
||||||
|
totalOriginal: 0
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// Track stats
|
||||||
|
let eligible = 0;
|
||||||
|
let ineligible = 0;
|
||||||
|
let noTranscript = 0;
|
||||||
|
const toolCounts = {};
|
||||||
|
|
||||||
|
// Analyze each tool use
|
||||||
|
obsByToolId.forEach((obsArray, toolUseId) => {
|
||||||
|
const original = originalContent.get(toolUseId);
|
||||||
|
const toolName = original?.name || 'unknown';
|
||||||
|
const outputLen = original?.output?.length || 0;
|
||||||
|
|
||||||
|
// Skip if no transcript data
|
||||||
|
if (!original || outputLen === 0) {
|
||||||
|
noTranscript++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip if tool not eligible for replacement
|
||||||
|
if (!REPLACEABLE_TOOLS.has(toolName)) {
|
||||||
|
ineligible++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
eligible++;
|
||||||
|
toolCounts[toolName] = (toolCounts[toolName] || 0) + 1;
|
||||||
|
|
||||||
|
// Sum lengths across ALL observations for this tool use (handles multiple obs per tool_use_id)
|
||||||
|
// Test each strategy - OUTPUT replacement only
|
||||||
|
strategies.forEach(strategy => {
|
||||||
|
const obsLen = obsArray.reduce((sum, obs) => sum + (obs[strategy.field] || 0), 0);
|
||||||
|
const r = results[strategy.name];
|
||||||
|
|
||||||
|
r.totalOriginal += outputLen;
|
||||||
|
|
||||||
|
if (obsLen > 0 && obsLen < outputLen) {
|
||||||
|
r.transforms++;
|
||||||
|
r.saved += (outputLen - obsLen);
|
||||||
|
} else {
|
||||||
|
r.noTransform++;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Print results
|
||||||
|
console.log('TOOL BREAKDOWN:');
|
||||||
|
Object.entries(toolCounts).sort((a, b) => b[1] - a[1]).forEach(([tool, count]) => {
|
||||||
|
console.log(` ${tool}: ${count}`);
|
||||||
|
});
|
||||||
|
console.log();
|
||||||
|
console.log('-'.repeat(100));
|
||||||
|
console.log(`Eligible tool uses: ${eligible}`);
|
||||||
|
console.log(`Ineligible (Edit/Write/etc): ${ineligible}`);
|
||||||
|
console.log(`No transcript data: ${noTranscript}`);
|
||||||
|
console.log('-'.repeat(100));
|
||||||
|
console.log();
|
||||||
|
console.log('Strategy Transforms No Transform Chars Saved Original Size Savings %');
|
||||||
|
console.log('-'.repeat(100));
|
||||||
|
|
||||||
|
strategies.forEach(strategy => {
|
||||||
|
const r = results[strategy.name];
|
||||||
|
const pct = r.totalOriginal > 0 ? ((r.saved / r.totalOriginal) * 100).toFixed(1) : '0.0';
|
||||||
|
console.log(
|
||||||
|
`${strategy.desc.padEnd(35)} ${String(r.transforms).padStart(10)} ${String(r.noTransform).padStart(12)} ${String(r.saved.toLocaleString()).padStart(13)} ${String(r.totalOriginal.toLocaleString()).padStart(15)} ${pct.padStart(8)}%`
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('-'.repeat(100));
|
||||||
|
console.log();
|
||||||
|
|
||||||
|
// Find best strategy
|
||||||
|
let bestStrategy = null;
|
||||||
|
let bestSavings = 0;
|
||||||
|
strategies.forEach(strategy => {
|
||||||
|
if (results[strategy.name].saved > bestSavings) {
|
||||||
|
bestSavings = results[strategy.name].saved;
|
||||||
|
bestStrategy = strategy;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (bestStrategy) {
|
||||||
|
const r = results[bestStrategy.name];
|
||||||
|
const pct = ((r.saved / r.totalOriginal) * 100).toFixed(1);
|
||||||
|
console.log(`BEST STRATEGY: ${bestStrategy.desc}`);
|
||||||
|
console.log(` - Transforms ${r.transforms} of ${eligible} eligible tool uses (${((r.transforms/eligible)*100).toFixed(1)}%)`);
|
||||||
|
console.log(` - Saves ${r.saved.toLocaleString()} of ${r.totalOriginal.toLocaleString()} chars (${pct}% reduction)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Main execution
|
||||||
|
async function main() {
|
||||||
|
await loadOriginalContent();
|
||||||
|
const observations = queryObservations();
|
||||||
|
analyzeTransformations(observations);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(error => {
|
||||||
|
console.error('Fatal error:', error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Protected sync-marketplace script
|
||||||
|
*
|
||||||
|
* Prevents accidental rsync overwrite when installed plugin is on beta branch.
|
||||||
|
* If on beta, the user should use the UI to update instead.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const { execSync } = require('child_process');
|
||||||
|
const { existsSync } = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const os = require('os');
|
||||||
|
|
||||||
|
const INSTALLED_PATH = path.join(os.homedir(), '.claude', 'plugins', 'marketplaces', 'thedotmack');
|
||||||
|
|
||||||
|
function getCurrentBranch() {
|
||||||
|
try {
|
||||||
|
if (!existsSync(path.join(INSTALLED_PATH, '.git'))) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return execSync('git rev-parse --abbrev-ref HEAD', {
|
||||||
|
cwd: INSTALLED_PATH,
|
||||||
|
encoding: 'utf-8',
|
||||||
|
stdio: ['pipe', 'pipe', 'pipe']
|
||||||
|
}).trim();
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const branch = getCurrentBranch();
|
||||||
|
|
||||||
|
if (branch && branch !== 'main') {
|
||||||
|
console.log('');
|
||||||
|
console.log('\x1b[33m%s\x1b[0m', `WARNING: Installed plugin is on beta branch: ${branch}`);
|
||||||
|
console.log('\x1b[33m%s\x1b[0m', 'Running rsync would overwrite beta code.');
|
||||||
|
console.log('');
|
||||||
|
console.log('Options:');
|
||||||
|
console.log(' 1. Use UI at http://localhost:37777 to update beta');
|
||||||
|
console.log(' 2. Switch to stable in UI first, then run sync');
|
||||||
|
console.log(' 3. Force rsync: npm run sync-marketplace:force');
|
||||||
|
console.log('');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normal rsync for main branch or fresh install
|
||||||
|
console.log('Syncing to marketplace...');
|
||||||
|
try {
|
||||||
|
execSync(
|
||||||
|
'rsync -av --delete --exclude=.git ./ ~/.claude/plugins/marketplaces/thedotmack/',
|
||||||
|
{ stdio: 'inherit' }
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log('Running npm install in marketplace...');
|
||||||
|
execSync(
|
||||||
|
'cd ~/.claude/plugins/marketplaces/thedotmack/ && npm install',
|
||||||
|
{ stdio: 'inherit' }
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log('\x1b[32m%s\x1b[0m', 'Sync complete!');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('\x1b[31m%s\x1b[0m', 'Sync failed:', error.message);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user