backup: Phase 1 agent work (security, persistence, batch endpoint)
This is a backup of all work done by the 3 Phase 1 agents: Agent A - Command Injection Fix (Issue #354): - Fixed command injection in BranchManager.ts - Fixed unnecessary shell usage in bun-path.ts - Added comprehensive security test suite - Created SECURITY.md and SECURITY_AUDIT_REPORT.md Agent B - Observation Persistence Fix (Issue #353): - Added PendingMessageStore from PR #335 - Integrated persistent queue into SessionManager - Modified SDKAgent to mark messages complete - Updated SessionStore with pending_messages migration - Updated worker-types.ts with new interfaces Agent C - Batch Endpoint Verification (Issue #348): - Created batch-observations.test.ts - Updated worker-service.mdx documentation Also includes: - Documentation context files (biomimetic, windows struggles) - Build artifacts from agent testing This work will be re-evaluated after v7.3.0 release.
This commit is contained in:
Alex Newman
@@ -0,0 +1,367 @@
|
||||
import { Database } from './sqlite-compat.js';
|
||||
import type { PendingMessage } from '../worker-types.js';
|
||||
|
||||
/**
|
||||
* Persistent pending message record from database
|
||||
*/
|
||||
export interface PersistentPendingMessage {
|
||||
id: number;
|
||||
session_db_id: number;
|
||||
claude_session_id: string;
|
||||
message_type: 'observation' | 'summarize';
|
||||
tool_name: string | null;
|
||||
tool_input: string | null;
|
||||
tool_response: string | null;
|
||||
cwd: string | null;
|
||||
last_user_message: string | null;
|
||||
last_assistant_message: string | null;
|
||||
prompt_number: number | null;
|
||||
status: 'pending' | 'processing' | 'processed' | 'failed';
|
||||
retry_count: number;
|
||||
created_at_epoch: number;
|
||||
started_processing_at_epoch: number | null;
|
||||
completed_at_epoch: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* PendingMessageStore - Persistent work queue for SDK messages
|
||||
*
|
||||
* Messages are persisted before processing and marked complete after success.
|
||||
* This enables recovery from SDK hangs and worker crashes.
|
||||
*
|
||||
* Lifecycle:
|
||||
* 1. enqueue() - Message persisted with status 'pending'
|
||||
* 2. markProcessing() - Status changes to 'processing' when yielded to SDK
|
||||
* 3. markProcessed() - Status changes to 'processed' after successful SDK response
|
||||
* 4. markFailed() - Status changes to 'failed' if max retries exceeded
|
||||
*
|
||||
* Recovery:
|
||||
* - resetStuckMessages() - Moves 'processing' messages back to 'pending' if stuck
|
||||
* - getSessionsWithPendingMessages() - Find sessions that need recovery on startup
|
||||
*/
|
||||
export class PendingMessageStore {
|
||||
private db: Database;
|
||||
private maxRetries: number;
|
||||
|
||||
constructor(db: Database, maxRetries: number = 3) {
|
||||
this.db = db;
|
||||
this.maxRetries = maxRetries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a new message (persist before processing)
|
||||
* @returns The database ID of the persisted message
|
||||
*/
|
||||
enqueue(sessionDbId: number, claudeSessionId: string, message: PendingMessage): number {
|
||||
const now = Date.now();
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO pending_messages (
|
||||
session_db_id, claude_session_id, message_type,
|
||||
tool_name, tool_input, tool_response, cwd,
|
||||
last_user_message, last_assistant_message,
|
||||
prompt_number, status, retry_count, created_at_epoch
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', 0, ?)
|
||||
`);
|
||||
|
||||
const result = stmt.run(
|
||||
sessionDbId,
|
||||
claudeSessionId,
|
||||
message.type,
|
||||
message.tool_name || null,
|
||||
message.tool_input ? JSON.stringify(message.tool_input) : null,
|
||||
message.tool_response ? JSON.stringify(message.tool_response) : null,
|
||||
message.cwd || null,
|
||||
message.last_user_message || null,
|
||||
message.last_assistant_message || null,
|
||||
message.prompt_number || null,
|
||||
now
|
||||
);
|
||||
|
||||
return result.lastInsertRowid as number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Peek at oldest pending message for session (does NOT change status)
|
||||
* @returns The oldest pending message or null if none
|
||||
*/
|
||||
peekPending(sessionDbId: number): PersistentPendingMessage | null {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT * FROM pending_messages
|
||||
WHERE session_db_id = ? AND status = 'pending'
|
||||
ORDER BY id ASC
|
||||
LIMIT 1
|
||||
`);
|
||||
return stmt.get(sessionDbId) as PersistentPendingMessage | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all pending messages for session (ordered by creation time)
|
||||
*/
|
||||
getAllPending(sessionDbId: number): PersistentPendingMessage[] {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT * FROM pending_messages
|
||||
WHERE session_db_id = ? AND status = 'pending'
|
||||
ORDER BY id ASC
|
||||
`);
|
||||
return stmt.all(sessionDbId) as PersistentPendingMessage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all queue messages (for UI display)
|
||||
* Returns pending, processing, and failed messages (not processed - they're deleted)
|
||||
* Joins with sdk_sessions to get project name
|
||||
*/
|
||||
getQueueMessages(): (PersistentPendingMessage & { project: string | null })[] {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT pm.*, ss.project
|
||||
FROM pending_messages pm
|
||||
LEFT JOIN sdk_sessions ss ON pm.claude_session_id = ss.claude_session_id
|
||||
WHERE pm.status IN ('pending', 'processing', 'failed')
|
||||
ORDER BY
|
||||
CASE pm.status
|
||||
WHEN 'failed' THEN 0
|
||||
WHEN 'processing' THEN 1
|
||||
WHEN 'pending' THEN 2
|
||||
END,
|
||||
pm.created_at_epoch ASC
|
||||
`);
|
||||
return stmt.all() as (PersistentPendingMessage & { project: string | null })[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of stuck messages (processing longer than threshold)
|
||||
*/
|
||||
getStuckCount(thresholdMs: number): number {
|
||||
const cutoff = Date.now() - thresholdMs;
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT COUNT(*) as count FROM pending_messages
|
||||
WHERE status = 'processing' AND started_processing_at_epoch < ?
|
||||
`);
|
||||
const result = stmt.get(cutoff) as { count: number };
|
||||
return result.count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry a specific message (reset to pending)
|
||||
* Works for pending (re-queue), processing (reset stuck), and failed messages
|
||||
*/
|
||||
retryMessage(messageId: number): boolean {
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'pending', started_processing_at_epoch = NULL
|
||||
WHERE id = ? AND status IN ('pending', 'processing', 'failed')
|
||||
`);
|
||||
const result = stmt.run(messageId);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset all processing messages for a session to pending
|
||||
* Used when force-restarting a stuck session
|
||||
*/
|
||||
resetProcessingToPending(sessionDbId: number): number {
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'pending', started_processing_at_epoch = NULL
|
||||
WHERE session_db_id = ? AND status = 'processing'
|
||||
`);
|
||||
const result = stmt.run(sessionDbId);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abort a specific message (delete from queue)
|
||||
*/
|
||||
abortMessage(messageId: number): boolean {
|
||||
const stmt = this.db.prepare('DELETE FROM pending_messages WHERE id = ?');
|
||||
const result = stmt.run(messageId);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry all stuck messages at once
|
||||
*/
|
||||
retryAllStuck(thresholdMs: number): number {
|
||||
const cutoff = Date.now() - thresholdMs;
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'pending', started_processing_at_epoch = NULL
|
||||
WHERE status = 'processing' AND started_processing_at_epoch < ?
|
||||
`);
|
||||
const result = stmt.run(cutoff);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recently processed messages (for UI feedback)
|
||||
* Shows messages completed in the last N minutes so users can see their stuck items were processed
|
||||
*/
|
||||
getRecentlyProcessed(limit: number = 10, withinMinutes: number = 30): (PersistentPendingMessage & { project: string | null })[] {
|
||||
const cutoff = Date.now() - (withinMinutes * 60 * 1000);
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT pm.*, ss.project
|
||||
FROM pending_messages pm
|
||||
LEFT JOIN sdk_sessions ss ON pm.claude_session_id = ss.claude_session_id
|
||||
WHERE pm.status = 'processed' AND pm.completed_at_epoch > ?
|
||||
ORDER BY pm.completed_at_epoch DESC
|
||||
LIMIT ?
|
||||
`);
|
||||
return stmt.all(cutoff, limit) as (PersistentPendingMessage & { project: string | null })[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark message as being processed (status: pending -> processing)
|
||||
*/
|
||||
markProcessing(messageId: number): void {
|
||||
const now = Date.now();
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'processing', started_processing_at_epoch = ?
|
||||
WHERE id = ? AND status = 'pending'
|
||||
`);
|
||||
stmt.run(now, messageId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark message as successfully processed (status: processing -> processed)
|
||||
*/
|
||||
markProcessed(messageId: number): void {
|
||||
const now = Date.now();
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'processed', completed_at_epoch = ?
|
||||
WHERE id = ? AND status = 'processing'
|
||||
`);
|
||||
stmt.run(now, messageId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark message as failed (status: processing -> failed or back to pending for retry)
|
||||
* If retry_count < maxRetries, moves back to 'pending' for retry
|
||||
* Otherwise marks as 'failed' permanently
|
||||
*/
|
||||
markFailed(messageId: number): void {
|
||||
const now = Date.now();
|
||||
|
||||
// Get current retry count
|
||||
const msg = this.db.prepare('SELECT retry_count FROM pending_messages WHERE id = ?').get(messageId) as { retry_count: number } | undefined;
|
||||
|
||||
if (!msg) return;
|
||||
|
||||
if (msg.retry_count < this.maxRetries) {
|
||||
// Move back to pending for retry
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'pending', retry_count = retry_count + 1, started_processing_at_epoch = NULL
|
||||
WHERE id = ?
|
||||
`);
|
||||
stmt.run(messageId);
|
||||
} else {
|
||||
// Max retries exceeded, mark as permanently failed
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'failed', completed_at_epoch = ?
|
||||
WHERE id = ?
|
||||
`);
|
||||
stmt.run(now, messageId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset stuck messages (processing -> pending if stuck longer than threshold)
|
||||
* @param thresholdMs Messages processing longer than this are considered stuck (0 = reset all)
|
||||
* @returns Number of messages reset
|
||||
*/
|
||||
resetStuckMessages(thresholdMs: number): number {
|
||||
const cutoff = thresholdMs === 0 ? Date.now() : Date.now() - thresholdMs;
|
||||
|
||||
const stmt = this.db.prepare(`
|
||||
UPDATE pending_messages
|
||||
SET status = 'pending', started_processing_at_epoch = NULL
|
||||
WHERE status = 'processing' AND started_processing_at_epoch < ?
|
||||
`);
|
||||
|
||||
const result = stmt.run(cutoff);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of pending messages for a session
|
||||
*/
|
||||
getPendingCount(sessionDbId: number): number {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT COUNT(*) as count FROM pending_messages
|
||||
WHERE session_db_id = ? AND status IN ('pending', 'processing')
|
||||
`);
|
||||
const result = stmt.get(sessionDbId) as { count: number };
|
||||
return result.count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if any session has pending work
|
||||
*/
|
||||
hasAnyPendingWork(): boolean {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT COUNT(*) as count FROM pending_messages
|
||||
WHERE status IN ('pending', 'processing')
|
||||
`);
|
||||
const result = stmt.get() as { count: number };
|
||||
return result.count > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all session IDs that have pending messages (for recovery on startup)
|
||||
*/
|
||||
getSessionsWithPendingMessages(): number[] {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT DISTINCT session_db_id FROM pending_messages
|
||||
WHERE status IN ('pending', 'processing')
|
||||
`);
|
||||
const results = stmt.all() as { session_db_id: number }[];
|
||||
return results.map(r => r.session_db_id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get session info for a pending message (for recovery)
|
||||
*/
|
||||
getSessionInfoForMessage(messageId: number): { sessionDbId: number; claudeSessionId: string } | null {
|
||||
const stmt = this.db.prepare(`
|
||||
SELECT session_db_id, claude_session_id FROM pending_messages WHERE id = ?
|
||||
`);
|
||||
const result = stmt.get(messageId) as { session_db_id: number; claude_session_id: string } | undefined;
|
||||
return result ? { sessionDbId: result.session_db_id, claudeSessionId: result.claude_session_id } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup old processed messages (retention policy)
|
||||
* @param retentionMs Delete processed messages older than this (0 = delete all processed)
|
||||
* @returns Number of messages deleted
|
||||
*/
|
||||
cleanupProcessed(retentionMs: number): number {
|
||||
const cutoff = retentionMs === 0 ? Date.now() : Date.now() - retentionMs;
|
||||
|
||||
const stmt = this.db.prepare(`
|
||||
DELETE FROM pending_messages
|
||||
WHERE status = 'processed' AND completed_at_epoch < ?
|
||||
`);
|
||||
|
||||
const result = stmt.run(cutoff);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a PersistentPendingMessage back to PendingMessage format
|
||||
*/
|
||||
toPendingMessage(persistent: PersistentPendingMessage): PendingMessage {
|
||||
return {
|
||||
type: persistent.message_type,
|
||||
tool_name: persistent.tool_name || undefined,
|
||||
tool_input: persistent.tool_input ? JSON.parse(persistent.tool_input) : undefined,
|
||||
tool_response: persistent.tool_response ? JSON.parse(persistent.tool_response) : undefined,
|
||||
prompt_number: persistent.prompt_number || undefined,
|
||||
cwd: persistent.cwd || undefined,
|
||||
last_user_message: persistent.last_user_message || undefined,
|
||||
last_assistant_message: persistent.last_assistant_message || undefined
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ export class SessionStore {
|
||||
this.makeObservationsTextNullable();
|
||||
this.createUserPromptsTable();
|
||||
this.ensureDiscoveryTokensColumn();
|
||||
this.createPendingMessagesTable();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -545,6 +546,61 @@ export class SessionStore {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create pending_messages table for persistent work queue (migration 16)
|
||||
* Messages are persisted before processing and deleted after success.
|
||||
* Enables recovery from SDK hangs and worker crashes.
|
||||
*/
|
||||
private createPendingMessagesTable(): void {
|
||||
try {
|
||||
// Check if migration already applied
|
||||
const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(16) as SchemaVersion | undefined;
|
||||
if (applied) return;
|
||||
|
||||
// Check if table already exists
|
||||
const tables = this.db.query("SELECT name FROM sqlite_master WHERE type='table' AND name='pending_messages'").all() as TableNameRow[];
|
||||
if (tables.length > 0) {
|
||||
this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(16, new Date().toISOString());
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('[SessionStore] Creating pending_messages table...');
|
||||
|
||||
this.db.run(`
|
||||
CREATE TABLE pending_messages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_db_id INTEGER NOT NULL,
|
||||
claude_session_id TEXT NOT NULL,
|
||||
message_type TEXT NOT NULL CHECK(message_type IN ('observation', 'summarize')),
|
||||
tool_name TEXT,
|
||||
tool_input TEXT,
|
||||
tool_response TEXT,
|
||||
cwd TEXT,
|
||||
last_user_message TEXT,
|
||||
last_assistant_message TEXT,
|
||||
prompt_number INTEGER,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'processing', 'processed', 'failed')),
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at_epoch INTEGER NOT NULL,
|
||||
started_processing_at_epoch INTEGER,
|
||||
completed_at_epoch INTEGER,
|
||||
FOREIGN KEY (session_db_id) REFERENCES sdk_sessions(id) ON DELETE CASCADE
|
||||
)
|
||||
`);
|
||||
|
||||
this.db.run('CREATE INDEX IF NOT EXISTS idx_pending_messages_session ON pending_messages(session_db_id)');
|
||||
this.db.run('CREATE INDEX IF NOT EXISTS idx_pending_messages_status ON pending_messages(status)');
|
||||
this.db.run('CREATE INDEX IF NOT EXISTS idx_pending_messages_claude_session ON pending_messages(claude_session_id)');
|
||||
|
||||
this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(16, new Date().toISOString());
|
||||
|
||||
console.log('[SessionStore] pending_messages table created successfully');
|
||||
} catch (error: any) {
|
||||
console.error('[SessionStore] Pending messages table migration error:', error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent session summaries for a project
|
||||
*/
|
||||
|
||||
@@ -14,13 +14,14 @@ export interface ActiveSession {
|
||||
sdkSessionId: string | null;
|
||||
project: string;
|
||||
userPrompt: string;
|
||||
pendingMessages: PendingMessage[];
|
||||
pendingMessages: PendingMessage[]; // Deprecated: now using persistent store, kept for compatibility
|
||||
abortController: AbortController;
|
||||
generatorPromise: Promise<void> | null;
|
||||
lastPromptNumber: number;
|
||||
startTime: number;
|
||||
cumulativeInputTokens: number; // Track input tokens for discovery cost
|
||||
cumulativeOutputTokens: number; // Track output tokens for discovery cost
|
||||
pendingProcessingIds: Set<number>; // Track ALL message IDs yielded but not yet processed
|
||||
}
|
||||
|
||||
export interface PendingMessage {
|
||||
@@ -34,6 +35,16 @@ export interface PendingMessage {
|
||||
last_assistant_message?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* PendingMessage with database ID for completion tracking.
|
||||
* The _persistentId is used to mark the message as processed after SDK success.
|
||||
* The _originalTimestamp is the epoch when the message was first queued (for accurate observation timestamps).
|
||||
*/
|
||||
export interface PendingMessageWithId extends PendingMessage {
|
||||
_persistentId: number;
|
||||
_originalTimestamp: number;
|
||||
}
|
||||
|
||||
export interface ObservationData {
|
||||
tool_name: string;
|
||||
tool_input: any;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* The installed plugin at ~/.claude/plugins/marketplaces/thedotmack/ is a git repo.
|
||||
*/
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import { execSync, spawnSync } from 'child_process';
|
||||
import { existsSync, unlinkSync } from 'fs';
|
||||
import { homedir } from 'os';
|
||||
import { join } from 'path';
|
||||
@@ -13,6 +13,21 @@ import { logger } from '../../utils/logger.js';
|
||||
|
||||
const INSTALLED_PLUGIN_PATH = join(homedir(), '.claude', 'plugins', 'marketplaces', 'thedotmack');
|
||||
|
||||
/**
|
||||
* Validate branch name to prevent command injection
|
||||
* Only allows alphanumeric, hyphens, underscores, forward slashes, and dots
|
||||
*/
|
||||
function isValidBranchName(branchName: string): boolean {
|
||||
if (!branchName || typeof branchName !== 'string') {
|
||||
return false;
|
||||
}
|
||||
// Git branch name validation: alphanumeric, hyphen, underscore, slash, dot
|
||||
// Must not start with dot, hyphen, or slash
|
||||
// Must not contain double dots (..)
|
||||
const validBranchRegex = /^[a-zA-Z0-9][a-zA-Z0-9._/-]*$/;
|
||||
return validBranchRegex.test(branchName) && !branchName.includes('..');
|
||||
}
|
||||
|
||||
// Timeout constants
|
||||
const GIT_COMMAND_TIMEOUT_MS = 30_000;
|
||||
const NPM_INSTALL_TIMEOUT_MS = 120_000;
|
||||
@@ -35,27 +50,54 @@ export interface SwitchResult {
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute git command in installed plugin directory
|
||||
* Execute git command in installed plugin directory using safe array-based arguments
|
||||
* SECURITY: Uses spawnSync with argument array to prevent command injection
|
||||
*/
|
||||
function execGit(command: string): string {
|
||||
return execSync(`git ${command}`, {
|
||||
function execGit(args: string[]): string {
|
||||
const result = spawnSync('git', args, {
|
||||
cwd: INSTALLED_PLUGIN_PATH,
|
||||
encoding: 'utf-8',
|
||||
timeout: GIT_COMMAND_TIMEOUT_MS,
|
||||
windowsHide: true
|
||||
}).trim();
|
||||
windowsHide: true,
|
||||
shell: false // CRITICAL: Never use shell with user input
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr || result.stdout || 'Git command failed');
|
||||
}
|
||||
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute shell command in installed plugin directory
|
||||
* Execute npm command in installed plugin directory using safe array-based arguments
|
||||
* SECURITY: Uses spawnSync with argument array to prevent command injection
|
||||
*/
|
||||
function execShell(command: string, timeoutMs: number = DEFAULT_SHELL_TIMEOUT_MS): string {
|
||||
return execSync(command, {
|
||||
function execNpm(args: string[], timeoutMs: number = NPM_INSTALL_TIMEOUT_MS): string {
|
||||
const isWindows = process.platform === 'win32';
|
||||
const npmCmd = isWindows ? 'npm.cmd' : 'npm';
|
||||
|
||||
const result = spawnSync(npmCmd, args, {
|
||||
cwd: INSTALLED_PLUGIN_PATH,
|
||||
encoding: 'utf-8',
|
||||
timeout: timeoutMs,
|
||||
windowsHide: true
|
||||
}).trim();
|
||||
windowsHide: true,
|
||||
shell: false // CRITICAL: Never use shell with user input
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr || result.stdout || 'npm command failed');
|
||||
}
|
||||
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -77,10 +119,10 @@ export function getBranchInfo(): BranchInfo {
|
||||
|
||||
try {
|
||||
// Get current branch
|
||||
const branch = execGit('rev-parse --abbrev-ref HEAD');
|
||||
const branch = execGit(['rev-parse', '--abbrev-ref', 'HEAD']);
|
||||
|
||||
// Check if dirty (has uncommitted changes)
|
||||
const status = execGit('status --porcelain');
|
||||
const status = execGit(['status', '--porcelain']);
|
||||
const isDirty = status.length > 0;
|
||||
|
||||
// Determine if on beta branch
|
||||
@@ -118,6 +160,14 @@ export function getBranchInfo(): BranchInfo {
|
||||
* 6. Restart worker (handled by caller after response)
|
||||
*/
|
||||
export async function switchBranch(targetBranch: string): Promise<SwitchResult> {
|
||||
// SECURITY: Validate branch name to prevent command injection
|
||||
if (!isValidBranchName(targetBranch)) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Invalid branch name: ${targetBranch}. Branch names must be alphanumeric with hyphens, underscores, slashes, or dots.`
|
||||
};
|
||||
}
|
||||
|
||||
const info = getBranchInfo();
|
||||
|
||||
if (!info.isGitRepo) {
|
||||
@@ -143,25 +193,25 @@ export async function switchBranch(targetBranch: string): Promise<SwitchResult>
|
||||
|
||||
// 1. Discard local changes (safe - user data is at ~/.claude-mem/)
|
||||
logger.debug('BRANCH', 'Discarding local changes');
|
||||
execGit('checkout -- .');
|
||||
execGit('clean -fd'); // Remove untracked files too
|
||||
execGit(['checkout', '--', '.']);
|
||||
execGit(['clean', '-fd']); // Remove untracked files too
|
||||
|
||||
// 2. Fetch latest
|
||||
logger.debug('BRANCH', 'Fetching from origin');
|
||||
execGit('fetch origin');
|
||||
execGit(['fetch', 'origin']);
|
||||
|
||||
// 3. Checkout target branch
|
||||
logger.debug('BRANCH', 'Checking out branch', { branch: targetBranch });
|
||||
try {
|
||||
execGit(`checkout ${targetBranch}`);
|
||||
execGit(['checkout', targetBranch]);
|
||||
} catch {
|
||||
// Branch might not exist locally, try tracking remote
|
||||
execGit(`checkout -b ${targetBranch} origin/${targetBranch}`);
|
||||
execGit(['checkout', '-b', targetBranch, `origin/${targetBranch}`]);
|
||||
}
|
||||
|
||||
// 4. Pull latest
|
||||
logger.debug('BRANCH', 'Pulling latest');
|
||||
execGit(`pull origin ${targetBranch}`);
|
||||
execGit(['pull', 'origin', targetBranch]);
|
||||
|
||||
// 5. Clear install marker and run npm install
|
||||
const installMarker = join(INSTALLED_PLUGIN_PATH, '.install-version');
|
||||
@@ -170,7 +220,7 @@ export async function switchBranch(targetBranch: string): Promise<SwitchResult>
|
||||
}
|
||||
|
||||
logger.debug('BRANCH', 'Running npm install');
|
||||
execShell('npm install', NPM_INSTALL_TIMEOUT_MS);
|
||||
execNpm(['install'], NPM_INSTALL_TIMEOUT_MS);
|
||||
|
||||
logger.success('BRANCH', 'Branch switch complete', {
|
||||
branch: targetBranch
|
||||
@@ -186,8 +236,8 @@ export async function switchBranch(targetBranch: string): Promise<SwitchResult>
|
||||
|
||||
// Try to recover by checking out original branch
|
||||
try {
|
||||
if (info.branch) {
|
||||
execGit(`checkout ${info.branch}`);
|
||||
if (info.branch && isValidBranchName(info.branch)) {
|
||||
execGit(['checkout', info.branch]);
|
||||
}
|
||||
} catch {
|
||||
// Recovery failed, user needs manual intervention
|
||||
@@ -214,21 +264,29 @@ export async function pullUpdates(): Promise<SwitchResult> {
|
||||
}
|
||||
|
||||
try {
|
||||
// SECURITY: Validate branch name before use
|
||||
if (!isValidBranchName(info.branch)) {
|
||||
return {
|
||||
success: false,
|
||||
error: `Invalid current branch name: ${info.branch}`
|
||||
};
|
||||
}
|
||||
|
||||
logger.info('BRANCH', 'Pulling updates', { branch: info.branch });
|
||||
|
||||
// Discard local changes first
|
||||
execGit('checkout -- .');
|
||||
execGit(['checkout', '--', '.']);
|
||||
|
||||
// Fetch and pull
|
||||
execGit('fetch origin');
|
||||
execGit(`pull origin ${info.branch}`);
|
||||
execGit(['fetch', 'origin']);
|
||||
execGit(['pull', 'origin', info.branch]);
|
||||
|
||||
// Clear install marker and reinstall
|
||||
const installMarker = join(INSTALLED_PLUGIN_PATH, '.install-version');
|
||||
if (existsSync(installMarker)) {
|
||||
unlinkSync(installMarker);
|
||||
}
|
||||
execShell('npm install', NPM_INSTALL_TIMEOUT_MS);
|
||||
execNpm(['install'], NPM_INSTALL_TIMEOUT_MS);
|
||||
|
||||
logger.success('BRANCH', 'Updates pulled', { branch: info.branch });
|
||||
|
||||
|
||||
@@ -396,6 +396,21 @@ export class SDKAgent {
|
||||
}
|
||||
}
|
||||
|
||||
// CRITICAL: Mark ALL pending messages as successfully processed
|
||||
// This prevents message loss if worker crashes before SDK finishes
|
||||
const pendingMessageStore = this.sessionManager.getPendingMessageStore();
|
||||
if (session.pendingProcessingIds.size > 0) {
|
||||
for (const messageId of session.pendingProcessingIds) {
|
||||
pendingMessageStore.markProcessed(messageId);
|
||||
}
|
||||
logger.debug('SDK', 'Messages marked as processed', {
|
||||
sessionId: session.sessionDbId,
|
||||
messageIds: Array.from(session.pendingProcessingIds),
|
||||
count: session.pendingProcessingIds.size
|
||||
});
|
||||
session.pendingProcessingIds.clear();
|
||||
}
|
||||
|
||||
// Broadcast activity status after processing (queue may have changed)
|
||||
if (worker && typeof worker.broadcastProcessingStatus === 'function') {
|
||||
worker.broadcastProcessingStatus();
|
||||
|
||||
@@ -11,18 +11,31 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { DatabaseManager } from './DatabaseManager.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
import type { ActiveSession, PendingMessage, ObservationData } from '../worker-types.js';
|
||||
import type { ActiveSession, PendingMessage, PendingMessageWithId, ObservationData } from '../worker-types.js';
|
||||
import { PendingMessageStore } from '../sqlite/PendingMessageStore.js';
|
||||
|
||||
export class SessionManager {
|
||||
private dbManager: DatabaseManager;
|
||||
private sessions: Map<number, ActiveSession> = new Map();
|
||||
private sessionQueues: Map<number, EventEmitter> = new Map();
|
||||
private onSessionDeletedCallback?: () => void;
|
||||
private pendingStore: PendingMessageStore | null = null;
|
||||
|
||||
constructor(dbManager: DatabaseManager) {
|
||||
this.dbManager = dbManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create PendingMessageStore (lazy initialization to avoid circular dependency)
|
||||
*/
|
||||
private getPendingStore(): PendingMessageStore {
|
||||
if (!this.pendingStore) {
|
||||
const sessionStore = this.dbManager.getSessionStore();
|
||||
this.pendingStore = new PendingMessageStore(sessionStore.db, 3);
|
||||
}
|
||||
return this.pendingStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set callback to be called when a session is deleted (for broadcasting status)
|
||||
*/
|
||||
@@ -103,7 +116,8 @@ export class SessionManager {
|
||||
lastPromptNumber: promptNumber || this.dbManager.getSessionStore().getPromptCounter(sessionDbId),
|
||||
startTime: Date.now(),
|
||||
cumulativeInputTokens: 0,
|
||||
cumulativeOutputTokens: 0
|
||||
cumulativeOutputTokens: 0,
|
||||
pendingProcessingIds: new Set()
|
||||
};
|
||||
|
||||
this.sessions.set(sessionDbId, session);
|
||||
@@ -133,6 +147,9 @@ export class SessionManager {
|
||||
/**
|
||||
* Queue an observation for processing (zero-latency notification)
|
||||
* Auto-initializes session if not in memory but exists in database
|
||||
*
|
||||
* CRITICAL: Persists to database FIRST before adding to in-memory queue.
|
||||
* This ensures observations survive worker crashes.
|
||||
*/
|
||||
queueObservation(sessionDbId: number, data: ObservationData): void {
|
||||
// Auto-initialize from database if needed (handles worker restarts)
|
||||
@@ -143,14 +160,33 @@ export class SessionManager {
|
||||
|
||||
const beforeDepth = session.pendingMessages.length;
|
||||
|
||||
session.pendingMessages.push({
|
||||
// CRITICAL: Persist to database FIRST
|
||||
const message: PendingMessage = {
|
||||
type: 'observation',
|
||||
tool_name: data.tool_name,
|
||||
tool_input: data.tool_input,
|
||||
tool_response: data.tool_response,
|
||||
prompt_number: data.prompt_number,
|
||||
cwd: data.cwd
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const messageId = this.getPendingStore().enqueue(sessionDbId, session.claudeSessionId, message);
|
||||
logger.debug('SESSION', `Observation persisted to DB`, {
|
||||
sessionId: sessionDbId,
|
||||
messageId,
|
||||
tool: data.tool_name
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('SESSION', 'Failed to persist observation to DB', {
|
||||
sessionId: sessionDbId,
|
||||
tool: data.tool_name
|
||||
}, error);
|
||||
throw error; // Don't continue if we can't persist
|
||||
}
|
||||
|
||||
// Add to in-memory queue (for backward compatibility with existing iterator)
|
||||
session.pendingMessages.push(message);
|
||||
|
||||
const afterDepth = session.pendingMessages.length;
|
||||
|
||||
@@ -171,6 +207,9 @@ export class SessionManager {
|
||||
/**
|
||||
* Queue a summarize request (zero-latency notification)
|
||||
* Auto-initializes session if not in memory but exists in database
|
||||
*
|
||||
* CRITICAL: Persists to database FIRST before adding to in-memory queue.
|
||||
* This ensures summarize requests survive worker crashes.
|
||||
*/
|
||||
queueSummarize(sessionDbId: number, lastUserMessage: string, lastAssistantMessage?: string): void {
|
||||
// Auto-initialize from database if needed (handles worker restarts)
|
||||
@@ -181,11 +220,28 @@ export class SessionManager {
|
||||
|
||||
const beforeDepth = session.pendingMessages.length;
|
||||
|
||||
session.pendingMessages.push({
|
||||
// CRITICAL: Persist to database FIRST
|
||||
const message: PendingMessage = {
|
||||
type: 'summarize',
|
||||
last_user_message: lastUserMessage,
|
||||
last_assistant_message: lastAssistantMessage
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const messageId = this.getPendingStore().enqueue(sessionDbId, session.claudeSessionId, message);
|
||||
logger.debug('SESSION', `Summarize persisted to DB`, {
|
||||
sessionId: sessionDbId,
|
||||
messageId
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('SESSION', 'Failed to persist summarize to DB', {
|
||||
sessionId: sessionDbId
|
||||
}, error);
|
||||
throw error; // Don't continue if we can't persist
|
||||
}
|
||||
|
||||
// Add to in-memory queue (for backward compatibility with existing iterator)
|
||||
session.pendingMessages.push(message);
|
||||
|
||||
const afterDepth = session.pendingMessages.length;
|
||||
|
||||
@@ -306,8 +362,12 @@ export class SessionManager {
|
||||
/**
|
||||
* Get message iterator for SDKAgent to consume (event-driven, no polling)
|
||||
* Auto-initializes session if not in memory but exists in database
|
||||
*
|
||||
* CRITICAL: Uses PendingMessageStore for crash-safe message persistence.
|
||||
* Messages are marked as 'processing' when yielded and must be marked 'processed'
|
||||
* by the SDK agent after successful completion.
|
||||
*/
|
||||
async *getMessageIterator(sessionDbId: number): AsyncIterableIterator<PendingMessage> {
|
||||
async *getMessageIterator(sessionDbId: number): AsyncIterableIterator<PendingMessageWithId> {
|
||||
// Auto-initialize from database if needed (handles worker restarts)
|
||||
let session = this.sessions.get(sessionDbId);
|
||||
if (!session) {
|
||||
@@ -319,32 +379,100 @@ export class SessionManager {
|
||||
throw new Error(`No emitter for session ${sessionDbId}`);
|
||||
}
|
||||
|
||||
// Linger timeout: how long to wait for new messages before exiting
|
||||
// This keeps the agent alive between messages, reducing "No active agent" windows
|
||||
const LINGER_TIMEOUT_MS = 5000; // 5 seconds
|
||||
|
||||
while (!session.abortController.signal.aborted) {
|
||||
// Wait for messages if queue is empty
|
||||
if (session.pendingMessages.length === 0) {
|
||||
await new Promise<void>(resolve => {
|
||||
const handler = () => resolve();
|
||||
emitter.once('message', handler);
|
||||
// Check for pending messages in persistent store
|
||||
const persistentMessage = this.getPendingStore().peekPending(sessionDbId);
|
||||
|
||||
if (!persistentMessage) {
|
||||
// Wait for new messages with timeout
|
||||
const gotMessage = await new Promise<boolean>(resolve => {
|
||||
let resolved = false;
|
||||
|
||||
const messageHandler = () => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
clearTimeout(timeoutId);
|
||||
resolve(true);
|
||||
}
|
||||
};
|
||||
|
||||
const timeoutHandler = () => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
emitter.off('message', messageHandler);
|
||||
resolve(false);
|
||||
}
|
||||
};
|
||||
|
||||
const timeoutId = setTimeout(timeoutHandler, LINGER_TIMEOUT_MS);
|
||||
|
||||
emitter.once('message', messageHandler);
|
||||
|
||||
// Also listen for abort
|
||||
session.abortController.signal.addEventListener('abort', () => {
|
||||
emitter.off('message', handler);
|
||||
resolve();
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
clearTimeout(timeoutId);
|
||||
emitter.off('message', messageHandler);
|
||||
resolve(false);
|
||||
}
|
||||
}, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
// Yield all pending messages
|
||||
while (session.pendingMessages.length > 0) {
|
||||
const message = session.pendingMessages.shift()!;
|
||||
yield message;
|
||||
// Re-check for messages after waking up (handles race condition)
|
||||
const recheckMessage = this.getPendingStore().peekPending(sessionDbId);
|
||||
if (recheckMessage) {
|
||||
// Got a message, continue processing
|
||||
continue;
|
||||
}
|
||||
|
||||
// If we just yielded a summary, that's the end of this batch - stop the iterator
|
||||
if (message.type === 'summarize') {
|
||||
logger.info('SESSION', `Summary yielded - ending generator`, { sessionId: sessionDbId });
|
||||
if (!gotMessage) {
|
||||
// Timeout or abort - exit the loop
|
||||
logger.info('SESSION', `Generator exiting after linger timeout`, { sessionId: sessionDbId });
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Mark as processing BEFORE yielding (status: pending -> processing)
|
||||
this.getPendingStore().markProcessing(persistentMessage.id);
|
||||
|
||||
// Track this message ID for completion marking
|
||||
session.pendingProcessingIds.add(persistentMessage.id);
|
||||
|
||||
// Convert to PendingMessageWithId and yield
|
||||
// Include original timestamp for accurate observation timestamps (survives stuck processing)
|
||||
const message: PendingMessageWithId = {
|
||||
_persistentId: persistentMessage.id,
|
||||
_originalTimestamp: persistentMessage.created_at_epoch,
|
||||
...this.getPendingStore().toPendingMessage(persistentMessage)
|
||||
};
|
||||
|
||||
// Also add to in-memory queue for backward compatibility (status tracking)
|
||||
session.pendingMessages.push(message);
|
||||
|
||||
yield message;
|
||||
|
||||
// Remove from in-memory queue after yielding
|
||||
session.pendingMessages.shift();
|
||||
|
||||
// If we just yielded a summary, that's the end of this batch - stop the iterator
|
||||
if (message.type === 'summarize') {
|
||||
logger.info('SESSION', `Summary yielded - ending generator`, { sessionId: sessionDbId });
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the PendingMessageStore (for SDKAgent to mark messages as processed)
|
||||
*/
|
||||
getPendingMessageStore(): PendingMessageStore {
|
||||
return this.getPendingStore();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ export function getBunPath(): string | null {
|
||||
const result = spawnSync('bun', ['--version'], {
|
||||
encoding: 'utf-8',
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
shell: isWindows
|
||||
shell: false // SECURITY: No need for shell, bun is the executable
|
||||
});
|
||||
if (result.status === 0) {
|
||||
return 'bun'; // Available in PATH
|
||||
|
||||
Reference in New Issue
Block a user