backup: Phase 1 agent work (security, persistence, batch endpoint)
This is a backup of all work done by the 3 Phase 1 agents: Agent A - Command Injection Fix (Issue #354): - Fixed command injection in BranchManager.ts - Fixed unnecessary shell usage in bun-path.ts - Added comprehensive security test suite - Created SECURITY.md and SECURITY_AUDIT_REPORT.md Agent B - Observation Persistence Fix (Issue #353): - Added PendingMessageStore from PR #335 - Integrated persistent queue into SessionManager - Modified SDKAgent to mark messages complete - Updated SessionStore with pending_messages migration - Updated worker-types.ts with new interfaces Agent C - Batch Endpoint Verification (Issue #348): - Created batch-observations.test.ts - Updated worker-service.mdx documentation Also includes: - Documentation context files (biomimetic, windows struggles) - Build artifacts from agent testing This work will be re-evaluated after v7.3.0 release.
This commit is contained in:
Alex Newman
@@ -11,18 +11,31 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { DatabaseManager } from './DatabaseManager.js';
|
||||
import { logger } from '../../utils/logger.js';
|
||||
import type { ActiveSession, PendingMessage, ObservationData } from '../worker-types.js';
|
||||
import type { ActiveSession, PendingMessage, PendingMessageWithId, ObservationData } from '../worker-types.js';
|
||||
import { PendingMessageStore } from '../sqlite/PendingMessageStore.js';
|
||||
|
||||
export class SessionManager {
|
||||
private dbManager: DatabaseManager;
|
||||
private sessions: Map<number, ActiveSession> = new Map();
|
||||
private sessionQueues: Map<number, EventEmitter> = new Map();
|
||||
private onSessionDeletedCallback?: () => void;
|
||||
private pendingStore: PendingMessageStore | null = null;
|
||||
|
||||
constructor(dbManager: DatabaseManager) {
|
||||
this.dbManager = dbManager;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create PendingMessageStore (lazy initialization to avoid circular dependency)
|
||||
*/
|
||||
private getPendingStore(): PendingMessageStore {
|
||||
if (!this.pendingStore) {
|
||||
const sessionStore = this.dbManager.getSessionStore();
|
||||
this.pendingStore = new PendingMessageStore(sessionStore.db, 3);
|
||||
}
|
||||
return this.pendingStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set callback to be called when a session is deleted (for broadcasting status)
|
||||
*/
|
||||
@@ -103,7 +116,8 @@ export class SessionManager {
|
||||
lastPromptNumber: promptNumber || this.dbManager.getSessionStore().getPromptCounter(sessionDbId),
|
||||
startTime: Date.now(),
|
||||
cumulativeInputTokens: 0,
|
||||
cumulativeOutputTokens: 0
|
||||
cumulativeOutputTokens: 0,
|
||||
pendingProcessingIds: new Set()
|
||||
};
|
||||
|
||||
this.sessions.set(sessionDbId, session);
|
||||
@@ -133,6 +147,9 @@ export class SessionManager {
|
||||
/**
|
||||
* Queue an observation for processing (zero-latency notification)
|
||||
* Auto-initializes session if not in memory but exists in database
|
||||
*
|
||||
* CRITICAL: Persists to database FIRST before adding to in-memory queue.
|
||||
* This ensures observations survive worker crashes.
|
||||
*/
|
||||
queueObservation(sessionDbId: number, data: ObservationData): void {
|
||||
// Auto-initialize from database if needed (handles worker restarts)
|
||||
@@ -143,14 +160,33 @@ export class SessionManager {
|
||||
|
||||
const beforeDepth = session.pendingMessages.length;
|
||||
|
||||
session.pendingMessages.push({
|
||||
// CRITICAL: Persist to database FIRST
|
||||
const message: PendingMessage = {
|
||||
type: 'observation',
|
||||
tool_name: data.tool_name,
|
||||
tool_input: data.tool_input,
|
||||
tool_response: data.tool_response,
|
||||
prompt_number: data.prompt_number,
|
||||
cwd: data.cwd
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const messageId = this.getPendingStore().enqueue(sessionDbId, session.claudeSessionId, message);
|
||||
logger.debug('SESSION', `Observation persisted to DB`, {
|
||||
sessionId: sessionDbId,
|
||||
messageId,
|
||||
tool: data.tool_name
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('SESSION', 'Failed to persist observation to DB', {
|
||||
sessionId: sessionDbId,
|
||||
tool: data.tool_name
|
||||
}, error);
|
||||
throw error; // Don't continue if we can't persist
|
||||
}
|
||||
|
||||
// Add to in-memory queue (for backward compatibility with existing iterator)
|
||||
session.pendingMessages.push(message);
|
||||
|
||||
const afterDepth = session.pendingMessages.length;
|
||||
|
||||
@@ -171,6 +207,9 @@ export class SessionManager {
|
||||
/**
|
||||
* Queue a summarize request (zero-latency notification)
|
||||
* Auto-initializes session if not in memory but exists in database
|
||||
*
|
||||
* CRITICAL: Persists to database FIRST before adding to in-memory queue.
|
||||
* This ensures summarize requests survive worker crashes.
|
||||
*/
|
||||
queueSummarize(sessionDbId: number, lastUserMessage: string, lastAssistantMessage?: string): void {
|
||||
// Auto-initialize from database if needed (handles worker restarts)
|
||||
@@ -181,11 +220,28 @@ export class SessionManager {
|
||||
|
||||
const beforeDepth = session.pendingMessages.length;
|
||||
|
||||
session.pendingMessages.push({
|
||||
// CRITICAL: Persist to database FIRST
|
||||
const message: PendingMessage = {
|
||||
type: 'summarize',
|
||||
last_user_message: lastUserMessage,
|
||||
last_assistant_message: lastAssistantMessage
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const messageId = this.getPendingStore().enqueue(sessionDbId, session.claudeSessionId, message);
|
||||
logger.debug('SESSION', `Summarize persisted to DB`, {
|
||||
sessionId: sessionDbId,
|
||||
messageId
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('SESSION', 'Failed to persist summarize to DB', {
|
||||
sessionId: sessionDbId
|
||||
}, error);
|
||||
throw error; // Don't continue if we can't persist
|
||||
}
|
||||
|
||||
// Add to in-memory queue (for backward compatibility with existing iterator)
|
||||
session.pendingMessages.push(message);
|
||||
|
||||
const afterDepth = session.pendingMessages.length;
|
||||
|
||||
@@ -306,8 +362,12 @@ export class SessionManager {
|
||||
/**
|
||||
* Get message iterator for SDKAgent to consume (event-driven, no polling)
|
||||
* Auto-initializes session if not in memory but exists in database
|
||||
*
|
||||
* CRITICAL: Uses PendingMessageStore for crash-safe message persistence.
|
||||
* Messages are marked as 'processing' when yielded and must be marked 'processed'
|
||||
* by the SDK agent after successful completion.
|
||||
*/
|
||||
async *getMessageIterator(sessionDbId: number): AsyncIterableIterator<PendingMessage> {
|
||||
async *getMessageIterator(sessionDbId: number): AsyncIterableIterator<PendingMessageWithId> {
|
||||
// Auto-initialize from database if needed (handles worker restarts)
|
||||
let session = this.sessions.get(sessionDbId);
|
||||
if (!session) {
|
||||
@@ -319,32 +379,100 @@ export class SessionManager {
|
||||
throw new Error(`No emitter for session ${sessionDbId}`);
|
||||
}
|
||||
|
||||
// Linger timeout: how long to wait for new messages before exiting
|
||||
// This keeps the agent alive between messages, reducing "No active agent" windows
|
||||
const LINGER_TIMEOUT_MS = 5000; // 5 seconds
|
||||
|
||||
while (!session.abortController.signal.aborted) {
|
||||
// Wait for messages if queue is empty
|
||||
if (session.pendingMessages.length === 0) {
|
||||
await new Promise<void>(resolve => {
|
||||
const handler = () => resolve();
|
||||
emitter.once('message', handler);
|
||||
// Check for pending messages in persistent store
|
||||
const persistentMessage = this.getPendingStore().peekPending(sessionDbId);
|
||||
|
||||
if (!persistentMessage) {
|
||||
// Wait for new messages with timeout
|
||||
const gotMessage = await new Promise<boolean>(resolve => {
|
||||
let resolved = false;
|
||||
|
||||
const messageHandler = () => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
clearTimeout(timeoutId);
|
||||
resolve(true);
|
||||
}
|
||||
};
|
||||
|
||||
const timeoutHandler = () => {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
emitter.off('message', messageHandler);
|
||||
resolve(false);
|
||||
}
|
||||
};
|
||||
|
||||
const timeoutId = setTimeout(timeoutHandler, LINGER_TIMEOUT_MS);
|
||||
|
||||
emitter.once('message', messageHandler);
|
||||
|
||||
// Also listen for abort
|
||||
session.abortController.signal.addEventListener('abort', () => {
|
||||
emitter.off('message', handler);
|
||||
resolve();
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
clearTimeout(timeoutId);
|
||||
emitter.off('message', messageHandler);
|
||||
resolve(false);
|
||||
}
|
||||
}, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
// Yield all pending messages
|
||||
while (session.pendingMessages.length > 0) {
|
||||
const message = session.pendingMessages.shift()!;
|
||||
yield message;
|
||||
// Re-check for messages after waking up (handles race condition)
|
||||
const recheckMessage = this.getPendingStore().peekPending(sessionDbId);
|
||||
if (recheckMessage) {
|
||||
// Got a message, continue processing
|
||||
continue;
|
||||
}
|
||||
|
||||
// If we just yielded a summary, that's the end of this batch - stop the iterator
|
||||
if (message.type === 'summarize') {
|
||||
logger.info('SESSION', `Summary yielded - ending generator`, { sessionId: sessionDbId });
|
||||
if (!gotMessage) {
|
||||
// Timeout or abort - exit the loop
|
||||
logger.info('SESSION', `Generator exiting after linger timeout`, { sessionId: sessionDbId });
|
||||
return;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Mark as processing BEFORE yielding (status: pending -> processing)
|
||||
this.getPendingStore().markProcessing(persistentMessage.id);
|
||||
|
||||
// Track this message ID for completion marking
|
||||
session.pendingProcessingIds.add(persistentMessage.id);
|
||||
|
||||
// Convert to PendingMessageWithId and yield
|
||||
// Include original timestamp for accurate observation timestamps (survives stuck processing)
|
||||
const message: PendingMessageWithId = {
|
||||
_persistentId: persistentMessage.id,
|
||||
_originalTimestamp: persistentMessage.created_at_epoch,
|
||||
...this.getPendingStore().toPendingMessage(persistentMessage)
|
||||
};
|
||||
|
||||
// Also add to in-memory queue for backward compatibility (status tracking)
|
||||
session.pendingMessages.push(message);
|
||||
|
||||
yield message;
|
||||
|
||||
// Remove from in-memory queue after yielding
|
||||
session.pendingMessages.shift();
|
||||
|
||||
// If we just yielded a summary, that's the end of this batch - stop the iterator
|
||||
if (message.type === 'summarize') {
|
||||
logger.info('SESSION', `Summary yielded - ending generator`, { sessionId: sessionDbId });
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the PendingMessageStore (for SDKAgent to mark messages as processed)
|
||||
*/
|
||||
getPendingMessageStore(): PendingMessageStore {
|
||||
return this.getPendingStore();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user