data pipeline refactor and fix

This commit is contained in:
2026-04-13 18:30:04 -04:00
parent 6418729b16
commit 326bf80846
96 changed files with 7107 additions and 1763 deletions

View File

@@ -72,7 +72,7 @@ export class Authenticator {
);
}
const sessionId = `ws_${userId}_${Date.now()}`;
const sessionId = `ws_${userId}`;
return {
authContext: {

View File

@@ -2,12 +2,14 @@ import type { FastifyInstance, FastifyRequest } from 'fastify';
import type { WebSocket } from '@fastify/websocket';
import type { Authenticator } from '../auth/authenticator.js';
import type { AgentHarness, HarnessFactory } from '../harness/agent-harness.js';
import type { HarnessEvent } from '../harness/harness-events.js';
import type { InboundMessage } from '../types/messages.js';
import { randomUUID } from 'crypto';
import type { SessionRegistry, EventSubscriber, Session } from '../events/index.js';
import type { OHLCService } from '../services/ohlc-service.js';
import type { OHLCService, BarUpdateCallback } from '../services/ohlc-service.js';
import type { SymbolIndexService } from '../services/symbol-index-service.js';
import type { ContainerManager } from '../k8s/container-manager.js';
import type { ConversationService } from '../services/conversation-service.js';
import {
WorkspaceManager,
ContainerSync,
@@ -42,6 +44,7 @@ export interface WebSocketHandlerConfig {
createHarness: HarnessFactory;
ohlcService?: OHLCService; // Optional for historical data support
symbolIndexService?: SymbolIndexService; // Optional for symbol search
conversationService?: ConversationService; // Optional for history replay on reconnect
}
/**
@@ -50,10 +53,18 @@ export interface WebSocketHandlerConfig {
* Handles WebSocket connections for chat and integrates with the event system
* for container-to-client notifications.
*/
interface BarSubscription {
ticker: string;
periodSeconds: number;
callback: BarUpdateCallback;
}
export class WebSocketHandler {
private config: WebSocketHandlerConfig;
private harnesses = new Map<string, AgentHarness>();
private workspaces = new Map<string, WorkspaceManager>();
/** Per-session realtime bar subscriptions for cleanup on disconnect */
private barSubscriptions = new Map<string, BarSubscription[]>();
constructor(config: WebSocketHandlerConfig) {
this.config = config;
@@ -106,17 +117,22 @@ export class WebSocketHandler {
// If container is spinning up, wait for it to be ready before continuing
if (isSpinningUp) {
sendStatus(socket, 'spinning_up', 'Your workspace is starting up, please wait...');
sendStatus(socket, 'spinning_up', 'Your personal agent is starting up, please wait...');
const startupPingInterval = setInterval(() => {
if (socket.readyState === 1) socket.ping();
}, 10000);
const ready = await this.config.containerManager.waitForContainerReady(authContext.userId, 120000);
clearInterval(startupPingInterval);
if (!ready) {
logger.warn({ userId: authContext.userId }, 'Container failed to become ready within timeout');
socket.send(JSON.stringify({ type: 'error', message: 'Workspace failed to start. Please try again later.' }));
logger.warn({ userId: authContext.userId }, 'Sandbox failed to become ready within timeout');
socket.send(JSON.stringify({ type: 'error', message: 'Agent workspace failed to start. Please try again later.' }));
socket.close(1011, 'Container startup timeout');
return;
}
logger.info({ userId: authContext.userId }, 'Container is ready, proceeding with session setup');
logger.info({ userId: authContext.userId }, 'Sandbox is ready, proceeding with session setup');
}
sendStatus(socket, 'initializing', 'Starting your workspace...');
@@ -241,6 +257,17 @@ export class WebSocketHandler {
})
);
// Replay conversation history so the UI pre-populates on reconnect
if (this.config.conversationService) {
const history = await this.config.conversationService.getHistory(
authContext.userId,
authContext.sessionId
);
if (history.length > 0) {
socket.send(JSON.stringify({ type: 'conversation_history', messages: history }));
}
}
// Handle messages
socket.on('message', async (data: Buffer) => {
try {
@@ -266,15 +293,45 @@ export class WebSocketHandler {
return;
}
// Chunks are streamed via channelAdapter.sendChunk() during handleMessage
try {
// Acknowledge receipt immediately so the client can show the seen indicator
socket.send(JSON.stringify({ type: 'agent_chunk', content: '', done: false }));
logger.info('Calling harness.handleMessage');
await harness.handleMessage(inboundMessage);
logger.info('Streaming harness response');
let fatalError = false;
for await (const event of harness.streamMessage(inboundMessage)) {
const e = event as HarnessEvent;
switch (e.type) {
case 'chunk':
socket.send(JSON.stringify({ type: 'agent_chunk', content: e.content, done: false }));
break;
case 'tool_call':
socket.send(JSON.stringify({ type: 'agent_tool_call', toolName: e.toolName, label: e.label }));
break;
case 'subagent_tool_call':
socket.send(JSON.stringify({ type: 'subagent_tool_call', agentName: e.agentName, toolName: e.toolName, label: e.label }));
break;
case 'subagent_chunk':
socket.send(JSON.stringify({ type: 'subagent_chunk', agentName: e.agentName, content: e.content }));
break;
case 'image':
socket.send(JSON.stringify({ type: 'image', data: e.data, mimeType: e.mimeType, caption: e.caption }));
break;
case 'error':
socket.send(JSON.stringify({ type: 'text', text: `An unrecoverable error occurred in the ${e.source}.` }));
if (e.fatal) fatalError = true;
break;
case 'done':
break;
}
}
// Send done marker after all chunks have been streamed
if (fatalError) {
socket.close(1011, 'Fatal error');
return;
}
// Send done marker after all events have been streamed
logger.debug('Sending done marker to client');
socket.send(
JSON.stringify({
@@ -332,6 +389,17 @@ export class WebSocketHandler {
await this.config.eventSubscriber.onSessionDisconnect(removedSession);
}
// Cleanup realtime bar subscriptions
const sessionId = authContext.sessionId;
const subs = this.barSubscriptions.get(sessionId);
if (subs && this.config.ohlcService) {
for (const { ticker, periodSeconds, callback } of subs) {
this.config.ohlcService.unsubscribeFromTicker(ticker, periodSeconds, callback);
}
this.barSubscriptions.delete(sessionId);
logger.info({ sessionId, count: subs.length }, 'Cleaned up realtime bar subscriptions');
}
// Cleanup workspace
await workspace!.shutdown();
this.workspaces.delete(authContext.sessionId);
@@ -356,6 +424,7 @@ export class WebSocketHandler {
}, 30000);
} catch (error) {
logger.error({ error }, 'Failed to initialize session');
socket.send(JSON.stringify({ type: 'text', text: 'An unrecoverable error occurred in the agent harness.' }));
socket.close(1011, 'Internal server error');
if (workspace) {
await workspace.shutdown();
@@ -527,19 +596,92 @@ export class WebSocketHandler {
break;
}
case 'subscribe_bars':
case 'unsubscribe_bars':
// TODO: Implement real-time subscriptions
socket.send(
JSON.stringify({
type: `${payload.type}_response`,
case 'subscribe_bars': {
if (!ohlcService || !authContext) {
socket.send(JSON.stringify({
type: 'subscribe_bars_response',
request_id: requestId,
subscription_id: payload.subscription_id,
success: false,
message: 'Real-time subscriptions not yet implemented',
})
);
message: 'Realtime service not available',
}));
break;
}
const subTicker: string = payload.symbol;
const subPeriod: number = payload.period_seconds ?? payload.resolution ?? 60;
const sessionId = authContext.sessionId;
// Create a per-subscription callback that forwards bars to this socket
const barCallback: BarUpdateCallback = (bar) => {
if (socket.readyState !== 1 /* OPEN */) return;
socket.send(JSON.stringify({
type: 'bar_update',
subscription_id: payload.subscription_id,
ticker: bar.ticker,
period_seconds: bar.periodSeconds,
bar: {
// Convert nanoseconds → seconds for client compatibility
time: Number(bar.timestamp / 1_000_000_000n),
open: bar.open,
high: bar.high,
low: bar.low,
close: bar.close,
volume: bar.volume,
},
}));
};
ohlcService.subscribeToTicker(subTicker, subPeriod, barCallback);
// Track for cleanup on disconnect
if (!this.barSubscriptions.has(sessionId)) {
this.barSubscriptions.set(sessionId, []);
}
this.barSubscriptions.get(sessionId)!.push({
ticker: subTicker,
periodSeconds: subPeriod,
callback: barCallback,
});
logger.info({ sessionId, ticker: subTicker, period: subPeriod }, 'Subscribed to realtime bars');
socket.send(JSON.stringify({
type: 'subscribe_bars_response',
request_id: requestId,
subscription_id: payload.subscription_id,
success: true,
}));
break;
}
case 'unsubscribe_bars': {
if (!ohlcService || !authContext) break;
const unsubTicker: string = payload.symbol;
const unsubPeriod: number = payload.period_seconds ?? payload.resolution ?? 60;
const sessionId = authContext.sessionId;
const subs = this.barSubscriptions.get(sessionId);
if (subs) {
const idx = subs.findIndex(
s => s.ticker === unsubTicker && s.periodSeconds === unsubPeriod
);
if (idx >= 0) {
const [removed] = subs.splice(idx, 1);
ohlcService.unsubscribeFromTicker(unsubTicker, unsubPeriod, removed.callback);
logger.info({ sessionId, ticker: unsubTicker, period: unsubPeriod }, 'Unsubscribed from realtime bars');
}
}
socket.send(JSON.stringify({
type: 'unsubscribe_bars_response',
request_id: requestId,
subscription_id: payload.subscription_id,
success: true,
}));
break;
}
case 'evaluate_indicator': {
// Direct MCP call — bypasses the agent/LLM for performance

View File

@@ -632,6 +632,118 @@ export class DuckDBClient {
}
}
/**
* Append a batch of image/audio blobs as a Parquet file in S3.
* Called once per assistant turn that produces binary output.
*/
async appendBlobs(
userId: string,
sessionId: string,
messageId: string,
blobs: Array<{
id: string;
user_id: string;
session_id: string;
message_id: string;
blob_type: string;
mime_type: string;
data: string;
caption: string | null;
timestamp: number;
}>
): Promise<void> {
await this.initialize();
if (!this.conversationsBucket || blobs.length === 0) {
return;
}
const now = new Date();
const year = now.getUTCFullYear();
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
const s3Path = `s3://${this.conversationsBucket}/gateway/blobs/year=${year}/month=${month}/user_id=${userId}/${sessionId}_${messageId}.parquet`;
const tempTable = `blob_flush_${Date.now()}`;
try {
await this.query(`
CREATE TEMP TABLE ${tempTable} (
id VARCHAR,
user_id VARCHAR,
session_id VARCHAR,
message_id VARCHAR,
blob_type VARCHAR,
mime_type VARCHAR,
data VARCHAR,
caption VARCHAR,
timestamp BIGINT
)
`);
for (const blob of blobs) {
await this.query(
`INSERT INTO ${tempTable} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[blob.id, blob.user_id, blob.session_id, blob.message_id, blob.blob_type, blob.mime_type, blob.data, blob.caption, blob.timestamp]
);
}
await this.query(`COPY ${tempTable} TO '${s3Path}' (FORMAT PARQUET)`);
this.logger.info({ userId, sessionId, messageId, count: blobs.length, s3Path }, 'Blobs flushed to Parquet');
} finally {
await this.query(`DROP TABLE IF EXISTS ${tempTable}`).catch(() => {});
}
}
/**
* Query blobs from S3 by userId/sessionId, optionally filtered to specific blob IDs.
*/
async queryBlobs(
userId: string,
sessionId: string,
blobIds?: string[]
): Promise<any[]> {
await this.initialize();
try {
const tablePath = await this.getTablePath(this.namespace, 'blobs', this.catalogUri);
if (!tablePath) {
// Fallback: scan per-turn Parquet files written directly to S3
if (this.conversationsBucket) {
this.logger.debug({ userId, sessionId }, 'REST catalog miss, scanning blob Parquet files');
const parquetPath = `s3://${this.conversationsBucket}/gateway/blobs/**/user_id=${userId}/${sessionId}_*.parquet`;
const idClause = blobIds?.length
? `WHERE id IN (${blobIds.map(id => `'${id.replace(/'/g, "''")}'`).join(', ')})`
: '';
try {
return await this.query(`SELECT * FROM read_parquet('${parquetPath}') ${idClause} ORDER BY timestamp ASC`);
} catch {
// No blobs yet for this session
}
}
return [];
}
const idFilter = blobIds?.length
? `AND id IN (${blobIds.map(() => '?').join(', ')})`
: '';
const params: any[] = [userId, sessionId, ...(blobIds ?? [])];
const sql = `
SELECT id, user_id, session_id, message_id, blob_type, mime_type, data, caption, timestamp
FROM iceberg_scan('${tablePath}')
WHERE user_id = ? AND session_id = ? ${idFilter}
ORDER BY timestamp ASC
`;
const rows = await this.query(sql, params);
this.logger.info({ userId, sessionId, count: rows.length }, 'Loaded blobs from Iceberg');
return rows.map((row: any) => ({ ...row, timestamp: Number(row.timestamp) }));
} catch (error: any) {
this.logger.error({ error: error.message, userId, sessionId }, 'Failed to query blobs');
return [];
}
}
/**
* Close the DuckDB connection
*/

View File

@@ -45,6 +45,21 @@ export interface IcebergMessage {
timestamp: number; // nanoseconds
}
/**
* Blob record for Iceberg storage (images, audio, etc.)
*/
export interface IcebergBlob {
id: string;
user_id: string;
session_id: string;
message_id: string;
blob_type: string;
mime_type: string;
data: string; // base64
caption: string | null;
timestamp: number; // microseconds
}
/**
* Checkpoint record for Iceberg storage
*/
@@ -153,6 +168,25 @@ export class IcebergClient {
return this.duckdb.appendMessages(userId, sessionId, messages);
}
/**
* Append blobs for one assistant turn as a Parquet file in S3.
*/
async appendBlobs(
userId: string,
sessionId: string,
messageId: string,
blobs: IcebergBlob[]
): Promise<void> {
return this.duckdb.appendBlobs(userId, sessionId, messageId, blobs);
}
/**
* Query blobs from S3/Iceberg, optionally filtered to specific blob IDs.
*/
async queryBlobs(userId: string, sessionId: string, blobIds?: string[]): Promise<IcebergBlob[]> {
return this.duckdb.queryBlobs(userId, sessionId, blobIds);
}
/**
* Get table metadata
*/

View File

@@ -298,6 +298,13 @@ export class QdrantClient {
pointsCount: info.points_count || 0,
};
} catch (error) {
// If the collection was lost (e.g. Qdrant restarted without the gateway restarting),
// recreate it and return zeroed stats rather than propagating the error.
if ((error as any)?.status === 404) {
this.logger.warn({ collection: this.collectionName }, 'Collection missing, recreating...');
await this.initialize();
return { vectorsCount: 0, indexedVectorsCount: 0, pointsCount: 0 };
}
this.logger.error({ error }, 'Failed to get collection info');
throw error;
}

View File

@@ -20,6 +20,22 @@ import type {
NotificationStatus,
} from '../types/ohlc.js';
export const OHLC_BAR_TOPIC_PATTERN = /^(.+)\|ohlc:(\d+)$/;
/** Decoded realtime OHLC bar received from the XPUB market data stream */
export interface RealtimeBar {
topic: string; // e.g., "BTC/USDT.BINANCE|ohlc:60"
ticker: string; // e.g., "BTC/USDT.BINANCE"
periodSeconds: number;
/** Window open time in nanoseconds since epoch */
timestamp: bigint;
open: number;
high: number;
low: number;
close: number;
volume: number;
}
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -39,14 +55,17 @@ export enum MessageType {
const protoDir = join(__dirname, '../..', 'protobuf');
const root = new protobuf.Root();
// Load proto file and parse it
// Load proto files
const ingestorProto = readFileSync(join(protoDir, 'ingestor.proto'), 'utf8');
const ohlcProto = readFileSync(join(protoDir, 'ohlc.proto'), 'utf8');
protobuf.parse(ingestorProto, root);
protobuf.parse(ohlcProto, root);
// Export message types
const SubmitHistoricalRequestType = root.lookupType('SubmitHistoricalRequest');
const SubmitResponseType = root.lookupType('SubmitResponse');
const HistoryReadyNotificationType = root.lookupType('HistoryReadyNotification');
const OHLCType = root.lookupType('OHLC');
/**
* Encode SubmitHistoricalRequest to ZMQ frames
@@ -178,3 +197,39 @@ export function decodeHistoryReadyNotification(frames: Buffer[]): HistoryReadyNo
completed_at: BigInt(payload.completedAt),
};
}
/**
* Decode a realtime OHLC bar from ZMQ SUB frames.
* Frame layout: [topic][version][0x04 OHLC type + OHLC protobuf bytes]
*
* Returns null if the topic doesn't match the realtime bar pattern or decoding fails.
*/
export function decodeRealtimeBar(frames: Buffer[]): RealtimeBar | null {
if (frames.length < 3) return null;
const topic = frames[0].toString();
const match = OHLC_BAR_TOPIC_PATTERN.exec(topic);
if (!match) return null;
const ticker = match[1];
const periodSeconds = parseInt(match[2], 10);
const messageFrame = frames[2];
if (messageFrame[0] !== 0x04) return null; // Must be OHLC type
const payloadBuffer = messageFrame.slice(1);
const decoded = OHLCType.decode(payloadBuffer);
const ohlc = OHLCType.toObject(decoded, { longs: String, defaults: true });
return {
topic,
ticker,
periodSeconds,
timestamp: BigInt(ohlc.timestamp ?? '0'),
open: Number(ohlc.open ?? 0),
high: Number(ohlc.high ?? 0),
low: Number(ohlc.low ?? 0),
close: Number(ohlc.close ?? 0),
volume: Number(ohlc.volume ?? 0),
};
}

View File

@@ -17,6 +17,9 @@ import {
encodeSubmitHistoricalRequest,
decodeSubmitResponse,
decodeHistoryReadyNotification,
decodeRealtimeBar,
OHLC_BAR_TOPIC_PATTERN,
type RealtimeBar,
} from './zmq-protocol.js';
import type {
SubmitHistoricalRequest,
@@ -27,6 +30,9 @@ import {
NotificationStatus,
} from '../types/ohlc.js';
export type BarUpdateCallback = (bar: RealtimeBar) => void;
export type { RealtimeBar };
export interface ZMQRelayConfig {
relayRequestEndpoint: string; // e.g., "tcp://relay:5559"
relayNotificationEndpoint: string; // e.g., "tcp://relay:5558"
@@ -57,6 +63,12 @@ export class ZMQRelayClient {
private notificationTopic: string;
private pendingRequests: Map<string, PendingRequest> = new Map();
/** Ref count per ZMQ topic (gateway-level dedup before ZMQ subscribe/unsubscribe) */
private topicRefs: Map<string, number> = new Map();
/** Callbacks registered by WebSocket sessions for realtime bar updates */
private barCallbacks: Map<string, Set<BarUpdateCallback>> = new Map();
private connected = false;
private notificationListenerRunning = false;
@@ -253,8 +265,6 @@ export class ZMQRelayClient {
// Handle metadata update notifications
if (topic === 'METADATA_UPDATE') {
this.logger.info('Received METADATA_UPDATE notification');
// Call the onMetadataUpdate callback if configured
if (this.config.onMetadataUpdate) {
try {
await this.config.onMetadataUpdate();
@@ -265,6 +275,20 @@ export class ZMQRelayClient {
continue;
}
// Handle realtime OHLC bar updates (topic pattern: "{ticker}|ohlc:{period}")
if (OHLC_BAR_TOPIC_PATTERN.test(topic)) {
const bar = decodeRealtimeBar(Array.from(frames));
if (bar) {
const callbacks = this.barCallbacks.get(topic);
if (callbacks) {
for (const cb of callbacks) {
try { cb(bar); } catch (e) { /* ignore callback errors */ }
}
}
}
continue;
}
// Handle history ready notifications
const notification = decodeHistoryReadyNotification(Array.from(frames));
@@ -308,6 +332,69 @@ export class ZMQRelayClient {
this.logger.debug('Notification listener started');
}
/**
* Subscribe to realtime OHLC bars for a ticker+period.
*
* ZMQ subscribe is only called on the 0→1 transition (first subscriber).
* This triggers the relay XPUB → Flink subscription detection → ingestor activation.
*
* @param callback Called whenever a new bar arrives for this topic
*/
subscribeToTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
const topic = `${ticker}|ohlc:${periodSeconds}`;
// Register callback
if (!this.barCallbacks.has(topic)) {
this.barCallbacks.set(topic, new Set());
}
this.barCallbacks.get(topic)!.add(callback);
// ZMQ subscribe on first ref
const prev = this.topicRefs.get(topic) ?? 0;
this.topicRefs.set(topic, prev + 1);
if (prev === 0 && this.subSocket) {
this.subSocket.subscribe(topic);
this.logger.info({ topic }, 'ZMQ subscribed to realtime topic');
}
}
/**
* Unsubscribe a callback from realtime OHLC bars.
* ZMQ unsubscribe is only called on the 1→0 transition (last subscriber).
*/
unsubscribeFromTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
const topic = `${ticker}|ohlc:${periodSeconds}`;
const callbacks = this.barCallbacks.get(topic);
if (callbacks) {
callbacks.delete(callback);
if (callbacks.size === 0) {
this.barCallbacks.delete(topic);
}
}
const prev = this.topicRefs.get(topic) ?? 0;
if (prev <= 1) {
this.topicRefs.delete(topic);
if (this.subSocket) {
this.subSocket.unsubscribe(topic);
this.logger.info({ topic }, 'ZMQ unsubscribed from realtime topic');
}
} else {
this.topicRefs.set(topic, prev - 1);
}
}
/**
* Remove all subscriptions for a set of (topic, callback) pairs.
* Convenience method for WebSocket disconnect cleanup.
*/
cleanupSubscriptions(subscriptions: Array<{ ticker: string; periodSeconds: number; callback: BarUpdateCallback }>): void {
for (const { ticker, periodSeconds, callback } of subscriptions) {
this.unsubscribeFromTicker(ticker, periodSeconds, callback);
}
}
/**
* Close the client and cleanup resources
*/

View File

@@ -4,6 +4,7 @@ import type { FastifyBaseLogger } from 'fastify';
import type { License } from '../types/user.js';
import { ChannelType } from '../types/user.js';
import type { ConversationStore } from './memory/conversation-store.js';
import type { BlobStore } from './memory/blob-store.js';
import type { InboundMessage, OutboundMessage } from '../types/messages.js';
import { MCPClientConnector } from './mcp-client.js';
import { LLMProviderFactory, type ProviderConfig } from '../llm/provider.js';
@@ -14,13 +15,16 @@ import type { ChannelAdapter, PathTriggerContext } from '../workspace/index.js';
import type { ResearchSubagent } from './subagents/research/index.js';
import type { IndicatorSubagent } from './subagents/indicator/index.js';
import type { WebExploreSubagent } from './subagents/web-explore/index.js';
import type { StrategySubagent } from './subagents/strategy/index.js';
import type { DynamicStructuredTool } from '@langchain/core/tools';
import { getToolRegistry } from '../tools/tool-registry.js';
import type { MCPToolInfo } from '../tools/mcp/mcp-tool-wrapper.js';
import { createResearchAgentTool } from '../tools/platform/research-agent.tool.js';
import { createIndicatorAgentTool } from '../tools/platform/indicator-agent.tool.js';
import { createWebExploreAgentTool } from '../tools/platform/web-explore-agent.tool.js';
import { createStrategyAgentTool } from '../tools/platform/strategy-agent.tool.js';
import { createUserContext } from './memory/session-context.js';
import type { HarnessEvent } from './harness-events.js';
import { readFile } from 'fs/promises';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
@@ -54,10 +58,12 @@ export type HarnessFactory = (sessionConfig: HarnessSessionConfig) => AgentHarne
export interface AgentHarnessConfig extends HarnessSessionConfig {
providerConfig: ProviderConfig;
conversationStore?: ConversationStore;
blobStore?: BlobStore;
historyLimit: number;
researchSubagent?: ResearchSubagent;
indicatorSubagent?: IndicatorSubagent;
webExploreSubagent?: WebExploreSubagent;
strategySubagent?: StrategySubagent;
}
/**
@@ -87,6 +93,8 @@ export class AgentHarness {
private conversationStore?: ConversationStore;
private indicatorSubagent?: IndicatorSubagent;
private webExploreSubagent?: WebExploreSubagent;
private strategySubagent?: StrategySubagent;
private blobStore?: BlobStore;
private abortController: AbortController | null = null;
constructor(config: AgentHarnessConfig) {
@@ -96,10 +104,12 @@ export class AgentHarness {
this.researchSubagent = config.researchSubagent;
this.indicatorSubagent = config.indicatorSubagent;
this.webExploreSubagent = config.webExploreSubagent;
this.strategySubagent = config.strategySubagent;
this.modelFactory = new LLMProviderFactory(config.providerConfig, config.logger);
this.modelRouter = new ModelRouter(this.modelFactory, config.logger);
this.conversationStore = config.conversationStore;
this.blobStore = config.blobStore;
this.mcpClient = new MCPClientConnector({
userId: config.userId,
@@ -419,17 +429,75 @@ export class AgentHarness {
}
}
/**
* Initialize strategy subagent
*/
private async initializeStrategySubagent(): Promise<void> {
if (this.strategySubagent) {
this.config.logger.debug('Strategy subagent already provided');
return;
}
this.config.logger.debug('Creating strategy subagent for session');
try {
const { createStrategySubagent } = await import('./subagents/strategy/index.js');
const { model } = await this.modelRouter.route(
'trading strategy writing and backtesting',
this.config.license,
RoutingStrategy.COMPLEXITY,
this.config.userId
);
const toolRegistry = getToolRegistry();
const strategyTools = await toolRegistry.getToolsForAgent(
'strategy',
this.mcpClient,
this.availableMCPTools,
this.workspaceManager,
undefined,
undefined
);
const strategySubagentPath = join(__dirname, 'subagents', 'strategy');
this.config.logger.debug({ strategySubagentPath }, 'Using strategy subagent path');
this.strategySubagent = await createStrategySubagent(
model,
this.config.logger,
strategySubagentPath,
this.mcpClient,
strategyTools
);
this.config.logger.info(
{
toolCount: strategyTools.length,
toolNames: strategyTools.map(t => t.name),
},
'Strategy subagent created successfully'
);
} catch (error) {
this.config.logger.error(
{ error, errorMessage: (error as Error).message, stack: (error as Error).stack },
'Failed to create strategy subagent'
);
// Don't throw — strategy subagent is optional
}
}
/**
* Execute model with tool calling loop
* Handles multi-turn tool calls until the model produces a final text response
*/
private async executeWithToolCalling(
private async *executeWithToolCalling(
model: any,
messages: BaseMessage[],
tools: DynamicStructuredTool[],
maxIterations: number = 2,
signal?: AbortSignal
): Promise<string> {
): AsyncGenerator<HarnessEvent> {
this.config.logger.info(
{ toolCount: tools.length, maxIterations },
'Starting tool calling loop'
@@ -437,6 +505,8 @@ export class AgentHarness {
const messagesCopy = [...messages];
let iterations = 0;
// Track last char of last yielded text chunk to detect missing spaces between tokens
let lastChunkTail = '';
while (iterations < maxIterations) {
if (signal?.aborted) break;
@@ -455,15 +525,24 @@ export class AgentHarness {
try {
const stream = await model.stream(messagesCopy, { signal });
for await (const chunk of stream) {
const contents: string[] = [];
if (typeof chunk.content === 'string' && chunk.content.length > 0) {
this.channelAdapter?.sendChunk(chunk.content);
contents.push(chunk.content);
} else if (Array.isArray(chunk.content)) {
for (const block of chunk.content) {
if (block.type === 'text' && block.text) {
this.channelAdapter?.sendChunk(block.text);
}
if (block.type === 'text' && block.text) contents.push(block.text);
}
}
for (const content of contents) {
// DeepInfra/GLM streams tokens without leading spaces; inject one when
// both the tail of the previous chunk and the head of this chunk are
// word characters (\w), which would otherwise merge two words.
if (lastChunkTail && /\w/.test(lastChunkTail) && /\w/.test(content[0])) {
yield { type: 'chunk', content: ' ' };
}
lastChunkTail = content[content.length - 1];
yield { type: 'chunk', content };
}
response = response ? response.concat(chunk) : chunk;
}
} catch (invokeError: any) {
@@ -486,6 +565,8 @@ export class AgentHarness {
contentLength: typeof response.content === 'string' ? response.content.length : 0,
hasToolCalls: !!response.tool_calls,
toolCallCount: response.tool_calls?.length || 0,
usageMetadata: (response as any).usage_metadata,
finishReason: (response as any).response_metadata?.finish_reason,
},
'Model response received'
);
@@ -508,7 +589,8 @@ export class AgentHarness {
{ finalContentLength: finalContent.length, iterations },
'Tool calling loop complete - no more tool calls'
);
return finalContent;
yield { type: 'done', content: finalContent };
return;
}
this.config.logger.info(
@@ -540,11 +622,32 @@ export class AgentHarness {
}
try {
this.channelAdapter?.sendToolCall?.(toolCall.name, this.getToolLabel(toolCall.name));
const result = await tool.func(toolCall.args);
yield { type: 'tool_call', toolName: toolCall.name, label: this.getToolLabel(toolCall.name) };
// Process result to extract images and send them via channel adapter
const processedResult = this.processToolResult(result, toolCall.name);
// Use streamFunc when available (subagent tools) to forward intermediate events inline
let result: string;
const streamFunc = (tool as any).streamFunc as ((args: any, signal?: AbortSignal) => AsyncGenerator<import('./harness-events.js').HarnessEvent, string>) | undefined;
if (streamFunc) {
const gen = streamFunc(toolCall.args, signal);
let next = await gen.next();
while (!next.done) {
if (signal?.aborted) {
gen.return?.('');
break;
}
yield next.value;
next = await gen.next();
}
result = next.done ? next.value : '';
} else {
result = await tool.func(toolCall.args);
}
// Extract images from result and yield them; get text-only version for LLM
const { cleanedResult: processedResult, images } = this.extractImagesFromToolResult(result, toolCall.name);
for (const img of images) {
yield { type: 'image', data: img.data, mimeType: img.mimeType, caption: img.caption };
}
this.config.logger.debug(
{
@@ -567,6 +670,12 @@ export class AgentHarness {
'Tool execution completed'
);
} catch (error) {
// Clean stop — abort signal fired during tool execution; exit without error message
if (signal?.aborted || (error as Error)?.name === 'AbortError') {
this.config.logger.info({ tool: toolCall.name }, 'Tool execution aborted by stop signal');
return;
}
this.config.logger.error(
{
error,
@@ -578,6 +687,8 @@ export class AgentHarness {
'Tool execution failed'
);
yield { type: 'error' as const, source: toolCall.name, fatal: false };
messagesCopy.push(
new ToolMessage({
content: `Error: ${error}`,
@@ -586,11 +697,15 @@ export class AgentHarness {
);
}
}
// After all tool calls complete, emit a space separator before the next LLM streaming pass
yield { type: 'chunk', content: ' ' };
lastChunkTail = ' ';
}
// Max iterations reached - return what we have
// Max iterations reached - yield done with apology
this.config.logger.warn('Max tool calling iterations reached');
return 'I apologize, but I encountered an issue processing your request. Please try rephrasing your question.';
yield { type: 'done', content: 'I apologize, but I encountered an issue processing your request. Please try rephrasing your question.' };
}
/**
@@ -617,162 +732,222 @@ export class AgentHarness {
}
/**
* Handle incoming message from user
* Stream events for an incoming user message.
* Yields typed HarnessEvents (chunk, tool_call, image, done) and saves the
* conversation to the store once the done event has been emitted.
*/
async handleMessage(message: InboundMessage): Promise<OutboundMessage> {
async *streamMessage(message: InboundMessage): AsyncGenerator<HarnessEvent> {
this.config.logger.info(
{ messageId: message.messageId, userId: message.userId, content: message.content.substring(0, 100) },
'Processing user message'
);
try {
// 1. Build system prompt from template
this.config.logger.debug('Building system prompt');
const systemPrompt = await this.buildSystemPrompt();
this.config.logger.debug({ systemPromptLength: systemPrompt.length }, 'System prompt built');
// 1. Build system prompt from template
this.config.logger.debug('Building system prompt');
const systemPrompt = await this.buildSystemPrompt();
this.config.logger.debug({ systemPromptLength: systemPrompt.length }, 'System prompt built');
// 2. Load recent conversation history
const channelKey = this.config.channelType ?? ChannelType.WEBSOCKET;
let storedMessages = this.conversationStore
? await this.conversationStore.getRecentMessages(
this.config.userId, this.config.sessionId, this.config.historyLimit, channelKey
)
: [];
// First turn: seed conversation history with current workspace state
if (storedMessages.length === 0 && this.workspaceManager && this.conversationStore) {
const workspaceJSON = this.workspaceManager.serializeState();
const content = `[Workspace State]\n\`\`\`json\n${workspaceJSON}\n\`\`\``;
await this.conversationStore.saveMessage(
this.config.userId, this.config.sessionId,
'workspace', content, { isWorkspaceContext: true }, channelKey
);
storedMessages = await this.conversationStore.getRecentMessages(
// 2. Load recent conversation history
const channelKey = this.config.channelType ?? ChannelType.WEBSOCKET;
let storedMessages = this.conversationStore
? await this.conversationStore.getRecentMessages(
this.config.userId, this.config.sessionId, this.config.historyLimit, channelKey
);
}
)
: [];
const history = this.conversationStore
? this.conversationStore.toLangChainMessages(storedMessages)
: [];
this.config.logger.debug({ historyLength: history.length }, 'Conversation history loaded');
// 4. Get the configured model
this.config.logger.debug('Routing to model');
const { model, middleware } = await this.modelRouter.route(
message.content,
this.config.license,
RoutingStrategy.COMPLEXITY,
this.config.userId
// First turn: seed conversation history with current workspace state
if (storedMessages.length === 0 && this.workspaceManager && this.conversationStore) {
const workspaceJSON = this.workspaceManager.serializeState();
const content = `[Workspace State]\n\`\`\`json\n${workspaceJSON}\n\`\`\``;
await this.conversationStore.saveMessage(
this.config.userId, this.config.sessionId,
'workspace', content, { isWorkspaceContext: true }, channelKey
);
this.middleware = middleware;
this.config.logger.info({ modelName: model.constructor.name }, 'Model selected');
// 5. Build LangChain messages
const langchainMessages = this.buildLangChainMessages(systemPrompt, history, message.content);
this.config.logger.debug({ messageCount: langchainMessages.length }, 'LangChain messages built');
// 6. Get tools for main agent from registry
const toolRegistry = getToolRegistry();
const tools = await toolRegistry.getToolsForAgent(
'main',
this.mcpClient,
this.availableMCPTools,
this.workspaceManager // Pass session workspace manager
storedMessages = await this.conversationStore.getRecentMessages(
this.config.userId, this.config.sessionId, this.config.historyLimit, channelKey
);
}
// Build shared subagent context
const subagentContext = {
userContext: createUserContext({
userId: this.config.userId,
sessionId: this.config.sessionId,
license: this.config.license,
channelType: this.config.channelType ?? ChannelType.WEBSOCKET,
channelUserId: this.config.channelUserId ?? this.config.userId,
}),
};
const history = this.conversationStore
? this.conversationStore.toLangChainMessages(storedMessages)
: [];
this.config.logger.debug({ historyLength: history.length }, 'Conversation history loaded');
// Add research subagent as a tool if available
if (this.researchSubagent) {
tools.push(createResearchAgentTool({
researchSubagent: this.researchSubagent,
context: subagentContext,
logger: this.config.logger,
}));
}
// 4. Get the configured model
this.config.logger.debug('Routing to model');
const { model, middleware } = await this.modelRouter.route(
message.content,
this.config.license,
RoutingStrategy.COMPLEXITY,
this.config.userId
);
this.middleware = middleware;
this.config.logger.info({ modelName: model.constructor.name }, 'Model selected');
// Add indicator subagent as a tool if available
if (this.indicatorSubagent) {
tools.push(createIndicatorAgentTool({
indicatorSubagent: this.indicatorSubagent,
context: subagentContext,
logger: this.config.logger,
}));
}
// 5. Build LangChain messages
const langchainMessages = this.buildLangChainMessages(systemPrompt, history, message.content);
this.config.logger.debug({ messageCount: langchainMessages.length }, 'LangChain messages built');
// Add web explore subagent as a tool if available
if (this.webExploreSubagent) {
tools.push(createWebExploreAgentTool({
webExploreSubagent: this.webExploreSubagent,
context: subagentContext,
logger: this.config.logger,
}));
}
// 6. Get tools for main agent from registry
const toolRegistry = getToolRegistry();
const tools = await toolRegistry.getToolsForAgent(
'main',
this.mcpClient,
this.availableMCPTools,
this.workspaceManager
);
// Build shared subagent context
const subagentContext = {
userContext: createUserContext({
userId: this.config.userId,
sessionId: this.config.sessionId,
license: this.config.license,
channelType: this.config.channelType ?? ChannelType.WEBSOCKET,
channelUserId: this.config.channelUserId ?? this.config.userId,
}),
};
if (this.researchSubagent) {
tools.push(createResearchAgentTool({
researchSubagent: this.researchSubagent,
context: subagentContext,
logger: this.config.logger,
}));
}
if (this.indicatorSubagent) {
tools.push(createIndicatorAgentTool({
indicatorSubagent: this.indicatorSubagent,
context: subagentContext,
logger: this.config.logger,
}));
}
if (this.webExploreSubagent) {
tools.push(createWebExploreAgentTool({
webExploreSubagent: this.webExploreSubagent,
context: subagentContext,
logger: this.config.logger,
}));
}
if (!this.strategySubagent) {
await this.initializeStrategySubagent();
}
if (this.strategySubagent) {
tools.push(createStrategyAgentTool({
strategySubagent: this.strategySubagent,
context: subagentContext,
logger: this.config.logger,
}));
}
this.config.logger.info(
{ toolCount: tools.length, toolNames: tools.map(t => t.name) },
'Tools loaded for main agent'
);
// Apply middleware (e.g. Anthropic prompt caching)
const processedMessages = this.middleware
? this.middleware.processMessages(langchainMessages, tools)
: langchainMessages;
// 7. Bind tools to model
const modelWithTools = tools.length > 0 && model.bindTools ? model.bindTools(tools) : model;
if (tools.length > 0) {
this.config.logger.info(
{
toolCount: tools.length,
toolNames: tools.map(t => t.name),
},
'Tools loaded for main agent'
{ modelType: modelWithTools.constructor.name, toolsBound: tools.length > 0 && !!model.bindTools },
'Model bound with tools'
);
}
// Apply middleware (e.g. Anthropic prompt caching)
const processedMessages = this.middleware
? this.middleware.processMessages(langchainMessages, tools)
: langchainMessages;
// 7. Bind tools to model
const modelWithTools = tools.length > 0 && model.bindTools ? model.bindTools(tools) : model;
if (tools.length > 0) {
this.config.logger.info(
{ modelType: modelWithTools.constructor.name, toolsBound: tools.length > 0 && !!model.bindTools },
'Model bound with tools'
);
// 8. Stream tool calling loop and save conversation on completion
this.config.logger.info('Invoking LLM with tool support');
this.abortController = new AbortController();
let finalContent = '';
const collectedImages: Array<{ data: string; mimeType: string; caption?: string }> = [];
try {
for await (const event of this.executeWithToolCalling(modelWithTools, processedMessages, tools, 10, this.abortController.signal)) {
if (event.type === 'done') {
finalContent = event.content;
this.config.logger.info({ responseLength: finalContent.length }, 'LLM response received');
} else if (event.type === 'image') {
collectedImages.push({ data: event.data, mimeType: event.mimeType, caption: event.caption });
}
yield event;
}
// 8. Call LLM with tool calling loop
this.config.logger.info('Invoking LLM with tool support');
this.abortController = new AbortController();
const assistantMessage = await this.executeWithToolCalling(modelWithTools, processedMessages, tools, 10, this.abortController.signal);
} catch (error) {
if ((error as Error)?.name === 'AbortError') {
this.config.logger.info('Agent harness interrupted by stop signal');
} else {
this.config.logger.error({ error }, 'Fatal error in agent harness');
yield { type: 'error' as const, source: 'agent harness', fatal: true };
}
} finally {
this.abortController = null;
if (finalContent && this.conversationStore) {
// Write blobs to S3 and capture their IDs for message metadata
let blobRefs: Array<{ id: string; mimeType: string; caption?: string }> = [];
if (collectedImages.length > 0 && this.blobStore) {
const assistantMsgId = `${this.config.userId}:${this.config.sessionId}:${Date.now()}`;
const blobIds = await this.blobStore.writeBlobs(
this.config.userId, this.config.sessionId, assistantMsgId,
collectedImages.map(img => ({ blobType: 'image' as const, mimeType: img.mimeType, data: img.data, caption: img.caption }))
);
blobRefs = blobIds.map((id, i) => ({ id, mimeType: collectedImages[i].mimeType, caption: collectedImages[i].caption }));
}
this.config.logger.info(
{ responseLength: assistantMessage.length },
'LLM response received'
);
// Save user message and assistant response to conversation store
if (this.conversationStore) {
await this.conversationStore.saveMessage(
this.config.userId, this.config.sessionId, 'user', message.content, undefined, channelKey
);
await this.conversationStore.saveMessage(
this.config.userId, this.config.sessionId, 'assistant', assistantMessage, undefined, channelKey
this.config.userId, this.config.sessionId, 'assistant', finalContent,
blobRefs.length > 0 ? { blobs: blobRefs } : undefined,
channelKey
);
}
}
}
return {
messageId: `msg_${Date.now()}`,
sessionId: message.sessionId,
content: assistantMessage,
timestamp: new Date(),
};
/**
* Handle incoming message from user.
* Consumes streamMessage and dispatches events to the channel adapter for
* backward compatibility with Telegram and other non-streaming callers.
*/
async handleMessage(message: InboundMessage): Promise<OutboundMessage> {
let finalContent = '';
try {
for await (const event of this.streamMessage(message)) {
switch (event.type) {
case 'chunk':
this.channelAdapter?.sendChunk(event.content);
break;
case 'tool_call':
this.channelAdapter?.sendToolCall?.(event.toolName, event.label);
break;
case 'image':
this.channelAdapter?.sendImage({ data: event.data, mimeType: event.mimeType, caption: event.caption });
break;
case 'error':
this.channelAdapter?.sendText?.({ text: `An unrecoverable error occurred in the ${event.source}.` });
break;
case 'done':
finalContent = event.content;
break;
}
}
} catch (error) {
this.config.logger.error({ error }, 'Error processing message');
throw error;
}
return {
messageId: `msg_${Date.now()}`,
sessionId: message.sessionId,
content: finalContent,
timestamp: new Date(),
};
}
/**
@@ -817,21 +992,27 @@ export class AgentHarness {
python_write: 'Coding...',
python_read: 'Inspecting...',
execute_research: 'Running script...',
backtest_strategy: 'Running backtest...',
backtest_strategy: 'Backtesting...',
list_active_strategies: 'Checking active strategies...',
web_explore: 'Searching the web...',
strategy: 'Coding a strategy...',
};
return labels[toolName] ?? `Running ${toolName}...`;
return labels[toolName] ?? `Running ${toolName} tool...`;
}
/**
* Process tool result to extract images and send via channel adapter.
* Returns text-only version for LLM context (no base64 image data).
*/
private processToolResult(result: string, toolName: string): string {
private extractImagesFromToolResult(
result: string,
toolName: string
): { cleanedResult: string; images: Array<{ data: string; mimeType: string; caption?: string }> } {
const noImages = { cleanedResult: String(result || ''), images: [] };
// Most tools return plain strings - only process JSON results
if (!result || typeof result !== 'string') {
return String(result || '');
return noImages;
}
// Try to parse as JSON
@@ -840,7 +1021,7 @@ export class AgentHarness {
parsedResult = JSON.parse(result);
} catch {
// Not JSON, return as-is
return result;
return noImages;
}
// Check if result has images array (from ResearchSubagent)
@@ -850,19 +1031,11 @@ export class AgentHarness {
'Extracting images from tool result'
);
// Send each image via channel adapter
const images: Array<{ data: string; mimeType: string; caption?: string }> = [];
for (const image of parsedResult.images) {
if (image.data && image.mimeType) {
if (this.channelAdapter) {
this.config.logger.debug({ mimeType: image.mimeType }, 'Sending image to channel');
this.channelAdapter.sendImage({
data: image.data,
mimeType: image.mimeType,
caption: undefined,
});
} else {
this.config.logger.warn('No channel adapter set, cannot send image');
}
this.config.logger.debug({ mimeType: image.mimeType }, 'Extracted image from tool result');
images.push({ data: image.data, mimeType: image.mimeType, caption: undefined });
}
}
@@ -872,15 +1045,13 @@ export class AgentHarness {
images: undefined,
imageCount: parsedResult.images.length,
};
// Clean up undefined values
Object.keys(textOnlyResult).forEach(key => {
if (textOnlyResult[key] === undefined) {
delete textOnlyResult[key];
}
});
return JSON.stringify(textOnlyResult);
return { cleanedResult: JSON.stringify(textOnlyResult), images };
}
// Check for nested chart_images object
@@ -890,20 +1061,12 @@ export class AgentHarness {
'Extracting chart images from tool result'
);
// Send each chart image via channel adapter
const images: Array<{ data: string; mimeType: string; caption?: string }> = [];
for (const [chartId, chartData] of Object.entries(parsedResult.chart_images)) {
const chart = chartData as any;
if (chart.type === 'image' && chart.data) {
if (this.channelAdapter) {
this.config.logger.debug({ chartId }, 'Sending chart image to channel');
this.channelAdapter.sendImage({
data: chart.data,
mimeType: 'image/png',
caption: undefined,
});
} else {
this.config.logger.warn('No channel adapter set, cannot send chart image');
}
this.config.logger.debug({ chartId }, 'Extracted chart image from tool result');
images.push({ data: chart.data, mimeType: 'image/png', caption: undefined });
}
}
@@ -913,19 +1076,17 @@ export class AgentHarness {
chart_images: undefined,
chartCount: Object.keys(parsedResult.chart_images).length,
};
// Clean up undefined values
Object.keys(textOnlyResult).forEach(key => {
if (textOnlyResult[key] === undefined) {
delete textOnlyResult[key];
}
});
return JSON.stringify(textOnlyResult);
return { cleanedResult: JSON.stringify(textOnlyResult), images };
}
// No images found, return stringified result
return result;
// No images found, return as-is
return { cleanedResult: result, images: [] };
}
/**

View File

@@ -0,0 +1,51 @@
export interface ChunkEvent {
type: 'chunk';
content: string;
}
export interface ToolCallEvent {
type: 'tool_call';
toolName: string;
label: string;
}
export interface ImageEvent {
type: 'image';
data: string;
mimeType: string;
caption?: string;
}
export interface DoneEvent {
type: 'done';
content: string;
}
export interface SubagentChunkEvent {
type: 'subagent_chunk';
agentName: string;
content: string;
}
export interface SubagentThinkingEvent {
type: 'subagent_thinking';
agentName: string;
content: string;
}
export interface SubagentToolCallEvent {
type: 'subagent_tool_call';
agentName: string;
toolName: string;
label: string;
}
export interface ErrorEvent {
type: 'error';
/** Name of the agent or tool where the error occurred */
source: string;
/** True if the error is unrecoverable and the chat session should end */
fatal: boolean;
}
export type HarnessEvent = ChunkEvent | ToolCallEvent | ImageEvent | DoneEvent | SubagentChunkEvent | SubagentThinkingEvent | SubagentToolCallEvent | ErrorEvent;

View File

@@ -57,57 +57,74 @@ export class MCPClientConnector {
this.client = null;
}
try {
this.config.logger.info(
{ userId: this.config.userId, url: this.config.mcpServerUrl },
'Connecting to user MCP server'
);
const maxAttempts = 5;
const retryDelayMs = 1500;
this.client = new Client(
{
name: 'dexorder-gateway',
version: '0.1.0',
},
{
capabilities: {
sampling: {},
this.config.logger.info(
{ userId: this.config.userId, url: this.config.mcpServerUrl },
'Connecting to user MCP server'
);
let lastError: unknown;
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
this.client = new Client(
{
name: 'dexorder-gateway',
version: '0.1.0',
},
}
);
{
capabilities: {
sampling: {},
},
}
);
// Streamable HTTP: single /mcp endpoint, session tracked via mcp-session-id header
const transport = new StreamableHTTPClientTransport(
new URL(`${this.config.mcpServerUrl}/mcp`)
);
// Streamable HTTP: single /mcp endpoint, session tracked via mcp-session-id header
const transport = new StreamableHTTPClientTransport(
new URL(`${this.config.mcpServerUrl}/mcp`)
);
await this.client.connect(transport);
await this.client.connect(transport);
// Hook client.onerror to detect transport failures (e.g. sandbox restart returning
// 404 "session not found"). When fired, mark disconnected so the next callTool /
// listTools call triggers a full reconnect + initialize handshake.
const connectedClient = this.client;
const origOnError = this.client.onerror;
this.client.onerror = (error) => {
origOnError?.(error);
// Only act on the currently-active client (ignore stale closures after reconnect)
if (this.client === connectedClient && this.connected) {
// Hook client.onerror to detect transport failures (e.g. sandbox restart returning
// 404 "session not found"). When fired, mark disconnected so the next callTool /
// listTools call triggers a full reconnect + initialize handshake.
const connectedClient = this.client;
const origOnError = this.client.onerror;
this.client.onerror = (error) => {
origOnError?.(error);
// Only act on the currently-active client (ignore stale closures after reconnect)
if (this.client === connectedClient && this.connected) {
this.config.logger.warn(
{ error },
'MCP transport error — marking disconnected for lazy reconnect'
);
this.connected = false;
}
};
this.connected = true;
this.config.logger.info('Connected to user MCP server');
return;
} catch (error) {
lastError = error;
this.client = null;
if (attempt < maxAttempts) {
this.config.logger.warn(
{ error },
'MCP transport error — marking disconnected for lazy reconnect'
{ error, userId: this.config.userId, attempt, maxAttempts },
'MCP connect attempt failed, retrying...'
);
this.connected = false;
await new Promise(resolve => setTimeout(resolve, retryDelayMs));
}
};
this.connected = true;
this.config.logger.info('Connected to user MCP server');
} catch (error) {
this.config.logger.error(
{ error, userId: this.config.userId },
'Failed to connect to user MCP server'
);
throw error;
}
}
this.config.logger.error(
{ error: lastError, userId: this.config.userId },
'Failed to connect to user MCP server'
);
throw lastError;
}
/**
@@ -134,7 +151,9 @@ export class MCPClientConnector {
try {
this.config.logger.debug({ tool: name, args }, 'Calling MCP tool');
const result = await this.client!.callTool({ name, arguments: args });
// Use a generous timeout: execute_research runs a subprocess with a 300s limit,
// so the default 60s MCP SDK timeout would fire before the script completes.
const result = await this.client!.callTool({ name, arguments: args }, undefined, { timeout: 330000 });
return result;
} catch (error) {
this.config.logger.error({ error, tool: name }, 'MCP tool call failed');

View File

@@ -0,0 +1,93 @@
import type { FastifyBaseLogger } from 'fastify';
import type { IcebergClient } from '../../clients/iceberg-client.js';
export interface StoredBlob {
id: string;
userId: string;
sessionId: string;
messageId: string;
blobType: 'image' | 'audio';
mimeType: string;
data: string; // base64
caption?: string;
timestamp: number; // microseconds
}
/**
* Blob store for binary attachments (images, audio) referenced by conversation messages.
*
* Unlike text messages (Redis hot + Iceberg cold), blobs write directly to S3 Parquet
* on each turn — they're infrequent enough that per-turn files don't cause fragmentation.
* Blob IDs are stored in the parent message's metadata field for later retrieval.
*/
export class BlobStore {
constructor(
private icebergClient: IcebergClient | undefined,
private logger: FastifyBaseLogger
) {}
/**
* Write all blobs for one assistant turn to a single S3 Parquet file.
* Returns the blob IDs assigned. Failures are logged but do not throw.
*/
async writeBlobs(
userId: string,
sessionId: string,
messageId: string,
blobs: Array<{ blobType: 'image' | 'audio'; mimeType: string; data: string; caption?: string }>
): Promise<string[]> {
if (!this.icebergClient || blobs.length === 0) {
return [];
}
const now = Date.now();
const stored = blobs.map((b, i) => ({
id: `blob_${userId}_${now}_${i}`,
user_id: userId,
session_id: sessionId,
message_id: messageId,
blob_type: b.blobType,
mime_type: b.mimeType,
data: b.data,
caption: b.caption ?? null,
timestamp: now * 1000, // microseconds
}));
try {
await this.icebergClient.appendBlobs(userId, sessionId, messageId, stored);
this.logger.info({ userId, sessionId, count: stored.length }, 'Blobs written to S3');
} catch (error) {
this.logger.error({ error }, 'Failed to write blobs to S3');
// Don't throw — blob failure should not break the conversation turn
}
return stored.map(b => b.id);
}
/**
* Retrieve blobs by their IDs from S3/Iceberg cold storage.
*/
async getBlobsByIds(userId: string, sessionId: string, blobIds: string[]): Promise<StoredBlob[]> {
if (!this.icebergClient || blobIds.length === 0) {
return [];
}
try {
const rows = await this.icebergClient.queryBlobs(userId, sessionId, blobIds);
return rows.map(r => ({
id: r.id,
userId: r.user_id,
sessionId: r.session_id,
messageId: r.message_id,
blobType: r.blob_type as 'image' | 'audio',
mimeType: r.mime_type,
data: r.data,
caption: r.caption ?? undefined,
timestamp: r.timestamp,
}));
} catch (error) {
this.logger.error({ error, blobIds }, 'Failed to retrieve blobs');
return [];
}
}
}

View File

@@ -39,9 +39,9 @@ If the user asks for a capability not provided by Dexorder, decline and explain
## Task Delegation
- For ANY research questions, deep analysis, statistical analysis, charting requests, or market data queries that require computation, you MUST use the 'research' tool
- For ANYTHING related to indicators on the chart — reading, adding, removing, modifying, or creating custom indicators — you MUST use the 'indicator' tool
- For ANY backtesting request — running a strategy against historical data — you MUST use the 'backtest_strategy' tool directly; NEVER use the research tool for backtesting
- For ANY request about trading strategies — writing, editing, backtesting, interpreting results, activating, deactivating, or monitoring — you MUST use the 'strategy' tool; NEVER write strategy Python code yourself
- NEVER write Python code directly in your responses to the user
- NEVER show code to the user — delegate to the research or indicator tool instead
- NEVER show code to the user — delegate to the research, indicator, or strategy tool instead
- NEVER attempt to do analysis yourself — let the subagents handle it
## Available Tools
@@ -110,46 +110,54 @@ Parameters:
- instruction: Natural language description of the analysis to perform (be specific!)
- name: A unique name for the research script (e.g., "BTC Weekly Analysis")
**Do NOT include any time range, history length, bar count, period size, resolution, or timestamp guidance in the instruction** — not as numbers, not as natural language ("3-6 months", "1 year", "sufficient data"), not at all. The research subagent has its own rules for selecting resolution and history window. If you add time guidance, the subagent will follow yours instead of its own (which uses much more data). Only pass time constraints if the user explicitly asked for a specific period (e.g. "last week", "show me 2023").
Example usage:
- User: "Does Friday price action correlate with Monday?"
- You: Call research tool with instruction="Analyze correlation between Friday and Monday price action during NY trading hours (9:30-4:00 ET)", name="Friday-Monday Correlation"
- WRONG: "...use hourly data and at least 3-6 months..." ← never add this
### strategy
**Use this tool for ALL trading strategy requests without exception.**
The strategy subagent handles the complete strategy lifecycle: writing PandasStrategy classes, running backtests, interpreting results, and activating/deactivating paper trading.
**ALWAYS use strategy for:**
- "Create a strategy that buys when RSI < 30" write a new strategy
- "Edit my momentum strategy to use a tighter stop" modify existing strategy
- "Backtest my RSI strategy over the last year" run backtest
- "How did this strategy perform on BTC?" interpret results
- "Activate my strategy for paper trading" start paper trading
- "What strategies are running?" list active strategies
- "Stop my momentum strategy" deactivate a strategy
- Any question about a strategy's PnL, trades, or performance
**NEVER call `backtest_strategy`, `activate_strategy`, `deactivate_strategy`, or `list_active_strategies` directly** always go through the strategy tool.
**Custom indicators in strategies:**
When writing a new strategy, the strategy subagent will first check for existing custom indicators via `python_list(category="indicator")`. Prefer using custom indicators (via `ta.custom_*`) over computing signals inline this promotes reuse and gives users better visibility into strategy components. If a needed indicator doesn't exist yet, the strategy subagent will create it first via the indicator workflow.
### backtest_strategy
**ALWAYS use this tool — and ONLY this tool — for any backtesting request.**
*(Called internally by the strategy tool do not call this directly.)*
Runs a saved trading strategy against historical OHLC data using the Nautilus Trader backtesting engine.
Returns structured performance metrics and an equity curve. Any charts generated are automatically sent to the user.
**ALWAYS use backtest_strategy for:**
- "Backtest my RSI strategy over the last year"
- "How did this strategy perform on BTC?"
- "Run a backtest from January to June"
- Any request to test or evaluate a strategy on historical data
**NEVER use research for backtesting** — the research tool cannot run strategies through the backtesting engine.
After the tool returns, summarize the results clearly: total return, Sharpe ratio, max drawdown, win rate, and trade count. Present the equity curve description in plain language.
Parameters:
- strategy_name: Display name of the saved strategy (use python_list with category="strategy" to check existing strategies)
- feeds: Array of `{symbol, period_seconds}` feed objects (e.g. `[{"symbol": "BTC/USDT.BINANCE", "period_seconds": 3600}]`)
- from_time / to_time: Date strings ("2024-01-01", "90 days ago", "now") or Unix timestamps
- initial_capital: Starting balance in quote currency (default 10,000)
Returns structured performance metrics including trade list, Sortino/Calmar ratios, and equity curve.
### list_active_strategies
*(Called internally by the strategy tool do not call this directly.)*
Lists all currently active (live or paper) strategies and their status.
Use this when the user asks what strategies are running.
### python_list
List existing scripts in a category ("strategy", "indicator", or "research").
Use this before calling the research tool to check whether a relevant script already exists.
If one does, pass its exact name to the research tool so the subagent updates it rather than creating a new one.
Also use before calling backtest_strategy to confirm the strategy name.
The strategy tool uses this internally to check strategy names before backtesting.
### symbol-lookup
Look up trading symbols and get metadata.
Use this when users mention tickers or need symbol information.
**Always use symbol_lookup to resolve a proper ticker before passing it to the research or get-chart-data tools.** Symbols must be in `SYMBOL.EXCHANGE` format (e.g., `BTC/USDT.BINANCE`). If the user says "ETHUSDT", "ETH", or any ambiguous ticker, resolve it first with symbol_lookup so the correct formatted ticker is passed downstream.
### get-chart-data
**IMPORTANT: This is for QUICK, CASUAL information ONLY. This tool just returns raw data - it does NOT create charts or plots.**

View File

@@ -7,6 +7,7 @@ import type { MCPClientConnector } from '../mcp-client.js';
import type { DynamicStructuredTool } from '@langchain/core/tools';
import { readFile } from 'fs/promises';
import { join } from 'path';
import type { HarnessEvent, SubagentChunkEvent, SubagentThinkingEvent } from '../harness-events.js';
/**
* Subagent configuration (loaded from config.yaml)
@@ -122,6 +123,65 @@ export abstract class BaseSubagent {
yield result;
}
/**
* Extract subagent_chunk / subagent_thinking events from a LangGraph `messages` stream datum.
*
* LangGraph emits `[message_chunk, metadata]` tuples in `messages` mode. The message content
* can be a plain string (normal text token) or an array of content blocks (extended thinking
* responses with `{type:"thinking", thinking:"..."}` and `{type:"text", text:"..."}`).
*/
static extractStreamChunks(
data: unknown,
agentName: string,
): Array<SubagentChunkEvent | SubagentThinkingEvent> {
const msg = Array.isArray(data) ? (data as unknown[])[0] : data;
const content = (msg as any)?.content;
if (typeof content === 'string') {
return content ? [{ type: 'subagent_chunk', agentName, content }] : [];
}
if (Array.isArray(content)) {
const chunks: Array<SubagentChunkEvent | SubagentThinkingEvent> = [];
for (const block of content as any[]) {
if (block?.type === 'thinking' && typeof block.thinking === 'string' && block.thinking) {
chunks.push({ type: 'subagent_thinking', agentName, content: block.thinking });
} else if (block?.type === 'text' && typeof block.text === 'string' && block.text) {
chunks.push({ type: 'subagent_chunk', agentName, content: block.text });
}
}
return chunks;
}
return [];
}
/**
* Extract the final text from an `updates`-mode agent message.
* Handles both plain string content and array content blocks (extended thinking).
*/
static extractFinalText(msg: any): string {
if (typeof msg?.content === 'string') return msg.content;
if (Array.isArray(msg?.content)) {
return (msg.content as any[])
.filter((b: any) => b?.type === 'text' && typeof b.text === 'string')
.map((b: any) => b.text as string)
.join('');
}
return '';
}
/**
* Stream typed HarnessEvents during execution.
* Subclasses override this to emit subagent_chunk / subagent_tool_call events
* using agent.stream() from LangGraph. Default falls back to execute().
*/
async *streamEvents(
context: SubagentContext,
input: string,
_signal?: AbortSignal,
): AsyncGenerator<HarnessEvent, string> {
const result = await this.execute(context, input);
return result;
}
/**
* Build messages with system prompt and memory context
*/

View File

@@ -11,3 +11,8 @@ export {
createResearchSubagent,
type ResearchResult,
} from './research/index.js';
export {
StrategySubagent,
createStrategySubagent,
} from './strategy/index.js';

View File

@@ -4,6 +4,7 @@ import { SystemMessage } from '@langchain/core/messages';
import { createReactAgent } from '@langchain/langgraph/prebuilt';
import type { FastifyBaseLogger } from 'fastify';
import type { MCPClientConnector } from '../../mcp-client.js';
import type { HarnessEvent } from '../../harness-events.js';
/**
* Indicator Subagent
@@ -84,6 +85,56 @@ export class IndicatorSubagent extends BaseSubagent {
return finalText;
}
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
if (!this.hasMCPClient()) {
throw new Error('MCP client not available for indicator subagent');
}
const initialMessages = this.buildMessages(context, instruction);
const systemMessage = initialMessages[0];
const humanMessage = initialMessages[initialMessages.length - 1];
const agent = createReactAgent({
llm: this.model,
tools: this.tools,
prompt: systemMessage as SystemMessage,
});
const stream = agent.stream(
{ messages: [humanMessage] },
{ streamMode: ['messages', 'updates'], recursionLimit: 25, signal }
);
let finalText = '';
for await (const [mode, data] of await stream) {
if (signal?.aborted) break;
if (mode === 'messages') {
for (const chunk of IndicatorSubagent.extractStreamChunks(data, this.config.name)) {
yield chunk;
}
} else if (mode === 'updates') {
if ((data as any).agent?.messages) {
for (const msg of (data as any).agent.messages as any[]) {
if (msg.tool_calls?.length) {
for (const tc of msg.tool_calls) {
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
}
} else {
const content = IndicatorSubagent.extractFinalText(msg);
if (content) finalText = content;
}
}
}
}
}
this.logger.info({ textLength: finalText.length }, 'streamEvents finished');
return finalText;
}
}
/**

View File

@@ -4,6 +4,7 @@ import { SystemMessage } from '@langchain/core/messages';
import { createReactAgent } from '@langchain/langgraph/prebuilt';
import type { FastifyBaseLogger } from 'fastify';
import type { MCPClientConnector } from '../../mcp-client.js';
import type { HarnessEvent } from '../../harness-events.js';
/**
* Result from research subagent execution
@@ -50,6 +51,58 @@ export class ResearchSubagent extends BaseSubagent {
this.imageCapture = capture;
}
/**
* Fetch custom indicators from the sandbox and return a formatted system prompt section.
* Returns empty string if there are no custom indicators or the call fails.
*/
private async fetchCustomIndicatorsSection(): Promise<string> {
try {
const raw = await this.callMCPTool('python_list', { category: 'indicator' });
const r = raw as any;
const text = r?.content?.[0]?.text ?? r?.[0]?.text;
const parsed = typeof text === 'string' ? JSON.parse(text) : raw;
const items: any[] = parsed?.items ?? [];
if (items.length === 0) return '';
const lines: string[] = ['\n\n## Custom Indicators\n'];
lines.push('The user has defined the following custom indicators. Use `ta.custom_<name>` where `<name>` is the lowercase sanitized function name shown below.\n');
for (const item of items) {
const displayName: string = item.name ?? 'unknown';
const description: string = item.description ?? '';
const meta: any = item.metadata ?? {};
// Derive the ta attribute name: sanitize display name to lowercase + underscores
const taAttr = `custom_${displayName.toLowerCase().replace(/[^\w]/g, '_').replace(/_+/g, '_').replace(/^_+|_+$/g, '')}`;
const inputSeries: string[] = meta.input_series ?? ['close'];
const params: Record<string, any> = meta.parameters ?? {};
const pane: string = meta.pane ?? 'separate';
const inputStr = inputSeries.map((s: string) => `df['${s}']`).join(', ');
const paramStr = Object.entries(params)
.map(([k, v]: [string, any]) => `${k}=${JSON.stringify(v?.default ?? null)}`)
.join(', ');
const callExample = paramStr
? `ta.${taAttr}(${inputStr}, ${paramStr})`
: `ta.${taAttr}(${inputStr})`;
const outputNames = (meta.output_columns ?? [{ name: 'value' }])
.map((c: any) => c.name)
.join(', ');
lines.push(`### ${displayName}`);
if (description) lines.push(description);
lines.push(`- **Call**: \`${callExample}\``);
lines.push(`- **Outputs**: ${outputNames} | **Pane**: ${pane}`);
lines.push('');
}
return lines.join('\n');
} catch (err) {
this.logger.warn({ err }, 'Failed to fetch custom indicators for prompt injection');
return '';
}
}
/**
* Execute research request using LangGraph's createReactAgent.
* This is the standard LangChain pattern for agents with tool access —
@@ -79,11 +132,17 @@ export class ResearchSubagent extends BaseSubagent {
this.imageCapture.length = 0;
this.lastImages = [];
const customIndicatorsSection = await this.fetchCustomIndicatorsSection();
// Build system prompt (with memory context appended)
const initialMessages = this.buildMessages(context, instruction);
// buildMessages returns [SystemMessage, ...history, HumanMessage]
// Extract system content for createReactAgent's prompt parameter
const systemMessage = initialMessages[0];
let systemMessage = initialMessages[0] as SystemMessage;
if (customIndicatorsSection) {
const base = typeof systemMessage.content === 'string' ? systemMessage.content : JSON.stringify(systemMessage.content);
systemMessage = new SystemMessage(base + customIndicatorsSection);
}
const humanMessage = initialMessages[initialMessages.length - 1];
// createReactAgent is the standard LangChain/LangGraph pattern for tool-using agents.
@@ -91,12 +150,12 @@ export class ResearchSubagent extends BaseSubagent {
const agent = createReactAgent({
llm: this.model,
tools: this.tools,
prompt: systemMessage as SystemMessage,
prompt: systemMessage,
});
const result = await agent.invoke(
{ messages: [humanMessage] },
{ recursionLimit: 20 }
{ recursionLimit: 40 }
);
// The final message in the graph output is the agent's last AIMessage
@@ -146,6 +205,109 @@ export class ResearchSubagent extends BaseSubagent {
return this.lastImages;
}
/**
* Stream typed HarnessEvents using LangGraph's agent.stream().
* Emits subagent_tool_call when tools fire, subagent_chunk for the final AI response.
* Returns the final text string as the generator return value.
*/
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
if (!this.hasMCPClient()) {
throw new Error('MCP client not available for research subagent');
}
this.imageCapture.length = 0;
this.lastImages = [];
// Emit immediately so the UI shows the subagent has started — LLM generation
// can take minutes with non-streaming models and nothing else reaches the UI until
// the first `updates` event fires (after the LLM finishes its first response).
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: 'Thinking...', label: 'Thinking...' };
const customIndicatorsSection = await this.fetchCustomIndicatorsSection();
const initialMessages = this.buildMessages(context, instruction);
let systemMessage = initialMessages[0] as SystemMessage;
if (customIndicatorsSection) {
const base = typeof systemMessage.content === 'string' ? systemMessage.content : JSON.stringify(systemMessage.content);
systemMessage = new SystemMessage(base + customIndicatorsSection);
}
const humanMessage = initialMessages[initialMessages.length - 1];
const agent = createReactAgent({
llm: this.model,
tools: this.tools,
prompt: systemMessage,
});
this.logger.debug(
{ toolCount: this.tools.length, toolNames: this.tools.map(t => t.name) },
'Research subagent: starting stream with tools'
);
const systemChars = typeof systemMessage.content === 'string'
? systemMessage.content.length
: JSON.stringify(systemMessage.content).length;
const humanChars = typeof humanMessage.content === 'string'
? humanMessage.content.length
: JSON.stringify(humanMessage.content).length;
this.logger.info(
{ systemChars, humanChars, approxInputKB: Math.round((systemChars + humanChars) / 1024) },
'Research subagent: input context size'
);
const stream = agent.stream(
{ messages: [humanMessage] },
{ streamMode: ['messages', 'updates'], recursionLimit: 40, signal }
);
let finalText = '';
let updateCount = 0;
for await (const [mode, data] of await stream) {
if (signal?.aborted) break;
if (mode === 'messages') {
// Real-time token streaming from the LLM — data is [BaseMessage, metadata]
for (const chunk of ResearchSubagent.extractStreamChunks(data, this.config.name)) {
yield chunk;
}
} else if (mode === 'updates') {
updateCount++;
const updateKeys = Object.keys(data as any);
this.logger.debug({ updateCount, updateKeys }, 'Research subagent: graph update');
// Agent node fired — yield tool call decisions before tools run
if ((data as any).agent?.messages) {
for (const msg of (data as any).agent.messages as any[]) {
if (msg.tool_calls?.length) {
for (const tc of msg.tool_calls) {
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
}
} else {
// Capture final text for return value (already streamed via messages above)
const content = ResearchSubagent.extractFinalText(msg);
if (content) finalText = content;
}
}
}
}
}
this.lastImages = [...this.imageCapture];
if (!finalText) {
this.logger.warn(
{ imageCount: this.lastImages.length },
'Research subagent: model returned empty output'
);
} else {
this.logger.info(
{ textLength: finalText.length, imageCount: this.lastImages.length },
'streamEvents finished'
);
}
return finalText;
}
/**
* Stream research execution
*/

View File

@@ -421,6 +421,7 @@ For research scripts, import and use get_api() to access the API:
"""
import logging
import threading
from typing import Optional
from dexorder.api.api import API
@@ -432,10 +433,13 @@ log = logging.getLogger(__name__)
# Global API instance - managed by main.py
_global_api: Optional[API] = None
# Thread-local API — used by harness threads so they don't overwrite the global
_thread_local = threading.local()
def get_api() -> API:
"""
Get the global API instance for accessing market data and charts.
Get the API instance for accessing market data and charts.
Use this in research scripts to access the data and charting APIs.
@@ -462,15 +466,27 @@ def get_api() -> API:
# Create chart
fig, ax = api.charting.plot_ohlc(df, title="BTC/USDT")
"""
# Thread-local takes priority (set by harness threads)
api = getattr(_thread_local, 'api', None)
if api is not None:
return api
if _global_api is None:
raise RuntimeError("API not initialized")
return _global_api
def set_api(api: API) -> None:
"""Set the global API instance. Internal use only."""
global _global_api
_global_api = api
"""Set the API instance.
When called from the main thread, sets the global API used by all threads.
When called from a non-main thread (e.g. harness threads), sets a thread-local
API so the global is not overwritten.
"""
if threading.current_thread() is threading.main_thread():
global _global_api
_global_api = api
else:
_thread_local.api = api
__all__ = ['API', 'ChartingAPI', 'DataAPI', 'get_api', 'set_api']

View File

@@ -28,11 +28,12 @@ from datetime import datetime
api = get_api()
# Method 1: Using Unix timestamps (seconds)
# 1609459200 = 2021-01-01, 1735689600 = 2025-01-01
df = asyncio.run(api.data.historical_ohlc(
ticker="BTC/USDT.BINANCE",
period_seconds=3600, # 1 hour candles
start_time=1640000000, # Unix timestamp in seconds
end_time=1640086400,
start_time=1609459200, # 2021-01-01
end_time=1735689600, # 2025-01-01 (~4 years, ~35,000 bars)
extra_columns=["volume"]
))
@@ -40,8 +41,8 @@ df = asyncio.run(api.data.historical_ohlc(
df = asyncio.run(api.data.historical_ohlc(
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20", # Simple date string
end_time="2021-12-21",
start_time="2021-01-01",
end_time="2025-01-01", # ~4 years of 1h bars ≈ 35,000 bars
extra_columns=["volume"]
))
@@ -49,21 +50,24 @@ df = asyncio.run(api.data.historical_ohlc(
df = asyncio.run(api.data.historical_ohlc(
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20 00:00:00",
end_time="2021-12-20 23:59:59",
start_time="2021-01-01 00:00:00",
end_time="2025-01-01 00:00:00",
extra_columns=["volume"]
))
# Method 4: Using datetime objects
from datetime import datetime, timedelta
end_time = datetime.now()
start_time = end_time - timedelta(days=4*365) # 4 years back
df = asyncio.run(api.data.historical_ohlc(
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time=datetime(2021, 12, 20),
end_time=datetime(2021, 12, 21),
start_time=start_time,
end_time=end_time,
extra_columns=["volume"]
))
print(f"Loaded {len(df)} candles")
print(f"Loaded {len(df)} candles from {df.index[0]} to {df.index[-1]}")
print(df.head())
```
@@ -94,8 +98,8 @@ api = get_api()
df = asyncio.run(api.data.historical_ohlc(
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20",
end_time="2021-12-21",
start_time="2021-01-01",
end_time="2025-01-01", # ~4 years of 1h bars
extra_columns=["volume"]
))
@@ -125,8 +129,8 @@ api = get_api()
df = asyncio.run(api.data.historical_ohlc(
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20",
end_time="2021-12-21"
start_time="2021-01-01",
end_time="2025-01-01"
))
# Calculate indicators using pandas-ta
@@ -190,14 +194,19 @@ import pandas_ta as ta
# Get API instance
api = get_api()
# Fetch historical data using date strings (easiest for research)
# Fetch historical data — use max history for research (target 100k-200k bars)
from datetime import datetime, timedelta
end_time = datetime.now()
start_time = end_time - timedelta(days=3*365) # 3 years of 1h bars ≈ 26,000 bars
df = asyncio.run(api.data.historical_ohlc(
ticker="BTC/USDT.BINANCE",
period_seconds=3600, # 1 hour
start_time="2021-12-20",
end_time="2021-12-21",
start_time=start_time,
end_time=end_time,
extra_columns=["volume"]
))
print(f"[Data] {len(df)} bars | {df.index[0]}{df.index[-1]} | period=3600s")
# Add moving averages using pandas-ta
df['sma_20'] = ta.sma(df['close'], length=20)
@@ -218,7 +227,7 @@ ax.plot(range(len(df)), df['ema_50'], label="EMA 50", color="red", linewidth=1.5
ax.legend()
# Print summary statistics
print(f"Period: {len(df)} candles")
print(f"[Data] {len(df)} bars | {df.index[0]}{df.index[-1]} | period=3600s")
print(f"High: {df['high'].max()}")
print(f"Low: {df['low'].min()}")
print(f"Mean Volume: {df['volume'].mean():.2f}")

View File

@@ -10,6 +10,33 @@ Create Python scripts that:
- Generate professional charts using matplotlib via the ChartingAPI
- All matplotlib figures are automatically captured and sent to the user as images
## Data Selection: Resolution and Time Window
> **Rule**: Every research script must fetch the maximum useful history — target 100,000200,000 bars, hard cap at 5 years. **Never** use short windows like "last 7 days" or "last 60 days" unless the user explicitly requests a specific recent period.
Choose the **coarsest** resolution that still captures the effect being studied:
| Phenomenon | Appropriate resolution |
|---|---|
| Intraday session opens/overlaps, hourly patterns | 15m (900s) |
| Short-term momentum, 530 min microstructure | 5m (300s) |
| Daily-level patterns (day-of-week, open/close effects) | 1h (3600s) |
| Multi-day / weekly effects | 4h (14400s) |
| Monthly / macro effects | 1d (86400s) |
Finer resolution than necessary adds noise and reduces statistical power. A session-open effect that plays out over 3060 minutes is fully visible on 15m bars.
Quick reference — approximate bars per resolution at various windows:
| Resolution | 1 year | 2 years | 5 years (max) |
|---|---|---|---|
| 5m | ~105,000 ✓ | ~210,000 → cap at ~1yr | ~525,000 → cap at ~1yr |
| 15m | ~35,000 | ~70,000 | ~175,000 ✓ |
| 1h | ~8,760 | ~17,520 | ~43,800 |
| 4h | ~2,190 | ~4,380 | ~10,950 |
**When to shorten the window**: only if 5 years at the chosen resolution would far exceed 200,000 bars (e.g., 5m over 5 years ≈ 525k → shorten to ~2 years). Otherwise always use the full 5 years.
## Available Tools
You have direct access to these MCP tools:
@@ -17,13 +44,15 @@ You have direct access to these MCP tools:
- **python_write**: Create a new script (research, strategy, or indicator category)
- Required: category, name, description, code
- Optional: metadata (category-specific fields — see below)
- For research: automatically executes the script after writing
- Returns validation results and execution output (text + images)
- **For research**: fully executes the script and returns all output (stdout, stderr) and captured chart images. The response IS the execution result — **do not call `execute_research` afterward**.
- **For indicator/strategy**: runs against synthetic test data to catch compile/runtime errors; no chart images are generated.
- Returns validation results and execution output (text + images for research)
- **python_edit**: Update an existing script
- Required: category, name
- Optional: code, description, metadata
- For research: automatically re-executes if code is updated
- **For research**: re-executes the script when code is changed and returns all output and images. **Do not call `execute_research` afterward**.
- **For indicator/strategy**: re-runs the validation test only.
- Returns validation results and execution output
- **python_read**: Read an existing research script
@@ -32,8 +61,9 @@ You have direct access to these MCP tools:
- **python_list**: List all research scripts
- Returns: array of {name, description, metadata}
- **execute_research**: Manually run a research script
- Note: Usually not needed since write/edit auto-execute
- **execute_research**: Run a research script that already exists on disk
- Use this **only** when the user explicitly asks to re-run a script, or to run a script that was written in a previous session and already exists
- **Do not call this after `python_write` or `python_edit`** — those tools already executed the script and returned its output
- Returns: text output and images
## Research Script API
@@ -55,180 +85,8 @@ See your knowledge base for complete API documentation, examples, and the full p
## Technical Indicators — pandas-ta
The sandbox environment uses **pandas-ta** as the standard indicator library. Always use it for technical indicator calculations; do not write manual rolling/ewm implementations.
Use `import pandas_ta as ta` for all indicator calculations. Never write manual rolling/ewm implementations. The full indicator catalog, calling conventions, column naming patterns, and default parameters are in `pandas-ta-reference.md` in your knowledge base.
```python
import pandas_ta as ta
```
### Calling Convention
pandas-ta functions accept a Series (or OHLCV columns) plus keyword parameters that match pandas-ta's documented argument names:
```python
# Single-series indicator
rsi = ta.rsi(df['close'], length=14) # returns Series
# OHLCV indicator
atr = ta.atr(df['high'], df['low'], df['close'], length=14)
# Multi-output indicator (returns DataFrame)
macd_df = ta.macd(df['close'], fast=12, slow=26, signal=9)
# columns: MACD_12_26_9, MACDh_12_26_9, MACDs_12_26_9
bbands_df = ta.bbands(df['close'], length=20, std=2.0)
# columns: BBL_20_2.0, BBM_20_2.0, BBU_20_2.0, BBB_20_2.0, BBP_20_2.0
```
### Available Indicators (canonical list)
These match the indicators supported by the TradingView web client. Use the pandas-ta function name shown here (lowercase):
**Overlap / Moving Averages** — plotted on the price pane
| Function | Description |
|----------|-------------|
| `sma` | Simple Moving Average — plain arithmetic mean over `length` periods |
| `ema` | Exponential Moving Average — more weight on recent prices |
| `wma` | Weighted Moving Average — linearly increasing weights |
| `dema` | Double EMA — two layers of EMA to reduce lag |
| `tema` | Triple EMA — three layers of EMA, even less lag than DEMA |
| `trima` | Triangular MA — double-smoothed SMA, very smooth |
| `kama` | Kaufman Adaptive MA — adapts speed to market noise/trending conditions |
| `t3` | T3 Moving Average — Tillson's smooth, low-lag MA using six EMAs |
| `hma` | Hull MA — very low-lag MA using WMAs |
| `alma` | Arnaud Legoux MA — Gaussian-weighted MA with reduced lag and noise |
| `midpoint` | Midpoint of close over `length` periods: (highest + lowest) / 2 |
| `midprice` | Midpoint of high/low over `length` periods |
| `supertrend` | Trend-following band (ATR-based) that flips above/below price |
| `ichimoku` | Ichimoku Cloud — multi-line Japanese trend/support/resistance system |
| `vwap` | Volume-Weighted Average Price — average price weighted by volume, resets on `anchor` |
| `vwma` | Volume-Weighted MA — like SMA but candles weighted by volume |
| `bbands` | Bollinger Bands — SMA ± N standard deviations; returns upper, mid, lower bands |
**Momentum** — typically plotted in a separate pane
| Function | Description |
|----------|-------------|
| `rsi` | Relative Strength Index — 0100 oscillator measuring speed of price changes |
| `macd` | MACD — difference of two EMAs plus signal line and histogram |
| `stoch` | Stochastic Oscillator — %K/%D, measures close vs recent high/low range |
| `stochrsi` | Stochastic RSI — applies stochastic formula to RSI values |
| `cci` | Commodity Channel Index — deviation of price from its statistical mean |
| `willr` | Williams %R — inverse stochastic, 100 to 0 oscillator |
| `mom` | Momentum — raw price change over `length` periods |
| `roc` | Rate of Change — percentage price change over `length` periods |
| `trix` | TRIX — 1-period % change of a triple-smoothed EMA |
| `cmo` | Chande Momentum Oscillator — ratio of up/down momentum, 100 to 100 |
| `adx` | Average Directional Index — strength of trend (0100, direction-agnostic) |
| `aroon` | Aroon — measures how recently the highest/lowest price occurred; returns Up, Down, Oscillator |
| `ao` | Awesome Oscillator — difference of 5- and 34-period simple MAs of midprice |
| `bop` | Balance of Power — measures buying vs selling pressure: (closeopen)/(highlow) |
| `uo` | Ultimate Oscillator — weighted combo of three period (fast/medium/slow) buying pressure ratios |
| `apo` | Absolute Price Oscillator — difference between two EMAs (like MACD without signal line) |
| `mfi` | Money Flow Index — RSI-like oscillator using price × volume |
| `coppock` | Coppock Curve — long-term momentum oscillator based on rate-of-change |
| `dpo` | Detrended Price Oscillator — removes trend to show cycle oscillations |
| `fisher` | Fisher Transform — converts price into a Gaussian normal distribution |
| `rvgi` | Relative Vigor Index — compares closeopen to highlow to measure trend vigor |
| `kst` | Know Sure Thing — momentum oscillator from four ROC periods, smoothed |
**Volatility** — plotted on price pane or separate
| Function | Description |
|----------|-------------|
| `atr` | Average True Range — average of true range (greatest of HL, HprevC, LprevC) |
| `kc` | Keltner Channels — EMA ± N × ATR bands around price |
| `donchian` | Donchian Channels — highest high / lowest low over `length` periods |
**Volume** — plotted in separate pane
| Function | Description |
|----------|-------------|
| `obv` | On Balance Volume — cumulative volume, added on up days, subtracted on down days |
| `ad` | Accumulation/Distribution — running total of the money flow multiplier × volume |
| `adosc` | Chaikin Oscillator — EMA difference of the A/D line |
| `cmf` | Chaikin Money Flow — sum of (money flow volume) / sum of volume over `length` |
| `eom` | Ease of Movement — relates price change to volume; high = price moves easily |
| `efi` | Elder's Force Index — combines price change direction with volume magnitude |
| `kvo` | Klinger Volume Oscillator — EMA difference of volume force |
| `pvt` | Price Volume Trend — cumulative: volume × percentage price change |
**Statistics / Price Transforms**
| Function | Description |
|----------|-------------|
| `stdev` | Standard Deviation of close over `length` periods |
| `linreg` | Linear Regression Curve — least-squares line endpoint value over `length` periods |
| `slope` | Linear Regression Slope — gradient of the regression line |
| `hl2` | Median Price — (high + low) / 2 |
| `hlc3` | Typical Price — (high + low + close) / 3 |
| `ohlc4` | Average Price — (open + high + low + close) / 4 |
**Trend**
| Function | Description |
|----------|-------------|
| `psar` | Parabolic SAR — trailing stop-and-reverse dots that follow price |
| `vortex` | Vortex Indicator — VI+ / VI lines measuring upward vs downward trend movement |
| `chop` | Choppiness Index — 0100, high = choppy/sideways, low = strong trend |
### Default Parameters
Key defaults to keep in mind:
- Most period/length indicators: `length=14` (use `length=` not `timeperiod=`)
- `bbands`: `length=20, std=2.0` (note: single `std`, not separate upper/lower)
- `macd`: `fast=12, slow=26, signal=9`
- `stoch`: `k=14, d=3, smooth_k=3`
- `psar`: `af0=0.02, af=0.02, max_af=0.2`
- `vwap`: `anchor='D'` (requires DatetimeIndex)
- `ichimoku`: `tenkan=9, kijun=26, senkou=52`
For multi-output indicator column extraction patterns and complete charting examples, fetch `pandas-ta-reference.md` from your knowledge base.
## Strategy Metadata Format
When writing or editing a strategy (`category="strategy"`), always include a `metadata` object with:
- **`data_feeds`** — list of feed descriptors the strategy requires:
```json
[
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 3600, "description": "Primary BTC/USDT hourly feed"},
{"symbol": "ETH/USDT.BINANCE", "period_seconds": 3600, "description": "ETH/USDT hourly for correlation"}
]
```
`period_seconds` must match what the strategy code expects. Use the same values when calling `backtest_strategy`.
- **`parameters`** — object documenting every configurable parameter in the strategy:
```json
{
"rsi_length": {"default": 14, "description": "RSI lookback period in bars"},
"overbought": {"default": 70, "description": "RSI level above which position is closed"},
"oversold": {"default": 30, "description": "RSI level below which long entry is triggered"},
"stop_pct": {"default": 0.02, "description": "Stop-loss as a fraction of entry price (e.g. 0.02 = 2%)"}
}
```
Include every parameter that appears as a constant in the strategy's `__init__` or class body — use the actual default values from the code.
Example `python_write` call for a strategy:
```json
{
"category": "strategy",
"name": "RSI Mean Reversion",
"description": "Long when RSI crosses above oversold; exit when overbought or stop hit",
"code": "...",
"metadata": {
"data_feeds": [
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 3600, "description": "BTC/USDT hourly OHLCV + order flow"}
],
"parameters": {
"rsi_length": {"default": 14, "description": "RSI lookback period"},
"overbought": {"default": 70, "description": "Exit long above this RSI level"},
"oversold": {"default": 30, "description": "Enter long below this RSI level"}
}
}
}
```
## Coding Loop Pattern
@@ -244,11 +102,11 @@ When a user requests analysis:
- Use appropriate ticker symbols, time ranges, and periods
- The script will auto-execute after writing
4. **Check execution results**: The tool returns:
- `validation.success`: Whether script ran without errors
- `validation.output`: Any stdout/stderr text output
- `execution.content`: Array of text and image results
- Note: Images are NOT included in your context - only text output is visible to you
4. **Check execution results**: The tool returns the execution result directly — this is the script's actual output:
- `success`: Whether the script ran without errors
- Text output from stdout/stderr is visible to you
- Chart images are captured and sent to the user (you cannot see them)
- **Do NOT call `execute_research` after this step** — the script has already run and the results are in the response above
5. **Iterate if needed**: If there are errors:
- Read the error message from validation.output or execution text
@@ -259,8 +117,28 @@ When a user requests analysis:
- The user will receive both your text response AND the chart images
- Don't try to describe the images in detail - the user can see them
## Ticker Format
All tickers passed to `api.data.historical_ohlc()` and other data methods **must** use the `SYMBOL.EXCHANGE` format, e.g.:
- `BTC/USDT.BINANCE`
- `ETH/USDT.BINANCE`
- `SOL/USDT.BINANCE`
**Never** use bare exchange-style tickers like `BTCUSDT`, `ETHUSDT`, or `BTCUSD` — these will fail with a format error.
If the instruction you receive includes a ticker in an incorrect format (e.g., `ETHUSDT`), convert it to the proper format (`ETH/USDT.BINANCE`) before writing the script. When in doubt about which exchange to use, default to `BINANCE`.
If you're unsure whether a given symbol exists or what its correct name is, print a clear error message from the script and ask the user to use the `symbol_lookup` tool at the top-level to find the correct ticker.
## Important Guidelines
- **Always print data stats after fetching**: Immediately after every `historical_ohlc` call, print the bar count and date range so it appears in the output:
```python
print(f"[Data] {len(df)} bars | {df.index[0]} → {df.index[-1]} | period={period_seconds}s")
```
This confirms the data window to both you and the user.
- **Images are pass-through only**: Chart images go directly to the user. You only see text output (print statements, errors). Don't try to analyze or describe images you can't see.
- **Async data fetching**: All `api.data` methods are async. Always use `asyncio.run()`:
@@ -268,15 +146,6 @@ When a user requests analysis:
df = asyncio.run(api.data.historical_ohlc(...))
```
- **Charting is sync**: All `api.charting` methods are synchronous:
```python
fig, ax = api.charting.plot_ohlc(df, title="BTC/USDT")
```
- **Automatic figure capture**: All matplotlib figures are automatically captured. Don't save manually.
- **Print for debugging**: Use `print()` statements for debugging - you'll see this output.
- **Package management**: If script needs packages beyond base environment (pandas, numpy, matplotlib):
- Add `conda_packages: ["package-name"]` to metadata
- Packages are auto-installed during validation
@@ -287,16 +156,18 @@ When a user requests analysis:
## Example Workflow
User: "Show me BTC price action for the last 7 days with volume"
User: "Show me BTC/ETH price correlation over time"
You:
1. Call `python_write` with:
- name: "BTC 7-Day Price Action"
- description: "BTC/USDT price and volume analysis for the last 7 days"
- code: (Python script that fetches data and creates chart)
2. Check execution results
3. If successful, respond: "I've created a 7-day BTC price chart with volume analysis. The chart shows [brief summary of what the script does]."
4. User receives: Your text response + the actual chart image
1. Identify timescale: daily return correlation → 1h bars are sufficient
2. Compute window: 1h bars × 5 years ≈ 43,800 bars (under 100k, but 5yr is the hard max — use it)
3. Call `python_write` with:
- name: "BTC ETH Price Correlation"
- description: "Rolling correlation of BTC/USDT and ETH/USDT daily returns using 5 years of 1h data"
- code: (Python script fetching 5yr of 1h OHLC for both tickers and plotting rolling correlation)
4. Check execution results
5. If successful, respond with a brief summary of what the script does
6. User receives: Your text response + the chart image
## Response Format

View File

@@ -0,0 +1,37 @@
name: strategy
description: Writes and manages PandasStrategy classes, runs backtests, and manages strategy activation
# Model configuration
model: claude-sonnet-4-6
temperature: 0.3
maxTokens: 16384
# Memory files loaded from memory/ directory
memoryFiles: []
# System prompt
systemPromptFile: system-prompt.md
# Capabilities
capabilities:
- strategy_writing
- backtesting
- strategy_lifecycle
# Tools available to this subagent
tools:
platform: []
mcp:
- python_write
- python_edit
- python_read
- python_list
- python_log
- python_revert
- backtest_strategy
- activate_strategy
- deactivate_strategy
- list_active_strategies
- get_backtest_results
- get_strategy_trades
- get_strategy_events

View File

@@ -0,0 +1,159 @@
import { BaseSubagent, type SubagentConfig, type SubagentContext } from '../base-subagent.js';
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
import { SystemMessage } from '@langchain/core/messages';
import { createReactAgent } from '@langchain/langgraph/prebuilt';
import type { FastifyBaseLogger } from 'fastify';
import type { MCPClientConnector } from '../../mcp-client.js';
import type { HarnessEvent } from '../../harness-events.js';
/**
* Strategy Subagent
*
* Specialized agent for writing PandasStrategy classes, running backtests,
* and managing strategy activation/deactivation.
*
* Mirrors the pattern of IndicatorSubagent in indicator/index.ts.
*/
export class StrategySubagent extends BaseSubagent {
constructor(
config: SubagentConfig,
model: BaseChatModel,
logger: FastifyBaseLogger,
mcpClient?: MCPClientConnector,
tools?: any[]
) {
super(config, model, logger, mcpClient, tools);
}
/**
* Execute a strategy request using LangGraph's createReactAgent.
*/
async execute(context: SubagentContext, instruction: string): Promise<string> {
this.logger.info(
{
subagent: this.getName(),
userId: context.userContext.userId,
instruction: instruction.substring(0, 200),
toolCount: this.tools.length,
toolNames: this.tools.map(t => t.name),
},
'Strategy subagent starting'
);
if (!this.hasMCPClient()) {
throw new Error('MCP client not available for strategy subagent');
}
if (this.tools.length === 0) {
this.logger.warn('Strategy subagent has no tools');
}
const initialMessages = this.buildMessages(context, instruction);
const systemMessage = initialMessages[0];
const humanMessage = initialMessages[initialMessages.length - 1];
const agent = createReactAgent({
llm: this.model,
tools: this.tools,
prompt: systemMessage as SystemMessage,
});
const result = await agent.invoke(
{ messages: [humanMessage] },
{ recursionLimit: 30 }
);
const allMessages: any[] = result.messages ?? [];
this.logger.info(
{ messageCount: allMessages.length },
'Strategy subagent graph completed'
);
const lastAI = [...allMessages].reverse().find(
(m: any) => m.constructor?.name === 'AIMessage' || m._getType?.() === 'ai'
);
const finalText = lastAI
? (typeof lastAI.content === 'string' ? lastAI.content : JSON.stringify(lastAI.content))
: 'Strategy task completed.';
this.logger.info({ textLength: finalText.length }, 'Strategy subagent finished');
return finalText;
}
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
if (!this.hasMCPClient()) {
throw new Error('MCP client not available for strategy subagent');
}
const initialMessages = this.buildMessages(context, instruction);
const systemMessage = initialMessages[0];
const humanMessage = initialMessages[initialMessages.length - 1];
const agent = createReactAgent({
llm: this.model,
tools: this.tools,
prompt: systemMessage as SystemMessage,
});
const stream = agent.stream(
{ messages: [humanMessage] },
{ streamMode: ['messages', 'updates'], recursionLimit: 30, signal }
);
let finalText = '';
for await (const [mode, data] of await stream) {
if (signal?.aborted) break;
if (mode === 'messages') {
for (const chunk of StrategySubagent.extractStreamChunks(data, this.config.name)) {
yield chunk;
}
} else if (mode === 'updates') {
if ((data as any).agent?.messages) {
for (const msg of (data as any).agent.messages as any[]) {
if (msg.tool_calls?.length) {
for (const tc of msg.tool_calls) {
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
}
} else {
const content = StrategySubagent.extractFinalText(msg);
if (content) finalText = content;
}
}
}
}
}
this.logger.info({ textLength: finalText.length }, 'streamEvents finished');
return finalText;
}
}
/**
* Factory function to create and initialize StrategySubagent
*/
export async function createStrategySubagent(
model: BaseChatModel,
logger: FastifyBaseLogger,
basePath: string,
mcpClient?: MCPClientConnector,
tools?: any[]
): Promise<StrategySubagent> {
const { readFile } = await import('fs/promises');
const { join } = await import('path');
const yaml = await import('js-yaml');
const configPath = join(basePath, 'config.yaml');
const configContent = await readFile(configPath, 'utf-8');
const config = yaml.load(configContent) as SubagentConfig;
const subagent = new StrategySubagent(config, model, logger, mcpClient, tools);
await subagent.initialize(basePath);
return subagent;
}

View File

@@ -0,0 +1,357 @@
# Strategy Subagent
You are a specialized assistant for writing, testing, and managing trading strategies on the Dexorder platform. You write `PandasStrategy` subclasses, run backtests, and manage strategy activation.
---
## Section A — PandasStrategy API
All strategies inherit from `PandasStrategy`. Users implement a single method, `evaluate(dfs)`, which is called on every new bar.
### Class structure
```python
from dexorder.nautilus.pandas_strategy import PandasStrategy, PandasStrategyConfig
class MyStrategy(PandasStrategy):
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
"""
Called after every new bar across all feeds.
Args:
dfs: dict mapping feed_key → pd.DataFrame with columns:
timestamp (nanoseconds), open, high, low, close, volume,
buy_vol, sell_vol, open_interest
Rows accumulate over time — the last row is always the latest bar.
"""
df = dfs.get("BTC/USDT.BINANCE:300")
if df is None or len(df) < 20:
return # Not enough data yet
close = df["close"]
# ... compute signals ...
if buy_signal:
self.buy(quantity=0.1)
elif sell_signal:
self.sell(quantity=0.1)
```
### Feed key format
Feed keys combine the ticker and period: `"{ticker}:{period_seconds}"`
Examples:
- `"BTC/USDT.BINANCE:300"` — BTC/USDT on Binance, 5-minute bars
- `"BTC/USDT.BINANCE:900"` — BTC/USDT on Binance, 15-minute bars
- `"BTC/USDT.BINANCE:3600"` — BTC/USDT on Binance, 1-hour bars
- `"ETH/USDT.BINANCE:900"` — ETH/USDT on Binance, 15-minute bars
Access the feed key from metadata: `self.config.feed_keys` is a tuple of all feed keys.
### Order API
```python
self.buy(quantity: float, feed_key: str = None)
self.sell(quantity: float, feed_key: str = None)
self.flatten(feed_key: str = None) # Close all open positions
```
If `feed_key` is None, the first feed in `feed_keys` is used.
`quantity` is in base currency units (e.g. 0.1 BTC). Use `self.config.initial_capital` to size appropriately.
### Configuration available inside evaluate()
```python
self.config.feed_keys # tuple of feed key strings
self.config.initial_capital # starting capital in quote currency
```
### DataFrame columns
| Column | Type | Description |
|--------|------|-------------|
| `timestamp` | int64 (ns) | Bar open time in nanoseconds |
| `open` | float | Open price |
| `high` | float | High price |
| `low` | float | Low price |
| `close` | float | Close price |
| `volume` | float | Total volume |
| `buy_vol` | float | Buy-side volume (taker buys) |
| `sell_vol` | float | Sell-side volume (taker sells) |
| `open_interest` | float | Open interest (futures only; NaN for spot) |
---
## Section B — Strategy Metadata
When writing a strategy with `python_write(category="strategy", ...)`, always provide complete metadata:
```python
python_write(
category="strategy",
name="RSI Mean Reversion",
description="Buy oversold, sell overbought based on RSI(14) on BTC/USDT 1h bars.",
code="""...""",
metadata={
"data_feeds": [
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 300, "description": "Primary BTC/USDT 5m feed"}
],
"parameters": {
"rsi_length": {"default": 14, "description": "RSI lookback period"},
"oversold": {"default": 30, "description": "RSI oversold threshold"},
"overbought": {"default": 70, "description": "RSI overbought threshold"},
"trade_qty": {"default": 0.01, "description": "Trade quantity in BTC"}
},
"conda_packages": []
}
)
```
### Metadata fields
| Field | Required | Description |
|-------|----------|-------------|
| `data_feeds` | yes | List of `{symbol, period_seconds, description}` — one per feed the strategy needs |
| `parameters` | yes | Dict of `{param_name: {default, description}}` for user-configurable values |
| `conda_packages` | no | Extra Python packages to install |
---
## Section C — Custom Indicators in Strategies
**Prefer using custom indicators defined in the `indicator` category rather than computing signals inline.**
Benefits:
- The indicator appears on the user's chart, making the signal transparent
- It can be reused across strategies without copy-pasting
- It is tested independently via the indicator harness
Before writing indicator logic, check if an indicator already exists:
```
python_list(category="indicator")
```
To use a custom indicator in a strategy:
```python
import pandas_ta as ta
def evaluate(self, dfs):
df = dfs.get("BTC/USDT.BINANCE:3600")
if df is None or len(df) < 20:
return
# Use a custom indicator registered as ta.custom_vw_rsi
vw_rsi = ta.custom_vw_rsi(df["close"], df["volume"], length=14)
if vw_rsi.iloc[-1] < 30:
self.buy(0.01)
elif vw_rsi.iloc[-1] > 70:
self.sell(0.01)
```
Custom indicator names follow the pattern `ta.custom_{sanitized_name}` where the sanitized name is the indicator's name lowercased with spaces replaced by underscores.
**When a user asks for a strategy that needs a novel signal, first create the indicator, then reference it in the strategy.**
---
## Section D — Complete Strategy Examples
### Example 1: RSI Mean Reversion (simple, single feed)
```python
import pandas as pd
import pandas_ta as ta
class RSIMeanReversion(PandasStrategy):
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
df = dfs.get("BTC/USDT.BINANCE:300")
if df is None or len(df) < 30:
return
rsi = ta.rsi(df["close"], length=14)
if rsi is None or rsi.isna().all():
return
last_rsi = rsi.iloc[-1]
trade_qty = 0.001 * self.config.initial_capital / df["close"].iloc[-1]
if last_rsi < 30:
self.buy(trade_qty)
elif last_rsi > 70:
self.sell(trade_qty)
```
Metadata:
```python
{
"data_feeds": [{"symbol": "BTC/USDT.BINANCE", "period_seconds": 300, "description": "BTC/USDT 5m"}],
"parameters": {
"rsi_length": {"default": 14, "description": "RSI period"},
"oversold": {"default": 30, "description": "Buy threshold"},
"overbought": {"default": 70, "description": "Sell threshold"}
},
"conda_packages": []
}
```
### Example 2: MACD Momentum (multi-feed dual timeframe)
```python
import pandas as pd
import pandas_ta as ta
class MACDMomentum(PandasStrategy):
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
df_15m = dfs.get("BTC/USDT.BINANCE:900")
df_4h = dfs.get("BTC/USDT.BINANCE:14400")
if df_15m is None or df_4h is None:
return
if len(df_15m) < 50 or len(df_4h) < 50:
return
# Higher-timeframe trend filter
ema_4h = ta.ema(df_4h["close"], length=20)
bullish_trend = df_4h["close"].iloc[-1] > ema_4h.iloc[-1]
# Entry signal on 15m
macd_df = ta.macd(df_15m["close"], fast=12, slow=26, signal=9)
if macd_df is None:
return
hist = macd_df.iloc[:, 2] # histogram
trade_qty = 0.002 * self.config.initial_capital / df_15m["close"].iloc[-1]
if bullish_trend and hist.iloc[-1] > 0 and hist.iloc[-2] <= 0:
self.buy(trade_qty, feed_key="BTC/USDT.BINANCE:900")
elif hist.iloc[-1] < 0 and hist.iloc[-2] >= 0:
self.flatten()
```
Metadata:
```python
{
"data_feeds": [
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 900, "description": "BTC/USDT 15m entry"},
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 14400, "description": "BTC/USDT 4h trend filter"}
],
"parameters": {},
"conda_packages": []
}
```
### Example 3: Volume Breakout (uses custom indicator)
```python
import pandas as pd
import pandas_ta as ta
class VolumeBreakout(PandasStrategy):
"""Breakout strategy using a custom volume-weighted RSI indicator."""
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
df = dfs.get("ETH/USDT.BINANCE:300")
if df is None or len(df) < 20:
return
# Custom indicator (must exist in the indicator category)
vw_rsi = ta.custom_vw_rsi(df["close"], df["volume"], length=14)
if vw_rsi is None:
return
donchian = ta.donchian(df["high"], df["low"], lower_length=20, upper_length=20)
if donchian is None:
return
upper = donchian.iloc[:, 0]
close = df["close"]
qty = 0.01 * self.config.initial_capital / close.iloc[-1]
if close.iloc[-1] > upper.iloc[-2] and vw_rsi.iloc[-1] > 60:
self.buy(qty)
elif close.iloc[-1] < donchian.iloc[:, 1].iloc[-1]:
self.flatten()
```
---
## Section E — Workflow
### Writing and validating a strategy
1. **Check for existing indicators first**: `python_list(category="indicator")` — reuse signals already defined rather than recomputing them inline.
2. **Write the strategy**:
```
python_write(category="strategy", name="...", description="...", code="...", metadata={...})
```
After writing, the system automatically runs the strategy against synthetic data. If validation fails, fix the reported error before proceeding.
3. **Run a backtest** — choose the window to target 100k200k bars at the strategy's resolution (max 5 years):
```
backtest_strategy(
strategy_name="RSI Mean Reversion",
feeds=[{"symbol": "BTC/USDT.BINANCE", "period_seconds": 900}], # 15m → 2 years ≈ 70k bars
from_time="2023-01-01",
to_time="2024-12-31",
initial_capital=10000
)
```
4. **Interpret results**:
- `summary.total_return` — total fractional return (0.15 = +15%)
- `summary.sharpe_ratio` — annualized Sharpe (>1.0 good, >2.0 excellent)
- `summary.max_drawdown` — maximum peak-to-trough loss (0.20 = 20%)
- `summary.win_rate` — fraction of trades profitable
- `statistics.profit_factor` — gross profit / gross loss (>1.5 good)
- `statistics.sortino_ratio` — Sharpe using only downside deviation
- `trades` — list of individual round-trip trades
- `equity_curve` — portfolio value over time
5. **Iterate**: edit with `python_edit`, re-run backtest, compare results. Use `get_backtest_results` to compare multiple runs.
6. **Activate** when satisfied:
```
activate_strategy(
strategy_name="RSI Mean Reversion",
feeds=[{"symbol": "BTC/USDT.BINANCE", "period_seconds": 900}],
allocation=5000.0,
paper=True
)
```
### Monitoring active strategies
```
list_active_strategies() # See all running strategies and PnL
get_strategy_trades(strategy_name) # View recent trade log
get_strategy_events(strategy_name) # View fills, errors, PnL updates
deactivate_strategy(strategy_name) # Stop and get final PnL
```
---
## Section F — Important Rules
1. **Always start with `python_list(category="indicator")`** before writing a new strategy. If the signals it needs already exist as custom indicators, use them via `ta.custom_*` rather than duplicating the computation.
2. **Wait for validation output** after `python_write` or `python_edit`. If the harness reports an error, fix it before running a backtest.
3. **Size positions conservatively** based on `self.config.initial_capital`. A typical trade quantity is `0.0010.01 * initial_capital / price`.
4. **Guard for insufficient data**: always check `len(df) >= min_required` before computing indicators that need a lookback period.
5. **Multi-feed strategies**: access each feed by its exact feed key. Missing feeds (not yet warmed up) will be absent from `dfs` — always use `.get()` and check for `None`.
6. **Bar resolution and backtest window**: Choose the bar resolution that fits the strategy's signal frequency and holding period. Once resolution is chosen, set the date window to target **100,000200,000 bars**. **Never request more than 5 years of data.** If 5 years at the chosen resolution would exceed 200,000 bars, shorten the window rather than coarsening the resolution. Quick reference:
- 5m bars: 100k bars ≈ 1 year; 200k bars ≈ 2 years
- 15m bars: 100k bars ≈ 2.9 years; 200k bars ≈ 5 years (at limit)
- 1h bars: 100k bars ≈ 11.4 years → cap at 5 years (≈ 43,800 bars)
- 4h bars: 100k bars ≈ 45 years → cap at 5 years (≈ 10,950 bars)
7. **Never `import` from `dexorder` inside `evaluate()`** — the strategy file is exec'd in a sandbox with PandasStrategy and pandas_ta pre-loaded. Standard library and pandas/numpy/pandas_ta are available.

View File

@@ -3,6 +3,7 @@ import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { SystemMessage } from '@langchain/core/messages';
import { createReactAgent } from '@langchain/langgraph/prebuilt';
import type { FastifyBaseLogger } from 'fastify';
import type { HarnessEvent } from '../../harness-events.js';
/**
* Web Explore Subagent
@@ -66,6 +67,52 @@ export class WebExploreSubagent extends BaseSubagent {
return finalText;
}
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
const initialMessages = this.buildMessages(context, instruction);
const systemMessage = initialMessages[0];
const humanMessage = initialMessages[initialMessages.length - 1];
const agent = createReactAgent({
llm: this.model,
tools: this.tools,
prompt: systemMessage as SystemMessage,
});
const stream = agent.stream(
{ messages: [humanMessage] },
{ streamMode: ['messages', 'updates'], recursionLimit: 15, signal }
);
let finalText = '';
for await (const [mode, data] of await stream) {
if (signal?.aborted) break;
if (mode === 'messages') {
for (const chunk of WebExploreSubagent.extractStreamChunks(data, this.config.name)) {
yield chunk;
}
} else if (mode === 'updates') {
if ((data as any).agent?.messages) {
for (const msg of (data as any).agent.messages as any[]) {
if (msg.tool_calls?.length) {
for (const tc of msg.tool_calls) {
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
}
} else {
const content = WebExploreSubagent.extractFinalText(msg);
if (content) finalText = content;
}
}
}
}
}
this.logger.info({ textLength: finalText.length }, 'streamEvents finished');
return finalText;
}
}
/**

View File

@@ -16,6 +16,8 @@ import { ContainerManager } from './k8s/container-manager.js';
import { ZMQRelayClient } from './clients/zmq-relay-client.js';
import { IcebergClient } from './clients/iceberg-client.js';
import { ConversationStore } from './harness/memory/conversation-store.js';
import { BlobStore } from './harness/memory/blob-store.js';
import { ConversationService } from './services/conversation-service.js';
import { AgentHarness, type HarnessSessionConfig } from './harness/agent-harness.js';
import { OHLCService } from './services/ohlc-service.js';
import { SymbolIndexService } from './services/symbol-index-service.js';
@@ -369,12 +371,17 @@ try {
const conversationStore = new ConversationStore(redis, app.log, icebergClient);
app.log.debug('Conversation store initialized');
const blobStore = new BlobStore(icebergClient, app.log);
const conversationService = new ConversationService(conversationStore, blobStore, app.log);
app.log.debug('Blob store and conversation service initialized');
// Harness factory: captures infrastructure deps; channel handlers stay infrastructure-free
function createHarness(sessionConfig: HarnessSessionConfig): AgentHarness {
return new AgentHarness({
...sessionConfig,
providerConfig: config.providerConfig,
conversationStore,
blobStore,
historyLimit: config.conversationHistoryLimit,
});
}
@@ -391,6 +398,7 @@ const websocketHandler = new WebSocketHandler({
createHarness,
ohlcService, // Optional
symbolIndexService, // Optional
conversationService, // Optional - for history replay on reconnect
});
app.log.debug('WebSocket handler initialized');
@@ -614,6 +622,19 @@ try {
mcpTools: [],
});
// Strategy subagent: all strategy-related MCP tools
toolRegistry.registerAgentTools({
agentName: 'strategy',
platformTools: [],
mcpTools: [
'python_write', 'python_edit', 'python_read', 'python_list',
'python_log', 'python_revert',
'backtest_strategy', 'activate_strategy', 'deactivate_strategy',
'list_active_strategies', 'get_backtest_results',
'get_strategy_trades', 'get_strategy_events',
],
});
app.log.info(
{
agents: toolRegistry.getRegisteredAgents(),

View File

@@ -0,0 +1,59 @@
import type { FastifyBaseLogger } from 'fastify';
import type { ConversationStore } from '../harness/memory/conversation-store.js';
import type { BlobStore, StoredBlob } from '../harness/memory/blob-store.js';
export interface EnrichedMessage {
id: string;
userId: string;
sessionId: string;
role: 'user' | 'assistant';
content: string;
timestamp: number; // microseconds
files: StoredBlob[];
}
/**
* Generic conversation history service.
*
* Combines text messages (ConversationStore) with binary blobs (BlobStore)
* into enriched message records. Used by:
* - WebSocket handler: replay history on reconnect
* - Future admin panel: conversation browser
*/
export class ConversationService {
constructor(
private conversationStore: ConversationStore,
private blobStore: BlobStore,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
_logger: FastifyBaseLogger
) {}
async getHistory(
userId: string,
sessionId: string,
limit = 50,
channelType = 'websocket'
): Promise<EnrichedMessage[]> {
const messages = await this.conversationStore.getFullHistory(userId, sessionId, limit, channelType);
const chatMessages = messages.filter(m => m.role === 'user' || m.role === 'assistant');
return Promise.all(
chatMessages.map(async (m) => {
const blobRefs = m.metadata?.blobs as Array<{ id: string; mimeType: string; caption?: string }> | undefined;
const files = blobRefs?.length
? await this.blobStore.getBlobsByIds(userId, sessionId, blobRefs.map(b => b.id))
: [];
return {
id: m.id,
userId: m.userId,
sessionId: m.sessionId,
role: m.role as 'user' | 'assistant',
content: m.content,
timestamp: m.timestamp,
files,
};
})
);
}
}

View File

@@ -16,7 +16,8 @@
import type { FastifyBaseLogger } from 'fastify';
import type { IcebergClient } from '../clients/iceberg-client.js';
import type { ZMQRelayClient } from '../clients/zmq-relay-client.js';
import type { ZMQRelayClient, BarUpdateCallback } from '../clients/zmq-relay-client.js';
export type { BarUpdateCallback } from '../clients/zmq-relay-client.js';
import type {
HistoryResult,
SymbolInfo,
@@ -53,6 +54,23 @@ export class OHLCService {
this.logger = config.logger;
}
/**
* Subscribe to realtime OHLC bar updates for a ticker+period.
* ZMQ subscribe is issued on the first call for a given topic; subsequent calls
* for the same topic only add the callback (no extra ZMQ events).
*/
subscribeToTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
this.relayClient.subscribeToTicker(ticker, periodSeconds, callback);
}
/**
* Unsubscribe a callback from realtime OHLC bar updates.
* ZMQ unsubscribe is issued when the last callback for a topic is removed.
*/
unsubscribeFromTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
this.relayClient.unsubscribeFromTicker(ticker, periodSeconds, callback);
}
/**
* Fetch OHLC data with smart caching
*

View File

@@ -28,23 +28,29 @@ export function createGetChartDataTool(config: GetChartDataToolConfig): DynamicS
**IMPORTANT: Use this tool ONLY for quick, casual data viewing. For any analysis, plotting, statistics, or deep research, use the 'research' tool instead.**
**Hard limit: returns at most 500 bars (the most recent 500). This tool is not suitable for analysis requiring longer sequences — use the 'research' tool for that.**
Parameters:
- ticker (optional): Market symbol (defaults to workspace chartState.symbol)
- ticker (optional): Market symbol in SYMBOL.EXCHANGE format, e.g. "BTC/USDT.BINANCE" (defaults to workspace chartState.symbol)
- period (optional): OHLC period in seconds (defaults to workspace chartState.period)
- from_time (optional): Start time as Unix timestamp (number or string like "1774126800") OR date string like "2 days ago", "2024-01-01" (defaults to workspace chartState.start_time)
- to_time (optional): End time as Unix timestamp (number or string like "1774732500") OR date string like "now", "yesterday" (defaults to workspace chartState.end_time)
- countback (optional): Limit number of bars returned
- countback (optional): Limit number of bars returned (max 500)
- columns (optional): Extra columns beyond OHLC: ["volume", "buy_vol", "sell_vol", "open_time", "high_time", "low_time", "close_time", "open_interest"]`,
schema: z.object({
ticker: z.string().optional().describe('Market symbol (defaults to workspace chartState.symbol)'),
period: z.number().optional().describe('OHLC period in seconds (defaults to workspace chartState.period)'),
from_time: z.union([z.number(), z.string()]).optional().describe('Start time: Unix seconds OR date string (defaults to workspace chartState.start_time)'),
to_time: z.union([z.number(), z.string()]).optional().describe('End time: Unix seconds OR date string (defaults to workspace chartState.end_time)'),
countback: z.number().optional().describe('Limit number of bars returned'),
countback: z.number().optional().describe('Limit number of bars returned (max 500)'),
columns: z.array(z.enum(['volume', 'buy_vol', 'sell_vol', 'open_time', 'high_time', 'low_time', 'close_time', 'open_interest'])).optional().describe('Extra columns beyond OHLC'),
}),
func: async ({ ticker, period, from_time, to_time, countback, columns }) => {
logger.debug({ ticker, period, from_time, to_time, countback, columns }, 'Executing get_chart_data tool');
const MAX_BARS = 500;
// Enforce hard cap — never return more than MAX_BARS bars
const effectiveCountback = countback !== undefined ? Math.min(countback, MAX_BARS) : MAX_BARS;
logger.debug({ ticker, period, from_time, to_time, countback: effectiveCountback, columns }, 'Executing get_chart_data tool');
try {
// Get workspace chart state
@@ -86,7 +92,7 @@ Parameters:
finalPeriod,
finalFromTime,
finalToTime,
countback
effectiveCountback
);
if (historyResult.noData || !historyResult.bars || historyResult.bars.length === 0) {
@@ -98,8 +104,13 @@ Parameters:
});
}
// Enforce hard cap — keep the most recent bars
const sourceBars = historyResult.bars.length > MAX_BARS
? historyResult.bars.slice(-MAX_BARS)
: historyResult.bars;
// Filter/format bars with requested columns
const bars = historyResult.bars.map(bar => {
const bars = sourceBars.map(bar => {
const result: any = {
time: bar.time,
open: bar.open,

View File

@@ -3,6 +3,7 @@ import { z } from 'zod';
import type { FastifyBaseLogger } from 'fastify';
import type { IndicatorSubagent } from '../../harness/subagents/indicator/index.js';
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
import type { HarnessEvent } from '../../harness/harness-events.js';
export interface IndicatorAgentToolConfig {
indicatorSubagent: IndicatorSubagent;
@@ -14,10 +15,20 @@ export interface IndicatorAgentToolConfig {
* Creates a LangChain tool that delegates to the indicator subagent.
* Mirrors the pattern of research-agent.tool.ts.
*/
export function createIndicatorAgentTool(config: IndicatorAgentToolConfig): DynamicStructuredTool {
export function createIndicatorAgentTool(config: IndicatorAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }) => AsyncGenerator<HarnessEvent, string> } {
const { indicatorSubagent, context, logger } = config;
return new DynamicStructuredTool({
async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming indicator subagent');
const gen = indicatorSubagent.streamEvents(context, instruction, signal);
let step: IteratorResult<HarnessEvent, string>;
while (!(step = await gen.next()).done) {
yield step.value;
}
return step.value;
}
const tool = new DynamicStructuredTool({
name: 'indicator',
description: `Delegate to the indicator subagent for all indicator-related tasks on the chart.
@@ -50,4 +61,6 @@ NEVER modify the indicators workspace store directly.`,
}
},
});
return Object.assign(tool, { streamFunc });
}

View File

@@ -3,6 +3,7 @@ import { z } from 'zod';
import type { FastifyBaseLogger } from 'fastify';
import type { ResearchSubagent } from '../../harness/subagents/research/index.js';
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
import type { HarnessEvent } from '../../harness/harness-events.js';
export interface ResearchAgentToolConfig {
researchSubagent: ResearchSubagent;
@@ -15,10 +16,24 @@ export interface ResearchAgentToolConfig {
* This is the standard LangChain pattern for exposing a subagent as a tool
* to a parent agent.
*/
export function createResearchAgentTool(config: ResearchAgentToolConfig): DynamicStructuredTool {
export function createResearchAgentTool(config: ResearchAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { name: string; instruction: string }) => AsyncGenerator<HarnessEvent, string> } {
const { researchSubagent, context, logger } = config;
return new DynamicStructuredTool({
const prompt = (name: string, instruction: string) => `Research script name: "${name}"\n\n${instruction}`;
async function* streamFunc({ name, instruction }: { name: string; instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
logger.info({ name, instruction: instruction.substring(0, 100) }, 'Streaming research subagent');
const gen = researchSubagent.streamEvents(context, prompt(name, instruction), signal);
let step: IteratorResult<HarnessEvent, string>;
while (!(step = await gen.next()).done) {
yield step.value;
}
const finalText = step.value;
const images = researchSubagent.getLastImages();
return JSON.stringify({ text: finalText, images });
}
const tool = new DynamicStructuredTool({
name: 'research',
description: `Delegate to the research subagent for data analysis, charting, statistics, and Python script execution.
@@ -36,21 +51,15 @@ The research subagent will write and execute Python scripts, capture output and
func: async ({ name, instruction }: { name: string; instruction: string }): Promise<string> => {
logger.info({ name, instruction: instruction.substring(0, 100) }, 'Delegating to research subagent');
const prompt = `Research script name: "${name}"\n\n${instruction}`;
try {
const result = await researchSubagent.executeWithImages(context, prompt);
// Return in the format that AgentHarness.processToolResult() knows how to handle
// (extracts images and passes them to channelAdapter)
return JSON.stringify({
text: result.text,
images: result.images,
});
const result = await researchSubagent.executeWithImages(context, prompt(name, instruction));
return JSON.stringify({ text: result.text, images: result.images });
} catch (error) {
logger.error({ error, errorMessage: (error as Error)?.message }, 'Research subagent failed');
throw error;
}
},
});
return Object.assign(tool, { streamFunc });
}

View File

@@ -0,0 +1,66 @@
import { DynamicStructuredTool } from '@langchain/core/tools';
import { z } from 'zod';
import type { FastifyBaseLogger } from 'fastify';
import type { StrategySubagent } from '../../harness/subagents/strategy/index.js';
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
import type { HarnessEvent } from '../../harness/harness-events.js';
export interface StrategyAgentToolConfig {
strategySubagent: StrategySubagent;
context: SubagentContext;
logger: FastifyBaseLogger;
}
/**
* Creates a LangChain tool that delegates to the strategy subagent.
* Mirrors the pattern of indicator-agent.tool.ts.
*/
export function createStrategyAgentTool(config: StrategyAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }, signal?: AbortSignal) => AsyncGenerator<HarnessEvent, string> } {
const { strategySubagent, context, logger } = config;
async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming strategy subagent');
const gen = strategySubagent.streamEvents(context, instruction, signal);
let step: IteratorResult<HarnessEvent, string>;
while (!(step = await gen.next()).done) {
yield step.value;
}
return step.value;
}
const tool = new DynamicStructuredTool({
name: 'strategy',
description: `Delegate to the strategy subagent for all trading strategy tasks.
Use this tool for:
- Writing new PandasStrategy classes ("create a strategy that...")
- Editing or improving existing strategies
- Running backtests on a strategy
- Interpreting backtest results (Sharpe ratio, drawdown, trade list)
- Activating or deactivating strategies for paper trading
- Monitoring running strategy PnL and trade logs
- Checking which strategies already exist
ALWAYS use this tool for any request about trading strategies, backtesting, or strategy activation.
NEVER write strategy Python code or call backtest_strategy directly — delegate here instead.`,
schema: z.object({
instruction: z.string().describe(
'The strategy task to perform. Be specific: include the strategy name, ' +
'desired signals (e.g. RSI < 30 = buy), timeframe, and symbol if known. ' +
'For backtest requests include the date range and starting capital.'
),
}),
func: async ({ instruction }: { instruction: string }): Promise<string> => {
logger.info({ instruction: instruction.substring(0, 100) }, 'Delegating to strategy subagent');
try {
return await strategySubagent.execute(context, instruction);
} catch (error) {
logger.error({ error, errorMessage: (error as Error)?.message }, 'Strategy subagent failed');
throw error;
}
},
});
return Object.assign(tool, { streamFunc });
}

View File

@@ -3,6 +3,7 @@ import { z } from 'zod';
import type { FastifyBaseLogger } from 'fastify';
import type { WebExploreSubagent } from '../../harness/subagents/web-explore/index.js';
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
import type { HarnessEvent } from '../../harness/harness-events.js';
export interface WebExploreAgentToolConfig {
webExploreSubagent: WebExploreSubagent;
@@ -14,10 +15,20 @@ export interface WebExploreAgentToolConfig {
* Creates a LangChain tool that delegates to the web-explore subagent.
* The subagent decides whether to use web search or arXiv based on the instruction.
*/
export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool {
export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }, signal?: AbortSignal) => AsyncGenerator<HarnessEvent, string> } {
const { webExploreSubagent, context, logger } = config;
return new DynamicStructuredTool({
async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming web-explore subagent');
const gen = webExploreSubagent.streamEvents(context, instruction, signal);
let step: IteratorResult<HarnessEvent, string>;
while (!(step = await gen.next()).done) {
yield step.value;
}
return step.value;
}
const tool = new DynamicStructuredTool({
name: 'web_explore',
description: `Search the web or academic databases and return a summarized answer.
@@ -46,4 +57,6 @@ The subagent will search the web (or arXiv for academic queries), fetch relevant
}
},
});
return Object.assign(tool, { streamFunc });
}

View File

@@ -103,6 +103,16 @@ export const DEFAULT_STORES: StoreConfig[] = [
persistent: true,
initialState: () => ({}),
},
{
name: 'strategy_types',
persistent: true,
initialState: () => ({}),
},
{
name: 'research_types',
persistent: true,
initialState: () => ({}),
},
{
name: 'channelState',
persistent: false,