data pipeline refactor and fix
This commit is contained in:
@@ -72,7 +72,7 @@ export class Authenticator {
|
||||
);
|
||||
}
|
||||
|
||||
const sessionId = `ws_${userId}_${Date.now()}`;
|
||||
const sessionId = `ws_${userId}`;
|
||||
|
||||
return {
|
||||
authContext: {
|
||||
|
||||
@@ -2,12 +2,14 @@ import type { FastifyInstance, FastifyRequest } from 'fastify';
|
||||
import type { WebSocket } from '@fastify/websocket';
|
||||
import type { Authenticator } from '../auth/authenticator.js';
|
||||
import type { AgentHarness, HarnessFactory } from '../harness/agent-harness.js';
|
||||
import type { HarnessEvent } from '../harness/harness-events.js';
|
||||
import type { InboundMessage } from '../types/messages.js';
|
||||
import { randomUUID } from 'crypto';
|
||||
import type { SessionRegistry, EventSubscriber, Session } from '../events/index.js';
|
||||
import type { OHLCService } from '../services/ohlc-service.js';
|
||||
import type { OHLCService, BarUpdateCallback } from '../services/ohlc-service.js';
|
||||
import type { SymbolIndexService } from '../services/symbol-index-service.js';
|
||||
import type { ContainerManager } from '../k8s/container-manager.js';
|
||||
import type { ConversationService } from '../services/conversation-service.js';
|
||||
import {
|
||||
WorkspaceManager,
|
||||
ContainerSync,
|
||||
@@ -42,6 +44,7 @@ export interface WebSocketHandlerConfig {
|
||||
createHarness: HarnessFactory;
|
||||
ohlcService?: OHLCService; // Optional for historical data support
|
||||
symbolIndexService?: SymbolIndexService; // Optional for symbol search
|
||||
conversationService?: ConversationService; // Optional for history replay on reconnect
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -50,10 +53,18 @@ export interface WebSocketHandlerConfig {
|
||||
* Handles WebSocket connections for chat and integrates with the event system
|
||||
* for container-to-client notifications.
|
||||
*/
|
||||
interface BarSubscription {
|
||||
ticker: string;
|
||||
periodSeconds: number;
|
||||
callback: BarUpdateCallback;
|
||||
}
|
||||
|
||||
export class WebSocketHandler {
|
||||
private config: WebSocketHandlerConfig;
|
||||
private harnesses = new Map<string, AgentHarness>();
|
||||
private workspaces = new Map<string, WorkspaceManager>();
|
||||
/** Per-session realtime bar subscriptions for cleanup on disconnect */
|
||||
private barSubscriptions = new Map<string, BarSubscription[]>();
|
||||
|
||||
constructor(config: WebSocketHandlerConfig) {
|
||||
this.config = config;
|
||||
@@ -106,17 +117,22 @@ export class WebSocketHandler {
|
||||
|
||||
// If container is spinning up, wait for it to be ready before continuing
|
||||
if (isSpinningUp) {
|
||||
sendStatus(socket, 'spinning_up', 'Your workspace is starting up, please wait...');
|
||||
sendStatus(socket, 'spinning_up', 'Your personal agent is starting up, please wait...');
|
||||
|
||||
const startupPingInterval = setInterval(() => {
|
||||
if (socket.readyState === 1) socket.ping();
|
||||
}, 10000);
|
||||
|
||||
const ready = await this.config.containerManager.waitForContainerReady(authContext.userId, 120000);
|
||||
clearInterval(startupPingInterval);
|
||||
if (!ready) {
|
||||
logger.warn({ userId: authContext.userId }, 'Container failed to become ready within timeout');
|
||||
socket.send(JSON.stringify({ type: 'error', message: 'Workspace failed to start. Please try again later.' }));
|
||||
logger.warn({ userId: authContext.userId }, 'Sandbox failed to become ready within timeout');
|
||||
socket.send(JSON.stringify({ type: 'error', message: 'Agent workspace failed to start. Please try again later.' }));
|
||||
socket.close(1011, 'Container startup timeout');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ userId: authContext.userId }, 'Container is ready, proceeding with session setup');
|
||||
logger.info({ userId: authContext.userId }, 'Sandbox is ready, proceeding with session setup');
|
||||
}
|
||||
|
||||
sendStatus(socket, 'initializing', 'Starting your workspace...');
|
||||
@@ -241,6 +257,17 @@ export class WebSocketHandler {
|
||||
})
|
||||
);
|
||||
|
||||
// Replay conversation history so the UI pre-populates on reconnect
|
||||
if (this.config.conversationService) {
|
||||
const history = await this.config.conversationService.getHistory(
|
||||
authContext.userId,
|
||||
authContext.sessionId
|
||||
);
|
||||
if (history.length > 0) {
|
||||
socket.send(JSON.stringify({ type: 'conversation_history', messages: history }));
|
||||
}
|
||||
}
|
||||
|
||||
// Handle messages
|
||||
socket.on('message', async (data: Buffer) => {
|
||||
try {
|
||||
@@ -266,15 +293,45 @@ export class WebSocketHandler {
|
||||
return;
|
||||
}
|
||||
|
||||
// Chunks are streamed via channelAdapter.sendChunk() during handleMessage
|
||||
try {
|
||||
// Acknowledge receipt immediately so the client can show the seen indicator
|
||||
socket.send(JSON.stringify({ type: 'agent_chunk', content: '', done: false }));
|
||||
|
||||
logger.info('Calling harness.handleMessage');
|
||||
await harness.handleMessage(inboundMessage);
|
||||
logger.info('Streaming harness response');
|
||||
let fatalError = false;
|
||||
for await (const event of harness.streamMessage(inboundMessage)) {
|
||||
const e = event as HarnessEvent;
|
||||
switch (e.type) {
|
||||
case 'chunk':
|
||||
socket.send(JSON.stringify({ type: 'agent_chunk', content: e.content, done: false }));
|
||||
break;
|
||||
case 'tool_call':
|
||||
socket.send(JSON.stringify({ type: 'agent_tool_call', toolName: e.toolName, label: e.label }));
|
||||
break;
|
||||
case 'subagent_tool_call':
|
||||
socket.send(JSON.stringify({ type: 'subagent_tool_call', agentName: e.agentName, toolName: e.toolName, label: e.label }));
|
||||
break;
|
||||
case 'subagent_chunk':
|
||||
socket.send(JSON.stringify({ type: 'subagent_chunk', agentName: e.agentName, content: e.content }));
|
||||
break;
|
||||
case 'image':
|
||||
socket.send(JSON.stringify({ type: 'image', data: e.data, mimeType: e.mimeType, caption: e.caption }));
|
||||
break;
|
||||
case 'error':
|
||||
socket.send(JSON.stringify({ type: 'text', text: `An unrecoverable error occurred in the ${e.source}.` }));
|
||||
if (e.fatal) fatalError = true;
|
||||
break;
|
||||
case 'done':
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Send done marker after all chunks have been streamed
|
||||
if (fatalError) {
|
||||
socket.close(1011, 'Fatal error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Send done marker after all events have been streamed
|
||||
logger.debug('Sending done marker to client');
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
@@ -332,6 +389,17 @@ export class WebSocketHandler {
|
||||
await this.config.eventSubscriber.onSessionDisconnect(removedSession);
|
||||
}
|
||||
|
||||
// Cleanup realtime bar subscriptions
|
||||
const sessionId = authContext.sessionId;
|
||||
const subs = this.barSubscriptions.get(sessionId);
|
||||
if (subs && this.config.ohlcService) {
|
||||
for (const { ticker, periodSeconds, callback } of subs) {
|
||||
this.config.ohlcService.unsubscribeFromTicker(ticker, periodSeconds, callback);
|
||||
}
|
||||
this.barSubscriptions.delete(sessionId);
|
||||
logger.info({ sessionId, count: subs.length }, 'Cleaned up realtime bar subscriptions');
|
||||
}
|
||||
|
||||
// Cleanup workspace
|
||||
await workspace!.shutdown();
|
||||
this.workspaces.delete(authContext.sessionId);
|
||||
@@ -356,6 +424,7 @@ export class WebSocketHandler {
|
||||
}, 30000);
|
||||
} catch (error) {
|
||||
logger.error({ error }, 'Failed to initialize session');
|
||||
socket.send(JSON.stringify({ type: 'text', text: 'An unrecoverable error occurred in the agent harness.' }));
|
||||
socket.close(1011, 'Internal server error');
|
||||
if (workspace) {
|
||||
await workspace.shutdown();
|
||||
@@ -527,19 +596,92 @@ export class WebSocketHandler {
|
||||
break;
|
||||
}
|
||||
|
||||
case 'subscribe_bars':
|
||||
case 'unsubscribe_bars':
|
||||
// TODO: Implement real-time subscriptions
|
||||
socket.send(
|
||||
JSON.stringify({
|
||||
type: `${payload.type}_response`,
|
||||
case 'subscribe_bars': {
|
||||
if (!ohlcService || !authContext) {
|
||||
socket.send(JSON.stringify({
|
||||
type: 'subscribe_bars_response',
|
||||
request_id: requestId,
|
||||
subscription_id: payload.subscription_id,
|
||||
success: false,
|
||||
message: 'Real-time subscriptions not yet implemented',
|
||||
})
|
||||
);
|
||||
message: 'Realtime service not available',
|
||||
}));
|
||||
break;
|
||||
}
|
||||
|
||||
const subTicker: string = payload.symbol;
|
||||
const subPeriod: number = payload.period_seconds ?? payload.resolution ?? 60;
|
||||
const sessionId = authContext.sessionId;
|
||||
|
||||
// Create a per-subscription callback that forwards bars to this socket
|
||||
const barCallback: BarUpdateCallback = (bar) => {
|
||||
if (socket.readyState !== 1 /* OPEN */) return;
|
||||
socket.send(JSON.stringify({
|
||||
type: 'bar_update',
|
||||
subscription_id: payload.subscription_id,
|
||||
ticker: bar.ticker,
|
||||
period_seconds: bar.periodSeconds,
|
||||
bar: {
|
||||
// Convert nanoseconds → seconds for client compatibility
|
||||
time: Number(bar.timestamp / 1_000_000_000n),
|
||||
open: bar.open,
|
||||
high: bar.high,
|
||||
low: bar.low,
|
||||
close: bar.close,
|
||||
volume: bar.volume,
|
||||
},
|
||||
}));
|
||||
};
|
||||
|
||||
ohlcService.subscribeToTicker(subTicker, subPeriod, barCallback);
|
||||
|
||||
// Track for cleanup on disconnect
|
||||
if (!this.barSubscriptions.has(sessionId)) {
|
||||
this.barSubscriptions.set(sessionId, []);
|
||||
}
|
||||
this.barSubscriptions.get(sessionId)!.push({
|
||||
ticker: subTicker,
|
||||
periodSeconds: subPeriod,
|
||||
callback: barCallback,
|
||||
});
|
||||
|
||||
logger.info({ sessionId, ticker: subTicker, period: subPeriod }, 'Subscribed to realtime bars');
|
||||
|
||||
socket.send(JSON.stringify({
|
||||
type: 'subscribe_bars_response',
|
||||
request_id: requestId,
|
||||
subscription_id: payload.subscription_id,
|
||||
success: true,
|
||||
}));
|
||||
break;
|
||||
}
|
||||
|
||||
case 'unsubscribe_bars': {
|
||||
if (!ohlcService || !authContext) break;
|
||||
|
||||
const unsubTicker: string = payload.symbol;
|
||||
const unsubPeriod: number = payload.period_seconds ?? payload.resolution ?? 60;
|
||||
const sessionId = authContext.sessionId;
|
||||
|
||||
const subs = this.barSubscriptions.get(sessionId);
|
||||
if (subs) {
|
||||
const idx = subs.findIndex(
|
||||
s => s.ticker === unsubTicker && s.periodSeconds === unsubPeriod
|
||||
);
|
||||
if (idx >= 0) {
|
||||
const [removed] = subs.splice(idx, 1);
|
||||
ohlcService.unsubscribeFromTicker(unsubTicker, unsubPeriod, removed.callback);
|
||||
logger.info({ sessionId, ticker: unsubTicker, period: unsubPeriod }, 'Unsubscribed from realtime bars');
|
||||
}
|
||||
}
|
||||
|
||||
socket.send(JSON.stringify({
|
||||
type: 'unsubscribe_bars_response',
|
||||
request_id: requestId,
|
||||
subscription_id: payload.subscription_id,
|
||||
success: true,
|
||||
}));
|
||||
break;
|
||||
}
|
||||
|
||||
case 'evaluate_indicator': {
|
||||
// Direct MCP call — bypasses the agent/LLM for performance
|
||||
|
||||
@@ -632,6 +632,118 @@ export class DuckDBClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a batch of image/audio blobs as a Parquet file in S3.
|
||||
* Called once per assistant turn that produces binary output.
|
||||
*/
|
||||
async appendBlobs(
|
||||
userId: string,
|
||||
sessionId: string,
|
||||
messageId: string,
|
||||
blobs: Array<{
|
||||
id: string;
|
||||
user_id: string;
|
||||
session_id: string;
|
||||
message_id: string;
|
||||
blob_type: string;
|
||||
mime_type: string;
|
||||
data: string;
|
||||
caption: string | null;
|
||||
timestamp: number;
|
||||
}>
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
if (!this.conversationsBucket || blobs.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const year = now.getUTCFullYear();
|
||||
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
|
||||
const s3Path = `s3://${this.conversationsBucket}/gateway/blobs/year=${year}/month=${month}/user_id=${userId}/${sessionId}_${messageId}.parquet`;
|
||||
const tempTable = `blob_flush_${Date.now()}`;
|
||||
|
||||
try {
|
||||
await this.query(`
|
||||
CREATE TEMP TABLE ${tempTable} (
|
||||
id VARCHAR,
|
||||
user_id VARCHAR,
|
||||
session_id VARCHAR,
|
||||
message_id VARCHAR,
|
||||
blob_type VARCHAR,
|
||||
mime_type VARCHAR,
|
||||
data VARCHAR,
|
||||
caption VARCHAR,
|
||||
timestamp BIGINT
|
||||
)
|
||||
`);
|
||||
|
||||
for (const blob of blobs) {
|
||||
await this.query(
|
||||
`INSERT INTO ${tempTable} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[blob.id, blob.user_id, blob.session_id, blob.message_id, blob.blob_type, blob.mime_type, blob.data, blob.caption, blob.timestamp]
|
||||
);
|
||||
}
|
||||
|
||||
await this.query(`COPY ${tempTable} TO '${s3Path}' (FORMAT PARQUET)`);
|
||||
this.logger.info({ userId, sessionId, messageId, count: blobs.length, s3Path }, 'Blobs flushed to Parquet');
|
||||
} finally {
|
||||
await this.query(`DROP TABLE IF EXISTS ${tempTable}`).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query blobs from S3 by userId/sessionId, optionally filtered to specific blob IDs.
|
||||
*/
|
||||
async queryBlobs(
|
||||
userId: string,
|
||||
sessionId: string,
|
||||
blobIds?: string[]
|
||||
): Promise<any[]> {
|
||||
await this.initialize();
|
||||
|
||||
try {
|
||||
const tablePath = await this.getTablePath(this.namespace, 'blobs', this.catalogUri);
|
||||
|
||||
if (!tablePath) {
|
||||
// Fallback: scan per-turn Parquet files written directly to S3
|
||||
if (this.conversationsBucket) {
|
||||
this.logger.debug({ userId, sessionId }, 'REST catalog miss, scanning blob Parquet files');
|
||||
const parquetPath = `s3://${this.conversationsBucket}/gateway/blobs/**/user_id=${userId}/${sessionId}_*.parquet`;
|
||||
const idClause = blobIds?.length
|
||||
? `WHERE id IN (${blobIds.map(id => `'${id.replace(/'/g, "''")}'`).join(', ')})`
|
||||
: '';
|
||||
try {
|
||||
return await this.query(`SELECT * FROM read_parquet('${parquetPath}') ${idClause} ORDER BY timestamp ASC`);
|
||||
} catch {
|
||||
// No blobs yet for this session
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
const idFilter = blobIds?.length
|
||||
? `AND id IN (${blobIds.map(() => '?').join(', ')})`
|
||||
: '';
|
||||
const params: any[] = [userId, sessionId, ...(blobIds ?? [])];
|
||||
|
||||
const sql = `
|
||||
SELECT id, user_id, session_id, message_id, blob_type, mime_type, data, caption, timestamp
|
||||
FROM iceberg_scan('${tablePath}')
|
||||
WHERE user_id = ? AND session_id = ? ${idFilter}
|
||||
ORDER BY timestamp ASC
|
||||
`;
|
||||
|
||||
const rows = await this.query(sql, params);
|
||||
this.logger.info({ userId, sessionId, count: rows.length }, 'Loaded blobs from Iceberg');
|
||||
return rows.map((row: any) => ({ ...row, timestamp: Number(row.timestamp) }));
|
||||
} catch (error: any) {
|
||||
this.logger.error({ error: error.message, userId, sessionId }, 'Failed to query blobs');
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the DuckDB connection
|
||||
*/
|
||||
|
||||
@@ -45,6 +45,21 @@ export interface IcebergMessage {
|
||||
timestamp: number; // nanoseconds
|
||||
}
|
||||
|
||||
/**
|
||||
* Blob record for Iceberg storage (images, audio, etc.)
|
||||
*/
|
||||
export interface IcebergBlob {
|
||||
id: string;
|
||||
user_id: string;
|
||||
session_id: string;
|
||||
message_id: string;
|
||||
blob_type: string;
|
||||
mime_type: string;
|
||||
data: string; // base64
|
||||
caption: string | null;
|
||||
timestamp: number; // microseconds
|
||||
}
|
||||
|
||||
/**
|
||||
* Checkpoint record for Iceberg storage
|
||||
*/
|
||||
@@ -153,6 +168,25 @@ export class IcebergClient {
|
||||
return this.duckdb.appendMessages(userId, sessionId, messages);
|
||||
}
|
||||
|
||||
/**
|
||||
* Append blobs for one assistant turn as a Parquet file in S3.
|
||||
*/
|
||||
async appendBlobs(
|
||||
userId: string,
|
||||
sessionId: string,
|
||||
messageId: string,
|
||||
blobs: IcebergBlob[]
|
||||
): Promise<void> {
|
||||
return this.duckdb.appendBlobs(userId, sessionId, messageId, blobs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query blobs from S3/Iceberg, optionally filtered to specific blob IDs.
|
||||
*/
|
||||
async queryBlobs(userId: string, sessionId: string, blobIds?: string[]): Promise<IcebergBlob[]> {
|
||||
return this.duckdb.queryBlobs(userId, sessionId, blobIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get table metadata
|
||||
*/
|
||||
|
||||
@@ -298,6 +298,13 @@ export class QdrantClient {
|
||||
pointsCount: info.points_count || 0,
|
||||
};
|
||||
} catch (error) {
|
||||
// If the collection was lost (e.g. Qdrant restarted without the gateway restarting),
|
||||
// recreate it and return zeroed stats rather than propagating the error.
|
||||
if ((error as any)?.status === 404) {
|
||||
this.logger.warn({ collection: this.collectionName }, 'Collection missing, recreating...');
|
||||
await this.initialize();
|
||||
return { vectorsCount: 0, indexedVectorsCount: 0, pointsCount: 0 };
|
||||
}
|
||||
this.logger.error({ error }, 'Failed to get collection info');
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -20,6 +20,22 @@ import type {
|
||||
NotificationStatus,
|
||||
} from '../types/ohlc.js';
|
||||
|
||||
export const OHLC_BAR_TOPIC_PATTERN = /^(.+)\|ohlc:(\d+)$/;
|
||||
|
||||
/** Decoded realtime OHLC bar received from the XPUB market data stream */
|
||||
export interface RealtimeBar {
|
||||
topic: string; // e.g., "BTC/USDT.BINANCE|ohlc:60"
|
||||
ticker: string; // e.g., "BTC/USDT.BINANCE"
|
||||
periodSeconds: number;
|
||||
/** Window open time in nanoseconds since epoch */
|
||||
timestamp: bigint;
|
||||
open: number;
|
||||
high: number;
|
||||
low: number;
|
||||
close: number;
|
||||
volume: number;
|
||||
}
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
@@ -39,14 +55,17 @@ export enum MessageType {
|
||||
const protoDir = join(__dirname, '../..', 'protobuf');
|
||||
const root = new protobuf.Root();
|
||||
|
||||
// Load proto file and parse it
|
||||
// Load proto files
|
||||
const ingestorProto = readFileSync(join(protoDir, 'ingestor.proto'), 'utf8');
|
||||
const ohlcProto = readFileSync(join(protoDir, 'ohlc.proto'), 'utf8');
|
||||
protobuf.parse(ingestorProto, root);
|
||||
protobuf.parse(ohlcProto, root);
|
||||
|
||||
// Export message types
|
||||
const SubmitHistoricalRequestType = root.lookupType('SubmitHistoricalRequest');
|
||||
const SubmitResponseType = root.lookupType('SubmitResponse');
|
||||
const HistoryReadyNotificationType = root.lookupType('HistoryReadyNotification');
|
||||
const OHLCType = root.lookupType('OHLC');
|
||||
|
||||
/**
|
||||
* Encode SubmitHistoricalRequest to ZMQ frames
|
||||
@@ -178,3 +197,39 @@ export function decodeHistoryReadyNotification(frames: Buffer[]): HistoryReadyNo
|
||||
completed_at: BigInt(payload.completedAt),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a realtime OHLC bar from ZMQ SUB frames.
|
||||
* Frame layout: [topic][version][0x04 OHLC type + OHLC protobuf bytes]
|
||||
*
|
||||
* Returns null if the topic doesn't match the realtime bar pattern or decoding fails.
|
||||
*/
|
||||
export function decodeRealtimeBar(frames: Buffer[]): RealtimeBar | null {
|
||||
if (frames.length < 3) return null;
|
||||
|
||||
const topic = frames[0].toString();
|
||||
const match = OHLC_BAR_TOPIC_PATTERN.exec(topic);
|
||||
if (!match) return null;
|
||||
|
||||
const ticker = match[1];
|
||||
const periodSeconds = parseInt(match[2], 10);
|
||||
|
||||
const messageFrame = frames[2];
|
||||
if (messageFrame[0] !== 0x04) return null; // Must be OHLC type
|
||||
|
||||
const payloadBuffer = messageFrame.slice(1);
|
||||
const decoded = OHLCType.decode(payloadBuffer);
|
||||
const ohlc = OHLCType.toObject(decoded, { longs: String, defaults: true });
|
||||
|
||||
return {
|
||||
topic,
|
||||
ticker,
|
||||
periodSeconds,
|
||||
timestamp: BigInt(ohlc.timestamp ?? '0'),
|
||||
open: Number(ohlc.open ?? 0),
|
||||
high: Number(ohlc.high ?? 0),
|
||||
low: Number(ohlc.low ?? 0),
|
||||
close: Number(ohlc.close ?? 0),
|
||||
volume: Number(ohlc.volume ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -17,6 +17,9 @@ import {
|
||||
encodeSubmitHistoricalRequest,
|
||||
decodeSubmitResponse,
|
||||
decodeHistoryReadyNotification,
|
||||
decodeRealtimeBar,
|
||||
OHLC_BAR_TOPIC_PATTERN,
|
||||
type RealtimeBar,
|
||||
} from './zmq-protocol.js';
|
||||
import type {
|
||||
SubmitHistoricalRequest,
|
||||
@@ -27,6 +30,9 @@ import {
|
||||
NotificationStatus,
|
||||
} from '../types/ohlc.js';
|
||||
|
||||
export type BarUpdateCallback = (bar: RealtimeBar) => void;
|
||||
export type { RealtimeBar };
|
||||
|
||||
export interface ZMQRelayConfig {
|
||||
relayRequestEndpoint: string; // e.g., "tcp://relay:5559"
|
||||
relayNotificationEndpoint: string; // e.g., "tcp://relay:5558"
|
||||
@@ -57,6 +63,12 @@ export class ZMQRelayClient {
|
||||
private notificationTopic: string;
|
||||
private pendingRequests: Map<string, PendingRequest> = new Map();
|
||||
|
||||
/** Ref count per ZMQ topic (gateway-level dedup before ZMQ subscribe/unsubscribe) */
|
||||
private topicRefs: Map<string, number> = new Map();
|
||||
|
||||
/** Callbacks registered by WebSocket sessions for realtime bar updates */
|
||||
private barCallbacks: Map<string, Set<BarUpdateCallback>> = new Map();
|
||||
|
||||
private connected = false;
|
||||
private notificationListenerRunning = false;
|
||||
|
||||
@@ -253,8 +265,6 @@ export class ZMQRelayClient {
|
||||
// Handle metadata update notifications
|
||||
if (topic === 'METADATA_UPDATE') {
|
||||
this.logger.info('Received METADATA_UPDATE notification');
|
||||
|
||||
// Call the onMetadataUpdate callback if configured
|
||||
if (this.config.onMetadataUpdate) {
|
||||
try {
|
||||
await this.config.onMetadataUpdate();
|
||||
@@ -265,6 +275,20 @@ export class ZMQRelayClient {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle realtime OHLC bar updates (topic pattern: "{ticker}|ohlc:{period}")
|
||||
if (OHLC_BAR_TOPIC_PATTERN.test(topic)) {
|
||||
const bar = decodeRealtimeBar(Array.from(frames));
|
||||
if (bar) {
|
||||
const callbacks = this.barCallbacks.get(topic);
|
||||
if (callbacks) {
|
||||
for (const cb of callbacks) {
|
||||
try { cb(bar); } catch (e) { /* ignore callback errors */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle history ready notifications
|
||||
const notification = decodeHistoryReadyNotification(Array.from(frames));
|
||||
|
||||
@@ -308,6 +332,69 @@ export class ZMQRelayClient {
|
||||
this.logger.debug('Notification listener started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to realtime OHLC bars for a ticker+period.
|
||||
*
|
||||
* ZMQ subscribe is only called on the 0→1 transition (first subscriber).
|
||||
* This triggers the relay XPUB → Flink subscription detection → ingestor activation.
|
||||
*
|
||||
* @param callback Called whenever a new bar arrives for this topic
|
||||
*/
|
||||
subscribeToTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
|
||||
const topic = `${ticker}|ohlc:${periodSeconds}`;
|
||||
|
||||
// Register callback
|
||||
if (!this.barCallbacks.has(topic)) {
|
||||
this.barCallbacks.set(topic, new Set());
|
||||
}
|
||||
this.barCallbacks.get(topic)!.add(callback);
|
||||
|
||||
// ZMQ subscribe on first ref
|
||||
const prev = this.topicRefs.get(topic) ?? 0;
|
||||
this.topicRefs.set(topic, prev + 1);
|
||||
if (prev === 0 && this.subSocket) {
|
||||
this.subSocket.subscribe(topic);
|
||||
this.logger.info({ topic }, 'ZMQ subscribed to realtime topic');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unsubscribe a callback from realtime OHLC bars.
|
||||
* ZMQ unsubscribe is only called on the 1→0 transition (last subscriber).
|
||||
*/
|
||||
unsubscribeFromTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
|
||||
const topic = `${ticker}|ohlc:${periodSeconds}`;
|
||||
|
||||
const callbacks = this.barCallbacks.get(topic);
|
||||
if (callbacks) {
|
||||
callbacks.delete(callback);
|
||||
if (callbacks.size === 0) {
|
||||
this.barCallbacks.delete(topic);
|
||||
}
|
||||
}
|
||||
|
||||
const prev = this.topicRefs.get(topic) ?? 0;
|
||||
if (prev <= 1) {
|
||||
this.topicRefs.delete(topic);
|
||||
if (this.subSocket) {
|
||||
this.subSocket.unsubscribe(topic);
|
||||
this.logger.info({ topic }, 'ZMQ unsubscribed from realtime topic');
|
||||
}
|
||||
} else {
|
||||
this.topicRefs.set(topic, prev - 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all subscriptions for a set of (topic, callback) pairs.
|
||||
* Convenience method for WebSocket disconnect cleanup.
|
||||
*/
|
||||
cleanupSubscriptions(subscriptions: Array<{ ticker: string; periodSeconds: number; callback: BarUpdateCallback }>): void {
|
||||
for (const { ticker, periodSeconds, callback } of subscriptions) {
|
||||
this.unsubscribeFromTicker(ticker, periodSeconds, callback);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the client and cleanup resources
|
||||
*/
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { License } from '../types/user.js';
|
||||
import { ChannelType } from '../types/user.js';
|
||||
import type { ConversationStore } from './memory/conversation-store.js';
|
||||
import type { BlobStore } from './memory/blob-store.js';
|
||||
import type { InboundMessage, OutboundMessage } from '../types/messages.js';
|
||||
import { MCPClientConnector } from './mcp-client.js';
|
||||
import { LLMProviderFactory, type ProviderConfig } from '../llm/provider.js';
|
||||
@@ -14,13 +15,16 @@ import type { ChannelAdapter, PathTriggerContext } from '../workspace/index.js';
|
||||
import type { ResearchSubagent } from './subagents/research/index.js';
|
||||
import type { IndicatorSubagent } from './subagents/indicator/index.js';
|
||||
import type { WebExploreSubagent } from './subagents/web-explore/index.js';
|
||||
import type { StrategySubagent } from './subagents/strategy/index.js';
|
||||
import type { DynamicStructuredTool } from '@langchain/core/tools';
|
||||
import { getToolRegistry } from '../tools/tool-registry.js';
|
||||
import type { MCPToolInfo } from '../tools/mcp/mcp-tool-wrapper.js';
|
||||
import { createResearchAgentTool } from '../tools/platform/research-agent.tool.js';
|
||||
import { createIndicatorAgentTool } from '../tools/platform/indicator-agent.tool.js';
|
||||
import { createWebExploreAgentTool } from '../tools/platform/web-explore-agent.tool.js';
|
||||
import { createStrategyAgentTool } from '../tools/platform/strategy-agent.tool.js';
|
||||
import { createUserContext } from './memory/session-context.js';
|
||||
import type { HarnessEvent } from './harness-events.js';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
@@ -54,10 +58,12 @@ export type HarnessFactory = (sessionConfig: HarnessSessionConfig) => AgentHarne
|
||||
export interface AgentHarnessConfig extends HarnessSessionConfig {
|
||||
providerConfig: ProviderConfig;
|
||||
conversationStore?: ConversationStore;
|
||||
blobStore?: BlobStore;
|
||||
historyLimit: number;
|
||||
researchSubagent?: ResearchSubagent;
|
||||
indicatorSubagent?: IndicatorSubagent;
|
||||
webExploreSubagent?: WebExploreSubagent;
|
||||
strategySubagent?: StrategySubagent;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -87,6 +93,8 @@ export class AgentHarness {
|
||||
private conversationStore?: ConversationStore;
|
||||
private indicatorSubagent?: IndicatorSubagent;
|
||||
private webExploreSubagent?: WebExploreSubagent;
|
||||
private strategySubagent?: StrategySubagent;
|
||||
private blobStore?: BlobStore;
|
||||
private abortController: AbortController | null = null;
|
||||
|
||||
constructor(config: AgentHarnessConfig) {
|
||||
@@ -96,10 +104,12 @@ export class AgentHarness {
|
||||
this.researchSubagent = config.researchSubagent;
|
||||
this.indicatorSubagent = config.indicatorSubagent;
|
||||
this.webExploreSubagent = config.webExploreSubagent;
|
||||
this.strategySubagent = config.strategySubagent;
|
||||
|
||||
this.modelFactory = new LLMProviderFactory(config.providerConfig, config.logger);
|
||||
this.modelRouter = new ModelRouter(this.modelFactory, config.logger);
|
||||
this.conversationStore = config.conversationStore;
|
||||
this.blobStore = config.blobStore;
|
||||
|
||||
this.mcpClient = new MCPClientConnector({
|
||||
userId: config.userId,
|
||||
@@ -419,17 +429,75 @@ export class AgentHarness {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize strategy subagent
|
||||
*/
|
||||
private async initializeStrategySubagent(): Promise<void> {
|
||||
if (this.strategySubagent) {
|
||||
this.config.logger.debug('Strategy subagent already provided');
|
||||
return;
|
||||
}
|
||||
|
||||
this.config.logger.debug('Creating strategy subagent for session');
|
||||
|
||||
try {
|
||||
const { createStrategySubagent } = await import('./subagents/strategy/index.js');
|
||||
|
||||
const { model } = await this.modelRouter.route(
|
||||
'trading strategy writing and backtesting',
|
||||
this.config.license,
|
||||
RoutingStrategy.COMPLEXITY,
|
||||
this.config.userId
|
||||
);
|
||||
|
||||
const toolRegistry = getToolRegistry();
|
||||
const strategyTools = await toolRegistry.getToolsForAgent(
|
||||
'strategy',
|
||||
this.mcpClient,
|
||||
this.availableMCPTools,
|
||||
this.workspaceManager,
|
||||
undefined,
|
||||
undefined
|
||||
);
|
||||
|
||||
const strategySubagentPath = join(__dirname, 'subagents', 'strategy');
|
||||
this.config.logger.debug({ strategySubagentPath }, 'Using strategy subagent path');
|
||||
|
||||
this.strategySubagent = await createStrategySubagent(
|
||||
model,
|
||||
this.config.logger,
|
||||
strategySubagentPath,
|
||||
this.mcpClient,
|
||||
strategyTools
|
||||
);
|
||||
|
||||
this.config.logger.info(
|
||||
{
|
||||
toolCount: strategyTools.length,
|
||||
toolNames: strategyTools.map(t => t.name),
|
||||
},
|
||||
'Strategy subagent created successfully'
|
||||
);
|
||||
} catch (error) {
|
||||
this.config.logger.error(
|
||||
{ error, errorMessage: (error as Error).message, stack: (error as Error).stack },
|
||||
'Failed to create strategy subagent'
|
||||
);
|
||||
// Don't throw — strategy subagent is optional
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute model with tool calling loop
|
||||
* Handles multi-turn tool calls until the model produces a final text response
|
||||
*/
|
||||
private async executeWithToolCalling(
|
||||
private async *executeWithToolCalling(
|
||||
model: any,
|
||||
messages: BaseMessage[],
|
||||
tools: DynamicStructuredTool[],
|
||||
maxIterations: number = 2,
|
||||
signal?: AbortSignal
|
||||
): Promise<string> {
|
||||
): AsyncGenerator<HarnessEvent> {
|
||||
this.config.logger.info(
|
||||
{ toolCount: tools.length, maxIterations },
|
||||
'Starting tool calling loop'
|
||||
@@ -437,6 +505,8 @@ export class AgentHarness {
|
||||
|
||||
const messagesCopy = [...messages];
|
||||
let iterations = 0;
|
||||
// Track last char of last yielded text chunk to detect missing spaces between tokens
|
||||
let lastChunkTail = '';
|
||||
|
||||
while (iterations < maxIterations) {
|
||||
if (signal?.aborted) break;
|
||||
@@ -455,15 +525,24 @@ export class AgentHarness {
|
||||
try {
|
||||
const stream = await model.stream(messagesCopy, { signal });
|
||||
for await (const chunk of stream) {
|
||||
const contents: string[] = [];
|
||||
if (typeof chunk.content === 'string' && chunk.content.length > 0) {
|
||||
this.channelAdapter?.sendChunk(chunk.content);
|
||||
contents.push(chunk.content);
|
||||
} else if (Array.isArray(chunk.content)) {
|
||||
for (const block of chunk.content) {
|
||||
if (block.type === 'text' && block.text) {
|
||||
this.channelAdapter?.sendChunk(block.text);
|
||||
}
|
||||
if (block.type === 'text' && block.text) contents.push(block.text);
|
||||
}
|
||||
}
|
||||
for (const content of contents) {
|
||||
// DeepInfra/GLM streams tokens without leading spaces; inject one when
|
||||
// both the tail of the previous chunk and the head of this chunk are
|
||||
// word characters (\w), which would otherwise merge two words.
|
||||
if (lastChunkTail && /\w/.test(lastChunkTail) && /\w/.test(content[0])) {
|
||||
yield { type: 'chunk', content: ' ' };
|
||||
}
|
||||
lastChunkTail = content[content.length - 1];
|
||||
yield { type: 'chunk', content };
|
||||
}
|
||||
response = response ? response.concat(chunk) : chunk;
|
||||
}
|
||||
} catch (invokeError: any) {
|
||||
@@ -486,6 +565,8 @@ export class AgentHarness {
|
||||
contentLength: typeof response.content === 'string' ? response.content.length : 0,
|
||||
hasToolCalls: !!response.tool_calls,
|
||||
toolCallCount: response.tool_calls?.length || 0,
|
||||
usageMetadata: (response as any).usage_metadata,
|
||||
finishReason: (response as any).response_metadata?.finish_reason,
|
||||
},
|
||||
'Model response received'
|
||||
);
|
||||
@@ -508,7 +589,8 @@ export class AgentHarness {
|
||||
{ finalContentLength: finalContent.length, iterations },
|
||||
'Tool calling loop complete - no more tool calls'
|
||||
);
|
||||
return finalContent;
|
||||
yield { type: 'done', content: finalContent };
|
||||
return;
|
||||
}
|
||||
|
||||
this.config.logger.info(
|
||||
@@ -540,11 +622,32 @@ export class AgentHarness {
|
||||
}
|
||||
|
||||
try {
|
||||
this.channelAdapter?.sendToolCall?.(toolCall.name, this.getToolLabel(toolCall.name));
|
||||
const result = await tool.func(toolCall.args);
|
||||
yield { type: 'tool_call', toolName: toolCall.name, label: this.getToolLabel(toolCall.name) };
|
||||
|
||||
// Process result to extract images and send them via channel adapter
|
||||
const processedResult = this.processToolResult(result, toolCall.name);
|
||||
// Use streamFunc when available (subagent tools) to forward intermediate events inline
|
||||
let result: string;
|
||||
const streamFunc = (tool as any).streamFunc as ((args: any, signal?: AbortSignal) => AsyncGenerator<import('./harness-events.js').HarnessEvent, string>) | undefined;
|
||||
if (streamFunc) {
|
||||
const gen = streamFunc(toolCall.args, signal);
|
||||
let next = await gen.next();
|
||||
while (!next.done) {
|
||||
if (signal?.aborted) {
|
||||
gen.return?.('');
|
||||
break;
|
||||
}
|
||||
yield next.value;
|
||||
next = await gen.next();
|
||||
}
|
||||
result = next.done ? next.value : '';
|
||||
} else {
|
||||
result = await tool.func(toolCall.args);
|
||||
}
|
||||
|
||||
// Extract images from result and yield them; get text-only version for LLM
|
||||
const { cleanedResult: processedResult, images } = this.extractImagesFromToolResult(result, toolCall.name);
|
||||
for (const img of images) {
|
||||
yield { type: 'image', data: img.data, mimeType: img.mimeType, caption: img.caption };
|
||||
}
|
||||
|
||||
this.config.logger.debug(
|
||||
{
|
||||
@@ -567,6 +670,12 @@ export class AgentHarness {
|
||||
'Tool execution completed'
|
||||
);
|
||||
} catch (error) {
|
||||
// Clean stop — abort signal fired during tool execution; exit without error message
|
||||
if (signal?.aborted || (error as Error)?.name === 'AbortError') {
|
||||
this.config.logger.info({ tool: toolCall.name }, 'Tool execution aborted by stop signal');
|
||||
return;
|
||||
}
|
||||
|
||||
this.config.logger.error(
|
||||
{
|
||||
error,
|
||||
@@ -578,6 +687,8 @@ export class AgentHarness {
|
||||
'Tool execution failed'
|
||||
);
|
||||
|
||||
yield { type: 'error' as const, source: toolCall.name, fatal: false };
|
||||
|
||||
messagesCopy.push(
|
||||
new ToolMessage({
|
||||
content: `Error: ${error}`,
|
||||
@@ -586,11 +697,15 @@ export class AgentHarness {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// After all tool calls complete, emit a space separator before the next LLM streaming pass
|
||||
yield { type: 'chunk', content: ' ' };
|
||||
lastChunkTail = ' ';
|
||||
}
|
||||
|
||||
// Max iterations reached - return what we have
|
||||
// Max iterations reached - yield done with apology
|
||||
this.config.logger.warn('Max tool calling iterations reached');
|
||||
return 'I apologize, but I encountered an issue processing your request. Please try rephrasing your question.';
|
||||
yield { type: 'done', content: 'I apologize, but I encountered an issue processing your request. Please try rephrasing your question.' };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -617,162 +732,222 @@ export class AgentHarness {
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle incoming message from user
|
||||
* Stream events for an incoming user message.
|
||||
* Yields typed HarnessEvents (chunk, tool_call, image, done) and saves the
|
||||
* conversation to the store once the done event has been emitted.
|
||||
*/
|
||||
async handleMessage(message: InboundMessage): Promise<OutboundMessage> {
|
||||
async *streamMessage(message: InboundMessage): AsyncGenerator<HarnessEvent> {
|
||||
this.config.logger.info(
|
||||
{ messageId: message.messageId, userId: message.userId, content: message.content.substring(0, 100) },
|
||||
'Processing user message'
|
||||
);
|
||||
|
||||
try {
|
||||
// 1. Build system prompt from template
|
||||
this.config.logger.debug('Building system prompt');
|
||||
const systemPrompt = await this.buildSystemPrompt();
|
||||
this.config.logger.debug({ systemPromptLength: systemPrompt.length }, 'System prompt built');
|
||||
// 1. Build system prompt from template
|
||||
this.config.logger.debug('Building system prompt');
|
||||
const systemPrompt = await this.buildSystemPrompt();
|
||||
this.config.logger.debug({ systemPromptLength: systemPrompt.length }, 'System prompt built');
|
||||
|
||||
// 2. Load recent conversation history
|
||||
const channelKey = this.config.channelType ?? ChannelType.WEBSOCKET;
|
||||
let storedMessages = this.conversationStore
|
||||
? await this.conversationStore.getRecentMessages(
|
||||
this.config.userId, this.config.sessionId, this.config.historyLimit, channelKey
|
||||
)
|
||||
: [];
|
||||
|
||||
// First turn: seed conversation history with current workspace state
|
||||
if (storedMessages.length === 0 && this.workspaceManager && this.conversationStore) {
|
||||
const workspaceJSON = this.workspaceManager.serializeState();
|
||||
const content = `[Workspace State]\n\`\`\`json\n${workspaceJSON}\n\`\`\``;
|
||||
await this.conversationStore.saveMessage(
|
||||
this.config.userId, this.config.sessionId,
|
||||
'workspace', content, { isWorkspaceContext: true }, channelKey
|
||||
);
|
||||
storedMessages = await this.conversationStore.getRecentMessages(
|
||||
// 2. Load recent conversation history
|
||||
const channelKey = this.config.channelType ?? ChannelType.WEBSOCKET;
|
||||
let storedMessages = this.conversationStore
|
||||
? await this.conversationStore.getRecentMessages(
|
||||
this.config.userId, this.config.sessionId, this.config.historyLimit, channelKey
|
||||
);
|
||||
}
|
||||
)
|
||||
: [];
|
||||
|
||||
const history = this.conversationStore
|
||||
? this.conversationStore.toLangChainMessages(storedMessages)
|
||||
: [];
|
||||
this.config.logger.debug({ historyLength: history.length }, 'Conversation history loaded');
|
||||
|
||||
// 4. Get the configured model
|
||||
this.config.logger.debug('Routing to model');
|
||||
const { model, middleware } = await this.modelRouter.route(
|
||||
message.content,
|
||||
this.config.license,
|
||||
RoutingStrategy.COMPLEXITY,
|
||||
this.config.userId
|
||||
// First turn: seed conversation history with current workspace state
|
||||
if (storedMessages.length === 0 && this.workspaceManager && this.conversationStore) {
|
||||
const workspaceJSON = this.workspaceManager.serializeState();
|
||||
const content = `[Workspace State]\n\`\`\`json\n${workspaceJSON}\n\`\`\``;
|
||||
await this.conversationStore.saveMessage(
|
||||
this.config.userId, this.config.sessionId,
|
||||
'workspace', content, { isWorkspaceContext: true }, channelKey
|
||||
);
|
||||
this.middleware = middleware;
|
||||
this.config.logger.info({ modelName: model.constructor.name }, 'Model selected');
|
||||
|
||||
// 5. Build LangChain messages
|
||||
const langchainMessages = this.buildLangChainMessages(systemPrompt, history, message.content);
|
||||
this.config.logger.debug({ messageCount: langchainMessages.length }, 'LangChain messages built');
|
||||
|
||||
// 6. Get tools for main agent from registry
|
||||
const toolRegistry = getToolRegistry();
|
||||
const tools = await toolRegistry.getToolsForAgent(
|
||||
'main',
|
||||
this.mcpClient,
|
||||
this.availableMCPTools,
|
||||
this.workspaceManager // Pass session workspace manager
|
||||
storedMessages = await this.conversationStore.getRecentMessages(
|
||||
this.config.userId, this.config.sessionId, this.config.historyLimit, channelKey
|
||||
);
|
||||
}
|
||||
|
||||
// Build shared subagent context
|
||||
const subagentContext = {
|
||||
userContext: createUserContext({
|
||||
userId: this.config.userId,
|
||||
sessionId: this.config.sessionId,
|
||||
license: this.config.license,
|
||||
channelType: this.config.channelType ?? ChannelType.WEBSOCKET,
|
||||
channelUserId: this.config.channelUserId ?? this.config.userId,
|
||||
}),
|
||||
};
|
||||
const history = this.conversationStore
|
||||
? this.conversationStore.toLangChainMessages(storedMessages)
|
||||
: [];
|
||||
this.config.logger.debug({ historyLength: history.length }, 'Conversation history loaded');
|
||||
|
||||
// Add research subagent as a tool if available
|
||||
if (this.researchSubagent) {
|
||||
tools.push(createResearchAgentTool({
|
||||
researchSubagent: this.researchSubagent,
|
||||
context: subagentContext,
|
||||
logger: this.config.logger,
|
||||
}));
|
||||
}
|
||||
// 4. Get the configured model
|
||||
this.config.logger.debug('Routing to model');
|
||||
const { model, middleware } = await this.modelRouter.route(
|
||||
message.content,
|
||||
this.config.license,
|
||||
RoutingStrategy.COMPLEXITY,
|
||||
this.config.userId
|
||||
);
|
||||
this.middleware = middleware;
|
||||
this.config.logger.info({ modelName: model.constructor.name }, 'Model selected');
|
||||
|
||||
// Add indicator subagent as a tool if available
|
||||
if (this.indicatorSubagent) {
|
||||
tools.push(createIndicatorAgentTool({
|
||||
indicatorSubagent: this.indicatorSubagent,
|
||||
context: subagentContext,
|
||||
logger: this.config.logger,
|
||||
}));
|
||||
}
|
||||
// 5. Build LangChain messages
|
||||
const langchainMessages = this.buildLangChainMessages(systemPrompt, history, message.content);
|
||||
this.config.logger.debug({ messageCount: langchainMessages.length }, 'LangChain messages built');
|
||||
|
||||
// Add web explore subagent as a tool if available
|
||||
if (this.webExploreSubagent) {
|
||||
tools.push(createWebExploreAgentTool({
|
||||
webExploreSubagent: this.webExploreSubagent,
|
||||
context: subagentContext,
|
||||
logger: this.config.logger,
|
||||
}));
|
||||
}
|
||||
// 6. Get tools for main agent from registry
|
||||
const toolRegistry = getToolRegistry();
|
||||
const tools = await toolRegistry.getToolsForAgent(
|
||||
'main',
|
||||
this.mcpClient,
|
||||
this.availableMCPTools,
|
||||
this.workspaceManager
|
||||
);
|
||||
|
||||
// Build shared subagent context
|
||||
const subagentContext = {
|
||||
userContext: createUserContext({
|
||||
userId: this.config.userId,
|
||||
sessionId: this.config.sessionId,
|
||||
license: this.config.license,
|
||||
channelType: this.config.channelType ?? ChannelType.WEBSOCKET,
|
||||
channelUserId: this.config.channelUserId ?? this.config.userId,
|
||||
}),
|
||||
};
|
||||
|
||||
if (this.researchSubagent) {
|
||||
tools.push(createResearchAgentTool({
|
||||
researchSubagent: this.researchSubagent,
|
||||
context: subagentContext,
|
||||
logger: this.config.logger,
|
||||
}));
|
||||
}
|
||||
|
||||
if (this.indicatorSubagent) {
|
||||
tools.push(createIndicatorAgentTool({
|
||||
indicatorSubagent: this.indicatorSubagent,
|
||||
context: subagentContext,
|
||||
logger: this.config.logger,
|
||||
}));
|
||||
}
|
||||
|
||||
if (this.webExploreSubagent) {
|
||||
tools.push(createWebExploreAgentTool({
|
||||
webExploreSubagent: this.webExploreSubagent,
|
||||
context: subagentContext,
|
||||
logger: this.config.logger,
|
||||
}));
|
||||
}
|
||||
|
||||
if (!this.strategySubagent) {
|
||||
await this.initializeStrategySubagent();
|
||||
}
|
||||
if (this.strategySubagent) {
|
||||
tools.push(createStrategyAgentTool({
|
||||
strategySubagent: this.strategySubagent,
|
||||
context: subagentContext,
|
||||
logger: this.config.logger,
|
||||
}));
|
||||
}
|
||||
|
||||
this.config.logger.info(
|
||||
{ toolCount: tools.length, toolNames: tools.map(t => t.name) },
|
||||
'Tools loaded for main agent'
|
||||
);
|
||||
|
||||
// Apply middleware (e.g. Anthropic prompt caching)
|
||||
const processedMessages = this.middleware
|
||||
? this.middleware.processMessages(langchainMessages, tools)
|
||||
: langchainMessages;
|
||||
|
||||
// 7. Bind tools to model
|
||||
const modelWithTools = tools.length > 0 && model.bindTools ? model.bindTools(tools) : model;
|
||||
|
||||
if (tools.length > 0) {
|
||||
this.config.logger.info(
|
||||
{
|
||||
toolCount: tools.length,
|
||||
toolNames: tools.map(t => t.name),
|
||||
},
|
||||
'Tools loaded for main agent'
|
||||
{ modelType: modelWithTools.constructor.name, toolsBound: tools.length > 0 && !!model.bindTools },
|
||||
'Model bound with tools'
|
||||
);
|
||||
}
|
||||
|
||||
// Apply middleware (e.g. Anthropic prompt caching)
|
||||
const processedMessages = this.middleware
|
||||
? this.middleware.processMessages(langchainMessages, tools)
|
||||
: langchainMessages;
|
||||
|
||||
// 7. Bind tools to model
|
||||
const modelWithTools = tools.length > 0 && model.bindTools ? model.bindTools(tools) : model;
|
||||
|
||||
if (tools.length > 0) {
|
||||
this.config.logger.info(
|
||||
{ modelType: modelWithTools.constructor.name, toolsBound: tools.length > 0 && !!model.bindTools },
|
||||
'Model bound with tools'
|
||||
);
|
||||
// 8. Stream tool calling loop and save conversation on completion
|
||||
this.config.logger.info('Invoking LLM with tool support');
|
||||
this.abortController = new AbortController();
|
||||
let finalContent = '';
|
||||
const collectedImages: Array<{ data: string; mimeType: string; caption?: string }> = [];
|
||||
try {
|
||||
for await (const event of this.executeWithToolCalling(modelWithTools, processedMessages, tools, 10, this.abortController.signal)) {
|
||||
if (event.type === 'done') {
|
||||
finalContent = event.content;
|
||||
this.config.logger.info({ responseLength: finalContent.length }, 'LLM response received');
|
||||
} else if (event.type === 'image') {
|
||||
collectedImages.push({ data: event.data, mimeType: event.mimeType, caption: event.caption });
|
||||
}
|
||||
yield event;
|
||||
}
|
||||
|
||||
// 8. Call LLM with tool calling loop
|
||||
this.config.logger.info('Invoking LLM with tool support');
|
||||
this.abortController = new AbortController();
|
||||
const assistantMessage = await this.executeWithToolCalling(modelWithTools, processedMessages, tools, 10, this.abortController.signal);
|
||||
} catch (error) {
|
||||
if ((error as Error)?.name === 'AbortError') {
|
||||
this.config.logger.info('Agent harness interrupted by stop signal');
|
||||
} else {
|
||||
this.config.logger.error({ error }, 'Fatal error in agent harness');
|
||||
yield { type: 'error' as const, source: 'agent harness', fatal: true };
|
||||
}
|
||||
} finally {
|
||||
this.abortController = null;
|
||||
if (finalContent && this.conversationStore) {
|
||||
// Write blobs to S3 and capture their IDs for message metadata
|
||||
let blobRefs: Array<{ id: string; mimeType: string; caption?: string }> = [];
|
||||
if (collectedImages.length > 0 && this.blobStore) {
|
||||
const assistantMsgId = `${this.config.userId}:${this.config.sessionId}:${Date.now()}`;
|
||||
const blobIds = await this.blobStore.writeBlobs(
|
||||
this.config.userId, this.config.sessionId, assistantMsgId,
|
||||
collectedImages.map(img => ({ blobType: 'image' as const, mimeType: img.mimeType, data: img.data, caption: img.caption }))
|
||||
);
|
||||
blobRefs = blobIds.map((id, i) => ({ id, mimeType: collectedImages[i].mimeType, caption: collectedImages[i].caption }));
|
||||
}
|
||||
|
||||
this.config.logger.info(
|
||||
{ responseLength: assistantMessage.length },
|
||||
'LLM response received'
|
||||
);
|
||||
|
||||
// Save user message and assistant response to conversation store
|
||||
if (this.conversationStore) {
|
||||
await this.conversationStore.saveMessage(
|
||||
this.config.userId, this.config.sessionId, 'user', message.content, undefined, channelKey
|
||||
);
|
||||
await this.conversationStore.saveMessage(
|
||||
this.config.userId, this.config.sessionId, 'assistant', assistantMessage, undefined, channelKey
|
||||
this.config.userId, this.config.sessionId, 'assistant', finalContent,
|
||||
blobRefs.length > 0 ? { blobs: blobRefs } : undefined,
|
||||
channelKey
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
messageId: `msg_${Date.now()}`,
|
||||
sessionId: message.sessionId,
|
||||
content: assistantMessage,
|
||||
timestamp: new Date(),
|
||||
};
|
||||
/**
|
||||
* Handle incoming message from user.
|
||||
* Consumes streamMessage and dispatches events to the channel adapter for
|
||||
* backward compatibility with Telegram and other non-streaming callers.
|
||||
*/
|
||||
async handleMessage(message: InboundMessage): Promise<OutboundMessage> {
|
||||
let finalContent = '';
|
||||
try {
|
||||
for await (const event of this.streamMessage(message)) {
|
||||
switch (event.type) {
|
||||
case 'chunk':
|
||||
this.channelAdapter?.sendChunk(event.content);
|
||||
break;
|
||||
case 'tool_call':
|
||||
this.channelAdapter?.sendToolCall?.(event.toolName, event.label);
|
||||
break;
|
||||
case 'image':
|
||||
this.channelAdapter?.sendImage({ data: event.data, mimeType: event.mimeType, caption: event.caption });
|
||||
break;
|
||||
case 'error':
|
||||
this.channelAdapter?.sendText?.({ text: `An unrecoverable error occurred in the ${event.source}.` });
|
||||
break;
|
||||
case 'done':
|
||||
finalContent = event.content;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.config.logger.error({ error }, 'Error processing message');
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
messageId: `msg_${Date.now()}`,
|
||||
sessionId: message.sessionId,
|
||||
content: finalContent,
|
||||
timestamp: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -817,21 +992,27 @@ export class AgentHarness {
|
||||
python_write: 'Coding...',
|
||||
python_read: 'Inspecting...',
|
||||
execute_research: 'Running script...',
|
||||
backtest_strategy: 'Running backtest...',
|
||||
backtest_strategy: 'Backtesting...',
|
||||
list_active_strategies: 'Checking active strategies...',
|
||||
web_explore: 'Searching the web...',
|
||||
strategy: 'Coding a strategy...',
|
||||
};
|
||||
return labels[toolName] ?? `Running ${toolName}...`;
|
||||
return labels[toolName] ?? `Running ${toolName} tool...`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process tool result to extract images and send via channel adapter.
|
||||
* Returns text-only version for LLM context (no base64 image data).
|
||||
*/
|
||||
private processToolResult(result: string, toolName: string): string {
|
||||
private extractImagesFromToolResult(
|
||||
result: string,
|
||||
toolName: string
|
||||
): { cleanedResult: string; images: Array<{ data: string; mimeType: string; caption?: string }> } {
|
||||
const noImages = { cleanedResult: String(result || ''), images: [] };
|
||||
|
||||
// Most tools return plain strings - only process JSON results
|
||||
if (!result || typeof result !== 'string') {
|
||||
return String(result || '');
|
||||
return noImages;
|
||||
}
|
||||
|
||||
// Try to parse as JSON
|
||||
@@ -840,7 +1021,7 @@ export class AgentHarness {
|
||||
parsedResult = JSON.parse(result);
|
||||
} catch {
|
||||
// Not JSON, return as-is
|
||||
return result;
|
||||
return noImages;
|
||||
}
|
||||
|
||||
// Check if result has images array (from ResearchSubagent)
|
||||
@@ -850,19 +1031,11 @@ export class AgentHarness {
|
||||
'Extracting images from tool result'
|
||||
);
|
||||
|
||||
// Send each image via channel adapter
|
||||
const images: Array<{ data: string; mimeType: string; caption?: string }> = [];
|
||||
for (const image of parsedResult.images) {
|
||||
if (image.data && image.mimeType) {
|
||||
if (this.channelAdapter) {
|
||||
this.config.logger.debug({ mimeType: image.mimeType }, 'Sending image to channel');
|
||||
this.channelAdapter.sendImage({
|
||||
data: image.data,
|
||||
mimeType: image.mimeType,
|
||||
caption: undefined,
|
||||
});
|
||||
} else {
|
||||
this.config.logger.warn('No channel adapter set, cannot send image');
|
||||
}
|
||||
this.config.logger.debug({ mimeType: image.mimeType }, 'Extracted image from tool result');
|
||||
images.push({ data: image.data, mimeType: image.mimeType, caption: undefined });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -872,15 +1045,13 @@ export class AgentHarness {
|
||||
images: undefined,
|
||||
imageCount: parsedResult.images.length,
|
||||
};
|
||||
|
||||
// Clean up undefined values
|
||||
Object.keys(textOnlyResult).forEach(key => {
|
||||
if (textOnlyResult[key] === undefined) {
|
||||
delete textOnlyResult[key];
|
||||
}
|
||||
});
|
||||
|
||||
return JSON.stringify(textOnlyResult);
|
||||
return { cleanedResult: JSON.stringify(textOnlyResult), images };
|
||||
}
|
||||
|
||||
// Check for nested chart_images object
|
||||
@@ -890,20 +1061,12 @@ export class AgentHarness {
|
||||
'Extracting chart images from tool result'
|
||||
);
|
||||
|
||||
// Send each chart image via channel adapter
|
||||
const images: Array<{ data: string; mimeType: string; caption?: string }> = [];
|
||||
for (const [chartId, chartData] of Object.entries(parsedResult.chart_images)) {
|
||||
const chart = chartData as any;
|
||||
if (chart.type === 'image' && chart.data) {
|
||||
if (this.channelAdapter) {
|
||||
this.config.logger.debug({ chartId }, 'Sending chart image to channel');
|
||||
this.channelAdapter.sendImage({
|
||||
data: chart.data,
|
||||
mimeType: 'image/png',
|
||||
caption: undefined,
|
||||
});
|
||||
} else {
|
||||
this.config.logger.warn('No channel adapter set, cannot send chart image');
|
||||
}
|
||||
this.config.logger.debug({ chartId }, 'Extracted chart image from tool result');
|
||||
images.push({ data: chart.data, mimeType: 'image/png', caption: undefined });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -913,19 +1076,17 @@ export class AgentHarness {
|
||||
chart_images: undefined,
|
||||
chartCount: Object.keys(parsedResult.chart_images).length,
|
||||
};
|
||||
|
||||
// Clean up undefined values
|
||||
Object.keys(textOnlyResult).forEach(key => {
|
||||
if (textOnlyResult[key] === undefined) {
|
||||
delete textOnlyResult[key];
|
||||
}
|
||||
});
|
||||
|
||||
return JSON.stringify(textOnlyResult);
|
||||
return { cleanedResult: JSON.stringify(textOnlyResult), images };
|
||||
}
|
||||
|
||||
// No images found, return stringified result
|
||||
return result;
|
||||
// No images found, return as-is
|
||||
return { cleanedResult: result, images: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
51
gateway/src/harness/harness-events.ts
Normal file
51
gateway/src/harness/harness-events.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
export interface ChunkEvent {
|
||||
type: 'chunk';
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ToolCallEvent {
|
||||
type: 'tool_call';
|
||||
toolName: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface ImageEvent {
|
||||
type: 'image';
|
||||
data: string;
|
||||
mimeType: string;
|
||||
caption?: string;
|
||||
}
|
||||
|
||||
export interface DoneEvent {
|
||||
type: 'done';
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface SubagentChunkEvent {
|
||||
type: 'subagent_chunk';
|
||||
agentName: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface SubagentThinkingEvent {
|
||||
type: 'subagent_thinking';
|
||||
agentName: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface SubagentToolCallEvent {
|
||||
type: 'subagent_tool_call';
|
||||
agentName: string;
|
||||
toolName: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface ErrorEvent {
|
||||
type: 'error';
|
||||
/** Name of the agent or tool where the error occurred */
|
||||
source: string;
|
||||
/** True if the error is unrecoverable and the chat session should end */
|
||||
fatal: boolean;
|
||||
}
|
||||
|
||||
export type HarnessEvent = ChunkEvent | ToolCallEvent | ImageEvent | DoneEvent | SubagentChunkEvent | SubagentThinkingEvent | SubagentToolCallEvent | ErrorEvent;
|
||||
@@ -57,57 +57,74 @@ export class MCPClientConnector {
|
||||
this.client = null;
|
||||
}
|
||||
|
||||
try {
|
||||
this.config.logger.info(
|
||||
{ userId: this.config.userId, url: this.config.mcpServerUrl },
|
||||
'Connecting to user MCP server'
|
||||
);
|
||||
const maxAttempts = 5;
|
||||
const retryDelayMs = 1500;
|
||||
|
||||
this.client = new Client(
|
||||
{
|
||||
name: 'dexorder-gateway',
|
||||
version: '0.1.0',
|
||||
},
|
||||
{
|
||||
capabilities: {
|
||||
sampling: {},
|
||||
this.config.logger.info(
|
||||
{ userId: this.config.userId, url: this.config.mcpServerUrl },
|
||||
'Connecting to user MCP server'
|
||||
);
|
||||
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
this.client = new Client(
|
||||
{
|
||||
name: 'dexorder-gateway',
|
||||
version: '0.1.0',
|
||||
},
|
||||
}
|
||||
);
|
||||
{
|
||||
capabilities: {
|
||||
sampling: {},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
// Streamable HTTP: single /mcp endpoint, session tracked via mcp-session-id header
|
||||
const transport = new StreamableHTTPClientTransport(
|
||||
new URL(`${this.config.mcpServerUrl}/mcp`)
|
||||
);
|
||||
// Streamable HTTP: single /mcp endpoint, session tracked via mcp-session-id header
|
||||
const transport = new StreamableHTTPClientTransport(
|
||||
new URL(`${this.config.mcpServerUrl}/mcp`)
|
||||
);
|
||||
|
||||
await this.client.connect(transport);
|
||||
await this.client.connect(transport);
|
||||
|
||||
// Hook client.onerror to detect transport failures (e.g. sandbox restart returning
|
||||
// 404 "session not found"). When fired, mark disconnected so the next callTool /
|
||||
// listTools call triggers a full reconnect + initialize handshake.
|
||||
const connectedClient = this.client;
|
||||
const origOnError = this.client.onerror;
|
||||
this.client.onerror = (error) => {
|
||||
origOnError?.(error);
|
||||
// Only act on the currently-active client (ignore stale closures after reconnect)
|
||||
if (this.client === connectedClient && this.connected) {
|
||||
// Hook client.onerror to detect transport failures (e.g. sandbox restart returning
|
||||
// 404 "session not found"). When fired, mark disconnected so the next callTool /
|
||||
// listTools call triggers a full reconnect + initialize handshake.
|
||||
const connectedClient = this.client;
|
||||
const origOnError = this.client.onerror;
|
||||
this.client.onerror = (error) => {
|
||||
origOnError?.(error);
|
||||
// Only act on the currently-active client (ignore stale closures after reconnect)
|
||||
if (this.client === connectedClient && this.connected) {
|
||||
this.config.logger.warn(
|
||||
{ error },
|
||||
'MCP transport error — marking disconnected for lazy reconnect'
|
||||
);
|
||||
this.connected = false;
|
||||
}
|
||||
};
|
||||
|
||||
this.connected = true;
|
||||
this.config.logger.info('Connected to user MCP server');
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
this.client = null;
|
||||
if (attempt < maxAttempts) {
|
||||
this.config.logger.warn(
|
||||
{ error },
|
||||
'MCP transport error — marking disconnected for lazy reconnect'
|
||||
{ error, userId: this.config.userId, attempt, maxAttempts },
|
||||
'MCP connect attempt failed, retrying...'
|
||||
);
|
||||
this.connected = false;
|
||||
await new Promise(resolve => setTimeout(resolve, retryDelayMs));
|
||||
}
|
||||
};
|
||||
|
||||
this.connected = true;
|
||||
this.config.logger.info('Connected to user MCP server');
|
||||
} catch (error) {
|
||||
this.config.logger.error(
|
||||
{ error, userId: this.config.userId },
|
||||
'Failed to connect to user MCP server'
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
this.config.logger.error(
|
||||
{ error: lastError, userId: this.config.userId },
|
||||
'Failed to connect to user MCP server'
|
||||
);
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -134,7 +151,9 @@ export class MCPClientConnector {
|
||||
try {
|
||||
this.config.logger.debug({ tool: name, args }, 'Calling MCP tool');
|
||||
|
||||
const result = await this.client!.callTool({ name, arguments: args });
|
||||
// Use a generous timeout: execute_research runs a subprocess with a 300s limit,
|
||||
// so the default 60s MCP SDK timeout would fire before the script completes.
|
||||
const result = await this.client!.callTool({ name, arguments: args }, undefined, { timeout: 330000 });
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.config.logger.error({ error, tool: name }, 'MCP tool call failed');
|
||||
|
||||
93
gateway/src/harness/memory/blob-store.ts
Normal file
93
gateway/src/harness/memory/blob-store.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { IcebergClient } from '../../clients/iceberg-client.js';
|
||||
|
||||
export interface StoredBlob {
|
||||
id: string;
|
||||
userId: string;
|
||||
sessionId: string;
|
||||
messageId: string;
|
||||
blobType: 'image' | 'audio';
|
||||
mimeType: string;
|
||||
data: string; // base64
|
||||
caption?: string;
|
||||
timestamp: number; // microseconds
|
||||
}
|
||||
|
||||
/**
|
||||
* Blob store for binary attachments (images, audio) referenced by conversation messages.
|
||||
*
|
||||
* Unlike text messages (Redis hot + Iceberg cold), blobs write directly to S3 Parquet
|
||||
* on each turn — they're infrequent enough that per-turn files don't cause fragmentation.
|
||||
* Blob IDs are stored in the parent message's metadata field for later retrieval.
|
||||
*/
|
||||
export class BlobStore {
|
||||
constructor(
|
||||
private icebergClient: IcebergClient | undefined,
|
||||
private logger: FastifyBaseLogger
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Write all blobs for one assistant turn to a single S3 Parquet file.
|
||||
* Returns the blob IDs assigned. Failures are logged but do not throw.
|
||||
*/
|
||||
async writeBlobs(
|
||||
userId: string,
|
||||
sessionId: string,
|
||||
messageId: string,
|
||||
blobs: Array<{ blobType: 'image' | 'audio'; mimeType: string; data: string; caption?: string }>
|
||||
): Promise<string[]> {
|
||||
if (!this.icebergClient || blobs.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const stored = blobs.map((b, i) => ({
|
||||
id: `blob_${userId}_${now}_${i}`,
|
||||
user_id: userId,
|
||||
session_id: sessionId,
|
||||
message_id: messageId,
|
||||
blob_type: b.blobType,
|
||||
mime_type: b.mimeType,
|
||||
data: b.data,
|
||||
caption: b.caption ?? null,
|
||||
timestamp: now * 1000, // microseconds
|
||||
}));
|
||||
|
||||
try {
|
||||
await this.icebergClient.appendBlobs(userId, sessionId, messageId, stored);
|
||||
this.logger.info({ userId, sessionId, count: stored.length }, 'Blobs written to S3');
|
||||
} catch (error) {
|
||||
this.logger.error({ error }, 'Failed to write blobs to S3');
|
||||
// Don't throw — blob failure should not break the conversation turn
|
||||
}
|
||||
|
||||
return stored.map(b => b.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve blobs by their IDs from S3/Iceberg cold storage.
|
||||
*/
|
||||
async getBlobsByIds(userId: string, sessionId: string, blobIds: string[]): Promise<StoredBlob[]> {
|
||||
if (!this.icebergClient || blobIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const rows = await this.icebergClient.queryBlobs(userId, sessionId, blobIds);
|
||||
return rows.map(r => ({
|
||||
id: r.id,
|
||||
userId: r.user_id,
|
||||
sessionId: r.session_id,
|
||||
messageId: r.message_id,
|
||||
blobType: r.blob_type as 'image' | 'audio',
|
||||
mimeType: r.mime_type,
|
||||
data: r.data,
|
||||
caption: r.caption ?? undefined,
|
||||
timestamp: r.timestamp,
|
||||
}));
|
||||
} catch (error) {
|
||||
this.logger.error({ error, blobIds }, 'Failed to retrieve blobs');
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,9 +39,9 @@ If the user asks for a capability not provided by Dexorder, decline and explain
|
||||
## Task Delegation
|
||||
- For ANY research questions, deep analysis, statistical analysis, charting requests, or market data queries that require computation, you MUST use the 'research' tool
|
||||
- For ANYTHING related to indicators on the chart — reading, adding, removing, modifying, or creating custom indicators — you MUST use the 'indicator' tool
|
||||
- For ANY backtesting request — running a strategy against historical data — you MUST use the 'backtest_strategy' tool directly; NEVER use the research tool for backtesting
|
||||
- For ANY request about trading strategies — writing, editing, backtesting, interpreting results, activating, deactivating, or monitoring — you MUST use the 'strategy' tool; NEVER write strategy Python code yourself
|
||||
- NEVER write Python code directly in your responses to the user
|
||||
- NEVER show code to the user — delegate to the research or indicator tool instead
|
||||
- NEVER show code to the user — delegate to the research, indicator, or strategy tool instead
|
||||
- NEVER attempt to do analysis yourself — let the subagents handle it
|
||||
|
||||
## Available Tools
|
||||
@@ -110,46 +110,54 @@ Parameters:
|
||||
- instruction: Natural language description of the analysis to perform (be specific!)
|
||||
- name: A unique name for the research script (e.g., "BTC Weekly Analysis")
|
||||
|
||||
**Do NOT include any time range, history length, bar count, period size, resolution, or timestamp guidance in the instruction** — not as numbers, not as natural language ("3-6 months", "1 year", "sufficient data"), not at all. The research subagent has its own rules for selecting resolution and history window. If you add time guidance, the subagent will follow yours instead of its own (which uses much more data). Only pass time constraints if the user explicitly asked for a specific period (e.g. "last week", "show me 2023").
|
||||
|
||||
Example usage:
|
||||
- User: "Does Friday price action correlate with Monday?"
|
||||
- You: Call research tool with instruction="Analyze correlation between Friday and Monday price action during NY trading hours (9:30-4:00 ET)", name="Friday-Monday Correlation"
|
||||
- WRONG: "...use hourly data and at least 3-6 months..." ← never add this
|
||||
|
||||
### strategy
|
||||
**Use this tool for ALL trading strategy requests without exception.**
|
||||
|
||||
The strategy subagent handles the complete strategy lifecycle: writing PandasStrategy classes, running backtests, interpreting results, and activating/deactivating paper trading.
|
||||
|
||||
**ALWAYS use strategy for:**
|
||||
- "Create a strategy that buys when RSI < 30" → write a new strategy
|
||||
- "Edit my momentum strategy to use a tighter stop" → modify existing strategy
|
||||
- "Backtest my RSI strategy over the last year" → run backtest
|
||||
- "How did this strategy perform on BTC?" → interpret results
|
||||
- "Activate my strategy for paper trading" → start paper trading
|
||||
- "What strategies are running?" → list active strategies
|
||||
- "Stop my momentum strategy" → deactivate a strategy
|
||||
- Any question about a strategy's PnL, trades, or performance
|
||||
|
||||
**NEVER call `backtest_strategy`, `activate_strategy`, `deactivate_strategy`, or `list_active_strategies` directly** — always go through the strategy tool.
|
||||
|
||||
**Custom indicators in strategies:**
|
||||
When writing a new strategy, the strategy subagent will first check for existing custom indicators via `python_list(category="indicator")`. Prefer using custom indicators (via `ta.custom_*`) over computing signals inline — this promotes reuse and gives users better visibility into strategy components. If a needed indicator doesn't exist yet, the strategy subagent will create it first via the indicator workflow.
|
||||
|
||||
### backtest_strategy
|
||||
**ALWAYS use this tool — and ONLY this tool — for any backtesting request.**
|
||||
|
||||
*(Called internally by the strategy tool — do not call this directly.)*
|
||||
Runs a saved trading strategy against historical OHLC data using the Nautilus Trader backtesting engine.
|
||||
Returns structured performance metrics and an equity curve. Any charts generated are automatically sent to the user.
|
||||
|
||||
**ALWAYS use backtest_strategy for:**
|
||||
- "Backtest my RSI strategy over the last year"
|
||||
- "How did this strategy perform on BTC?"
|
||||
- "Run a backtest from January to June"
|
||||
- Any request to test or evaluate a strategy on historical data
|
||||
|
||||
**NEVER use research for backtesting** — the research tool cannot run strategies through the backtesting engine.
|
||||
|
||||
After the tool returns, summarize the results clearly: total return, Sharpe ratio, max drawdown, win rate, and trade count. Present the equity curve description in plain language.
|
||||
|
||||
Parameters:
|
||||
- strategy_name: Display name of the saved strategy (use python_list with category="strategy" to check existing strategies)
|
||||
- feeds: Array of `{symbol, period_seconds}` feed objects (e.g. `[{"symbol": "BTC/USDT.BINANCE", "period_seconds": 3600}]`)
|
||||
- from_time / to_time: Date strings ("2024-01-01", "90 days ago", "now") or Unix timestamps
|
||||
- initial_capital: Starting balance in quote currency (default 10,000)
|
||||
Returns structured performance metrics including trade list, Sortino/Calmar ratios, and equity curve.
|
||||
|
||||
### list_active_strategies
|
||||
*(Called internally by the strategy tool — do not call this directly.)*
|
||||
Lists all currently active (live or paper) strategies and their status.
|
||||
Use this when the user asks what strategies are running.
|
||||
|
||||
### python_list
|
||||
List existing scripts in a category ("strategy", "indicator", or "research").
|
||||
Use this before calling the research tool to check whether a relevant script already exists.
|
||||
If one does, pass its exact name to the research tool so the subagent updates it rather than creating a new one.
|
||||
Also use before calling backtest_strategy to confirm the strategy name.
|
||||
The strategy tool uses this internally to check strategy names before backtesting.
|
||||
|
||||
### symbol-lookup
|
||||
Look up trading symbols and get metadata.
|
||||
Use this when users mention tickers or need symbol information.
|
||||
|
||||
**Always use symbol_lookup to resolve a proper ticker before passing it to the research or get-chart-data tools.** Symbols must be in `SYMBOL.EXCHANGE` format (e.g., `BTC/USDT.BINANCE`). If the user says "ETHUSDT", "ETH", or any ambiguous ticker, resolve it first with symbol_lookup so the correct formatted ticker is passed downstream.
|
||||
|
||||
### get-chart-data
|
||||
**IMPORTANT: This is for QUICK, CASUAL information ONLY. This tool just returns raw data - it does NOT create charts or plots.**
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { MCPClientConnector } from '../mcp-client.js';
|
||||
import type { DynamicStructuredTool } from '@langchain/core/tools';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import type { HarnessEvent, SubagentChunkEvent, SubagentThinkingEvent } from '../harness-events.js';
|
||||
|
||||
/**
|
||||
* Subagent configuration (loaded from config.yaml)
|
||||
@@ -122,6 +123,65 @@ export abstract class BaseSubagent {
|
||||
yield result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract subagent_chunk / subagent_thinking events from a LangGraph `messages` stream datum.
|
||||
*
|
||||
* LangGraph emits `[message_chunk, metadata]` tuples in `messages` mode. The message content
|
||||
* can be a plain string (normal text token) or an array of content blocks (extended thinking
|
||||
* responses with `{type:"thinking", thinking:"..."}` and `{type:"text", text:"..."}`).
|
||||
*/
|
||||
static extractStreamChunks(
|
||||
data: unknown,
|
||||
agentName: string,
|
||||
): Array<SubagentChunkEvent | SubagentThinkingEvent> {
|
||||
const msg = Array.isArray(data) ? (data as unknown[])[0] : data;
|
||||
const content = (msg as any)?.content;
|
||||
if (typeof content === 'string') {
|
||||
return content ? [{ type: 'subagent_chunk', agentName, content }] : [];
|
||||
}
|
||||
if (Array.isArray(content)) {
|
||||
const chunks: Array<SubagentChunkEvent | SubagentThinkingEvent> = [];
|
||||
for (const block of content as any[]) {
|
||||
if (block?.type === 'thinking' && typeof block.thinking === 'string' && block.thinking) {
|
||||
chunks.push({ type: 'subagent_thinking', agentName, content: block.thinking });
|
||||
} else if (block?.type === 'text' && typeof block.text === 'string' && block.text) {
|
||||
chunks.push({ type: 'subagent_chunk', agentName, content: block.text });
|
||||
}
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the final text from an `updates`-mode agent message.
|
||||
* Handles both plain string content and array content blocks (extended thinking).
|
||||
*/
|
||||
static extractFinalText(msg: any): string {
|
||||
if (typeof msg?.content === 'string') return msg.content;
|
||||
if (Array.isArray(msg?.content)) {
|
||||
return (msg.content as any[])
|
||||
.filter((b: any) => b?.type === 'text' && typeof b.text === 'string')
|
||||
.map((b: any) => b.text as string)
|
||||
.join('');
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream typed HarnessEvents during execution.
|
||||
* Subclasses override this to emit subagent_chunk / subagent_tool_call events
|
||||
* using agent.stream() from LangGraph. Default falls back to execute().
|
||||
*/
|
||||
async *streamEvents(
|
||||
context: SubagentContext,
|
||||
input: string,
|
||||
_signal?: AbortSignal,
|
||||
): AsyncGenerator<HarnessEvent, string> {
|
||||
const result = await this.execute(context, input);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build messages with system prompt and memory context
|
||||
*/
|
||||
|
||||
@@ -11,3 +11,8 @@ export {
|
||||
createResearchSubagent,
|
||||
type ResearchResult,
|
||||
} from './research/index.js';
|
||||
|
||||
export {
|
||||
StrategySubagent,
|
||||
createStrategySubagent,
|
||||
} from './strategy/index.js';
|
||||
|
||||
@@ -4,6 +4,7 @@ import { SystemMessage } from '@langchain/core/messages';
|
||||
import { createReactAgent } from '@langchain/langgraph/prebuilt';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { MCPClientConnector } from '../../mcp-client.js';
|
||||
import type { HarnessEvent } from '../../harness-events.js';
|
||||
|
||||
/**
|
||||
* Indicator Subagent
|
||||
@@ -84,6 +85,56 @@ export class IndicatorSubagent extends BaseSubagent {
|
||||
|
||||
return finalText;
|
||||
}
|
||||
|
||||
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
|
||||
|
||||
if (!this.hasMCPClient()) {
|
||||
throw new Error('MCP client not available for indicator subagent');
|
||||
}
|
||||
|
||||
const initialMessages = this.buildMessages(context, instruction);
|
||||
const systemMessage = initialMessages[0];
|
||||
const humanMessage = initialMessages[initialMessages.length - 1];
|
||||
|
||||
const agent = createReactAgent({
|
||||
llm: this.model,
|
||||
tools: this.tools,
|
||||
prompt: systemMessage as SystemMessage,
|
||||
});
|
||||
|
||||
const stream = agent.stream(
|
||||
{ messages: [humanMessage] },
|
||||
{ streamMode: ['messages', 'updates'], recursionLimit: 25, signal }
|
||||
);
|
||||
|
||||
let finalText = '';
|
||||
|
||||
for await (const [mode, data] of await stream) {
|
||||
if (signal?.aborted) break;
|
||||
if (mode === 'messages') {
|
||||
for (const chunk of IndicatorSubagent.extractStreamChunks(data, this.config.name)) {
|
||||
yield chunk;
|
||||
}
|
||||
} else if (mode === 'updates') {
|
||||
if ((data as any).agent?.messages) {
|
||||
for (const msg of (data as any).agent.messages as any[]) {
|
||||
if (msg.tool_calls?.length) {
|
||||
for (const tc of msg.tool_calls) {
|
||||
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
|
||||
}
|
||||
} else {
|
||||
const content = IndicatorSubagent.extractFinalText(msg);
|
||||
if (content) finalText = content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info({ textLength: finalText.length }, 'streamEvents finished');
|
||||
return finalText;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -4,6 +4,7 @@ import { SystemMessage } from '@langchain/core/messages';
|
||||
import { createReactAgent } from '@langchain/langgraph/prebuilt';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { MCPClientConnector } from '../../mcp-client.js';
|
||||
import type { HarnessEvent } from '../../harness-events.js';
|
||||
|
||||
/**
|
||||
* Result from research subagent execution
|
||||
@@ -50,6 +51,58 @@ export class ResearchSubagent extends BaseSubagent {
|
||||
this.imageCapture = capture;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch custom indicators from the sandbox and return a formatted system prompt section.
|
||||
* Returns empty string if there are no custom indicators or the call fails.
|
||||
*/
|
||||
private async fetchCustomIndicatorsSection(): Promise<string> {
|
||||
try {
|
||||
const raw = await this.callMCPTool('python_list', { category: 'indicator' });
|
||||
const r = raw as any;
|
||||
const text = r?.content?.[0]?.text ?? r?.[0]?.text;
|
||||
const parsed = typeof text === 'string' ? JSON.parse(text) : raw;
|
||||
const items: any[] = parsed?.items ?? [];
|
||||
if (items.length === 0) return '';
|
||||
|
||||
const lines: string[] = ['\n\n## Custom Indicators\n'];
|
||||
lines.push('The user has defined the following custom indicators. Use `ta.custom_<name>` where `<name>` is the lowercase sanitized function name shown below.\n');
|
||||
|
||||
for (const item of items) {
|
||||
const displayName: string = item.name ?? 'unknown';
|
||||
const description: string = item.description ?? '';
|
||||
const meta: any = item.metadata ?? {};
|
||||
// Derive the ta attribute name: sanitize display name to lowercase + underscores
|
||||
const taAttr = `custom_${displayName.toLowerCase().replace(/[^\w]/g, '_').replace(/_+/g, '_').replace(/^_+|_+$/g, '')}`;
|
||||
const inputSeries: string[] = meta.input_series ?? ['close'];
|
||||
const params: Record<string, any> = meta.parameters ?? {};
|
||||
const pane: string = meta.pane ?? 'separate';
|
||||
|
||||
const inputStr = inputSeries.map((s: string) => `df['${s}']`).join(', ');
|
||||
const paramStr = Object.entries(params)
|
||||
.map(([k, v]: [string, any]) => `${k}=${JSON.stringify(v?.default ?? null)}`)
|
||||
.join(', ');
|
||||
const callExample = paramStr
|
||||
? `ta.${taAttr}(${inputStr}, ${paramStr})`
|
||||
: `ta.${taAttr}(${inputStr})`;
|
||||
|
||||
const outputNames = (meta.output_columns ?? [{ name: 'value' }])
|
||||
.map((c: any) => c.name)
|
||||
.join(', ');
|
||||
|
||||
lines.push(`### ${displayName}`);
|
||||
if (description) lines.push(description);
|
||||
lines.push(`- **Call**: \`${callExample}\``);
|
||||
lines.push(`- **Outputs**: ${outputNames} | **Pane**: ${pane}`);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, 'Failed to fetch custom indicators for prompt injection');
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute research request using LangGraph's createReactAgent.
|
||||
* This is the standard LangChain pattern for agents with tool access —
|
||||
@@ -79,11 +132,17 @@ export class ResearchSubagent extends BaseSubagent {
|
||||
this.imageCapture.length = 0;
|
||||
this.lastImages = [];
|
||||
|
||||
const customIndicatorsSection = await this.fetchCustomIndicatorsSection();
|
||||
|
||||
// Build system prompt (with memory context appended)
|
||||
const initialMessages = this.buildMessages(context, instruction);
|
||||
// buildMessages returns [SystemMessage, ...history, HumanMessage]
|
||||
// Extract system content for createReactAgent's prompt parameter
|
||||
const systemMessage = initialMessages[0];
|
||||
let systemMessage = initialMessages[0] as SystemMessage;
|
||||
if (customIndicatorsSection) {
|
||||
const base = typeof systemMessage.content === 'string' ? systemMessage.content : JSON.stringify(systemMessage.content);
|
||||
systemMessage = new SystemMessage(base + customIndicatorsSection);
|
||||
}
|
||||
const humanMessage = initialMessages[initialMessages.length - 1];
|
||||
|
||||
// createReactAgent is the standard LangChain/LangGraph pattern for tool-using agents.
|
||||
@@ -91,12 +150,12 @@ export class ResearchSubagent extends BaseSubagent {
|
||||
const agent = createReactAgent({
|
||||
llm: this.model,
|
||||
tools: this.tools,
|
||||
prompt: systemMessage as SystemMessage,
|
||||
prompt: systemMessage,
|
||||
});
|
||||
|
||||
const result = await agent.invoke(
|
||||
{ messages: [humanMessage] },
|
||||
{ recursionLimit: 20 }
|
||||
{ recursionLimit: 40 }
|
||||
);
|
||||
|
||||
// The final message in the graph output is the agent's last AIMessage
|
||||
@@ -146,6 +205,109 @@ export class ResearchSubagent extends BaseSubagent {
|
||||
return this.lastImages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream typed HarnessEvents using LangGraph's agent.stream().
|
||||
* Emits subagent_tool_call when tools fire, subagent_chunk for the final AI response.
|
||||
* Returns the final text string as the generator return value.
|
||||
*/
|
||||
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
|
||||
|
||||
if (!this.hasMCPClient()) {
|
||||
throw new Error('MCP client not available for research subagent');
|
||||
}
|
||||
|
||||
this.imageCapture.length = 0;
|
||||
this.lastImages = [];
|
||||
|
||||
// Emit immediately so the UI shows the subagent has started — LLM generation
|
||||
// can take minutes with non-streaming models and nothing else reaches the UI until
|
||||
// the first `updates` event fires (after the LLM finishes its first response).
|
||||
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: 'Thinking...', label: 'Thinking...' };
|
||||
|
||||
const customIndicatorsSection = await this.fetchCustomIndicatorsSection();
|
||||
|
||||
const initialMessages = this.buildMessages(context, instruction);
|
||||
let systemMessage = initialMessages[0] as SystemMessage;
|
||||
if (customIndicatorsSection) {
|
||||
const base = typeof systemMessage.content === 'string' ? systemMessage.content : JSON.stringify(systemMessage.content);
|
||||
systemMessage = new SystemMessage(base + customIndicatorsSection);
|
||||
}
|
||||
const humanMessage = initialMessages[initialMessages.length - 1];
|
||||
|
||||
const agent = createReactAgent({
|
||||
llm: this.model,
|
||||
tools: this.tools,
|
||||
prompt: systemMessage,
|
||||
});
|
||||
|
||||
this.logger.debug(
|
||||
{ toolCount: this.tools.length, toolNames: this.tools.map(t => t.name) },
|
||||
'Research subagent: starting stream with tools'
|
||||
);
|
||||
|
||||
const systemChars = typeof systemMessage.content === 'string'
|
||||
? systemMessage.content.length
|
||||
: JSON.stringify(systemMessage.content).length;
|
||||
const humanChars = typeof humanMessage.content === 'string'
|
||||
? humanMessage.content.length
|
||||
: JSON.stringify(humanMessage.content).length;
|
||||
this.logger.info(
|
||||
{ systemChars, humanChars, approxInputKB: Math.round((systemChars + humanChars) / 1024) },
|
||||
'Research subagent: input context size'
|
||||
);
|
||||
|
||||
const stream = agent.stream(
|
||||
{ messages: [humanMessage] },
|
||||
{ streamMode: ['messages', 'updates'], recursionLimit: 40, signal }
|
||||
);
|
||||
|
||||
let finalText = '';
|
||||
let updateCount = 0;
|
||||
|
||||
for await (const [mode, data] of await stream) {
|
||||
if (signal?.aborted) break;
|
||||
if (mode === 'messages') {
|
||||
// Real-time token streaming from the LLM — data is [BaseMessage, metadata]
|
||||
for (const chunk of ResearchSubagent.extractStreamChunks(data, this.config.name)) {
|
||||
yield chunk;
|
||||
}
|
||||
} else if (mode === 'updates') {
|
||||
updateCount++;
|
||||
const updateKeys = Object.keys(data as any);
|
||||
this.logger.debug({ updateCount, updateKeys }, 'Research subagent: graph update');
|
||||
// Agent node fired — yield tool call decisions before tools run
|
||||
if ((data as any).agent?.messages) {
|
||||
for (const msg of (data as any).agent.messages as any[]) {
|
||||
if (msg.tool_calls?.length) {
|
||||
for (const tc of msg.tool_calls) {
|
||||
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
|
||||
}
|
||||
} else {
|
||||
// Capture final text for return value (already streamed via messages above)
|
||||
const content = ResearchSubagent.extractFinalText(msg);
|
||||
if (content) finalText = content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.lastImages = [...this.imageCapture];
|
||||
if (!finalText) {
|
||||
this.logger.warn(
|
||||
{ imageCount: this.lastImages.length },
|
||||
'Research subagent: model returned empty output'
|
||||
);
|
||||
} else {
|
||||
this.logger.info(
|
||||
{ textLength: finalText.length, imageCount: this.lastImages.length },
|
||||
'streamEvents finished'
|
||||
);
|
||||
}
|
||||
return finalText;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream research execution
|
||||
*/
|
||||
|
||||
@@ -421,6 +421,7 @@ For research scripts, import and use get_api() to access the API:
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
from dexorder.api.api import API
|
||||
@@ -432,10 +433,13 @@ log = logging.getLogger(__name__)
|
||||
# Global API instance - managed by main.py
|
||||
_global_api: Optional[API] = None
|
||||
|
||||
# Thread-local API — used by harness threads so they don't overwrite the global
|
||||
_thread_local = threading.local()
|
||||
|
||||
|
||||
def get_api() -> API:
|
||||
"""
|
||||
Get the global API instance for accessing market data and charts.
|
||||
Get the API instance for accessing market data and charts.
|
||||
|
||||
Use this in research scripts to access the data and charting APIs.
|
||||
|
||||
@@ -462,15 +466,27 @@ def get_api() -> API:
|
||||
# Create chart
|
||||
fig, ax = api.charting.plot_ohlc(df, title="BTC/USDT")
|
||||
"""
|
||||
# Thread-local takes priority (set by harness threads)
|
||||
api = getattr(_thread_local, 'api', None)
|
||||
if api is not None:
|
||||
return api
|
||||
if _global_api is None:
|
||||
raise RuntimeError("API not initialized")
|
||||
return _global_api
|
||||
|
||||
|
||||
def set_api(api: API) -> None:
|
||||
"""Set the global API instance. Internal use only."""
|
||||
global _global_api
|
||||
_global_api = api
|
||||
"""Set the API instance.
|
||||
|
||||
When called from the main thread, sets the global API used by all threads.
|
||||
When called from a non-main thread (e.g. harness threads), sets a thread-local
|
||||
API so the global is not overwritten.
|
||||
"""
|
||||
if threading.current_thread() is threading.main_thread():
|
||||
global _global_api
|
||||
_global_api = api
|
||||
else:
|
||||
_thread_local.api = api
|
||||
|
||||
|
||||
__all__ = ['API', 'ChartingAPI', 'DataAPI', 'get_api', 'set_api']
|
||||
|
||||
@@ -28,11 +28,12 @@ from datetime import datetime
|
||||
api = get_api()
|
||||
|
||||
# Method 1: Using Unix timestamps (seconds)
|
||||
# 1609459200 = 2021-01-01, 1735689600 = 2025-01-01
|
||||
df = asyncio.run(api.data.historical_ohlc(
|
||||
ticker="BTC/USDT.BINANCE",
|
||||
period_seconds=3600, # 1 hour candles
|
||||
start_time=1640000000, # Unix timestamp in seconds
|
||||
end_time=1640086400,
|
||||
start_time=1609459200, # 2021-01-01
|
||||
end_time=1735689600, # 2025-01-01 (~4 years, ~35,000 bars)
|
||||
extra_columns=["volume"]
|
||||
))
|
||||
|
||||
@@ -40,8 +41,8 @@ df = asyncio.run(api.data.historical_ohlc(
|
||||
df = asyncio.run(api.data.historical_ohlc(
|
||||
ticker="BTC/USDT.BINANCE",
|
||||
period_seconds=3600,
|
||||
start_time="2021-12-20", # Simple date string
|
||||
end_time="2021-12-21",
|
||||
start_time="2021-01-01",
|
||||
end_time="2025-01-01", # ~4 years of 1h bars ≈ 35,000 bars
|
||||
extra_columns=["volume"]
|
||||
))
|
||||
|
||||
@@ -49,21 +50,24 @@ df = asyncio.run(api.data.historical_ohlc(
|
||||
df = asyncio.run(api.data.historical_ohlc(
|
||||
ticker="BTC/USDT.BINANCE",
|
||||
period_seconds=3600,
|
||||
start_time="2021-12-20 00:00:00",
|
||||
end_time="2021-12-20 23:59:59",
|
||||
start_time="2021-01-01 00:00:00",
|
||||
end_time="2025-01-01 00:00:00",
|
||||
extra_columns=["volume"]
|
||||
))
|
||||
|
||||
# Method 4: Using datetime objects
|
||||
from datetime import datetime, timedelta
|
||||
end_time = datetime.now()
|
||||
start_time = end_time - timedelta(days=4*365) # 4 years back
|
||||
df = asyncio.run(api.data.historical_ohlc(
|
||||
ticker="BTC/USDT.BINANCE",
|
||||
period_seconds=3600,
|
||||
start_time=datetime(2021, 12, 20),
|
||||
end_time=datetime(2021, 12, 21),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
extra_columns=["volume"]
|
||||
))
|
||||
|
||||
print(f"Loaded {len(df)} candles")
|
||||
print(f"Loaded {len(df)} candles from {df.index[0]} to {df.index[-1]}")
|
||||
print(df.head())
|
||||
```
|
||||
|
||||
@@ -94,8 +98,8 @@ api = get_api()
|
||||
df = asyncio.run(api.data.historical_ohlc(
|
||||
ticker="BTC/USDT.BINANCE",
|
||||
period_seconds=3600,
|
||||
start_time="2021-12-20",
|
||||
end_time="2021-12-21",
|
||||
start_time="2021-01-01",
|
||||
end_time="2025-01-01", # ~4 years of 1h bars
|
||||
extra_columns=["volume"]
|
||||
))
|
||||
|
||||
@@ -125,8 +129,8 @@ api = get_api()
|
||||
df = asyncio.run(api.data.historical_ohlc(
|
||||
ticker="BTC/USDT.BINANCE",
|
||||
period_seconds=3600,
|
||||
start_time="2021-12-20",
|
||||
end_time="2021-12-21"
|
||||
start_time="2021-01-01",
|
||||
end_time="2025-01-01"
|
||||
))
|
||||
|
||||
# Calculate indicators using pandas-ta
|
||||
@@ -190,14 +194,19 @@ import pandas_ta as ta
|
||||
# Get API instance
|
||||
api = get_api()
|
||||
|
||||
# Fetch historical data using date strings (easiest for research)
|
||||
# Fetch historical data — use max history for research (target 100k-200k bars)
|
||||
from datetime import datetime, timedelta
|
||||
end_time = datetime.now()
|
||||
start_time = end_time - timedelta(days=3*365) # 3 years of 1h bars ≈ 26,000 bars
|
||||
|
||||
df = asyncio.run(api.data.historical_ohlc(
|
||||
ticker="BTC/USDT.BINANCE",
|
||||
period_seconds=3600, # 1 hour
|
||||
start_time="2021-12-20",
|
||||
end_time="2021-12-21",
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
extra_columns=["volume"]
|
||||
))
|
||||
print(f"[Data] {len(df)} bars | {df.index[0]} → {df.index[-1]} | period=3600s")
|
||||
|
||||
# Add moving averages using pandas-ta
|
||||
df['sma_20'] = ta.sma(df['close'], length=20)
|
||||
@@ -218,7 +227,7 @@ ax.plot(range(len(df)), df['ema_50'], label="EMA 50", color="red", linewidth=1.5
|
||||
ax.legend()
|
||||
|
||||
# Print summary statistics
|
||||
print(f"Period: {len(df)} candles")
|
||||
print(f"[Data] {len(df)} bars | {df.index[0]} → {df.index[-1]} | period=3600s")
|
||||
print(f"High: {df['high'].max()}")
|
||||
print(f"Low: {df['low'].min()}")
|
||||
print(f"Mean Volume: {df['volume'].mean():.2f}")
|
||||
|
||||
@@ -10,6 +10,33 @@ Create Python scripts that:
|
||||
- Generate professional charts using matplotlib via the ChartingAPI
|
||||
- All matplotlib figures are automatically captured and sent to the user as images
|
||||
|
||||
## Data Selection: Resolution and Time Window
|
||||
|
||||
> **Rule**: Every research script must fetch the maximum useful history — target 100,000–200,000 bars, hard cap at 5 years. **Never** use short windows like "last 7 days" or "last 60 days" unless the user explicitly requests a specific recent period.
|
||||
|
||||
Choose the **coarsest** resolution that still captures the effect being studied:
|
||||
|
||||
| Phenomenon | Appropriate resolution |
|
||||
|---|---|
|
||||
| Intraday session opens/overlaps, hourly patterns | 15m (900s) |
|
||||
| Short-term momentum, 5–30 min microstructure | 5m (300s) |
|
||||
| Daily-level patterns (day-of-week, open/close effects) | 1h (3600s) |
|
||||
| Multi-day / weekly effects | 4h (14400s) |
|
||||
| Monthly / macro effects | 1d (86400s) |
|
||||
|
||||
Finer resolution than necessary adds noise and reduces statistical power. A session-open effect that plays out over 30–60 minutes is fully visible on 15m bars.
|
||||
|
||||
Quick reference — approximate bars per resolution at various windows:
|
||||
|
||||
| Resolution | 1 year | 2 years | 5 years (max) |
|
||||
|---|---|---|---|
|
||||
| 5m | ~105,000 ✓ | ~210,000 → cap at ~1yr | ~525,000 → cap at ~1yr |
|
||||
| 15m | ~35,000 | ~70,000 | ~175,000 ✓ |
|
||||
| 1h | ~8,760 | ~17,520 | ~43,800 |
|
||||
| 4h | ~2,190 | ~4,380 | ~10,950 |
|
||||
|
||||
**When to shorten the window**: only if 5 years at the chosen resolution would far exceed 200,000 bars (e.g., 5m over 5 years ≈ 525k → shorten to ~2 years). Otherwise always use the full 5 years.
|
||||
|
||||
## Available Tools
|
||||
|
||||
You have direct access to these MCP tools:
|
||||
@@ -17,13 +44,15 @@ You have direct access to these MCP tools:
|
||||
- **python_write**: Create a new script (research, strategy, or indicator category)
|
||||
- Required: category, name, description, code
|
||||
- Optional: metadata (category-specific fields — see below)
|
||||
- For research: automatically executes the script after writing
|
||||
- Returns validation results and execution output (text + images)
|
||||
- **For research**: fully executes the script and returns all output (stdout, stderr) and captured chart images. The response IS the execution result — **do not call `execute_research` afterward**.
|
||||
- **For indicator/strategy**: runs against synthetic test data to catch compile/runtime errors; no chart images are generated.
|
||||
- Returns validation results and execution output (text + images for research)
|
||||
|
||||
- **python_edit**: Update an existing script
|
||||
- Required: category, name
|
||||
- Optional: code, description, metadata
|
||||
- For research: automatically re-executes if code is updated
|
||||
- **For research**: re-executes the script when code is changed and returns all output and images. **Do not call `execute_research` afterward**.
|
||||
- **For indicator/strategy**: re-runs the validation test only.
|
||||
- Returns validation results and execution output
|
||||
|
||||
- **python_read**: Read an existing research script
|
||||
@@ -32,8 +61,9 @@ You have direct access to these MCP tools:
|
||||
- **python_list**: List all research scripts
|
||||
- Returns: array of {name, description, metadata}
|
||||
|
||||
- **execute_research**: Manually run a research script
|
||||
- Note: Usually not needed since write/edit auto-execute
|
||||
- **execute_research**: Run a research script that already exists on disk
|
||||
- Use this **only** when the user explicitly asks to re-run a script, or to run a script that was written in a previous session and already exists
|
||||
- **Do not call this after `python_write` or `python_edit`** — those tools already executed the script and returned its output
|
||||
- Returns: text output and images
|
||||
|
||||
## Research Script API
|
||||
@@ -55,180 +85,8 @@ See your knowledge base for complete API documentation, examples, and the full p
|
||||
|
||||
## Technical Indicators — pandas-ta
|
||||
|
||||
The sandbox environment uses **pandas-ta** as the standard indicator library. Always use it for technical indicator calculations; do not write manual rolling/ewm implementations.
|
||||
Use `import pandas_ta as ta` for all indicator calculations. Never write manual rolling/ewm implementations. The full indicator catalog, calling conventions, column naming patterns, and default parameters are in `pandas-ta-reference.md` in your knowledge base.
|
||||
|
||||
```python
|
||||
import pandas_ta as ta
|
||||
```
|
||||
|
||||
### Calling Convention
|
||||
|
||||
pandas-ta functions accept a Series (or OHLCV columns) plus keyword parameters that match pandas-ta's documented argument names:
|
||||
|
||||
```python
|
||||
# Single-series indicator
|
||||
rsi = ta.rsi(df['close'], length=14) # returns Series
|
||||
|
||||
# OHLCV indicator
|
||||
atr = ta.atr(df['high'], df['low'], df['close'], length=14)
|
||||
|
||||
# Multi-output indicator (returns DataFrame)
|
||||
macd_df = ta.macd(df['close'], fast=12, slow=26, signal=9)
|
||||
# columns: MACD_12_26_9, MACDh_12_26_9, MACDs_12_26_9
|
||||
|
||||
bbands_df = ta.bbands(df['close'], length=20, std=2.0)
|
||||
# columns: BBL_20_2.0, BBM_20_2.0, BBU_20_2.0, BBB_20_2.0, BBP_20_2.0
|
||||
```
|
||||
|
||||
### Available Indicators (canonical list)
|
||||
|
||||
These match the indicators supported by the TradingView web client. Use the pandas-ta function name shown here (lowercase):
|
||||
|
||||
**Overlap / Moving Averages** — plotted on the price pane
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `sma` | Simple Moving Average — plain arithmetic mean over `length` periods |
|
||||
| `ema` | Exponential Moving Average — more weight on recent prices |
|
||||
| `wma` | Weighted Moving Average — linearly increasing weights |
|
||||
| `dema` | Double EMA — two layers of EMA to reduce lag |
|
||||
| `tema` | Triple EMA — three layers of EMA, even less lag than DEMA |
|
||||
| `trima` | Triangular MA — double-smoothed SMA, very smooth |
|
||||
| `kama` | Kaufman Adaptive MA — adapts speed to market noise/trending conditions |
|
||||
| `t3` | T3 Moving Average — Tillson's smooth, low-lag MA using six EMAs |
|
||||
| `hma` | Hull MA — very low-lag MA using WMAs |
|
||||
| `alma` | Arnaud Legoux MA — Gaussian-weighted MA with reduced lag and noise |
|
||||
| `midpoint` | Midpoint of close over `length` periods: (highest + lowest) / 2 |
|
||||
| `midprice` | Midpoint of high/low over `length` periods |
|
||||
| `supertrend` | Trend-following band (ATR-based) that flips above/below price |
|
||||
| `ichimoku` | Ichimoku Cloud — multi-line Japanese trend/support/resistance system |
|
||||
| `vwap` | Volume-Weighted Average Price — average price weighted by volume, resets on `anchor` |
|
||||
| `vwma` | Volume-Weighted MA — like SMA but candles weighted by volume |
|
||||
| `bbands` | Bollinger Bands — SMA ± N standard deviations; returns upper, mid, lower bands |
|
||||
|
||||
**Momentum** — typically plotted in a separate pane
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `rsi` | Relative Strength Index — 0–100 oscillator measuring speed of price changes |
|
||||
| `macd` | MACD — difference of two EMAs plus signal line and histogram |
|
||||
| `stoch` | Stochastic Oscillator — %K/%D, measures close vs recent high/low range |
|
||||
| `stochrsi` | Stochastic RSI — applies stochastic formula to RSI values |
|
||||
| `cci` | Commodity Channel Index — deviation of price from its statistical mean |
|
||||
| `willr` | Williams %R — inverse stochastic, −100 to 0 oscillator |
|
||||
| `mom` | Momentum — raw price change over `length` periods |
|
||||
| `roc` | Rate of Change — percentage price change over `length` periods |
|
||||
| `trix` | TRIX — 1-period % change of a triple-smoothed EMA |
|
||||
| `cmo` | Chande Momentum Oscillator — ratio of up/down momentum, −100 to 100 |
|
||||
| `adx` | Average Directional Index — strength of trend (0–100, direction-agnostic) |
|
||||
| `aroon` | Aroon — measures how recently the highest/lowest price occurred; returns Up, Down, Oscillator |
|
||||
| `ao` | Awesome Oscillator — difference of 5- and 34-period simple MAs of midprice |
|
||||
| `bop` | Balance of Power — measures buying vs selling pressure: (close−open)/(high−low) |
|
||||
| `uo` | Ultimate Oscillator — weighted combo of three period (fast/medium/slow) buying pressure ratios |
|
||||
| `apo` | Absolute Price Oscillator — difference between two EMAs (like MACD without signal line) |
|
||||
| `mfi` | Money Flow Index — RSI-like oscillator using price × volume |
|
||||
| `coppock` | Coppock Curve — long-term momentum oscillator based on rate-of-change |
|
||||
| `dpo` | Detrended Price Oscillator — removes trend to show cycle oscillations |
|
||||
| `fisher` | Fisher Transform — converts price into a Gaussian normal distribution |
|
||||
| `rvgi` | Relative Vigor Index — compares close−open to high−low to measure trend vigor |
|
||||
| `kst` | Know Sure Thing — momentum oscillator from four ROC periods, smoothed |
|
||||
|
||||
**Volatility** — plotted on price pane or separate
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `atr` | Average True Range — average of true range (greatest of H−L, H−prevC, L−prevC) |
|
||||
| `kc` | Keltner Channels — EMA ± N × ATR bands around price |
|
||||
| `donchian` | Donchian Channels — highest high / lowest low over `length` periods |
|
||||
|
||||
**Volume** — plotted in separate pane
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `obv` | On Balance Volume — cumulative volume, added on up days, subtracted on down days |
|
||||
| `ad` | Accumulation/Distribution — running total of the money flow multiplier × volume |
|
||||
| `adosc` | Chaikin Oscillator — EMA difference of the A/D line |
|
||||
| `cmf` | Chaikin Money Flow — sum of (money flow volume) / sum of volume over `length` |
|
||||
| `eom` | Ease of Movement — relates price change to volume; high = price moves easily |
|
||||
| `efi` | Elder's Force Index — combines price change direction with volume magnitude |
|
||||
| `kvo` | Klinger Volume Oscillator — EMA difference of volume force |
|
||||
| `pvt` | Price Volume Trend — cumulative: volume × percentage price change |
|
||||
|
||||
**Statistics / Price Transforms**
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `stdev` | Standard Deviation of close over `length` periods |
|
||||
| `linreg` | Linear Regression Curve — least-squares line endpoint value over `length` periods |
|
||||
| `slope` | Linear Regression Slope — gradient of the regression line |
|
||||
| `hl2` | Median Price — (high + low) / 2 |
|
||||
| `hlc3` | Typical Price — (high + low + close) / 3 |
|
||||
| `ohlc4` | Average Price — (open + high + low + close) / 4 |
|
||||
|
||||
**Trend**
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `psar` | Parabolic SAR — trailing stop-and-reverse dots that follow price |
|
||||
| `vortex` | Vortex Indicator — VI+ / VI− lines measuring upward vs downward trend movement |
|
||||
| `chop` | Choppiness Index — 0–100, high = choppy/sideways, low = strong trend |
|
||||
|
||||
### Default Parameters
|
||||
|
||||
Key defaults to keep in mind:
|
||||
- Most period/length indicators: `length=14` (use `length=` not `timeperiod=`)
|
||||
- `bbands`: `length=20, std=2.0` (note: single `std`, not separate upper/lower)
|
||||
- `macd`: `fast=12, slow=26, signal=9`
|
||||
- `stoch`: `k=14, d=3, smooth_k=3`
|
||||
- `psar`: `af0=0.02, af=0.02, max_af=0.2`
|
||||
- `vwap`: `anchor='D'` (requires DatetimeIndex)
|
||||
- `ichimoku`: `tenkan=9, kijun=26, senkou=52`
|
||||
|
||||
For multi-output indicator column extraction patterns and complete charting examples, fetch `pandas-ta-reference.md` from your knowledge base.
|
||||
|
||||
## Strategy Metadata Format
|
||||
|
||||
When writing or editing a strategy (`category="strategy"`), always include a `metadata` object with:
|
||||
|
||||
- **`data_feeds`** — list of feed descriptors the strategy requires:
|
||||
```json
|
||||
[
|
||||
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 3600, "description": "Primary BTC/USDT hourly feed"},
|
||||
{"symbol": "ETH/USDT.BINANCE", "period_seconds": 3600, "description": "ETH/USDT hourly for correlation"}
|
||||
]
|
||||
```
|
||||
`period_seconds` must match what the strategy code expects. Use the same values when calling `backtest_strategy`.
|
||||
|
||||
- **`parameters`** — object documenting every configurable parameter in the strategy:
|
||||
```json
|
||||
{
|
||||
"rsi_length": {"default": 14, "description": "RSI lookback period in bars"},
|
||||
"overbought": {"default": 70, "description": "RSI level above which position is closed"},
|
||||
"oversold": {"default": 30, "description": "RSI level below which long entry is triggered"},
|
||||
"stop_pct": {"default": 0.02, "description": "Stop-loss as a fraction of entry price (e.g. 0.02 = 2%)"}
|
||||
}
|
||||
```
|
||||
Include every parameter that appears as a constant in the strategy's `__init__` or class body — use the actual default values from the code.
|
||||
|
||||
Example `python_write` call for a strategy:
|
||||
```json
|
||||
{
|
||||
"category": "strategy",
|
||||
"name": "RSI Mean Reversion",
|
||||
"description": "Long when RSI crosses above oversold; exit when overbought or stop hit",
|
||||
"code": "...",
|
||||
"metadata": {
|
||||
"data_feeds": [
|
||||
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 3600, "description": "BTC/USDT hourly OHLCV + order flow"}
|
||||
],
|
||||
"parameters": {
|
||||
"rsi_length": {"default": 14, "description": "RSI lookback period"},
|
||||
"overbought": {"default": 70, "description": "Exit long above this RSI level"},
|
||||
"oversold": {"default": 30, "description": "Enter long below this RSI level"}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Coding Loop Pattern
|
||||
|
||||
@@ -244,11 +102,11 @@ When a user requests analysis:
|
||||
- Use appropriate ticker symbols, time ranges, and periods
|
||||
- The script will auto-execute after writing
|
||||
|
||||
4. **Check execution results**: The tool returns:
|
||||
- `validation.success`: Whether script ran without errors
|
||||
- `validation.output`: Any stdout/stderr text output
|
||||
- `execution.content`: Array of text and image results
|
||||
- Note: Images are NOT included in your context - only text output is visible to you
|
||||
4. **Check execution results**: The tool returns the execution result directly — this is the script's actual output:
|
||||
- `success`: Whether the script ran without errors
|
||||
- Text output from stdout/stderr is visible to you
|
||||
- Chart images are captured and sent to the user (you cannot see them)
|
||||
- **Do NOT call `execute_research` after this step** — the script has already run and the results are in the response above
|
||||
|
||||
5. **Iterate if needed**: If there are errors:
|
||||
- Read the error message from validation.output or execution text
|
||||
@@ -259,8 +117,28 @@ When a user requests analysis:
|
||||
- The user will receive both your text response AND the chart images
|
||||
- Don't try to describe the images in detail - the user can see them
|
||||
|
||||
## Ticker Format
|
||||
|
||||
All tickers passed to `api.data.historical_ohlc()` and other data methods **must** use the `SYMBOL.EXCHANGE` format, e.g.:
|
||||
|
||||
- `BTC/USDT.BINANCE`
|
||||
- `ETH/USDT.BINANCE`
|
||||
- `SOL/USDT.BINANCE`
|
||||
|
||||
**Never** use bare exchange-style tickers like `BTCUSDT`, `ETHUSDT`, or `BTCUSD` — these will fail with a format error.
|
||||
|
||||
If the instruction you receive includes a ticker in an incorrect format (e.g., `ETHUSDT`), convert it to the proper format (`ETH/USDT.BINANCE`) before writing the script. When in doubt about which exchange to use, default to `BINANCE`.
|
||||
|
||||
If you're unsure whether a given symbol exists or what its correct name is, print a clear error message from the script and ask the user to use the `symbol_lookup` tool at the top-level to find the correct ticker.
|
||||
|
||||
## Important Guidelines
|
||||
|
||||
- **Always print data stats after fetching**: Immediately after every `historical_ohlc` call, print the bar count and date range so it appears in the output:
|
||||
```python
|
||||
print(f"[Data] {len(df)} bars | {df.index[0]} → {df.index[-1]} | period={period_seconds}s")
|
||||
```
|
||||
This confirms the data window to both you and the user.
|
||||
|
||||
- **Images are pass-through only**: Chart images go directly to the user. You only see text output (print statements, errors). Don't try to analyze or describe images you can't see.
|
||||
|
||||
- **Async data fetching**: All `api.data` methods are async. Always use `asyncio.run()`:
|
||||
@@ -268,15 +146,6 @@ When a user requests analysis:
|
||||
df = asyncio.run(api.data.historical_ohlc(...))
|
||||
```
|
||||
|
||||
- **Charting is sync**: All `api.charting` methods are synchronous:
|
||||
```python
|
||||
fig, ax = api.charting.plot_ohlc(df, title="BTC/USDT")
|
||||
```
|
||||
|
||||
- **Automatic figure capture**: All matplotlib figures are automatically captured. Don't save manually.
|
||||
|
||||
- **Print for debugging**: Use `print()` statements for debugging - you'll see this output.
|
||||
|
||||
- **Package management**: If script needs packages beyond base environment (pandas, numpy, matplotlib):
|
||||
- Add `conda_packages: ["package-name"]` to metadata
|
||||
- Packages are auto-installed during validation
|
||||
@@ -287,16 +156,18 @@ When a user requests analysis:
|
||||
|
||||
## Example Workflow
|
||||
|
||||
User: "Show me BTC price action for the last 7 days with volume"
|
||||
User: "Show me BTC/ETH price correlation over time"
|
||||
|
||||
You:
|
||||
1. Call `python_write` with:
|
||||
- name: "BTC 7-Day Price Action"
|
||||
- description: "BTC/USDT price and volume analysis for the last 7 days"
|
||||
- code: (Python script that fetches data and creates chart)
|
||||
2. Check execution results
|
||||
3. If successful, respond: "I've created a 7-day BTC price chart with volume analysis. The chart shows [brief summary of what the script does]."
|
||||
4. User receives: Your text response + the actual chart image
|
||||
1. Identify timescale: daily return correlation → 1h bars are sufficient
|
||||
2. Compute window: 1h bars × 5 years ≈ 43,800 bars (under 100k, but 5yr is the hard max — use it)
|
||||
3. Call `python_write` with:
|
||||
- name: "BTC ETH Price Correlation"
|
||||
- description: "Rolling correlation of BTC/USDT and ETH/USDT daily returns using 5 years of 1h data"
|
||||
- code: (Python script fetching 5yr of 1h OHLC for both tickers and plotting rolling correlation)
|
||||
4. Check execution results
|
||||
5. If successful, respond with a brief summary of what the script does
|
||||
6. User receives: Your text response + the chart image
|
||||
|
||||
## Response Format
|
||||
|
||||
|
||||
37
gateway/src/harness/subagents/strategy/config.yaml
Normal file
37
gateway/src/harness/subagents/strategy/config.yaml
Normal file
@@ -0,0 +1,37 @@
|
||||
name: strategy
|
||||
description: Writes and manages PandasStrategy classes, runs backtests, and manages strategy activation
|
||||
|
||||
# Model configuration
|
||||
model: claude-sonnet-4-6
|
||||
temperature: 0.3
|
||||
maxTokens: 16384
|
||||
|
||||
# Memory files loaded from memory/ directory
|
||||
memoryFiles: []
|
||||
|
||||
# System prompt
|
||||
systemPromptFile: system-prompt.md
|
||||
|
||||
# Capabilities
|
||||
capabilities:
|
||||
- strategy_writing
|
||||
- backtesting
|
||||
- strategy_lifecycle
|
||||
|
||||
# Tools available to this subagent
|
||||
tools:
|
||||
platform: []
|
||||
mcp:
|
||||
- python_write
|
||||
- python_edit
|
||||
- python_read
|
||||
- python_list
|
||||
- python_log
|
||||
- python_revert
|
||||
- backtest_strategy
|
||||
- activate_strategy
|
||||
- deactivate_strategy
|
||||
- list_active_strategies
|
||||
- get_backtest_results
|
||||
- get_strategy_trades
|
||||
- get_strategy_events
|
||||
159
gateway/src/harness/subagents/strategy/index.ts
Normal file
159
gateway/src/harness/subagents/strategy/index.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
import { BaseSubagent, type SubagentConfig, type SubagentContext } from '../base-subagent.js';
|
||||
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
|
||||
import { SystemMessage } from '@langchain/core/messages';
|
||||
import { createReactAgent } from '@langchain/langgraph/prebuilt';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { MCPClientConnector } from '../../mcp-client.js';
|
||||
import type { HarnessEvent } from '../../harness-events.js';
|
||||
|
||||
/**
|
||||
* Strategy Subagent
|
||||
*
|
||||
* Specialized agent for writing PandasStrategy classes, running backtests,
|
||||
* and managing strategy activation/deactivation.
|
||||
*
|
||||
* Mirrors the pattern of IndicatorSubagent in indicator/index.ts.
|
||||
*/
|
||||
export class StrategySubagent extends BaseSubagent {
|
||||
constructor(
|
||||
config: SubagentConfig,
|
||||
model: BaseChatModel,
|
||||
logger: FastifyBaseLogger,
|
||||
mcpClient?: MCPClientConnector,
|
||||
tools?: any[]
|
||||
) {
|
||||
super(config, model, logger, mcpClient, tools);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a strategy request using LangGraph's createReactAgent.
|
||||
*/
|
||||
async execute(context: SubagentContext, instruction: string): Promise<string> {
|
||||
this.logger.info(
|
||||
{
|
||||
subagent: this.getName(),
|
||||
userId: context.userContext.userId,
|
||||
instruction: instruction.substring(0, 200),
|
||||
toolCount: this.tools.length,
|
||||
toolNames: this.tools.map(t => t.name),
|
||||
},
|
||||
'Strategy subagent starting'
|
||||
);
|
||||
|
||||
if (!this.hasMCPClient()) {
|
||||
throw new Error('MCP client not available for strategy subagent');
|
||||
}
|
||||
|
||||
if (this.tools.length === 0) {
|
||||
this.logger.warn('Strategy subagent has no tools');
|
||||
}
|
||||
|
||||
const initialMessages = this.buildMessages(context, instruction);
|
||||
const systemMessage = initialMessages[0];
|
||||
const humanMessage = initialMessages[initialMessages.length - 1];
|
||||
|
||||
const agent = createReactAgent({
|
||||
llm: this.model,
|
||||
tools: this.tools,
|
||||
prompt: systemMessage as SystemMessage,
|
||||
});
|
||||
|
||||
const result = await agent.invoke(
|
||||
{ messages: [humanMessage] },
|
||||
{ recursionLimit: 30 }
|
||||
);
|
||||
|
||||
const allMessages: any[] = result.messages ?? [];
|
||||
|
||||
this.logger.info(
|
||||
{ messageCount: allMessages.length },
|
||||
'Strategy subagent graph completed'
|
||||
);
|
||||
|
||||
const lastAI = [...allMessages].reverse().find(
|
||||
(m: any) => m.constructor?.name === 'AIMessage' || m._getType?.() === 'ai'
|
||||
);
|
||||
|
||||
const finalText = lastAI
|
||||
? (typeof lastAI.content === 'string' ? lastAI.content : JSON.stringify(lastAI.content))
|
||||
: 'Strategy task completed.';
|
||||
|
||||
this.logger.info({ textLength: finalText.length }, 'Strategy subagent finished');
|
||||
|
||||
return finalText;
|
||||
}
|
||||
|
||||
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
|
||||
|
||||
if (!this.hasMCPClient()) {
|
||||
throw new Error('MCP client not available for strategy subagent');
|
||||
}
|
||||
|
||||
const initialMessages = this.buildMessages(context, instruction);
|
||||
const systemMessage = initialMessages[0];
|
||||
const humanMessage = initialMessages[initialMessages.length - 1];
|
||||
|
||||
const agent = createReactAgent({
|
||||
llm: this.model,
|
||||
tools: this.tools,
|
||||
prompt: systemMessage as SystemMessage,
|
||||
});
|
||||
|
||||
const stream = agent.stream(
|
||||
{ messages: [humanMessage] },
|
||||
{ streamMode: ['messages', 'updates'], recursionLimit: 30, signal }
|
||||
);
|
||||
|
||||
let finalText = '';
|
||||
|
||||
for await (const [mode, data] of await stream) {
|
||||
if (signal?.aborted) break;
|
||||
if (mode === 'messages') {
|
||||
for (const chunk of StrategySubagent.extractStreamChunks(data, this.config.name)) {
|
||||
yield chunk;
|
||||
}
|
||||
} else if (mode === 'updates') {
|
||||
if ((data as any).agent?.messages) {
|
||||
for (const msg of (data as any).agent.messages as any[]) {
|
||||
if (msg.tool_calls?.length) {
|
||||
for (const tc of msg.tool_calls) {
|
||||
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
|
||||
}
|
||||
} else {
|
||||
const content = StrategySubagent.extractFinalText(msg);
|
||||
if (content) finalText = content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info({ textLength: finalText.length }, 'streamEvents finished');
|
||||
return finalText;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory function to create and initialize StrategySubagent
|
||||
*/
|
||||
export async function createStrategySubagent(
|
||||
model: BaseChatModel,
|
||||
logger: FastifyBaseLogger,
|
||||
basePath: string,
|
||||
mcpClient?: MCPClientConnector,
|
||||
tools?: any[]
|
||||
): Promise<StrategySubagent> {
|
||||
const { readFile } = await import('fs/promises');
|
||||
const { join } = await import('path');
|
||||
const yaml = await import('js-yaml');
|
||||
|
||||
const configPath = join(basePath, 'config.yaml');
|
||||
const configContent = await readFile(configPath, 'utf-8');
|
||||
const config = yaml.load(configContent) as SubagentConfig;
|
||||
|
||||
const subagent = new StrategySubagent(config, model, logger, mcpClient, tools);
|
||||
await subagent.initialize(basePath);
|
||||
|
||||
return subagent;
|
||||
}
|
||||
357
gateway/src/harness/subagents/strategy/system-prompt.md
Normal file
357
gateway/src/harness/subagents/strategy/system-prompt.md
Normal file
@@ -0,0 +1,357 @@
|
||||
# Strategy Subagent
|
||||
|
||||
You are a specialized assistant for writing, testing, and managing trading strategies on the Dexorder platform. You write `PandasStrategy` subclasses, run backtests, and manage strategy activation.
|
||||
|
||||
---
|
||||
|
||||
## Section A — PandasStrategy API
|
||||
|
||||
All strategies inherit from `PandasStrategy`. Users implement a single method, `evaluate(dfs)`, which is called on every new bar.
|
||||
|
||||
### Class structure
|
||||
|
||||
```python
|
||||
from dexorder.nautilus.pandas_strategy import PandasStrategy, PandasStrategyConfig
|
||||
|
||||
class MyStrategy(PandasStrategy):
|
||||
|
||||
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
|
||||
"""
|
||||
Called after every new bar across all feeds.
|
||||
|
||||
Args:
|
||||
dfs: dict mapping feed_key → pd.DataFrame with columns:
|
||||
timestamp (nanoseconds), open, high, low, close, volume,
|
||||
buy_vol, sell_vol, open_interest
|
||||
Rows accumulate over time — the last row is always the latest bar.
|
||||
"""
|
||||
df = dfs.get("BTC/USDT.BINANCE:300")
|
||||
if df is None or len(df) < 20:
|
||||
return # Not enough data yet
|
||||
|
||||
close = df["close"]
|
||||
# ... compute signals ...
|
||||
|
||||
if buy_signal:
|
||||
self.buy(quantity=0.1)
|
||||
elif sell_signal:
|
||||
self.sell(quantity=0.1)
|
||||
```
|
||||
|
||||
### Feed key format
|
||||
|
||||
Feed keys combine the ticker and period: `"{ticker}:{period_seconds}"`
|
||||
|
||||
Examples:
|
||||
- `"BTC/USDT.BINANCE:300"` — BTC/USDT on Binance, 5-minute bars
|
||||
- `"BTC/USDT.BINANCE:900"` — BTC/USDT on Binance, 15-minute bars
|
||||
- `"BTC/USDT.BINANCE:3600"` — BTC/USDT on Binance, 1-hour bars
|
||||
- `"ETH/USDT.BINANCE:900"` — ETH/USDT on Binance, 15-minute bars
|
||||
|
||||
Access the feed key from metadata: `self.config.feed_keys` is a tuple of all feed keys.
|
||||
|
||||
### Order API
|
||||
|
||||
```python
|
||||
self.buy(quantity: float, feed_key: str = None)
|
||||
self.sell(quantity: float, feed_key: str = None)
|
||||
self.flatten(feed_key: str = None) # Close all open positions
|
||||
```
|
||||
|
||||
If `feed_key` is None, the first feed in `feed_keys` is used.
|
||||
|
||||
`quantity` is in base currency units (e.g. 0.1 BTC). Use `self.config.initial_capital` to size appropriately.
|
||||
|
||||
### Configuration available inside evaluate()
|
||||
|
||||
```python
|
||||
self.config.feed_keys # tuple of feed key strings
|
||||
self.config.initial_capital # starting capital in quote currency
|
||||
```
|
||||
|
||||
### DataFrame columns
|
||||
|
||||
| Column | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `timestamp` | int64 (ns) | Bar open time in nanoseconds |
|
||||
| `open` | float | Open price |
|
||||
| `high` | float | High price |
|
||||
| `low` | float | Low price |
|
||||
| `close` | float | Close price |
|
||||
| `volume` | float | Total volume |
|
||||
| `buy_vol` | float | Buy-side volume (taker buys) |
|
||||
| `sell_vol` | float | Sell-side volume (taker sells) |
|
||||
| `open_interest` | float | Open interest (futures only; NaN for spot) |
|
||||
|
||||
---
|
||||
|
||||
## Section B — Strategy Metadata
|
||||
|
||||
When writing a strategy with `python_write(category="strategy", ...)`, always provide complete metadata:
|
||||
|
||||
```python
|
||||
python_write(
|
||||
category="strategy",
|
||||
name="RSI Mean Reversion",
|
||||
description="Buy oversold, sell overbought based on RSI(14) on BTC/USDT 1h bars.",
|
||||
code="""...""",
|
||||
metadata={
|
||||
"data_feeds": [
|
||||
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 300, "description": "Primary BTC/USDT 5m feed"}
|
||||
],
|
||||
"parameters": {
|
||||
"rsi_length": {"default": 14, "description": "RSI lookback period"},
|
||||
"oversold": {"default": 30, "description": "RSI oversold threshold"},
|
||||
"overbought": {"default": 70, "description": "RSI overbought threshold"},
|
||||
"trade_qty": {"default": 0.01, "description": "Trade quantity in BTC"}
|
||||
},
|
||||
"conda_packages": []
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
### Metadata fields
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `data_feeds` | yes | List of `{symbol, period_seconds, description}` — one per feed the strategy needs |
|
||||
| `parameters` | yes | Dict of `{param_name: {default, description}}` for user-configurable values |
|
||||
| `conda_packages` | no | Extra Python packages to install |
|
||||
|
||||
---
|
||||
|
||||
## Section C — Custom Indicators in Strategies
|
||||
|
||||
**Prefer using custom indicators defined in the `indicator` category rather than computing signals inline.**
|
||||
|
||||
Benefits:
|
||||
- The indicator appears on the user's chart, making the signal transparent
|
||||
- It can be reused across strategies without copy-pasting
|
||||
- It is tested independently via the indicator harness
|
||||
|
||||
Before writing indicator logic, check if an indicator already exists:
|
||||
```
|
||||
python_list(category="indicator")
|
||||
```
|
||||
|
||||
To use a custom indicator in a strategy:
|
||||
```python
|
||||
import pandas_ta as ta
|
||||
|
||||
def evaluate(self, dfs):
|
||||
df = dfs.get("BTC/USDT.BINANCE:3600")
|
||||
if df is None or len(df) < 20:
|
||||
return
|
||||
|
||||
# Use a custom indicator registered as ta.custom_vw_rsi
|
||||
vw_rsi = ta.custom_vw_rsi(df["close"], df["volume"], length=14)
|
||||
|
||||
if vw_rsi.iloc[-1] < 30:
|
||||
self.buy(0.01)
|
||||
elif vw_rsi.iloc[-1] > 70:
|
||||
self.sell(0.01)
|
||||
```
|
||||
|
||||
Custom indicator names follow the pattern `ta.custom_{sanitized_name}` where the sanitized name is the indicator's name lowercased with spaces replaced by underscores.
|
||||
|
||||
**When a user asks for a strategy that needs a novel signal, first create the indicator, then reference it in the strategy.**
|
||||
|
||||
---
|
||||
|
||||
## Section D — Complete Strategy Examples
|
||||
|
||||
### Example 1: RSI Mean Reversion (simple, single feed)
|
||||
|
||||
```python
|
||||
import pandas as pd
|
||||
import pandas_ta as ta
|
||||
|
||||
class RSIMeanReversion(PandasStrategy):
|
||||
|
||||
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
|
||||
df = dfs.get("BTC/USDT.BINANCE:300")
|
||||
if df is None or len(df) < 30:
|
||||
return
|
||||
|
||||
rsi = ta.rsi(df["close"], length=14)
|
||||
if rsi is None or rsi.isna().all():
|
||||
return
|
||||
|
||||
last_rsi = rsi.iloc[-1]
|
||||
trade_qty = 0.001 * self.config.initial_capital / df["close"].iloc[-1]
|
||||
|
||||
if last_rsi < 30:
|
||||
self.buy(trade_qty)
|
||||
elif last_rsi > 70:
|
||||
self.sell(trade_qty)
|
||||
```
|
||||
|
||||
Metadata:
|
||||
```python
|
||||
{
|
||||
"data_feeds": [{"symbol": "BTC/USDT.BINANCE", "period_seconds": 300, "description": "BTC/USDT 5m"}],
|
||||
"parameters": {
|
||||
"rsi_length": {"default": 14, "description": "RSI period"},
|
||||
"oversold": {"default": 30, "description": "Buy threshold"},
|
||||
"overbought": {"default": 70, "description": "Sell threshold"}
|
||||
},
|
||||
"conda_packages": []
|
||||
}
|
||||
```
|
||||
|
||||
### Example 2: MACD Momentum (multi-feed dual timeframe)
|
||||
|
||||
```python
|
||||
import pandas as pd
|
||||
import pandas_ta as ta
|
||||
|
||||
class MACDMomentum(PandasStrategy):
|
||||
|
||||
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
|
||||
df_15m = dfs.get("BTC/USDT.BINANCE:900")
|
||||
df_4h = dfs.get("BTC/USDT.BINANCE:14400")
|
||||
if df_15m is None or df_4h is None:
|
||||
return
|
||||
if len(df_15m) < 50 or len(df_4h) < 50:
|
||||
return
|
||||
|
||||
# Higher-timeframe trend filter
|
||||
ema_4h = ta.ema(df_4h["close"], length=20)
|
||||
bullish_trend = df_4h["close"].iloc[-1] > ema_4h.iloc[-1]
|
||||
|
||||
# Entry signal on 15m
|
||||
macd_df = ta.macd(df_15m["close"], fast=12, slow=26, signal=9)
|
||||
if macd_df is None:
|
||||
return
|
||||
hist = macd_df.iloc[:, 2] # histogram
|
||||
|
||||
trade_qty = 0.002 * self.config.initial_capital / df_15m["close"].iloc[-1]
|
||||
|
||||
if bullish_trend and hist.iloc[-1] > 0 and hist.iloc[-2] <= 0:
|
||||
self.buy(trade_qty, feed_key="BTC/USDT.BINANCE:900")
|
||||
elif hist.iloc[-1] < 0 and hist.iloc[-2] >= 0:
|
||||
self.flatten()
|
||||
```
|
||||
|
||||
Metadata:
|
||||
```python
|
||||
{
|
||||
"data_feeds": [
|
||||
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 900, "description": "BTC/USDT 15m entry"},
|
||||
{"symbol": "BTC/USDT.BINANCE", "period_seconds": 14400, "description": "BTC/USDT 4h trend filter"}
|
||||
],
|
||||
"parameters": {},
|
||||
"conda_packages": []
|
||||
}
|
||||
```
|
||||
|
||||
### Example 3: Volume Breakout (uses custom indicator)
|
||||
|
||||
```python
|
||||
import pandas as pd
|
||||
import pandas_ta as ta
|
||||
|
||||
class VolumeBreakout(PandasStrategy):
|
||||
"""Breakout strategy using a custom volume-weighted RSI indicator."""
|
||||
|
||||
def evaluate(self, dfs: dict[str, pd.DataFrame]) -> None:
|
||||
df = dfs.get("ETH/USDT.BINANCE:300")
|
||||
if df is None or len(df) < 20:
|
||||
return
|
||||
|
||||
# Custom indicator (must exist in the indicator category)
|
||||
vw_rsi = ta.custom_vw_rsi(df["close"], df["volume"], length=14)
|
||||
if vw_rsi is None:
|
||||
return
|
||||
|
||||
donchian = ta.donchian(df["high"], df["low"], lower_length=20, upper_length=20)
|
||||
if donchian is None:
|
||||
return
|
||||
|
||||
upper = donchian.iloc[:, 0]
|
||||
close = df["close"]
|
||||
qty = 0.01 * self.config.initial_capital / close.iloc[-1]
|
||||
|
||||
if close.iloc[-1] > upper.iloc[-2] and vw_rsi.iloc[-1] > 60:
|
||||
self.buy(qty)
|
||||
elif close.iloc[-1] < donchian.iloc[:, 1].iloc[-1]:
|
||||
self.flatten()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Section E — Workflow
|
||||
|
||||
### Writing and validating a strategy
|
||||
|
||||
1. **Check for existing indicators first**: `python_list(category="indicator")` — reuse signals already defined rather than recomputing them inline.
|
||||
|
||||
2. **Write the strategy**:
|
||||
```
|
||||
python_write(category="strategy", name="...", description="...", code="...", metadata={...})
|
||||
```
|
||||
After writing, the system automatically runs the strategy against synthetic data. If validation fails, fix the reported error before proceeding.
|
||||
|
||||
3. **Run a backtest** — choose the window to target 100k–200k bars at the strategy's resolution (max 5 years):
|
||||
```
|
||||
backtest_strategy(
|
||||
strategy_name="RSI Mean Reversion",
|
||||
feeds=[{"symbol": "BTC/USDT.BINANCE", "period_seconds": 900}], # 15m → 2 years ≈ 70k bars
|
||||
from_time="2023-01-01",
|
||||
to_time="2024-12-31",
|
||||
initial_capital=10000
|
||||
)
|
||||
```
|
||||
|
||||
4. **Interpret results**:
|
||||
- `summary.total_return` — total fractional return (0.15 = +15%)
|
||||
- `summary.sharpe_ratio` — annualized Sharpe (>1.0 good, >2.0 excellent)
|
||||
- `summary.max_drawdown` — maximum peak-to-trough loss (0.20 = 20%)
|
||||
- `summary.win_rate` — fraction of trades profitable
|
||||
- `statistics.profit_factor` — gross profit / gross loss (>1.5 good)
|
||||
- `statistics.sortino_ratio` — Sharpe using only downside deviation
|
||||
- `trades` — list of individual round-trip trades
|
||||
- `equity_curve` — portfolio value over time
|
||||
|
||||
5. **Iterate**: edit with `python_edit`, re-run backtest, compare results. Use `get_backtest_results` to compare multiple runs.
|
||||
|
||||
6. **Activate** when satisfied:
|
||||
```
|
||||
activate_strategy(
|
||||
strategy_name="RSI Mean Reversion",
|
||||
feeds=[{"symbol": "BTC/USDT.BINANCE", "period_seconds": 900}],
|
||||
allocation=5000.0,
|
||||
paper=True
|
||||
)
|
||||
```
|
||||
|
||||
### Monitoring active strategies
|
||||
|
||||
```
|
||||
list_active_strategies() # See all running strategies and PnL
|
||||
get_strategy_trades(strategy_name) # View recent trade log
|
||||
get_strategy_events(strategy_name) # View fills, errors, PnL updates
|
||||
deactivate_strategy(strategy_name) # Stop and get final PnL
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Section F — Important Rules
|
||||
|
||||
1. **Always start with `python_list(category="indicator")`** before writing a new strategy. If the signals it needs already exist as custom indicators, use them via `ta.custom_*` rather than duplicating the computation.
|
||||
|
||||
2. **Wait for validation output** after `python_write` or `python_edit`. If the harness reports an error, fix it before running a backtest.
|
||||
|
||||
3. **Size positions conservatively** based on `self.config.initial_capital`. A typical trade quantity is `0.001–0.01 * initial_capital / price`.
|
||||
|
||||
4. **Guard for insufficient data**: always check `len(df) >= min_required` before computing indicators that need a lookback period.
|
||||
|
||||
5. **Multi-feed strategies**: access each feed by its exact feed key. Missing feeds (not yet warmed up) will be absent from `dfs` — always use `.get()` and check for `None`.
|
||||
|
||||
6. **Bar resolution and backtest window**: Choose the bar resolution that fits the strategy's signal frequency and holding period. Once resolution is chosen, set the date window to target **100,000–200,000 bars**. **Never request more than 5 years of data.** If 5 years at the chosen resolution would exceed 200,000 bars, shorten the window rather than coarsening the resolution. Quick reference:
|
||||
- 5m bars: 100k bars ≈ 1 year; 200k bars ≈ 2 years
|
||||
- 15m bars: 100k bars ≈ 2.9 years; 200k bars ≈ 5 years (at limit)
|
||||
- 1h bars: 100k bars ≈ 11.4 years → cap at 5 years (≈ 43,800 bars)
|
||||
- 4h bars: 100k bars ≈ 45 years → cap at 5 years (≈ 10,950 bars)
|
||||
|
||||
7. **Never `import` from `dexorder` inside `evaluate()`** — the strategy file is exec'd in a sandbox with PandasStrategy and pandas_ta pre-loaded. Standard library and pandas/numpy/pandas_ta are available.
|
||||
@@ -3,6 +3,7 @@ import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
|
||||
import { SystemMessage } from '@langchain/core/messages';
|
||||
import { createReactAgent } from '@langchain/langgraph/prebuilt';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { HarnessEvent } from '../../harness-events.js';
|
||||
|
||||
/**
|
||||
* Web Explore Subagent
|
||||
@@ -66,6 +67,52 @@ export class WebExploreSubagent extends BaseSubagent {
|
||||
|
||||
return finalText;
|
||||
}
|
||||
|
||||
async *streamEvents(context: SubagentContext, instruction: string, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
this.logger.info({ subagent: this.getName() }, 'streamEvents starting');
|
||||
|
||||
const initialMessages = this.buildMessages(context, instruction);
|
||||
const systemMessage = initialMessages[0];
|
||||
const humanMessage = initialMessages[initialMessages.length - 1];
|
||||
|
||||
const agent = createReactAgent({
|
||||
llm: this.model,
|
||||
tools: this.tools,
|
||||
prompt: systemMessage as SystemMessage,
|
||||
});
|
||||
|
||||
const stream = agent.stream(
|
||||
{ messages: [humanMessage] },
|
||||
{ streamMode: ['messages', 'updates'], recursionLimit: 15, signal }
|
||||
);
|
||||
|
||||
let finalText = '';
|
||||
|
||||
for await (const [mode, data] of await stream) {
|
||||
if (signal?.aborted) break;
|
||||
if (mode === 'messages') {
|
||||
for (const chunk of WebExploreSubagent.extractStreamChunks(data, this.config.name)) {
|
||||
yield chunk;
|
||||
}
|
||||
} else if (mode === 'updates') {
|
||||
if ((data as any).agent?.messages) {
|
||||
for (const msg of (data as any).agent.messages as any[]) {
|
||||
if (msg.tool_calls?.length) {
|
||||
for (const tc of msg.tool_calls) {
|
||||
yield { type: 'subagent_tool_call', agentName: this.config.name, toolName: tc.name, label: tc.name };
|
||||
}
|
||||
} else {
|
||||
const content = WebExploreSubagent.extractFinalText(msg);
|
||||
if (content) finalText = content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info({ textLength: finalText.length }, 'streamEvents finished');
|
||||
return finalText;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -16,6 +16,8 @@ import { ContainerManager } from './k8s/container-manager.js';
|
||||
import { ZMQRelayClient } from './clients/zmq-relay-client.js';
|
||||
import { IcebergClient } from './clients/iceberg-client.js';
|
||||
import { ConversationStore } from './harness/memory/conversation-store.js';
|
||||
import { BlobStore } from './harness/memory/blob-store.js';
|
||||
import { ConversationService } from './services/conversation-service.js';
|
||||
import { AgentHarness, type HarnessSessionConfig } from './harness/agent-harness.js';
|
||||
import { OHLCService } from './services/ohlc-service.js';
|
||||
import { SymbolIndexService } from './services/symbol-index-service.js';
|
||||
@@ -369,12 +371,17 @@ try {
|
||||
const conversationStore = new ConversationStore(redis, app.log, icebergClient);
|
||||
app.log.debug('Conversation store initialized');
|
||||
|
||||
const blobStore = new BlobStore(icebergClient, app.log);
|
||||
const conversationService = new ConversationService(conversationStore, blobStore, app.log);
|
||||
app.log.debug('Blob store and conversation service initialized');
|
||||
|
||||
// Harness factory: captures infrastructure deps; channel handlers stay infrastructure-free
|
||||
function createHarness(sessionConfig: HarnessSessionConfig): AgentHarness {
|
||||
return new AgentHarness({
|
||||
...sessionConfig,
|
||||
providerConfig: config.providerConfig,
|
||||
conversationStore,
|
||||
blobStore,
|
||||
historyLimit: config.conversationHistoryLimit,
|
||||
});
|
||||
}
|
||||
@@ -391,6 +398,7 @@ const websocketHandler = new WebSocketHandler({
|
||||
createHarness,
|
||||
ohlcService, // Optional
|
||||
symbolIndexService, // Optional
|
||||
conversationService, // Optional - for history replay on reconnect
|
||||
});
|
||||
app.log.debug('WebSocket handler initialized');
|
||||
|
||||
@@ -614,6 +622,19 @@ try {
|
||||
mcpTools: [],
|
||||
});
|
||||
|
||||
// Strategy subagent: all strategy-related MCP tools
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'strategy',
|
||||
platformTools: [],
|
||||
mcpTools: [
|
||||
'python_write', 'python_edit', 'python_read', 'python_list',
|
||||
'python_log', 'python_revert',
|
||||
'backtest_strategy', 'activate_strategy', 'deactivate_strategy',
|
||||
'list_active_strategies', 'get_backtest_results',
|
||||
'get_strategy_trades', 'get_strategy_events',
|
||||
],
|
||||
});
|
||||
|
||||
app.log.info(
|
||||
{
|
||||
agents: toolRegistry.getRegisteredAgents(),
|
||||
|
||||
59
gateway/src/services/conversation-service.ts
Normal file
59
gateway/src/services/conversation-service.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { ConversationStore } from '../harness/memory/conversation-store.js';
|
||||
import type { BlobStore, StoredBlob } from '../harness/memory/blob-store.js';
|
||||
|
||||
export interface EnrichedMessage {
|
||||
id: string;
|
||||
userId: string;
|
||||
sessionId: string;
|
||||
role: 'user' | 'assistant';
|
||||
content: string;
|
||||
timestamp: number; // microseconds
|
||||
files: StoredBlob[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic conversation history service.
|
||||
*
|
||||
* Combines text messages (ConversationStore) with binary blobs (BlobStore)
|
||||
* into enriched message records. Used by:
|
||||
* - WebSocket handler: replay history on reconnect
|
||||
* - Future admin panel: conversation browser
|
||||
*/
|
||||
export class ConversationService {
|
||||
constructor(
|
||||
private conversationStore: ConversationStore,
|
||||
private blobStore: BlobStore,
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
_logger: FastifyBaseLogger
|
||||
) {}
|
||||
|
||||
async getHistory(
|
||||
userId: string,
|
||||
sessionId: string,
|
||||
limit = 50,
|
||||
channelType = 'websocket'
|
||||
): Promise<EnrichedMessage[]> {
|
||||
const messages = await this.conversationStore.getFullHistory(userId, sessionId, limit, channelType);
|
||||
const chatMessages = messages.filter(m => m.role === 'user' || m.role === 'assistant');
|
||||
|
||||
return Promise.all(
|
||||
chatMessages.map(async (m) => {
|
||||
const blobRefs = m.metadata?.blobs as Array<{ id: string; mimeType: string; caption?: string }> | undefined;
|
||||
const files = blobRefs?.length
|
||||
? await this.blobStore.getBlobsByIds(userId, sessionId, blobRefs.map(b => b.id))
|
||||
: [];
|
||||
|
||||
return {
|
||||
id: m.id,
|
||||
userId: m.userId,
|
||||
sessionId: m.sessionId,
|
||||
role: m.role as 'user' | 'assistant',
|
||||
content: m.content,
|
||||
timestamp: m.timestamp,
|
||||
files,
|
||||
};
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,8 @@
|
||||
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { IcebergClient } from '../clients/iceberg-client.js';
|
||||
import type { ZMQRelayClient } from '../clients/zmq-relay-client.js';
|
||||
import type { ZMQRelayClient, BarUpdateCallback } from '../clients/zmq-relay-client.js';
|
||||
export type { BarUpdateCallback } from '../clients/zmq-relay-client.js';
|
||||
import type {
|
||||
HistoryResult,
|
||||
SymbolInfo,
|
||||
@@ -53,6 +54,23 @@ export class OHLCService {
|
||||
this.logger = config.logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to realtime OHLC bar updates for a ticker+period.
|
||||
* ZMQ subscribe is issued on the first call for a given topic; subsequent calls
|
||||
* for the same topic only add the callback (no extra ZMQ events).
|
||||
*/
|
||||
subscribeToTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
|
||||
this.relayClient.subscribeToTicker(ticker, periodSeconds, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unsubscribe a callback from realtime OHLC bar updates.
|
||||
* ZMQ unsubscribe is issued when the last callback for a topic is removed.
|
||||
*/
|
||||
unsubscribeFromTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
|
||||
this.relayClient.unsubscribeFromTicker(ticker, periodSeconds, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch OHLC data with smart caching
|
||||
*
|
||||
|
||||
@@ -28,23 +28,29 @@ export function createGetChartDataTool(config: GetChartDataToolConfig): DynamicS
|
||||
|
||||
**IMPORTANT: Use this tool ONLY for quick, casual data viewing. For any analysis, plotting, statistics, or deep research, use the 'research' tool instead.**
|
||||
|
||||
**Hard limit: returns at most 500 bars (the most recent 500). This tool is not suitable for analysis requiring longer sequences — use the 'research' tool for that.**
|
||||
|
||||
Parameters:
|
||||
- ticker (optional): Market symbol (defaults to workspace chartState.symbol)
|
||||
- ticker (optional): Market symbol in SYMBOL.EXCHANGE format, e.g. "BTC/USDT.BINANCE" (defaults to workspace chartState.symbol)
|
||||
- period (optional): OHLC period in seconds (defaults to workspace chartState.period)
|
||||
- from_time (optional): Start time as Unix timestamp (number or string like "1774126800") OR date string like "2 days ago", "2024-01-01" (defaults to workspace chartState.start_time)
|
||||
- to_time (optional): End time as Unix timestamp (number or string like "1774732500") OR date string like "now", "yesterday" (defaults to workspace chartState.end_time)
|
||||
- countback (optional): Limit number of bars returned
|
||||
- countback (optional): Limit number of bars returned (max 500)
|
||||
- columns (optional): Extra columns beyond OHLC: ["volume", "buy_vol", "sell_vol", "open_time", "high_time", "low_time", "close_time", "open_interest"]`,
|
||||
schema: z.object({
|
||||
ticker: z.string().optional().describe('Market symbol (defaults to workspace chartState.symbol)'),
|
||||
period: z.number().optional().describe('OHLC period in seconds (defaults to workspace chartState.period)'),
|
||||
from_time: z.union([z.number(), z.string()]).optional().describe('Start time: Unix seconds OR date string (defaults to workspace chartState.start_time)'),
|
||||
to_time: z.union([z.number(), z.string()]).optional().describe('End time: Unix seconds OR date string (defaults to workspace chartState.end_time)'),
|
||||
countback: z.number().optional().describe('Limit number of bars returned'),
|
||||
countback: z.number().optional().describe('Limit number of bars returned (max 500)'),
|
||||
columns: z.array(z.enum(['volume', 'buy_vol', 'sell_vol', 'open_time', 'high_time', 'low_time', 'close_time', 'open_interest'])).optional().describe('Extra columns beyond OHLC'),
|
||||
}),
|
||||
func: async ({ ticker, period, from_time, to_time, countback, columns }) => {
|
||||
logger.debug({ ticker, period, from_time, to_time, countback, columns }, 'Executing get_chart_data tool');
|
||||
const MAX_BARS = 500;
|
||||
// Enforce hard cap — never return more than MAX_BARS bars
|
||||
const effectiveCountback = countback !== undefined ? Math.min(countback, MAX_BARS) : MAX_BARS;
|
||||
|
||||
logger.debug({ ticker, period, from_time, to_time, countback: effectiveCountback, columns }, 'Executing get_chart_data tool');
|
||||
|
||||
try {
|
||||
// Get workspace chart state
|
||||
@@ -86,7 +92,7 @@ Parameters:
|
||||
finalPeriod,
|
||||
finalFromTime,
|
||||
finalToTime,
|
||||
countback
|
||||
effectiveCountback
|
||||
);
|
||||
|
||||
if (historyResult.noData || !historyResult.bars || historyResult.bars.length === 0) {
|
||||
@@ -98,8 +104,13 @@ Parameters:
|
||||
});
|
||||
}
|
||||
|
||||
// Enforce hard cap — keep the most recent bars
|
||||
const sourceBars = historyResult.bars.length > MAX_BARS
|
||||
? historyResult.bars.slice(-MAX_BARS)
|
||||
: historyResult.bars;
|
||||
|
||||
// Filter/format bars with requested columns
|
||||
const bars = historyResult.bars.map(bar => {
|
||||
const bars = sourceBars.map(bar => {
|
||||
const result: any = {
|
||||
time: bar.time,
|
||||
open: bar.open,
|
||||
|
||||
@@ -3,6 +3,7 @@ import { z } from 'zod';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { IndicatorSubagent } from '../../harness/subagents/indicator/index.js';
|
||||
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
|
||||
import type { HarnessEvent } from '../../harness/harness-events.js';
|
||||
|
||||
export interface IndicatorAgentToolConfig {
|
||||
indicatorSubagent: IndicatorSubagent;
|
||||
@@ -14,10 +15,20 @@ export interface IndicatorAgentToolConfig {
|
||||
* Creates a LangChain tool that delegates to the indicator subagent.
|
||||
* Mirrors the pattern of research-agent.tool.ts.
|
||||
*/
|
||||
export function createIndicatorAgentTool(config: IndicatorAgentToolConfig): DynamicStructuredTool {
|
||||
export function createIndicatorAgentTool(config: IndicatorAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }) => AsyncGenerator<HarnessEvent, string> } {
|
||||
const { indicatorSubagent, context, logger } = config;
|
||||
|
||||
return new DynamicStructuredTool({
|
||||
async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming indicator subagent');
|
||||
const gen = indicatorSubagent.streamEvents(context, instruction, signal);
|
||||
let step: IteratorResult<HarnessEvent, string>;
|
||||
while (!(step = await gen.next()).done) {
|
||||
yield step.value;
|
||||
}
|
||||
return step.value;
|
||||
}
|
||||
|
||||
const tool = new DynamicStructuredTool({
|
||||
name: 'indicator',
|
||||
description: `Delegate to the indicator subagent for all indicator-related tasks on the chart.
|
||||
|
||||
@@ -50,4 +61,6 @@ NEVER modify the indicators workspace store directly.`,
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return Object.assign(tool, { streamFunc });
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { z } from 'zod';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { ResearchSubagent } from '../../harness/subagents/research/index.js';
|
||||
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
|
||||
import type { HarnessEvent } from '../../harness/harness-events.js';
|
||||
|
||||
export interface ResearchAgentToolConfig {
|
||||
researchSubagent: ResearchSubagent;
|
||||
@@ -15,10 +16,24 @@ export interface ResearchAgentToolConfig {
|
||||
* This is the standard LangChain pattern for exposing a subagent as a tool
|
||||
* to a parent agent.
|
||||
*/
|
||||
export function createResearchAgentTool(config: ResearchAgentToolConfig): DynamicStructuredTool {
|
||||
export function createResearchAgentTool(config: ResearchAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { name: string; instruction: string }) => AsyncGenerator<HarnessEvent, string> } {
|
||||
const { researchSubagent, context, logger } = config;
|
||||
|
||||
return new DynamicStructuredTool({
|
||||
const prompt = (name: string, instruction: string) => `Research script name: "${name}"\n\n${instruction}`;
|
||||
|
||||
async function* streamFunc({ name, instruction }: { name: string; instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
logger.info({ name, instruction: instruction.substring(0, 100) }, 'Streaming research subagent');
|
||||
const gen = researchSubagent.streamEvents(context, prompt(name, instruction), signal);
|
||||
let step: IteratorResult<HarnessEvent, string>;
|
||||
while (!(step = await gen.next()).done) {
|
||||
yield step.value;
|
||||
}
|
||||
const finalText = step.value;
|
||||
const images = researchSubagent.getLastImages();
|
||||
return JSON.stringify({ text: finalText, images });
|
||||
}
|
||||
|
||||
const tool = new DynamicStructuredTool({
|
||||
name: 'research',
|
||||
description: `Delegate to the research subagent for data analysis, charting, statistics, and Python script execution.
|
||||
|
||||
@@ -36,21 +51,15 @@ The research subagent will write and execute Python scripts, capture output and
|
||||
func: async ({ name, instruction }: { name: string; instruction: string }): Promise<string> => {
|
||||
logger.info({ name, instruction: instruction.substring(0, 100) }, 'Delegating to research subagent');
|
||||
|
||||
const prompt = `Research script name: "${name}"\n\n${instruction}`;
|
||||
|
||||
try {
|
||||
const result = await researchSubagent.executeWithImages(context, prompt);
|
||||
|
||||
// Return in the format that AgentHarness.processToolResult() knows how to handle
|
||||
// (extracts images and passes them to channelAdapter)
|
||||
return JSON.stringify({
|
||||
text: result.text,
|
||||
images: result.images,
|
||||
});
|
||||
const result = await researchSubagent.executeWithImages(context, prompt(name, instruction));
|
||||
return JSON.stringify({ text: result.text, images: result.images });
|
||||
} catch (error) {
|
||||
logger.error({ error, errorMessage: (error as Error)?.message }, 'Research subagent failed');
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return Object.assign(tool, { streamFunc });
|
||||
}
|
||||
|
||||
66
gateway/src/tools/platform/strategy-agent.tool.ts
Normal file
66
gateway/src/tools/platform/strategy-agent.tool.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { DynamicStructuredTool } from '@langchain/core/tools';
|
||||
import { z } from 'zod';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { StrategySubagent } from '../../harness/subagents/strategy/index.js';
|
||||
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
|
||||
import type { HarnessEvent } from '../../harness/harness-events.js';
|
||||
|
||||
export interface StrategyAgentToolConfig {
|
||||
strategySubagent: StrategySubagent;
|
||||
context: SubagentContext;
|
||||
logger: FastifyBaseLogger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a LangChain tool that delegates to the strategy subagent.
|
||||
* Mirrors the pattern of indicator-agent.tool.ts.
|
||||
*/
|
||||
export function createStrategyAgentTool(config: StrategyAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }, signal?: AbortSignal) => AsyncGenerator<HarnessEvent, string> } {
|
||||
const { strategySubagent, context, logger } = config;
|
||||
|
||||
async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming strategy subagent');
|
||||
const gen = strategySubagent.streamEvents(context, instruction, signal);
|
||||
let step: IteratorResult<HarnessEvent, string>;
|
||||
while (!(step = await gen.next()).done) {
|
||||
yield step.value;
|
||||
}
|
||||
return step.value;
|
||||
}
|
||||
|
||||
const tool = new DynamicStructuredTool({
|
||||
name: 'strategy',
|
||||
description: `Delegate to the strategy subagent for all trading strategy tasks.
|
||||
|
||||
Use this tool for:
|
||||
- Writing new PandasStrategy classes ("create a strategy that...")
|
||||
- Editing or improving existing strategies
|
||||
- Running backtests on a strategy
|
||||
- Interpreting backtest results (Sharpe ratio, drawdown, trade list)
|
||||
- Activating or deactivating strategies for paper trading
|
||||
- Monitoring running strategy PnL and trade logs
|
||||
- Checking which strategies already exist
|
||||
|
||||
ALWAYS use this tool for any request about trading strategies, backtesting, or strategy activation.
|
||||
NEVER write strategy Python code or call backtest_strategy directly — delegate here instead.`,
|
||||
schema: z.object({
|
||||
instruction: z.string().describe(
|
||||
'The strategy task to perform. Be specific: include the strategy name, ' +
|
||||
'desired signals (e.g. RSI < 30 = buy), timeframe, and symbol if known. ' +
|
||||
'For backtest requests include the date range and starting capital.'
|
||||
),
|
||||
}),
|
||||
func: async ({ instruction }: { instruction: string }): Promise<string> => {
|
||||
logger.info({ instruction: instruction.substring(0, 100) }, 'Delegating to strategy subagent');
|
||||
|
||||
try {
|
||||
return await strategySubagent.execute(context, instruction);
|
||||
} catch (error) {
|
||||
logger.error({ error, errorMessage: (error as Error)?.message }, 'Strategy subagent failed');
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return Object.assign(tool, { streamFunc });
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { z } from 'zod';
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { WebExploreSubagent } from '../../harness/subagents/web-explore/index.js';
|
||||
import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
|
||||
import type { HarnessEvent } from '../../harness/harness-events.js';
|
||||
|
||||
export interface WebExploreAgentToolConfig {
|
||||
webExploreSubagent: WebExploreSubagent;
|
||||
@@ -14,10 +15,20 @@ export interface WebExploreAgentToolConfig {
|
||||
* Creates a LangChain tool that delegates to the web-explore subagent.
|
||||
* The subagent decides whether to use web search or arXiv based on the instruction.
|
||||
*/
|
||||
export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool {
|
||||
export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }, signal?: AbortSignal) => AsyncGenerator<HarnessEvent, string> } {
|
||||
const { webExploreSubagent, context, logger } = config;
|
||||
|
||||
return new DynamicStructuredTool({
|
||||
async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
|
||||
logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming web-explore subagent');
|
||||
const gen = webExploreSubagent.streamEvents(context, instruction, signal);
|
||||
let step: IteratorResult<HarnessEvent, string>;
|
||||
while (!(step = await gen.next()).done) {
|
||||
yield step.value;
|
||||
}
|
||||
return step.value;
|
||||
}
|
||||
|
||||
const tool = new DynamicStructuredTool({
|
||||
name: 'web_explore',
|
||||
description: `Search the web or academic databases and return a summarized answer.
|
||||
|
||||
@@ -46,4 +57,6 @@ The subagent will search the web (or arXiv for academic queries), fetch relevant
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return Object.assign(tool, { streamFunc });
|
||||
}
|
||||
|
||||
@@ -103,6 +103,16 @@ export const DEFAULT_STORES: StoreConfig[] = [
|
||||
persistent: true,
|
||||
initialState: () => ({}),
|
||||
},
|
||||
{
|
||||
name: 'strategy_types',
|
||||
persistent: true,
|
||||
initialState: () => ({}),
|
||||
},
|
||||
{
|
||||
name: 'research_types',
|
||||
persistent: true,
|
||||
initialState: () => ({}),
|
||||
},
|
||||
{
|
||||
name: 'channelState',
|
||||
persistent: false,
|
||||
|
||||
Reference in New Issue
Block a user