data pipeline refactor and fix

gateway/src/tools/platform/get-chart-data.tool.ts

@@ -28,23 +28,29 @@ export function createGetChartDataTool(config: GetChartDataToolConfig): DynamicS
 
 **IMPORTANT: Use this tool ONLY for quick, casual data viewing. For any analysis, plotting, statistics, or deep research, use the 'research' tool instead.**
 
+**Hard limit: returns at most 500 bars (the most recent 500). This tool is not suitable for analysis requiring longer sequences — use the 'research' tool for that.**
+
 Parameters:
-- ticker (optional): Market symbol (defaults to workspace chartState.symbol)
+- ticker (optional): Market symbol in SYMBOL.EXCHANGE format, e.g. "BTC/USDT.BINANCE" (defaults to workspace chartState.symbol)
 - period (optional): OHLC period in seconds (defaults to workspace chartState.period)
 - from_time (optional): Start time as Unix timestamp (number or string like "1774126800") OR date string like "2 days ago", "2024-01-01" (defaults to workspace chartState.start_time)
 - to_time (optional): End time as Unix timestamp (number or string like "1774732500") OR date string like "now", "yesterday" (defaults to workspace chartState.end_time)
-- countback (optional): Limit number of bars returned
+- countback (optional): Limit number of bars returned (max 500)
 - columns (optional): Extra columns beyond OHLC: ["volume", "buy_vol", "sell_vol", "open_time", "high_time", "low_time", "close_time", "open_interest"]`,
     schema: z.object({
       ticker: z.string().optional().describe('Market symbol (defaults to workspace chartState.symbol)'),
       period: z.number().optional().describe('OHLC period in seconds (defaults to workspace chartState.period)'),
       from_time: z.union([z.number(), z.string()]).optional().describe('Start time: Unix seconds OR date string (defaults to workspace chartState.start_time)'),
       to_time: z.union([z.number(), z.string()]).optional().describe('End time: Unix seconds OR date string (defaults to workspace chartState.end_time)'),
-      countback: z.number().optional().describe('Limit number of bars returned'),
+      countback: z.number().optional().describe('Limit number of bars returned (max 500)'),
       columns: z.array(z.enum(['volume', 'buy_vol', 'sell_vol', 'open_time', 'high_time', 'low_time', 'close_time', 'open_interest'])).optional().describe('Extra columns beyond OHLC'),
     }),
     func: async ({ ticker, period, from_time, to_time, countback, columns }) => {
-      logger.debug({ ticker, period, from_time, to_time, countback, columns }, 'Executing get_chart_data tool');
+      const MAX_BARS = 500;
+      // Enforce hard cap — never return more than MAX_BARS bars
+      const effectiveCountback = countback !== undefined ? Math.min(countback, MAX_BARS) : MAX_BARS;
+
+      logger.debug({ ticker, period, from_time, to_time, countback: effectiveCountback, columns }, 'Executing get_chart_data tool');
 
       try {
         // Get workspace chart state
@@ -86,7 +92,7 @@ Parameters:
           finalPeriod,
           finalFromTime,
           finalToTime,
-          countback
+          effectiveCountback
         );
 
         if (historyResult.noData || !historyResult.bars || historyResult.bars.length === 0) {
@@ -98,8 +104,13 @@ Parameters:
           });
         }
 
+        // Enforce hard cap — keep the most recent bars
+        const sourceBars = historyResult.bars.length > MAX_BARS
+          ? historyResult.bars.slice(-MAX_BARS)
+          : historyResult.bars;
+
         // Filter/format bars with requested columns
-        const bars = historyResult.bars.map(bar => {
+        const bars = sourceBars.map(bar => {
           const result: any = {
             time: bar.time,
             open: bar.open,

gateway/src/tools/platform/indicator-agent.tool.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { FastifyBaseLogger } from 'fastify';
 import type { IndicatorSubagent } from '../../harness/subagents/indicator/index.js';
 import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
+import type { HarnessEvent } from '../../harness/harness-events.js';
 
 export interface IndicatorAgentToolConfig {
   indicatorSubagent: IndicatorSubagent;
@@ -14,10 +15,20 @@ export interface IndicatorAgentToolConfig {
  * Creates a LangChain tool that delegates to the indicator subagent.
  * Mirrors the pattern of research-agent.tool.ts.
  */
-export function createIndicatorAgentTool(config: IndicatorAgentToolConfig): DynamicStructuredTool {
+export function createIndicatorAgentTool(config: IndicatorAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }) => AsyncGenerator<HarnessEvent, string> } {
   const { indicatorSubagent, context, logger } = config;
 
-  return new DynamicStructuredTool({
+  async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
+    logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming indicator subagent');
+    const gen = indicatorSubagent.streamEvents(context, instruction, signal);
+    let step: IteratorResult<HarnessEvent, string>;
+    while (!(step = await gen.next()).done) {
+      yield step.value;
+    }
+    return step.value;
+  }
+
+  const tool = new DynamicStructuredTool({
     name: 'indicator',
     description: `Delegate to the indicator subagent for all indicator-related tasks on the chart.
 
@@ -50,4 +61,6 @@ NEVER modify the indicators workspace store directly.`,
       }
     },
   });
+
+  return Object.assign(tool, { streamFunc });
 }

gateway/src/tools/platform/research-agent.tool.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { FastifyBaseLogger } from 'fastify';
 import type { ResearchSubagent } from '../../harness/subagents/research/index.js';
 import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
+import type { HarnessEvent } from '../../harness/harness-events.js';
 
 export interface ResearchAgentToolConfig {
   researchSubagent: ResearchSubagent;
@@ -15,10 +16,24 @@ export interface ResearchAgentToolConfig {
  * This is the standard LangChain pattern for exposing a subagent as a tool
  * to a parent agent.
  */
-export function createResearchAgentTool(config: ResearchAgentToolConfig): DynamicStructuredTool {
+export function createResearchAgentTool(config: ResearchAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { name: string; instruction: string }) => AsyncGenerator<HarnessEvent, string> } {
   const { researchSubagent, context, logger } = config;
 
-  return new DynamicStructuredTool({
+  const prompt = (name: string, instruction: string) => `Research script name: "${name}"\n\n${instruction}`;
+
+  async function* streamFunc({ name, instruction }: { name: string; instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
+    logger.info({ name, instruction: instruction.substring(0, 100) }, 'Streaming research subagent');
+    const gen = researchSubagent.streamEvents(context, prompt(name, instruction), signal);
+    let step: IteratorResult<HarnessEvent, string>;
+    while (!(step = await gen.next()).done) {
+      yield step.value;
+    }
+    const finalText = step.value;
+    const images = researchSubagent.getLastImages();
+    return JSON.stringify({ text: finalText, images });
+  }
+
+  const tool = new DynamicStructuredTool({
     name: 'research',
     description: `Delegate to the research subagent for data analysis, charting, statistics, and Python script execution.
 
@@ -36,21 +51,15 @@ The research subagent will write and execute Python scripts, capture output and
     func: async ({ name, instruction }: { name: string; instruction: string }): Promise<string> => {
       logger.info({ name, instruction: instruction.substring(0, 100) }, 'Delegating to research subagent');
 
-      const prompt = `Research script name: "${name}"\n\n${instruction}`;
-
       try {
-        const result = await researchSubagent.executeWithImages(context, prompt);
-
-        // Return in the format that AgentHarness.processToolResult() knows how to handle
-        // (extracts images and passes them to channelAdapter)
-        return JSON.stringify({
-          text: result.text,
-          images: result.images,
-        });
+        const result = await researchSubagent.executeWithImages(context, prompt(name, instruction));
+        return JSON.stringify({ text: result.text, images: result.images });
       } catch (error) {
         logger.error({ error, errorMessage: (error as Error)?.message }, 'Research subagent failed');
         throw error;
       }
     },
   });
+
+  return Object.assign(tool, { streamFunc });
 }

gateway/src/tools/platform/strategy-agent.tool.ts

@@ -0,0 +1,66 @@
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import { z } from 'zod';
+import type { FastifyBaseLogger } from 'fastify';
+import type { StrategySubagent } from '../../harness/subagents/strategy/index.js';
+import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
+import type { HarnessEvent } from '../../harness/harness-events.js';
+
+export interface StrategyAgentToolConfig {
+  strategySubagent: StrategySubagent;
+  context: SubagentContext;
+  logger: FastifyBaseLogger;
+}
+
+/**
+ * Creates a LangChain tool that delegates to the strategy subagent.
+ * Mirrors the pattern of indicator-agent.tool.ts.
+ */
+export function createStrategyAgentTool(config: StrategyAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }, signal?: AbortSignal) => AsyncGenerator<HarnessEvent, string> } {
+  const { strategySubagent, context, logger } = config;
+
+  async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
+    logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming strategy subagent');
+    const gen = strategySubagent.streamEvents(context, instruction, signal);
+    let step: IteratorResult<HarnessEvent, string>;
+    while (!(step = await gen.next()).done) {
+      yield step.value;
+    }
+    return step.value;
+  }
+
+  const tool = new DynamicStructuredTool({
+    name: 'strategy',
+    description: `Delegate to the strategy subagent for all trading strategy tasks.
+
+Use this tool for:
+- Writing new PandasStrategy classes ("create a strategy that...")
+- Editing or improving existing strategies
+- Running backtests on a strategy
+- Interpreting backtest results (Sharpe ratio, drawdown, trade list)
+- Activating or deactivating strategies for paper trading
+- Monitoring running strategy PnL and trade logs
+- Checking which strategies already exist
+
+ALWAYS use this tool for any request about trading strategies, backtesting, or strategy activation.
+NEVER write strategy Python code or call backtest_strategy directly — delegate here instead.`,
+    schema: z.object({
+      instruction: z.string().describe(
+        'The strategy task to perform. Be specific: include the strategy name, ' +
+        'desired signals (e.g. RSI < 30 = buy), timeframe, and symbol if known. ' +
+        'For backtest requests include the date range and starting capital.'
+      ),
+    }),
+    func: async ({ instruction }: { instruction: string }): Promise<string> => {
+      logger.info({ instruction: instruction.substring(0, 100) }, 'Delegating to strategy subagent');
+
+      try {
+        return await strategySubagent.execute(context, instruction);
+      } catch (error) {
+        logger.error({ error, errorMessage: (error as Error)?.message }, 'Strategy subagent failed');
+        throw error;
+      }
+    },
+  });
+
+  return Object.assign(tool, { streamFunc });
+}

gateway/src/tools/platform/web-explore-agent.tool.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { FastifyBaseLogger } from 'fastify';
 import type { WebExploreSubagent } from '../../harness/subagents/web-explore/index.js';
 import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
+import type { HarnessEvent } from '../../harness/harness-events.js';
 
 export interface WebExploreAgentToolConfig {
   webExploreSubagent: WebExploreSubagent;
@@ -14,10 +15,20 @@ export interface WebExploreAgentToolConfig {
  * Creates a LangChain tool that delegates to the web-explore subagent.
  * The subagent decides whether to use web search or arXiv based on the instruction.
  */
-export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool {
+export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }, signal?: AbortSignal) => AsyncGenerator<HarnessEvent, string> } {
   const { webExploreSubagent, context, logger } = config;
 
-  return new DynamicStructuredTool({
+  async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
+    logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming web-explore subagent');
+    const gen = webExploreSubagent.streamEvents(context, instruction, signal);
+    let step: IteratorResult<HarnessEvent, string>;
+    while (!(step = await gen.next()).done) {
+      yield step.value;
+    }
+    return step.value;
+  }
+
+  const tool = new DynamicStructuredTool({
     name: 'web_explore',
     description: `Search the web or academic databases and return a summarized answer.
 
@@ -46,4 +57,6 @@ The subagent will search the web (or arXiv for academic queries), fetch relevant
       }
     },
   });
+
+  return Object.assign(tool, { streamFunc });
 }