data pipeline refactor and fix

gateway/src/tools/platform/web-explore-agent.tool.ts

@@ -3,6 +3,7 @@ import { z } from 'zod';
 import type { FastifyBaseLogger } from 'fastify';
 import type { WebExploreSubagent } from '../../harness/subagents/web-explore/index.js';
 import type { SubagentContext } from '../../harness/subagents/base-subagent.js';
+import type { HarnessEvent } from '../../harness/harness-events.js';
 
 export interface WebExploreAgentToolConfig {
   webExploreSubagent: WebExploreSubagent;
@@ -14,10 +15,20 @@ export interface WebExploreAgentToolConfig {
  * Creates a LangChain tool that delegates to the web-explore subagent.
  * The subagent decides whether to use web search or arXiv based on the instruction.
  */
-export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool {
+export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): DynamicStructuredTool & { streamFunc: (args: { instruction: string }, signal?: AbortSignal) => AsyncGenerator<HarnessEvent, string> } {
   const { webExploreSubagent, context, logger } = config;
 
-  return new DynamicStructuredTool({
+  async function* streamFunc({ instruction }: { instruction: string }, signal?: AbortSignal): AsyncGenerator<HarnessEvent, string> {
+    logger.info({ instruction: instruction.substring(0, 100) }, 'Streaming web-explore subagent');
+    const gen = webExploreSubagent.streamEvents(context, instruction, signal);
+    let step: IteratorResult<HarnessEvent, string>;
+    while (!(step = await gen.next()).done) {
+      yield step.value;
+    }
+    return step.value;
+  }
+
+  const tool = new DynamicStructuredTool({
     name: 'web_explore',
     description: `Search the web or academic databases and return a summarized answer.
 
@@ -46,4 +57,6 @@ The subagent will search the web (or arXiv for academic queries), fetch relevant
       }
     },
   });
+
+  return Object.assign(tool, { streamFunc });
 }