dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data timeout fixes; research agent improvements

Browse Source
This commit is contained in:
Tim Olson
2026-04-24 20:43:42 -04:00
parent 1800363566
commit 319d81c41f
37 changed files with 672 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
gateway/src/clients/duckdb-client.ts
View File

@@ -44,6 +44,7 @@ export class DuckDBClient {
private conversationsBucket?: string;
private logger: FastifyBaseLogger;
private initialized = false;
private initPromise: Promise<void> | null = null;
constructor(config: DuckDBConfig, logger: FastifyBaseLogger) {
this.logger = logger;
@@ -63,10 +64,16 @@ export class DuckDBClient {
* Initialize DuckDB connection and configure S3/Iceberg extensions
*/
async initialize(): Promise<void> {
if (this.initialized) {
return;
if (this.initialized) return;
if (!this.initPromise) {
this.initPromise = this._initialize().finally(() => {
this.initPromise = null;
});
}
await this.initPromise;
}
private async _initialize(): Promise<void> {
try {
this.db = new Database(':memory:');
this.conn = this.db.connect();
@@ -409,10 +416,12 @@ export class DuckDBClient {
// duplicate parquet files (e.g. from repeated Flink job runs on the same key
// range) never produce more than one row per (ticker, period_seconds, timestamp).
const sql = `
SELECT timestamp, ticker, period_seconds, open, high, low, close, volume
SELECT timestamp, ticker, period_seconds, open, high, low, close,
volume, buy_vol, sell_vol, open_time, close_time, num_trades, quote_volume
FROM (
SELECT
timestamp, ticker, period_seconds, open, high, low, close, volume, ingested_at,
timestamp, ticker, period_seconds, open, high, low, close,
volume, buy_vol, sell_vol, open_time, close_time, num_trades, quote_volume, ingested_at,
ROW_NUMBER() OVER (
PARTITION BY timestamp
ORDER BY ingested_at DESC

4
gateway/src/clients/zmq-relay-client.ts
View File

@@ -37,7 +37,7 @@ export interface ZMQRelayConfig {
relayRequestEndpoint: string; // e.g., "tcp://relay:5559"
relayNotificationEndpoint: string; // e.g., "tcp://relay:5558"
clientId?: string; // Optional client ID, will generate if not provided
requestTimeout?: number; // Request timeout in ms (default: 60000)
requestTimeout?: number; // Request timeout in ms (default: 120000)
onMetadataUpdate?: () => Promise<void>; // Callback when symbol metadata updates
}
@@ -77,7 +77,7 @@ export class ZMQRelayClient {
relayRequestEndpoint: config.relayRequestEndpoint,
relayNotificationEndpoint: config.relayNotificationEndpoint,
clientId: config.clientId || `gateway-${randomUUID().slice(0, 8)}`,
requestTimeout: config.requestTimeout || 60000,
requestTimeout: config.requestTimeout || 120000,
onMetadataUpdate: config.onMetadataUpdate || (async () => {}),
};
this.logger = logger;

52
gateway/src/harness/agent-harness.ts
View File

@@ -138,14 +138,29 @@ export class AgentHarness {
}
});
// Register the user-preferences virtual wiki page (loaded fresh each turn)
this.wikiLoader.registerVirtual('user-preferences', async (ctx) => {
// Register existing research scripts as a virtual wiki page
this.wikiLoader.registerVirtual('research-scripts', async (ctx) => {
if (!ctx.mcpClient) return '';
return this.fetchResearchScriptsSection(ctx.mcpClient);
});
this.registerFileDocument('user-preferences', 'PreferencesRead', 'User Preferences');
this.registerFileDocument('research-summary', 'ResearchSummaryRead', 'Research Summary');
}
/**
* Register a virtual wiki page backed by a sandbox file-document tool.
* The tool must return { content: string, exists: boolean }.
* Pages are loaded fresh every turn (never cached).
*/
private registerFileDocument(virtualName: string, toolName: string, heading: string): void {
this.wikiLoader.registerVirtual(virtualName, async (ctx) => {
if (!ctx.mcpClient) return '';
try {
const result = await ctx.mcpClient.callTool('PreferencesRead', {});
const result = await ctx.mcpClient.callTool(toolName, {});
const parsed = JSON.parse(String(result));
if (!parsed.exists || !parsed.content?.trim()) return '';
return `## User Preferences\n\n${parsed.content}`;
return `## ${heading}\n\n${parsed.content}`;
} catch {
return '';
}
@@ -235,6 +250,35 @@ export class AgentHarness {
}
}
/**
* Fetch existing research scripts and return a formatted markdown section.
* Used as the virtual wiki page 'research-scripts'.
*/
private async fetchResearchScriptsSection(mcpClient: MCPClientConnector): Promise<string> {
try {
const raw = await mcpClient.callTool('PythonList', { category: 'research' });
const r = raw as any;
const text = r?.content?.[0]?.text ?? r?.[0]?.text;
const parsed = typeof text === 'string' ? JSON.parse(text) : raw;
const items: any[] = parsed?.items ?? [];
if (items.length === 0) return '';
const lines: string[] = ['## Existing Research Scripts', ''];
lines.push('The following research scripts already exist. Before creating a new one, check whether any of these already covers the requested analysis.\n');
lines.push('| Name | Description |');
lines.push('|------|-------------|');
for (const item of items) {
const name: string = item.name ?? 'unknown';
const desc: string = item.description ?? '';
lines.push(`| ${name} | ${desc} |`);
}
return lines.join('\n');
} catch (err) {
this.config.logger.warn({ err }, 'Failed to fetch research scripts for wiki page');
return '';
}
}
/**
* Fetch custom indicators from the sandbox and return a formatted markdown section.
* Used as the virtual wiki page 'custom-indicators'.

59
gateway/src/harness/spawn/wiki-loader.ts
View File

@@ -1,5 +1,5 @@
import { readFile, readdir } from 'fs/promises';
import { join, dirname } from 'path';
import { join, dirname, relative } from 'path';
import { fileURLToPath } from 'url';
import yaml from 'js-yaml';
import type { MCPClientConnector } from '../mcp-client.js';
@@ -16,6 +16,7 @@ const KNOWLEDGE_DIR = join(__dirname, '..', '..', '..', 'knowledge');
const PROMPT_DIR = join(__dirname, '..', '..', '..', 'prompt');
export interface WikiFrontmatter {
description?: string;
maxTokens?: number;
recursionLimit?: number;
spawnsImages?: boolean;
@@ -157,36 +158,74 @@ export class WikiLoader {
}
/**
* Return the base prompt text: prompt/index.md body + prompt/tools.md body, concatenated.
* Return the base prompt text: prompt/index.md body + prompt/tools.md body + KB catalog.
* Result is cached for the lifetime of this WikiLoader instance.
*/
async getBasePrompt(): Promise<string> {
if (this.basePromptCache !== null) return this.basePromptCache;
const [index, tools] = await Promise.all([
const [index, tools, catalog] = await Promise.all([
this.loadPromptPage('index'),
this.loadPromptPage('tools'),
this.getKBCatalog(),
]);
const parts: string[] = [];
if (index) parts.push(index.body);
if (tools) parts.push(tools.body);
parts.push(catalog);
this.basePromptCache = parts.join('\n\n');
return this.basePromptCache;
}
/**
* List all .md file names available in the knowledge directory (without extension).
* Returns paths relative to KNOWLEDGE_DIR, e.g. "api-reference", "platform/workspace".
*/
async listPages(): Promise<string[]> {
try {
const files = await readdir(KNOWLEDGE_DIR);
return files
.filter(f => f.endsWith('.md'))
.map(f => f.slice(0, -3));
} catch {
return [];
const results: string[] = [];
const scan = async (dir: string): Promise<void> => {
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch {
return;
}
for (const entry of entries) {
const full = join(dir, entry.name);
if (entry.isDirectory()) {
await scan(full);
} else if (entry.isFile() && entry.name.endsWith('.md')) {
const rel = relative(KNOWLEDGE_DIR, full);
results.push(rel.slice(0, -3)); // strip .md
}
}
};
await scan(KNOWLEDGE_DIR);
return results.sort();
}
/**
* Build a markdown catalog table of all KB pages with their descriptions.
* Used by getBasePrompt() to inject into the cached system prompt.
*/
async getKBCatalog(): Promise<string> {
const pages = await this.listPages();
const rows: string[] = [];
for (const name of pages) {
const page = await this.loadPage(name);
const desc = page?.frontmatter.description ?? '';
rows.push(`| ${name} | ${desc} |`);
}
return [
'## Knowledge Base',
'',
'The following reference articles are available via `MemoryLookup({page: "..."})`. Read an article when you need detailed reference information.',
'',
'| Page | Description |',
'|------|-------------|',
...rows,
].join('\n');
}
/**

2
gateway/src/services/ohlc-service.ts
View File

@@ -277,7 +277,7 @@ export class OHLCService {
pricescale: 100,
has_intraday: true,
has_daily: true,
has_weekly_and_monthly: false,
has_weekly_and_monthly: true,
supported_resolutions: DEFAULT_SUPPORTED_RESOLUTIONS,
data_status: 'streaming',
};

19
gateway/src/services/symbol-index-service.ts
View File

@@ -25,6 +25,7 @@ export class SymbolIndexService {
private logger: FastifyBaseLogger;
private symbols: Map<string, SymbolMetadata> = new Map(); // key: "MARKET_ID.EXCHANGE" (Nautilus format)
private initialized: boolean = false;
private initPromise: Promise<void> | null = null;
constructor(config: SymbolIndexServiceConfig) {
this.icebergClient = config.icebergClient;
@@ -57,7 +58,9 @@ export class SymbolIndexService {
this.symbols.set(key, symbol);
}
this.initialized = true;
if (symbols.length > 0) {
this.initialized = true;
}
this.logger.info({
count: this.symbols.size,
totalRows: symbols.length,
@@ -74,12 +77,14 @@ export class SymbolIndexService {
* Ensure index is initialized (with retry on failure)
*/
private async ensureInitialized(): Promise<void> {
if (this.initialized) {
return;
if (this.initialized) return;
if (!this.initPromise) {
this.logger.info('Lazy-loading symbol index');
this.initPromise = this.initialize().finally(() => {
this.initPromise = null;
});
}
this.logger.info('Lazy-loading symbol index');
await this.initialize();
await this.initPromise;
}
/**
@@ -213,7 +218,7 @@ export class SymbolIndexService {
supported_resolutions: supportedResolutions.length > 0 ? supportedResolutions : DEFAULT_SUPPORTED_RESOLUTIONS,
has_intraday: true,
has_daily: true,
has_weekly_and_monthly: false,
has_weekly_and_monthly: true,
pricescale,
minmov: 1,
base_currency: metadata.base_asset,

22
gateway/src/tools/mcp/mcp-tool-wrapper.ts
View File

@@ -125,6 +125,14 @@ export function createMCPToolWrapper(
});
}
/** Silently parse a JSON string; pass non-strings through unchanged. */
function tryParseJson(val: unknown): unknown {
if (typeof val === 'string') {
try { return JSON.parse(val); } catch { /* fall through */ }
}
return val;
}
/**
* Convert MCP input schema to Zod schema
*/
@@ -156,17 +164,13 @@ function mcpInputSchemaToZod(inputSchema?: MCPToolInfo['inputSchema']): z.ZodObj
case 'boolean':
zodType = z.boolean().describe(prop.description || '');
break;
case 'array':
// Handle array items
if (prop.items) {
const itemType = getZodTypeForProperty(prop.items);
zodType = z.array(itemType).describe(prop.description || '');
} else {
zodType = z.array(z.any()).describe(prop.description || '');
}
case 'array': {
const itemType = prop.items ? getZodTypeForProperty(prop.items) : z.any();
zodType = z.preprocess(tryParseJson, z.array(itemType)).describe(prop.description || '');
break;
}
case 'object':
zodType = z.object({}).passthrough().describe(prop.description || '');
zodType = z.preprocess(tryParseJson, z.object({}).passthrough()).describe(prop.description || '');
break;
default:
zodType = z.any().describe(prop.description || '');

5
gateway/src/tools/platform/get-chart-data.tool.ts
View File

@@ -43,7 +43,10 @@ Parameters:
from_time: z.union([z.number(), z.string()]).optional().describe('Start time: Unix seconds OR date string (defaults to workspace chartState.start_time)'),
to_time: z.union([z.number(), z.string()]).optional().describe('End time: Unix seconds OR date string (defaults to workspace chartState.end_time)'),
countback: z.number().optional().describe('Limit number of bars returned (max 500)'),
columns: z.array(z.enum(['volume', 'buy_vol', 'sell_vol', 'open_time', 'high_time', 'low_time', 'close_time', 'open_interest'])).optional().describe('Extra columns beyond OHLC'),
columns: z.preprocess(
(val) => { if (typeof val === 'string') { try { return JSON.parse(val); } catch { /* fall through */ } } return val; },
z.array(z.enum(['volume', 'buy_vol', 'sell_vol', 'open_time', 'high_time', 'low_time', 'close_time', 'open_interest'])).optional()
).describe('Extra columns beyond OHLC'),
}),
func: async ({ ticker, period, from_time, to_time, countback, columns }) => {
const MAX_BARS = 500;

19
gateway/src/tools/platform/memory-lookup.tool.ts
View File

@@ -19,31 +19,18 @@ export function createMemoryLookupTool(config: MemoryLookupToolConfig): DynamicS
return new DynamicStructuredTool({
name: 'MemoryLookup',
description: `Read a knowledge wiki page by name to get detailed reference information.
Pass "index" to list all available pages.
Example pages:
- "api-reference" — DataAPI and ChartingAPI reference for research scripts
- "usage-examples" — Example research scripts
- "pandas-ta-reference" — Full pandas-ta indicator catalog`,
description: `Read a knowledge wiki page by name to get detailed reference information. Available pages are listed in the Knowledge Base section of your system prompt.`,
schema: z.object({
page: z.string().describe(
'Wiki page name to read (without .md extension). Pass "index" to list all pages.'
'Wiki page name to read (without .md extension), e.g. "api-reference" or "platform/workspace".'
),
}),
func: async ({ page }: { page: string }): Promise<string> => {
logger.info({ page }, 'memory_lookup: reading page');
if (page === 'index') {
const pages = await wikiLoader.listPages();
return `Available wiki pages:\n${pages.map(p => `- ${p}`).join('\n')}`;
}
const wikiPage = await wikiLoader.loadPage(page);
if (!wikiPage) {
const pages = await wikiLoader.listPages();
return `Page "${page}" not found.\n\nAvailable pages:\n${pages.map(p => `- ${p}`).join('\n')}`;
return `Page "${page}" not found. Refer to the Knowledge Base table in your system prompt for available page names.`;
}
return wikiPage.body;