data timeout fixes; research agent improvements
This commit is contained in:
@@ -17,14 +17,21 @@ Documents should be in Markdown format with:
|
||||
- Code examples where relevant
|
||||
- Cross-references to other docs
|
||||
|
||||
### Frontmatter Fields
|
||||
|
||||
`description` (required) — One or two sentences describing what the article covers. This is injected into every agent's system prompt as a KB catalog so agents know what to look up without making an extra tool call.
|
||||
|
||||
`tags` (optional) — List of topic tags for categorization.
|
||||
|
||||
### Example with Frontmatter
|
||||
|
||||
```markdown
|
||||
---
|
||||
tags: [trading, risk-management, position-sizing]
|
||||
description: "Patterns for writing custom Python indicator scripts that compute values from OHLCV data and plot live on the chart."
|
||||
tags: [indicators, python, development]
|
||||
---
|
||||
|
||||
# Risk Management
|
||||
# Custom Indicator Development
|
||||
|
||||
Content here...
|
||||
```
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "Complete Python API reference (DataAPI and ChartingAPI) with full source and docstrings for use in research scripts."
|
||||
---
|
||||
|
||||
# Dexorder Research API Reference
|
||||
|
||||
This file contains the complete Python API source code with full docstrings.
|
||||
@@ -124,11 +128,14 @@ class DataAPI(ABC):
|
||||
- pandas Timestamp: pd.Timestamp("2021-12-20")
|
||||
end_time: End of time range. Same formats as start_time.
|
||||
extra_columns: Optional additional columns to include beyond the standard
|
||||
OHLC columns. Available options:
|
||||
- "volume" - Total volume (decimal float)
|
||||
- "buy_vol" - Buy-side volume (decimal float)
|
||||
- "sell_vol" - Sell-side volume (decimal float)
|
||||
- "open_time", "high_time", "low_time", "close_time" (timestamps)
|
||||
OHLC columns. Available options (all populated for Binance data):
|
||||
- "volume" - Total base-asset volume (decimal float)
|
||||
- "buy_vol" - Taker buy volume in base asset (decimal float)
|
||||
- "sell_vol" - Taker sell volume in base asset (decimal float)
|
||||
- "quote_volume" - Total quote-asset volume, e.g. USDT (decimal float)
|
||||
- "num_trades" - Number of trades in the candle (integer)
|
||||
- "open_time", "close_time" (nanosecond timestamps; Binance only)
|
||||
- "high_time", "low_time" (not provided by any exchange; always null)
|
||||
- "open_interest" (for futures markets)
|
||||
- "ticker", "period_seconds"
|
||||
|
||||
@@ -201,8 +208,8 @@ class DataAPI(ABC):
|
||||
length: Number of most recent candles to return (default: 1)
|
||||
extra_columns: Optional list of additional column names to include.
|
||||
Same column options as historical_ohlc:
|
||||
- "volume", "buy_vol", "sell_vol"
|
||||
- "open_time", "high_time", "low_time", "close_time"
|
||||
- "volume", "buy_vol", "sell_vol", "quote_volume", "num_trades"
|
||||
- "open_time", "close_time", "high_time", "low_time"
|
||||
- "open_interest", "ticker", "period_seconds"
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "API and patterns for writing custom Python indicator scripts that compute values from OHLCV data and plot live on the chart."
|
||||
---
|
||||
|
||||
# Custom Indicator Development
|
||||
|
||||
Custom indicators are Python scripts saved in the `indicator` category. They compute values from OHLCV data and are plotted live on the TradingView chart alongside built-in indicators.
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "Full catalog of technical indicators available via pandas-ta, with parameters and usage for research scripts and custom indicators."
|
||||
---
|
||||
|
||||
# pandas-ta Reference for Research Scripts
|
||||
|
||||
This catalog applies to both research scripts and custom indicators. For usage in research scripts see [`usage-examples.md`](usage-examples.md). For writing custom indicator scripts (with metadata for the TradingView plotter) see [`indicators/indicator-development.md`](indicators/indicator-development.md).
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "User sandbox lifecycle, persistent script storage categories, and session management for indicator, strategy, and research scripts."
|
||||
---
|
||||
|
||||
# User Sandbox
|
||||
|
||||
Each user has a dedicated sandbox environment that persists their data across sessions.
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "Workspace store schema, available stores, and WorkspaceRead/WorkspacePatch usage for reading and updating the user's UI state."
|
||||
---
|
||||
|
||||
# Workspace
|
||||
|
||||
The Workspace is the user's current UI context — what they are looking at, what is selected, and what persistent state belongs to their session. It is a collection of named **stores** that are kept in sync between the web client, the gateway, and the user's sandbox container.
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "PandasStrategy class API, order placement, backtesting, and paper trading patterns for automated crypto strategy development."
|
||||
---
|
||||
|
||||
# Strategy Development Guide
|
||||
|
||||
Strategies on Dexorder are `PandasStrategy` subclasses that receive a live stream of OHLCV bars and call `self.buy()` / `self.sell()` / `self.flatten()` to place orders.
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "Technical analysis fundamentals covering chart patterns, trend analysis, and TA concepts applicable to crypto OHLCV data."
|
||||
---
|
||||
|
||||
# Technical Analysis Fundamentals
|
||||
|
||||
Technical analysis is the study of historical price and volume data to identify patterns and predict future market movements.
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
---
|
||||
description: "Example research scripts showing common patterns for data retrieval, statistical analysis, and visualization."
|
||||
---
|
||||
|
||||
# Research Script API Usage
|
||||
|
||||
See [`api-reference.md`](api-reference.md) for the full DataAPI and ChartingAPI source with all method signatures and docstrings. See [`pandas-ta-reference.md`](pandas-ta-reference.md) for the indicator catalog.
|
||||
@@ -75,11 +79,16 @@ print(df.head())
|
||||
|
||||
### Available Extra Columns
|
||||
|
||||
- `"volume"` - Total volume
|
||||
- `"buy_vol"` - Buy-side volume
|
||||
- `"sell_vol"` - Sell-side volume
|
||||
- `"open_time"`, `"high_time"`, `"low_time"`, `"close_time"` - Timestamps for each price point
|
||||
- `"open_interest"` - Open interest (for futures)
|
||||
All columns below are fully populated for Binance data. Other exchanges provide only `volume`.
|
||||
|
||||
- `"volume"` - Total base-asset volume
|
||||
- `"buy_vol"` - Taker buy volume (base asset) — order flow long pressure
|
||||
- `"sell_vol"` - Taker sell volume (base asset) — order flow short pressure
|
||||
- `"quote_volume"` - Total quote-asset volume (e.g. USDT traded)
|
||||
- `"num_trades"` - Number of individual trades in the candle
|
||||
- `"open_time"`, `"close_time"` - Exact candle open/close timestamps (Binance only)
|
||||
- `"high_time"`, `"low_time"` - Not provided by any exchange (always null)
|
||||
- `"open_interest"` - Open interest (futures only)
|
||||
- `"ticker"` - Market identifier
|
||||
- `"period_seconds"` - Period in seconds
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
---
|
||||
dynamic_imports:
|
||||
- user-preferences
|
||||
- research-summary
|
||||
- research-scripts
|
||||
---
|
||||
|
||||
# Main Agent Instructions
|
||||
@@ -60,7 +62,9 @@ Use research for exploratory or one-off analysis. Use indicator whenever the use
|
||||
|
||||
## Pre-delegation Checks
|
||||
|
||||
Before calling research, call `PythonList(category="research")` to check if a relevant script already exists. If it does, pass its name to the research instruction so the agent updates it rather than creating a duplicate.
|
||||
Before calling research, check the **Existing Research Scripts** list above. If a relevant script already exists, pass its exact name to the research instruction so the agent updates it rather than creating a duplicate.
|
||||
|
||||
**Iterating on an idea across turns**: When the user refines, tweaks, or asks follow-up questions about an analysis already performed this session (e.g. "now do it with a 30-day window", "can you add a volume subplot", "try with ETH instead"), pass the **same script name** as before in the research instruction. The agent will update the existing script in place. Old versions are preserved in git history and do not need to be kept as separate scripts.
|
||||
|
||||
Before calling strategy, call `PythonList(category="strategy")` similarly.
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ static_imports:
|
||||
dynamic_imports:
|
||||
- conda-environment
|
||||
- custom-indicators
|
||||
- research-scripts
|
||||
---
|
||||
# Research Script Assistant
|
||||
|
||||
@@ -22,6 +23,22 @@ Create Python scripts that:
|
||||
- Generate professional charts using matplotlib via the ChartingAPI
|
||||
- All matplotlib figures are automatically captured and sent to the user as images
|
||||
|
||||
## Exploratory Mindset
|
||||
|
||||
Go beyond the literal request. The user's question is a starting point, not a ceiling. Adjacent analysis — things the user didn't ask for but that naturally illuminate the same topic — often produces the most valuable insights and can reframe or deepen the interpretation of the original result.
|
||||
|
||||
**Always ask**: *What else is related to this that would be worth knowing?* Then include it.
|
||||
|
||||
If the user asks about Monday morning opening price trends, also plot order flow imbalance, session volatility, and volume — these directly affect how the price trend should be interpreted. If the user asks about RSI divergences, also show the distribution of returns following each divergence type. If asked about a specific symbol's correlation with BTC, also show correlation stability over time and during high-volatility regimes.
|
||||
|
||||
Concretely:
|
||||
- **Add subplots** for related metrics (volume, volatility, spread, order flow) alongside the primary chart
|
||||
- **Include summary statistics** the user didn't ask for but that contextualize the result (e.g. sample size, statistical significance, base rates, regime breakdowns)
|
||||
- **Surface anomalies or surprises** you notice in the data, even if tangential
|
||||
- **Stratify results** by relevant dimensions (time of day, day of week, bull/bear regime, high/low volatility) when the sample is large enough
|
||||
|
||||
Keep it focused — adjacent analysis should feel like natural extensions of the same question, not a data dump. Two or three well-chosen additions are better than ten loosely related ones.
|
||||
|
||||
## Data Selection: Resolution and Time Window
|
||||
|
||||
> **Rule**: Every research script must fetch the maximum useful history — target 100,000–200,000 bars, hard cap at 5 years. **Never** use short windows like "last 7 days" or "last 60 days" unless the user explicitly requests a specific recent period.
|
||||
@@ -49,36 +66,11 @@ Quick reference — approximate bars per resolution at various windows:
|
||||
|
||||
**When to shorten the window**: only if 5 years at the chosen resolution would far exceed 200,000 bars (e.g., 5m over 5 years ≈ 525k → shorten to ~2 years). Otherwise always use the full 5 years.
|
||||
|
||||
## Available Tools
|
||||
## Tool Behavior Notes
|
||||
|
||||
You have direct access to these MCP tools:
|
||||
|
||||
- **PythonWrite**: Create a new script (research, strategy, or indicator category)
|
||||
- Required: category, name, description, details, code
|
||||
- Optional: metadata (category-specific fields — see below)
|
||||
- **For research**: fully executes the script and returns all output (stdout, stderr) and captured chart images. The response IS the execution result — **do not call `ExecuteResearch` afterward**.
|
||||
- **For indicator/strategy**: runs against synthetic test data to catch compile/runtime errors; no chart images are generated.
|
||||
- Returns validation results and execution output (text + images for research)
|
||||
|
||||
- **PythonEdit**: Update an existing script
|
||||
- Required: category, name
|
||||
- Optional: code, patches, description, details (full replacement), detail_patches (targeted text replacements in details), metadata
|
||||
- **For research**: re-executes the script when code is changed and returns all output and images. **Do not call `ExecuteResearch` afterward**.
|
||||
- **For indicator/strategy**: re-runs the validation test only.
|
||||
- Returns validation results and execution output
|
||||
|
||||
- **PythonRead**: Read an existing research script
|
||||
- Returns: code, metadata
|
||||
|
||||
- **PythonList**: List all research scripts
|
||||
- Returns: array of {name, description, metadata}
|
||||
|
||||
- **ExecuteResearch**: Run a research script that already exists on disk
|
||||
- Use this **only** when the user explicitly asks to re-run a script, or to run a script that was written in a previous session and already exists
|
||||
- **Do not call this after `PythonWrite` or `PythonEdit`** — those tools already executed the script and returned its output
|
||||
- Returns: text output and images
|
||||
|
||||
- **WebSearch**, **FetchPage**, **ArxivSearch**: Search the web or fetch pages for reference information when researching methodologies or indicators
|
||||
- **`PythonWrite` / `PythonEdit` for research**: auto-executes the script and returns all output (stdout, stderr) and captured images. **Do not call `ExecuteResearch` afterward** — the script has already run.
|
||||
- **`PythonWrite` / `PythonEdit` for indicator/strategy**: runs against synthetic test data only; no chart images are generated.
|
||||
- **`ExecuteResearch`**: use **only** when the user explicitly asks to re-run a script, or to run one written in a previous session. Never call it after `PythonWrite` or `PythonEdit`.
|
||||
|
||||
## Research Script API
|
||||
|
||||
@@ -109,6 +101,10 @@ When a user requests analysis:
|
||||
|
||||
2. **Use the provided name**: The instruction will begin with `Research script name: "<name>"`. Always use that exact name when calling `PythonWrite` or `PythonEdit`. Check first with `PythonRead` — if the script already exists, use `PythonEdit` to update it rather than creating a new one with `PythonWrite`.
|
||||
|
||||
**One script per analysis idea**: If the name matches an existing script, the user is iterating on that idea — update it in place rather than creating a variant with a different name. Old versions are preserved in git history; there is no need to keep multiple scripts for variations of the same analysis.
|
||||
|
||||
**Duplicate detection**: Also review the **Existing Research Scripts** list above. If a script already exists there that appears to cover the same analysis as your current instruction — even under a different name — note this in your response after completing the task, so the user can decide whether to consolidate.
|
||||
|
||||
3. **Write the script**: Use `PythonWrite` (new) or `PythonEdit` (existing)
|
||||
- Write clean, well-commented Python code
|
||||
- Include proper error handling
|
||||
@@ -127,7 +123,17 @@ When a user requests analysis:
|
||||
- Use `PythonEdit` to fix the script
|
||||
- The script will auto-execute again
|
||||
|
||||
6. **Return results**: Once successful, summarize what was done
|
||||
6. **Summarize findings**: After successful execution, update the research summary entry
|
||||
using `ResearchSummaryPatch`:
|
||||
- Replace the `**Findings:**` line(s) with 3–5 concise bullet points of key results
|
||||
- Include only **statistically significant or practically notable** findings —
|
||||
p-values, effect sizes, actionable patterns
|
||||
- If nothing notable emerged: a single bullet `No significant patterns found`
|
||||
- Keep the entire findings block under ~100 words; full output is always readable via
|
||||
`PythonReadOutput(category="research", name="<script-name>")`
|
||||
- This applies after `PythonWrite`, `PythonEdit`, and `ExecuteResearch` runs
|
||||
|
||||
7. **Return results**: Once successful, summarize what was done
|
||||
- The user will receive both your text response AND the chart images
|
||||
- Don't try to describe the images in detail - the user can see them
|
||||
|
||||
|
||||
@@ -6,14 +6,6 @@ recursionLimit: 15
|
||||
|
||||
You are a research assistant that searches the web and academic databases to answer questions or gather information according to the given instructions.
|
||||
|
||||
## Tools
|
||||
|
||||
You have three tools:
|
||||
|
||||
- **`WebSearch`** — Search the web broadly (Tavily). Returns titles, URLs, and content summaries. Best for general information, news, documentation, proprietary/niche topics, trading indicators, software papers, and anything not likely to be on arXiv.
|
||||
- **`ArxivSearch`** — Search arXiv for academic preprints. Returns titles, authors, abstracts, and PDF links. Use this **only** for peer-reviewed or academic research (e.g. machine learning, statistics, finance theory). Most trading indicators, technical analysis tools, and proprietary methods are NOT on arXiv.
|
||||
- **`FetchPage`** — Fetch the full content of a URL (web page or PDF). PDFs are automatically converted to text. Use this after searching to read the complete content of a promising result.
|
||||
|
||||
## Strategy
|
||||
|
||||
1. **Choose the right search tool first:**
|
||||
|
||||
@@ -44,6 +44,7 @@ export class DuckDBClient {
|
||||
private conversationsBucket?: string;
|
||||
private logger: FastifyBaseLogger;
|
||||
private initialized = false;
|
||||
private initPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(config: DuckDBConfig, logger: FastifyBaseLogger) {
|
||||
this.logger = logger;
|
||||
@@ -63,10 +64,16 @@ export class DuckDBClient {
|
||||
* Initialize DuckDB connection and configure S3/Iceberg extensions
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
if (this.initialized) return;
|
||||
if (!this.initPromise) {
|
||||
this.initPromise = this._initialize().finally(() => {
|
||||
this.initPromise = null;
|
||||
});
|
||||
}
|
||||
await this.initPromise;
|
||||
}
|
||||
|
||||
private async _initialize(): Promise<void> {
|
||||
try {
|
||||
this.db = new Database(':memory:');
|
||||
this.conn = this.db.connect();
|
||||
@@ -409,10 +416,12 @@ export class DuckDBClient {
|
||||
// duplicate parquet files (e.g. from repeated Flink job runs on the same key
|
||||
// range) never produce more than one row per (ticker, period_seconds, timestamp).
|
||||
const sql = `
|
||||
SELECT timestamp, ticker, period_seconds, open, high, low, close, volume
|
||||
SELECT timestamp, ticker, period_seconds, open, high, low, close,
|
||||
volume, buy_vol, sell_vol, open_time, close_time, num_trades, quote_volume
|
||||
FROM (
|
||||
SELECT
|
||||
timestamp, ticker, period_seconds, open, high, low, close, volume, ingested_at,
|
||||
timestamp, ticker, period_seconds, open, high, low, close,
|
||||
volume, buy_vol, sell_vol, open_time, close_time, num_trades, quote_volume, ingested_at,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY timestamp
|
||||
ORDER BY ingested_at DESC
|
||||
|
||||
@@ -37,7 +37,7 @@ export interface ZMQRelayConfig {
|
||||
relayRequestEndpoint: string; // e.g., "tcp://relay:5559"
|
||||
relayNotificationEndpoint: string; // e.g., "tcp://relay:5558"
|
||||
clientId?: string; // Optional client ID, will generate if not provided
|
||||
requestTimeout?: number; // Request timeout in ms (default: 60000)
|
||||
requestTimeout?: number; // Request timeout in ms (default: 120000)
|
||||
onMetadataUpdate?: () => Promise<void>; // Callback when symbol metadata updates
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ export class ZMQRelayClient {
|
||||
relayRequestEndpoint: config.relayRequestEndpoint,
|
||||
relayNotificationEndpoint: config.relayNotificationEndpoint,
|
||||
clientId: config.clientId || `gateway-${randomUUID().slice(0, 8)}`,
|
||||
requestTimeout: config.requestTimeout || 60000,
|
||||
requestTimeout: config.requestTimeout || 120000,
|
||||
onMetadataUpdate: config.onMetadataUpdate || (async () => {}),
|
||||
};
|
||||
this.logger = logger;
|
||||
|
||||
@@ -138,14 +138,29 @@ export class AgentHarness {
|
||||
}
|
||||
});
|
||||
|
||||
// Register the user-preferences virtual wiki page (loaded fresh each turn)
|
||||
this.wikiLoader.registerVirtual('user-preferences', async (ctx) => {
|
||||
// Register existing research scripts as a virtual wiki page
|
||||
this.wikiLoader.registerVirtual('research-scripts', async (ctx) => {
|
||||
if (!ctx.mcpClient) return '';
|
||||
return this.fetchResearchScriptsSection(ctx.mcpClient);
|
||||
});
|
||||
|
||||
this.registerFileDocument('user-preferences', 'PreferencesRead', 'User Preferences');
|
||||
this.registerFileDocument('research-summary', 'ResearchSummaryRead', 'Research Summary');
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a virtual wiki page backed by a sandbox file-document tool.
|
||||
* The tool must return { content: string, exists: boolean }.
|
||||
* Pages are loaded fresh every turn (never cached).
|
||||
*/
|
||||
private registerFileDocument(virtualName: string, toolName: string, heading: string): void {
|
||||
this.wikiLoader.registerVirtual(virtualName, async (ctx) => {
|
||||
if (!ctx.mcpClient) return '';
|
||||
try {
|
||||
const result = await ctx.mcpClient.callTool('PreferencesRead', {});
|
||||
const result = await ctx.mcpClient.callTool(toolName, {});
|
||||
const parsed = JSON.parse(String(result));
|
||||
if (!parsed.exists || !parsed.content?.trim()) return '';
|
||||
return `## User Preferences\n\n${parsed.content}`;
|
||||
return `## ${heading}\n\n${parsed.content}`;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
@@ -235,6 +250,35 @@ export class AgentHarness {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch existing research scripts and return a formatted markdown section.
|
||||
* Used as the virtual wiki page 'research-scripts'.
|
||||
*/
|
||||
private async fetchResearchScriptsSection(mcpClient: MCPClientConnector): Promise<string> {
|
||||
try {
|
||||
const raw = await mcpClient.callTool('PythonList', { category: 'research' });
|
||||
const r = raw as any;
|
||||
const text = r?.content?.[0]?.text ?? r?.[0]?.text;
|
||||
const parsed = typeof text === 'string' ? JSON.parse(text) : raw;
|
||||
const items: any[] = parsed?.items ?? [];
|
||||
if (items.length === 0) return '';
|
||||
|
||||
const lines: string[] = ['## Existing Research Scripts', ''];
|
||||
lines.push('The following research scripts already exist. Before creating a new one, check whether any of these already covers the requested analysis.\n');
|
||||
lines.push('| Name | Description |');
|
||||
lines.push('|------|-------------|');
|
||||
for (const item of items) {
|
||||
const name: string = item.name ?? 'unknown';
|
||||
const desc: string = item.description ?? '';
|
||||
lines.push(`| ${name} | ${desc} |`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
} catch (err) {
|
||||
this.config.logger.warn({ err }, 'Failed to fetch research scripts for wiki page');
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch custom indicators from the sandbox and return a formatted markdown section.
|
||||
* Used as the virtual wiki page 'custom-indicators'.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { readFile, readdir } from 'fs/promises';
|
||||
import { join, dirname } from 'path';
|
||||
import { join, dirname, relative } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import yaml from 'js-yaml';
|
||||
import type { MCPClientConnector } from '../mcp-client.js';
|
||||
@@ -16,6 +16,7 @@ const KNOWLEDGE_DIR = join(__dirname, '..', '..', '..', 'knowledge');
|
||||
const PROMPT_DIR = join(__dirname, '..', '..', '..', 'prompt');
|
||||
|
||||
export interface WikiFrontmatter {
|
||||
description?: string;
|
||||
maxTokens?: number;
|
||||
recursionLimit?: number;
|
||||
spawnsImages?: boolean;
|
||||
@@ -157,36 +158,74 @@ export class WikiLoader {
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the base prompt text: prompt/index.md body + prompt/tools.md body, concatenated.
|
||||
* Return the base prompt text: prompt/index.md body + prompt/tools.md body + KB catalog.
|
||||
* Result is cached for the lifetime of this WikiLoader instance.
|
||||
*/
|
||||
async getBasePrompt(): Promise<string> {
|
||||
if (this.basePromptCache !== null) return this.basePromptCache;
|
||||
|
||||
const [index, tools] = await Promise.all([
|
||||
const [index, tools, catalog] = await Promise.all([
|
||||
this.loadPromptPage('index'),
|
||||
this.loadPromptPage('tools'),
|
||||
this.getKBCatalog(),
|
||||
]);
|
||||
|
||||
const parts: string[] = [];
|
||||
if (index) parts.push(index.body);
|
||||
if (tools) parts.push(tools.body);
|
||||
parts.push(catalog);
|
||||
this.basePromptCache = parts.join('\n\n');
|
||||
return this.basePromptCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all .md file names available in the knowledge directory (without extension).
|
||||
* Returns paths relative to KNOWLEDGE_DIR, e.g. "api-reference", "platform/workspace".
|
||||
*/
|
||||
async listPages(): Promise<string[]> {
|
||||
try {
|
||||
const files = await readdir(KNOWLEDGE_DIR);
|
||||
return files
|
||||
.filter(f => f.endsWith('.md'))
|
||||
.map(f => f.slice(0, -3));
|
||||
} catch {
|
||||
return [];
|
||||
const results: string[] = [];
|
||||
const scan = async (dir: string): Promise<void> => {
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const full = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await scan(full);
|
||||
} else if (entry.isFile() && entry.name.endsWith('.md')) {
|
||||
const rel = relative(KNOWLEDGE_DIR, full);
|
||||
results.push(rel.slice(0, -3)); // strip .md
|
||||
}
|
||||
}
|
||||
};
|
||||
await scan(KNOWLEDGE_DIR);
|
||||
return results.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a markdown catalog table of all KB pages with their descriptions.
|
||||
* Used by getBasePrompt() to inject into the cached system prompt.
|
||||
*/
|
||||
async getKBCatalog(): Promise<string> {
|
||||
const pages = await this.listPages();
|
||||
const rows: string[] = [];
|
||||
for (const name of pages) {
|
||||
const page = await this.loadPage(name);
|
||||
const desc = page?.frontmatter.description ?? '';
|
||||
rows.push(`| ${name} | ${desc} |`);
|
||||
}
|
||||
return [
|
||||
'## Knowledge Base',
|
||||
'',
|
||||
'The following reference articles are available via `MemoryLookup({page: "..."})`. Read an article when you need detailed reference information.',
|
||||
'',
|
||||
'| Page | Description |',
|
||||
'|------|-------------|',
|
||||
...rows,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -277,7 +277,7 @@ export class OHLCService {
|
||||
pricescale: 100,
|
||||
has_intraday: true,
|
||||
has_daily: true,
|
||||
has_weekly_and_monthly: false,
|
||||
has_weekly_and_monthly: true,
|
||||
supported_resolutions: DEFAULT_SUPPORTED_RESOLUTIONS,
|
||||
data_status: 'streaming',
|
||||
};
|
||||
|
||||
@@ -25,6 +25,7 @@ export class SymbolIndexService {
|
||||
private logger: FastifyBaseLogger;
|
||||
private symbols: Map<string, SymbolMetadata> = new Map(); // key: "MARKET_ID.EXCHANGE" (Nautilus format)
|
||||
private initialized: boolean = false;
|
||||
private initPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(config: SymbolIndexServiceConfig) {
|
||||
this.icebergClient = config.icebergClient;
|
||||
@@ -57,7 +58,9 @@ export class SymbolIndexService {
|
||||
this.symbols.set(key, symbol);
|
||||
}
|
||||
|
||||
this.initialized = true;
|
||||
if (symbols.length > 0) {
|
||||
this.initialized = true;
|
||||
}
|
||||
this.logger.info({
|
||||
count: this.symbols.size,
|
||||
totalRows: symbols.length,
|
||||
@@ -74,12 +77,14 @@ export class SymbolIndexService {
|
||||
* Ensure index is initialized (with retry on failure)
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
if (this.initialized) return;
|
||||
if (!this.initPromise) {
|
||||
this.logger.info('Lazy-loading symbol index');
|
||||
this.initPromise = this.initialize().finally(() => {
|
||||
this.initPromise = null;
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.info('Lazy-loading symbol index');
|
||||
await this.initialize();
|
||||
await this.initPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -213,7 +218,7 @@ export class SymbolIndexService {
|
||||
supported_resolutions: supportedResolutions.length > 0 ? supportedResolutions : DEFAULT_SUPPORTED_RESOLUTIONS,
|
||||
has_intraday: true,
|
||||
has_daily: true,
|
||||
has_weekly_and_monthly: false,
|
||||
has_weekly_and_monthly: true,
|
||||
pricescale,
|
||||
minmov: 1,
|
||||
base_currency: metadata.base_asset,
|
||||
|
||||
@@ -125,6 +125,14 @@ export function createMCPToolWrapper(
|
||||
});
|
||||
}
|
||||
|
||||
/** Silently parse a JSON string; pass non-strings through unchanged. */
|
||||
function tryParseJson(val: unknown): unknown {
|
||||
if (typeof val === 'string') {
|
||||
try { return JSON.parse(val); } catch { /* fall through */ }
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert MCP input schema to Zod schema
|
||||
*/
|
||||
@@ -156,17 +164,13 @@ function mcpInputSchemaToZod(inputSchema?: MCPToolInfo['inputSchema']): z.ZodObj
|
||||
case 'boolean':
|
||||
zodType = z.boolean().describe(prop.description || '');
|
||||
break;
|
||||
case 'array':
|
||||
// Handle array items
|
||||
if (prop.items) {
|
||||
const itemType = getZodTypeForProperty(prop.items);
|
||||
zodType = z.array(itemType).describe(prop.description || '');
|
||||
} else {
|
||||
zodType = z.array(z.any()).describe(prop.description || '');
|
||||
}
|
||||
case 'array': {
|
||||
const itemType = prop.items ? getZodTypeForProperty(prop.items) : z.any();
|
||||
zodType = z.preprocess(tryParseJson, z.array(itemType)).describe(prop.description || '');
|
||||
break;
|
||||
}
|
||||
case 'object':
|
||||
zodType = z.object({}).passthrough().describe(prop.description || '');
|
||||
zodType = z.preprocess(tryParseJson, z.object({}).passthrough()).describe(prop.description || '');
|
||||
break;
|
||||
default:
|
||||
zodType = z.any().describe(prop.description || '');
|
||||
|
||||
@@ -43,7 +43,10 @@ Parameters:
|
||||
from_time: z.union([z.number(), z.string()]).optional().describe('Start time: Unix seconds OR date string (defaults to workspace chartState.start_time)'),
|
||||
to_time: z.union([z.number(), z.string()]).optional().describe('End time: Unix seconds OR date string (defaults to workspace chartState.end_time)'),
|
||||
countback: z.number().optional().describe('Limit number of bars returned (max 500)'),
|
||||
columns: z.array(z.enum(['volume', 'buy_vol', 'sell_vol', 'open_time', 'high_time', 'low_time', 'close_time', 'open_interest'])).optional().describe('Extra columns beyond OHLC'),
|
||||
columns: z.preprocess(
|
||||
(val) => { if (typeof val === 'string') { try { return JSON.parse(val); } catch { /* fall through */ } } return val; },
|
||||
z.array(z.enum(['volume', 'buy_vol', 'sell_vol', 'open_time', 'high_time', 'low_time', 'close_time', 'open_interest'])).optional()
|
||||
).describe('Extra columns beyond OHLC'),
|
||||
}),
|
||||
func: async ({ ticker, period, from_time, to_time, countback, columns }) => {
|
||||
const MAX_BARS = 500;
|
||||
|
||||
@@ -19,31 +19,18 @@ export function createMemoryLookupTool(config: MemoryLookupToolConfig): DynamicS
|
||||
|
||||
return new DynamicStructuredTool({
|
||||
name: 'MemoryLookup',
|
||||
description: `Read a knowledge wiki page by name to get detailed reference information.
|
||||
|
||||
Pass "index" to list all available pages.
|
||||
|
||||
Example pages:
|
||||
- "api-reference" — DataAPI and ChartingAPI reference for research scripts
|
||||
- "usage-examples" — Example research scripts
|
||||
- "pandas-ta-reference" — Full pandas-ta indicator catalog`,
|
||||
description: `Read a knowledge wiki page by name to get detailed reference information. Available pages are listed in the Knowledge Base section of your system prompt.`,
|
||||
schema: z.object({
|
||||
page: z.string().describe(
|
||||
'Wiki page name to read (without .md extension). Pass "index" to list all pages.'
|
||||
'Wiki page name to read (without .md extension), e.g. "api-reference" or "platform/workspace".'
|
||||
),
|
||||
}),
|
||||
func: async ({ page }: { page: string }): Promise<string> => {
|
||||
logger.info({ page }, 'memory_lookup: reading page');
|
||||
|
||||
if (page === 'index') {
|
||||
const pages = await wikiLoader.listPages();
|
||||
return `Available wiki pages:\n${pages.map(p => `- ${p}`).join('\n')}`;
|
||||
}
|
||||
|
||||
const wikiPage = await wikiLoader.loadPage(page);
|
||||
if (!wikiPage) {
|
||||
const pages = await wikiLoader.listPages();
|
||||
return `Page "${page}" not found.\n\nAvailable pages:\n${pages.map(p => `- ${p}`).join('\n')}`;
|
||||
return `Page "${page}" not found. Refer to the Knowledge Base table in your system prompt for available page names.`;
|
||||
}
|
||||
|
||||
return wikiPage.body;
|
||||
|
||||
Reference in New Issue
Block a user