bugfixes; research subproc; higher sandbox limits

This commit is contained in:
2026-04-16 18:11:26 -04:00
parent f80c943dc3
commit 3153e89d4f
54 changed files with 1947 additions and 498 deletions

View File

@@ -595,28 +595,28 @@ export class WebSocketHandler {
case 'get_bars': {
if (!ohlcService) {
socket.send(JSON.stringify({
type: 'error',
type: 'get_bars_response',
request_id: requestId,
error_message: 'OHLC service not available'
error: 'OHLC service not available',
}));
break;
}
const history = await ohlcService.fetchOHLC(
payload.symbol,
payload.period_seconds,
payload.from_time,
payload.to_time,
payload.countback
);
logger.info({ requestId, barCount: history.bars?.length ?? 0, noData: history.noData, socketState: socket.readyState }, 'Sending get_bars_response');
socket.send(
jsonStringifySafe({
type: 'get_bars_response',
request_id: requestId,
history,
})
);
logger.info({ requestId }, 'get_bars_response sent');
try {
const history = await ohlcService.fetchOHLC(
payload.symbol,
payload.period_seconds,
payload.from_time,
payload.to_time,
payload.countback
);
logger.info({ requestId, barCount: history.bars?.length ?? 0, noData: history.noData, socketState: socket.readyState }, 'Sending get_bars_response');
socket.send(jsonStringifySafe({ type: 'get_bars_response', request_id: requestId, history }));
logger.info({ requestId }, 'get_bars_response sent');
} catch (err: any) {
const errorMessage = err?.message ?? String(err);
logger.error({ requestId, ticker: payload.symbol, errorMessage }, 'get_bars failed');
socket.send(JSON.stringify({ type: 'get_bars_response', request_id: requestId, error: errorMessage }));
}
break;
}

View File

@@ -1,6 +1,6 @@
import { Pool } from 'pg';
import type { UserLicense } from '../types/user.js';
import { UserLicenseSchema } from '../types/user.js';
import type { UserLicense, License, LicenseTier } from '../types/user.js';
import { UserLicenseSchema, LICENSE_TIER_TEMPLATES } from '../types/user.js';
import type { AuthService } from '../auth/auth-service.js';
export class UserService {
@@ -114,6 +114,54 @@ export class UserService {
return await this.authService.verifyToken(token);
}
/**
* Re-apply the current canonical template for every user's declared licenseType.
* Updates only the DB — does not touch deployments, so running pods are unaffected
* until their next natural restart.
*/
async migrateAllLicenses(): Promise<{ updated: number }> {
const client = await this.pool.connect();
try {
const rows = await client.query(
`SELECT user_id, license->>'licenseType' AS tier FROM user_licenses`
);
let updated = 0;
for (const row of rows.rows) {
const tier = row.tier as LicenseTier;
if (!LICENSE_TIER_TEMPLATES[tier]) continue;
await client.query(
`UPDATE user_licenses SET license = $1::jsonb, updated_at = NOW() WHERE user_id = $2`,
[JSON.stringify(LICENSE_TIER_TEMPLATES[tier]), row.user_id]
);
updated++;
}
return { updated };
} finally {
client.release();
}
}
/**
* Set a user's license to a canonical tier template.
* Overwrites the existing license with the current template for that tier.
*/
async setUserLicenseTier(userId: string, tier: LicenseTier): Promise<License> {
const license = LICENSE_TIER_TEMPLATES[tier];
const client = await this.pool.connect();
try {
await client.query(
`INSERT INTO user_licenses (user_id, license, mcp_server_url, updated_at)
VALUES ($1, $2::jsonb, 'pending', NOW())
ON CONFLICT (user_id) DO UPDATE
SET license = EXCLUDED.license, updated_at = NOW()`,
[userId, JSON.stringify(license)]
);
} finally {
client.release();
}
return license;
}
/**
* Close database pool
*/

View File

@@ -16,6 +16,7 @@ import type { ResearchSubagent } from './subagents/research/index.js';
import type { IndicatorSubagent } from './subagents/indicator/index.js';
import type { WebExploreSubagent } from './subagents/web-explore/index.js';
import type { StrategySubagent } from './subagents/strategy/index.js';
import { BaseSubagent } from './subagents/base-subagent.js';
import type { DynamicStructuredTool } from '@langchain/core/tools';
import { getToolRegistry } from '../tools/tool-registry.js';
import type { MCPToolInfo } from '../tools/mcp/mcp-tool-wrapper.js';
@@ -237,12 +238,22 @@ export class AgentHarness {
try {
const { createResearchSubagent } = await import('./subagents/research/index.js');
// Create a model for the research subagent
// Path resolution: use the compiled output path
const researchSubagentPath = join(__dirname, 'subagents', 'research');
this.config.logger.debug({ researchSubagentPath }, 'Using research subagent path');
// Load the subagent config to get maxTokens — research scripts require more tokens
// than the provider default (4096) because python_write arguments include full code bodies
const researchSubagentConfig = await BaseSubagent.loadConfig(researchSubagentPath);
// Create a model for the research subagent — always use the complex model
// since research tasks involve data analysis, charting, and code generation
const { model } = await this.modelRouter.route(
'research analysis', // dummy query
'analyze and backtest research data', // triggers complex routing
this.config.license,
RoutingStrategy.COMPLEXITY,
this.config.userId
this.config.userId,
researchSubagentConfig.maxTokens // honour the subagent's maxTokens (e.g. 8192)
);
// Get tools for research subagent from registry
@@ -274,10 +285,6 @@ export class AgentHarness {
}));
}
// Path resolution: use the compiled output path
const researchSubagentPath = join(__dirname, 'subagents', 'research');
this.config.logger.debug({ researchSubagentPath }, 'Using research subagent path');
this.researchSubagent = await createResearchSubagent(
model,
this.config.logger,
@@ -535,10 +542,12 @@ export class AgentHarness {
const stream = await model.stream(messagesCopy, { signal });
for await (const chunk of stream) {
if (typeof chunk.content === 'string' && chunk.content.length > 0) {
this.config.logger.trace({ content: chunk.content }, 'raw chunk');
yield { type: 'chunk', content: chunk.content };
} else if (Array.isArray(chunk.content)) {
for (const block of chunk.content) {
if (block.type === 'text' && block.text) {
this.config.logger.trace({ content: block.text }, 'raw chunk');
yield { type: 'chunk', content: block.text };
}
}

View File

@@ -18,8 +18,11 @@ Dexorder trading platform provides OHLC data at a 1-minute resolution and suppor
Dexorder does not support:
* tick-by-tick trading or high-frequency strategies.
* long-running computations like paramater optimizations or training machine learning models.
* long-running computations like parameter optimizations or training machine learning models during live execution.
* portfolio optimization or trading strategies that require a large number of symbols.
* LLM calls inside strategy scripts — strategies must be deterministic and lightweight for backtesting to be reliable and repeatable. LLMs are slow, expensive, and introduce temperature-based non-determinism that breaks backtesting. (Walk-forward LLM integration via timer/data triggers is planned but not yet available.)
* TradFi data (equities, forex, bonds, options, etc.) — only crypto pricing data is available.
* Alternative data sources such as news feeds, Twitter/social sentiment, on-chain data, or economic calendars — these are not yet available.
Dexorder does support:
* backtesting strategies against historical data.
@@ -33,6 +36,27 @@ If the user asks for a capability not provided by Dexorder, decline and explain
# Important Instructions
## Switching Chart Symbol or Timeframe
**IMPORTANT: When the user asks to switch, change, or update the chart symbol or timeframe, you MUST call `workspace_patch` directly. Do NOT use web_explore, do NOT delegate to the indicator tool.**
Call `workspace_patch` with `store_name = "chartState"` and the appropriate JSON patch:
To switch symbol only:
```json
[{ "op": "replace", "path": "/symbol", "value": "SOL/USDT.BINANCE" }]
```
To switch symbol and period (period is seconds: 60=1m, 300=5m, 900=15m, 3600=1h, 86400=1D):
```json
[
{ "op": "replace", "path": "/symbol", "value": "SOL/USDT.BINANCE" },
{ "op": "replace", "path": "/period", "value": 900 }
]
```
You already know this format — do not search for it. After patching, confirm the change to the user.
## Investment Advice
**NEVER** recommend any specific ticker, trade, or position. You may suggest mechanical adjustments or improvements to strategies, but you must **NEVER** offer an opinion on a specific trade or position. You are **NOT** a registered investment advisor.

View File

@@ -1 +1 @@
This is your first chat with a new user. Welcome them to Dexorder and describe who are you and what can you do.
This is your first chat with a new user. Welcome them to Dexorder, and describe who you are and what can you do.

View File

@@ -83,6 +83,15 @@ self.config.initial_capital # starting capital in quote currency
| `sell_vol` | float | Sell-side volume (taker sells) |
| `open_interest` | float | Open interest (futures only; NaN for spot) |
### Available data — crypto only
Strategies have access **only** to crypto OHLC feeds with volume, buy/sell volume split, and open interest. The following are **not available** and must never be referenced in a strategy:
- **TradFi data** — equities, forex, bonds, futures spreads, options, macro indicators, interest rates, etc.
- **Alternative data** — news feeds, social sentiment (Twitter/Reddit), on-chain metrics, economic calendars, earnings, etc.
If a user requests a strategy that depends on unavailable data, explain the limitation and offer a crypto-native alternative (e.g. use order-flow imbalance instead of news sentiment).
---
## Section B — Strategy Metadata
@@ -355,3 +364,16 @@ deactivate_strategy(strategy_name) # Stop and get final PnL
- 4h bars: 100k bars ≈ 45 years → cap at 5 years (≈ 10,950 bars)
7. **Never `import` from `dexorder` inside `evaluate()`** — the strategy file is exec'd in a sandbox with PandasStrategy and pandas_ta pre-loaded. Standard library and pandas/numpy/pandas_ta are available.
8. **No LLM calls inside strategies** — strategies must be fully deterministic. LLM invocations are prohibited because:
- They are slow and expensive, making backtesting impractical.
- Any temperature > 0 produces non-repeatable outputs, breaking backtest reproducibility.
- The correct model is: the LLM *writes* the strategy; the strategy runs without LLM involvement.
- Walk-forward LLM integration (via timer or data triggers) is a planned feature but is **not yet implemented**. Do not attempt to approximate it now.
9. **`evaluate()` must be fast, lightweight, and deterministic** — it is called on every bar during backtesting across potentially hundreds of thousands of bars. Specifically:
- **No heavy computation at runtime**: model inference, large matrix operations, file I/O, network calls, or database queries are forbidden inside `evaluate()`.
- **ML is allowed with restrictions**: a model may be trained offline (e.g. in `__init__` using warm-up data), but inference in `evaluate()` must be fast (microseconds, not milliseconds). If training is compute-intensive, note this clearly in the strategy description.
- **No randomness**: do not use `random`, `np.random`, or any non-seeded stochastic operation. All outputs given the same data must be identical across runs.
10. **Data scope** — strategies may only use data available in the `dfs` feeds. Do not attempt to fetch external data, call APIs, read files, or access anything outside the provided DataFrames. Crypto OHLCV + buy/sell volume + open interest is what is available; nothing else.

View File

@@ -306,6 +306,25 @@ export class KubernetesClient {
}
}
/**
* Delete only the Deployment, preserving PVC (user data) and Service (stable DNS).
* Used when applying a license tier change — next ensureContainerRunning recreates
* the deployment with updated resource limits.
*/
async deleteDeploymentOnly(userId: string): Promise<void> {
const deploymentName = KubernetesClient.getDeploymentName(userId);
try {
await this.appsApi.deleteNamespacedDeployment({
name: deploymentName,
namespace: this.config.namespace
});
this.config.logger.info({ deploymentName }, 'Deleted deployment (tier change)');
} catch (error: any) {
const is404 = error.code === 404 || error.response?.statusCode === 404 || error.statusCode === 404;
if (!is404) throw error;
}
}
/**
* Delete deployment and associated resources
* (Used for cleanup/testing - normally handled by lifecycle sidecar)

View File

@@ -1,9 +1,11 @@
import type { FastifyBaseLogger } from 'fastify';
import { KubernetesClient, type DeploymentSpec } from './client.js';
import type { License } from '../types/user.js';
import type { License, LicenseTier } from '../types/user.js';
import type { UserService } from '../db/user-service.js';
export interface ContainerManagerConfig {
k8sClient: KubernetesClient;
userService: UserService;
sandboxImage: string;
sidecarImage: string;
storageClass: string;
@@ -139,6 +141,17 @@ export class ContainerManager {
return { exists: true, ready, mcpEndpoint };
}
/**
* Apply a canonical license tier to a user: updates DB and deletes the deployment
* so it is recreated with the new resource limits on next connect.
*/
async applyLicenseTier(userId: string, tier: LicenseTier): Promise<License> {
const license = await this.config.userService.setUserLicenseTier(userId, tier);
await this.config.k8sClient.deleteDeploymentOnly(userId);
this.config.logger.info({ userId, tier }, 'License tier applied; deployment will recreate on next connect');
return license;
}
/**
* Delete container (for cleanup/testing)
*/

View File

@@ -42,7 +42,8 @@ export class ModelRouter {
message: string,
license: License,
strategy: RoutingStrategy = RoutingStrategy.USER_PREFERENCE,
userId?: string
userId?: string,
maxTokens?: number
): Promise<{ model: BaseChatModel; middleware: ModelMiddleware }> {
let modelConfig: ModelConfig;
@@ -67,12 +68,17 @@ export class ModelRouter {
modelConfig = this.defaultModel;
}
if (maxTokens !== undefined) {
modelConfig = { ...modelConfig, maxTokens };
}
this.logger.info(
{
userId,
strategy,
provider: modelConfig.provider,
model: modelConfig.model,
maxTokens: modelConfig.maxTokens,
},
'Routing to model'
);

View File

@@ -22,6 +22,7 @@ import { AgentHarness, type HarnessSessionConfig } from './harness/agent-harness
import { OHLCService } from './services/ohlc-service.js';
import { SymbolIndexService } from './services/symbol-index-service.js';
import { SymbolRoutes } from './routes/symbol-routes.js';
import { AdminRoutes } from './routes/admin-routes.js';
// Catch unhandled promise rejections for better debugging
process.on('unhandledRejection', (reason: any, promise) => {
@@ -309,6 +310,7 @@ const k8sClient = new KubernetesClient({
const containerManager = new ContainerManager({
k8sClient,
userService,
sandboxImage: config.kubernetes.sandboxImage,
sidecarImage: config.kubernetes.sidecarImage,
storageClass: config.kubernetes.storageClass,
@@ -439,6 +441,9 @@ const getSymbolService = () => symbolIndexService;
const symbolRoutes = new SymbolRoutes({ getSymbolIndexService: getSymbolService });
symbolRoutes.register(app);
// Register admin routes
new AdminRoutes(containerManager, userService).register(app);
app.log.debug('All routes registered');
// Health check
@@ -715,7 +720,6 @@ try {
icebergClient,
logger: app.log,
});
await indexService.initialize();
// Assign to module-level variable so onMetadataUpdate callback can use it
symbolIndexService = indexService;
@@ -723,7 +727,17 @@ try {
// Update websocket handler's config so it can use the service
(websocketHandler as any).config.symbolIndexService = indexService;
app.log.info({ stats: symbolIndexService.getStats() }, 'Symbol index service initialized');
// Retry until we get at least some symbol metadata
while (true) {
await indexService.initialize();
const stats = indexService.getStats();
if (stats.symbolCount > 0) {
app.log.info({ stats }, 'Symbol index service initialized');
break;
}
app.log.warn('Symbol index has no metadata yet, retrying in 5 seconds...');
await new Promise(resolve => setTimeout(resolve, 5000));
}
} catch (error) {
app.log.warn({ error }, 'Failed to initialize symbol index service - symbol search will not be available');
}

View File

@@ -0,0 +1,35 @@
import type { FastifyInstance } from 'fastify';
import type { ContainerManager } from '../k8s/container-manager.js';
import type { UserService } from '../db/user-service.js';
import type { LicenseTier } from '../types/user.js';
const VALID_TIERS: LicenseTier[] = ['free', 'pro', 'enterprise'];
export class AdminRoutes {
private containerManager: ContainerManager;
private userService: UserService;
constructor(containerManager: ContainerManager, userService: UserService) {
this.containerManager = containerManager;
this.userService = userService;
}
register(app: FastifyInstance): void {
app.post<{ Params: { userId: string }; Body: { tier: string } }>(
'/admin/users/:userId/set-tier',
async (req, reply) => {
const { userId } = req.params;
const { tier } = req.body;
if (!VALID_TIERS.includes(tier as LicenseTier)) {
return reply.code(400).send({ error: `Invalid tier. Must be one of: ${VALID_TIERS.join(', ')}` });
}
const license = await this.containerManager.applyLicenseTier(userId, tier as LicenseTier);
return { userId, tier, license };
}
);
app.post('/admin/migrate-licenses', async () => {
return await this.userService.migrateAllLicenses();
});
}
}

View File

@@ -167,11 +167,7 @@ export class OHLCService {
period_seconds,
}, 'Failed to fetch historical data');
// Return empty result on error
return {
bars: [],
noData: true,
};
throw error;
}
}

View File

@@ -0,0 +1,87 @@
/**
* Direct DeepInfra streaming test — bypasses LangChain entirely.
* Logs each delta.content with JSON.stringify so spaces are unambiguous.
*
* Usage:
* DEEPINFRA_API_KEY=$(op read "op://Private/DeepInfra/credential") npx tsx src/test-deepinfra-chunks.ts
*/
export {};
const DEEP_INFRA_URL = 'https://api.deepinfra.com/v1/openai/chat/completions';
const MODEL = 'zai-org/GLM-5';
const apiKey = process.env.DEEPINFRA_API_KEY;
if (!apiKey) {
console.error('DEEPINFRA_API_KEY is not set');
process.exit(1);
}
const res = await fetch(DEEP_INFRA_URL, {
method: 'POST',
headers: {
Authorization: `Bearer ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: MODEL,
stream: true,
messages: [
{ role: 'user', content: 'Write two sentences about ETH price analysis.' },
],
}),
});
if (!res.ok || !res.body) {
console.error(`HTTP ${res.status}: ${await res.text()}`);
process.exit(1);
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let chunkIndex = 0;
let assembled = '';
console.log(`Testing model: ${MODEL}`);
console.log('--- chunks ---');
while (true) {
const { value, done } = await reader.read();
if (done) break;
const text = decoder.decode(value, { stream: true });
for (const line of text.split('\n')) {
const trimmed = line.trim();
if (!trimmed.startsWith('data:')) continue;
const data = trimmed.slice(5).trimStart();
if (data === '[DONE]') break;
let parsed: unknown;
try {
parsed = JSON.parse(data);
} catch {
continue;
}
const choice = (parsed as { choices?: Array<{ delta?: Record<string, unknown> }> })
?.choices?.[0];
const delta = choice?.delta;
const content = delta?.content as string | undefined;
if (content !== undefined) {
const endsSpace = content.endsWith(' ');
const startsSpace = content.startsWith(' ');
// Log full delta so we can see all available fields (logprobs, token_ids, etc.)
console.log(
`chunk[${chunkIndex++}]: ${JSON.stringify(content)} ` +
`(len=${content.length}, startsSpace=${startsSpace}, endsSpace=${endsSpace}) ` +
`delta=${JSON.stringify(delta)}`,
);
assembled += content;
}
}
}
console.log('--- assembled ---');
console.log(assembled);

View File

@@ -42,7 +42,8 @@ Use this tool for:
- Recommending indicators for a given strategy or analysis goal
ALWAYS use this tool for any request about the chart's indicators.
NEVER modify the indicators workspace store directly.`,
NEVER modify the indicators workspace store directly.
NEVER use this tool to switch the chart symbol or timeframe — that is done via workspace_patch on chartState.`,
schema: z.object({
instruction: z.string().describe(
'The indicator task to perform. Be specific about which indicators, parameters, ' +

View File

@@ -30,13 +30,18 @@ export function createWebExploreAgentTool(config: WebExploreAgentToolConfig): Dy
const tool = new DynamicStructuredTool({
name: 'web_explore',
description: `Search the web or academic databases and return a summarized answer.
description: `Search the EXTERNAL web or academic databases and return a summarized answer.
Use this tool when the user asks about:
Use this tool ONLY for external, public information:
- Current events, news, or real-time information
- Documentation, tutorials, or how-to guides
- External documentation, tutorials, or how-to guides for third-party libraries/tools
- Academic papers, research findings, or scientific topics
- Any topic that benefits from external sources
- Any topic requiring external sources
NEVER use this tool for:
- Questions about the Dexorder platform itself (workspace tools, chartState, indicators, strategies)
- Internal API usage (workspace_patch, workspace_read, etc.) — consult the system prompt instead
- Anything that can be answered from the context already available
The subagent will search the web (or arXiv for academic queries), fetch relevant content, and return a markdown summary with cited sources.`,
schema: z.object({

View File

@@ -76,7 +76,7 @@ export const LICENSE_TIER_TEMPLATES: Record<LicenseTier, License> = {
maxTokensPerMessage: 4096, rateLimitPerMinute: 10,
},
k8sResources: {
memoryRequest: '256Mi', memoryLimit: '512Mi',
memoryRequest: '256Mi', memoryLimit: '8Gi',
cpuRequest: '100m', cpuLimit: '500m',
storage: '1Gi', tmpSizeLimit: '128Mi',
enableIdleShutdown: true, idleTimeoutMinutes: 15,
@@ -93,7 +93,7 @@ export const LICENSE_TIER_TEMPLATES: Record<LicenseTier, License> = {
maxTokensPerMessage: 8192, rateLimitPerMinute: 60,
},
k8sResources: {
memoryRequest: '512Mi', memoryLimit: '2Gi',
memoryRequest: '512Mi', memoryLimit: '8Gi',
cpuRequest: '250m', cpuLimit: '2000m',
storage: '10Gi', tmpSizeLimit: '256Mi',
enableIdleShutdown: false, idleTimeoutMinutes: 0,
@@ -110,7 +110,7 @@ export const LICENSE_TIER_TEMPLATES: Record<LicenseTier, License> = {
maxTokensPerMessage: 32768, rateLimitPerMinute: 300,
},
k8sResources: {
memoryRequest: '1Gi', memoryLimit: '4Gi',
memoryRequest: '1Gi', memoryLimit: '8Gi',
cpuRequest: '500m', cpuLimit: '4000m',
storage: '50Gi', tmpSizeLimit: '512Mi',
enableIdleShutdown: false, idleTimeoutMinutes: 0,

View File

@@ -79,12 +79,12 @@ export interface StoreConfig {
export const DEFAULT_STORES: StoreConfig[] = [
{
name: 'chartState',
persistent: false,
persistent: true,
initialState: () => ({
symbol: 'BTC/USDT.BINANCE',
start_time: null,
end_time: null,
period: '15',
period: 900,
selected_shapes: [],
}),
},