dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Symbol & data refactoring for Nautilus

Browse Source
This commit is contained in:
Tim Olson
2026-04-01 00:59:13 -04:00
parent cd28e18e52
commit 93bc8a3a4f
55 changed files with 537 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gateway/src/clients/duckdb-client.ts
View File

@@ -388,8 +388,8 @@ export class DuckDBClient {
async queryOHLC(
ticker: string,
period_seconds: number,
start_time: bigint, // microseconds
end_time: bigint // microseconds
start_time: bigint, // nanoseconds
end_time: bigint // nanoseconds
): Promise<any[]> {
await this.initialize();

8
gateway/src/clients/iceberg-client.ts
View File

@@ -42,7 +42,7 @@ export interface IcebergMessage {
role: 'user' | 'assistant' | 'system' | 'workspace';
content: string;
metadata: string; // JSON string
timestamp: number; // microseconds
timestamp: number; // nanoseconds
}
/**
@@ -54,7 +54,7 @@ export interface IcebergCheckpoint {
checkpoint_id: string;
checkpoint_data: string; // JSON string
metadata: string; // JSON string
timestamp: number; // microseconds
timestamp: number; // nanoseconds
}
/**
@@ -213,8 +213,8 @@ export class IcebergClient {
async queryOHLC(
ticker: string,
period_seconds: number,
start_time: bigint, // microseconds
end_time: bigint // microseconds
start_time: bigint, // nanoseconds
end_time: bigint // nanoseconds
): Promise<any[]> {
return this.duckdb.queryOHLC(ticker, period_seconds, start_time, end_time);
}

2
gateway/src/clients/zmq-relay-client.ts
View File

@@ -124,7 +124,7 @@ export class ZMQRelayClient {
*
* IMPORTANT: Call connect() before using this method.
*
* @param ticker Market identifier (e.g., "BINANCE:BTC/USDT")
* @param ticker Market identifier (e.g., "BTC/USDT.BINANCE")
* @param period_seconds OHLC period in seconds
* @param start_time Start timestamp in MICROSECONDS
* @param end_time End timestamp in MICROSECONDS

2
gateway/src/harness/agent-harness.ts
View File

@@ -588,7 +588,7 @@ export class AgentHarness {
const labels: Record<string, string> = {
research: 'Researching...',
get_chart_data: 'Fetching chart data...',
symbol_lookup: 'Looking up symbol...',
symbol_lookup: 'Searching symbol...',
category_list: 'Seeing what we have...',
category_edit: 'Coding...',
category_write: 'Coding...',

26
gateway/src/harness/subagents/research/memory/api-reference.md
View File

@@ -60,7 +60,7 @@ class API:
# Fetch data
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20",
end_time="2021-12-21"
@@ -107,8 +107,8 @@ class DataAPI(ABC):
Fetch historical OHLC candlestick data for a market.
Args:
ticker: Market identifier in format "EXCHANGE:SYMBOL"
Examples: "BINANCE:BTC/USDT", "COINBASE:ETH/USD"
ticker: Market identifier in format "MARKET.EXCHANGE"
Examples: "BTC/USDT.BINANCE", "ETH/USD.COINBASE"
period_seconds: Candle period in seconds
Common values:
- 60 (1 minute)
@@ -135,7 +135,7 @@ class DataAPI(ABC):
Returns:
DataFrame with candlestick data sorted by timestamp (ascending).
Standard columns (always included):
- timestamp: Period start time in microseconds
- timestamp: Period start time in nanoseconds
- open: Opening price (decimal float)
- high: Highest price (decimal float)
- low: Lowest price (decimal float)
@@ -151,7 +151,7 @@ class DataAPI(ABC):
Examples:
# Basic OHLC with Unix timestamp
df = await api.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time=1640000000,
end_time=1640086400
@@ -159,7 +159,7 @@ class DataAPI(ABC):
# Using date strings with volume
df = await api.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20",
end_time="2021-12-21",
@@ -169,7 +169,7 @@ class DataAPI(ABC):
# Using datetime objects
from datetime import datetime
df = await api.historical_ohlc(
ticker="COINBASE:ETH/USD",
ticker="ETH/USD.COINBASE",
period_seconds=300,
start_time=datetime(2021, 12, 20, 9, 30),
end_time=datetime(2021, 12, 20, 16, 30),
@@ -193,8 +193,8 @@ class DataAPI(ABC):
specify exact timestamps. Useful for real-time analysis and indicators.
Args:
ticker: Market identifier in format "EXCHANGE:SYMBOL"
Examples: "BINANCE:BTC/USDT", "COINBASE:ETH/USD"
ticker: Market identifier in format "MARKET.EXCHANGE"
Examples: "BTC/USDT.BINANCE", "ETH/USD.COINBASE"
period_seconds: OHLC candle period in seconds
Common values: 60 (1m), 300 (5m), 900 (15m), 3600 (1h),
86400 (1d), 604800 (1w)
@@ -213,14 +213,14 @@ class DataAPI(ABC):
Examples:
# Get the last candle
df = await api.latest_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600
)
# Returns: timestamp, open, high, low, close
# Get the last 50 5-minute candles with volume
df = await api.latest_ohlc(
ticker="COINBASE:ETH/USD",
ticker="ETH/USD.COINBASE",
period_seconds=300,
length=50,
extra_columns=["volume", "buy_vol", "sell_vol"]
@@ -228,7 +228,7 @@ class DataAPI(ABC):
# Get recent candles with all timing data
df = await api.latest_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=60,
length=100,
extra_columns=["open_time", "high_time", "low_time", "close_time"]
@@ -451,7 +451,7 @@ def get_api() -> API:
# Fetch data
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20",
end_time="2021-12-21"

2
gateway/src/harness/subagents/research/memory/pandas-ta-reference.md
View File

@@ -198,7 +198,7 @@ import asyncio
api = get_api()
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2024-01-01",
end_time="2024-01-08",

14
gateway/src/harness/subagents/research/memory/usage-examples.md
View File

@@ -29,7 +29,7 @@ api = get_api()
# Method 1: Using Unix timestamps (seconds)
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600, # 1 hour candles
start_time=1640000000, # Unix timestamp in seconds
end_time=1640086400,
@@ -38,7 +38,7 @@ df = asyncio.run(api.data.historical_ohlc(
# Method 2: Using date strings
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20", # Simple date string
end_time="2021-12-21",
@@ -47,7 +47,7 @@ df = asyncio.run(api.data.historical_ohlc(
# Method 3: Using date strings with time
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20 00:00:00",
end_time="2021-12-20 23:59:59",
@@ -56,7 +56,7 @@ df = asyncio.run(api.data.historical_ohlc(
# Method 4: Using datetime objects
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time=datetime(2021, 12, 20),
end_time=datetime(2021, 12, 21),
@@ -92,7 +92,7 @@ api = get_api()
# Fetch data
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20",
end_time="2021-12-21",
@@ -123,7 +123,7 @@ api = get_api()
# Fetch data
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600,
start_time="2021-12-20",
end_time="2021-12-21"
@@ -191,7 +191,7 @@ api = get_api()
# Fetch historical data using date strings (easiest for research)
df = asyncio.run(api.data.historical_ohlc(
ticker="BINANCE:BTC/USDT",
ticker="BTC/USDT.BINANCE",
period_seconds=3600, # 1 hour
start_time="2021-12-20",
end_time="2021-12-21",

2
gateway/src/routes/symbol-routes.ts
View File

@@ -55,7 +55,7 @@ export class SymbolRoutes {
}
});
// Resolve symbol (use wildcard to capture ticker with slashes like BINANCE:BTC/USDT)
// Resolve symbol (use wildcard to capture ticker with slashes like BTC/USDT.BINANCE)
app.get('/symbols/*', async (request, reply) => {
const symbolIndexService = this.getSymbolIndexService();

30
gateway/src/services/ohlc-service.ts
View File

@@ -25,7 +25,7 @@ import type {
TradingViewBar,
} from '../types/ohlc.js';
import {
secondsToMicros,
secondsToNanos,
backendToTradingView,
DEFAULT_SUPPORTED_RESOLUTIONS,
} from '../types/ohlc.js';
@@ -79,19 +79,19 @@ export class OHLCService {
countback,
}, 'Fetching OHLC data');
// Convert times to microseconds, then align to period boundaries using
// Convert times to nanoseconds, then align to period boundaries using
// [ceil(start), ceil(end)) semantics:
// - start: ceil to next period boundary — excludes any in-progress candle whose
// official timestamp is before from_time.
// - end: ceil to next period boundary, used as EXCLUSIVE upper bound — includes
// the last candle whose timestamp < to_time, excludes one sitting exactly on
// to_time (which would be the next candle, not yet started).
const periodMicros = BigInt(period_seconds) * 1_000_000n;
const raw_start = secondsToMicros(from_time);
const raw_end = secondsToMicros(to_time);
const periodNanos = BigInt(period_seconds) * 1_000_000_000n;
const raw_start = secondsToNanos(from_time);
const raw_end = secondsToNanos(to_time);
// bigint ceiling: ceil(a/b)*b = ((a + b - 1) / b) * b
const start_time = ((raw_start + periodMicros - 1n) / periodMicros) * periodMicros;
const end_time = ((raw_end + periodMicros - 1n) / periodMicros) * periodMicros; // exclusive
const start_time = ((raw_start + periodNanos - 1n) / periodNanos) * periodNanos;
const end_time = ((raw_end + periodNanos - 1n) / periodNanos) * periodNanos; // exclusive
// Step 1: Check Iceberg for existing data
let data = await this.icebergClient.queryOHLC(ticker, period_seconds, start_time, end_time);
@@ -220,11 +220,11 @@ export class OHLCService {
// For now, return default symbol if query matches
if (query.toLowerCase().includes('btc') || query.toLowerCase().includes('binance')) {
return [{
symbol: 'BINANCE:BTC/USDT',
full_name: 'BINANCE:BTC/USDT',
symbol: 'BTC/USDT',
full_name: 'BTC/USDT (BINANCE)',
description: 'Bitcoin / Tether USD',
exchange: 'BINANCE',
ticker: 'BINANCE:BTC/USDT',
ticker: 'BTC/USDT.BINANCE',
type: 'crypto',
}];
}
@@ -241,12 +241,12 @@ export class OHLCService {
this.logger.debug({ symbol }, 'Resolving symbol');
// TODO: Implement central symbol registry
// For now, return default symbol info for BINANCE:BTC/USDT
if (symbol === 'BINANCE:BTC/USDT' || symbol === 'BTC/USDT') {
// For now, return default symbol info for BTC/USDT.BINANCE
if (symbol === 'BTC/USDT.BINANCE' || symbol === 'BTC/USDT') {
return {
symbol: 'BINANCE:BTC/USDT',
name: 'BINANCE:BTC/USDT',
ticker: 'BINANCE:BTC/USDT',
symbol: 'BTC/USDT',
name: 'BTC/USDT',
ticker: 'BTC/USDT.BINANCE',
description: 'Bitcoin / Tether USD',
type: 'crypto',
session: '24x7',

32
gateway/src/services/symbol-index-service.ts
View File

@@ -23,7 +23,7 @@ export interface SymbolIndexServiceConfig {
export class SymbolIndexService {
private icebergClient: IcebergClient;
private logger: FastifyBaseLogger;
private symbols: Map<string, SymbolMetadata> = new Map(); // key: "EXCHANGE:MARKET_ID"
private symbols: Map<string, SymbolMetadata> = new Map(); // key: "MARKET_ID.EXCHANGE" (Nautilus format)
private initialized: boolean = false;
constructor(config: SymbolIndexServiceConfig) {
@@ -52,7 +52,7 @@ export class SymbolIndexService {
const uniqueKeys = new Set<string>();
for (const symbol of symbols) {
const key = `${symbol.exchange_id}:${symbol.market_id}`;
const key = `${symbol.market_id}.${symbol.exchange_id}`;
uniqueKeys.add(key);
this.symbols.set(key, symbol);
}
@@ -86,7 +86,7 @@ export class SymbolIndexService {
* Update or add a symbol to the index
*/
updateSymbol(symbol: SymbolMetadata): void {
const key = `${symbol.exchange_id}:${symbol.market_id}`;
const key = `${symbol.market_id}.${symbol.exchange_id}`;
this.symbols.set(key, symbol);
this.logger.debug({ key }, 'Updated symbol in index');
}
@@ -149,11 +149,11 @@ export class SymbolIndexService {
return null;
}
// ticker format: "EXCHANGE:MARKET_ID" or just "MARKET_ID"
// ticker format: "MARKET_ID.EXCHANGE" (Nautilus) or just "MARKET_ID"
let key = ticker;
// If no exchange prefix, search for first match
if (!ticker.includes(':')) {
// If no dot separator, search for first match by market_id
if (!ticker.includes('.')) {
for (const [k, metadata] of this.symbols) {
if (metadata.market_id === ticker) {
key = k;
@@ -176,7 +176,7 @@ export class SymbolIndexService {
*/
private metadataToSearchResult(metadata: SymbolMetadata): SearchResult {
const symbol = metadata.market_id; // Clean format: "BTC/USDT"
const ticker = `${metadata.exchange_id}:${metadata.market_id}`; // "BINANCE:BTC/USDT"
const ticker = `${metadata.market_id}.${metadata.exchange_id}`; // "BTC/USDT.BINANCE"
const fullName = `${metadata.market_id} (${metadata.exchange_id})`;
return {
@@ -194,15 +194,12 @@ export class SymbolIndexService {
*/
private metadataToSymbolInfo(metadata: SymbolMetadata): SymbolInfo {
const symbol = metadata.market_id;
const ticker = `${metadata.exchange_id}:${metadata.market_id}`;
const ticker = `${metadata.market_id}.${metadata.exchange_id}`; // "BTC/USDT.BINANCE"
// Convert supported_period_seconds to resolution strings
const supportedResolutions = this.periodSecondsToResolutions(metadata.supported_period_seconds || []);
// Calculate pricescale from tick_denom
// tick_denom is 10^n where n is the number of decimal places
// pricescale is the same value
const pricescale = metadata.tick_denom ? Number(metadata.tick_denom) : 100;
// pricescale = 10^price_precision (e.g., price_precision=2 → pricescale=100)
const pricescale = metadata.price_precision != null ? Math.pow(10, metadata.price_precision) : 100;
return {
symbol,
@@ -222,9 +219,12 @@ export class SymbolIndexService {
base_currency: metadata.base_asset,
quote_currency: metadata.quote_asset,
data_status: 'streaming',
tick_denominator: metadata.tick_denom ? Number(metadata.tick_denom) : undefined,
base_denominator: metadata.base_denom ? Number(metadata.base_denom) : undefined,
quote_denominator: metadata.quote_denom ? Number(metadata.quote_denom) : undefined,
price_precision: metadata.price_precision,
size_precision: metadata.size_precision,
tick_size: metadata.tick_size,
lot_size: metadata.lot_size,
maker_fee: metadata.maker_fee,
taker_fee: metadata.taker_fee,
};
}

4
gateway/src/tools/platform/get-chart-data.tool.ts
View File

@@ -164,7 +164,7 @@ async function getChartState(workspaceManager: WorkspaceManager, logger: Fastify
if (!chartState) {
// Return default chart state
return {
symbol: 'BINANCE:BTC/USDT',
symbol: 'BTC/USDT.BINANCE',
start_time: null,
end_time: null,
period: 900,
@@ -177,7 +177,7 @@ async function getChartState(workspaceManager: WorkspaceManager, logger: Fastify
logger.error({ error }, 'Failed to get chart state from workspace');
// Return default chart state
return {
symbol: 'BINANCE:BTC/USDT',
symbol: 'BTC/USDT.BINANCE',
start_time: null,
end_time: null,
period: 900,

65
gateway/src/types/ohlc.ts
View File

@@ -3,7 +3,7 @@
*
* Handles conversion between:
* - TradingView datafeed format (seconds, OHLCV structure)
* - Backend/Iceberg format (microseconds, ticker prefix)
* - Backend/Iceberg format (nanoseconds, Nautilus ticker)
* - ZMQ protocol format (protobuf messages)
*/
@@ -31,8 +31,8 @@ export interface TradingViewBar {
* Backend OHLC format (from Iceberg)
*/
export interface BackendOHLC {
timestamp: bigint; // Unix timestamp in MICROSECONDS — kept as bigint to preserve precision
ticker: string;
timestamp: bigint; // Unix timestamp in NANOSECONDS — kept as bigint to preserve precision
ticker: string; // Nautilus format: "BTC/USDT.BINANCE"
period_seconds: number;
open: number | null; // null for gap bars (no trades that period)
high: number | null;
@@ -59,7 +59,7 @@ export interface DatafeedConfig {
*/
export interface SymbolInfo {
symbol: string; // Clean format (e.g., "BTC/USDT")
ticker: string; // With exchange prefix (e.g., "BINANCE:BTC/USDT")
ticker: string; // Nautilus format (e.g., "BTC/USDT.BINANCE")
name: string; // Display name
description: string; // Human-readable description
type: string; // "crypto", "spot", "futures", etc.
@@ -70,14 +70,18 @@ export interface SymbolInfo {
has_intraday: boolean;
has_daily: boolean;
has_weekly_and_monthly: boolean;
pricescale: number; // Price scale factor
pricescale: number; // 10^price_precision
minmov: number; // Minimum price movement
base_currency?: string; // Base asset (e.g., "BTC")
quote_currency?: string; // Quote asset (e.g., "USDT")
data_status?: string; // "streaming", "delayed", etc.
tick_denominator?: number; // Denominator for price scaling (e.g., 1e6)
base_denominator?: number; // Denominator for base asset
quote_denominator?: number; // Denominator for quote asset
// Nautilus Instrument fields
price_precision?: number;
size_precision?: number;
tick_size?: number;
lot_size?: number;
maker_fee?: number;
taker_fee?: number;
}
/**
@@ -95,7 +99,7 @@ export interface HistoryResult {
*/
export interface SearchResult {
symbol: string; // Clean format (e.g., "BTC/USDT")
ticker: string; // With exchange prefix for routing (e.g., "BINANCE:BTC/USDT")
ticker: string; // Nautilus format (e.g., "BTC/USDT.BINANCE")
full_name: string; // Full display name (e.g., "BTC/USDT (BINANCE)")
description: string; // Human-readable description
exchange: string; // Exchange identifier
@@ -122,9 +126,9 @@ export enum NotificationStatus {
export interface SubmitHistoricalRequest {
request_id: string;
ticker: string;
start_time: bigint; // microseconds
end_time: bigint; // microseconds
ticker: string; // Nautilus format: "BTC/USDT.BINANCE"
start_time: bigint; // nanoseconds
end_time: bigint; // nanoseconds
period_seconds: number;
limit?: number;
client_id?: string;
@@ -139,34 +143,33 @@ export interface SubmitResponse {
export interface HistoryReadyNotification {
request_id: string;
ticker: string;
ticker: string; // Nautilus format: "BTC/USDT.BINANCE"
period_seconds: number;
start_time: bigint; // microseconds
end_time: bigint; // microseconds
start_time: bigint; // nanoseconds
end_time: bigint; // nanoseconds
status: NotificationStatus;
error_message?: string;
iceberg_namespace: string;
iceberg_table: string;
row_count: number;
completed_at: bigint; // microseconds
completed_at: bigint; // nanoseconds
}
/**
* Conversion utilities
*/
export function secondsToMicros(seconds: number): bigint {
return BigInt(Math.floor(seconds)) * 1000000n;
export function secondsToNanos(seconds: number): bigint {
return BigInt(Math.floor(seconds)) * 1_000_000_000n;
}
export function microsToSeconds(micros: bigint | number): number {
// Integer division: convert microseconds to seconds (truncates to integer)
return Number(BigInt(micros) / 1000000n);
export function nanosToSeconds(nanos: bigint | number): number {
return Number(BigInt(nanos) / 1_000_000_000n);
}
export function backendToTradingView(backend: BackendOHLC): TradingViewBar {
return {
time: microsToSeconds(backend.timestamp),
time: nanosToSeconds(backend.timestamp),
open: backend.open,
high: backend.high,
low: backend.low,
@@ -220,10 +223,18 @@ export interface SymbolMetadata {
description?: string;
base_asset?: string;
quote_asset?: string;
tick_denom?: bigint;
base_denom?: bigint;
quote_denom?: bigint;
supported_period_seconds?: number[];
earliest_time?: bigint;
updated_at: bigint;
earliest_time?: bigint; // nanoseconds
updated_at: bigint; // nanoseconds
// Nautilus Instrument fields
price_precision?: number;
size_precision?: number;
tick_size?: number;
lot_size?: number;
min_notional?: number;
margin_init?: number;
margin_maint?: number;
maker_fee?: number;
taker_fee?: number;
contract_multiplier?: number;
}

2
gateway/src/workspace/types.ts
View File

@@ -81,7 +81,7 @@ export const DEFAULT_STORES: StoreConfig[] = [
name: 'chartState',
persistent: false,
initialState: () => ({
symbol: 'BINANCE:BTC/USDT',
symbol: 'BTC/USDT.BINANCE',
start_time: null,
end_time: null,
period: '15',