dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Symbol & data refactoring for Nautilus

Browse Source
This commit is contained in:
Tim Olson
2026-04-01 00:59:13 -04:00
parent cd28e18e52
commit 93bc8a3a4f
55 changed files with 537 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ingestor/README.md
View File

@@ -75,7 +75,7 @@ docker run -v /path/to/config:/config ccxt-ingestor
Tickers must be in the format: `EXCHANGE:SYMBOL`
Examples:
- `BINANCE:BTC/USDT`
- `BTC/USDT.BINANCE`
- `COINBASE:ETH/USD`
- `KRAKEN:XRP/EUR`

69
ingestor/src/ccxt-fetcher.js
View File

@@ -12,17 +12,17 @@ export class CCXTFetcher {
/**
* Parse ticker string to exchange and symbol
* Expected format: "EXCHANGE:SYMBOL" (e.g., "BINANCE:BTC/USDT")
* Expected format: "SYMBOL.EXCHANGE" (e.g., "BTC/USDT.BINANCE")
*/
parseTicker(ticker) {
const parts = ticker.split(':');
if (parts.length !== 2) {
throw new Error(`Invalid ticker format: ${ticker}. Expected "EXCHANGE:SYMBOL"`);
const lastDot = ticker.lastIndexOf('.');
if (lastDot === -1) {
throw new Error(`Invalid ticker format: ${ticker}. Expected "SYMBOL.EXCHANGE"`);
}
return {
exchange: parts[0].toLowerCase(),
symbol: parts[1]
exchange: ticker.slice(lastDot + 1).toLowerCase(),
symbol: ticker.slice(0, lastDot)
};
}
@@ -101,9 +101,9 @@ export class CCXTFetcher {
const { exchange: exchangeName, symbol } = this.parseTicker(ticker);
const exchange = this.getExchange(exchangeName);
// Convert microseconds to milliseconds
const startMs = Math.floor(parseInt(startTime) / 1000);
const endMs = Math.floor(parseInt(endTime) / 1000);
// Convert nanoseconds to milliseconds
const startMs = Math.floor(parseInt(startTime) / 1_000_000);
const endMs = Math.floor(parseInt(endTime) / 1_000_000);
// Map period seconds to CCXT timeframe
const timeframe = this.secondsToTimeframe(periodSeconds);
@@ -208,7 +208,7 @@ export class CCXTFetcher {
/**
* Fetch recent trades for realtime tick data
* @param {string} ticker - Ticker in format "EXCHANGE:SYMBOL"
* @param {string} since - Optional timestamp in microseconds to fetch from
* @param {string} since - Optional timestamp in nanoseconds to fetch from
* @returns {Promise<Array>} Array of trade ticks
*/
async fetchRecentTrades(ticker, since = null) {
@@ -216,8 +216,8 @@ export class CCXTFetcher {
const exchange = this.getExchange(exchangeName);
try {
// Convert microseconds to milliseconds if provided
const sinceMs = since ? Math.floor(parseInt(since) / 1000) : undefined;
// Convert nanoseconds to milliseconds if provided
const sinceMs = since ? Math.floor(parseInt(since) / 1_000_000) : undefined;
const trades = await exchange.fetchTrades(symbol, sinceMs, 1000);
@@ -243,25 +243,24 @@ export class CCXTFetcher {
/**
* Convert CCXT OHLCV array to our OHLC format
* CCXT format: [timestamp, open, high, low, close, volume]
* Uses denominators from market metadata for proper integer representation
* Uses precision fields from market metadata for proper integer representation
*/
convertToOHLC(candle, ticker, periodSeconds, metadata) {
const [timestamp, open, high, low, close, volume] = candle;
// Use denominators from metadata
const tickDenom = metadata.tickDenom || 100;
const baseDenom = metadata.baseDenom || 100000000;
const priceMult = Math.pow(10, metadata.pricePrecision ?? 2);
const sizeMult = Math.pow(10, metadata.sizePrecision ?? 8);
return {
ticker,
timestamp: (timestamp * 1000).toString(), // Convert ms to microseconds
open: Math.round(open * tickDenom).toString(),
high: Math.round(high * tickDenom).toString(),
low: Math.round(low * tickDenom).toString(),
close: Math.round(close * tickDenom).toString(),
volume: Math.round(volume * baseDenom).toString(),
open_time: (timestamp * 1000).toString(),
close_time: ((timestamp + periodSeconds * 1000) * 1000).toString()
timestamp: (timestamp * 1_000_000).toString(), // Convert ms to nanoseconds
open: Math.round(open * priceMult).toString(),
high: Math.round(high * priceMult).toString(),
low: Math.round(low * priceMult).toString(),
close: Math.round(close * priceMult).toString(),
volume: Math.round(volume * sizeMult).toString(),
open_time: (timestamp * 1_000_000).toString(),
close_time: ((timestamp + periodSeconds * 1000) * 1_000_000).toString()
};
}
@@ -272,35 +271,33 @@ export class CCXTFetcher {
createGapBar(timestampMs, ticker, periodSeconds, metadata) {
return {
ticker,
timestamp: (timestampMs * 1000).toString(), // Convert ms to microseconds
timestamp: (timestampMs * 1_000_000).toString(), // Convert ms to nanoseconds
open: null,
high: null,
low: null,
close: null,
volume: null,
open_time: (timestampMs * 1000).toString(),
close_time: ((timestampMs + periodSeconds * 1000) * 1000).toString()
open_time: (timestampMs * 1_000_000).toString(),
close_time: ((timestampMs + periodSeconds * 1000) * 1_000_000).toString()
};
}
/**
* Convert CCXT trade to our Tick format
* Uses denominators from market metadata for proper integer representation
* Uses precision fields from market metadata for proper integer representation
*/
convertToTick(trade, ticker, metadata) {
// Use denominators from metadata
const tickDenom = metadata.tickDenom || 100;
const baseDenom = metadata.baseDenom || 100000000;
const quoteDenom = metadata.quoteDenom || tickDenom;
const priceMult = Math.pow(10, metadata.pricePrecision ?? 2);
const sizeMult = Math.pow(10, metadata.sizePrecision ?? 8);
const price = Math.round(trade.price * tickDenom);
const amount = Math.round(trade.amount * baseDenom);
const quoteAmount = Math.round((trade.price * trade.amount) * quoteDenom);
const price = Math.round(trade.price * priceMult);
const amount = Math.round(trade.amount * sizeMult);
const quoteAmount = Math.round((trade.price * trade.amount) * priceMult);
return {
trade_id: trade.id || `${trade.timestamp}`,
ticker,
timestamp: (trade.timestamp * 1000).toString(), // Convert ms to microseconds
timestamp: (trade.timestamp * 1_000_000).toString(), // Convert ms to nanoseconds
price: price.toString(),
amount: amount.toString(),
quote_amount: quoteAmount.toString(),

76
ingestor/src/symbol-metadata-generator.js
View File

@@ -141,59 +141,50 @@ export class SymbolMetadataGenerator {
const precision = market.precision || {};
const limits = market.limits || {};
// Get tick_denom from price precision
// This tells us the denominator for price values.
// For example, if BTC/USDT trades with 2 decimals (0.01 precision), tick_denom = 100
// Derive Nautilus Instrument fields from CCXT market data
//
// CCXT precision.price can be:
// - Integer (decimal places): 2 means 0.01 tick size -> denominator 100
// - Float (tick size): 0.01 -> invert to get denominator 100
let tick_denom;
// - Integer (decimal places): 2 means 0.01 tick size -> price_precision=2, tick_size=0.01
// - Float (tick size): 0.01 -> tick_size=0.01, price_precision=2
let price_precision;
let tick_size;
if (precision.price !== undefined) {
if (Number.isInteger(precision.price)) {
// Integer: number of decimal places
// e.g., precision.price = 2 means 2 decimal places = 0.01 tick = 100 denom
tick_denom = Math.pow(10, precision.price);
price_precision = precision.price;
tick_size = Math.pow(10, -precision.price);
} else {
// Float: actual tick size, need to invert and round
// e.g., precision.price = 0.01 -> 1/0.01 = 100
tick_denom = Math.round(1 / precision.price);
tick_size = precision.price;
price_precision = Math.round(-Math.log10(precision.price));
}
} else if (limits.price?.min !== undefined) {
// Fallback: use minimum price as tick size
tick_denom = Math.round(1 / limits.price.min);
tick_size = limits.price.min;
price_precision = Math.round(-Math.log10(tick_size));
} else {
// Default to 2 decimals (pennies)
tick_denom = 100;
price_precision = 2;
tick_size = 0.01;
}
// Get base_denom from amount precision (for volumes)
let base_denom;
let size_precision;
let lot_size;
if (precision.amount !== undefined) {
if (Number.isInteger(precision.amount)) {
base_denom = Math.pow(10, precision.amount);
size_precision = precision.amount;
lot_size = Math.pow(10, -precision.amount);
} else {
base_denom = Math.round(1 / precision.amount);
lot_size = precision.amount;
size_precision = Math.round(-Math.log10(precision.amount));
}
} else if (limits.amount?.min !== undefined) {
base_denom = Math.round(1 / limits.amount.min);
lot_size = limits.amount.min;
size_precision = Math.round(-Math.log10(lot_size));
} else {
// Default to 8 decimals (standard for crypto)
base_denom = 100000000;
size_precision = 8;
lot_size = 0.00000001;
}
// Get quote_denom from cost precision (price * amount)
let quote_denom;
if (precision.cost !== undefined) {
if (Number.isInteger(precision.cost)) {
quote_denom = Math.pow(10, precision.cost);
} else {
quote_denom = Math.round(1 / precision.cost);
}
} else {
// Default: typically tick_denom for most exchanges
quote_denom = tick_denom;
}
const min_notional = limits.cost?.min || 0;
const maker_fee = market.maker !== undefined ? market.maker : 0.001;
const taker_fee = market.taker !== undefined ? market.taker : 0.001;
// Standard supported periods (in seconds)
// Most exchanges support these timeframes
@@ -218,9 +209,14 @@ export class SymbolMetadataGenerator {
description,
baseAsset: base,
quoteAsset: quote,
tickDenom: tick_denom,
baseDenom: base_denom,
quoteDenom: quote_denom,
pricePrecision: price_precision,
sizePrecision: size_precision,
tickSize: tick_size,
lotSize: lot_size,
minNotional: min_notional,
makerFee: maker_fee,
takerFee: taker_fee,
contractMultiplier: 1.0,
supportedPeriodSeconds: supported_period_seconds,
// earliestTime can be added later if we track it
};
@@ -238,7 +234,7 @@ export class SymbolMetadataGenerator {
let duplicateCount = 0;
for (const metadata of metadataList) {
const key = `${metadata.exchangeId}:${metadata.marketId}`;
const key = `${metadata.marketId}.${metadata.exchangeId}`;
// Debug first few to understand duplication
if (uniqueMetadata.length < 3 || (uniqueMetadata.length === 0 && duplicateCount < 3)) {
@@ -269,7 +265,7 @@ export class SymbolMetadataGenerator {
// Convert each metadata to protobuf Market message
const messages = uniqueMetadata.map(metadata => {
const key = `${metadata.exchangeId}:${metadata.marketId}`;
const key = `${metadata.marketId}.${metadata.exchangeId}`;
return {
key,

4
ingestor/src/zmq-client.js
View File

@@ -29,9 +29,9 @@ export class ZmqClient {
const workEndpoint = `tcp://${flink_hostname}:${ingestor_work_port}`;
await this.workSocket.connect(workEndpoint);
// Subscribe to each supported exchange prefix
// Subscribe to each supported exchange suffix (Nautilus format: "BTC/USDT.BINANCE")
for (const exchange of this.supportedExchanges) {
const prefix = `${exchange}:`;
const prefix = `${exchange}.`;
this.workSocket.subscribe(prefix);
this.logger.info(`Subscribed to exchange prefix: ${prefix}`);
}