dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chart data loading

Browse Source
This commit is contained in:
Tim Olson
2026-03-24 21:37:49 -04:00
parent f6bd22a8ef
commit c76887ab92
65 changed files with 6350 additions and 713 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
ingestor/src/ccxt-fetcher.js
View File

@@ -2,10 +2,12 @@
import ccxt from 'ccxt';
export class CCXTFetcher {
constructor(config, logger) {
constructor(config, logger, metadataGenerator = null) {
this.config = config;
this.logger = logger;
this.exchanges = new Map();
this.metadataGenerator = metadataGenerator;
this.metadataCache = new Map(); // Cache metadata by ticker
}
/**
@@ -24,6 +26,41 @@ export class CCXTFetcher {
};
}
/**
* Get metadata for a ticker (cached or generate on-the-fly)
*/
async getMetadata(ticker) {
// Check cache first
if (this.metadataCache.has(ticker)) {
return this.metadataCache.get(ticker);
}
// Generate metadata on-the-fly
if (!this.metadataGenerator) {
throw new Error('Metadata generator not available');
}
const { exchange: exchangeName, symbol } = this.parseTicker(ticker);
const exchangeUpper = exchangeName.toUpperCase();
const exchange = this.getExchange(exchangeName);
// Load market info from CCXT
await exchange.loadMarkets();
const market = exchange.market(symbol);
if (!market) {
throw new Error(`Market not found: ${symbol} on ${exchangeUpper}`);
}
// Convert to our metadata format
const metadata = this.metadataGenerator.convertMarketToMetadata(exchangeUpper, symbol, market);
// Cache it
this.metadataCache.set(ticker, metadata);
return metadata;
}
/**
* Get or create CCXT exchange instance
*/
@@ -123,8 +160,11 @@ export class CCXTFetcher {
}
}
// Get metadata for proper denomination
const metadata = await this.getMetadata(ticker);
// Convert to our OHLC format
return allCandles.map(candle => this.convertToOHLC(candle, ticker, periodSeconds));
return allCandles.map(candle => this.convertToOHLC(candle, ticker, periodSeconds, metadata));
}
/**
@@ -148,8 +188,11 @@ export class CCXTFetcher {
'Fetched recent trades'
);
// Get metadata for proper denomination
const metadata = await this.getMetadata(ticker);
// Convert to our Tick format
return trades.map(trade => this.convertToTick(trade, ticker));
return trades.map(trade => this.convertToTick(trade, ticker, metadata));
} catch (error) {
this.logger.error(
{ error: error.message, ticker },
@@ -162,21 +205,23 @@ export class CCXTFetcher {
/**
* Convert CCXT OHLCV array to our OHLC format
* CCXT format: [timestamp, open, high, low, close, volume]
* Uses denominators from market metadata for proper integer representation
*/
convertToOHLC(candle, ticker, periodSeconds) {
convertToOHLC(candle, ticker, periodSeconds, metadata) {
const [timestamp, open, high, low, close, volume] = candle;
// Convert to fixed-point integers (using 8 decimal places = 10^8)
const DENOM = 100000000;
// Use denominators from metadata
const tickDenom = metadata.tickDenom || 100;
const baseDenom = metadata.baseDenom || 100000000;
return {
ticker,
timestamp: (timestamp * 1000).toString(), // Convert ms to microseconds
open: Math.round(open * DENOM).toString(),
high: Math.round(high * DENOM).toString(),
low: Math.round(low * DENOM).toString(),
close: Math.round(close * DENOM).toString(),
volume: Math.round(volume * DENOM).toString(),
open: Math.round(open * tickDenom).toString(),
high: Math.round(high * tickDenom).toString(),
low: Math.round(low * tickDenom).toString(),
close: Math.round(close * tickDenom).toString(),
volume: Math.round(volume * baseDenom).toString(),
open_time: (timestamp * 1000).toString(),
close_time: ((timestamp + periodSeconds * 1000) * 1000).toString()
};
@@ -184,14 +229,17 @@ export class CCXTFetcher {
/**
* Convert CCXT trade to our Tick format
* Uses denominators from market metadata for proper integer representation
*/
convertToTick(trade, ticker) {
// Convert to fixed-point integers (using 8 decimal places = 10^8)
const DENOM = 100000000;
convertToTick(trade, ticker, metadata) {
// Use denominators from metadata
const tickDenom = metadata.tickDenom || 100;
const baseDenom = metadata.baseDenom || 100000000;
const quoteDenom = metadata.quoteDenom || tickDenom;
const price = Math.round(trade.price * DENOM);
const amount = Math.round(trade.amount * DENOM);
const quoteAmount = Math.round((trade.price * trade.amount) * DENOM);
const price = Math.round(trade.price * tickDenom);
const amount = Math.round(trade.amount * baseDenom);
const quoteAmount = Math.round((trade.price * trade.amount) * quoteDenom);
return {
trade_id: trade.id || `${trade.timestamp}`,

47
ingestor/src/index.js
View File

@@ -8,6 +8,7 @@ import { ZmqClient } from './zmq-client.js';
import { KafkaProducer } from './kafka-producer.js';
import { CCXTFetcher } from './ccxt-fetcher.js';
import { RealtimePoller } from './realtime-poller.js';
import { SymbolMetadataGenerator } from './symbol-metadata-generator.js';
// Logger setup
const logger = pino({
@@ -61,6 +62,10 @@ function loadConfig() {
max_concurrent: config.max_concurrent || 10,
poll_interval_ms: config.poll_interval_ms || 10000,
// Symbol metadata configuration
supported_exchanges: config.supported_exchanges || ['binance', 'coinbase', 'kraken'],
symbol_metadata_interval_ms: config.symbol_metadata_interval_ms || 6 * 60 * 60 * 1000, // 6 hours
...secrets
};
}
@@ -75,9 +80,16 @@ class IngestorWorker {
config,
logger.child({ component: 'kafka' })
);
// Create metadata generator first so ccxtFetcher can use it
this.metadataGenerator = new SymbolMetadataGenerator(
config,
this.kafkaProducer,
logger.child({ component: 'metadata' })
);
this.ccxtFetcher = new CCXTFetcher(
config,
logger.child({ component: 'ccxt' })
logger.child({ component: 'ccxt' }),
this.metadataGenerator
);
this.realtimePoller = new RealtimePoller(
this.ccxtFetcher,
@@ -88,6 +100,10 @@ class IngestorWorker {
// Track active requests
this.activeRequests = new Map();
this.isShutdown = false;
// Metadata generation interval
this.metadataIntervalMs = config.symbol_metadata_interval_ms;
this.metadataInterval = null;
}
/**
@@ -106,6 +122,26 @@ class IngestorWorker {
// Start work loop
this.workLoop();
// Generate symbol metadata on startup
this.logger.info('Generating initial symbol metadata');
try {
const results = await this.metadataGenerator.generateAll();
this.logger.info({ results }, 'Initial symbol metadata generated');
} catch (error) {
this.logger.error({ error: error.message }, 'Failed to generate initial symbol metadata');
}
// Schedule periodic metadata generation
this.metadataInterval = setInterval(async () => {
this.logger.info('Periodic symbol metadata generation');
try {
const results = await this.metadataGenerator.generateAll();
this.logger.info({ results }, 'Periodic symbol metadata generated');
} catch (error) {
this.logger.error({ error: error.message }, 'Failed to generate periodic symbol metadata');
}
}, this.metadataIntervalMs);
this.logger.info('Ingestor worker started successfully');
}
@@ -347,7 +383,8 @@ class IngestorWorker {
return {
activeRequests: this.activeRequests.size,
maxConcurrent: this.config.max_concurrent,
pollerStats: this.realtimePoller.getStats()
pollerStats: this.realtimePoller.getStats(),
metadataStatus: this.metadataGenerator.getStatus()
};
}
@@ -362,11 +399,17 @@ class IngestorWorker {
this.isShutdown = true;
this.logger.info('Shutting down ingestor worker');
// Stop metadata generation interval
if (this.metadataInterval) {
clearInterval(this.metadataInterval);
}
// Stop polling
this.realtimePoller.shutdown();
// Close connections
await this.ccxtFetcher.close();
await this.metadataGenerator.close();
await this.kafkaProducer.disconnect();
await this.zmqClient.shutdown();

37
ingestor/src/kafka-producer.js
View File

@@ -1,6 +1,6 @@
// Kafka producer for writing market data
import { Kafka } from 'kafkajs';
import { encodeMessage, MessageTypeId, Tick, OHLC, OHLCBatch } from './proto/messages.js';
import { encodeMessage, MessageTypeId, Tick, OHLC, OHLCBatch, Market } from './proto/messages.js';
export class KafkaProducer {
constructor(config, logger) {
@@ -257,6 +257,41 @@ export class KafkaProducer {
);
}
/**
* Write market metadata messages to Kafka
* @param {string} topic - Kafka topic name
* @param {Array<object>} messages - Array of {key, value} objects where value is Market metadata
*/
async writeMarketMetadata(topic, messages) {
if (!this.isConnected) {
throw new Error('Kafka producer not connected');
}
if (messages.length === 0) {
return;
}
const kafkaMessages = messages.map(({ key, value }) => {
const [frame1, frame2] = encodeMessage(MessageTypeId.MARKET, value, Market);
const encodedValue = Buffer.concat([frame1, frame2]);
return {
key,
value: encodedValue
};
});
await this.producer.send({
topic,
messages: kafkaMessages
});
this.logger.debug(
{ count: messages.length, topic },
'Wrote market metadata to Kafka'
);
}
/**
* Disconnect from Kafka
*/

313
ingestor/src/symbol-metadata-generator.js Normal file
View File

@@ -0,0 +1,313 @@
// Symbol Metadata Generator - extracts market metadata from CCXT exchanges
import ccxt from 'ccxt';
export class SymbolMetadataGenerator {
constructor(config, kafkaProducer, logger) {
this.config = config;
this.kafkaProducer = kafkaProducer;
this.logger = logger;
this.exchanges = new Map();
this.lastGenerated = new Map(); // Track last generation time per exchange
this.publishedSymbols = new Set(); // Track published symbols to prevent duplicates
}
/**
* Get or create CCXT exchange instance
*/
getExchange(exchangeName) {
if (this.exchanges.has(exchangeName)) {
return this.exchanges.get(exchangeName);
}
const ExchangeClass = ccxt[exchangeName];
if (!ExchangeClass) {
throw new Error(`Unsupported exchange: ${exchangeName}`);
}
const exchange = new ExchangeClass({
enableRateLimit: true,
options: {
defaultType: 'spot'
}
});
this.exchanges.set(exchangeName, exchange);
this.logger.info({ exchange: exchangeName }, 'Created CCXT exchange instance for metadata');
return exchange;
}
/**
* Generate symbol metadata for all configured exchanges
*/
async generateAll() {
const supportedExchanges = this.config.supported_exchanges || ['binance'];
this.logger.info({ exchanges: supportedExchanges }, 'Generating symbol metadata for all exchanges');
const results = [];
for (const exchangeName of supportedExchanges) {
try {
const metadata = await this.generateForExchange(exchangeName);
results.push({ exchange: exchangeName, count: metadata.length, success: true });
} catch (error) {
this.logger.error(
{ error: error.message, exchange: exchangeName },
'Failed to generate metadata for exchange'
);
results.push({ exchange: exchangeName, error: error.message, success: false });
}
}
return results;
}
/**
* Generate symbol metadata for a single exchange
*/
async generateForExchange(exchangeName) {
const exchangeLower = exchangeName.toLowerCase();
const exchangeUpper = exchangeName.toUpperCase();
this.logger.info({ exchange: exchangeUpper }, 'Loading markets from exchange');
const exchange = this.getExchange(exchangeLower);
// Load all markets from the exchange
const markets = await exchange.loadMarkets();
this.logger.info(
{ exchange: exchangeUpper, marketCount: Object.keys(markets).length },
'Loaded markets from exchange'
);
const metadataList = [];
for (const [symbol, market] of Object.entries(markets)) {
try {
const metadata = this.convertMarketToMetadata(exchangeUpper, symbol, market);
// Debug log first few symbols
if (metadataList.length < 5) {
this.logger.info({
exchange: exchangeUpper,
symbol,
metadata
}, `Symbol metadata ${metadataList.length + 1} generated`);
}
metadataList.push(metadata);
} catch (error) {
this.logger.warn(
{ exchange: exchangeUpper, symbol, error: error.message },
'Failed to convert market to metadata'
);
}
}
// Write all metadata to Kafka
if (metadataList.length > 0) {
try {
await this.writeMetadataToKafka(metadataList);
} catch (error) {
this.logger.error(
{ exchange: exchangeUpper, error: error.message, stack: error.stack },
'Failed to write metadata to Kafka'
);
throw error;
}
}
// Update last generated timestamp
this.lastGenerated.set(exchangeUpper, Date.now());
this.logger.info(
{ exchange: exchangeUpper, count: metadataList.length },
'Generated and published symbol metadata'
);
return metadataList;
}
/**
* Convert CCXT market object to our Market metadata format
*/
convertMarketToMetadata(exchangeId, symbol, market) {
const base = market.base || '';
const quote = market.quote || '';
const marketType = market.type || 'spot';
// Extract precision information
const precision = market.precision || {};
const limits = market.limits || {};
// Get tick_denom from price precision
// This tells us the denominator for price values.
// For example, if BTC/USDT trades with 2 decimals (0.01 precision), tick_denom = 100
//
// CCXT precision.price can be:
// - Integer (decimal places): 2 means 0.01 tick size -> denominator 100
// - Float (tick size): 0.01 -> invert to get denominator 100
let tick_denom;
if (precision.price !== undefined) {
if (Number.isInteger(precision.price)) {
// Integer: number of decimal places
// e.g., precision.price = 2 means 2 decimal places = 0.01 tick = 100 denom
tick_denom = Math.pow(10, precision.price);
} else {
// Float: actual tick size, need to invert and round
// e.g., precision.price = 0.01 -> 1/0.01 = 100
tick_denom = Math.round(1 / precision.price);
}
} else if (limits.price?.min !== undefined) {
// Fallback: use minimum price as tick size
tick_denom = Math.round(1 / limits.price.min);
} else {
// Default to 2 decimals (pennies)
tick_denom = 100;
}
// Get base_denom from amount precision (for volumes)
let base_denom;
if (precision.amount !== undefined) {
if (Number.isInteger(precision.amount)) {
base_denom = Math.pow(10, precision.amount);
} else {
base_denom = Math.round(1 / precision.amount);
}
} else if (limits.amount?.min !== undefined) {
base_denom = Math.round(1 / limits.amount.min);
} else {
// Default to 8 decimals (standard for crypto)
base_denom = 100000000;
}
// Get quote_denom from cost precision (price * amount)
let quote_denom;
if (precision.cost !== undefined) {
if (Number.isInteger(precision.cost)) {
quote_denom = Math.pow(10, precision.cost);
} else {
quote_denom = Math.round(1 / precision.cost);
}
} else {
// Default: typically tick_denom for most exchanges
quote_denom = tick_denom;
}
// Standard supported periods (in seconds)
// Most exchanges support these timeframes
const supported_period_seconds = [
60, // 1m
300, // 5m
900, // 15m
1800, // 30m
3600, // 1h
14400, // 4h
86400, // 1d
];
// Build description
const description = `${base}/${quote} ${marketType} trading pair on ${exchangeId}`;
// NOTE: protobufjs expects camelCase field names, not snake_case!
return {
exchangeId: exchangeId,
marketId: symbol,
marketType: marketType,
description,
baseAsset: base,
quoteAsset: quote,
tickDenom: tick_denom,
baseDenom: base_denom,
quoteDenom: quote_denom,
supportedPeriodSeconds: supported_period_seconds,
// earliestTime can be added later if we track it
};
}
/**
* Write metadata list to Kafka topic
* Deduplicates symbols to prevent duplicate writes
*/
async writeMetadataToKafka(metadataList) {
const topic = 'symbol-metadata';
// Filter out duplicates
const uniqueMetadata = [];
let duplicateCount = 0;
for (const metadata of metadataList) {
const key = `${metadata.exchangeId}:${metadata.marketId}`;
// Debug first few to understand duplication
if (uniqueMetadata.length < 3 || (uniqueMetadata.length === 0 && duplicateCount < 3)) {
this.logger.info({
key,
exchangeId: metadata.exchangeId,
marketId: metadata.marketId,
isDuplicate: this.publishedSymbols.has(key),
setSize: this.publishedSymbols.size
}, 'Deduplication check');
}
if (!this.publishedSymbols.has(key)) {
uniqueMetadata.push(metadata);
this.publishedSymbols.add(key);
} else {
duplicateCount++;
}
}
if (uniqueMetadata.length === 0) {
this.logger.debug(
{ duplicateCount, topic },
'All symbols already published, skipping'
);
return;
}
// Convert each metadata to protobuf Market message
const messages = uniqueMetadata.map(metadata => {
const key = `${metadata.exchangeId}:${metadata.marketId}`;
return {
key,
value: metadata,
};
});
await this.kafkaProducer.writeMarketMetadata(topic, messages);
this.logger.info(
{ count: messages.length, duplicateCount, topic },
'Wrote symbol metadata to Kafka'
);
}
/**
* Get status of metadata generation
*/
getStatus() {
const status = {};
for (const [exchange, timestamp] of this.lastGenerated.entries()) {
status[exchange] = {
lastGenerated: new Date(timestamp).toISOString(),
ageMinutes: Math.floor((Date.now() - timestamp) / 60000),
};
}
return status;
}
/**
* Close all exchange connections
*/
async close() {
for (const [name, exchange] of this.exchanges) {
if (exchange.close) {
await exchange.close();
}
}
this.exchanges.clear();
}
}