chart data loading
This commit is contained in:
313
ingestor/src/symbol-metadata-generator.js
Normal file
313
ingestor/src/symbol-metadata-generator.js
Normal file
@@ -0,0 +1,313 @@
|
||||
// Symbol Metadata Generator - extracts market metadata from CCXT exchanges
|
||||
import ccxt from 'ccxt';
|
||||
|
||||
export class SymbolMetadataGenerator {
|
||||
constructor(config, kafkaProducer, logger) {
|
||||
this.config = config;
|
||||
this.kafkaProducer = kafkaProducer;
|
||||
this.logger = logger;
|
||||
this.exchanges = new Map();
|
||||
this.lastGenerated = new Map(); // Track last generation time per exchange
|
||||
this.publishedSymbols = new Set(); // Track published symbols to prevent duplicates
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create CCXT exchange instance
|
||||
*/
|
||||
getExchange(exchangeName) {
|
||||
if (this.exchanges.has(exchangeName)) {
|
||||
return this.exchanges.get(exchangeName);
|
||||
}
|
||||
|
||||
const ExchangeClass = ccxt[exchangeName];
|
||||
if (!ExchangeClass) {
|
||||
throw new Error(`Unsupported exchange: ${exchangeName}`);
|
||||
}
|
||||
|
||||
const exchange = new ExchangeClass({
|
||||
enableRateLimit: true,
|
||||
options: {
|
||||
defaultType: 'spot'
|
||||
}
|
||||
});
|
||||
|
||||
this.exchanges.set(exchangeName, exchange);
|
||||
this.logger.info({ exchange: exchangeName }, 'Created CCXT exchange instance for metadata');
|
||||
|
||||
return exchange;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate symbol metadata for all configured exchanges
|
||||
*/
|
||||
async generateAll() {
|
||||
const supportedExchanges = this.config.supported_exchanges || ['binance'];
|
||||
|
||||
this.logger.info({ exchanges: supportedExchanges }, 'Generating symbol metadata for all exchanges');
|
||||
|
||||
const results = [];
|
||||
for (const exchangeName of supportedExchanges) {
|
||||
try {
|
||||
const metadata = await this.generateForExchange(exchangeName);
|
||||
results.push({ exchange: exchangeName, count: metadata.length, success: true });
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ error: error.message, exchange: exchangeName },
|
||||
'Failed to generate metadata for exchange'
|
||||
);
|
||||
results.push({ exchange: exchangeName, error: error.message, success: false });
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate symbol metadata for a single exchange
|
||||
*/
|
||||
async generateForExchange(exchangeName) {
|
||||
const exchangeLower = exchangeName.toLowerCase();
|
||||
const exchangeUpper = exchangeName.toUpperCase();
|
||||
|
||||
this.logger.info({ exchange: exchangeUpper }, 'Loading markets from exchange');
|
||||
|
||||
const exchange = this.getExchange(exchangeLower);
|
||||
|
||||
// Load all markets from the exchange
|
||||
const markets = await exchange.loadMarkets();
|
||||
|
||||
this.logger.info(
|
||||
{ exchange: exchangeUpper, marketCount: Object.keys(markets).length },
|
||||
'Loaded markets from exchange'
|
||||
);
|
||||
|
||||
const metadataList = [];
|
||||
|
||||
for (const [symbol, market] of Object.entries(markets)) {
|
||||
try {
|
||||
const metadata = this.convertMarketToMetadata(exchangeUpper, symbol, market);
|
||||
|
||||
// Debug log first few symbols
|
||||
if (metadataList.length < 5) {
|
||||
this.logger.info({
|
||||
exchange: exchangeUpper,
|
||||
symbol,
|
||||
metadata
|
||||
}, `Symbol metadata ${metadataList.length + 1} generated`);
|
||||
}
|
||||
|
||||
metadataList.push(metadata);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
{ exchange: exchangeUpper, symbol, error: error.message },
|
||||
'Failed to convert market to metadata'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Write all metadata to Kafka
|
||||
if (metadataList.length > 0) {
|
||||
try {
|
||||
await this.writeMetadataToKafka(metadataList);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ exchange: exchangeUpper, error: error.message, stack: error.stack },
|
||||
'Failed to write metadata to Kafka'
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Update last generated timestamp
|
||||
this.lastGenerated.set(exchangeUpper, Date.now());
|
||||
|
||||
this.logger.info(
|
||||
{ exchange: exchangeUpper, count: metadataList.length },
|
||||
'Generated and published symbol metadata'
|
||||
);
|
||||
|
||||
return metadataList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert CCXT market object to our Market metadata format
|
||||
*/
|
||||
convertMarketToMetadata(exchangeId, symbol, market) {
|
||||
const base = market.base || '';
|
||||
const quote = market.quote || '';
|
||||
const marketType = market.type || 'spot';
|
||||
|
||||
// Extract precision information
|
||||
const precision = market.precision || {};
|
||||
const limits = market.limits || {};
|
||||
|
||||
// Get tick_denom from price precision
|
||||
// This tells us the denominator for price values.
|
||||
// For example, if BTC/USDT trades with 2 decimals (0.01 precision), tick_denom = 100
|
||||
//
|
||||
// CCXT precision.price can be:
|
||||
// - Integer (decimal places): 2 means 0.01 tick size -> denominator 100
|
||||
// - Float (tick size): 0.01 -> invert to get denominator 100
|
||||
let tick_denom;
|
||||
if (precision.price !== undefined) {
|
||||
if (Number.isInteger(precision.price)) {
|
||||
// Integer: number of decimal places
|
||||
// e.g., precision.price = 2 means 2 decimal places = 0.01 tick = 100 denom
|
||||
tick_denom = Math.pow(10, precision.price);
|
||||
} else {
|
||||
// Float: actual tick size, need to invert and round
|
||||
// e.g., precision.price = 0.01 -> 1/0.01 = 100
|
||||
tick_denom = Math.round(1 / precision.price);
|
||||
}
|
||||
} else if (limits.price?.min !== undefined) {
|
||||
// Fallback: use minimum price as tick size
|
||||
tick_denom = Math.round(1 / limits.price.min);
|
||||
} else {
|
||||
// Default to 2 decimals (pennies)
|
||||
tick_denom = 100;
|
||||
}
|
||||
|
||||
// Get base_denom from amount precision (for volumes)
|
||||
let base_denom;
|
||||
if (precision.amount !== undefined) {
|
||||
if (Number.isInteger(precision.amount)) {
|
||||
base_denom = Math.pow(10, precision.amount);
|
||||
} else {
|
||||
base_denom = Math.round(1 / precision.amount);
|
||||
}
|
||||
} else if (limits.amount?.min !== undefined) {
|
||||
base_denom = Math.round(1 / limits.amount.min);
|
||||
} else {
|
||||
// Default to 8 decimals (standard for crypto)
|
||||
base_denom = 100000000;
|
||||
}
|
||||
|
||||
// Get quote_denom from cost precision (price * amount)
|
||||
let quote_denom;
|
||||
if (precision.cost !== undefined) {
|
||||
if (Number.isInteger(precision.cost)) {
|
||||
quote_denom = Math.pow(10, precision.cost);
|
||||
} else {
|
||||
quote_denom = Math.round(1 / precision.cost);
|
||||
}
|
||||
} else {
|
||||
// Default: typically tick_denom for most exchanges
|
||||
quote_denom = tick_denom;
|
||||
}
|
||||
|
||||
// Standard supported periods (in seconds)
|
||||
// Most exchanges support these timeframes
|
||||
const supported_period_seconds = [
|
||||
60, // 1m
|
||||
300, // 5m
|
||||
900, // 15m
|
||||
1800, // 30m
|
||||
3600, // 1h
|
||||
14400, // 4h
|
||||
86400, // 1d
|
||||
];
|
||||
|
||||
// Build description
|
||||
const description = `${base}/${quote} ${marketType} trading pair on ${exchangeId}`;
|
||||
|
||||
// NOTE: protobufjs expects camelCase field names, not snake_case!
|
||||
return {
|
||||
exchangeId: exchangeId,
|
||||
marketId: symbol,
|
||||
marketType: marketType,
|
||||
description,
|
||||
baseAsset: base,
|
||||
quoteAsset: quote,
|
||||
tickDenom: tick_denom,
|
||||
baseDenom: base_denom,
|
||||
quoteDenom: quote_denom,
|
||||
supportedPeriodSeconds: supported_period_seconds,
|
||||
// earliestTime can be added later if we track it
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Write metadata list to Kafka topic
|
||||
* Deduplicates symbols to prevent duplicate writes
|
||||
*/
|
||||
async writeMetadataToKafka(metadataList) {
|
||||
const topic = 'symbol-metadata';
|
||||
|
||||
// Filter out duplicates
|
||||
const uniqueMetadata = [];
|
||||
let duplicateCount = 0;
|
||||
|
||||
for (const metadata of metadataList) {
|
||||
const key = `${metadata.exchangeId}:${metadata.marketId}`;
|
||||
|
||||
// Debug first few to understand duplication
|
||||
if (uniqueMetadata.length < 3 || (uniqueMetadata.length === 0 && duplicateCount < 3)) {
|
||||
this.logger.info({
|
||||
key,
|
||||
exchangeId: metadata.exchangeId,
|
||||
marketId: metadata.marketId,
|
||||
isDuplicate: this.publishedSymbols.has(key),
|
||||
setSize: this.publishedSymbols.size
|
||||
}, 'Deduplication check');
|
||||
}
|
||||
|
||||
if (!this.publishedSymbols.has(key)) {
|
||||
uniqueMetadata.push(metadata);
|
||||
this.publishedSymbols.add(key);
|
||||
} else {
|
||||
duplicateCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (uniqueMetadata.length === 0) {
|
||||
this.logger.debug(
|
||||
{ duplicateCount, topic },
|
||||
'All symbols already published, skipping'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert each metadata to protobuf Market message
|
||||
const messages = uniqueMetadata.map(metadata => {
|
||||
const key = `${metadata.exchangeId}:${metadata.marketId}`;
|
||||
|
||||
return {
|
||||
key,
|
||||
value: metadata,
|
||||
};
|
||||
});
|
||||
|
||||
await this.kafkaProducer.writeMarketMetadata(topic, messages);
|
||||
|
||||
this.logger.info(
|
||||
{ count: messages.length, duplicateCount, topic },
|
||||
'Wrote symbol metadata to Kafka'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get status of metadata generation
|
||||
*/
|
||||
getStatus() {
|
||||
const status = {};
|
||||
for (const [exchange, timestamp] of this.lastGenerated.entries()) {
|
||||
status[exchange] = {
|
||||
lastGenerated: new Date(timestamp).toISOString(),
|
||||
ageMinutes: Math.floor((Date.now() - timestamp) / 60000),
|
||||
};
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close all exchange connections
|
||||
*/
|
||||
async close() {
|
||||
for (const [name, exchange] of this.exchanges) {
|
||||
if (exchange.close) {
|
||||
await exchange.close();
|
||||
}
|
||||
}
|
||||
this.exchanges.clear();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user