310 lines
8.8 KiB
JavaScript
310 lines
8.8 KiB
JavaScript
// Symbol Metadata Generator - extracts market metadata from CCXT exchanges
|
|
import ccxt from 'ccxt';
|
|
|
|
export class SymbolMetadataGenerator {
|
|
constructor(config, kafkaProducer, logger) {
|
|
this.config = config;
|
|
this.kafkaProducer = kafkaProducer;
|
|
this.logger = logger;
|
|
this.exchanges = new Map();
|
|
this.lastGenerated = new Map(); // Track last generation time per exchange
|
|
this.publishedSymbols = new Set(); // Track published symbols to prevent duplicates
|
|
}
|
|
|
|
/**
|
|
* Get or create CCXT exchange instance
|
|
*/
|
|
getExchange(exchangeName) {
|
|
if (this.exchanges.has(exchangeName)) {
|
|
return this.exchanges.get(exchangeName);
|
|
}
|
|
|
|
const ExchangeClass = ccxt[exchangeName];
|
|
if (!ExchangeClass) {
|
|
throw new Error(`Unsupported exchange: ${exchangeName}`);
|
|
}
|
|
|
|
const exchange = new ExchangeClass({
|
|
enableRateLimit: true,
|
|
options: {
|
|
defaultType: 'spot'
|
|
}
|
|
});
|
|
|
|
this.exchanges.set(exchangeName, exchange);
|
|
this.logger.info({ exchange: exchangeName }, 'Created CCXT exchange instance for metadata');
|
|
|
|
return exchange;
|
|
}
|
|
|
|
/**
|
|
* Generate symbol metadata for all configured exchanges
|
|
*/
|
|
async generateAll() {
|
|
const supportedExchanges = this.config.supported_exchanges || ['binance'];
|
|
|
|
this.logger.info({ exchanges: supportedExchanges }, 'Generating symbol metadata for all exchanges');
|
|
|
|
const results = [];
|
|
for (const exchangeName of supportedExchanges) {
|
|
try {
|
|
const metadata = await this.generateForExchange(exchangeName);
|
|
results.push({ exchange: exchangeName, count: metadata.length, success: true });
|
|
} catch (error) {
|
|
this.logger.error(
|
|
{ error: error.message, exchange: exchangeName },
|
|
'Failed to generate metadata for exchange'
|
|
);
|
|
results.push({ exchange: exchangeName, error: error.message, success: false });
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/**
|
|
* Generate symbol metadata for a single exchange
|
|
*/
|
|
async generateForExchange(exchangeName) {
|
|
const exchangeLower = exchangeName.toLowerCase();
|
|
const exchangeUpper = exchangeName.toUpperCase();
|
|
|
|
this.logger.info({ exchange: exchangeUpper }, 'Loading markets from exchange');
|
|
|
|
const exchange = this.getExchange(exchangeLower);
|
|
|
|
// Load all markets from the exchange
|
|
const markets = await exchange.loadMarkets();
|
|
|
|
this.logger.info(
|
|
{ exchange: exchangeUpper, marketCount: Object.keys(markets).length },
|
|
'Loaded markets from exchange'
|
|
);
|
|
|
|
const metadataList = [];
|
|
|
|
for (const [symbol, market] of Object.entries(markets)) {
|
|
try {
|
|
const metadata = this.convertMarketToMetadata(exchangeUpper, symbol, market);
|
|
|
|
// Debug log first few symbols
|
|
if (metadataList.length < 5) {
|
|
this.logger.info({
|
|
exchange: exchangeUpper,
|
|
symbol,
|
|
metadata
|
|
}, `Symbol metadata ${metadataList.length + 1} generated`);
|
|
}
|
|
|
|
metadataList.push(metadata);
|
|
} catch (error) {
|
|
this.logger.warn(
|
|
{ exchange: exchangeUpper, symbol, error: error.message },
|
|
'Failed to convert market to metadata'
|
|
);
|
|
}
|
|
}
|
|
|
|
// Write all metadata to Kafka
|
|
if (metadataList.length > 0) {
|
|
try {
|
|
await this.writeMetadataToKafka(metadataList);
|
|
} catch (error) {
|
|
this.logger.error(
|
|
{ exchange: exchangeUpper, error: error.message, stack: error.stack },
|
|
'Failed to write metadata to Kafka'
|
|
);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
// Update last generated timestamp
|
|
this.lastGenerated.set(exchangeUpper, Date.now());
|
|
|
|
this.logger.info(
|
|
{ exchange: exchangeUpper, count: metadataList.length },
|
|
'Generated and published symbol metadata'
|
|
);
|
|
|
|
return metadataList;
|
|
}
|
|
|
|
/**
|
|
* Convert CCXT market object to our Market metadata format
|
|
*/
|
|
convertMarketToMetadata(exchangeId, symbol, market) {
|
|
const base = market.base || '';
|
|
const quote = market.quote || '';
|
|
const marketType = market.type || 'spot';
|
|
|
|
// Extract precision information
|
|
const precision = market.precision || {};
|
|
const limits = market.limits || {};
|
|
|
|
// Derive Nautilus Instrument fields from CCXT market data
|
|
//
|
|
// CCXT precision.price can be:
|
|
// - Integer (decimal places): 2 means 0.01 tick size -> price_precision=2, tick_size=0.01
|
|
// - Float (tick size): 0.01 -> tick_size=0.01, price_precision=2
|
|
let price_precision;
|
|
let tick_size;
|
|
if (precision.price !== undefined) {
|
|
if (Number.isInteger(precision.price)) {
|
|
price_precision = precision.price;
|
|
tick_size = Math.pow(10, -precision.price);
|
|
} else {
|
|
tick_size = precision.price;
|
|
price_precision = Math.round(-Math.log10(precision.price));
|
|
}
|
|
} else if (limits.price?.min !== undefined) {
|
|
tick_size = limits.price.min;
|
|
price_precision = Math.round(-Math.log10(tick_size));
|
|
} else {
|
|
price_precision = 2;
|
|
tick_size = 0.01;
|
|
}
|
|
|
|
let size_precision;
|
|
let lot_size;
|
|
if (precision.amount !== undefined) {
|
|
if (Number.isInteger(precision.amount)) {
|
|
size_precision = precision.amount;
|
|
lot_size = Math.pow(10, -precision.amount);
|
|
} else {
|
|
lot_size = precision.amount;
|
|
size_precision = Math.round(-Math.log10(precision.amount));
|
|
}
|
|
} else if (limits.amount?.min !== undefined) {
|
|
lot_size = limits.amount.min;
|
|
size_precision = Math.round(-Math.log10(lot_size));
|
|
} else {
|
|
size_precision = 8;
|
|
lot_size = 0.00000001;
|
|
}
|
|
|
|
const min_notional = limits.cost?.min || 0;
|
|
const maker_fee = market.maker !== undefined ? market.maker : 0.001;
|
|
const taker_fee = market.taker !== undefined ? market.taker : 0.001;
|
|
|
|
// Standard supported periods (in seconds)
|
|
// Most exchanges support these timeframes
|
|
const supported_period_seconds = [
|
|
60, // 1m
|
|
300, // 5m
|
|
900, // 15m
|
|
1800, // 30m
|
|
3600, // 1h
|
|
14400, // 4h
|
|
86400, // 1d
|
|
];
|
|
|
|
// Build description
|
|
const description = `${base}/${quote} ${marketType} trading pair on ${exchangeId}`;
|
|
|
|
// NOTE: protobufjs expects camelCase field names, not snake_case!
|
|
return {
|
|
exchangeId: exchangeId,
|
|
marketId: symbol,
|
|
marketType: marketType,
|
|
description,
|
|
baseAsset: base,
|
|
quoteAsset: quote,
|
|
pricePrecision: price_precision,
|
|
sizePrecision: size_precision,
|
|
tickSize: tick_size,
|
|
lotSize: lot_size,
|
|
minNotional: min_notional,
|
|
makerFee: maker_fee,
|
|
takerFee: taker_fee,
|
|
contractMultiplier: 1.0,
|
|
supportedPeriodSeconds: supported_period_seconds,
|
|
// earliestTime can be added later if we track it
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Write metadata list to Kafka topic
|
|
* Deduplicates symbols to prevent duplicate writes
|
|
*/
|
|
async writeMetadataToKafka(metadataList) {
|
|
const topic = 'symbol-metadata';
|
|
|
|
// Filter out duplicates
|
|
const uniqueMetadata = [];
|
|
let duplicateCount = 0;
|
|
|
|
for (const metadata of metadataList) {
|
|
const key = `${metadata.marketId}.${metadata.exchangeId}`;
|
|
|
|
// Debug first few to understand duplication
|
|
if (uniqueMetadata.length < 3 || (uniqueMetadata.length === 0 && duplicateCount < 3)) {
|
|
this.logger.info({
|
|
key,
|
|
exchangeId: metadata.exchangeId,
|
|
marketId: metadata.marketId,
|
|
isDuplicate: this.publishedSymbols.has(key),
|
|
setSize: this.publishedSymbols.size
|
|
}, 'Deduplication check');
|
|
}
|
|
|
|
if (!this.publishedSymbols.has(key)) {
|
|
uniqueMetadata.push(metadata);
|
|
this.publishedSymbols.add(key);
|
|
} else {
|
|
duplicateCount++;
|
|
}
|
|
}
|
|
|
|
if (uniqueMetadata.length === 0) {
|
|
this.logger.debug(
|
|
{ duplicateCount, topic },
|
|
'All symbols already published, skipping'
|
|
);
|
|
return;
|
|
}
|
|
|
|
// Convert each metadata to protobuf Market message
|
|
const messages = uniqueMetadata.map(metadata => {
|
|
const key = `${metadata.marketId}.${metadata.exchangeId}`;
|
|
|
|
return {
|
|
key,
|
|
value: metadata,
|
|
};
|
|
});
|
|
|
|
await this.kafkaProducer.writeMarketMetadata(topic, messages);
|
|
|
|
this.logger.info(
|
|
{ count: messages.length, duplicateCount, topic },
|
|
'Wrote symbol metadata to Kafka'
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get status of metadata generation
|
|
*/
|
|
getStatus() {
|
|
const status = {};
|
|
for (const [exchange, timestamp] of this.lastGenerated.entries()) {
|
|
status[exchange] = {
|
|
lastGenerated: new Date(timestamp).toISOString(),
|
|
ageMinutes: Math.floor((Date.now() - timestamp) / 60000),
|
|
};
|
|
}
|
|
return status;
|
|
}
|
|
|
|
/**
|
|
* Close all exchange connections
|
|
*/
|
|
async close() {
|
|
for (const [name, exchange] of this.exchanges) {
|
|
if (exchange.close) {
|
|
await exchange.close();
|
|
}
|
|
}
|
|
this.exchanges.clear();
|
|
}
|
|
}
|