// Symbol Metadata Generator - extracts market metadata from CCXT exchanges import ccxt from 'ccxt'; export class SymbolMetadataGenerator { constructor(config, kafkaProducer, logger) { this.config = config; this.kafkaProducer = kafkaProducer; this.logger = logger; this.exchanges = new Map(); this.lastGenerated = new Map(); // Track last generation time per exchange this.publishedSymbols = new Set(); // Track published symbols to prevent duplicates } /** * Get or create CCXT exchange instance */ getExchange(exchangeName) { if (this.exchanges.has(exchangeName)) { return this.exchanges.get(exchangeName); } const ExchangeClass = ccxt[exchangeName]; if (!ExchangeClass) { throw new Error(`Unsupported exchange: ${exchangeName}`); } const exchange = new ExchangeClass({ enableRateLimit: true, options: { defaultType: 'spot' } }); this.exchanges.set(exchangeName, exchange); this.logger.info({ exchange: exchangeName }, 'Created CCXT exchange instance for metadata'); return exchange; } /** * Generate symbol metadata for all configured exchanges */ async generateAll() { const supportedExchanges = this.config.supported_exchanges || ['binance']; this.logger.info({ exchanges: supportedExchanges }, 'Generating symbol metadata for all exchanges'); const results = []; for (const exchangeName of supportedExchanges) { try { const metadata = await this.generateForExchange(exchangeName); results.push({ exchange: exchangeName, count: metadata.length, success: true }); } catch (error) { this.logger.error( { error: error.message, exchange: exchangeName }, 'Failed to generate metadata for exchange' ); results.push({ exchange: exchangeName, error: error.message, success: false }); } } return results; } /** * Generate symbol metadata for a single exchange */ async generateForExchange(exchangeName) { const exchangeLower = exchangeName.toLowerCase(); const exchangeUpper = exchangeName.toUpperCase(); this.logger.info({ exchange: exchangeUpper }, 'Loading markets from exchange'); const exchange = this.getExchange(exchangeLower); // Load all markets from the exchange const markets = await exchange.loadMarkets(); this.logger.info( { exchange: exchangeUpper, marketCount: Object.keys(markets).length }, 'Loaded markets from exchange' ); const metadataList = []; for (const [symbol, market] of Object.entries(markets)) { try { const metadata = this.convertMarketToMetadata(exchangeUpper, symbol, market); // Debug log first few symbols if (metadataList.length < 5) { this.logger.info({ exchange: exchangeUpper, symbol, metadata }, `Symbol metadata ${metadataList.length + 1} generated`); } metadataList.push(metadata); } catch (error) { this.logger.warn( { exchange: exchangeUpper, symbol, error: error.message }, 'Failed to convert market to metadata' ); } } // Write all metadata to Kafka if (metadataList.length > 0) { try { await this.writeMetadataToKafka(metadataList); } catch (error) { this.logger.error( { exchange: exchangeUpper, error: error.message, stack: error.stack }, 'Failed to write metadata to Kafka' ); throw error; } } // Update last generated timestamp this.lastGenerated.set(exchangeUpper, Date.now()); this.logger.info( { exchange: exchangeUpper, count: metadataList.length }, 'Generated and published symbol metadata' ); return metadataList; } /** * Convert CCXT market object to our Market metadata format */ convertMarketToMetadata(exchangeId, symbol, market) { const base = market.base || ''; const quote = market.quote || ''; const marketType = market.type || 'spot'; // Extract precision information const precision = market.precision || {}; const limits = market.limits || {}; // Derive Nautilus Instrument fields from CCXT market data // // CCXT precision.price can be: // - Integer (decimal places): 2 means 0.01 tick size -> price_precision=2, tick_size=0.01 // - Float (tick size): 0.01 -> tick_size=0.01, price_precision=2 let price_precision; let tick_size; if (precision.price !== undefined) { if (Number.isInteger(precision.price)) { price_precision = precision.price; tick_size = Math.pow(10, -precision.price); } else { tick_size = precision.price; price_precision = Math.round(-Math.log10(precision.price)); } } else if (limits.price?.min !== undefined) { tick_size = limits.price.min; price_precision = Math.round(-Math.log10(tick_size)); } else { price_precision = 2; tick_size = 0.01; } let size_precision; let lot_size; if (precision.amount !== undefined) { if (Number.isInteger(precision.amount)) { size_precision = precision.amount; lot_size = Math.pow(10, -precision.amount); } else { lot_size = precision.amount; size_precision = Math.round(-Math.log10(precision.amount)); } } else if (limits.amount?.min !== undefined) { lot_size = limits.amount.min; size_precision = Math.round(-Math.log10(lot_size)); } else { size_precision = 8; lot_size = 0.00000001; } const min_notional = limits.cost?.min || 0; const maker_fee = market.maker !== undefined ? market.maker : 0.001; const taker_fee = market.taker !== undefined ? market.taker : 0.001; // Standard supported periods (in seconds) // Most exchanges support these timeframes const supported_period_seconds = [ 60, // 1m 300, // 5m 900, // 15m 1800, // 30m 3600, // 1h 14400, // 4h 86400, // 1d ]; // Build description const description = `${base}/${quote} ${marketType} trading pair on ${exchangeId}`; // NOTE: protobufjs expects camelCase field names, not snake_case! return { exchangeId: exchangeId, marketId: symbol, marketType: marketType, description, baseAsset: base, quoteAsset: quote, pricePrecision: price_precision, sizePrecision: size_precision, tickSize: tick_size, lotSize: lot_size, minNotional: min_notional, makerFee: maker_fee, takerFee: taker_fee, contractMultiplier: 1.0, supportedPeriodSeconds: supported_period_seconds, // earliestTime can be added later if we track it }; } /** * Write metadata list to Kafka topic * Deduplicates symbols to prevent duplicate writes */ async writeMetadataToKafka(metadataList) { const topic = 'symbol-metadata'; // Filter out duplicates const uniqueMetadata = []; let duplicateCount = 0; for (const metadata of metadataList) { const key = `${metadata.marketId}.${metadata.exchangeId}`; // Debug first few to understand duplication if (uniqueMetadata.length < 3 || (uniqueMetadata.length === 0 && duplicateCount < 3)) { this.logger.info({ key, exchangeId: metadata.exchangeId, marketId: metadata.marketId, isDuplicate: this.publishedSymbols.has(key), setSize: this.publishedSymbols.size }, 'Deduplication check'); } if (!this.publishedSymbols.has(key)) { uniqueMetadata.push(metadata); this.publishedSymbols.add(key); } else { duplicateCount++; } } if (uniqueMetadata.length === 0) { this.logger.debug( { duplicateCount, topic }, 'All symbols already published, skipping' ); return; } // Convert each metadata to protobuf Market message const messages = uniqueMetadata.map(metadata => { const key = `${metadata.marketId}.${metadata.exchangeId}`; return { key, value: metadata, }; }); await this.kafkaProducer.writeMarketMetadata(topic, messages); this.logger.info( { count: messages.length, duplicateCount, topic }, 'Wrote symbol metadata to Kafka' ); } /** * Get status of metadata generation */ getStatus() { const status = {}; for (const [exchange, timestamp] of this.lastGenerated.entries()) { status[exchange] = { lastGenerated: new Date(timestamp).toISOString(), ageMinutes: Math.floor((Date.now() - timestamp) / 60000), }; } return status; } /** * Close all exchange connections */ async close() { for (const [name, exchange] of this.exchanges) { if (exchange.close) { await exchange.close(); } } this.exchanges.clear(); } }