chart data loading

ingestor/src/ccxt-fetcher.js

@@ -2,10 +2,12 @@
 import ccxt from 'ccxt';
 
 export class CCXTFetcher {
-  constructor(config, logger) {
+  constructor(config, logger, metadataGenerator = null) {
     this.config = config;
     this.logger = logger;
     this.exchanges = new Map();
+    this.metadataGenerator = metadataGenerator;
+    this.metadataCache = new Map(); // Cache metadata by ticker
   }
 
   /**
@@ -24,6 +26,41 @@ export class CCXTFetcher {
     };
   }
 
+  /**
+   * Get metadata for a ticker (cached or generate on-the-fly)
+   */
+  async getMetadata(ticker) {
+    // Check cache first
+    if (this.metadataCache.has(ticker)) {
+      return this.metadataCache.get(ticker);
+    }
+
+    // Generate metadata on-the-fly
+    if (!this.metadataGenerator) {
+      throw new Error('Metadata generator not available');
+    }
+
+    const { exchange: exchangeName, symbol } = this.parseTicker(ticker);
+    const exchangeUpper = exchangeName.toUpperCase();
+    const exchange = this.getExchange(exchangeName);
+
+    // Load market info from CCXT
+    await exchange.loadMarkets();
+    const market = exchange.market(symbol);
+
+    if (!market) {
+      throw new Error(`Market not found: ${symbol} on ${exchangeUpper}`);
+    }
+
+    // Convert to our metadata format
+    const metadata = this.metadataGenerator.convertMarketToMetadata(exchangeUpper, symbol, market);
+
+    // Cache it
+    this.metadataCache.set(ticker, metadata);
+
+    return metadata;
+  }
+
   /**
    * Get or create CCXT exchange instance
    */
@@ -123,8 +160,11 @@ export class CCXTFetcher {
       }
     }
 
+    // Get metadata for proper denomination
+    const metadata = await this.getMetadata(ticker);
+
     // Convert to our OHLC format
-    return allCandles.map(candle => this.convertToOHLC(candle, ticker, periodSeconds));
+    return allCandles.map(candle => this.convertToOHLC(candle, ticker, periodSeconds, metadata));
   }
 
   /**
@@ -148,8 +188,11 @@ export class CCXTFetcher {
         'Fetched recent trades'
       );
 
+      // Get metadata for proper denomination
+      const metadata = await this.getMetadata(ticker);
+
       // Convert to our Tick format
-      return trades.map(trade => this.convertToTick(trade, ticker));
+      return trades.map(trade => this.convertToTick(trade, ticker, metadata));
     } catch (error) {
       this.logger.error(
         { error: error.message, ticker },
@@ -162,21 +205,23 @@ export class CCXTFetcher {
   /**
    * Convert CCXT OHLCV array to our OHLC format
    * CCXT format: [timestamp, open, high, low, close, volume]
+   * Uses denominators from market metadata for proper integer representation
    */
-  convertToOHLC(candle, ticker, periodSeconds) {
+  convertToOHLC(candle, ticker, periodSeconds, metadata) {
     const [timestamp, open, high, low, close, volume] = candle;
 
-    // Convert to fixed-point integers (using 8 decimal places = 10^8)
-    const DENOM = 100000000;
+    // Use denominators from metadata
+    const tickDenom = metadata.tickDenom || 100;
+    const baseDenom = metadata.baseDenom || 100000000;
 
     return {
       ticker,
       timestamp: (timestamp * 1000).toString(), // Convert ms to microseconds
-      open: Math.round(open * DENOM).toString(),
-      high: Math.round(high * DENOM).toString(),
-      low: Math.round(low * DENOM).toString(),
-      close: Math.round(close * DENOM).toString(),
-      volume: Math.round(volume * DENOM).toString(),
+      open: Math.round(open * tickDenom).toString(),
+      high: Math.round(high * tickDenom).toString(),
+      low: Math.round(low * tickDenom).toString(),
+      close: Math.round(close * tickDenom).toString(),
+      volume: Math.round(volume * baseDenom).toString(),
       open_time: (timestamp * 1000).toString(),
       close_time: ((timestamp + periodSeconds * 1000) * 1000).toString()
     };
@@ -184,14 +229,17 @@ export class CCXTFetcher {
 
   /**
    * Convert CCXT trade to our Tick format
+   * Uses denominators from market metadata for proper integer representation
    */
-  convertToTick(trade, ticker) {
-    // Convert to fixed-point integers (using 8 decimal places = 10^8)
-    const DENOM = 100000000;
+  convertToTick(trade, ticker, metadata) {
+    // Use denominators from metadata
+    const tickDenom = metadata.tickDenom || 100;
+    const baseDenom = metadata.baseDenom || 100000000;
+    const quoteDenom = metadata.quoteDenom || tickDenom;
 
-    const price = Math.round(trade.price * DENOM);
-    const amount = Math.round(trade.amount * DENOM);
-    const quoteAmount = Math.round((trade.price * trade.amount) * DENOM);
+    const price = Math.round(trade.price * tickDenom);
+    const amount = Math.round(trade.amount * baseDenom);
+    const quoteAmount = Math.round((trade.price * trade.amount) * quoteDenom);
 
     return {
       trade_id: trade.id || `${trade.timestamp}`,