Add Ticker24h support: hourly market snapshots with USD-normalized volume filtering

ingestor/src/ccxt-fetcher.js

@@ -462,6 +462,80 @@ export class CCXTFetcher {
     return timeframe;
   }
 
+  /**
+   * Fetch 24h rolling ticker stats for all symbols on an exchange.
+   * Uses exchange.fetchTickers() — single API call, very rate-limit efficient.
+   * Returns an array of TickerStats-compatible objects, or throws if unsupported.
+   *
+   * @param {string} exchangeName - lowercase exchange name (e.g. "binance")
+   * @returns {Promise<Array>} Array of TickerStats objects
+   */
+  async fetchAllTickers(exchangeName) {
+    const exchange = this.getExchange(exchangeName);
+    const exchangeUpper = exchangeName.toUpperCase();
+
+    if (!exchange.has['fetchTickers']) {
+      throw new Error(`Exchange ${exchangeUpper} does not support fetchTickers()`);
+    }
+
+    this.logger.info({ exchange: exchangeUpper }, 'Fetching all 24h tickers');
+
+    let rawTickers;
+    try {
+      rawTickers = await exchange.fetchTickers();
+    } catch (error) {
+      if (error.constructor?.name === 'RateLimitExceeded') {
+        const retryAfterMs = extractRetryAfterMs(exchange, error);
+        this.logger.warn({ exchange: exchangeUpper, retryAfterMs }, 'fetchTickers rate-limited');
+        throw new ExchangeRateLimitError(exchangeName, retryAfterMs, error.message);
+      }
+      this.logger.error({ error: error.message, exchange: exchangeUpper }, 'Error fetching tickers');
+      throw error;
+    }
+
+    const nowNs = (BigInt(Date.now()) * 1_000_000n).toString();
+    const tickers = [];
+
+    for (const [symbol, t] of Object.entries(rawTickers)) {
+      if (!t || t.last == null) continue;
+
+      // Build Nautilus-format ticker: "BASE/QUOTE.EXCHANGE"
+      const ticker = `${symbol}.${exchangeUpper}`;
+
+      // Extract base/quote from the CCXT market info
+      const market = exchange.markets?.[symbol];
+      const baseAsset = market?.base ?? symbol.split('/')[0] ?? '';
+      const quoteAsset = market?.quote ?? symbol.split('/')[1] ?? '';
+
+      // protobufjs camelCase: only removes '_' before LETTERS, not digits.
+      // quote_volume_24h → quoteVolume_24h  (underscore before '24' is preserved)
+      // open_24h → open_24h, high_24h → high_24h, etc.
+      const stat = {
+        ticker,
+        exchangeId: exchangeUpper,
+        baseAsset,
+        quoteAsset,
+        lastPrice: t.last ?? 0,
+        priceChangePct: t.percentage ?? 0,
+        'quoteVolume_24h': t.quoteVolume ?? 0,
+        timestamp: nowNs,
+      };
+
+      if (t.bid != null) stat.bidPrice = t.bid;
+      if (t.ask != null) stat.askPrice = t.ask;
+      if (t.open != null) stat['open_24h'] = t.open;
+      if (t.high != null) stat['high_24h'] = t.high;
+      if (t.low != null) stat['low_24h'] = t.low;
+      if (t.baseVolume != null) stat['volume_24h'] = t.baseVolume;
+      if (t.info?.count != null) stat.numTrades = Number(t.info.count);
+
+      tickers.push(stat);
+    }
+
+    this.logger.info({ exchange: exchangeUpper, count: tickers.length }, 'Fetched all tickers');
+    return tickers;
+  }
+
   /**
    * Close all exchange connections
    */

ingestor/src/index.js

@@ -57,6 +57,7 @@ function loadConfig() {
     kafka_brokers: config.kafka_brokers || ['localhost:9092'],
     kafka_ohlc_topic: config.kafka_ohlc_topic || 'market-ohlc',
     kafka_tick_topic: config.kafka_tick_topic || 'market-tick',
+    kafka_ticker_topic: config.kafka_ticker_topic || 'market-ticker',
 
     // Worker configuration
     poll_interval_ms: config.poll_interval_ms || 10000,
@@ -316,6 +317,7 @@ class IngestorWorker {
 
     const isHistorical = !type || type === 'HISTORICAL_OHLC' || type === 0;
     const isRealtime = type === 'REALTIME_TICKS' || type === 1;
+    const isTickerSnapshot = type === 'TICKER_SNAPSHOT' || type === 2;
 
     if (isHistorical) {
       if (!this.pool.consumeSlot(jobId, exchange, 'HISTORICAL')) {
@@ -331,6 +333,14 @@ class IngestorWorker {
         return;
       }
       this.handleRealtimeRequest(request);
+    } else if (isTickerSnapshot) {
+      if (!this.pool.consumeSlot(jobId, exchange, 'HISTORICAL')) {
+        this.zmqClient.sendReject(jobId, 'Slot capacity exceeded').catch(() => {});
+        return;
+      }
+      this.handleTicker24hRequest(request).catch(err => {
+        this.logger.error({ jobId, requestId, error: err.message }, 'Unexpected error in ticker24h handler');
+      });
     } else {
       this.logger.warn({ jobId, type }, 'Unknown request type — rejecting');
       this.zmqClient.sendReject(jobId, `Unknown request type: ${type}`).catch(() => {});
@@ -430,6 +440,44 @@ class IngestorWorker {
     this.realtimePoller.startSubscription(jobId, requestId, ticker, this.config.kafka_tick_topic);
   }
 
+  /**
+   * Fetch all tickers (24h stats) for an exchange and write to Kafka.
+   * Triggered by TICKER_SNAPSHOT request with sentinel ticker @TICKER24H.{EXCHANGE}.
+   */
+  async handleTicker24hRequest(request) {
+    const { jobId, requestId, ticker, clientId } = request;
+    const exchangeId = exchangeOf(ticker); // e.g. "BINANCE" from "@TICKER24H.BINANCE"
+    const exchangeName = exchangeId.toLowerCase();
+
+    this.logger.info({ jobId, requestId, ticker, exchangeId }, 'Processing TICKER_SNAPSHOT request');
+
+    // Immediately ack to reset Flink's dispatch-time timeout clock.
+    await this.zmqClient.sendHeartbeat(jobId);
+
+    try {
+      const tickers = await this.ccxtFetcher.fetchAllTickers(exchangeName);
+
+      this.logger.info({ jobId, requestId, exchangeId, count: tickers.length }, 'Fetched tickers from exchange');
+
+      await this.kafkaProducer.writeTickerBatch(this.config.kafka_ticker_topic, exchangeId, tickers, clientId, requestId);
+
+      this.logger.info({ jobId, requestId, exchangeId }, 'Ticker24h request complete — sending WorkComplete');
+      await this.zmqClient.sendComplete(jobId, true);
+
+    } catch (error) {
+      this.logger.error({ jobId, requestId, exchangeId, error: error.message }, 'Ticker24h request failed');
+
+      if (error instanceof ExchangeRateLimitError) {
+        this.pool.reportRateLimit(exchangeId, 'HISTORICAL', error.retryAfterMs);
+      }
+
+      await this.zmqClient.sendComplete(jobId, false, error.message);
+    }
+
+    this.pool.releaseSlot(jobId).catch(err =>
+      this.logger.error({ jobId, error: err.message }, 'Failed to release ticker24h slot'));
+  }
+
   getStatus() {
     return {
       activeRealtime: this.activeRealtime.size,

ingestor/src/kafka-producer.js

@@ -1,6 +1,6 @@
 // Kafka producer for writing market data
 import { Kafka } from 'kafkajs';
-import { encodeMessage, MessageTypeId, Tick, OHLC, OHLCBatch, Market } from './proto/messages.js';
+import { encodeMessage, MessageTypeId, Tick, OHLC, OHLCBatch, Market, TickerBatch } from './proto/messages.js';
 
 export class KafkaProducer {
   constructor(config, logger) {
@@ -302,6 +302,39 @@ export class KafkaProducer {
     );
   }
 
+  /**
+   * Write a TickerBatch (all-ticker snapshot for one exchange) to Kafka.
+   * @param {string} topic - Kafka topic name (e.g. "market-ticker")
+   * @param {string} exchangeId - Exchange identifier (e.g. "BINANCE")
+   * @param {Array<object>} tickers - Array of TickerStats-compatible objects
+   * @param {string} [clientId] - Non-empty = client-initiated; absent = scheduled broadcast
+   * @param {string} [requestId] - Echoed for tracing
+   */
+  async writeTickerBatch(topic, exchangeId, tickers, clientId, requestId) {
+    if (!this.isConnected) {
+      throw new Error('Kafka producer not connected');
+    }
+
+    const nowNs = (BigInt(Date.now()) * 1_000_000n).toString();
+    const batch = {
+      exchangeId,
+      tickers,
+      fetchedAt: nowNs,
+    };
+    if (clientId) batch.clientId = clientId;
+    if (requestId) batch.requestId = requestId;
+
+    const [frame1, frame2] = encodeMessage(MessageTypeId.TICKER_BATCH, batch, TickerBatch);
+    const value = Buffer.concat([frame1, frame2]);
+
+    await this.producer.send({
+      topic,
+      messages: [{ key: exchangeId, value }],
+    });
+
+    this.logger.info({ exchange: exchangeId, count: tickers.length, topic }, 'Wrote TickerBatch to Kafka');
+  }
+
   /**
    * Disconnect from Kafka
    */