dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Ticker24h support: hourly market snapshots with USD-normalized volume filtering

Browse Source
This commit is contained in:
Tim Olson
2026-04-26 18:39:52 -04:00
parent 85fcbe1330
commit 0178b5d29d
45 changed files with 1995 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
ingestor/src/index.js
View File

@@ -57,6 +57,7 @@ function loadConfig() {
kafka_brokers: config.kafka_brokers || ['localhost:9092'],
kafka_ohlc_topic: config.kafka_ohlc_topic || 'market-ohlc',
kafka_tick_topic: config.kafka_tick_topic || 'market-tick',
kafka_ticker_topic: config.kafka_ticker_topic || 'market-ticker',
// Worker configuration
poll_interval_ms: config.poll_interval_ms || 10000,
@@ -316,6 +317,7 @@ class IngestorWorker {
const isHistorical = !type || type === 'HISTORICAL_OHLC' || type === 0;
const isRealtime = type === 'REALTIME_TICKS' || type === 1;
const isTickerSnapshot = type === 'TICKER_SNAPSHOT' || type === 2;
if (isHistorical) {
if (!this.pool.consumeSlot(jobId, exchange, 'HISTORICAL')) {
@@ -331,6 +333,14 @@ class IngestorWorker {
return;
}
this.handleRealtimeRequest(request);
} else if (isTickerSnapshot) {
if (!this.pool.consumeSlot(jobId, exchange, 'HISTORICAL')) {
this.zmqClient.sendReject(jobId, 'Slot capacity exceeded').catch(() => {});
return;
}
this.handleTicker24hRequest(request).catch(err => {
this.logger.error({ jobId, requestId, error: err.message }, 'Unexpected error in ticker24h handler');
});
} else {
this.logger.warn({ jobId, type }, 'Unknown request type — rejecting');
this.zmqClient.sendReject(jobId, `Unknown request type: ${type}`).catch(() => {});
@@ -430,6 +440,44 @@ class IngestorWorker {
this.realtimePoller.startSubscription(jobId, requestId, ticker, this.config.kafka_tick_topic);
}
/**
* Fetch all tickers (24h stats) for an exchange and write to Kafka.
* Triggered by TICKER_SNAPSHOT request with sentinel ticker @TICKER24H.{EXCHANGE}.
*/
async handleTicker24hRequest(request) {
const { jobId, requestId, ticker, clientId } = request;
const exchangeId = exchangeOf(ticker); // e.g. "BINANCE" from "@TICKER24H.BINANCE"
const exchangeName = exchangeId.toLowerCase();
this.logger.info({ jobId, requestId, ticker, exchangeId }, 'Processing TICKER_SNAPSHOT request');
// Immediately ack to reset Flink's dispatch-time timeout clock.
await this.zmqClient.sendHeartbeat(jobId);
try {
const tickers = await this.ccxtFetcher.fetchAllTickers(exchangeName);
this.logger.info({ jobId, requestId, exchangeId, count: tickers.length }, 'Fetched tickers from exchange');
await this.kafkaProducer.writeTickerBatch(this.config.kafka_ticker_topic, exchangeId, tickers, clientId, requestId);
this.logger.info({ jobId, requestId, exchangeId }, 'Ticker24h request complete — sending WorkComplete');
await this.zmqClient.sendComplete(jobId, true);
} catch (error) {
this.logger.error({ jobId, requestId, exchangeId, error: error.message }, 'Ticker24h request failed');
if (error instanceof ExchangeRateLimitError) {
this.pool.reportRateLimit(exchangeId, 'HISTORICAL', error.retryAfterMs);
}
await this.zmqClient.sendComplete(jobId, false, error.message);
}
this.pool.releaseSlot(jobId).catch(err =>
this.logger.error({ jobId, error: err.message }, 'Failed to release ticker24h slot'));
}
getStatus() {
return {
activeRealtime: this.activeRealtime.size,