backend redesign

ingestor/src/realtime-poller.js

@@ -0,0 +1,217 @@
+// Realtime tick data poller using 10-second polling
+export class RealtimePoller {
+  constructor(ccxtFetcher, kafkaProducer, logger) {
+    this.ccxtFetcher = ccxtFetcher;
+    this.kafkaProducer = kafkaProducer;
+    this.logger = logger;
+
+    // Active subscriptions: requestId -> subscription info
+    this.subscriptions = new Map();
+
+    // Poll interval in milliseconds (10 seconds)
+    this.pollInterval = 10000;
+
+    // Main polling loop
+    this.pollingLoop = null;
+  }
+
+  /**
+   * Start a realtime subscription
+   * @param {string} requestId - Unique request ID
+   * @param {string} ticker - Ticker to subscribe to
+   * @param {string} kafkaTopic - Kafka topic to write to
+   */
+  startSubscription(requestId, ticker, kafkaTopic) {
+    if (this.subscriptions.has(requestId)) {
+      this.logger.warn({ requestId }, 'Subscription already exists');
+      return;
+    }
+
+    const subscription = {
+      requestId,
+      ticker,
+      kafkaTopic,
+      lastTimestamp: null,
+      isActive: true,
+      errorCount: 0
+    };
+
+    this.subscriptions.set(requestId, subscription);
+
+    this.logger.info(
+      { requestId, ticker, kafkaTopic },
+      'Started realtime subscription'
+    );
+
+    // Start polling loop if not already running
+    if (!this.pollingLoop) {
+      this.startPollingLoop();
+    }
+  }
+
+  /**
+   * Cancel a realtime subscription
+   * @param {string} requestId - Request ID to cancel
+   */
+  cancelSubscription(requestId) {
+    const subscription = this.subscriptions.get(requestId);
+    if (subscription) {
+      subscription.isActive = false;
+      this.subscriptions.delete(requestId);
+
+      this.logger.info(
+        { requestId, ticker: subscription.ticker },
+        'Cancelled realtime subscription'
+      );
+    }
+
+    // Stop polling loop if no active subscriptions
+    if (this.subscriptions.size === 0 && this.pollingLoop) {
+      clearInterval(this.pollingLoop);
+      this.pollingLoop = null;
+      this.logger.info('Stopped polling loop - no active subscriptions');
+    }
+  }
+
+  /**
+   * Start the main polling loop
+   */
+  startPollingLoop() {
+    this.logger.info({ interval: this.pollInterval }, 'Starting polling loop');
+
+    this.pollingLoop = setInterval(async () => {
+      await this.pollAllSubscriptions();
+    }, this.pollInterval);
+
+    // Do an immediate poll
+    this.pollAllSubscriptions();
+  }
+
+  /**
+   * Poll all active subscriptions
+   */
+  async pollAllSubscriptions() {
+    const subscriptions = Array.from(this.subscriptions.values());
+
+    // Poll subscriptions in parallel
+    await Promise.allSettled(
+      subscriptions.map(sub => this.pollSubscription(sub))
+    );
+  }
+
+  /**
+   * Poll a single subscription
+   * @param {object} subscription - Subscription object
+   */
+  async pollSubscription(subscription) {
+    if (!subscription.isActive) {
+      return;
+    }
+
+    const { requestId, ticker, kafkaTopic, lastTimestamp } = subscription;
+
+    try {
+      // Fetch trades since last timestamp
+      const trades = await this.ccxtFetcher.fetchRecentTrades(
+        ticker,
+        lastTimestamp
+      );
+
+      if (trades.length === 0) {
+        this.logger.debug({ requestId, ticker }, 'No new trades');
+        return;
+      }
+
+      // Filter out trades we've already seen
+      let newTrades = trades;
+      if (lastTimestamp) {
+        const lastTs = BigInt(lastTimestamp);
+        newTrades = trades.filter(t => BigInt(t.timestamp) > lastTs);
+      }
+
+      if (newTrades.length > 0) {
+        // Write trades to Kafka
+        await this.kafkaProducer.writeTicks(kafkaTopic, newTrades);
+
+        // Update last timestamp
+        const latestTrade = newTrades[newTrades.length - 1];
+        subscription.lastTimestamp = latestTrade.timestamp;
+
+        this.logger.info(
+          {
+            requestId,
+            ticker,
+            count: newTrades.length,
+            kafkaTopic
+          },
+          'Wrote new trades to Kafka'
+        );
+      }
+
+      // Reset error count on success
+      subscription.errorCount = 0;
+    } catch (error) {
+      subscription.errorCount++;
+
+      this.logger.error(
+        {
+          error: error.message,
+          requestId,
+          ticker,
+          errorCount: subscription.errorCount
+        },
+        'Error polling subscription'
+      );
+
+      // Cancel subscription after too many errors
+      if (subscription.errorCount >= 5) {
+        this.logger.error(
+          { requestId, ticker },
+          'Cancelling subscription due to repeated errors'
+        );
+        this.cancelSubscription(requestId);
+      }
+    }
+  }
+
+  /**
+   * Get subscription statistics
+   */
+  getStats() {
+    const stats = {
+      totalSubscriptions: this.subscriptions.size,
+      subscriptions: []
+    };
+
+    for (const [requestId, sub] of this.subscriptions) {
+      stats.subscriptions.push({
+        requestId,
+        ticker: sub.ticker,
+        isActive: sub.isActive,
+        errorCount: sub.errorCount,
+        lastTimestamp: sub.lastTimestamp
+      });
+    }
+
+    return stats;
+  }
+
+  /**
+   * Shutdown poller and cancel all subscriptions
+   */
+  shutdown() {
+    this.logger.info('Shutting down realtime poller');
+
+    if (this.pollingLoop) {
+      clearInterval(this.pollingLoop);
+      this.pollingLoop = null;
+    }
+
+    // Mark all subscriptions as inactive
+    for (const subscription of this.subscriptions.values()) {
+      subscription.isActive = false;
+    }
+
+    this.subscriptions.clear();
+  }
+}