data pipeline refactor and fix

gateway/src/clients/zmq-relay-client.ts

@@ -17,6 +17,9 @@ import {
   encodeSubmitHistoricalRequest,
   decodeSubmitResponse,
   decodeHistoryReadyNotification,
+  decodeRealtimeBar,
+  OHLC_BAR_TOPIC_PATTERN,
+  type RealtimeBar,
 } from './zmq-protocol.js';
 import type {
   SubmitHistoricalRequest,
@@ -27,6 +30,9 @@ import {
   NotificationStatus,
 } from '../types/ohlc.js';
 
+export type BarUpdateCallback = (bar: RealtimeBar) => void;
+export type { RealtimeBar };
+
 export interface ZMQRelayConfig {
   relayRequestEndpoint: string;       // e.g., "tcp://relay:5559"
   relayNotificationEndpoint: string;  // e.g., "tcp://relay:5558"
@@ -57,6 +63,12 @@ export class ZMQRelayClient {
   private notificationTopic: string;
   private pendingRequests: Map<string, PendingRequest> = new Map();
 
+  /** Ref count per ZMQ topic (gateway-level dedup before ZMQ subscribe/unsubscribe) */
+  private topicRefs: Map<string, number> = new Map();
+
+  /** Callbacks registered by WebSocket sessions for realtime bar updates */
+  private barCallbacks: Map<string, Set<BarUpdateCallback>> = new Map();
+
   private connected = false;
   private notificationListenerRunning = false;
 
@@ -253,8 +265,6 @@ export class ZMQRelayClient {
             // Handle metadata update notifications
             if (topic === 'METADATA_UPDATE') {
               this.logger.info('Received METADATA_UPDATE notification');
-
-              // Call the onMetadataUpdate callback if configured
               if (this.config.onMetadataUpdate) {
                 try {
                   await this.config.onMetadataUpdate();
@@ -265,6 +275,20 @@ export class ZMQRelayClient {
               continue;
             }
 
+            // Handle realtime OHLC bar updates (topic pattern: "{ticker}|ohlc:{period}")
+            if (OHLC_BAR_TOPIC_PATTERN.test(topic)) {
+              const bar = decodeRealtimeBar(Array.from(frames));
+              if (bar) {
+                const callbacks = this.barCallbacks.get(topic);
+                if (callbacks) {
+                  for (const cb of callbacks) {
+                    try { cb(bar); } catch (e) { /* ignore callback errors */ }
+                  }
+                }
+              }
+              continue;
+            }
+
             // Handle history ready notifications
             const notification = decodeHistoryReadyNotification(Array.from(frames));
 
@@ -308,6 +332,69 @@ export class ZMQRelayClient {
     this.logger.debug('Notification listener started');
   }
 
+  /**
+   * Subscribe to realtime OHLC bars for a ticker+period.
+   *
+   * ZMQ subscribe is only called on the 0→1 transition (first subscriber).
+   * This triggers the relay XPUB → Flink subscription detection → ingestor activation.
+   *
+   * @param callback  Called whenever a new bar arrives for this topic
+   */
+  subscribeToTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
+    const topic = `${ticker}|ohlc:${periodSeconds}`;
+
+    // Register callback
+    if (!this.barCallbacks.has(topic)) {
+      this.barCallbacks.set(topic, new Set());
+    }
+    this.barCallbacks.get(topic)!.add(callback);
+
+    // ZMQ subscribe on first ref
+    const prev = this.topicRefs.get(topic) ?? 0;
+    this.topicRefs.set(topic, prev + 1);
+    if (prev === 0 && this.subSocket) {
+      this.subSocket.subscribe(topic);
+      this.logger.info({ topic }, 'ZMQ subscribed to realtime topic');
+    }
+  }
+
+  /**
+   * Unsubscribe a callback from realtime OHLC bars.
+   * ZMQ unsubscribe is only called on the 1→0 transition (last subscriber).
+   */
+  unsubscribeFromTicker(ticker: string, periodSeconds: number, callback: BarUpdateCallback): void {
+    const topic = `${ticker}|ohlc:${periodSeconds}`;
+
+    const callbacks = this.barCallbacks.get(topic);
+    if (callbacks) {
+      callbacks.delete(callback);
+      if (callbacks.size === 0) {
+        this.barCallbacks.delete(topic);
+      }
+    }
+
+    const prev = this.topicRefs.get(topic) ?? 0;
+    if (prev <= 1) {
+      this.topicRefs.delete(topic);
+      if (this.subSocket) {
+        this.subSocket.unsubscribe(topic);
+        this.logger.info({ topic }, 'ZMQ unsubscribed from realtime topic');
+      }
+    } else {
+      this.topicRefs.set(topic, prev - 1);
+    }
+  }
+
+  /**
+   * Remove all subscriptions for a set of (topic, callback) pairs.
+   * Convenience method for WebSocket disconnect cleanup.
+   */
+  cleanupSubscriptions(subscriptions: Array<{ ticker: string; periodSeconds: number; callback: BarUpdateCallback }>): void {
+    for (const { ticker, periodSeconds, callback } of subscriptions) {
+      this.unsubscribeFromTicker(ticker, periodSeconds, callback);
+    }
+  }
+
   /**
    * Close the client and cleanup resources
    */