backend redesign

ingestor/src/zmq-client.js

@@ -0,0 +1,116 @@
+// ZeroMQ client for connecting to Flink control channels
+import * as zmq from 'zeromq';
+import { decodeMessage } from './proto/messages.js';
+
+export class ZmqClient {
+  constructor(config, logger) {
+    this.config = config;
+    this.logger = logger;
+
+    // Work queue - SUB socket to receive data requests with exchange prefix filtering
+    this.workSocket = null;
+
+    // NOTE: NO RESPONSE SOCKET - Async architecture via Kafka!
+    // Ingestors write data to Kafka only
+    // Flink processes and publishes notifications
+
+    this.isShutdown = false;
+    this.supportedExchanges = config.supported_exchanges || ['BINANCE', 'COINBASE'];
+  }
+
+  /**
+   * Connect to Relay ZMQ endpoints
+   */
+  async connect() {
+    const { flink_hostname, ingestor_work_port } = this.config;
+
+    // Connect to work queue (SUB with exchange prefix filtering)
+    this.workSocket = new zmq.Subscriber();
+    const workEndpoint = `tcp://${flink_hostname}:${ingestor_work_port}`;
+    await this.workSocket.connect(workEndpoint);
+
+    // Subscribe to each supported exchange prefix
+    for (const exchange of this.supportedExchanges) {
+      const prefix = `${exchange}:`;
+      this.workSocket.subscribe(prefix);
+      this.logger.info(`Subscribed to exchange prefix: ${prefix}`);
+    }
+    this.logger.info(`Connected to work queue at ${workEndpoint}`);
+    this.logger.info('ASYNC MODE: No response socket - data flows via Kafka → Flink → pub/sub notification');
+  }
+
+  /**
+   * Pull a data request from the work queue
+   * @returns {Promise<object>} Decoded DataRequest message
+   */
+  async pullDataRequest() {
+    if (this.isShutdown) {
+      return null;
+    }
+
+    try {
+      const frames = await this.workSocket.receive();
+      this.logger.info({
+        frameCount: frames.length,
+        frame0Len: frames[0]?.length,
+        frame1Len: frames[1]?.length,
+        frame2Len: frames[2]?.length,
+        frame0: frames[0]?.toString('utf8').substring(0, 50),
+        frame1Hex: frames[1]?.toString('hex').substring(0, 20),
+        frame2Hex: frames[2]?.toString('hex').substring(0, 20)
+      }, 'Received raw ZMQ frames');
+
+      // First frame is the topic (exchange prefix), skip it
+      // Remaining frames are: [version_frame, message_frame]
+      if (frames.length < 3) {
+        this.logger.warn({ frameCount: frames.length }, 'Unexpected frame count');
+        return null;
+      }
+      const messageFrames = frames.slice(1); // Skip topic, keep version + message
+      const { version, typeId, message } = decodeMessage(messageFrames);
+      this.logger.info({
+        version,
+        typeId: `0x${typeId.toString(16)}`,
+        requestId: message.requestId,
+        type: message.type,
+        typeOf: typeof message.type,
+        ticker: message.ticker
+      }, 'Decoded data request');
+      return message;
+    } catch (error) {
+      if (!this.isShutdown) {
+        this.logger.error({ error: error.message, stack: error.stack }, 'Error receiving data request');
+      }
+      return null;
+    }
+  }
+
+
+  /**
+   * Start listening for control messages in the background
+   * @param {Function} handler - Callback function to handle control messages
+   *
+   * NOTE: Control channel not implemented yet. This is a stub for future use.
+   * For now, just log and ignore.
+   */
+  startControlListener(handler) {
+    this.logger.info('Control channel listener stub - not implemented yet');
+    // TODO: Implement control channel when needed
+    // Control messages would be used for:
+    // - Canceling realtime subscriptions
+    // - Graceful shutdown signals
+    // - Configuration updates
+  }
+
+  /**
+   * Shutdown and close connections
+   */
+  async shutdown() {
+    this.isShutdown = true;
+    this.logger.info('Shutting down ZMQ connections');
+
+    if (this.workSocket) {
+      await this.workSocket.close();
+    }
+  }
+}