data pipeline refactor and fix

ingestor/src/kafka-producer.js

@@ -116,12 +116,17 @@ export class KafkaProducer {
   }
 
   /**
-   * Write multiple OHLC candles to Kafka as an OHLCBatch message
-   * Uses protobuf encoding with metadata in batch wrapper
+   * Write multiple OHLC candles to Kafka as an OHLCBatch message.
+   *
+   * Historical mode: pass explicit metadata and isLastPage flag.
+   * Realtime mode: omit metadata (null/undefined) — writes individual OHLC messages instead.
+   *
    * @param {string} topic - Kafka topic name
-   * @param {Array<object>} ohlcData - Array of OHLC data objects (may include __metadata in first record)
+   * @param {Array<object>} ohlcData - Array of OHLC candle objects
+   * @param {object|null} metadata - Request metadata for historical batches; null for realtime
+   * @param {boolean} isLastPage - True if this is the final page of a historical query
    */
-  async writeOHLCs(topic, ohlcData) {
+  async writeOHLCs(topic, ohlcData, metadata = null, isLastPage = false) {
     if (!this.isConnected) {
       throw new Error('Kafka producer not connected');
     }
@@ -130,12 +135,8 @@ export class KafkaProducer {
       return;
     }
 
-    // Extract metadata from first record if present
-    const firstCandle = ohlcData[0];
-    const metadata = firstCandle.__metadata;
-
     if (!metadata) {
-      // No metadata - write individual OHLC messages (realtime mode)
+      // Realtime mode — write individual OHLC messages (no batch wrapper)
       const messages = ohlcData.map(candle => {
         const protoCandle = {
           timestamp: candle.timestamp,
@@ -156,10 +157,7 @@ export class KafkaProducer {
         };
       });
 
-      await this.producer.send({
-        topic,
-        messages
-      });
+      await this.producer.send({ topic, messages });
 
       this.logger.debug(
         { count: ohlcData.length, topic, type: 'individual' },
@@ -168,7 +166,7 @@ export class KafkaProducer {
       return;
     }
 
-    // Historical mode - write as OHLCBatch with metadata
+    // Historical mode — write as OHLCBatch with metadata
     const batch = {
       metadata: {
         requestId: metadata.request_id,
@@ -178,7 +176,8 @@ export class KafkaProducer {
         startTime: metadata.start_time,
         endTime: metadata.end_time,
         status: metadata.status || 'OK',
-        errorMessage: metadata.error_message
+        errorMessage: metadata.error_message,
+        isLastPage
       },
       rows: ohlcData.map(candle => {
         const row = {
@@ -194,22 +193,16 @@ export class KafkaProducer {
       })
     };
 
-    // Encode as protobuf OHLCBatch with ZMQ envelope
     const [frame1, frame2] = encodeMessage(MessageTypeId.OHLC_BATCH, batch, OHLCBatch);
     const value = Buffer.concat([frame1, frame2]);
 
     await this.producer.send({
       topic,
-      messages: [
-        {
-          key: metadata.ticker,
-          value
-        }
-      ]
+      messages: [{ key: metadata.ticker, value }]
     });
 
     this.logger.debug(
-      { request_id: metadata.request_id, count: ohlcData.length, topic, type: 'batch' },
+      { request_id: metadata.request_id, count: ohlcData.length, isLastPage, topic },
       'Wrote OHLCBatch to Kafka'
     );
   }
@@ -225,7 +218,8 @@ export class KafkaProducer {
       throw new Error('Kafka producer not connected');
     }
 
-    // Create an empty OHLCBatch with status in metadata
+    // Create an empty OHLCBatch with status in metadata.
+    // Markers are always the terminal message for a request (is_last_page = true).
     const batch = {
       metadata: {
         requestId: marker.request_id,
@@ -235,7 +229,8 @@ export class KafkaProducer {
         startTime: marker.start_time,
         endTime: marker.end_time,
         status: marker.status, // 'NOT_FOUND' or 'ERROR'
-        errorMessage: marker.error_message || marker.message
+        errorMessage: marker.error_message || marker.message,
+        isLastPage: true
       },
       rows: [] // Empty rows array indicates marker message
     };