bugfixes

ingestor/src/index.js

@@ -372,7 +372,10 @@ class IngestorWorker {
 
       if (candles.length > 0) {
         const metadata = { request_id: requestId, client_id, ticker, period_seconds, start_time, end_time };
-        const PAGE_SIZE = 1000;
+        // 8000 rows/page: each OHLC row is ~77 bytes typical (9 populated fields as
+        // protobuf varints + ticker string). Worst-case is ~124 bytes, so 8000 rows
+        // stays safely under Kafka's 1MB message limit in all realistic scenarios.
+        const PAGE_SIZE = 8000;
         for (let i = 0; i < candles.length; i += PAGE_SIZE) {
           const page = candles.slice(i, i + PAGE_SIZE);
           const isLastPage = (i + PAGE_SIZE) >= candles.length;

ingestor/src/symbol-metadata-generator.js

@@ -249,7 +249,6 @@ export class SymbolMetadataGenerator {
 
       if (!this.publishedSymbols.has(key)) {
         uniqueMetadata.push(metadata);
-        this.publishedSymbols.add(key);
       } else {
         duplicateCount++;
       }
@@ -275,6 +274,12 @@ export class SymbolMetadataGenerator {
 
     await this.kafkaProducer.writeMarketMetadata(topic, messages);
 
+    // Mark as published only after successful Kafka write
+    for (const metadata of uniqueMetadata) {
+      const key = `${metadata.marketId}.${metadata.exchangeId}`;
+      this.publishedSymbols.add(key);
+    }
+
     this.logger.info(
       { count: messages.length, duplicateCount, topic },
       'Wrote symbol metadata to Kafka'