data fixes; indicator=>workspace sync

ingestor/src/ccxt-fetcher.js

@@ -113,11 +113,12 @@ export class CCXTFetcher {
       'Fetching historical OHLC'
     );
 
-    const allCandles = [];
+    const fetchedCandles = [];
     let since = startMs;
 
-    // CCXT typically limits to 1000 candles per request
-    const batchSize = limit || 1000;
+    // Always page in fixed batches of 1000 regardless of any limit hint.
+    // The caller's limit/countback is irrelevant to how much we need to fetch from the exchange.
+    const PAGE_SIZE = 1000;
 
     while (since < endMs) {
       try {
@@ -125,27 +126,26 @@ export class CCXTFetcher {
           symbol,
           timeframe,
           since,
-          batchSize
+          PAGE_SIZE
         );
 
         if (candles.length === 0) {
           break;
         }
 
-        // Filter candles within the time range
+        // Filter candles within the requested time range
         const filteredCandles = candles.filter(c => {
           const timestamp = c[0];
-          return timestamp >= startMs && timestamp <= endMs;
+          return timestamp >= startMs && timestamp < endMs;  // endMs is exclusive
         });
 
-        allCandles.push(...filteredCandles);
+        fetchedCandles.push(...filteredCandles);
 
-        // Move to next batch
+        // Advance to next batch start
         const lastTimestamp = candles[candles.length - 1][0];
         since = lastTimestamp + (periodSeconds * 1000);
 
-        // Break if we've reached the end time or limit
-        if (since >= endMs || (limit && allCandles.length >= limit)) {
+        if (since >= endMs) {
           break;
         }
 
@@ -163,8 +163,46 @@ export class CCXTFetcher {
     // Get metadata for proper denomination
     const metadata = await this.getMetadata(ticker);
 
-    // Convert to our OHLC format
-    return allCandles.map(candle => this.convertToOHLC(candle, ticker, periodSeconds, metadata));
+    // Build a map of fetched candles by timestamp (ms)
+    const fetchedByTs = new Map(fetchedCandles.map(c => [c[0], c]));
+
+    if (fetchedCandles.length === 0) {
+      // No data from exchange — return empty so caller writes a NOT_FOUND marker.
+      return [];
+    }
+
+    const periodMs = periodSeconds * 1000;
+
+    // Only create null gap bars for interior gaps — periods where real data exists
+    // on BOTH sides (i.e., between the first and last real bar). Do NOT append
+    // null bars before the first real bar or after the last real bar: those edge
+    // positions may be in-progress candles or simply outside the exchange's history,
+    // and we have no positive signal that a gap exists there.
+    const realTimestamps = [...fetchedByTs.keys()].sort((a, b) => a - b);
+    const firstRealTs = realTimestamps[0];
+    const lastRealTs  = realTimestamps[realTimestamps.length - 1];
+
+    const allCandles = [];
+    let gapCount = 0;
+
+    for (let ts = firstRealTs; ts <= lastRealTs; ts += periodMs) {
+      if (fetchedByTs.has(ts)) {
+        allCandles.push(this.convertToOHLC(fetchedByTs.get(ts), ticker, periodSeconds, metadata));
+      } else {
+        // Interior gap — confirmed by real bars on both sides
+        gapCount++;
+        allCandles.push(this.createGapBar(ts, ticker, periodSeconds, metadata));
+      }
+    }
+
+    if (gapCount > 0) {
+      this.logger.info(
+        { ticker, gapCount, total: allCandles.length },
+        'Filled interior gap bars for missing periods in source data'
+      );
+    }
+
+    return allCandles;
   }
 
   /**
@@ -227,6 +265,24 @@ export class CCXTFetcher {
     };
   }
 
+  /**
+   * Create a gap bar for a period with no trade data.
+   * All OHLC/volume fields are null — the timestamp slot is reserved but unpopulated.
+   */
+  createGapBar(timestampMs, ticker, periodSeconds, metadata) {
+    return {
+      ticker,
+      timestamp: (timestampMs * 1000).toString(),   // Convert ms to microseconds
+      open: null,
+      high: null,
+      low: null,
+      close: null,
+      volume: null,
+      open_time: (timestampMs * 1000).toString(),
+      close_time: ((timestampMs + periodSeconds * 1000) * 1000).toString()
+    };
+  }
+
   /**
    * Convert CCXT trade to our Tick format
    * Uses denominators from market metadata for proper integer representation

ingestor/src/kafka-producer.js

@@ -180,15 +180,20 @@ export class KafkaProducer {
         status: metadata.status || 'OK',
         errorMessage: metadata.error_message
       },
-      rows: ohlcData.map(candle => ({
-        timestamp: candle.timestamp,
-        ticker: candle.ticker,
-        open: candle.open,
-        high: candle.high,
-        low: candle.low,
-        close: candle.close,
-        volume: candle.volume
-      }))
+      rows: ohlcData.map(candle => {
+        // null open/high/low/close signals a gap bar (no trades that period).
+        // Omit fields from the protobuf message when null so hasOpen() etc. return false.
+        const row = {
+          timestamp: candle.timestamp,
+          ticker: candle.ticker,
+        };
+        if (candle.open  != null) row.open   = candle.open;
+        if (candle.high  != null) row.high   = candle.high;
+        if (candle.low   != null) row.low    = candle.low;
+        if (candle.close != null) row.close  = candle.close;
+        if (candle.volume != null) row.volume = candle.volume;
+        return row;
+      })
     };
 
     // Encode as protobuf OHLCBatch with ZMQ envelope