data fixes, partial custom indicator support

gateway/src/clients/duckdb-client.ts

@@ -504,7 +504,11 @@ export class DuckDBClient {
   }
 
   /**
-   * Find missing OHLC data ranges
+   * Find missing OHLC data ranges by checking for absent timestamps.
+   *
+   * Any timestamp slot in [start_time, min(end_time, now)) that has no row in
+   * Iceberg is treated as missing and collected into contiguous ranges that the
+   * caller should request from the relay/ingestor.
    */
   async findMissingOHLCRanges(
     ticker: string,
@@ -517,32 +521,51 @@ export class DuckDBClient {
     try {
       const data = await this.queryOHLC(ticker, period_seconds, start_time, end_time);
 
-      if (data.length === 0) {
-        // All data is missing
-        return [[start_time, end_time]];
-      }
-
-      // Check if we have continuous data
-      // For now, simple check: if we have any data, assume complete
-      // TODO: Implement proper gap detection by checking for missing periods
       const periodNanos = BigInt(period_seconds) * 1_000_000_000n;
-      // end_time is exclusive, so expected count = (end - start) / period (no +1)
-      const expectedBars = Number((end_time - start_time) / periodNanos);
 
-      if (data.length < expectedBars * 0.95) { // Allow 5% tolerance
-        this.logger.debug({
-          ticker,
-          expected: expectedBars,
-          actual: data.length,
-        }, 'Incomplete OHLC data detected');
-        return [[start_time, end_time]]; // Request full range
+      // Cap at current time — future slots are not "missing", they don't exist yet.
+      const nowNanos = BigInt(Date.now()) * 1_000_000n;
+      const effectiveEnd = end_time < nowNanos ? end_time : nowNanos;
+
+      // Build a set of timestamps we already have (all rows are non-null now).
+      const present = new Set(data.map((row: any) => row.timestamp));
+
+      // Collect every expected slot that is absent.
+      const missing: bigint[] = [];
+      for (let t = start_time; t < effectiveEnd; t += periodNanos) {
+        if (!present.has(t)) {
+          missing.push(t);
+        }
       }
 
-      // Data appears complete
-      return [];
+      if (missing.length === 0) {
+        return [];
+      }
+
+      // Coalesce adjacent missing slots into contiguous [rangeStart, rangeEnd) intervals.
+      const ranges: Array<[bigint, bigint]> = [];
+      let rangeStart = missing[0];
+      let prev = missing[0];
+      for (let i = 1; i < missing.length; i++) {
+        if (missing[i] !== prev + periodNanos) {
+          ranges.push([rangeStart, prev + periodNanos]);
+          rangeStart = missing[i];
+        }
+        prev = missing[i];
+      }
+      ranges.push([rangeStart, prev + periodNanos]);
+
+      this.logger.debug({
+        ticker,
+        period_seconds,
+        missingSlots: missing.length,
+        ranges: ranges.length,
+      }, 'OHLC gap detection complete');
+
+      return ranges;
     } catch (error: any) {
       this.logger.error({ error: error.message }, 'Failed to find missing OHLC ranges');
-      // Return full range on error (safe default)
+      // Return full range on error (safe default — triggers a backfill)
       return [[start_time, end_time]];
     }
   }