dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data fixes, partial custom indicator support

Browse Source
This commit is contained in:
Tim Olson
2026-04-08 21:28:31 -04:00
parent b701554996
commit a70dcd954f
81 changed files with 5438 additions and 1852 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
ingestor/src/ccxt-fetcher.js
View File

@@ -45,6 +45,7 @@ export class CCXTFetcher {
const exchange = this.getExchange(exchangeName);
// Load market info from CCXT
this.logger.info({ exchangeName, symbol }, 'Loading markets for metadata');
await exchange.loadMarkets();
const market = exchange.market(symbol);
@@ -108,8 +109,9 @@ export class CCXTFetcher {
// Map period seconds to CCXT timeframe
const timeframe = this.secondsToTimeframe(periodSeconds);
const marketsLoaded = exchange.markets != null && Object.keys(exchange.markets).length > 0;
this.logger.info(
{ ticker, timeframe, startMs, endMs, limit },
{ ticker, timeframe, startMs, endMs, limit, marketsLoaded },
'Fetching historical OHLC'
);
@@ -120,44 +122,76 @@ export class CCXTFetcher {
// The caller's limit/countback is irrelevant to how much we need to fetch from the exchange.
const PAGE_SIZE = 1000;
const FETCH_RETRIES = 3;
const FETCH_RETRY_DELAY_MS = 5000;
while (since < endMs) {
try {
const candles = await exchange.fetchOHLCV(
symbol,
timeframe,
since,
PAGE_SIZE
);
if (candles.length === 0) {
let candles;
let lastError;
for (let attempt = 1; attempt <= FETCH_RETRIES; attempt++) {
try {
candles = await exchange.fetchOHLCV(symbol, timeframe, since, PAGE_SIZE);
lastError = null;
break;
} catch (error) {
lastError = error;
const isRetryable = error.constructor?.name === 'NetworkError' ||
error.constructor?.name === 'RequestTimeout' ||
error.constructor?.name === 'ExchangeNotAvailable';
this.logger.warn(
{
errorType: error.constructor?.name,
error: error.message,
errorUrl: error.url,
ticker,
since,
attempt,
retryable: isRetryable
},
'OHLC fetch attempt failed'
);
if (!isRetryable || attempt === FETCH_RETRIES) break;
await exchange.sleep(FETCH_RETRY_DELAY_MS * attempt);
}
// Filter candles within the requested time range
const filteredCandles = candles.filter(c => {
const timestamp = c[0];
return timestamp >= startMs && timestamp < endMs; // endMs is exclusive
});
fetchedCandles.push(...filteredCandles);
// Advance to next batch start
const lastTimestamp = candles[candles.length - 1][0];
since = lastTimestamp + (periodSeconds * 1000);
if (since >= endMs) {
break;
}
// Apply rate limiting
await exchange.sleep(exchange.rateLimit);
} catch (error) {
}
if (lastError) {
this.logger.error(
{ error: error.message, ticker, since },
{
errorType: lastError.constructor?.name,
error: lastError.message,
errorUrl: lastError.url,
ticker,
since,
marketsLoaded: exchange.markets != null && Object.keys(exchange.markets).length > 0,
stack: lastError.stack
},
'Error fetching OHLC'
);
throw error;
throw lastError;
}
if (candles.length === 0) {
break;
}
// Filter candles within the requested time range
const filteredCandles = candles.filter(c => {
const timestamp = c[0];
return timestamp >= startMs && timestamp < endMs; // endMs is exclusive
});
fetchedCandles.push(...filteredCandles);
// Advance to next batch start
const lastTimestamp = candles[candles.length - 1][0];
since = lastTimestamp + (periodSeconds * 1000);
if (since >= endMs) {
break;
}
// Apply rate limiting
await exchange.sleep(exchange.rateLimit);
}
// Get metadata for proper denomination
@@ -173,32 +207,44 @@ export class CCXTFetcher {
const periodMs = periodSeconds * 1000;
// Only create null gap bars for interior gaps — periods where real data exists
// on BOTH sides (i.e., between the first and last real bar). Do NOT append
// null bars before the first real bar or after the last real bar: those edge
// positions may be in-progress candles or simply outside the exchange's history,
// and we have no positive signal that a gap exists there.
// Forward-fill interior gaps — periods between the first and last real bar
// where the exchange returned no candle. Edge gaps (before firstRealTs or
// after lastRealTs) are left absent; they'll be caught by gap detection and
// trigger a targeted backfill request.
const realTimestamps = [...fetchedByTs.keys()].sort((a, b) => a - b);
const firstRealTs = realTimestamps[0];
const lastRealTs = realTimestamps[realTimestamps.length - 1];
const allCandles = [];
let gapCount = 0;
let prevClose = null;
for (let ts = firstRealTs; ts <= lastRealTs; ts += periodMs) {
if (fetchedByTs.has(ts)) {
allCandles.push(this.convertToOHLC(fetchedByTs.get(ts), ticker, periodSeconds, metadata));
} else {
// Interior gap — confirmed by real bars on both sides
const bar = this.convertToOHLC(fetchedByTs.get(ts), ticker, periodSeconds, metadata);
prevClose = bar.close;
allCandles.push(bar);
} else if (prevClose !== null) {
// Interior gap — forward-fill with previous close, zero volume
gapCount++;
allCandles.push(this.createGapBar(ts, ticker, periodSeconds, metadata));
allCandles.push({
ticker,
timestamp: (ts * 1_000_000).toString(),
open: prevClose,
high: prevClose,
low: prevClose,
close: prevClose,
volume: '0',
open_time: (ts * 1_000_000).toString(),
close_time: ((ts + periodSeconds * 1000) * 1_000_000).toString()
});
}
}
if (gapCount > 0) {
this.logger.info(
{ ticker, gapCount, total: allCandles.length },
'Filled interior gap bars for missing periods in source data'
'Forward-filled interior gap bars with previous close price'
);
}
@@ -264,24 +310,6 @@ export class CCXTFetcher {
};
}
/**
* Create a gap bar for a period with no trade data.
* All OHLC/volume fields are null — the timestamp slot is reserved but unpopulated.
*/
createGapBar(timestampMs, ticker, periodSeconds, metadata) {
return {
ticker,
timestamp: (timestampMs * 1_000_000).toString(), // Convert ms to nanoseconds
open: null,
high: null,
low: null,
close: null,
volume: null,
open_time: (timestampMs * 1_000_000).toString(),
close_time: ((timestampMs + periodSeconds * 1000) * 1_000_000).toString()
};
}
/**
* Convert CCXT trade to our Tick format
* Uses precision fields from market metadata for proper integer representation

9
ingestor/src/index.js
View File

@@ -285,7 +285,14 @@ class IngestorWorker {
} catch (error) {
this.logger.error(
{ error: error.message, request_id, ticker },
{
errorType: error.constructor?.name,
error: error.message,
errorUrl: error.url,
request_id,
ticker,
stack: error.stack
},
'Failed to process historical request'
);

12
ingestor/src/kafka-producer.js
View File

@@ -181,16 +181,14 @@ export class KafkaProducer {
errorMessage: metadata.error_message
},
rows: ohlcData.map(candle => {
// null open/high/low/close signals a gap bar (no trades that period).
// Omit fields from the protobuf message when null so hasOpen() etc. return false.
const row = {
timestamp: candle.timestamp,
ticker: candle.ticker,
ticker: candle.ticker,
open: candle.open,
high: candle.high,
low: candle.low,
close: candle.close,
};
if (candle.open != null) row.open = candle.open;
if (candle.high != null) row.high = candle.high;
if (candle.low != null) row.low = candle.low;
if (candle.close != null) row.close = candle.close;
if (candle.volume != null) row.volume = candle.volume;
return row;
})