dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data fixes, partial custom indicator support

Browse Source
This commit is contained in:
Tim Olson
2026-04-08 21:28:31 -04:00
parent b701554996
commit a70dcd954f
81 changed files with 5438 additions and 1852 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
ingestor/src/ccxt-fetcher.js
View File

@@ -45,6 +45,7 @@ export class CCXTFetcher {
const exchange = this.getExchange(exchangeName);
// Load market info from CCXT
this.logger.info({ exchangeName, symbol }, 'Loading markets for metadata');
await exchange.loadMarkets();
const market = exchange.market(symbol);
@@ -108,8 +109,9 @@ export class CCXTFetcher {
// Map period seconds to CCXT timeframe
const timeframe = this.secondsToTimeframe(periodSeconds);
const marketsLoaded = exchange.markets != null && Object.keys(exchange.markets).length > 0;
this.logger.info(
{ ticker, timeframe, startMs, endMs, limit },
{ ticker, timeframe, startMs, endMs, limit, marketsLoaded },
'Fetching historical OHLC'
);
@@ -120,44 +122,76 @@ export class CCXTFetcher {
// The caller's limit/countback is irrelevant to how much we need to fetch from the exchange.
const PAGE_SIZE = 1000;
const FETCH_RETRIES = 3;
const FETCH_RETRY_DELAY_MS = 5000;
while (since < endMs) {
try {
const candles = await exchange.fetchOHLCV(
symbol,
timeframe,
since,
PAGE_SIZE
);
if (candles.length === 0) {
let candles;
let lastError;
for (let attempt = 1; attempt <= FETCH_RETRIES; attempt++) {
try {
candles = await exchange.fetchOHLCV(symbol, timeframe, since, PAGE_SIZE);
lastError = null;
break;
} catch (error) {
lastError = error;
const isRetryable = error.constructor?.name === 'NetworkError' ||
error.constructor?.name === 'RequestTimeout' ||
error.constructor?.name === 'ExchangeNotAvailable';
this.logger.warn(
{
errorType: error.constructor?.name,
error: error.message,
errorUrl: error.url,
ticker,
since,
attempt,
retryable: isRetryable
},
'OHLC fetch attempt failed'
);
if (!isRetryable || attempt === FETCH_RETRIES) break;
await exchange.sleep(FETCH_RETRY_DELAY_MS * attempt);
}
// Filter candles within the requested time range
const filteredCandles = candles.filter(c => {
const timestamp = c[0];
return timestamp >= startMs && timestamp < endMs; // endMs is exclusive
});
fetchedCandles.push(...filteredCandles);
// Advance to next batch start
const lastTimestamp = candles[candles.length - 1][0];
since = lastTimestamp + (periodSeconds * 1000);
if (since >= endMs) {
break;
}
// Apply rate limiting
await exchange.sleep(exchange.rateLimit);
} catch (error) {
}
if (lastError) {
this.logger.error(
{ error: error.message, ticker, since },
{
errorType: lastError.constructor?.name,
error: lastError.message,
errorUrl: lastError.url,
ticker,
since,
marketsLoaded: exchange.markets != null && Object.keys(exchange.markets).length > 0,
stack: lastError.stack
},
'Error fetching OHLC'
);
throw error;
throw lastError;
}
if (candles.length === 0) {
break;
}
// Filter candles within the requested time range
const filteredCandles = candles.filter(c => {
const timestamp = c[0];
return timestamp >= startMs && timestamp < endMs; // endMs is exclusive
});
fetchedCandles.push(...filteredCandles);
// Advance to next batch start
const lastTimestamp = candles[candles.length - 1][0];
since = lastTimestamp + (periodSeconds * 1000);
if (since >= endMs) {
break;
}
// Apply rate limiting
await exchange.sleep(exchange.rateLimit);
}
// Get metadata for proper denomination
@@ -173,32 +207,44 @@ export class CCXTFetcher {
const periodMs = periodSeconds * 1000;
// Only create null gap bars for interior gaps — periods where real data exists
// on BOTH sides (i.e., between the first and last real bar). Do NOT append
// null bars before the first real bar or after the last real bar: those edge
// positions may be in-progress candles or simply outside the exchange's history,
// and we have no positive signal that a gap exists there.
// Forward-fill interior gaps — periods between the first and last real bar
// where the exchange returned no candle. Edge gaps (before firstRealTs or
// after lastRealTs) are left absent; they'll be caught by gap detection and
// trigger a targeted backfill request.
const realTimestamps = [...fetchedByTs.keys()].sort((a, b) => a - b);
const firstRealTs = realTimestamps[0];
const lastRealTs = realTimestamps[realTimestamps.length - 1];
const allCandles = [];
let gapCount = 0;
let prevClose = null;
for (let ts = firstRealTs; ts <= lastRealTs; ts += periodMs) {
if (fetchedByTs.has(ts)) {
allCandles.push(this.convertToOHLC(fetchedByTs.get(ts), ticker, periodSeconds, metadata));
} else {
// Interior gap — confirmed by real bars on both sides
const bar = this.convertToOHLC(fetchedByTs.get(ts), ticker, periodSeconds, metadata);
prevClose = bar.close;
allCandles.push(bar);
} else if (prevClose !== null) {
// Interior gap — forward-fill with previous close, zero volume
gapCount++;
allCandles.push(this.createGapBar(ts, ticker, periodSeconds, metadata));
allCandles.push({
ticker,
timestamp: (ts * 1_000_000).toString(),
open: prevClose,
high: prevClose,
low: prevClose,
close: prevClose,
volume: '0',
open_time: (ts * 1_000_000).toString(),
close_time: ((ts + periodSeconds * 1000) * 1_000_000).toString()
});
}
}
if (gapCount > 0) {
this.logger.info(
{ ticker, gapCount, total: allCandles.length },
'Filled interior gap bars for missing periods in source data'
'Forward-filled interior gap bars with previous close price'
);
}
@@ -264,24 +310,6 @@ export class CCXTFetcher {
};
}
/**
* Create a gap bar for a period with no trade data.
* All OHLC/volume fields are null — the timestamp slot is reserved but unpopulated.
*/
createGapBar(timestampMs, ticker, periodSeconds, metadata) {
return {
ticker,
timestamp: (timestampMs * 1_000_000).toString(), // Convert ms to nanoseconds
open: null,
high: null,
low: null,
close: null,
volume: null,
open_time: (timestampMs * 1_000_000).toString(),
close_time: ((timestampMs + periodSeconds * 1000) * 1_000_000).toString()
};
}
/**
* Convert CCXT trade to our Tick format
* Uses precision fields from market metadata for proper integer representation