dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data timeout fixes; research agent improvements

Browse Source
This commit is contained in:
Tim Olson
2026-04-24 20:43:42 -04:00
parent 1800363566
commit 319d81c41f
37 changed files with 672 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
ingestor/src/ccxt-fetcher.js
View File

@@ -156,12 +156,32 @@ export class CCXTFetcher {
const FETCH_RETRIES = 3;
const FETCH_RETRY_DELAY_MS = 5000;
// Binance provides extended kline data (buy/sell volume split, quote volume, trade count).
// We use the raw klines endpoint directly to capture all available fields.
const isBinance = exchangeName === 'binance';
let binanceMarketId = null;
if (isBinance) {
if (!exchange.markets || Object.keys(exchange.markets).length === 0) {
await exchange.loadMarkets();
}
binanceMarketId = exchange.market(symbol).id;
}
while (since < endMs) {
let candles;
let lastError;
for (let attempt = 1; attempt <= FETCH_RETRIES; attempt++) {
try {
candles = await exchange.fetchOHLCV(symbol, timeframe, since, PAGE_SIZE);
if (isBinance) {
candles = await exchange.publicGetKlines({
symbol: binanceMarketId,
interval: timeframe,
startTime: since,
limit: PAGE_SIZE
});
} else {
candles = await exchange.fetchOHLCV(symbol, timeframe, since, PAGE_SIZE);
}
lastError = null;
break;
} catch (error) {
@@ -267,7 +287,7 @@ export class CCXTFetcher {
} else if (prevClose !== null) {
// Interior gap — forward-fill with previous close, zero volume
gapCount++;
allCandles.push({
const gapBar = {
ticker,
timestamp: (ts * 1_000_000).toString(),
open: prevClose,
@@ -277,7 +297,14 @@ export class CCXTFetcher {
volume: '0',
open_time: (ts * 1_000_000).toString(),
close_time: ((ts + periodSeconds * 1000) * 1_000_000).toString()
});
};
if (isBinance) {
gapBar.buy_vol = '0';
gapBar.sell_vol = '0';
gapBar.num_trades = '0';
gapBar.quote_volume = '0';
}
allCandles.push(gapBar);
}
}
@@ -332,27 +359,54 @@ export class CCXTFetcher {
}
/**
* Convert CCXT OHLCV array to our OHLC format
* CCXT format: [timestamp, open, high, low, close, volume]
* Uses precision fields from market metadata for proper integer representation
* Convert OHLCV array to our OHLC format.
*
* Accepts two formats:
* - Standard CCXT (6 elements): [timestamp, open, high, low, close, volume]
* - Binance raw klines (12 elements): [openTime, open, high, low, close, baseVolume,
* closeTime, quoteVolume, numTrades, takerBuyBaseVol, takerBuyQuoteVol, ignore]
*
* Prices/volumes use integer representation scaled by market metadata precision.
*/
convertToOHLC(candle, ticker, periodSeconds, metadata) {
const [timestamp, open, high, low, close, volume] = candle;
const timestamp = Number(candle[0]);
const open = parseFloat(candle[1]);
const high = parseFloat(candle[2]);
const low = parseFloat(candle[3]);
const close = parseFloat(candle[4]);
const volume = parseFloat(candle[5]);
const priceMult = Math.pow(10, metadata.pricePrecision ?? 2);
const sizeMult = Math.pow(10, metadata.sizePrecision ?? 8);
const sizeMult = Math.pow(10, metadata.sizePrecision ?? 8);
return {
const result = {
ticker,
timestamp: (timestamp * 1_000_000).toString(), // Convert ms to nanoseconds
open: Math.round(open * priceMult).toString(),
high: Math.round(high * priceMult).toString(),
low: Math.round(low * priceMult).toString(),
timestamp: (timestamp * 1_000_000).toString(),
open: Math.round(open * priceMult).toString(),
high: Math.round(high * priceMult).toString(),
low: Math.round(low * priceMult).toString(),
close: Math.round(close * priceMult).toString(),
volume: Math.round(volume * sizeMult).toString(),
open_time: (timestamp * 1_000_000).toString(),
close_time: ((timestamp + periodSeconds * 1000) * 1_000_000).toString()
};
if (candle.length >= 10) {
// Binance extended klines format
const closeTimeMs = Number(candle[6]);
const quoteVolRaw = parseFloat(candle[7]);
const numTrades = Number(candle[8]);
const takerBuyBase = parseFloat(candle[9]);
result.close_time = (closeTimeMs * 1_000_000).toString();
result.quote_volume = Math.round(quoteVolRaw * priceMult).toString();
result.num_trades = numTrades.toString();
result.buy_vol = Math.round(takerBuyBase * sizeMult).toString();
result.sell_vol = Math.round((volume - takerBuyBase) * sizeMult).toString();
} else {
result.close_time = ((timestamp + periodSeconds * 1000) * 1_000_000).toString();
}
return result;
}
/**

14
ingestor/src/kafka-producer.js
View File

@@ -147,6 +147,12 @@ export class KafkaProducer {
close: candle.close,
volume: candle.volume
};
if (candle.buy_vol != null) protoCandle.buy_vol = candle.buy_vol;
if (candle.sell_vol != null) protoCandle.sell_vol = candle.sell_vol;
if (candle.open_time != null) protoCandle.open_time = candle.open_time;
if (candle.close_time != null) protoCandle.close_time = candle.close_time;
if (candle.num_trades != null) protoCandle.num_trades = candle.num_trades;
if (candle.quote_volume != null) protoCandle.quote_volume = candle.quote_volume;
const [frame1, frame2] = encodeMessage(MessageTypeId.OHLC, protoCandle, OHLC);
const value = Buffer.concat([frame1, frame2]);
@@ -188,7 +194,13 @@ export class KafkaProducer {
low: candle.low,
close: candle.close,
};
if (candle.volume != null) row.volume = candle.volume;
if (candle.volume != null) row.volume = candle.volume;
if (candle.buy_vol != null) row.buy_vol = candle.buy_vol;
if (candle.sell_vol != null) row.sell_vol = candle.sell_vol;
if (candle.open_time != null) row.open_time = candle.open_time;
if (candle.close_time != null) row.close_time = candle.close_time;
if (candle.num_trades != null) row.num_trades = candle.num_trades;
if (candle.quote_volume != null) row.quote_volume = candle.quote_volume;
return row;
})
};