backend redesign
This commit is contained in:
248
ingestor/src/ccxt-fetcher.js
Normal file
248
ingestor/src/ccxt-fetcher.js
Normal file
@@ -0,0 +1,248 @@
|
||||
// CCXT data fetcher for historical OHLC and realtime ticks
|
||||
import ccxt from 'ccxt';
|
||||
|
||||
export class CCXTFetcher {
|
||||
constructor(config, logger) {
|
||||
this.config = config;
|
||||
this.logger = logger;
|
||||
this.exchanges = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse ticker string to exchange and symbol
|
||||
* Expected format: "EXCHANGE:SYMBOL" (e.g., "BINANCE:BTC/USDT")
|
||||
*/
|
||||
parseTicker(ticker) {
|
||||
const parts = ticker.split(':');
|
||||
if (parts.length !== 2) {
|
||||
throw new Error(`Invalid ticker format: ${ticker}. Expected "EXCHANGE:SYMBOL"`);
|
||||
}
|
||||
|
||||
return {
|
||||
exchange: parts[0].toLowerCase(),
|
||||
symbol: parts[1]
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create CCXT exchange instance
|
||||
*/
|
||||
getExchange(exchangeName) {
|
||||
if (this.exchanges.has(exchangeName)) {
|
||||
return this.exchanges.get(exchangeName);
|
||||
}
|
||||
|
||||
// Create exchange instance
|
||||
const ExchangeClass = ccxt[exchangeName];
|
||||
if (!ExchangeClass) {
|
||||
throw new Error(`Unsupported exchange: ${exchangeName}`);
|
||||
}
|
||||
|
||||
const exchange = new ExchangeClass({
|
||||
enableRateLimit: true,
|
||||
options: {
|
||||
defaultType: 'spot'
|
||||
}
|
||||
});
|
||||
|
||||
this.exchanges.set(exchangeName, exchange);
|
||||
this.logger.info({ exchange: exchangeName }, 'Created CCXT exchange instance');
|
||||
|
||||
return exchange;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch historical OHLC data
|
||||
* @param {string} ticker - Ticker in format "EXCHANGE:SYMBOL"
|
||||
* @param {string} startTime - Start time in microseconds
|
||||
* @param {string} endTime - End time in microseconds
|
||||
* @param {number} periodSeconds - OHLC period in seconds
|
||||
* @param {number} limit - Optional limit on number of candles
|
||||
* @returns {Promise<Array>} Array of OHLC candles
|
||||
*/
|
||||
async fetchHistoricalOHLC(ticker, startTime, endTime, periodSeconds, limit) {
|
||||
const { exchange: exchangeName, symbol } = this.parseTicker(ticker);
|
||||
const exchange = this.getExchange(exchangeName);
|
||||
|
||||
// Convert microseconds to milliseconds
|
||||
const startMs = Math.floor(parseInt(startTime) / 1000);
|
||||
const endMs = Math.floor(parseInt(endTime) / 1000);
|
||||
|
||||
// Map period seconds to CCXT timeframe
|
||||
const timeframe = this.secondsToTimeframe(periodSeconds);
|
||||
|
||||
this.logger.info(
|
||||
{ ticker, timeframe, startMs, endMs, limit },
|
||||
'Fetching historical OHLC'
|
||||
);
|
||||
|
||||
const allCandles = [];
|
||||
let since = startMs;
|
||||
|
||||
// CCXT typically limits to 1000 candles per request
|
||||
const batchSize = limit || 1000;
|
||||
|
||||
while (since < endMs) {
|
||||
try {
|
||||
const candles = await exchange.fetchOHLCV(
|
||||
symbol,
|
||||
timeframe,
|
||||
since,
|
||||
batchSize
|
||||
);
|
||||
|
||||
if (candles.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Filter candles within the time range
|
||||
const filteredCandles = candles.filter(c => {
|
||||
const timestamp = c[0];
|
||||
return timestamp >= startMs && timestamp <= endMs;
|
||||
});
|
||||
|
||||
allCandles.push(...filteredCandles);
|
||||
|
||||
// Move to next batch
|
||||
const lastTimestamp = candles[candles.length - 1][0];
|
||||
since = lastTimestamp + (periodSeconds * 1000);
|
||||
|
||||
// Break if we've reached the end time or limit
|
||||
if (since >= endMs || (limit && allCandles.length >= limit)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Apply rate limiting
|
||||
await exchange.sleep(exchange.rateLimit);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ error: error.message, ticker, since },
|
||||
'Error fetching OHLC'
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to our OHLC format
|
||||
return allCandles.map(candle => this.convertToOHLC(candle, ticker, periodSeconds));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch recent trades for realtime tick data
|
||||
* @param {string} ticker - Ticker in format "EXCHANGE:SYMBOL"
|
||||
* @param {string} since - Optional timestamp in microseconds to fetch from
|
||||
* @returns {Promise<Array>} Array of trade ticks
|
||||
*/
|
||||
async fetchRecentTrades(ticker, since = null) {
|
||||
const { exchange: exchangeName, symbol } = this.parseTicker(ticker);
|
||||
const exchange = this.getExchange(exchangeName);
|
||||
|
||||
try {
|
||||
// Convert microseconds to milliseconds if provided
|
||||
const sinceMs = since ? Math.floor(parseInt(since) / 1000) : undefined;
|
||||
|
||||
const trades = await exchange.fetchTrades(symbol, sinceMs, 1000);
|
||||
|
||||
this.logger.debug(
|
||||
{ ticker, count: trades.length },
|
||||
'Fetched recent trades'
|
||||
);
|
||||
|
||||
// Convert to our Tick format
|
||||
return trades.map(trade => this.convertToTick(trade, ticker));
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
{ error: error.message, ticker },
|
||||
'Error fetching trades'
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert CCXT OHLCV array to our OHLC format
|
||||
* CCXT format: [timestamp, open, high, low, close, volume]
|
||||
*/
|
||||
convertToOHLC(candle, ticker, periodSeconds) {
|
||||
const [timestamp, open, high, low, close, volume] = candle;
|
||||
|
||||
// Convert to fixed-point integers (using 8 decimal places = 10^8)
|
||||
const DENOM = 100000000;
|
||||
|
||||
return {
|
||||
ticker,
|
||||
timestamp: (timestamp * 1000).toString(), // Convert ms to microseconds
|
||||
open: Math.round(open * DENOM).toString(),
|
||||
high: Math.round(high * DENOM).toString(),
|
||||
low: Math.round(low * DENOM).toString(),
|
||||
close: Math.round(close * DENOM).toString(),
|
||||
volume: Math.round(volume * DENOM).toString(),
|
||||
open_time: (timestamp * 1000).toString(),
|
||||
close_time: ((timestamp + periodSeconds * 1000) * 1000).toString()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert CCXT trade to our Tick format
|
||||
*/
|
||||
convertToTick(trade, ticker) {
|
||||
// Convert to fixed-point integers (using 8 decimal places = 10^8)
|
||||
const DENOM = 100000000;
|
||||
|
||||
const price = Math.round(trade.price * DENOM);
|
||||
const amount = Math.round(trade.amount * DENOM);
|
||||
const quoteAmount = Math.round((trade.price * trade.amount) * DENOM);
|
||||
|
||||
return {
|
||||
trade_id: trade.id || `${trade.timestamp}`,
|
||||
ticker,
|
||||
timestamp: (trade.timestamp * 1000).toString(), // Convert ms to microseconds
|
||||
price: price.toString(),
|
||||
amount: amount.toString(),
|
||||
quote_amount: quoteAmount.toString(),
|
||||
taker_buy: trade.side === 'buy',
|
||||
sequence: trade.order ? trade.order.toString() : undefined
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert period seconds to CCXT timeframe string
|
||||
*/
|
||||
secondsToTimeframe(seconds) {
|
||||
const timeframes = {
|
||||
60: '1m',
|
||||
300: '5m',
|
||||
900: '15m',
|
||||
1800: '30m',
|
||||
3600: '1h',
|
||||
7200: '2h',
|
||||
14400: '4h',
|
||||
21600: '6h',
|
||||
28800: '8h',
|
||||
43200: '12h',
|
||||
86400: '1d',
|
||||
259200: '3d',
|
||||
604800: '1w',
|
||||
2592000: '1M'
|
||||
};
|
||||
|
||||
const timeframe = timeframes[seconds];
|
||||
if (!timeframe) {
|
||||
throw new Error(`Unsupported period: ${seconds} seconds`);
|
||||
}
|
||||
|
||||
return timeframe;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close all exchange connections
|
||||
*/
|
||||
async close() {
|
||||
for (const [name, exchange] of this.exchanges) {
|
||||
if (exchange.close) {
|
||||
await exchange.close();
|
||||
}
|
||||
}
|
||||
this.exchanges.clear();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user