chart data loading
This commit is contained in:
249
gateway/src/services/ohlc-service.ts
Normal file
249
gateway/src/services/ohlc-service.ts
Normal file
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* OHLC Service - High-level API for historical market data
|
||||
*
|
||||
* Workflow (mirroring client-py/dexorder/ohlc_client.py):
|
||||
* 1. Check Iceberg for existing data
|
||||
* 2. Identify missing ranges
|
||||
* 3. If complete, return immediately
|
||||
* 4. Otherwise, request missing data via relay
|
||||
* 5. Wait for completion notification
|
||||
* 6. Query Iceberg again for complete dataset
|
||||
* 7. Return results
|
||||
*
|
||||
* This provides transparent caching - clients don't need to know
|
||||
* whether data came from cache or was fetched on-demand.
|
||||
*/
|
||||
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { IcebergClient } from '../clients/iceberg-client.js';
|
||||
import type { ZMQRelayClient } from '../clients/zmq-relay-client.js';
|
||||
import type {
|
||||
HistoryResult,
|
||||
SymbolInfo,
|
||||
SearchResult,
|
||||
DatafeedConfig,
|
||||
TradingViewBar,
|
||||
} from '../types/ohlc.js';
|
||||
import {
|
||||
secondsToMicros,
|
||||
backendToTradingView,
|
||||
resolutionToSeconds,
|
||||
DEFAULT_SUPPORTED_RESOLUTIONS,
|
||||
} from '../types/ohlc.js';
|
||||
|
||||
export interface OHLCServiceConfig {
|
||||
icebergClient: IcebergClient;
|
||||
relayClient: ZMQRelayClient;
|
||||
logger: FastifyBaseLogger;
|
||||
requestTimeout?: number; // Request timeout in ms (default: 30000)
|
||||
}
|
||||
|
||||
/**
|
||||
* OHLC Service
|
||||
*
|
||||
* Provides high-level API for fetching OHLC data with smart caching.
|
||||
*/
|
||||
export class OHLCService {
|
||||
private icebergClient: IcebergClient;
|
||||
private relayClient: ZMQRelayClient;
|
||||
private logger: FastifyBaseLogger;
|
||||
|
||||
constructor(config: OHLCServiceConfig) {
|
||||
this.icebergClient = config.icebergClient;
|
||||
this.relayClient = config.relayClient;
|
||||
this.logger = config.logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch OHLC data with smart caching
|
||||
*
|
||||
* Steps:
|
||||
* 1. Query Iceberg for existing data
|
||||
* 2. If complete, return immediately
|
||||
* 3. If missing data, request via relay
|
||||
* 4. Wait for completion notification
|
||||
* 5. Query Iceberg again for complete dataset
|
||||
* 6. Return results
|
||||
*/
|
||||
async fetchOHLC(
|
||||
ticker: string,
|
||||
resolution: string,
|
||||
from_time: number, // Unix timestamp in SECONDS
|
||||
to_time: number, // Unix timestamp in SECONDS
|
||||
countback?: number
|
||||
): Promise<HistoryResult> {
|
||||
this.logger.debug({
|
||||
ticker,
|
||||
resolution,
|
||||
from_time,
|
||||
to_time,
|
||||
countback,
|
||||
}, 'Fetching OHLC data');
|
||||
|
||||
// Convert resolution to period_seconds
|
||||
const period_seconds = resolutionToSeconds(resolution);
|
||||
|
||||
// Convert times to microseconds
|
||||
const start_time = secondsToMicros(from_time);
|
||||
const end_time = secondsToMicros(to_time);
|
||||
|
||||
// Step 1: Check Iceberg for existing data
|
||||
let data = await this.icebergClient.queryOHLC(ticker, period_seconds, start_time, end_time);
|
||||
|
||||
// Step 2: Identify missing ranges
|
||||
const missingRanges = await this.icebergClient.findMissingOHLCRanges(
|
||||
ticker,
|
||||
period_seconds,
|
||||
start_time,
|
||||
end_time
|
||||
);
|
||||
|
||||
if (missingRanges.length === 0 && data.length > 0) {
|
||||
// All data exists in Iceberg
|
||||
this.logger.debug({ ticker, resolution, cached: true }, 'OHLC data found in cache');
|
||||
return this.formatHistoryResult(data, countback);
|
||||
}
|
||||
|
||||
// Step 3: Request missing data via relay
|
||||
this.logger.debug({ ticker, resolution, missingRanges: missingRanges.length }, 'Requesting missing OHLC data');
|
||||
|
||||
try {
|
||||
const notification = await this.relayClient.requestHistoricalOHLC(
|
||||
ticker,
|
||||
period_seconds,
|
||||
start_time,
|
||||
end_time,
|
||||
countback
|
||||
);
|
||||
|
||||
this.logger.info({
|
||||
ticker,
|
||||
resolution,
|
||||
row_count: notification.row_count,
|
||||
status: notification.status,
|
||||
}, 'Historical data request completed');
|
||||
|
||||
// Step 4: Query Iceberg again for complete dataset
|
||||
data = await this.icebergClient.queryOHLC(ticker, period_seconds, start_time, end_time);
|
||||
|
||||
return this.formatHistoryResult(data, countback);
|
||||
|
||||
} catch (error: any) {
|
||||
this.logger.error({
|
||||
error,
|
||||
ticker,
|
||||
resolution,
|
||||
}, 'Failed to fetch historical data');
|
||||
|
||||
// Return empty result on error
|
||||
return {
|
||||
bars: [],
|
||||
noData: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format OHLC data as TradingView history result
|
||||
*/
|
||||
private formatHistoryResult(data: any[], countback?: number): HistoryResult {
|
||||
if (data.length === 0) {
|
||||
return {
|
||||
bars: [],
|
||||
noData: true,
|
||||
};
|
||||
}
|
||||
|
||||
// Convert to TradingView format
|
||||
let bars: TradingViewBar[] = data.map(backendToTradingView);
|
||||
|
||||
// Sort by time
|
||||
bars.sort((a, b) => a.time - b.time);
|
||||
|
||||
// Apply countback limit if specified
|
||||
if (countback && bars.length > countback) {
|
||||
bars = bars.slice(-countback);
|
||||
}
|
||||
|
||||
return {
|
||||
bars,
|
||||
noData: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get datafeed configuration
|
||||
*/
|
||||
async getConfig(): Promise<DatafeedConfig> {
|
||||
return {
|
||||
supported_resolutions: DEFAULT_SUPPORTED_RESOLUTIONS,
|
||||
supports_search: true,
|
||||
supports_group_request: false,
|
||||
supports_marks: false,
|
||||
supports_timescale_marks: false,
|
||||
supports_time: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Search symbols
|
||||
*
|
||||
* For now, stub with default symbol
|
||||
*/
|
||||
async searchSymbols(
|
||||
query: string,
|
||||
type?: string,
|
||||
exchange?: string,
|
||||
limit: number = 30
|
||||
): Promise<SearchResult[]> {
|
||||
this.logger.debug({ query, type, exchange, limit }, 'Searching symbols');
|
||||
|
||||
// TODO: Implement central symbol registry
|
||||
// For now, return default symbol if query matches
|
||||
if (query.toLowerCase().includes('btc') || query.toLowerCase().includes('binance')) {
|
||||
return [{
|
||||
symbol: 'BINANCE:BTC/USDT',
|
||||
full_name: 'BINANCE:BTC/USDT',
|
||||
description: 'Bitcoin / Tether USD',
|
||||
exchange: 'BINANCE',
|
||||
ticker: 'BINANCE:BTC/USDT',
|
||||
type: 'crypto',
|
||||
}];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve symbol metadata
|
||||
*
|
||||
* For now, stub with default symbol
|
||||
*/
|
||||
async resolveSymbol(symbol: string): Promise<SymbolInfo> {
|
||||
this.logger.debug({ symbol }, 'Resolving symbol');
|
||||
|
||||
// TODO: Implement central symbol registry
|
||||
// For now, return default symbol info for BINANCE:BTC/USDT
|
||||
if (symbol === 'BINANCE:BTC/USDT' || symbol === 'BTC/USDT') {
|
||||
return {
|
||||
symbol: 'BINANCE:BTC/USDT',
|
||||
name: 'BINANCE:BTC/USDT',
|
||||
ticker: 'BINANCE:BTC/USDT',
|
||||
description: 'Bitcoin / Tether USD',
|
||||
type: 'crypto',
|
||||
session: '24x7',
|
||||
timezone: 'Etc/UTC',
|
||||
exchange: 'BINANCE',
|
||||
minmov: 1,
|
||||
pricescale: 100,
|
||||
has_intraday: true,
|
||||
has_daily: true,
|
||||
has_weekly_and_monthly: false,
|
||||
supported_resolutions: DEFAULT_SUPPORTED_RESOLUTIONS,
|
||||
data_status: 'streaming',
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Symbol not found: ${symbol}`);
|
||||
}
|
||||
}
|
||||
266
gateway/src/services/symbol-index-service.ts
Normal file
266
gateway/src/services/symbol-index-service.ts
Normal file
@@ -0,0 +1,266 @@
|
||||
/**
|
||||
* Symbol Index Service
|
||||
*
|
||||
* Provides fast in-memory search for symbol metadata.
|
||||
* Loads initial data from Iceberg and stays synced via Kafka subscription.
|
||||
*/
|
||||
|
||||
import type { FastifyBaseLogger } from 'fastify';
|
||||
import type { IcebergClient } from '../clients/iceberg-client.js';
|
||||
import type { SearchResult, SymbolInfo, SymbolMetadata } from '../types/ohlc.js';
|
||||
import { DEFAULT_SUPPORTED_RESOLUTIONS } from '../types/ohlc.js';
|
||||
|
||||
export interface SymbolIndexServiceConfig {
|
||||
icebergClient: IcebergClient;
|
||||
logger: FastifyBaseLogger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Symbol Index Service
|
||||
*
|
||||
* Maintains an in-memory index of all available symbols for fast search.
|
||||
*/
|
||||
export class SymbolIndexService {
|
||||
private icebergClient: IcebergClient;
|
||||
private logger: FastifyBaseLogger;
|
||||
private symbols: Map<string, SymbolMetadata> = new Map(); // key: "EXCHANGE:MARKET_ID"
|
||||
private initialized: boolean = false;
|
||||
|
||||
constructor(config: SymbolIndexServiceConfig) {
|
||||
this.icebergClient = config.icebergClient;
|
||||
this.logger = config.logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the index by loading symbols from Iceberg
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
this.logger.info('Initializing symbol index from Iceberg');
|
||||
|
||||
try {
|
||||
// Load all symbols from Iceberg symbol_metadata table
|
||||
const symbols = await this.icebergClient.queryAllSymbols();
|
||||
|
||||
this.logger.info({
|
||||
symbolsType: typeof symbols,
|
||||
symbolsIsArray: Array.isArray(symbols),
|
||||
symbolsLength: symbols?.length,
|
||||
firstSymbol: symbols[0]
|
||||
}, 'Loaded symbols debug info');
|
||||
|
||||
// Track unique keys for debugging
|
||||
const uniqueKeys = new Set<string>();
|
||||
|
||||
for (const symbol of symbols) {
|
||||
const key = `${symbol.exchange_id}:${symbol.market_id}`;
|
||||
uniqueKeys.add(key);
|
||||
this.symbols.set(key, symbol);
|
||||
}
|
||||
|
||||
this.initialized = true;
|
||||
this.logger.info({
|
||||
count: this.symbols.size,
|
||||
totalRows: symbols.length,
|
||||
uniqueKeys: uniqueKeys.size,
|
||||
sampleKeys: Array.from(uniqueKeys).slice(0, 5)
|
||||
}, 'Symbol index initialized');
|
||||
} catch (error: any) {
|
||||
this.logger.warn({ error: error.message }, 'Failed to initialize symbol index (will retry on first request)');
|
||||
// Don't throw - allow lazy loading
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure index is initialized (with retry on failure)
|
||||
*/
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (this.initialized) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.info('Lazy-loading symbol index');
|
||||
await this.initialize();
|
||||
}
|
||||
|
||||
/**
|
||||
* Update or add a symbol to the index
|
||||
*/
|
||||
updateSymbol(symbol: SymbolMetadata): void {
|
||||
const key = `${symbol.exchange_id}:${symbol.market_id}`;
|
||||
this.symbols.set(key, symbol);
|
||||
this.logger.debug({ key }, 'Updated symbol in index');
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for symbols matching a query
|
||||
*
|
||||
* Simple case-insensitive substring matching across:
|
||||
* - Ticker (EXCHANGE:MARKET_ID)
|
||||
* - Base asset
|
||||
* - Quote asset
|
||||
* - Description
|
||||
*/
|
||||
async search(query: string, limit: number = 30): Promise<SearchResult[]> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
if (!this.initialized) {
|
||||
this.logger.warn('Symbol index not initialized, returning empty results');
|
||||
return [];
|
||||
}
|
||||
|
||||
const queryLower = query.toLowerCase();
|
||||
const results: SearchResult[] = [];
|
||||
|
||||
for (const [key, metadata] of this.symbols) {
|
||||
// Match against various fields
|
||||
const ticker = key;
|
||||
const base = metadata.base_asset || '';
|
||||
const quote = metadata.quote_asset || '';
|
||||
const desc = metadata.description || '';
|
||||
const marketId = metadata.market_id || '';
|
||||
|
||||
if (
|
||||
ticker.toLowerCase().includes(queryLower) ||
|
||||
base.toLowerCase().includes(queryLower) ||
|
||||
quote.toLowerCase().includes(queryLower) ||
|
||||
desc.toLowerCase().includes(queryLower) ||
|
||||
marketId.toLowerCase().includes(queryLower)
|
||||
) {
|
||||
results.push(this.metadataToSearchResult(metadata));
|
||||
|
||||
if (results.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.debug({ query, count: results.length }, 'Symbol search completed');
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve symbol metadata by ticker
|
||||
*/
|
||||
async resolveSymbol(ticker: string): Promise<SymbolInfo | null> {
|
||||
await this.ensureInitialized();
|
||||
|
||||
if (!this.initialized) {
|
||||
this.logger.warn('Symbol index not initialized after retry');
|
||||
return null;
|
||||
}
|
||||
|
||||
// ticker format: "EXCHANGE:MARKET_ID" or just "MARKET_ID"
|
||||
let key = ticker;
|
||||
|
||||
// If no exchange prefix, search for first match
|
||||
if (!ticker.includes(':')) {
|
||||
for (const [k, metadata] of this.symbols) {
|
||||
if (metadata.market_id === ticker) {
|
||||
key = k;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const metadata = this.symbols.get(key);
|
||||
if (!metadata) {
|
||||
this.logger.debug({ ticker }, 'Symbol not found');
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.metadataToSymbolInfo(metadata);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert SymbolMetadata to SearchResult
|
||||
*/
|
||||
private metadataToSearchResult(metadata: SymbolMetadata): SearchResult {
|
||||
const symbol = metadata.market_id; // Clean format: "BTC/USDT"
|
||||
const ticker = `${metadata.exchange_id}:${metadata.market_id}`; // "BINANCE:BTC/USDT"
|
||||
const fullName = `${metadata.market_id} (${metadata.exchange_id})`;
|
||||
|
||||
return {
|
||||
symbol,
|
||||
ticker,
|
||||
full_name: fullName,
|
||||
description: metadata.description || `${metadata.base_asset}/${metadata.quote_asset} ${metadata.market_type || 'spot'} pair on ${metadata.exchange_id}`,
|
||||
exchange: metadata.exchange_id,
|
||||
type: metadata.market_type || 'spot',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert SymbolMetadata to SymbolInfo
|
||||
*/
|
||||
private metadataToSymbolInfo(metadata: SymbolMetadata): SymbolInfo {
|
||||
const symbol = metadata.market_id;
|
||||
const ticker = `${metadata.exchange_id}:${metadata.market_id}`;
|
||||
|
||||
// Convert supported_period_seconds to resolution strings
|
||||
const supportedResolutions = this.periodSecondsToResolutions(metadata.supported_period_seconds || []);
|
||||
|
||||
// Calculate pricescale from tick_denom
|
||||
// tick_denom is 10^n where n is the number of decimal places
|
||||
// pricescale is the same value
|
||||
const pricescale = metadata.tick_denom ? Number(metadata.tick_denom) : 100;
|
||||
|
||||
return {
|
||||
symbol,
|
||||
ticker,
|
||||
name: symbol,
|
||||
description: metadata.description || `${metadata.base_asset}/${metadata.quote_asset} ${metadata.market_type || 'spot'} pair on ${metadata.exchange_id}`,
|
||||
type: metadata.market_type || 'spot',
|
||||
exchange: metadata.exchange_id,
|
||||
timezone: 'Etc/UTC',
|
||||
session: '24x7',
|
||||
supported_resolutions: supportedResolutions.length > 0 ? supportedResolutions : DEFAULT_SUPPORTED_RESOLUTIONS,
|
||||
has_intraday: true,
|
||||
has_daily: true,
|
||||
has_weekly_and_monthly: false,
|
||||
pricescale,
|
||||
minmov: 1,
|
||||
base_currency: metadata.base_asset,
|
||||
quote_currency: metadata.quote_asset,
|
||||
data_status: 'streaming',
|
||||
tick_denominator: metadata.tick_denom ? Number(metadata.tick_denom) : undefined,
|
||||
base_denominator: metadata.base_denom ? Number(metadata.base_denom) : undefined,
|
||||
quote_denominator: metadata.quote_denom ? Number(metadata.quote_denom) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert period_seconds array to TradingView resolution strings
|
||||
*/
|
||||
private periodSecondsToResolutions(periods: number[]): string[] {
|
||||
const resolutions: string[] = [];
|
||||
|
||||
for (const seconds of periods) {
|
||||
if (seconds < 3600) {
|
||||
// Minutes
|
||||
resolutions.push(String(seconds / 60));
|
||||
} else if (seconds === 3600) {
|
||||
resolutions.push('60');
|
||||
} else if (seconds === 14400) {
|
||||
resolutions.push('240');
|
||||
} else if (seconds === 86400) {
|
||||
resolutions.push('1D');
|
||||
} else if (seconds === 604800) {
|
||||
resolutions.push('1W');
|
||||
} else if (seconds === 2592000) {
|
||||
resolutions.push('1M');
|
||||
}
|
||||
}
|
||||
|
||||
return resolutions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics about the symbol index
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
symbolCount: this.symbols.size,
|
||||
initialized: this.initialized,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user