import {convertTvResolution, loadOHLC} from './ohlc.js'; import {metadata} from "@/version.js"; import FlexSearch from "flexsearch"; import {useChartOrderStore} from "@/orderbuild.js"; import {useStore} from "@/store/store.js"; import {subOHLC, unsubOHLC} from "@/blockchain/ohlcs.js"; import {ohlcStart} from "@/charts/chart-misc.js"; import {timestamp} from "@/misc.js"; const DEBUG_LOGGING = false const log = DEBUG_LOGGING ? console.log : ()=>{} // this file manages connecting data to TradingView using their DataFeed API. // https://www.tradingview.com/charting-library-docs/latest/connecting_data/Datafeed-API/#integrate-datafeed-api // see ohlc.js for fetching dexorder ohlc files // in order of priority const quoteSymbols = [ 'USDT', 'USDC', 'TUSD', 'GUSD', 'BUSD', 'MUSD', 'DAI', 'CRVUSD', 'EURC', 'EURS', 'EURI', 'WBTC', 'WETH', ] let widget = null // Python code to generate VARIABLE_TICK_SIZE: // from decimal import Decimal // K=12 # powers above and below to include // D=5 # number of significant digits to display // ' '.join(f'{Decimal(10)**(k-D):f} {Decimal(10)**k:f}' for k in range(-K,K+1))+f' {Decimal(10)**(K+1-D)}' const VARIABLE_TICK_SIZE = '0.00000000000000001 0.000000000001 0.0000000000000001 0.00000000001 0.000000000000001 0.0000000001 0.00000000000001 0.000000001 0.0000000000001 0.00000001 0.000000000001 0.0000001 0.00000000001 0.000001 0.0000000001 0.00001 0.000000001 0.0001 0.00000001 0.001 0.0000001 0.01 0.000001 0.1 0.00001 1 0.0001 10 0.001 100 0.01 1000 0.1 10000 1 100000 10 1000000 100 10000000 1000 100000000 10000 1000000000 100000 10000000000 1000000 100000000000 10000000 1000000000000 100000000' // DatafeedConfiguration implementation const configurationData = { // Represents the resolutions for bars supported by your datafeed supported_resolutions: ['1', '3', '5', '10', '15', '30', '60', '120', '240', '480', '720', '1D', '2D', '3D', '1W'], // The `exchanges` arguments are used for the `searchSymbols` method if a user selects the exchange exchanges: [ // { // value: 'UNIv2', // name: 'Uniswap v2', // desc: 'Uniswap v2', // }, { value: 'UNIv3', name: 'Uniswap v3', desc: 'Uniswap v3', logo: 'https://upload.wikimedia.org/wikipedia/commons/e/e7/Uniswap_Logo.svg', }, ], // The `symbols_types` arguments are used for the `searchSymbols` method if a user selects this symbol type symbols_types: [ {name: 'swap', value: 'swap',}, ], }; const tokenMap = {} // todo needs chainId const poolMap = {} // todo needs chainId let _symbols = null // keyed by the concatenated hex addrs of the token pair e.g. '0xf3Ed85D882b5d9A67fC10dBf8f9AA991212983aA' + '0x6cdC5106DC100115E6C310539Fe44a61b3EEa6C4' const indexer = new FlexSearch.Document({ document: {id: 'id', index: ['fn', 'as[]', 'b', 'q', 'bs', 'qs', 'e', 'd']}, // this must match what is generated for the index object in addSymbol() charset: {split: /\W+/}, tokenize: 'forward', }) const indexes = {} const feeGroups = {} // keyed by ticker without the final fee field. values are a list of pools [[addr,fee],...] // The symbol key is the chain and base/quote: basically a "pair." It is only used by dexorder. The TradingView // symbol is keyed by the `ticker` which is defined as 'chain_id|pool_addr' for absolute uniqueness. export function tickerForOrder(chainId, order) { const [exchange, fee] = order.route return tickerKey(chainId, exchange, order.tokenIn, order.tokenOut, fee, true) } export function tickerKey(chainId, exchange, tokenAddrA, tokenAddrB, fee, chooseInversion=false ) { if (chooseInversion && invertedDefault(tokenAddrA, tokenAddrB) ) [tokenAddrA, tokenAddrB] = [tokenAddrB, tokenAddrA] // NOTE: the ticker key specifies a base and quote ordering, so there are two tickers per pool return `${chainId}|${exchange}|${tokenAddrA}|${tokenAddrB}|${fee}` } export function feelessTickerKey(ticker) { return ticker.split('|').slice(0, -1).join('|'); } function addSymbol(chainId, p, base, quote, inverted) { const symbol = base.s + quote.s const exchange = ['UNIv2', 'UNIv3'][p.e] const full_name = exchange + ':' + symbol // + '%' + formatFee(fee) const ticker = tickerKey(chainId, p.e, base.a, quote.a, p.f) // add the search index only if this is the natural, noninverted base/quote pair log('addSymbol', p, base, quote, inverted, ticker) const description = `${base.n} / ${quote.n} ${(p.f/10000).toFixed(2)}%` const type = 'swap' const decimals = inverted ? -p.d : p.d const symbolInfo = { key: ticker, ticker, chainId, address: p.a, exchangeId: p.e, full_name, symbol, description, exchange, type, inverted, base, quote, decimals, x:p.x, fee:p.f, }; _symbols[ticker] = symbolInfo const feelessKey = feelessTickerKey(ticker) if (feelessKey in feeGroups) { feeGroups[feelessKey].push([symbolInfo.address, symbolInfo.fee]) feeGroups[feelessKey].sort((a,b)=>a[1]-b[1]) } else feeGroups[feelessKey] = [[symbolInfo.address, symbolInfo.fee]] symbolInfo.feeGroup = feeGroups[feelessKey] if (defaultSymbol===null && !invertedDefault(symbolInfo.base.a, symbolInfo.quote.a)) defaultSymbol = _symbols[ticker] log('new symbol', ticker, _symbols[ticker]) } function buildSymbolIndex() { for (const symbol of Object.values(_symbols)) { if (invertedDefault(symbol.base.a, symbol.quote.a)) continue // don't search "upside down" pairs const feelessKey = feelessTickerKey(symbol.ticker) const feeGroup = feeGroups[feelessKey] const [_addr, medianFee] = feeGroup[Math.floor((feeGroup.length-1)/2)] if (symbol.fee !== medianFee) continue // show the pool with the median fee by default const ticker = symbol.ticker const longExchange = ['Uniswap v2', 'Uniswap v3',][symbol.exchangeId] if (ticker in indexes) { indexes[ticker].as.push(symbol.address) // add the pool address index } else { indexes[ticker] = { // key id: ticker, // addresses a: symbol.address, b: symbol.base.a, q: symbol.quote.a, // symbols fn: symbol.full_name, bs: symbol.base.s, qs: symbol.quote.s, e: symbol.exchange + ' ' + longExchange, d: symbol.description, } } } } // function formatFee(fee) { // let str = (fee / 10000).toFixed(2); // if (str.startsWith('0')) // start with the decimal point not a zero // str = str.slice(1) // return str // } export function invertedDefault(tokenAddrA, tokenAddrB) { // lower priority is more important (earlier in the list) const a = tokenMap[tokenAddrA]; const b = tokenMap[tokenAddrB]; if (!a) { log(`No token ${tokenAddrA} found`) return } if (!b) { log(`No token ${tokenAddrB} found`) return } let basePriority = quoteSymbols.indexOf(a.s) let quotePriority = quoteSymbols.indexOf(b.s) if (basePriority === -1) basePriority = Number.MAX_SAFE_INTEGER if (quotePriority === -1) quotePriority = Number.MAX_SAFE_INTEGER return basePriority < quotePriority } export function getAllSymbols() { if (_symbols===null) { const chainId = useStore().chainId; const md = metadata[chainId] if(!md) { log('could not get metadata for chain', chainId) return [] } _symbols = {} for (const t of md.t) tokenMap[t.a] = t md.p.forEach((p)=>{ poolMap[p.a] = p const base = tokenMap[p.b]; const quote = tokenMap[p.q]; if (!base) { log(`No token ${p.b} found`) return } if (!quote) { log(`No token ${p.q} found`) return } addSymbol(chainId, p, quote, base, true); addSymbol(chainId, p, base, quote, false); }) buildSymbolIndex() log('indexes', indexes) Object.values(indexes).forEach(indexer.add.bind(indexer)) } log('symbols', _symbols) return _symbols } function invertTicker(ticker) { const [chainId, exchange, base, quote, fee] = ticker.split('|') return tickerKey(chainId, exchange, quote, base, fee) } export function lookupSymbol(ticker) { // lookup by ticker which is "0xbaseAddress/0xquoteAddress" // todo tim lookup default base/quote pool const symbols = getAllSymbols(); if (!(ticker in symbols)) { // check the inverted symbol const orig = ticker ticker = invertTicker(ticker); if (!(ticker in symbols)) { console.error('no symbol found for ticker', orig, symbols) return null } } return symbols[ticker] } function checkBar(bar, msg) { // Everything should be positive let o_l = bar.open - bar.low let c_l = bar.close - bar.low let h_o = bar.high - bar.open let h_c = bar.high - bar.close let h_l = bar.high - bar.low if (o_l<0||c_l<0||h_o<0||h_c<0||h_l<0) { log(msg, "bar.high/low inconsistent:", bar) if (o_l<0) log("bar inconsistent: open-low: ", o_l) if (c_l<0) log("bar inconsistent: close-low: ", c_l) if (h_o<0) log("bar inconsistent: high-open: ", h_o) if (h_c<0) log("bar inconsistent: high-close:", h_c) if (h_l<0) log("bar inconsistent: high-low: ", h_l) } else { log(msg, "bar diffs:", bar) log("bar diff: open-low: ", o_l) log("bar diff: close-low: ", c_l) log("bar diff: high-open: ", h_o) log("bar diff: high-close:", h_c) log("bar diff: high-low: ", h_l) } } function invertOhlcs(bars) { const result = [] for (const bar of bars) { const h = bar.high result.push({ time: bar.time, open: 1/bar.open, high: 1/bar.low, low: 1/h, close: 1/bar.close, }) } return result } const subByTvSubId = {} const subByKey = {} class RealtimeSubscription { constructor(chainId, poolAddr, res, symbol, tvSubId, onRealtimeCb, onResetCacheCb ) { this.chainId = chainId this.poolAddr = poolAddr this.res = res this.symbol = symbol this.tvSubId = tvSubId this.onRealtimeCb = onRealtimeCb this.onResetCacheCb = onResetCacheCb this.key = `${chainId}|${poolAddr}|${res.name}` subByTvSubId[this.tvSubId] = this subByKey[this.key] = this } close() { delete subByTvSubId[this.tvSubId] delete subByKey[this.key] } } export const DataFeed = { onReady(callback) { log('[onReady]: Method call'); setTimeout(() => callback(configurationData)); }, async searchSymbols( userInput, exchange, symbolType, onResultReadyCallback, ) { log('[searchSymbols]: Method call'); // todo limit results by exchange. use a separate indexer per exchange? const found = indexer.search(userInput, 10) log('found', found) const result = [] const seen = {} for (const f of found) for (const ticker of f.result) if (!(ticker in seen)) { result.push(_symbols[ticker]) seen[ticker] = true } onResultReadyCallback(result); }, async resolveSymbol( symbolName, onSymbolResolvedCallback, onResolveErrorCallback, extension ) { setTimeout(async ()=> await this.doResolveSymbol(symbolName,onSymbolResolvedCallback,onResolveErrorCallback,extension), 0) }, async doResolveSymbol( symbolName, onSymbolResolvedCallback, onResolveErrorCallback, extension ) { log('[resolveSymbol]: Method call', symbolName); const symbols = getAllSymbols(); const symbolItem = symbolName === 'default' ? defaultSymbol : symbols[symbolName] if (!symbolItem) { log('[resolveSymbol]: Cannot resolve symbol', symbolName); onResolveErrorCallback('cannot resolve symbol'); return; } const co = useChartOrderStore(); co.selectedSymbol = symbolItem const feelessKey = feelessTickerKey(symbolItem.ticker) const symbolsByFee = feeGroups[feelessKey] symbolsByFee.sort((a,b)=>a.fee-b.fee) const pool = symbolsByFee[Math.floor((symbolsByFee.length - 1)/2)] // median rounded down // noinspection JSValidateTypes co.selectedPool = pool // todo remove // LibrarySymbolInfo // https://www.tradingview.com/charting-library-docs/latest/api/interfaces/Charting_Library.LibrarySymbolInfo const symbolInfo = { ticker: symbolItem.ticker, name: symbolItem.symbol, pro_name: symbolItem.full_name, description: symbolItem.description, type: symbolItem.type, session: '24x7', timezone: 'Etc/UTC', exchange: symbolItem.exchange, minmov: .00000000000000001, pricescale: 1, variable_tick_size: VARIABLE_TICK_SIZE, has_intraday: true, // Added to allow less than one day to work visible_plots_set: 'ohlc', has_weekly_and_monthly: true, // Added to allow greater than one day to work supported_resolutions: configurationData.supported_resolutions, // volume_precision: 2, data_status: 'streaming', }; log('[resolveSymbol]: Symbol resolved', symbolName); onSymbolResolvedCallback(symbolInfo) }, async getBars(symbolInfo, resolution, periodParams, onHistoryCallback, onErrorCallback) { const { from, to, firstDataRequest } = periodParams; log('[getBars]: Method call', symbolInfo, resolution, from, to); try { // todo need to consider the selected fee tier await getAllSymbols() const symbol = lookupSymbol(symbolInfo.ticker); const poolAddr = symbol.address const res = convertTvResolution(resolution) const key = `${useStore().chainId}|${poolAddr}|${res.name}` let bars = await loadOHLC(symbol, poolAddr, from, to, resolution); // This is the one that does all the work this.updatePushedCache(key, bars) if (symbol.inverted) bars = invertOhlcs(bars) log(`[getBars]: returned ${bars.length} bar(s), and metadata ${metadata}`); log('[getBars]: bars=', bars); // noData should be set only if no bars are in the requested period and earlier. // In our case, we are guaranteed to have contiguous samples. // So we only return no bars (bars.length==0) if: // 1. period is entirely before first data available. // 2. period is entirely after last data available. // Returning noData based on bars.length works perfectly assuming that TV never asks for case 2. // This is probably not a safe assumption. The alternative would be to search // backward to find beginning of history. How far to search? const noData = bars.length === 0; onHistoryCallback(bars, {noData}); } catch (error) { log('[getBars]: Get error', error); onErrorCallback(error); } }, subscribeBarsOnRealtimeCallback: null, subscribeBars( symbolInfo, resolution, onRealtimeCallback2, subscriberUID, onResetCacheNeededCallback, ) { const oldCb = onRealtimeCallback2 function onRealtimeCallback() { log('push bar', ...arguments) oldCb(...arguments) } log('[subscribeBars]', symbolInfo, resolution, subscriberUID); const symbol = getAllSymbols()[symbolInfo.ticker] const poolAddr = symbol.address const chainId = useStore().chainId; const res = convertTvResolution(resolution) const period = res.name log('subscription symbol', symbol, chainId, poolAddr, res) const sub = new RealtimeSubscription(chainId, poolAddr, res, symbol, subscriberUID, onRealtimeCallback, onResetCacheNeededCallback) log('sub', sub.key) subOHLC(chainId, poolAddr, period) this.startOHLCBumper() }, unsubscribeBars(subscriberUID) { log('[unsubscribeBars]', subscriberUID); const sub = subByTvSubId[subscriberUID] if (sub===undefined) { console.log(`warning: no subscription found for tvSubId ${subscriberUID}`) return } unsubOHLC(sub.chainId, sub.poolAddr, sub.res.name) log('unsub',sub.key) sub.close() delete this.pushedBars[sub.key] delete this.recentBars[sub.key] }, // key-value format: // 'chain|pool|period': [[javatime, open, high, low, close], ...] pushedBars: {}, // bars actually sent to TradingView recentBars:{}, // bars received from server notifications pushRecentBars(key, recent) { this.recentBars[key] = recent const historical = this.pushedBars[key] if (historical!==undefined) this.overlapAndPush(key, recent) }, overlapAndPush(key, ohlcs) { log('overlapAndPush',key,ohlcs) if (!ohlcs || ohlcs.length === 0) return const sub = subByKey[key] // do not check reorgs on mock symbols because the dev environment price will be different from the main chain // price and reorgs would happen every time. const checkReorg = sub.symbol.x?.data === undefined const res = sub.res const period = res.seconds * 1000 const bars = [] // to push const pushed = this.pushedBars[key] log('got pushed bars', pushed) let pi = 0 // pushed index let time let price if (pushed === undefined) { time = ohlcs[0].time price = ohlcs[0].close } else { const last = pushed.length - 1 time = pushed[last].time price = pushed[last].close } log('last time/price', time, price) for( const ohlc of ohlcs ) { log('handle ohlc', ohlc) // forward the pi index to at least the current ohlc time while( pi < pushed.length && pushed[pi].time < ohlc.time ) pi++ if (pi < pushed.length-1) { // finalized bars must match the previous push exactly or else there was a reorg and we need to reset const p = pushed[pi] log('check reorg', checkReorg, pi, p, ohlc) if (checkReorg && (p.time !== ohlc.time || p.open !== ohlc.open || p.high !== ohlc.high || p.low !== ohlc.low || p.close !== ohlc.close) ) { console.log('RESET TV CACHE') return this.resetCache(key) } } else { // the last pushed bar and anything after it can be sent to TV while (time + period < ohlc.time) { // fill gap time += period const bar = {time, open: price, high: price, low: price, close: price}; log('fill', bar) bars.push(bar) } bars.push(ohlc) time = ohlc.time price = ohlc.close } } return this.pushToTV(key, bars) }, resetCache(key) { log('resetting TV data cache') const sub = subByKey[key] if (sub===undefined) { console.log(`warning: no subscription found for dexorder key ${key}`) return } sub.onResetCacheCb() }, updatePushedCache(key, ohlcs) { if (ohlcs.length===0) return log('updatePushedCache', key, ohlcs) if (!(key in this.pushedBars)) this.pushedBars[key] = ohlcs else { const prev = this.pushedBars[key] log('prev pushed', prev) const endTime = prev[prev.length - 1].time; if (endTime === ohlcs[0].time) // the new ohlc's overlap the old time, so exclude the most recent historical item, which gets replaced this.pushedBars[key] = [...prev.slice(0, -1), ...ohlcs] else if (endTime <= ohlcs[ohlcs.length-1].time) { // no overlap of any bars. full append. this.pushedBars[key] = [...prev, ...ohlcs] } // otherwise the ohlc's being pushed are =>older<= than the ones in the pushedBars cache. This happens // because TV can request data pages (using getBars()) out of chronological order. } }, pushToTV(key, ohlcs) { log('pushing bars to tv', ohlcs) if (ohlcs.length===0) return this.updatePushedCache(key, ohlcs); // we cache the raw bars before inversion so they match dexorder data sources const sub = subByKey[key] if (sub===undefined) { console.log(`warning: could not find subscription for dexorder sub key ${key}`) return } if (sub.symbol.inverted) ohlcs = invertOhlcs(ohlcs) for (const ohlc of ohlcs) sub.onRealtimeCb(ohlc) }, poolCallback(chainId, poolPeriod, ohlcs) { if (!ohlcs || ohlcs.length===0) return const key = `${chainId}|${poolPeriod}`; const bars = [] for (const ohlc of ohlcs) { let close = parseFloat(ohlc[4]) // close const bar = { time: ohlc[0] * 1000, open: ohlc[1] ? parseFloat(ohlc[1]) : close, // open high: ohlc[2] ? parseFloat(ohlc[2]) : close, // high low: ohlc[3] ? parseFloat(ohlc[3]) : close, // low close: close, } bars.push(bar) } return this.pushRecentBars(key, bars) }, intervalChanged(seconds) { // rollover bumper // this timer function creates a new bar when the period rolls over this.startOHLCBumper() }, // The OHLC bumper advances bars at the end of the period. If the price doesn't change, no new data will arrive, so the // new bar is implicit and must be generated dynamically. startOHLCBumper() { const co = useChartOrderStore() if (_rolloverBumper !== null) clearTimeout(_rolloverBumper) const period = co.intervalSecs; if (period === 0) return const now = Date.now() const nextRollover = ohlcStart(now/1000 + period)*1000 + 2000 // two second delay to wait for server data const delay = nextRollover - now _rolloverBumper = setTimeout(this.bumpOHLC.bind(this), delay) }, bumpOHLC() { log('bumpOHLC') _rolloverBumper = null const secs = useChartOrderStore().intervalSecs; for (const sub of Object.values(subByKey)) { log('check bump', sub.res.seconds, secs) if (sub.res.seconds === secs) { const pushed = this.pushedBars[sub.key] log('check pushed', pushed) if (pushed !== undefined && pushed.length > 0) { const lastBar = pushed[pushed.length - 1] const price = lastBar.close const now = timestamp() * 1000 const period = sub.res.seconds * 1000 const fills = [] for( let time=lastBar.time + period; time < now; time += period ) { log('pushing bump', time, price) const bar = {time, open: price, high: price, low: price, close: price} fills.push(bar) } this.pushToTV(sub.key, fills) } else { log('warning: bumpOHLC() found no previous bars') } } } this.startOHLCBumper() }, } let _rolloverBumper = null let defaultSymbol = null