import {convertTvResolution, loadOHLC} from './ohlc.js'; import {metadata} from "@/version.js"; import FlexSearch from "flexsearch"; import {useChartOrderStore} from "@/orderbuild.js"; import {useStore} from "@/store/store.js"; import {subOHLC, unsubOHLC} from "@/blockchain/ohlcs.js"; import {ohlcStart} from "@/charts/chart-misc.js"; import {timestamp} from "@/misc.js"; const DEBUG_LOGGING = false const log = DEBUG_LOGGING ? console.log : ()=>{} // this file manages connecting data to TradingView using their DataFeed API. // https://www.tradingview.com/charting-library-docs/latest/connecting_data/Datafeed-API/#integrate-datafeed-api // see ohlc.js for fetching dexorder ohlc files // in order of priority const quoteSymbols = [ 'USDT', 'USDC', 'TUSD', 'GUSD', 'BUSD', 'MUSD', 'DAI', 'CRVUSD', 'EURC', 'EURS', 'EURI', 'WBTC', 'WETH', ] let feeDropdown = null let widget = null // Python code to generate VARIABLE_TICK_SIZE: // from decimal import Decimal // K=12 # powers above and below to include // D=5 # number of significant digits to display // ' '.join(f'{Decimal(10)**(k-D):f} {Decimal(10)**k:f}' for k in range(-K,K+1))+f' {Decimal(10)**(K+1-D)}' const VARIABLE_TICK_SIZE = '0.00000000000000001 0.000000000001 0.0000000000000001 0.00000000001 0.000000000000001 0.0000000001 0.00000000000001 0.000000001 0.0000000000001 0.00000001 0.000000000001 0.0000001 0.00000000001 0.000001 0.0000000001 0.00001 0.000000001 0.0001 0.00000001 0.001 0.0000001 0.01 0.000001 0.1 0.00001 1 0.0001 10 0.001 100 0.01 1000 0.1 10000 1 100000 10 1000000 100 10000000 1000 100000000 10000 1000000000 100000 10000000000 1000000 100000000000 10000000 1000000000000 100000000' export function initFeeDropdown(w) { widget = w widget.createDropdown( { title: 'Fees', tooltip: 'Choose Fee Tier', items: [/*{title: 'Automatic Fee Selection', onSelect: () => {log('autofees')}}*/], icon: ``, } ).then(dropdown => {feeDropdown = dropdown; updateFeeDropdown()}) } function updateFeeDropdown() { if (feeDropdown===null) return const symbolItem = useChartOrderStore().selectedSymbol let items if (symbolItem===null) items = [{title: '0.00%'}] else { items = symbolItem.pools.map((p)=> { return { title: (p[1]/10000).toFixed(2)+'%', onSelect: ()=>selectPool(p), } }) } feeDropdown.applyOptions({items}) } function selectPool(p) { const co = useChartOrderStore(); if ( co.selectedPool === null || co.selectedPool[0] !== p[0] || co.selectedPool[1] !== p[1]) { co.selectedPool = p } } const lastBarsCache = new Map(); // DatafeedConfiguration implementation const configurationData = { // Represents the resolutions for bars supported by your datafeed supported_resolutions: ['1', '3', '5', '10', '15', '30', '60', '120', '240', '480', '720', '1D', '2D', '3D', '1W'], // The `exchanges` arguments are used for the `searchSymbols` method if a user selects the exchange exchanges: [ // { // value: 'UNIv2', // name: 'Uniswap v2', // desc: 'Uniswap v2', // }, { value: 'UNIv3', name: 'Uniswap v3', desc: 'Uniswap v3', logo: 'https://upload.wikimedia.org/wikipedia/commons/e/e7/Uniswap_Logo.svg', }, ], // The `symbols_types` arguments are used for the `searchSymbols` method if a user selects this symbol type symbols_types: [ {name: 'swap', value: 'swap',}, ], }; const tokenMap = {} const poolMap = {} let _symbols = null // keyed by the concatenated hex addrs of the token pair e.g. '0xf3Ed85D882b5d9A67fC10dBf8f9AA991212983aA' + '0x6cdC5106DC100115E6C310539Fe44a61b3EEa6C4' const indexer = new FlexSearch.Document({ document: {id: 'id', index: ['fn', 'as[]', 'b', 'q', 'bs', 'qs', 'e', 'd']}, // this must match what is generated for the index object in addSymbol() charset: {split: /\W+/}, tokenize: 'forward', }) const indexes = {} const symbolsSeen = {} // keyed by (base,quote) so we only list one pool per pair even if there are many fee tiers // todo add chainIds to all these keys function addSymbol(p, base, quote, inverted) { const symbol = base.s + quote.s const exchange = ['UNIv2', 'UNIv3'][p.e] const full_name = exchange + ':' + symbol // + '%' + formatFee(fee) const key = `${base.a}/${quote.a}` log('addSymbol', p, base, quote, inverted, key) if (symbolsSeen[key]) { // add this pool's address to the existing symbol as an additional fee tier const symbolInfo = _symbols[key]; symbolInfo.pools.push([p.a, p.f]) symbolInfo.pools.sort((a,b)=>a[1]-b[1]) log('integrated symbol', symbolInfo) indexes[key].as.push(p.a) return } symbolsSeen[key] = true const longExchange = ['Uniswap v2', 'Uniswap v3',][p.e] const description = `${base.n} / ${quote.n}` const type = 'swap' const pools = [[p.a, p.f]] const decimals = p.d _symbols[key] = { ticker: key, full_name, symbol, description, exchange, type, inverted, base, quote, pools, decimals, x:p.x } if (defaultSymbol===null) defaultSymbol = _symbols[key] log('new symbol', key, _symbols[key]) indexes[key] = { // key id: key, // addresses as: [p.a], // multiple pool addrs for each fee tier b: p.b, q: p.q, // symbols fn: full_name, bs: base.s, qs: quote.s, e: exchange + ' ' + longExchange, d: description, } } // function formatFee(fee) { // let str = (fee / 10000).toFixed(2); // if (str.startsWith('0')) // start with the decimal point not a zero // str = str.slice(1) // return str // } function getAllSymbols() { if (_symbols===null) { const chainId = useStore().chainId; const md = metadata[chainId] if(!md) { log('could not get metadata for chain', chainId) return [] } _symbols = {} for (const t of md.t) tokenMap[t.a] = t md.p.forEach((p)=>{ poolMap[p.a] = p const base = tokenMap[p.b]; const quote = tokenMap[p.q]; if (!base) { log(`No token ${p.b} found`) return } if (!quote) { log(`No token ${p.q} found`) return } // todo check quotes symbol list for inversion hint let basePriority = quoteSymbols.indexOf(base.s) let quotePriority = quoteSymbols.indexOf(quote.s) if (basePriority === -1) basePriority = Number.MAX_SAFE_INTEGER if (quotePriority === -1) quotePriority = Number.MAX_SAFE_INTEGER const showInverted = basePriority < quotePriority if (showInverted) addSymbol(p, quote, base, true); else addSymbol(p, base, quote, false); }) log('indexes', indexes) Object.values(indexes).forEach(indexer.add.bind(indexer)) } log('symbols', _symbols) return _symbols } export function lookupSymbol(key) { // lookup by fullname return getAllSymbols()[key] } function checkBar(bar, msg) { // Everything should be positive let o_l = bar.open - bar.low let c_l = bar.close - bar.low let h_o = bar.high - bar.open let h_c = bar.high - bar.close let h_l = bar.high - bar.low if (o_l<0||c_l<0||h_o<0||h_c<0||h_l<0) { log(msg, "bar.high/low inconsistent:", bar) if (o_l<0) log("bar inconsistent: open-low: ", o_l) if (c_l<0) log("bar inconsistent: close-low: ", c_l) if (h_o<0) log("bar inconsistent: high-open: ", h_o) if (h_c<0) log("bar inconsistent: high-close:", h_c) if (h_l<0) log("bar inconsistent: high-low: ", h_l) } else { log(msg, "bar diffs:", bar) log("bar diff: open-low: ", o_l) log("bar diff: close-low: ", c_l) log("bar diff: high-open: ", h_o) log("bar diff: high-close:", h_c) log("bar diff: high-low: ", h_l) } } function invertOhlcs(bars) { const result = [] for (const bar of bars) { const h = bar.high result.push({ time: bar.time, open: 1/bar.open, high: 1/bar.low, low: 1/h, close: 1/bar.close, }) } return result } const subByTvSubId = {} const subByKey = {} class RealtimeSubscription { constructor(chainId, poolAddr, res, symbol, tvSubId, onRealtimeCb, onResetCacheCb ) { this.chainId = chainId this.poolAddr = poolAddr this.res = res this.symbol = symbol this.tvSubId = tvSubId this.onRealtimeCb = onRealtimeCb this.onResetCacheCb = onResetCacheCb this.key = `${chainId}|${poolAddr}|${res.name}` subByTvSubId[this.tvSubId] = this subByKey[this.key] = this } close() { delete subByTvSubId[this.tvSubId] delete subByKey[this.key] } } export const DataFeed = { onReady(callback) { log('[onReady]: Method call'); setTimeout(() => callback(configurationData)); }, async searchSymbols( userInput, exchange, symbolType, onResultReadyCallback, ) { log('[searchSymbols]: Method call'); // todo limit results by exchange. use a separate indexer per exchange? const found = indexer.search(userInput, 10) log('found', found) const result = [] for (const f of found) for (const key of f.result) result.push(_symbols[key]) onResultReadyCallback(result); }, async resolveSymbol( symbolName, onSymbolResolvedCallback, onResolveErrorCallback, extension ) { setTimeout(async ()=> await this.doResolveSymbol(symbolName,onSymbolResolvedCallback,onResolveErrorCallback,extension), 0) }, async doResolveSymbol( symbolName, onSymbolResolvedCallback, onResolveErrorCallback, extension ) { log('[resolveSymbol]: Method call', symbolName); const symbols = getAllSymbols(); const symbolItem = symbolName === 'default' ? defaultSymbol : symbols[symbolName] log('symbol resolved?', symbolItem) if (!symbolItem) { log('[resolveSymbol]: Cannot resolve symbol', symbolName); onResolveErrorCallback('cannot resolve symbol'); return; } const co = useChartOrderStore(); co.selectedSymbol = symbolItem const pool = symbolItem.pools[0]; // choose the first-listed pool. server will adjust metadata accordingly. // noinspection JSValidateTypes co.selectedPool = pool updateFeeDropdown() const priceScale = '100' // LibrarySymbolInfo // https://www.tradingview.com/charting-library-docs/latest/api/interfaces/Charting_Library.LibrarySymbolInfo const symbolInfo = { ticker: symbolItem.ticker, name: symbolItem.symbol, pro_name: symbolItem.full_name, // we add the fee to the description after the specific pool has been resolved description: symbolItem.description + ` ${(pool[1] / 10000).toFixed(2)}%`, type: symbolItem.type, session: '24x7', timezone: 'Etc/UTC', exchange: symbolItem.exchange, minmov: .00000000000000001, pricescale: 1, variable_tick_size: VARIABLE_TICK_SIZE, has_intraday: true, // Added to allow less than one day to work visible_plots_set: 'ohlc', has_weekly_and_monthly: true, // Added to allow greater than one day to work supported_resolutions: configurationData.supported_resolutions, // volume_precision: 2, data_status: 'streaming', }; log('[resolveSymbol]: Symbol resolved', symbolName); onSymbolResolvedCallback(symbolInfo) }, async getBars(symbolInfo, resolution, periodParams, onHistoryCallback, onErrorCallback) { const { from, to, firstDataRequest } = periodParams; log('[getBars]: Method call', symbolInfo, resolution, from, to); try { // todo need to consider the selected fee tier const [poolAddr, poolFee] = useChartOrderStore().selectedPool; await getAllSymbols() const symbol = lookupSymbol(symbolInfo.ticker); const res = convertTvResolution(resolution) const key = `${useStore().chainId}|${poolAddr}|${res.name}` let bars = await loadOHLC(symbol, poolAddr, from, to, resolution); // This is the one that does all the work this.updatePushedCache(key, bars) if (symbol.inverted) bars = invertOhlcs(bars) log(`[getBars]: returned ${bars.length} bar(s), and metadata ${metadata}`); log('[getBars]: bars=', bars); // noData should be set only if no bars are in the requested period and earlier. // In our case, we are guaranteed to have contiguous samples. // So we only return no bars (bars.length==0) if: // 1. period is entirely before first data available. // 2. period is entirely after last data available. // Returning noData based on bars.length works perfectly assuming that TV never asks for case 2. // This is probably not a safe assumption. The alternative would be to search // backward to find beginning of history. How far to search? const noData = bars.length === 0; onHistoryCallback(bars, {noData}); } catch (error) { log('[getBars]: Get error', error); onErrorCallback(error); } }, subscribeBarsOnRealtimeCallback: null, subscribeBars( symbolInfo, resolution, onRealtimeCallback2, subscriberUID, onResetCacheNeededCallback, ) { const oldCb = onRealtimeCallback2 function onRealtimeCallback() { log('push bar', ...arguments) oldCb(...arguments) } log('[subscribeBars]', symbolInfo, resolution, subscriberUID); const symbol = getAllSymbols()[symbolInfo.full_name] const co = useChartOrderStore() // todo redo symbolInfo: the full_name passed to TV should just be the pool addr. when searching a symbol, go ahead and select the first/most liquid automatically let poolAddr = null const selectedFee = co.selectedPool[1]; for (const [addr, fee] of symbol.pools) { if (fee === selectedFee) { poolAddr = addr break } } if (poolAddr===null) { console.error(`Could not find pool for fee ${selectedFee}: ${symbol.pools}`) return } const chainId = useStore().chainId; const res = convertTvResolution(resolution) const period = res.name console.log('subscription symbol', symbol, getAllSymbols()) const sub = new RealtimeSubscription(chainId, poolAddr, res, symbol, subscriberUID, onRealtimeCallback, onResetCacheNeededCallback) log('sub', sub.key) subOHLC(chainId, poolAddr, period) this.startOHLCBumper() }, unsubscribeBars(subscriberUID) { log('[unsubscribeBars]', subscriberUID); const sub = subByTvSubId[subscriberUID] if (sub===undefined) { console.log(`warning: no subscription found for tvSubId ${subscriberUID}`) return } unsubOHLC(sub.chainId, sub.poolAddr, sub.res.name) log('unsub',sub.key) sub.close() delete this.pushedBars[sub.key] delete this.recentBars[sub.key] }, // key-value format: // 'chain|pool|period': [[javatime, open, high, low, close], ...] pushedBars: {}, // bars actually sent to TradingView recentBars:{}, // bars received from server notifications pushRecentBars(key, recent) { this.recentBars[key] = recent const historical = this.pushedBars[key] if (historical!==undefined) this.overlapAndPush(key, recent) }, overlapAndPush(key, ohlcs) { log('overlapAndPush',key,ohlcs) if (!ohlcs || ohlcs.length === 0) return const sub = subByKey[key] // do not check reorgs on mock symbols because the dev environment price will be different from the main chain // price and reorgs would happen every time. const checkReorg = sub.symbol.x?.data === undefined const res = sub.res const period = res.seconds * 1000 const bars = [] // to push const pushed = this.pushedBars[key] log('got pushed bars', pushed) let pi = 0 // pushed index let time let price if (pushed === undefined) { time = ohlcs[0].time price = ohlcs[0].close } else { const last = pushed.length - 1 time = pushed[last].time price = pushed[last].close } log('last time/price', time, price) for( const ohlc of ohlcs ) { log('handle ohlc', ohlc) // forward the pi index to at least the current ohlc time while( pi < pushed.length && pushed[pi].time < ohlc.time ) pi++ if (pi < pushed.length-1) { // finalized bars must match the previous push exactly or else there was a reorg and we need to reset const p = pushed[pi] log('check reorg', checkReorg, pi, p, ohlc) if (checkReorg && (p.time !== ohlc.time || p.open !== ohlc.open || p.high !== ohlc.high || p.low !== ohlc.low || p.close !== ohlc.close) ) { console.log('RESET TV CACHE') return this.resetCache(key) } } else { // the last pushed bar and anything after it can be sent to TV while (time + period < ohlc.time) { // fill gap time += period const bar = {time, open: price, high: price, low: price, close: price}; log('fill', bar) bars.push(bar) } bars.push(ohlc) time = ohlc.time price = ohlc.close } } return this.pushToTV(key, bars) }, resetCache(key) { log('resetting TV data cache') const sub = subByKey[key] if (sub===undefined) { console.log(`warning: no subscription found for dexorder key ${key}`) return } sub.onResetCacheCb() }, updatePushedCache(key, ohlcs) { log('updatePushedCache', key, ohlcs) if (!(key in this.pushedBars)) this.pushedBars[key] = ohlcs else { const prev = this.pushedBars[key] log('prev pushed', prev) const endTime = prev[prev.length - 1].time; if (endTime === ohlcs[0].time) // the new ohlc's overlap the old time, so exclude the most recent historical item, which gets replaced this.pushedBars[key] = [...prev.slice(0, -1), ...ohlcs] else if (endTime <= ohlcs[ohlcs.length-1].time) { // no overlap of any bars. full append. this.pushedBars[key] = [...prev, ...ohlcs] } // otherwise the ohlc's being pushed are =>older<= than the ones in the pushedBars cache. This happens // because TV can request data pages (using getBars()) out of chronological order. } }, pushToTV(key, ohlcs) { if (ohlcs.length===0) return log('pushing bars to tv', ohlcs) this.updatePushedCache(key, ohlcs); // we cache the raw bars before inversion so they match dexorder data sources const sub = subByKey[key] if (sub===undefined) { console.log(`warning: could not find subscription for dexorder sub key ${key}`) return } if (sub.symbol.inverted) ohlcs = invertOhlcs(ohlcs) for (const ohlc of ohlcs) sub.onRealtimeCb(ohlc) }, poolCallback(chainId, poolPeriod, ohlcs) { if (!ohlcs || ohlcs.length===0) return const key = `${chainId}|${poolPeriod}`; const bars = [] for (const ohlc of ohlcs) { let close = parseFloat(ohlc[4]) // close const bar = { time: ohlc[0] * 1000, open: ohlc[1] ? parseFloat(ohlc[1]) : close, // open high: ohlc[2] ? parseFloat(ohlc[2]) : close, // high low: ohlc[3] ? parseFloat(ohlc[3]) : close, // low close: close, } bars.push(bar) } return this.pushRecentBars(key, bars) }, intervalChanged(seconds) { // rollover bumper // this timer function creates a new bar when the period rolls over this.startOHLCBumper() }, // The OHLC bumper advances bars at the end of the period. If the price doesn't change, no new data will arrive, so the // new bar is implicit and must be generated dynamically. startOHLCBumper() { const co = useChartOrderStore() if (_rolloverBumper !== null) clearTimeout(_rolloverBumper) const period = co.intervalSecs; if (period === 0) return const now = Date.now() const nextRollover = ohlcStart(now/1000 + period)*1000 + 2000 // two second delay to wait for server data const delay = nextRollover - now _rolloverBumper = setTimeout(this.bumpOHLC.bind(this), delay) }, bumpOHLC() { log('bumpOHLC') _rolloverBumper = null const secs = useChartOrderStore().intervalSecs; for (const sub of Object.values(subByKey)) { log('check bump', sub.res.seconds, secs) if (sub.res.seconds === secs) { const pushed = this.pushedBars[sub.key] log('check pushed', pushed) if (pushed !== undefined && pushed.length > 0) { const lastBar = pushed[pushed.length - 1] const price = lastBar.close const now = timestamp() * 1000 const period = sub.res.seconds * 1000 const fills = [] for( let time=lastBar.time + period; time < now; time += period ) { log('pushing bump', time, price) const bar = {time, open: price, high: price, low: price, close: price} fills.push(bar) } this.pushToTV(sub.key, fills) } else { log('warning: bumpOHLC() found no previous bars') } } } this.startOHLCBumper() }, } let _rolloverBumper = null let defaultSymbol = null