Files
web/src/charts/datafeed.js
2024-10-23 23:08:03 -04:00

682 lines
21 KiB
JavaScript

import {convertTvResolution, loadOHLC} from './ohlc.js';
import {metadata} from "@/version.js";
import FlexSearch from "flexsearch";
import {useChartOrderStore} from "@/orderbuild.js";
import {useStore} from "@/store/store.js";
import {subOHLC, unsubOHLC} from "@/blockchain/ohlcs.js";
import {ohlcStart} from "@/charts/chart-misc.js";
import {timestamp} from "@/misc.js";
const DEBUG_LOGGING = false
const log = DEBUG_LOGGING ? console.log : ()=>{}
// this file manages connecting data to TradingView using their DataFeed API.
// https://www.tradingview.com/charting-library-docs/latest/connecting_data/Datafeed-API/#integrate-datafeed-api
// see ohlc.js for fetching dexorder ohlc files
// in order of priority
const quoteSymbols = [
'USDT',
'USDC',
'TUSD',
'GUSD',
'BUSD',
'MUSD',
'DAI',
'CRVUSD',
'EURC',
'EURS',
'EURI',
'WBTC',
'WETH',
]
let widget = null
// Python code to generate VARIABLE_TICK_SIZE:
// from decimal import Decimal
// K=12 # powers above and below to include
// D=5 # number of significant digits to display
// ' '.join(f'{Decimal(10)**(k-D):f} {Decimal(10)**k:f}' for k in range(-K,K+1))+f' {Decimal(10)**(K+1-D)}'
const VARIABLE_TICK_SIZE = '0.00000000000000001 0.000000000001 0.0000000000000001 0.00000000001 0.000000000000001 0.0000000001 0.00000000000001 0.000000001 0.0000000000001 0.00000001 0.000000000001 0.0000001 0.00000000001 0.000001 0.0000000001 0.00001 0.000000001 0.0001 0.00000001 0.001 0.0000001 0.01 0.000001 0.1 0.00001 1 0.0001 10 0.001 100 0.01 1000 0.1 10000 1 100000 10 1000000 100 10000000 1000 100000000 10000 1000000000 100000 10000000000 1000000 100000000000 10000000 1000000000000 100000000'
// DatafeedConfiguration implementation
const configurationData = {
// Represents the resolutions for bars supported by your datafeed
supported_resolutions:
['1', '3', '5', '10', '15', '30', '60', '120', '240', '480', '720', '1D', '2D', '3D', '1W'],
// The `exchanges` arguments are used for the `searchSymbols` method if a user selects the exchange
exchanges: [
// {
// value: 'UNIv2',
// name: 'Uniswap v2',
// desc: 'Uniswap v2',
// },
{
value: 'UNIv3',
name: 'Uniswap v3',
desc: 'Uniswap v3',
logo: 'https://upload.wikimedia.org/wikipedia/commons/e/e7/Uniswap_Logo.svg',
},
],
// The `symbols_types` arguments are used for the `searchSymbols` method if a user selects this symbol type
symbols_types: [
{name: 'swap', value: 'swap',},
],
};
const tokenMap = {} // todo needs chainId
const poolMap = {} // todo needs chainId
let _symbols = null // keyed by the concatenated hex addrs of the token pair e.g. '0xf3Ed85D882b5d9A67fC10dBf8f9AA991212983aA' + '0x6cdC5106DC100115E6C310539Fe44a61b3EEa6C4'
const indexer = new FlexSearch.Document({
document: {id: 'id', index: ['fn', 'as[]', 'b', 'q', 'bs', 'qs', 'e', 'd']}, // this must match what is generated for the index object in addSymbol()
charset: {split: /\W+/},
tokenize: 'forward',
})
const indexes = {}
const feeGroups = {} // keyed by ticker without the final fee field. values are a list of pools [[addr,fee],...]
// The symbol key is the chain and base/quote: basically a "pair." It is only used by dexorder. The TradingView
// symbol is keyed by the `ticker` which is defined as 'chain_id|pool_addr' for absolute uniqueness.
export function tickerForOrder(chainId, order) {
return tickerKey(chainId, order.route.exchange, order.tokenIn, order.tokenOut, order.route.fee, true)
}
export function tickerKey(chainId, exchange, tokenAddrA, tokenAddrB, fee, chooseInversion=false ) {
if (chooseInversion && invertedDefault(tokenAddrA, tokenAddrB) )
[tokenAddrA, tokenAddrB] = [tokenAddrB, tokenAddrA]
// NOTE: the ticker key specifies a base and quote ordering, so there are two tickers per pool
return `${chainId}|${exchange}|${tokenAddrA}|${tokenAddrB}|${fee}`
}
export function feelessTickerKey(ticker) {
return ticker.split('|').slice(0, -1).join('|');
}
function addSymbol(chainId, p, base, quote, inverted) {
const symbol = base.s + '/' + quote.s
const exchange = ['UNIv2', 'UNIv3'][p.e]
const full_name = exchange + ':' + symbol // + '%' + formatFee(fee)
const ticker = tickerKey(chainId, p.e, base.a, quote.a, p.f)
// add the search index only if this is the natural, noninverted base/quote pair
log('addSymbol', p, base, quote, inverted, ticker)
const description = `${base.n} / ${quote.n} ${(p.f/10000).toFixed(2)}%`
const type = 'swap'
const decimals = inverted ? -p.d : p.d
const symbolInfo = {
key: ticker, ticker,
chainId, address: p.a, exchangeId: p.e,
full_name, symbol, description, exchange, type, inverted, base, quote, decimals, x:p.x, fee:p.f,
};
_symbols[ticker] = symbolInfo
const feelessKey = feelessTickerKey(ticker)
if (feelessKey in feeGroups) {
feeGroups[feelessKey].push([symbolInfo.address, symbolInfo.fee])
feeGroups[feelessKey].sort((a,b)=>a[1]-b[1])
}
else
feeGroups[feelessKey] = [[symbolInfo.address, symbolInfo.fee]]
symbolInfo.feeGroup = feeGroups[feelessKey]
if (defaultSymbol===null && !invertedDefault(symbolInfo.base.a, symbolInfo.quote.a))
defaultSymbol = _symbols[ticker]
log('new symbol', ticker, _symbols[ticker])
}
function buildSymbolIndex() {
for (const symbol of Object.values(_symbols)) {
if (invertedDefault(symbol.base.a, symbol.quote.a))
continue // don't search "upside down" pairs
const feelessKey = feelessTickerKey(symbol.ticker)
const feeGroup = feeGroups[feelessKey]
const [_addr, medianFee] = feeGroup[Math.floor((feeGroup.length-1)/2)]
if (symbol.fee !== medianFee)
continue // show the pool with the median fee by default
const ticker = symbol.ticker
const longExchange = ['Uniswap v2', 'Uniswap v3',][symbol.exchangeId]
if (ticker in indexes) {
indexes[ticker].as.push(symbol.address) // add the pool address index
} else {
indexes[ticker] = {
// key
id: ticker,
// addresses
a: symbol.address,
b: symbol.base.a,
q: symbol.quote.a,
// symbols
fn: symbol.full_name,
bs: symbol.base.s,
qs: symbol.quote.s,
e: symbol.exchange + ' ' + longExchange,
d: symbol.description,
}
}
}
}
// function formatFee(fee) {
// let str = (fee / 10000).toFixed(2);
// if (str.startsWith('0')) // start with the decimal point not a zero
// str = str.slice(1)
// return str
// }
export function invertedDefault(tokenAddrA, tokenAddrB) {
// lower priority is more important (earlier in the list)
const a = tokenMap[tokenAddrA];
const b = tokenMap[tokenAddrB];
if (!a) {
log(`No token ${tokenAddrA} found`)
return
}
if (!b) {
log(`No token ${tokenAddrB} found`)
return
}
let basePriority = quoteSymbols.indexOf(a.s)
let quotePriority = quoteSymbols.indexOf(b.s)
if (basePriority === -1)
basePriority = Number.MAX_SAFE_INTEGER
if (quotePriority === -1)
quotePriority = Number.MAX_SAFE_INTEGER
return basePriority < quotePriority
}
export function getAllSymbols() {
if (_symbols===null) {
const chainId = useStore().chainId;
const md = metadata[chainId]
if(!md) {
log('could not get metadata for chain', chainId)
return []
}
_symbols = {}
for (const t of md.t)
tokenMap[t.a] = t
md.p.forEach((p)=>{
poolMap[p.a] = p
const base = tokenMap[p.b];
const quote = tokenMap[p.q];
if (!base) {
log(`No token ${p.b} found`)
return
}
if (!quote) {
log(`No token ${p.q} found`)
return
}
addSymbol(chainId, p, quote, base, true);
addSymbol(chainId, p, base, quote, false);
})
buildSymbolIndex()
log('indexes', indexes)
Object.values(indexes).forEach(indexer.add.bind(indexer))
}
log('symbols', _symbols)
return _symbols
}
function invertTicker(ticker) {
const [chainId, exchange, base, quote, fee] = ticker.split('|')
return tickerKey(chainId, exchange, quote, base, fee)
}
export function lookupSymbol(ticker) { // lookup by ticker which is "0xbaseAddress/0xquoteAddress"
// todo tim lookup default base/quote pool
const symbols = getAllSymbols();
if (!(ticker in symbols)) {
// check the inverted symbol
const orig = ticker
ticker = invertTicker(ticker);
if (!(ticker in symbols)) {
console.error('no symbol found for ticker', orig, symbols)
return null
}
}
return symbols[ticker]
}
function checkBar(bar, msg) {
// Everything should be positive
let o_l = bar.open - bar.low
let c_l = bar.close - bar.low
let h_o = bar.high - bar.open
let h_c = bar.high - bar.close
let h_l = bar.high - bar.low
if (o_l<0||c_l<0||h_o<0||h_c<0||h_l<0) {
log(msg, "bar.high/low inconsistent:", bar)
if (o_l<0) log("bar inconsistent: open-low: ", o_l)
if (c_l<0) log("bar inconsistent: close-low: ", c_l)
if (h_o<0) log("bar inconsistent: high-open: ", h_o)
if (h_c<0) log("bar inconsistent: high-close:", h_c)
if (h_l<0) log("bar inconsistent: high-low: ", h_l)
} else {
log(msg, "bar diffs:", bar)
log("bar diff: open-low: ", o_l)
log("bar diff: close-low: ", c_l)
log("bar diff: high-open: ", h_o)
log("bar diff: high-close:", h_c)
log("bar diff: high-low: ", h_l)
}
}
function invertOhlcs(bars) {
const result = []
for (const bar of bars) {
const h = bar.high
result.push({
time: bar.time,
open: 1/bar.open,
high: 1/bar.low,
low: 1/h,
close: 1/bar.close,
})
}
return result
}
const subByTvSubId = {}
const subByKey = {}
class RealtimeSubscription {
constructor(chainId, poolAddr, res, symbol, tvSubId, onRealtimeCb, onResetCacheCb ) {
this.chainId = chainId
this.poolAddr = poolAddr
this.res = res
this.symbol = symbol
this.tvSubId = tvSubId
this.onRealtimeCb = onRealtimeCb
this.onResetCacheCb = onResetCacheCb
this.key = `${chainId}|${poolAddr}|${res.name}`
subByTvSubId[this.tvSubId] = this
subByKey[this.key] = this
}
close() {
delete subByTvSubId[this.tvSubId]
delete subByKey[this.key]
}
}
export const DataFeed = {
onReady(callback) {
log('[onReady]: Method call');
setTimeout(() => callback(configurationData));
},
async searchSymbols(
userInput,
exchange,
symbolType,
onResultReadyCallback,
) {
log('[searchSymbols]: Method call');
// todo limit results by exchange. use a separate indexer per exchange?
const found = indexer.search(userInput, 10)
log('found', found)
const result = []
const seen = {}
for (const f of found)
for (const ticker of f.result)
if (!(ticker in seen)) {
result.push(_symbols[ticker])
seen[ticker] = true
}
onResultReadyCallback(result);
},
async resolveSymbol(
symbolName,
onSymbolResolvedCallback,
onResolveErrorCallback,
extension
) {
setTimeout(async ()=>
await this.doResolveSymbol(symbolName,onSymbolResolvedCallback,onResolveErrorCallback,extension),
0)
},
async doResolveSymbol(
symbolName,
onSymbolResolvedCallback,
onResolveErrorCallback,
extension
) {
log('[resolveSymbol]: Method call', symbolName);
const symbols = getAllSymbols();
const symbolItem = symbolName === 'default' ? defaultSymbol : symbols[symbolName]
if (!symbolItem) {
log('[resolveSymbol]: Cannot resolve symbol', symbolName);
onResolveErrorCallback('cannot resolve symbol');
return;
}
const co = useChartOrderStore();
co.selectedSymbol = symbolItem
const feelessKey = feelessTickerKey(symbolItem.ticker)
const symbolsByFee = feeGroups[feelessKey]
symbolsByFee.sort((a,b)=>a.fee-b.fee)
const pool = symbolsByFee[Math.floor((symbolsByFee.length - 1)/2)] // median rounded down
// noinspection JSValidateTypes
co.selectedPool = pool // todo remove
// LibrarySymbolInfo
// https://www.tradingview.com/charting-library-docs/latest/api/interfaces/Charting_Library.LibrarySymbolInfo
const symbolInfo = {
ticker: symbolItem.ticker,
name: symbolItem.symbol,
pro_name: symbolItem.full_name,
description: symbolItem.description,
type: symbolItem.type,
session: '24x7',
timezone: 'Etc/UTC',
exchange: symbolItem.exchange,
minmov: .00000000000000001,
pricescale: 1,
variable_tick_size: VARIABLE_TICK_SIZE,
has_intraday: true, // Added to allow less than one day to work
visible_plots_set: 'ohlc',
has_weekly_and_monthly: true, // Added to allow greater than one day to work
supported_resolutions: configurationData.supported_resolutions,
// volume_precision: 2,
data_status: 'streaming',
};
log('[resolveSymbol]: Symbol resolved', symbolName);
onSymbolResolvedCallback(symbolInfo)
},
async getBars(symbolInfo, resolution, periodParams, onHistoryCallback, onErrorCallback) {
const { from, to, firstDataRequest } = periodParams;
log('[getBars]: Method call', symbolInfo, resolution, from, to);
try {
// todo need to consider the selected fee tier
await getAllSymbols()
const symbol = lookupSymbol(symbolInfo.ticker);
const poolAddr = symbol.address
const res = convertTvResolution(resolution)
const key = `${useStore().chainId}|${poolAddr}|${res.name}`
let bars = await loadOHLC(symbol, poolAddr, from, to, resolution); // This is the one that does all the work
this.updatePushedCache(key, bars)
if (symbol.inverted)
bars = invertOhlcs(bars)
log(`[getBars]: returned ${bars.length} bar(s), and metadata ${metadata}`);
log('[getBars]: bars=', bars);
// noData should be set only if no bars are in the requested period and earlier.
// In our case, we are guaranteed to have contiguous samples.
// So we only return no bars (bars.length==0) if:
// 1. period is entirely before first data available.
// 2. period is entirely after last data available.
// Returning noData based on bars.length works perfectly assuming that TV never asks for case 2.
// This is probably not a safe assumption. The alternative would be to search
// backward to find beginning of history. How far to search?
const noData = bars.length === 0;
onHistoryCallback(bars, {noData});
} catch (error) {
log('[getBars]: Get error', error);
onErrorCallback(error);
}
},
subscribeBarsOnRealtimeCallback: null,
subscribeBars(
symbolInfo,
resolution,
onRealtimeCallback2,
subscriberUID,
onResetCacheNeededCallback,
) {
const oldCb = onRealtimeCallback2
function onRealtimeCallback() {
log('push bar', ...arguments)
oldCb(...arguments)
}
log('[subscribeBars]', symbolInfo, resolution, subscriberUID);
const symbol = getAllSymbols()[symbolInfo.ticker]
const poolAddr = symbol.address
const chainId = useStore().chainId;
const res = convertTvResolution(resolution)
const period = res.name
log('subscription symbol', symbol, chainId, poolAddr, res)
const sub = new RealtimeSubscription(chainId, poolAddr, res, symbol, subscriberUID, onRealtimeCallback, onResetCacheNeededCallback)
log('sub', sub.key)
subOHLC(chainId, poolAddr, period)
this.startOHLCBumper()
},
unsubscribeBars(subscriberUID) {
log('[unsubscribeBars]', subscriberUID);
const sub = subByTvSubId[subscriberUID]
if (sub===undefined) {
console.log(`warning: no subscription found for tvSubId ${subscriberUID}`)
return
}
unsubOHLC(sub.chainId, sub.poolAddr, sub.res.name)
log('unsub',sub.key)
sub.close()
delete this.pushedBars[sub.key]
delete this.recentBars[sub.key]
},
// key-value format:
// 'chain|pool|period': [[javatime, open, high, low, close], ...]
pushedBars: {}, // bars actually sent to TradingView
recentBars:{}, // bars received from server notifications
pushRecentBars(key, recent) {
this.recentBars[key] = recent
const historical = this.pushedBars[key]
if (historical!==undefined)
this.overlapAndPush(key, recent)
},
overlapAndPush(key, ohlcs) {
log('overlapAndPush',key,ohlcs)
if (!ohlcs || ohlcs.length === 0) return
const sub = subByKey[key]
// do not check reorgs on mock symbols because the dev environment price will be different from the main chain
// price and reorgs would happen every time.
const checkReorg = sub.symbol.x?.data === undefined
const res = sub.res
const period = res.seconds * 1000
const bars = [] // to push
const pushed = this.pushedBars[key]
log('got pushed bars', pushed)
let pi = 0 // pushed index
let time
let price
if (pushed === undefined) {
time = ohlcs[0].time
price = ohlcs[0].close
}
else {
const last = pushed.length - 1
time = pushed[last].time
price = pushed[last].close
}
log('last time/price', time, price)
for( const ohlc of ohlcs ) {
log('handle ohlc', ohlc)
// forward the pi index to at least the current ohlc time
while( pi < pushed.length && pushed[pi].time < ohlc.time ) pi++
if (pi < pushed.length-1) {
// finalized bars must match the previous push exactly or else there was a reorg and we need to reset
const p = pushed[pi]
log('check reorg', checkReorg, pi, p, ohlc)
if (checkReorg &&
(p.time !== ohlc.time || p.open !== ohlc.open || p.high !== ohlc.high ||
p.low !== ohlc.low || p.close !== ohlc.close) )
{
console.log('RESET TV CACHE')
return this.resetCache(key)
}
}
else {
// the last pushed bar and anything after it can be sent to TV
while (time + period < ohlc.time) {
// fill gap
time += period
const bar = {time, open: price, high: price, low: price, close: price};
log('fill', bar)
bars.push(bar)
}
bars.push(ohlc)
time = ohlc.time
price = ohlc.close
}
}
return this.pushToTV(key, bars)
},
resetCache(key) {
log('resetting TV data cache')
const sub = subByKey[key]
if (sub===undefined) {
console.log(`warning: no subscription found for dexorder key ${key}`)
return
}
sub.onResetCacheCb()
},
updatePushedCache(key, ohlcs) {
if (ohlcs.length===0) return
log('updatePushedCache', key, ohlcs)
if (!(key in this.pushedBars))
this.pushedBars[key] = ohlcs
else {
const prev = this.pushedBars[key]
log('prev pushed', prev)
const endTime = prev[prev.length - 1].time;
if (endTime === ohlcs[0].time)
// the new ohlc's overlap the old time, so exclude the most recent historical item, which gets replaced
this.pushedBars[key] = [...prev.slice(0, -1), ...ohlcs]
else if (endTime <= ohlcs[ohlcs.length-1].time) {
// no overlap of any bars. full append.
this.pushedBars[key] = [...prev, ...ohlcs]
}
// otherwise the ohlc's being pushed are =>older<= than the ones in the pushedBars cache. This happens
// because TV can request data pages (using getBars()) out of chronological order.
}
},
pushToTV(key, ohlcs) {
log('pushing bars to tv', ohlcs)
if (ohlcs.length===0) return
this.updatePushedCache(key, ohlcs); // we cache the raw bars before inversion so they match dexorder data sources
const sub = subByKey[key]
if (sub===undefined) {
console.log(`warning: could not find subscription for dexorder sub key ${key}`)
return
}
if (sub.symbol.inverted)
ohlcs = invertOhlcs(ohlcs)
for (const ohlc of ohlcs)
sub.onRealtimeCb(ohlc)
},
poolCallback(chainId, poolPeriod, ohlcs) {
if (!ohlcs || ohlcs.length===0) return
const key = `${chainId}|${poolPeriod}`;
const bars = []
for (const ohlc of ohlcs) {
let close = parseFloat(ohlc[4]) // close
const bar = {
time: ohlc[0] * 1000,
open: ohlc[1] ? parseFloat(ohlc[1]) : close, // open
high: ohlc[2] ? parseFloat(ohlc[2]) : close, // high
low: ohlc[3] ? parseFloat(ohlc[3]) : close, // low
close: close,
}
bars.push(bar)
}
return this.pushRecentBars(key, bars)
},
intervalChanged(seconds) {
// rollover bumper
// this timer function creates a new bar when the period rolls over
this.startOHLCBumper()
},
// The OHLC bumper advances bars at the end of the period. If the price doesn't change, no new data will arrive, so the
// new bar is implicit and must be generated dynamically.
startOHLCBumper() {
const co = useChartOrderStore()
if (_rolloverBumper !== null)
clearTimeout(_rolloverBumper)
const period = co.intervalSecs;
if (period === 0)
return
const now = Date.now()
const nextRollover = ohlcStart(now/1000 + period)*1000 + 2000 // two second delay to wait for server data
const delay = nextRollover - now
_rolloverBumper = setTimeout(this.bumpOHLC.bind(this), delay)
},
bumpOHLC() {
log('bumpOHLC')
_rolloverBumper = null
const secs = useChartOrderStore().intervalSecs;
for (const sub of Object.values(subByKey)) {
log('check bump', sub.res.seconds, secs)
if (sub.res.seconds === secs) {
const pushed = this.pushedBars[sub.key]
log('check pushed', pushed)
if (pushed !== undefined && pushed.length > 0) {
const lastBar = pushed[pushed.length - 1]
const price = lastBar.close
const now = timestamp() * 1000
const period = sub.res.seconds * 1000
const fills = []
for( let time=lastBar.time + period; time < now; time += period ) {
log('pushing bump', time, price)
const bar = {time, open: price, high: price, low: price, close: price}
fills.push(bar)
}
this.pushToTV(sub.key, fills)
}
else {
log('warning: bumpOHLC() found no previous bars')
}
}
}
this.startOHLCBumper()
},
}
let _rolloverBumper = null
let defaultSymbol = null