Files
web/src/charts/datafeed.js
2024-08-30 16:45:50 -04:00

672 lines
20 KiB
JavaScript

import {convertTvResolution, loadOHLC} from './ohlc.js';
import {metadata} from "@/version.js";
import FlexSearch from "flexsearch";
import {useChartOrderStore} from "@/orderbuild.js";
import {useStore} from "@/store/store.js";
import {subOHLC, unsubOHLC} from "@/blockchain/ohlcs.js";
import {ohlcStart} from "@/charts/chart-misc.js";
import {timestamp} from "@/misc.js";
const DEBUG_LOGGING = false
const log = DEBUG_LOGGING ? console.log : ()=>{}
// this file manages connecting data to TradingView using their DataFeed API.
// https://www.tradingview.com/charting-library-docs/latest/connecting_data/Datafeed-API/#integrate-datafeed-api
// see ohlc.js for fetching dexorder ohlc files
// in order of priority
const quoteSymbols = [
'USDT',
'USDC',
'TUSD',
'GUSD',
'BUSD',
'MUSD',
'DAI',
'CRVUSD',
'EURC',
'EURS',
'EURI',
'WBTC',
'WETH',
]
let feeDropdown = null
let widget = null
// Python code to generate VARIABLE_TICK_SIZE:
// from decimal import Decimal
// K=12 # powers above and below to include
// D=5 # number of significant digits to display
// ' '.join(f'{Decimal(10)**(k-D):f} {Decimal(10)**k:f}' for k in range(-K,K+1))+f' {Decimal(10)**(K+1-D)}'
const VARIABLE_TICK_SIZE = '0.00000000000000001 0.000000000001 0.0000000000000001 0.00000000001 0.000000000000001 0.0000000001 0.00000000000001 0.000000001 0.0000000000001 0.00000001 0.000000000001 0.0000001 0.00000000001 0.000001 0.0000000001 0.00001 0.000000001 0.0001 0.00000001 0.001 0.0000001 0.01 0.000001 0.1 0.00001 1 0.0001 10 0.001 100 0.01 1000 0.1 10000 1 100000 10 1000000 100 10000000 1000 100000000 10000 1000000000 100000 10000000000 1000000 100000000000 10000000 1000000000000 100000000'
export function initFeeDropdown(w) {
widget = w
widget.createDropdown(
{
title: 'Fees',
tooltip: 'Choose Fee Tier',
items: [/*{title: 'Automatic Fee Selection', onSelect: () => {log('autofees')}}*/],
icon: `<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28"><g fill="none" stroke="currentColor"><circle cx="10" cy="10" r="2.5"/><circle cx="18" cy="18" r="2.5"/><path stroke-linecap="square" d="M17.5 7.5l-7 13"/></g></svg>`,
}
).then(dropdown => {feeDropdown = dropdown; updateFeeDropdown()})
}
function updateFeeDropdown() {
if (feeDropdown===null) return
const symbolItem = useChartOrderStore().selectedSymbol
let items
if (symbolItem===null)
items = [{title: '0.00%'}]
else {
items = symbolItem.pools.map((p)=> {
return {
title: (p[1]/10000).toFixed(2)+'%',
onSelect: ()=>selectPool(p),
}
})
}
feeDropdown.applyOptions({items})
}
function selectPool(p) {
const co = useChartOrderStore();
if ( co.selectedPool === null || co.selectedPool[0] !== p[0] || co.selectedPool[1] !== p[1]) {
co.selectedPool = p
}
}
const lastBarsCache = new Map();
// DatafeedConfiguration implementation
const configurationData = {
// Represents the resolutions for bars supported by your datafeed
supported_resolutions:
['1', '3', '5', '10', '15', '30', '60', '120', '240', '480', '720', '1D', '2D', '3D', '1W'],
// The `exchanges` arguments are used for the `searchSymbols` method if a user selects the exchange
exchanges: [
// {
// value: 'UNIv2',
// name: 'Uniswap v2',
// desc: 'Uniswap v2',
// },
{
value: 'UNIv3',
name: 'Uniswap v3',
desc: 'Uniswap v3',
logo: 'https://upload.wikimedia.org/wikipedia/commons/e/e7/Uniswap_Logo.svg',
},
],
// The `symbols_types` arguments are used for the `searchSymbols` method if a user selects this symbol type
symbols_types: [
{name: 'swap', value: 'swap',},
],
};
const tokenMap = {}
const poolMap = {}
let _symbols = null // keyed by the concatenated hex addrs of the token pair e.g. '0xf3Ed85D882b5d9A67fC10dBf8f9AA991212983aA' + '0x6cdC5106DC100115E6C310539Fe44a61b3EEa6C4'
const indexer = new FlexSearch.Document({
document: {id: 'id', index: ['fn', 'as[]', 'b', 'q', 'bs', 'qs', 'e', 'd']}, // this must match what is generated for the index object in addSymbol()
charset: {split: /\W+/},
tokenize: 'forward',
})
const indexes = {}
const symbolsSeen = {} // keyed by (base,quote) so we only list one pool per pair even if there are many fee tiers
// todo add chainIds to all these keys
function addSymbol(p, base, quote, inverted) {
const symbol = base.s + quote.s
const exchange = ['UNIv2', 'UNIv3'][p.e]
const full_name = exchange + ':' + symbol // + '%' + formatFee(fee)
const key = `${base.a}/${quote.a}`
log('addSymbol', p, base, quote, inverted, key)
if (symbolsSeen[key]) {
// add this pool's address to the existing symbol as an additional fee tier
const symbolInfo = _symbols[key];
symbolInfo.pools.push([p.a, p.f])
symbolInfo.pools.sort((a,b)=>a[1]-b[1])
log('integrated symbol', symbolInfo)
indexes[key].as.push(p.a)
return
}
symbolsSeen[key] = true
const longExchange = ['Uniswap v2', 'Uniswap v3',][p.e]
const description = `${base.n} / ${quote.n}`
const type = 'swap'
const pools = [[p.a, p.f]]
const decimals = p.d
_symbols[key] = {
ticker: key, full_name, symbol, description,
exchange, type, inverted, base, quote, pools, decimals, x:p.x
}
if (defaultSymbol===null)
defaultSymbol = _symbols[key]
log('new symbol', key, _symbols[key])
indexes[key] = {
// key
id: key,
// addresses
as: [p.a], // multiple pool addrs for each fee tier
b: p.b,
q: p.q,
// symbols
fn: full_name,
bs: base.s,
qs: quote.s,
e: exchange + ' ' + longExchange,
d: description,
}
}
// function formatFee(fee) {
// let str = (fee / 10000).toFixed(2);
// if (str.startsWith('0')) // start with the decimal point not a zero
// str = str.slice(1)
// return str
// }
function getAllSymbols() {
if (_symbols===null) {
const chainId = useStore().chainId;
const md = metadata[chainId]
if(!md) {
log('could not get metadata for chain', chainId)
return []
}
_symbols = {}
for (const t of md.t)
tokenMap[t.a] = t
md.p.forEach((p)=>{
poolMap[p.a] = p
const base = tokenMap[p.b];
const quote = tokenMap[p.q];
if (!base) {
log(`No token ${p.b} found`)
return
}
if (!quote) {
log(`No token ${p.q} found`)
return
}
// todo check quotes symbol list for inversion hint
let basePriority = quoteSymbols.indexOf(base.s)
let quotePriority = quoteSymbols.indexOf(quote.s)
if (basePriority === -1)
basePriority = Number.MAX_SAFE_INTEGER
if (quotePriority === -1)
quotePriority = Number.MAX_SAFE_INTEGER
const showInverted = basePriority < quotePriority
if (showInverted)
addSymbol(p, quote, base, true);
else
addSymbol(p, base, quote, false);
})
log('indexes', indexes)
Object.values(indexes).forEach(indexer.add.bind(indexer))
}
log('symbols', _symbols)
return _symbols
}
export function lookupSymbol(key) { // lookup by fullname
return getAllSymbols()[key]
}
function checkBar(bar, msg) {
// Everything should be positive
let o_l = bar.open - bar.low
let c_l = bar.close - bar.low
let h_o = bar.high - bar.open
let h_c = bar.high - bar.close
let h_l = bar.high - bar.low
if (o_l<0||c_l<0||h_o<0||h_c<0||h_l<0) {
log(msg, "bar.high/low inconsistent:", bar)
if (o_l<0) log("bar inconsistent: open-low: ", o_l)
if (c_l<0) log("bar inconsistent: close-low: ", c_l)
if (h_o<0) log("bar inconsistent: high-open: ", h_o)
if (h_c<0) log("bar inconsistent: high-close:", h_c)
if (h_l<0) log("bar inconsistent: high-low: ", h_l)
} else {
log(msg, "bar diffs:", bar)
log("bar diff: open-low: ", o_l)
log("bar diff: close-low: ", c_l)
log("bar diff: high-open: ", h_o)
log("bar diff: high-close:", h_c)
log("bar diff: high-low: ", h_l)
}
}
function invertOhlcs(bars) {
const result = []
for (const bar of bars) {
const h = bar.high
result.push({
time: bar.time,
open: 1/bar.open,
high: 1/bar.low,
low: 1/h,
close: 1/bar.close,
})
}
return result
}
const subByTvSubId = {}
const subByKey = {}
class RealtimeSubscription {
constructor(chainId, poolAddr, res, symbol, tvSubId, onRealtimeCb, onResetCacheCb ) {
this.chainId = chainId
this.poolAddr = poolAddr
this.res = res
this.symbol = symbol
this.tvSubId = tvSubId
this.onRealtimeCb = onRealtimeCb
this.onResetCacheCb = onResetCacheCb
this.key = `${chainId}|${poolAddr}|${res.name}`
subByTvSubId[this.tvSubId] = this
subByKey[this.key] = this
}
close() {
delete subByTvSubId[this.tvSubId]
delete subByKey[this.key]
}
}
export const DataFeed = {
onReady(callback) {
log('[onReady]: Method call');
setTimeout(() => callback(configurationData));
},
async searchSymbols(
userInput,
exchange,
symbolType,
onResultReadyCallback,
) {
log('[searchSymbols]: Method call');
// todo limit results by exchange. use a separate indexer per exchange?
const found = indexer.search(userInput, 10)
log('found', found)
const result = []
for (const f of found)
for (const key of f.result)
result.push(_symbols[key])
onResultReadyCallback(result);
},
async resolveSymbol(
symbolName,
onSymbolResolvedCallback,
onResolveErrorCallback,
extension
) {
setTimeout(async ()=>
await this.doResolveSymbol(symbolName,onSymbolResolvedCallback,onResolveErrorCallback,extension),
0)
},
async doResolveSymbol(
symbolName,
onSymbolResolvedCallback,
onResolveErrorCallback,
extension
) {
log('[resolveSymbol]: Method call', symbolName);
const symbols = getAllSymbols();
const symbolItem = symbolName === 'default' ? defaultSymbol : symbols[symbolName]
log('symbol resolved?', symbolItem)
if (!symbolItem) {
log('[resolveSymbol]: Cannot resolve symbol', symbolName);
onResolveErrorCallback('cannot resolve symbol');
return;
}
const co = useChartOrderStore();
co.selectedSymbol = symbolItem
const pool = symbolItem.pools[0]; // choose the first-listed pool. server will adjust metadata accordingly.
// noinspection JSValidateTypes
co.selectedPool = pool
updateFeeDropdown()
const priceScale = '100'
// LibrarySymbolInfo
// https://www.tradingview.com/charting-library-docs/latest/api/interfaces/Charting_Library.LibrarySymbolInfo
const symbolInfo = {
ticker: symbolItem.ticker,
name: symbolItem.symbol,
pro_name: symbolItem.full_name,
// we add the fee to the description after the specific pool has been resolved
description: symbolItem.description + ` ${(pool[1] / 10000).toFixed(2)}%`,
type: symbolItem.type,
session: '24x7',
timezone: 'Etc/UTC',
exchange: symbolItem.exchange,
minmov: .00000000000000001,
pricescale: 1,
variable_tick_size: VARIABLE_TICK_SIZE,
has_intraday: true, // Added to allow less than one day to work
visible_plots_set: 'ohlc',
has_weekly_and_monthly: true, // Added to allow greater than one day to work
supported_resolutions: configurationData.supported_resolutions,
// volume_precision: 2,
data_status: 'streaming',
};
log('[resolveSymbol]: Symbol resolved', symbolName);
onSymbolResolvedCallback(symbolInfo)
},
async getBars(symbolInfo, resolution, periodParams, onHistoryCallback, onErrorCallback) {
const { from, to, firstDataRequest } = periodParams;
log('[getBars]: Method call', symbolInfo, resolution, from, to);
try {
// todo need to consider the selected fee tier
const [poolAddr, poolFee] = useChartOrderStore().selectedPool;
await getAllSymbols()
const symbol = lookupSymbol(symbolInfo.ticker);
const res = convertTvResolution(resolution)
const key = `${useStore().chainId}|${poolAddr}|${res.name}`
let bars = await loadOHLC(symbol, poolAddr, from, to, resolution); // This is the one that does all the work
this.updatePushedCache(key, bars)
if (symbol.inverted)
bars = invertOhlcs(bars)
log(`[getBars]: returned ${bars.length} bar(s), and metadata ${metadata}`);
log('[getBars]: bars=', bars);
// noData should be set only if no bars are in the requested period and earlier.
// In our case, we are guaranteed to have contiguous samples.
// So we only return no bars (bars.length==0) if:
// 1. period is entirely before first data available.
// 2. period is entirely after last data available.
// Returning noData based on bars.length works perfectly assuming that TV never asks for case 2.
// This is probably not a safe assumption. The alternative would be to search
// backward to find beginning of history. How far to search?
const noData = bars.length === 0;
onHistoryCallback(bars, {noData});
} catch (error) {
log('[getBars]: Get error', error);
onErrorCallback(error);
}
},
subscribeBarsOnRealtimeCallback: null,
subscribeBars(
symbolInfo,
resolution,
onRealtimeCallback2,
subscriberUID,
onResetCacheNeededCallback,
) {
const oldCb = onRealtimeCallback2
function onRealtimeCallback() {
log('push bar', ...arguments)
oldCb(...arguments)
}
log('[subscribeBars]', symbolInfo, resolution, subscriberUID);
const symbol = getAllSymbols()[symbolInfo.full_name]
const co = useChartOrderStore()
// todo redo symbolInfo: the full_name passed to TV should just be the pool addr. when searching a symbol, go ahead and select the first/most liquid automatically
let poolAddr = null
const selectedFee = co.selectedPool[1];
for (const [addr, fee] of symbol.pools) {
if (fee === selectedFee) {
poolAddr = addr
break
}
}
if (poolAddr===null) {
console.error(`Could not find pool for fee ${selectedFee}: ${symbol.pools}`)
return
}
const chainId = useStore().chainId;
const res = convertTvResolution(resolution)
const period = res.name
console.log('subscription symbol', symbol, getAllSymbols())
const sub = new RealtimeSubscription(chainId, poolAddr, res, symbol, subscriberUID, onRealtimeCallback, onResetCacheNeededCallback)
log('sub', sub.key)
subOHLC(chainId, poolAddr, period)
this.startOHLCBumper()
},
unsubscribeBars(subscriberUID) {
log('[unsubscribeBars]', subscriberUID);
const sub = subByTvSubId[subscriberUID]
if (sub===undefined) {
console.log(`warning: no subscription found for tvSubId ${subscriberUID}`)
return
}
unsubOHLC(sub.chainId, sub.poolAddr, sub.res.name)
log('unsub',sub.key)
sub.close()
delete this.pushedBars[sub.key]
delete this.recentBars[sub.key]
},
// key-value format:
// 'chain|pool|period': [[javatime, open, high, low, close], ...]
pushedBars: {}, // bars actually sent to TradingView
recentBars:{}, // bars received from server notifications
pushRecentBars(key, recent) {
this.recentBars[key] = recent
const historical = this.pushedBars[key]
if (historical!==undefined)
this.overlapAndPush(key, recent)
},
overlapAndPush(key, ohlcs) {
log('overlapAndPush',key,ohlcs)
if (!ohlcs || ohlcs.length === 0) return
const sub = subByKey[key]
// do not check reorgs on mock symbols because the dev environment price will be different from the main chain
// price and reorgs would happen every time.
const checkReorg = sub.symbol.x?.data === undefined
const res = sub.res
const period = res.seconds * 1000
const bars = [] // to push
const pushed = this.pushedBars[key]
log('got pushed bars', pushed)
let pi = 0 // pushed index
let time
let price
if (pushed === undefined) {
time = ohlcs[0].time
price = ohlcs[0].close
}
else {
const last = pushed.length - 1
time = pushed[last].time
price = pushed[last].close
}
log('last time/price', time, price)
for( const ohlc of ohlcs ) {
log('handle ohlc', ohlc)
// forward the pi index to at least the current ohlc time
while( pi < pushed.length && pushed[pi].time < ohlc.time ) pi++
if (pi < pushed.length-1) {
// finalized bars must match the previous push exactly or else there was a reorg and we need to reset
const p = pushed[pi]
log('check reorg', checkReorg, pi, p, ohlc)
if (checkReorg &&
(p.time !== ohlc.time || p.open !== ohlc.open || p.high !== ohlc.high ||
p.low !== ohlc.low || p.close !== ohlc.close) )
{
console.log('RESET TV CACHE')
return this.resetCache(key)
}
}
else {
// the last pushed bar and anything after it can be sent to TV
while (time + period < ohlc.time) {
// fill gap
time += period
const bar = {time, open: price, high: price, low: price, close: price};
log('fill', bar)
bars.push(bar)
}
bars.push(ohlc)
time = ohlc.time
price = ohlc.close
}
}
return this.pushToTV(key, bars)
},
resetCache(key) {
log('resetting TV data cache')
const sub = subByKey[key]
if (sub===undefined) {
console.log(`warning: no subscription found for dexorder key ${key}`)
return
}
sub.onResetCacheCb()
},
updatePushedCache(key, ohlcs) {
log('updatePushedCache', key, ohlcs)
if (!(key in this.pushedBars))
this.pushedBars[key] = ohlcs
else {
const prev = this.pushedBars[key]
log('prev pushed', prev)
const endTime = prev[prev.length - 1].time;
if (endTime === ohlcs[0].time)
// the new ohlc's overlap the old time, so exclude the most recent historical item, which gets replaced
this.pushedBars[key] = [...prev.slice(0, -1), ...ohlcs]
else if (endTime <= ohlcs[ohlcs.length-1].time) {
// no overlap of any bars. full append.
this.pushedBars[key] = [...prev, ...ohlcs]
}
// otherwise the ohlc's being pushed are =>older<= than the ones in the pushedBars cache. This happens
// because TV can request data pages (using getBars()) out of chronological order.
}
},
pushToTV(key, ohlcs) {
if (ohlcs.length===0) return
log('pushing bars to tv', ohlcs)
this.updatePushedCache(key, ohlcs); // we cache the raw bars before inversion so they match dexorder data sources
const sub = subByKey[key]
if (sub===undefined) {
console.log(`warning: could not find subscription for dexorder sub key ${key}`)
return
}
if (sub.symbol.inverted)
ohlcs = invertOhlcs(ohlcs)
for (const ohlc of ohlcs)
sub.onRealtimeCb(ohlc)
},
poolCallback(chainId, poolPeriod, ohlcs) {
if (!ohlcs || ohlcs.length===0) return
const key = `${chainId}|${poolPeriod}`;
const bars = []
for (const ohlc of ohlcs) {
let close = parseFloat(ohlc[4]) // close
const bar = {
time: ohlc[0] * 1000,
open: ohlc[1] ? parseFloat(ohlc[1]) : close, // open
high: ohlc[2] ? parseFloat(ohlc[2]) : close, // high
low: ohlc[3] ? parseFloat(ohlc[3]) : close, // low
close: close,
}
bars.push(bar)
}
return this.pushRecentBars(key, bars)
},
intervalChanged(seconds) {
// rollover bumper
// this timer function creates a new bar when the period rolls over
this.startOHLCBumper()
},
// The OHLC bumper advances bars at the end of the period. If the price doesn't change, no new data will arrive, so the
// new bar is implicit and must be generated dynamically.
startOHLCBumper() {
const co = useChartOrderStore()
if (_rolloverBumper !== null)
clearTimeout(_rolloverBumper)
const period = co.intervalSecs;
if (period === 0)
return
const now = Date.now()
const nextRollover = ohlcStart(now/1000 + period)*1000 + 2000 // two second delay to wait for server data
const delay = nextRollover - now
_rolloverBumper = setTimeout(this.bumpOHLC.bind(this), delay)
},
bumpOHLC() {
log('bumpOHLC')
_rolloverBumper = null
const secs = useChartOrderStore().intervalSecs;
for (const sub of Object.values(subByKey)) {
log('check bump', sub.res.seconds, secs)
if (sub.res.seconds === secs) {
const pushed = this.pushedBars[sub.key]
log('check pushed', pushed)
if (pushed !== undefined && pushed.length > 0) {
const lastBar = pushed[pushed.length - 1]
const price = lastBar.close
const now = timestamp() * 1000
const period = sub.res.seconds * 1000
const fills = []
for( let time=lastBar.time + period; time < now; time += period ) {
log('pushing bump', time, price)
const bar = {time, open: price, high: price, low: price, close: price}
fills.push(bar)
}
this.pushToTV(sub.key, fills)
}
else {
log('warning: bumpOHLC() found no previous bars')
}
}
}
this.startOHLCBumper()
},
}
let _rolloverBumper = null
let defaultSymbol = null