From 9b6a9c3cda84c743112ba7e746319d03d9c17ec4 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 13 Feb 2024 23:50:57 -0400 Subject: [PATCH] OHLC's use timestamps instead of string dates --- src/dexorder/__init__.py | 6 ++++-- src/dexorder/configuration/load.py | 11 ----------- src/dexorder/configuration/schema.py | 11 ----------- src/dexorder/ohlc.py | 28 ++++++++++++++-------------- 4 files changed, 18 insertions(+), 38 deletions(-) diff --git a/src/dexorder/__init__.py b/src/dexorder/__init__.py index 4c3edbd..e9c652d 100644 --- a/src/dexorder/__init__.py +++ b/src/dexorder/__init__.py @@ -10,8 +10,10 @@ dec = Decimal def now(): return datetime.now(timezone.utc) -def timestamp(): - return int(datetime.now().timestamp()) +def timestamp(when=None): + if when is None: + when = datetime.now() + return int(when.timestamp()) def from_timestamp(ts): return datetime.fromtimestamp(ts, timezone.utc) diff --git a/src/dexorder/configuration/load.py b/src/dexorder/configuration/load.py index a440d03..662c6e0 100644 --- a/src/dexorder/configuration/load.py +++ b/src/dexorder/configuration/load.py @@ -7,7 +7,6 @@ from omegaconf.errors import OmegaConfBaseException from .schema import Config from .standard_accounts import default_accounts_config -from .standard_tokens import default_token_config schema = OmegaConf.structured(Config()) @@ -20,7 +19,6 @@ def load_config(): # noinspection PyTypeChecker result:ConfigDict = OmegaConf.merge( schema, - load_tokens(), load_accounts(), from_toml('pool.toml'), from_toml('dexorder.toml'), @@ -31,15 +29,6 @@ def load_config(): return result -def load_tokens(): - token_conf = OmegaConf.create({'tokens': default_token_config}) - try: - OmegaConf.merge(schema, token_conf) - return token_conf - except OmegaConfBaseException as _x: - raise ConfigException(f'Error while processing default tokens:\n{_x}') - - def load_accounts(): accounts_conf = OmegaConf.create({'accounts': default_accounts_config}) try: diff --git a/src/dexorder/configuration/schema.py b/src/dexorder/configuration/schema.py index 9853c40..9581877 100644 --- a/src/dexorder/configuration/schema.py +++ b/src/dexorder/configuration/schema.py @@ -21,19 +21,8 @@ class Config: polling: float = 0 # seconds between queries for a new block. 0 disables polling and uses a websocket subscription on ws_url instead backfill: int = 0 # if not 0, then runner will initialize an empty database by backfilling from the given block height - tokens: list['TokenConfig'] = field(default_factory=list) - account: Optional[str] = None # may be a private key or an account alias accounts: Optional[dict[str,str]] = field(default_factory=dict) # account aliases deployments: Optional[dict[str,str]] = field(default_factory=dict) min_gas: str = '0' - -@dataclass -class TokenConfig: - name: str - symbol: str - decimals: int - chain: str - address: str - abi: Optional[str] = None diff --git a/src/dexorder/ohlc.py b/src/dexorder/ohlc.py index 5640441..3b8553c 100644 --- a/src/dexorder/ohlc.py +++ b/src/dexorder/ohlc.py @@ -7,7 +7,7 @@ from typing import Optional, NamedTuple, Reversible, Union from cachetools import LFUCache -from dexorder import dec, config, from_isotime, minutely, from_timestamp +from dexorder import dec, config, from_timestamp, timestamp from dexorder.base.chain import current_chain from dexorder.blockstate import BlockDict, DiffItem, current_blockstate from dexorder.blockstate.diff import DiffEntryItem @@ -24,16 +24,16 @@ OHLC_PERIODS = [ OHLC_DATE_ROOT = datetime(2009, 1, 4, tzinfo=timezone.utc) # Sunday before Bitcoin Genesis -# OHLC's are stored as [time, open, high, low, close] string values. If there was no data during the interval, -# then open, high, and low are None but the close value is carried over from the previous interval. -OHLC = list[str] # typedef +# OHLC's are stored as [timestamp, open, high, low, close] with prices as string values. If there was no data during +# the interval, then open, high, and low are None but the close value is carried over from the previous interval. +OHLC = list[Union[None,int,str]] # typedef def opt_dec(v): return None if v is None else dec(v) def dt(v): - return v if isinstance(v, datetime) else from_isotime(v) + return v if isinstance(v, datetime) else from_timestamp(v) @dataclass class NativeOHLC: @@ -50,7 +50,7 @@ class NativeOHLC: @property def ohlc(self) -> OHLC: return [ - minutely(self.start), + timestamp(self.start), None if self.open is None else str(self.open), None if self.high is None else str(self.high), None if self.low is None else str(self.low), @@ -154,13 +154,13 @@ class OHLCRepository: return # do not track symbols which have not been explicity set up p = str(price) historical = [] - updated = [OHLC((minutely(ohlc_start_time(time, period)), p, p, p, p))] + updated = [OHLC((timestamp(ohlc_start_time(time, period)), p, p, p, p))] # log.debug(f'\tcreated new bars {updated}') else: updated = update_ohlc(historical[-1], period, time, price) # drop any historical bars that are older than we need oldest_needed = from_timestamp(current_blockstate.get().root_block.timestamp) - period # cover the root block time plus one period prior - trim = (oldest_needed - from_isotime(historical[0][0])) // period + trim = (oldest_needed - from_timestamp(historical[0][0])) // period if trim > 0: historical = historical[trim:] @@ -168,8 +168,8 @@ class OHLCRepository: if not historical or not updated: updated = historical + updated else: - last_bar = from_isotime(historical[-1][0]) - first_updated = from_isotime(updated[0][0]) + last_bar = from_timestamp(historical[-1][0]) + first_updated = from_timestamp(updated[0][0]) overlap = (first_updated - last_bar) // period + 1 updated = historical[:-overlap] + updated if overlap > 0 else historical + updated # log.debug(f'\tnew recents: {updated}') @@ -187,17 +187,17 @@ class OHLCRepository: if not chunk: chunk = [ohlc] else: - start = from_isotime(chunk[0][0]) + start = from_timestamp(chunk[0][0]) index = (time - start) // period # log.debug(f'save {symbol} {ohlc_name(period)} chunk {start} index {index} <= {len(chunk)}') if index > len(chunk): log.error(f'chunk gap: {index} > {len(chunk)} {symbol} {period_name(period)} {ohlc}'+''.join(f'\n\t{c}' for c in chunk)) exit(1) if index == len(chunk): - assert from_isotime(chunk[-1][0]) + period == time + assert from_timestamp(chunk[-1][0]) + period == time chunk.append(ohlc) else: - assert from_isotime(chunk[index][0]) == time + assert from_timestamp(chunk[index][0]) == time chunk[index] = ohlc self.save_chunk(symbol, period, chunk) @@ -222,7 +222,7 @@ class OHLCRepository: def save_chunk(self, symbol: str, period: timedelta, chunk: list[OHLC]): if not chunk: return - path = self.chunk_path(symbol, period, from_isotime(chunk[0][0])) + path = self.chunk_path(symbol, period, from_timestamp(chunk[0][0])) for _ in range(2): try: with open(path, 'w') as file: