commit 68647364cd6fe5bc77b62e1c9181d25fd92350bd Author: Tim Olson <> Date: Mon Sep 18 17:41:56 2023 -0400 BlockState working in memory with basic event triggers diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d785257 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +venv/ +*secret* +dexorder.toml +./contract +.idea diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..9a536d1 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,111 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# this is set programatically in the env.py +sqlalchemy.url = + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..8afde53 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,79 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# DEXORDER SETUP +from sys import path +path.append('src') +import dexorder.db.model +target_metadata = dexorder.db.model.Base.metadata +config.set_main_option('sqlalchemy.url', dexorder.config.db_url) + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..d5b9f93 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import dexorder.db +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1163dbc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +sqlalchemy~=2.0.20 +alembic~=1.11.3 +omegaconf~=2.3.0 +web3==6.9.0 +psycopg2-binary +orjson~=3.9.7 +sortedcontainers +hexbytes~=0.3.1 +defaultlist~=1.0.0 diff --git a/src/dexorder/__init__.py b/src/dexorder/__init__.py new file mode 100644 index 0000000..324458f --- /dev/null +++ b/src/dexorder/__init__.py @@ -0,0 +1,21 @@ +# NARG is used in argument defaults to mean "not specified" rather than "specified as None" +class _NARG: + def __bool__(self): return False +NARG = _NARG() +ADDRESS_0 = '0x0000000000000000000000000000000000000000' +WEI = 1 +GWEI = 1_000_000_000 +ETH = 1_000_000_000_000_000_000 + +# noinspection PyProtectedMember +from .util.cwd import _cwd +_cwd() # do this first so that config has the right current working directory + +# ordering here is important! +from .base.chain import Blockchain # the singletons are loaded into the dexorder.blockchain.* namespace +from .util import async_yield +from .base.fixed import Fixed2, FixedDecimals, Dec18 +from .configuration import config +from .base.account import Account # must come before context +from .base.context import ctx +from .base.token import Token, tokens diff --git a/src/dexorder/base/__init__.py b/src/dexorder/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dexorder/base/account.py b/src/dexorder/base/account.py new file mode 100644 index 0000000..97d0242 --- /dev/null +++ b/src/dexorder/base/account.py @@ -0,0 +1,66 @@ +from typing import Union, Optional + +import eth_account +from eth_account.signers.local import LocalAccount +from web3.middleware import construct_sign_and_send_raw_middleware + +from dexorder import NARG, config + + +# this is just here for typing the extra .name. the __new__() function returns an eth_account...LocalAccount +# we do it this way because web3py expects a LocalAccount object but we cannot construct one directly with a super() +# call but must instead use a factory :( +class Account (LocalAccount): + + @staticmethod + # noinspection PyInitNewSignature + def get(account:[Union,str]=NARG) -> Optional[LocalAccount]: + if account is NARG: + account = config.account + if type(account) is not str: + return account + + key_str = config.accounts.get(account, account) + try: + local_account = eth_account.Account.from_key(key_str) + return Account(local_account, key_str, account) + except ValueError: + try: + # was the key missing a leading '0x'? + fixed = '0x' + key_str + local_account = eth_account.Account.from_key(fixed) + print(f'WARNING: account "{account}" is missing a leading "0x"') + return Account(local_account, fixed, account) + except ValueError: + pass + try: + # was the key an integer posing as a string? + converted = f'{int(key_str):#0{66}x}' + local_account = eth_account.Account.from_key(converted) + print(f'WARNING: account "{account}" is set as an integer instead of a string. Converted to: {converted}') + return Account(local_account, converted, account) + except ValueError: + pass + raise ValueError(f'Could not construct account for name "{account}"') + + def __init__(self, local_account: LocalAccount, key_str, name: str): # todo chain_id? + super().__init__(local_account._key_obj, local_account._publicapi) # from digging into the source code + self.name = name + self.transaction_counter = 0 # used by GasHandler to detect when new transactions were fired + self.key_str = key_str + self.signing_middleware = construct_sign_and_send_raw_middleware(self) + + def attach(self, w3): + w3.eth.default_account = self.address + try: + w3.middleware_onion.remove('account_signer') + except ValueError: + pass + w3.middleware_onion.add(self.signing_middleware, 'account_signer') + + def balance(self): + return ctx.w3.eth.get_balance(self.address) + + def __str__(self): + return self.name + diff --git a/src/dexorder/base/blockstate.py b/src/dexorder/base/blockstate.py new file mode 100644 index 0000000..d034d2a --- /dev/null +++ b/src/dexorder/base/blockstate.py @@ -0,0 +1,220 @@ +from collections import defaultdict +from contextvars import ContextVar +from logging import Logger +from typing import Union, TypeVar, Generic, Any + +from sortedcontainers import SortedList + +from dexorder import NARG +from dexorder.db.model.block import Block + +log = Logger('dexorder.blockstate') + + +class BlockState: + DELETE = object() + + by_chain: dict[int, 'BlockState'] = {} + + """ + Since recent blocks can be part of temporary forks, we need to be able to undo certain operations if they were part of a reorg. Instead of implementing + undo, we recover state via snapshot plus replay of recent diffs. When old blocks become low enough in the blockheight they may be considered canonical + at which point the deltas may be reliably incorporated into a new snapshot or rolling permanent collection. BlockState manages separate memory areas + for every block, per-block state that defaults to its parent's state, up the ancestry tree to the root. State clients may read the state for their block, + applying any diffs from the root state to the target block. + """ + + @staticmethod + def cur() -> 'BlockState': + return _cur.get() + + @staticmethod + def set_cur(value: 'BlockState'): + _cur.set(value) + + def __init__(self, root_block: Block, root_state: dict): + self.root_block: Block = root_block + self.root_state: dict = root_state + self.by_height: SortedList[tuple[int, Block]] = SortedList(key=lambda x: x[0]) + self.by_hash: dict[bytes, Block] = {root_block.hash: root_block} + self.diffs: dict[bytes, dict[Any, dict[Any, Union[Any, BlockState.DELETE]]]] = defaultdict(dict) # by series + self.ancestors: dict[bytes, Block] = {} + BlockState.by_chain[root_block.chain] = self + + def add_block(self, block: Block) -> Union[int, Block, None]: + """ + If block is the same age as root_height or older, it is ignored and None is returned. Otherwise, returns the found parent block if available + or else self.root_height. + The ancestor block is set in the ancestors dictionary and any state updates to block are considered to have occured between the registered ancestor + block and the given block. This could be an interval of many blocks, and the ancestor does not need to be the block's immediate parent. + """ + # check height + height_diff = block.height - self.root_block.height + if height_diff <= 0: + log.debug(f'IGNORING old block {block}') + return None + if block.hash not in self.by_hash: + self.by_hash[block.hash] = block + self.by_height.add((block.height, block)) + log.debug(f'new block state {block}') + parent = self.by_hash.get(block.parent) + if parent is None: + self.ancestors[block.hash] = self.root_block + return self.root_block.height + else: + self.ancestors[block.hash] = parent + return parent + + def promote_root(self, block): + assert block.hash in self.by_hash + diffs = self.collect_diffs(block) + BlockState.apply_diffs(self.root_state, diffs) + del self.by_hash[self.root_block.hash] + while self.by_height and self.by_height[0][0] <= block.height: + height, dead = self.by_height.pop(0) + if dead is not block: + try: + del self.by_hash[dead.hash] + except KeyError: + pass + try: + del self.diffs[dead.hash] + except KeyError: + pass + try: + del self.ancestors[dead.hash] + except KeyError: + pass + self.root_block = block + + @staticmethod + def apply_diffs(obj, diffs): + for series_key, series in diffs.items(): + for key, value in series.items(): + if value is BlockState.DELETE: + try: + del obj[series_key][key] + except KeyError: + pass + else: + series_obj = obj.get(series_key) + if series_obj is None: + obj[series_key] = series_obj = {} + series_obj[key] = value + + + def collect_diffs(self, block, series_key=NARG): + diffs = {} + while block is not self.root_block: + block_diffs = self.diffs.get(block.hash) + if block_diffs is not None: + if series_key is NARG: + for s_key, series in block_diffs.items(): + series_diffs = diffs.get(s_key) + if series_diffs is None: + series_diffs = diffs[s_key] = {} + for k, v in series.items(): + series_diffs.setdefault(k, v) + else: + series = block_diffs.get(series_key) + if series is not None: + for k, v in series.items(): + diffs.setdefault(k, v) + block = self.ancestors[block.hash] + return diffs + + +_cur = ContextVar[BlockState]('BlockState.cur') + + +T = TypeVar('T') + + +class BlockDict(Generic[T]): + + def __init__(self, series_key): + self.series_key = series_key + + def __setitem__(self, item, value): + BlockDict.setitem(self.series_key, item, value) + + def __getitem__(self, item): + return BlockDict.getitem(self.series_key, item) + + def __delitem__(self, item): + BlockDict.delitem(self.series_key, item) + + def __contains__(self, item): + return BlockDict.contains(self.series_key, item) + + def add(self, item): + """ set-like semantics. the item key is added with a value of None. """ + BlockDict.setitem(self.series_key, item, None) + + def items(self): + return BlockDict.iter_items(self.series_key) + + + @staticmethod + def setitem(series_key, item, value): + state = BlockState.cur() + block = Block.cur() + if block.height > state.root_block.height: + diffs = state.diffs[block.hash] + series = diffs.get(series_key) + if series is None: + series = diffs[series_key] = {} + else: + series = state.root_state.get(series_key) + if series is None: + series = state.root_state[series_key] = {} + series[item] = value + + @staticmethod + def getitem(series_key, item): + state = BlockState.cur() + block = Block.cur() + while block.height > state.root_block.height: + diffs = state.diffs.get(block.hash) + if diffs is not None: + series = diffs.get(series_key) + if series is not None: + value = series.get(item, NARG) + if value is BlockState.DELETE: + raise KeyError + if value is not NARG: + return value + block = state.ancestors[block.hash] + if block is not state.root_block: + raise ValueError('Orphaned block is invalid',Block.cur().hash) + root_series = state.root_state.get(series_key) + if root_series is not None: + value = root_series.get(item, NARG) + if value is BlockState.DELETE: + raise KeyError + if value is not NARG: + return value + raise KeyError + + @staticmethod + def delitem(series_key, item): + BlockDict.setitem(series_key, item, BlockState.DELETE) + + @staticmethod + def contains(series_key, item): + try: + BlockDict.getitem(series_key, item) + return True + except KeyError: + return False + + @staticmethod + def iter_items(series_key): + state = BlockState.cur() + block = Block.cur() + root = state.root_state.get(series_key,{}) + diffs = state.collect_diffs(block, series_key) + # first output recent changes in the diff obj + yield from ((k,v) for k,v in diffs.items() if v is not BlockState.DELETE) + # then all the items not diffed + yield from ((k,v) for k,v in root.items() if k not in diffs) diff --git a/src/dexorder/base/chain.py b/src/dexorder/base/chain.py new file mode 100644 index 0000000..f9d192e --- /dev/null +++ b/src/dexorder/base/chain.py @@ -0,0 +1,50 @@ +from contextvars import ContextVar + + +class Blockchain: + @staticmethod + def cur() -> 'Blockchain': + return _cur.get() + + @staticmethod + def set_cur(value: 'Blockchain'): + _cur.set(value) + + @staticmethod + def for_id(chain_id): + result = Blockchain._instances_by_id.get(chain_id) + if result is None: + result = Blockchain(chain_id, 'Unknown') + return result + + @staticmethod + def for_name(chain_name): + return Blockchain._instances_by_name[chain_name] + + @staticmethod + def get(name_or_id): + return Blockchain.for_name(name_or_id) if type(name_or_id) is str else Blockchain.for_id(name_or_id) + + def __init__(self, chain_id, name): + self.chain_id = chain_id + self.name = name + Blockchain._instances_by_id[chain_id] = self + Blockchain._instances_by_name[name] = self + + def __str__(self): + return self.name + + _instances_by_id = {} + _instances_by_name = {} + + +# https://chainlist.org/ + +Ethereum = Blockchain(1, 'Ethereum') +Goerli = Blockchain(5, 'Goerli') +Polygon = Blockchain(137, 'Polygon') # POS not zkEVM +Mumbai = Blockchain(80001, 'Mumbai') +BSC = Blockchain(56, 'BSC') +Arbitrum = ArbitrumOne = Blockchain(42161, 'ArbitrumOne') + +_cur = ContextVar[Blockchain]('Blockchain.cur') diff --git a/src/dexorder/base/event_manager.py b/src/dexorder/base/event_manager.py new file mode 100644 index 0000000..4fcc5a3 --- /dev/null +++ b/src/dexorder/base/event_manager.py @@ -0,0 +1,24 @@ +from eth_utils import keccak + +from dexorder.base.blockstate import BlockDict + + +class EventManager: + def __init__(self): + self.all_topics = set() + self.triggers:dict[str,BlockDict] = {} + + def add_handler(self, topic: str, callback): + if not topic.startswith('0x'): + topic = '0x'+keccak(text=topic).hex().lower() + triggers = self.triggers.get(topic) + if triggers is None: + triggers = self.triggers[topic] = BlockDict(topic) + triggers.add(callback) + self.all_topics.add(topic) + + def handle_logs(self, logs): + for log in logs: + for callback, _ in self.triggers.get(log.topics[0].hex(), []).items(): + callback(log) + diff --git a/src/dexorder/base/fixed.py b/src/dexorder/base/fixed.py new file mode 100644 index 0000000..347a197 --- /dev/null +++ b/src/dexorder/base/fixed.py @@ -0,0 +1,121 @@ +# binary fixed point math +from _decimal import Decimal +from typing import Tuple + + +class FixedDecimals: + def __init__(self, decimals): + self._denom = Decimal(10) ** Decimal(decimals) + + def dec(self, amount: int) -> Decimal: + return Decimal(amount) / self._denom + + def amount(self, decimal_like: [Decimal, int, str, float]) -> int: + return round(Decimal(decimal_like) * self._denom) + + +class Fixed2: + def __init__(self, int_value, denom_bits): + assert (int_value == int(int_value)) + self.int_value = int_value + self.dbits = denom_bits + + @property + def value(self): + return self.int_value / 2 ** self.dbits + + def round(self, denom_bits): + if self.dbits > denom_bits: + int_value = round(self.int_value / 2 ** (self.dbits - denom_bits)) + elif self.dbits < denom_bits: + int_value = self.int_value * 2 ** (denom_bits - self.dbits) + else: + int_value = self.int_value + return Fixed2(int_value, self.dbits) + + def __format__(self, format_spec): + return self.value.__format__(format_spec) if format_spec.endswith('f') \ + else self.int_value.__format__(format_spec[:-1]) + + def __str__(self) -> str: + return str(self.int_value) + + def __repr__(self) -> str: + return f'Fixed({self.int_value},{self.dbits})' + + def __hash__(self): + return hash(self.int_value) ^ hash(self.dbits) + + def __int__(self) -> int: + return self.int_value + + def __float__(self) -> float: + return self.value + + def __abs__(self) -> 'Fixed2': + return Fixed2(abs(self.int_value), self.dbits) + + def __hex__(self) -> str: + return hex(self.int_value) + + def __neg__(self) -> 'Fixed2': + return Fixed2(-self.int_value, self.dbits) + + def __bool__(self) -> bool: + return bool(self.int_value) + + def __add__(self, other: 'Fixed2') -> 'Fixed2': + dbits = max(self.dbits, other.dbits) + return Fixed2(self.round(dbits).int_value + other.round(dbits).int_value, dbits) + + def __sub__(self, other: 'Fixed2') -> 'Fixed2': + dbits = max(self.dbits, other.dbits) + return Fixed2(self.round(dbits).int_value - other.round(dbits).int_value, dbits) + + def __mul__(self, other: 'Fixed2') -> 'Fixed2': + dbits = max(self.dbits, other.dbits) + self_up = self.round(dbits) + other_up = other.round(dbits) + return Fixed2(self_up.int_value * other_up.int_value // 2 ** dbits, dbits) + + def __floordiv__(self, other): + dbits = max(self.dbits, other.dbits) + self_up = self.round(dbits) + other_up = other.round(dbits) + return self_up.int_value // other_up.int_value + + def __divmod__(self, other: 'Fixed2') -> Tuple['Fixed2', 'Fixed2']: + dbits = max(self.dbits, other.dbits) + self_up = self.round(dbits) + other_up = other.round(dbits) + div, mod = self_up.int_value.__divmod__(other_up.int_value) + return Fixed2(div, dbits), Fixed2(mod, dbits) # mod not supported + + def __eq__(self, other): + return (self - other).int_value == 0 + + def __le__(self, other): + return (self - other).int_value <= 0 + + def __gt__(self, other): + return (self - other).int_value > 0 + + +class Fixed2Type: + def __init__(self, denominator_bits): + self.dbits = denominator_bits + + def __call__(self, int_value): + return Fixed2(int_value, self.dbits) + + def from_string(self, str_value): + return self(round(Decimal(str_value) * 2 ** self.dbits)) + + def from_number(self, value): + return self(round(value) * 2 ** self.dbits) + + +X96 = Fixed2Type(96) +X128 = Fixed2Type(128) + +Dec18 = FixedDecimals(18) diff --git a/src/dexorder/base/token.py b/src/dexorder/base/token.py new file mode 100644 index 0000000..2b7cc41 --- /dev/null +++ b/src/dexorder/base/token.py @@ -0,0 +1,120 @@ +from collections import defaultdict +from decimal import Decimal + +from sqlalchemy.orm import Mapped +from web3 import Web3 + +from dexorder import config, ctx, Blockchain, NARG, FixedDecimals, ADDRESS_0 +from dexorder.blockchain import ByBlockchainDict +from dexorder.base.chain import Polygon, ArbitrumOne, Ethereum +from dexorder.contract import ContractProxy, abis +import dexorder.db.column as col + + +class Token (ContractProxy, FixedDecimals): + chain: Mapped[col.Blockchain] + address: Mapped[col.Address] + decimals: Mapped[col.Uint8] + name: Mapped[str] + symbol: Mapped[str] + + + @staticmethod + def get(name_or_address:str, *, chain_id=None) -> 'Token': + try: + return tokens.get(name_or_address, default=NARG, chain_id=chain_id) # default=NARG will raise + except KeyError: + try: + # noinspection PyTypeChecker + return Web3.to_checksum_address(name_or_address) + except ValueError: + raise ValueError(f'Could not resolve token {name_or_address} for chain {ctx.chain_id}') + + def __init__(self, chain_id, address, decimals, symbol, name, *, abi=None): + FixedDecimals.__init__(self, decimals) + if abi is None: + load = 'ERC20' + else: + load = None + abi = abis.get(abi,abi) + ContractProxy.__init__(self, address, load, abi=abi) + self.chain_id = chain_id + self.address = address + self.decimals = decimals + self.symbol = symbol + self.name = name + + def balance(self, address: str = None) -> int: + if address is None: + address = ctx.address + return self.balanceOf(address) + + def balance_dec(self, address: str = None) -> Decimal: + return self.dec(self.balance(address)) + + + def __str__(self): + return self.symbol + + def __repr__(self): + return f'{self.symbol}({self.address},{self.decimals})' + + def __eq__(self, other): + return self.chain_id == other.chain_id and self.address == other.address + + def __hash__(self): + return hash((self.chain_id,self.address)) + + +class NativeToken (FixedDecimals): + """ Token-like but not a contract. """ + + @staticmethod + def get( chain_id = None) -> 'NativeToken': + if chain_id is None: + chain_id = ctx.chain_id + return _native_tokens[chain_id] + + def __init__(self, chain_id, decimals, symbol, name, *, wrapper_token = None): + self.chain_id = chain_id + self.address = ADDRESS_0 # todo i think there's actually an address? like 0x11 or something? + super().__init__(decimals) + self.symbol = symbol + self.name = name + self._wrapper_token = wrapper_token if wrapper_token is not None else _tokens_by_chain[chain_id]['W'+symbol] + + def balance(self, address: str = None) -> int: + if address is None: + address = ctx.address + assert ctx.chain_id == self.chain_id + return ctx.w3.eth.get_balance(address) + + def balance_dec(self, address: str = None) -> Decimal: + return self.dec(self.balance(address)) + + @property + def wrapper(self) -> Token: + return self._wrapper_token + + def __repr__(self): + return self.symbol + + + +# convert TokenConfigs to Tokens +_tokens_by_chain:dict[int,dict[str,Token]] = defaultdict(dict) +for _c in config.tokens: + _chain_id = Blockchain.get(_c.chain).chain_id + _tokens_by_chain[_chain_id][_c.symbol] = Token(_chain_id, _c.address, _c.decimals, _c.symbol, _c.name, abi=_c.abi) + +_native_tokens: dict[int, NativeToken] = { + # Ethereum.chain_id: NativeToken(Ethereum.chain_id, 18, 'ETH', 'Ether'), # todo need WETH on Ethereum + # Polygon.chain_id: NativeToken(Polygon.chain_id, 18, 'MATIC', 'Polygon'), +} + +for _chain_id, _native in _native_tokens.items(): + # noinspection PyTypeChecker + _tokens_by_chain[_chain_id][_native.symbol] = _native + +tokens = ByBlockchainDict[Token](_tokens_by_chain) + diff --git a/src/dexorder/bin/__init__.py b/src/dexorder/bin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/dexorder/bin/executable.py b/src/dexorder/bin/executable.py new file mode 100644 index 0000000..e477fbc --- /dev/null +++ b/src/dexorder/bin/executable.py @@ -0,0 +1,50 @@ +import logging +from asyncio import CancelledError +from traceback import print_exception +import asyncio +from signal import Signals +from typing import Coroutine + +from dexorder import configuration + +if __name__ == '__main__': + raise Exception('this file is meant to be imported not executed') + + +ignorable_exceptions = [CancelledError] + +log = logging.getLogger(__name__) + +async def _shutdown_coro(_sig, loop, extra_shutdown): + log.info('shutting down') + if extra_shutdown is not None: + extra_shutdown() + tasks = [t for t in asyncio.all_tasks() if t is not + asyncio.current_task()] + for task in tasks: + task.cancel() + exceptions = await asyncio.gather(*tasks, return_exceptions=True) + loop.stop() + for x in exceptions: + if x is not None and x.__class__ not in ignorable_exceptions: + print_exception(x) + + +def execute(main:Coroutine, shutdown=None, parse_args=True): + if parse_args: + configuration.parse_args() + loop = asyncio.get_event_loop() + signals = Signals.SIGQUIT, Signals.SIGTERM, Signals.SIGINT + for s in signals: + loop.add_signal_handler(s, lambda sig=s: asyncio.create_task(_shutdown_coro(sig, loop, shutdown))) + task = loop.create_task(main) + loop.run_until_complete(task) + x = task.exception() + if x is not None: + print_exception(x) + for t in asyncio.all_tasks(): + t.cancel() + else: + loop.run_forever() + loop.stop() + loop.close() diff --git a/src/dexorder/bin/main.py b/src/dexorder/bin/main.py new file mode 100644 index 0000000..6017ae4 --- /dev/null +++ b/src/dexorder/bin/main.py @@ -0,0 +1,122 @@ +import logging +from asyncio import CancelledError + +from hexbytes import HexBytes +from web3 import AsyncWeb3, WebsocketProviderV2, AsyncHTTPProvider +from web3.types import FilterParams + +from dexorder import config, Blockchain +from dexorder.base.blockstate import BlockState, BlockDict +from dexorder.base.event_manager import EventManager +from dexorder.bin.executable import execute +from dexorder.configuration import resolve_rpc_url +from dexorder.db.model import Block + +log = logging.getLogger('dexorder') + +ROOT_AGE = 10 # todo set per chain + + +wallets = BlockDict('wallets') + +def handle_transfer(event): + to_address = event.topics[2].hex() + wallets.add(to_address) + +def setup_triggers(event_manager: EventManager): + event_manager.add_handler('Transfer(address,address,uint256)', handle_transfer) + + +async def main(): + """ + 1. load root stateBlockchain + a. if no root, init from head + b. if root is old, batch forward by height + 2. discover new heads + 2b. find in-memory ancestor else use root + 3. context = ancestor->head diff + 4. query global log filter + 5. process new vaults + 6. process new orders and cancels + a. new pools + 7. process Swap events and generate pool prices + 8. process price horizons + 9. process token movement + 10. process swap triggers (zero constraint tranches) + 11. process price tranche triggers + 12. process horizon tranche triggers + 13. filter by time tranche triggers + 14. bundle execution requests and send tx. tx has require(block ROOT_AGE: + b = block + try: + for _ in range(1,ROOT_AGE): + # we walk backwards ROOT_AGE and promote what's there + b = state.by_hash[b.parent] + except KeyError: + pass + else: + log.debug(f'promoting root {b}') + state.promote_root(b) + log.debug('wallets: '+' '.join(k for k,_ in wallets.items())) + except CancelledError: + pass + finally: + if ws_provider.is_connected(): + await ws_provider.disconnect() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + log = logging.getLogger('dexorder') + log.setLevel(logging.DEBUG) + execute(main()) + log.info('exiting') diff --git a/src/dexorder/blockchain/__init__.py b/src/dexorder/blockchain/__init__.py new file mode 100644 index 0000000..7017c84 --- /dev/null +++ b/src/dexorder/blockchain/__init__.py @@ -0,0 +1,4 @@ +from .old_dispatch import OldDispatcher +from .by_blockchain import ByBlockchainDict, ByBlockchainList, ByBlockchainCollection +from .connection import connect +from dexorder.base.chain import Ethereum, Polygon, Goerli, Mumbai, ArbitrumOne, BSC diff --git a/src/dexorder/blockchain/by_blockchain.py b/src/dexorder/blockchain/by_blockchain.py new file mode 100644 index 0000000..11b8449 --- /dev/null +++ b/src/dexorder/blockchain/by_blockchain.py @@ -0,0 +1,55 @@ +from typing import Generic, TypeVar, Any, Iterator + +from dexorder import ctx, NARG + +_T = TypeVar('_T') + + +class ByBlockchainSingleton: + def __init__(self, by_blockchain:dict[int,Any]): + self.by_blockchain = by_blockchain + + def __getattr__(self, item): + return self.by_blockchain.__getattribute__(item) + + +class ByBlockchainCollection (Generic[_T]): + def __init__(self, by_blockchain:dict[int,dict[Any,_T]]=None): + self.by_blockchain = by_blockchain if by_blockchain is not None else {} + + def __getitem__(self, item) -> _T: + return self.by_blockchain[ctx.chain_id][item] + + +class ByBlockchainDict (ByBlockchainCollection[_T], Generic[_T]): + + def __getattr__(self, name: str) -> _T: + return self.by_blockchain[ctx.chain_id][name] + + def get(self, item, default=None, *, chain_id=None) -> _T: + # will raise if default is NARG + if chain_id is None: + chain_id = ctx.chain_id + if chain_id is None: + raise KeyError('no ctx.chain_id set') + found = self.by_blockchain.get(chain_id, {}).get(item, default) + if found is NARG: + raise KeyError + return found + + +class ByBlockchainList (ByBlockchainCollection[_T], Generic[_T]): + def __iter__(self) -> Iterator[_T]: + return iter(self.by_blockchain[ctx.chain_id]) + + def iter(self, *, chain_id=None) -> Iterator[_T]: + if chain_id is None: + chain_id = ctx.chain_id + return iter(self.by_blockchain[chain_id]) + + def get(self, index, *, chain_id=None) -> _T: + if chain_id is None: + chain_id = ctx.chain_id + if chain_id is None: + raise KeyError('no ctx.chain_id set') + return self.by_blockchain[chain_id][index] diff --git a/src/dexorder/blockchain/chain_singletons.py b/src/dexorder/blockchain/chain_singletons.py new file mode 100644 index 0000000..ffe1c28 --- /dev/null +++ b/src/dexorder/blockchain/chain_singletons.py @@ -0,0 +1,2 @@ +from dexorder import Blockchain + diff --git a/src/dexorder/blockchain/connection.py b/src/dexorder/blockchain/connection.py new file mode 100644 index 0000000..cf716b1 --- /dev/null +++ b/src/dexorder/blockchain/connection.py @@ -0,0 +1,50 @@ +from web3 import HTTPProvider, Web3 +from web3.middleware import geth_poa_middleware, simple_cache_middleware + +from dexorder import ctx +from dexorder.blockchain.util import get_contract_data +from ..configuration import resolve_rpc_url + + +def connect(rpc_url=None): + """ + connects to the rpc_url and configures the context + if you don't want to use ctx.account for this w3, either set ctx.account first or + use create_w3() and set w3.eth.default_account separately + """ + w3 = create_w3(rpc_url) + ctx.w3 = w3 + return w3 + + +def create_w3(rpc_url=None): + """ + this constructs a Web3 object but does NOT attach it to the context. consider using connect(...) instead + this does *not* attach any signer to the w3. make sure to inject the proper middleware with Account.attach(w3) + """ + # todo create a proxy w3 that rotates among rpc urls + # self.w3s = tuple(create_w3(url) for url in rpc_url_or_tag) + # chain_id = self.w3s[0].eth.chain_id + # assert all(w3.eth.chain_id == chain_id for w3 in self.w3s) # all rpc urls must be the same blockchain + # self.w3iter = itertools.cycle(self.w3s) + + url = resolve_rpc_url(rpc_url) + w3 = Web3(HTTPProvider(url)) + w3.middleware_onion.inject(geth_poa_middleware, layer=0) + w3.middleware_onion.add(simple_cache_middleware) + w3.eth.Contract = _make_contract(w3.eth) + return w3 + + +def _make_contract(w3_eth): + def f(address, abi_or_name): # if abi, then it must already be in native object format, not a string + if type(abi_or_name) is str: + data = get_contract_data(abi_or_name) + abi = data['abi'] + bytecode = data['bytecode']['object'] if address is None else None + else: + abi = abi_or_name + bytecode = None + return w3_eth.contract(address,abi=abi,bytecode=bytecode) + return f + diff --git a/src/dexorder/blockchain/util.py b/src/dexorder/blockchain/util.py new file mode 100644 index 0000000..94cf53f --- /dev/null +++ b/src/dexorder/blockchain/util.py @@ -0,0 +1,7 @@ +import json + + +def get_contract_data(name): + with open(f'contract/out/{name}.sol/{name}.json') as file: + return json.load(file) + diff --git a/src/dexorder/configuration/__init__.py b/src/dexorder/configuration/__init__.py new file mode 100644 index 0000000..a319d02 --- /dev/null +++ b/src/dexorder/configuration/__init__.py @@ -0,0 +1,3 @@ +from .standard_accounts import test_accounts +from .load import config, parse_args +from .resolve import resolve_rpc_url diff --git a/src/dexorder/configuration/load.py b/src/dexorder/configuration/load.py new file mode 100644 index 0000000..41e5117 --- /dev/null +++ b/src/dexorder/configuration/load.py @@ -0,0 +1,94 @@ +import os +import tomllib +from tomllib import TOMLDecodeError + +from omegaconf import OmegaConf, DictConfig +from omegaconf.errors import OmegaConfBaseException + +from .schema import Config +from .standard_accounts import default_accounts_config +from .standard_tokens import default_token_config + +schema = OmegaConf.structured(Config()) + + +class ConfigException (Exception): + pass + + +def load_config(): + +# noinspection PyTypeChecker + result:ConfigDict = OmegaConf.merge( + schema, + load_tokens(), + load_accounts(), + from_toml('pool.toml'), + from_toml('dexorder.toml'), + from_toml('config.toml'), + from_toml('.secret.toml'), + from_env() + ) + return result + + +def load_tokens(): + token_conf = OmegaConf.create({'tokens': default_token_config}) + try: + OmegaConf.merge(schema, token_conf) + return token_conf + except OmegaConfBaseException as _x: + raise ConfigException(f'Error while processing default tokens:\n{_x}') + + +def load_accounts(): + accounts_conf = OmegaConf.create({'accounts': default_accounts_config}) + try: + OmegaConf.merge(schema, accounts_conf) + return accounts_conf + except OmegaConfBaseException as _x: + raise ConfigException(f'Error while processing default accounts:\n{_x}') + + +def from_env(prefix='DEXORDER_'): + dotlist = [] + for key, value in os.environ.items(): + if key.startswith(prefix): + key = key[len(prefix):].lower().replace('__','.') + dotlist.append(key+'='+value) + result = OmegaConf.from_dotlist(dotlist) + try: + OmegaConf.merge(schema, result) + return result + except OmegaConfBaseException as x: + raise ConfigException(f'Error while parsing environment config:\n{x}') + + +def from_toml(filename): + try: + try: + with open(filename, 'rb') as file: + toml = tomllib.load(file) + result = OmegaConf.create(toml) + except FileNotFoundError: + return OmegaConf.create() + OmegaConf.merge(schema, result) + return result + except (OmegaConfBaseException, TOMLDecodeError) as x: + raise ConfigException(f'Error while loading {filename}:\n{x}') + + +def parse_args(args=None): + """ should be called from binaries to parse args as command-line config settings """ + # noinspection PyTypeChecker + try: + config.merge_with(OmegaConf.from_cli(args)) # updates config in-place. THANK YOU OmegaConf! + except OmegaConfBaseException as x: + raise ConfigException(f'Could not parse command-line args:\n{x}') + + +class ConfigDict (Config, DictConfig): # give type hints from Config plus methods from DictConfig + pass + + +config = load_config() diff --git a/src/dexorder/configuration/postprocess.py b/src/dexorder/configuration/postprocess.py new file mode 100644 index 0000000..e5170ac --- /dev/null +++ b/src/dexorder/configuration/postprocess.py @@ -0,0 +1,13 @@ +from typing import Callable +from .schema import Config + + +_handlers = [] + +def add_config_handler(callback:Callable[[Config],None]): + _handlers.append(callback) + +def postprocess_config(conf:Config): + for callback in _handlers: + callback(conf) + diff --git a/src/dexorder/configuration/resolve.py b/src/dexorder/configuration/resolve.py new file mode 100644 index 0000000..88b8ca2 --- /dev/null +++ b/src/dexorder/configuration/resolve.py @@ -0,0 +1,13 @@ +from .load import config + + +def resolve_rpc_url(rpc_url=None): + if rpc_url is None: + rpc_url = config.rpc_url + if rpc_url == 'test': + return 'http://localhost:8545' + try: + return config.rpc_urls[rpc_url] # look up aliases + except KeyError: + pass + return rpc_url diff --git a/src/dexorder/configuration/schema.py b/src/dexorder/configuration/schema.py new file mode 100644 index 0000000..7aee0a4 --- /dev/null +++ b/src/dexorder/configuration/schema.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass, field +from typing import Optional, Union + + +# SCHEMA NOTES: +# - avoid using int keys since (1) they are hard to decipher by a human and (2) the Python TOML parser mistypes int keys +# as strings in certain situations +# - do not nest structured types more than one level deep. it confuses the config's typing system + +@dataclass +class Config: + db_url: str = 'postgresql://dexorder:redroxed@localhost/dexorder' + dump_sql: bool = False + chain: Union[int,str] = 'Arbitrum' + + # rpc_url may also reference the aliases from the foundry.toml's rpc_endpoints section + rpc_url: str = 'http://localhost:8545' + ws_url: str = 'ws://localhost:8545' + rpc_urls: Optional[dict[str,str]] = field(default_factory=dict) + + account: Optional[str] = None # may be a private key or an account alias + accounts: Optional[dict[str,str]] = field(default_factory=dict) # account aliases + min_gas: str = '0' + + tokens: list['TokenConfig'] = field(default_factory=list) + dexorders: list['DexorderConfig'] = field(default_factory=list) + pools: list['PoolConfig'] = field(default_factory=list) + query_helpers: dict[str,str] = field(default_factory=dict) + + # Dispatcher + polling_interval: float = 0.2 + backoff_factor: float = 1.5 + max_interval: float = 10 + + # positive numbers are absolute block numbers and negative numbers are relative to the latest block + backfill: int = 0 + + +@dataclass +class TokenConfig: + name: str + symbol: str + decimals: int + chain: str + address: str + abi: Optional[str] = None + + +@dataclass +class PoolConfig: + chain: str + address: str + token_a: str + token_b: str + fee: int + enabled: bool = False + + +@dataclass +class DexorderConfig: + chain: str + address: str + pool: str + owner: str + name: Optional[str] = None + width: Optional[int] = None # in bps aka ticks + width_above: Optional[int] = None # defaults to width + width_below: Optional[int] = None # defaults to width + offset: Optional[int] = None # in bps aka ticks + offset_above: Optional[int] = None # defaults to offset + offset_below: Optional[int] = None # defaults to offset + ema: Optional[int] = None + diff --git a/src/dexorder/configuration/standard_accounts.py b/src/dexorder/configuration/standard_accounts.py new file mode 100644 index 0000000..29d4b65 --- /dev/null +++ b/src/dexorder/configuration/standard_accounts.py @@ -0,0 +1,18 @@ +test_accounts = { + 'test0': '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', + 'test1': '0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d', + 'test2': '0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a', + 'test3': '0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6', + 'test4': '0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a', + 'test5': '0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba', + 'test6': '0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e', + 'test7': '0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356', + 'test8': '0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97', + 'test9': '0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6', +} + +default_accounts_config = {} + +default_accounts_config.update(test_accounts) + +default_accounts_config['test'] = default_accounts_config['test0'] diff --git a/src/dexorder/configuration/standard_tokens.py b/src/dexorder/configuration/standard_tokens.py new file mode 100644 index 0000000..1d02e55 --- /dev/null +++ b/src/dexorder/configuration/standard_tokens.py @@ -0,0 +1,8 @@ +from .schema import TokenConfig + +default_token_config = [ + # TokenConfig('Wrapped Matic', 'WMATIC', 18, 'Polygon', '0x0d500B1d8E8eF31E21C99d1Db9A6444d3ADf1270', abi='WMATIC'), + # TokenConfig('Wrapped Ethereum','WETH', 18, 'Polygon', '0x7ceB23fD6bC0adD59E62ac25578270cFf1b9f619'), + # TokenConfig('Wrapped Bitcoin', 'WBTC', 8, 'Polygon', '0x1BFD67037B42Cf73acF2047067bd4F2C47D9BfD6'), + # TokenConfig('USD Coin', 'USDC', 6, 'Polygon', '0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174'), +] diff --git a/src/dexorder/db/__init__.py b/src/dexorder/db/__init__.py new file mode 100644 index 0000000..daac667 --- /dev/null +++ b/src/dexorder/db/__init__.py @@ -0,0 +1,25 @@ +import sqlalchemy + +from .migrate import migrate_database +from .. import config, ctx + + +# noinspection PyShadowingNames +def connect(url=None, migrate=True, reconnect=False, dump_sql=None): + if ctx.engine is not None and not reconnect: + return + if url is None: + url = config.db_url + if dump_sql is None: + dump_sql = config.dump_sql + engine = sqlalchemy.create_engine(url, echo=dump_sql) + if migrate: + migrate_database() + with engine.connect() as connection: + connection.execute(sqlalchemy.text("SET TIME ZONE 'UTC'")) + result = connection.execute(sqlalchemy.text("select version_num from alembic_version")) + for row in result: + print(f'database revision {row[0]}') + ctx.engine = engine + return + raise Exception('database version not found') diff --git a/src/dexorder/db/column.py b/src/dexorder/db/column.py new file mode 100644 index 0000000..0a05486 --- /dev/null +++ b/src/dexorder/db/column.py @@ -0,0 +1,87 @@ +from sqlalchemy import SMALLINT, INTEGER, BIGINT +from sqlalchemy.orm import mapped_column +from typing_extensions import Annotated + +from dexorder import Fixed2, Blockchain as NativeBlockchain +from . import column_types as t + +# noinspection DuplicatedCode +Uint8 = Annotated[int, mapped_column(SMALLINT)] +Uint16 = Annotated[int, mapped_column(SMALLINT)] +Uint24 = Annotated[int, mapped_column(INTEGER)] +Uint32 = Annotated[int, mapped_column(INTEGER)] +Uint40 = Annotated[int, mapped_column(BIGINT)] +Uint48 = Annotated[int, mapped_column(BIGINT)] +Uint56 = Annotated[int, mapped_column(BIGINT)] +Uint64 = Annotated[int, mapped_column(t.IntBits(64, False))] # cannot use BIGINT since an unsigned value could overflow it +Uint72 = Annotated[int, mapped_column(t.IntBits(72, False))] +Uint80 = Annotated[int, mapped_column(t.IntBits(80, False))] +Uint88 = Annotated[int, mapped_column(t.IntBits(88, False))] +Uint96 = Annotated[int, mapped_column(t.IntBits(96, False))] +Uint104 = Annotated[int, mapped_column(t.IntBits(104, False))] +Uint112 = Annotated[int, mapped_column(t.IntBits(112, False))] +Uint120 = Annotated[int, mapped_column(t.IntBits(120, False))] +Uint128 = Annotated[int, mapped_column(t.IntBits(128, False))] +Uint136 = Annotated[int, mapped_column(t.IntBits(136, False))] +Uint144 = Annotated[int, mapped_column(t.IntBits(144, False))] +Uint152 = Annotated[int, mapped_column(t.IntBits(152, False))] +Uint160 = Annotated[int, mapped_column(t.IntBits(160, False))] +Uint168 = Annotated[int, mapped_column(t.IntBits(168, False))] +Uint176 = Annotated[int, mapped_column(t.IntBits(176, False))] +Uint184 = Annotated[int, mapped_column(t.IntBits(184, False))] +Uint192 = Annotated[int, mapped_column(t.IntBits(192, False))] +Uint200 = Annotated[int, mapped_column(t.IntBits(200, False))] +Uint208 = Annotated[int, mapped_column(t.IntBits(208, False))] +Uint216 = Annotated[int, mapped_column(t.IntBits(216, False))] +Uint224 = Annotated[int, mapped_column(t.IntBits(224, False))] +Uint232 = Annotated[int, mapped_column(t.IntBits(232, False))] +Uint240 = Annotated[int, mapped_column(t.IntBits(240, False))] +Uint248 = Annotated[int, mapped_column(t.IntBits(248, False))] +Uint256 = Annotated[int, mapped_column(t.IntBits(256, False))] + +# noinspection DuplicatedCode +Int8 = Annotated[int, mapped_column(SMALLINT)] +Int16 = Annotated[int, mapped_column(SMALLINT)] +Int24 = Annotated[int, mapped_column(INTEGER)] +Int32 = Annotated[int, mapped_column(INTEGER)] +Int40 = Annotated[int, mapped_column(BIGINT)] +Int48 = Annotated[int, mapped_column(BIGINT)] +Int56 = Annotated[int, mapped_column(BIGINT)] +Int64 = Annotated[int, mapped_column(BIGINT)] +Int72 = Annotated[int, mapped_column(t.IntBits(72, True))] +Int80 = Annotated[int, mapped_column(t.IntBits(80, True))] +Int88 = Annotated[int, mapped_column(t.IntBits(88, True))] +Int96 = Annotated[int, mapped_column(t.IntBits(96, True))] +Int104 = Annotated[int, mapped_column(t.IntBits(104, True))] +Int112 = Annotated[int, mapped_column(t.IntBits(112, True))] +Int120 = Annotated[int, mapped_column(t.IntBits(120, True))] +Int128 = Annotated[int, mapped_column(t.IntBits(128, True))] +Int136 = Annotated[int, mapped_column(t.IntBits(136, True))] +Int144 = Annotated[int, mapped_column(t.IntBits(144, True))] +Int152 = Annotated[int, mapped_column(t.IntBits(152, True))] +Int160 = Annotated[int, mapped_column(t.IntBits(160, True))] +Int168 = Annotated[int, mapped_column(t.IntBits(168, True))] +Int176 = Annotated[int, mapped_column(t.IntBits(176, True))] +Int184 = Annotated[int, mapped_column(t.IntBits(184, True))] +Int192 = Annotated[int, mapped_column(t.IntBits(192, True))] +Int200 = Annotated[int, mapped_column(t.IntBits(200, True))] +Int208 = Annotated[int, mapped_column(t.IntBits(208, True))] +Int216 = Annotated[int, mapped_column(t.IntBits(216, True))] +Int224 = Annotated[int, mapped_column(t.IntBits(224, True))] +Int232 = Annotated[int, mapped_column(t.IntBits(232, True))] +Int240 = Annotated[int, mapped_column(t.IntBits(240, True))] +Int248 = Annotated[int, mapped_column(t.IntBits(248, True))] +Int256 = Annotated[int, mapped_column(t.IntBits(256, True))] + +Address = Annotated[str, mapped_column(t.Address())] + +BlockCol = Annotated[int, mapped_column(BIGINT)] + +Blockchain = Annotated[NativeBlockchain, mapped_column(t.Blockchain)] + +# Uniswap aliases +Tick = Int24 +SqrtPriceX96 = Uint160 +Liquidity = Uint128 +Q128X96 = Annotated[Fixed2, mapped_column(t.Fixed(128, 96))] +Q256X128 = Annotated[Fixed2, mapped_column(t.Fixed(256, 128))] diff --git a/src/dexorder/db/column_types.py b/src/dexorder/db/column_types.py new file mode 100644 index 0000000..c3fdbb4 --- /dev/null +++ b/src/dexorder/db/column_types.py @@ -0,0 +1,71 @@ +import math + +from sqlalchemy import TypeDecorator, BIGINT +from sqlalchemy.dialects.postgresql import BYTEA +from web3 import Web3 + +from dexorder import Fixed2 as NativeFixed, Blockchain as NativeBlockchain + + +class Address(TypeDecorator): + impl = BYTEA + cache_ok = True + + def process_bind_param(self, value, dialect): + return bytes.fromhex(value[2:] if value.startswith('0x') else value) + + def process_result_value(self, value: bytes, dialect): + return Web3.to_checksum_address(value.hex()) + + +class Blockchain(TypeDecorator): + impl = BIGINT + cache_ok = True + + def process_bind_param(self, value: NativeBlockchain, dialect): + return value.chain_id + + def process_result_value(self, value: int, dialect): + return Blockchain.for_id(value) + + + +# Alembic instantiates these custom column types by calling their __init__ function as if they were the underlying +# type. Therefore, we cannot change the init function signature, so we use wrapper methods to set member variables +# after instance creation. This way, Alembic calls __init__ with the params for the underlying type, but at runtime +# when we construct our custom column types, they have the extra members set by the wrapper function. + +class _IntBits(TypeDecorator): + impl = BYTEA + cache_ok = True + + def process_bind_param(self, value, dialect): + assert( not self.signed or value >= 0) + return int(value).to_bytes(self.length, 'big', signed=self.signed) + + def process_result_value(self, value, dialect): + return int.from_bytes(value, 'big', signed=self.signed) + + +def IntBits(bits, signed): + result = _IntBits(math.ceil(bits/8)) + result.bits = bits + result.signed = signed + return result + +class _Fixed(TypeDecorator): + impl = BYTEA + cache_ok = True + + def process_bind_param(self, value, dialect): + assert( not self.signed or value >= 0) + return int(value).to_bytes(self.length, 'big', signed=self.signed) + def process_result_value(self, value, dialect): + return NativeFixed(int.from_bytes(value, 'big', signed=self.signed), self.dbits) + +def Fixed(bits, dbits, signed=False): + result = _Fixed(math.ceil(bits/8)) + result.bits = bits + result.dbits = dbits + result.signed = signed + return result diff --git a/src/dexorder/db/migrate.py b/src/dexorder/db/migrate.py new file mode 100644 index 0000000..f434fb5 --- /dev/null +++ b/src/dexorder/db/migrate.py @@ -0,0 +1,10 @@ +import subprocess +import sys + + +def migrate_database(): + completed = subprocess.run('alembic upgrade head', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if completed.returncode != 0: + print(completed.stdout.decode(), file=sys.stderr) + print('FATAL: database migration failed!', file=sys.stderr) + exit(1) diff --git a/src/dexorder/db/model/__init__.py b/src/dexorder/db/model/__init__.py new file mode 100644 index 0000000..ffeb25f --- /dev/null +++ b/src/dexorder/db/model/__init__.py @@ -0,0 +1,2 @@ +from .base import Base +from .block import Block diff --git a/src/dexorder/db/model/base.py b/src/dexorder/db/model/base.py new file mode 100644 index 0000000..be80cb9 --- /dev/null +++ b/src/dexorder/db/model/base.py @@ -0,0 +1,16 @@ +from sqlalchemy.orm import DeclarativeBase, declared_attr + +from dexorder import ctx + + +# add Base as the -last- class inherited on classes which should get tables +class Base(DeclarativeBase): + # noinspection PyMethodParameters + @declared_attr.directive + def __tablename__(cls) -> str: + return cls.__name__.lower() + + @classmethod + def get(cls, **kwargs): + return ctx.session.get(cls, kwargs) + diff --git a/src/dexorder/db/model/block.py b/src/dexorder/db/model/block.py new file mode 100644 index 0000000..aa0cc92 --- /dev/null +++ b/src/dexorder/db/model/block.py @@ -0,0 +1,37 @@ +from contextvars import ContextVar + +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from dexorder.db.model import Base + + +class Block(Base): + chain: Mapped[int] = mapped_column(primary_key=True) + height: Mapped[int] = mapped_column(primary_key=True) # timescaledb index + hash: Mapped[bytes] = mapped_column(primary_key=True) + parent: Mapped[bytes] + data: Mapped[dict] = mapped_column(JSONB) + + def __str__(self): + return f'{self.height}_{self.hash.hex()}' + + @staticmethod + def cur() -> 'Block': + return _cur.get() + + @staticmethod + def set_cur(value: 'Block'): + _cur.set(value) + + @staticmethod + def latest() -> 'Block': + return _latest.get() + + @staticmethod + def set_latest(value: 'Block'): + _latest.set(value) + + +_cur = ContextVar[Block]('Block.cur') +_latest = ContextVar[Block]('Block.latest') diff --git a/src/dexorder/util/__init__.py b/src/dexorder/util/__init__.py new file mode 100644 index 0000000..6fb21ce --- /dev/null +++ b/src/dexorder/util/__init__.py @@ -0,0 +1,14 @@ +import re + +from .async_yield import async_yield +from .tick_math import nearest_available_ticks, round_tick, spans_tick, spans_range + + +def align_decimal(value, left_columns) -> str: + """ + returns a string where the decimal point in value is aligned to have left_columns of characters before it + """ + s = str(value) + pad = max(left_columns - len(re.sub(r'[^0-9]*$','',s.split('.')[0])), 0) + return ' ' * pad + s + diff --git a/src/dexorder/util/async_yield.py b/src/dexorder/util/async_yield.py new file mode 100644 index 0000000..9bbe0ae --- /dev/null +++ b/src/dexorder/util/async_yield.py @@ -0,0 +1,5 @@ +import asyncio + +async def async_yield(): + # a value of exactly 0 doesn't seem to work as well, so we set 1 nanosecond + await asyncio.sleep(1e-9) diff --git a/src/dexorder/util/convert.py b/src/dexorder/util/convert.py new file mode 100644 index 0000000..bfae166 --- /dev/null +++ b/src/dexorder/util/convert.py @@ -0,0 +1,20 @@ +import math + + +def price_to_tick(p): + return round(math.log(p)/math.log(1.0001)) + +def tick_to_price(t): + return math.pow(1.0001,t) + +def tick_to_sqrt_price(t): + return math.pow(1.0001**.5,t) + +def to_fixed(value, decimals): + return round(value * 10**decimals) + +def from_fixed(value, decimals): + return value / 10**decimals + +def tick_to_sqrtPriceX96(tick): + return round(math.sqrt(tick_to_price(tick)) * 2**96) diff --git a/src/dexorder/util/cwd.py b/src/dexorder/util/cwd.py new file mode 100644 index 0000000..8ad240b --- /dev/null +++ b/src/dexorder/util/cwd.py @@ -0,0 +1,9 @@ +import os + + +def _cwd(): + while 'alembic.ini' not in os.listdir(): + if os.getcwd() == '/': + print('FATAL: could not find project root directory') + exit(1) + os.chdir(os.path.normpath('..')) diff --git a/src/dexorder/util/json.py b/src/dexorder/util/json.py new file mode 100644 index 0000000..4eb239f --- /dev/null +++ b/src/dexorder/util/json.py @@ -0,0 +1,16 @@ +from hexbytes import HexBytes +from orjson import orjson +from web3.datastructures import ReadableAttributeDict + + +def _serialize(v): + # todo wrap json.dumps() + if isinstance(v,HexBytes): + return v.hex() + if isinstance(v,ReadableAttributeDict): + return v.__dict__ + raise ValueError(v) + + +def dumps(obj): + return orjson.dumps(obj, default=_serialize) diff --git a/src/dexorder/util/shutdown.py b/src/dexorder/util/shutdown.py new file mode 100644 index 0000000..c5de24e --- /dev/null +++ b/src/dexorder/util/shutdown.py @@ -0,0 +1,9 @@ +import logging + +log = logging.getLogger('dexorder') + +def fatal(message, exception=None): + if exception is None and isinstance(message, (BaseException,RuntimeError)): + exception = message + log.exception(message, exc_info=exception) + exit(1) diff --git a/src/dexorder/util/sql.py b/src/dexorder/util/sql.py new file mode 100644 index 0000000..62a6581 --- /dev/null +++ b/src/dexorder/util/sql.py @@ -0,0 +1,14 @@ +from datetime import datetime, timedelta +from typing import Union + + +def where_time_range(sql, time_column, start: Union[datetime,timedelta,None] = None, end: Union[datetime,timedelta,None] = None): + if start is not None: + if isinstance(start, timedelta): + start = datetime.now() - abs(start) + sql = sql.where(time_column >= start) + if end is not None: + if isinstance(end, timedelta): + end = datetime.now() - abs(end) + sql = sql.where(time_column < end) + return sql diff --git a/src/dexorder/util/tick_math.py b/src/dexorder/util/tick_math.py new file mode 100644 index 0000000..b0516d2 --- /dev/null +++ b/src/dexorder/util/tick_math.py @@ -0,0 +1,24 @@ + +def round_tick(tick, tick_spacing): + """ + returns the nearest available tick + """ + return round(tick/tick_spacing) * tick_spacing + + +def nearest_available_ticks(tick, tick_spacing): + """ + returns the two available ticks just below and above the given tick + """ + lower = tick // tick_spacing * tick_spacing + upper = lower + tick_spacing + return lower, upper + + +def spans_tick(tick, lower, upper): + return spans_range( *nearest_available_ticks(tick), lower, upper) + + +def spans_range(below, above, lower, upper): + return lower < above and upper > below + diff --git a/test/test_blockstate.py b/test/test_blockstate.py new file mode 100644 index 0000000..cac00be --- /dev/null +++ b/test/test_blockstate.py @@ -0,0 +1,62 @@ +from dexorder.base.blockstate import BlockState, BlockDict +from dexorder.db.model.block import Block + +block_10 = Block(chain=1, height=10, hash=bytes.fromhex('10'), parent=bytes.fromhex('09'), data=None) +block_11a = Block(chain=1, height=11, hash=bytes.fromhex('1a'), parent=block_10.hash, data=None) +block_11b = Block(chain=1, height=11, hash=bytes.fromhex('1b'), parent=block_10.hash, data=None) +block_12a = Block(chain=1, height=12, hash=bytes.fromhex('12'), parent=block_11a.hash, data=None) +state = BlockState(block_10, {'series':{'foo':'bar'}}) +BlockState.set_cur(state) +d = BlockDict('series') + +def start_block(b): + Block.set_cur(b) + state.add_block(b) + +start_block(block_11a) +del d['foo'] +d['foue'] = 'barre' + +start_block(block_12a) +d['foo'] = 'bar2' + +start_block(block_11b) +d['fu'] = 'ku' + +def print_dict(x:dict=d): + for k, v in x.items(): + print(f'{k:>10} : {v}') + +for block in [block_10,block_11a,block_12a,block_11b]: + Block.set_cur(block) + print() + print(Block.cur().hash) + print_dict() + +def test11b(): + Block.set_cur(block_11b) + assert 'fu' in d + assert d['fu'] == 'ku' + assert 'foo' in d + assert d['foo'] == 'bar' + +def test12a(): + Block.set_cur(block_12a) + assert 'fu' not in d + assert 'foo' in d + assert d['foo'] == 'bar2' + assert 'foue' in d + assert d['foue'] == 'barre' + +test11b() +test12a() +state.promote_root(block_11a) +print() +print('promoted root') +print_dict(state.root_state) +test12a() +state.promote_root(block_12a) +print() +print('promoted root') +print_dict(state.root_state) +test12a()