From bb2d84a1e7e938749a93726570c6a89e26ebaeb0 Mon Sep 17 00:00:00 2001 From: Tim Date: Mon, 17 Jun 2024 02:39:12 -0400 Subject: [PATCH] vault upgrades; fees; refactoring --- dexorder-alpha.toml | 25 ------------ dexorder-separb.toml | 31 ++++++++++++++ src/dexorder/base/chain.py | 4 +- src/dexorder/base/orderlib.py | 67 ++++++++++++++++--------------- src/dexorder/bin/main.py | 12 ++---- src/dexorder/contract/__init__.py | 14 +++++-- src/dexorder/contract/dexorder.py | 6 +-- src/dexorder/event_handler.py | 34 +++++++++++++--- src/dexorder/logics.py | 17 ++++++++ src/dexorder/progressor.py | 4 +- src/dexorder/uniswap.py | 4 +- src/dexorder/vault_blockdata.py | 1 + test/test_blockstate.py | 4 +- 13 files changed, 136 insertions(+), 87 deletions(-) delete mode 100644 dexorder-alpha.toml create mode 100644 dexorder-separb.toml create mode 100644 src/dexorder/logics.py diff --git a/dexorder-alpha.toml b/dexorder-alpha.toml deleted file mode 100644 index 3367b57..0000000 --- a/dexorder-alpha.toml +++ /dev/null @@ -1,25 +0,0 @@ -account='test1' # Dev Account #1 -rpc_url='http://alpharpc:8545' -ws_url='ws://alpharpc:8545' -db_url='postgresql://dexorder:redroxed@postgres/dexorder' -redis_url='redis://redis:6379' - -mirror_pools=[ - '0xC6962004f452bE9203591991D15f6b388e09E8D0', # WETH/USDC 0.05% - '0x2f5e87C9312fa29aed5c179E456625D79015299c', # WBTC/WETH 0.05% - '0x0d94947374cbc779a0FB4D1bfF795C0Af6Dfae25', # USDC/UNI 1.00% - '0x689C96ceAb93f5E131631D225D75DeA3fD37747E', # WBTC/ARB 0.30% - '0x0E4831319A50228B9e450861297aB92dee15B44F', # WBTC/USDC 0.05% - '0x2038eEAa7100E08739352a37Ed67852E8529E8ED', # ARB/UNI 1.00% - '0x468b88941e7Cc0B88c1869d68ab6b570bCEF62Ff', # WETH/LINK 0.30% - '0xC24f7d8E51A64dc1238880BD00bb961D54cbeb29', # WETH/UNI 0.30% - '0xbBe36e6f0331C6a36AB44Bc8421E28E1a1871C1e', # USDC/LINK 0.30% - '0xa79fD76cA2b24631Ec3151f10c0660a30Bc946E7', # WBTC/LINK 0.30% - '0xb0f6cA40411360c03d41C5fFc5F179b8403CdcF8', # ARB/USDC 0.05% - '0xC6F780497A95e246EB9449f5e4770916DCd6396A', # WETH/ARB 0.05% - '0x8b6149aF984140BD3F8e158CcDCD05984a4ad0f5', # ARB/LINK 0.30% - '0xEd701Ba0cec723d85B7d96c80C21148E49D2Bf05', # LINK/UNI 1.00% -] - -[deployments] -1337='alpha' diff --git a/dexorder-separb.toml b/dexorder-separb.toml new file mode 100644 index 0000000..c166865 --- /dev/null +++ b/dexorder-separb.toml @@ -0,0 +1,31 @@ +account = 'test1' # Dev Account #1 +rpc_url = 'http://alpharpc:8545' +ws_url = 'ws://alpharpc:8545' +db_url = 'postgresql://dexorder:redroxed@postgres/dexorder' +redis_url = 'redis://redis:6379' + +mirror_pools = [ + # Arbitrum Pools + '0xC6962004f452bE9203591991D15f6b388e09E8D0', # WETH/USDC 0.05% + # '0x2f5e87C9312fa29aed5c179E456625D79015299c', # WBTC/WETH 0.05% + # '0x0d94947374cbc779a0FB4D1bfF795C0Af6Dfae25', # USDC/UNI 1.00% + # '0x689C96ceAb93f5E131631D225D75DeA3fD37747E', # WBTC/ARB 0.30% + # '0x0E4831319A50228B9e450861297aB92dee15B44F', # WBTC/USDC 0.05% + # '0x2038eEAa7100E08739352a37Ed67852E8529E8ED', # ARB/UNI 1.00% + # '0x468b88941e7Cc0B88c1869d68ab6b570bCEF62Ff', # WETH/LINK 0.30% + # '0xC24f7d8E51A64dc1238880BD00bb961D54cbeb29', # WETH/UNI 0.30% + # '0xbBe36e6f0331C6a36AB44Bc8421E28E1a1871C1e', # USDC/LINK 0.30% + # '0xa79fD76cA2b24631Ec3151f10c0660a30Bc946E7', # WBTC/LINK 0.30% + # '0xb0f6cA40411360c03d41C5fFc5F179b8403CdcF8', # ARB/USDC 0.05% + # '0xC6F780497A95e246EB9449f5e4770916DCd6396A', # WETH/ARB 0.05% + # '0x8b6149aF984140BD3F8e158CcDCD05984a4ad0f5', # ARB/LINK 0.30% + # '0xEd701Ba0cec723d85B7d96c80C21148E49D2Bf05', # LINK/UNI 1.00% + + # Base Pools + # '0xd0b53D9277642d899DF5C87A3966A349A798F224', # WETH/USDC + # '0xb4CB800910B228ED3d0834cF79D697127BBB00e5', # WETH/USDC + +] + +[deployments] +1337 = 'alpha' diff --git a/src/dexorder/base/chain.py b/src/dexorder/base/chain.py index ee2948d..0ae84ff 100644 --- a/src/dexorder/base/chain.py +++ b/src/dexorder/base/chain.py @@ -49,10 +49,10 @@ Polygon = Blockchain(137, 'Polygon') # POS not zkEVM Mumbai = Blockchain(80001, 'Mumbai') BSC = Blockchain(56, 'BSC') Arbitrum = Blockchain(42161, 'Arbitrum', 3, batch_size=2000) -Mock = Blockchain(31337, 'Mock', 3, batch_size=2000) +Mockchain = Blockchain(31337, 'Mockchain', 3, batch_size=2000) Alpha = Blockchain(1337, 'Dexorder Alpha', 3, batch_size=1000) -current_chain = ContextVar[Blockchain]('current_chain', default=Mock) +current_chain = ContextVar[Blockchain]('current_chain', default=Mockchain) class Clock: diff --git a/src/dexorder/base/orderlib.py b/src/dexorder/base/orderlib.py index 41ae63f..5842a5f 100644 --- a/src/dexorder/base/orderlib.py +++ b/src/dexorder/base/orderlib.py @@ -54,7 +54,7 @@ class SwapOrder: minFillAmount: int amountIsInput: bool outputDirectlyToOwner: bool - chainOrder: int + conditionalOrder: int tranches: list['Tranche'] @staticmethod @@ -64,7 +64,7 @@ class SwapOrder: def dump(self): return (self.tokenIn, self.tokenOut, self.route.dump(), str(self.amount), str(self.minFillAmount), self.amountIsInput, - self.outputDirectlyToOwner, self.chainOrder, [t.dump() for t in self.tranches]) + self.outputDirectlyToOwner, self.conditionalOrder, [t.dump() for t in self.tranches]) def __str__(self): msg = f''' @@ -83,7 +83,7 @@ SwapOrder @dataclass class SwapStatus: # this is an elaborated version of the on-chain status - fillFeeBP: int + fillFeeHalfBps: int state: SwapOrderState start: int ocoGroup: Optional[int] @@ -91,6 +91,7 @@ class SwapStatus: filledOut: Optional[int] # if None then look in the order_filled blockstate trancheFilledIn: Optional[list[int]] # if None then look in the tranche_filled blockstate trancheFilledOut: Optional[list[int]] # if None then look in the tranche_filled blockstate + trancheActivationTime: list[int] @dataclass @@ -105,7 +106,7 @@ class SwapOrderStatus(SwapStatus): @staticmethod def load(obj): order = SwapOrder.load(obj[0]) - fillFeeBP = int(obj[1]) + fillFeeHalfBps = int(obj[1]) state = SwapOrderState(obj[2]) start = obj[3] ocoGroup = None if obj[4] == NO_OCO else obj[4] @@ -113,20 +114,23 @@ class SwapOrderStatus(SwapStatus): filledOut = int(obj[6]) trancheFilledIn = [int(f) for f in obj[7]] trancheFilledOut = [int(f) for f in obj[8]] - return SwapOrderStatus(order, fillFeeBP, state, start, ocoGroup, filledIn, filledOut, trancheFilledIn, trancheFilledOut) + trancheActivationTime = [int(f) for f in obj[9]] + return SwapOrderStatus(order, fillFeeHalfBps, state, start, ocoGroup, + filledIn, filledOut, trancheFilledIn, trancheFilledOut, trancheActivationTime) @staticmethod def load_from_chain(obj): # 0 SwapOrder order; - # 1 int fillFeeBP + # 1 int fillFeeHalfBps # 2 bool canceled; # 3 uint32 start; # 4 uint64 ocoGroup; # 5 uint256 filled; // total # 6 uint256[] trancheFilled; // sum(trancheFilled) == filled + # 7 uint32[] trancheActivationTime; order = SwapOrder.load(obj[0]) - fillFeeBP = obj[1] + fillFeeHalfBps = obj[1] state = SwapOrderState.Canceled if obj[2] else SwapOrderState.Open start = obj[3] ocoGroup = None if obj[4] == NO_OCO else obj[4] @@ -135,12 +139,17 @@ class SwapOrderStatus(SwapStatus): filledOut = 0 trancheFilledIn = [0 for _ in range(len(obj[6]))] trancheFilledOut = [0 for _ in range(len(obj[6]))] - return SwapOrderStatus(order, fillFeeBP, state, start, ocoGroup, filledIn, filledOut, trancheFilledIn, trancheFilledOut) + trancheActivationTime = [int(i) for i in obj[7]] + return SwapOrderStatus(order, fillFeeHalfBps, state, start, ocoGroup, + filledIn, filledOut, trancheFilledIn, trancheFilledOut, trancheActivationTime) def dump(self): - return (self.order.dump(), self.fillFeeBP, self.state.value, self.start, self.ocoGroup, - str(self.filledIn), str(self.filledOut), - [str(f) for f in self.trancheFilledIn], [str(f) for f in self.trancheFilledOut]) + return ( + self.order.dump(), self.fillFeeHalfBps, self.state.value, self.start, self.ocoGroup, + str(self.filledIn), str(self.filledOut), + [str(f) for f in self.trancheFilledIn], [str(f) for f in self.trancheFilledOut], + [int(i) for i in self.trancheActivationTime] + ) def copy(self): return copy.deepcopy(self) @@ -168,11 +177,11 @@ class Tranche: maxIsBarrier: bool marketOrder: bool - _reserved5: bool - _reserved6: bool + minIsRatio: bool + maxIsRatio: bool _reserved7: bool - _reserved8: int - _reserved16: int + rateLimitFraction: int + rateLimitPeriod: int startTime: int endTime: int @@ -196,11 +205,11 @@ class Tranche: obj[3], # minIsBarrier obj[4], # maxIsBarrier obj[5], # marketOrder - obj[6], # reserved - obj[7], # reserved - obj[8], # reserved - obj[9], # reserved - obj[10], # reserved + obj[6], # minIsRatio + obj[7], # maxIsRatio + obj[8], # _reserved7 + obj[9], # rateLimitFraction + obj[10], # rateLimitPeriod obj[11], # startTime obj[12], # endTime decode_IEEE754(obj[13]), # minIntercept @@ -208,10 +217,6 @@ class Tranche: decode_IEEE754(obj[15]), # maxIntercept decode_IEEE754(obj[16]), # maxSlope ) - result._origMinIntercept = obj[13] - result._origMinSlope = obj[14] - result._origMaxIntercept = obj[15] - result._origMaxSlope = obj[16] return result def dump(self): @@ -219,17 +224,11 @@ class Tranche: minM = encode_IEEE754(self.minSlope) maxB = encode_IEEE754(self.maxIntercept) maxM = encode_IEEE754(self.maxSlope) - if hasattr(self, '_origMinIntercept'): - assert minB == self._origMinIntercept - # noinspection PyUnresolvedReferences - assert minM == self._origMinSlope - # noinspection PyUnresolvedReferences - assert maxB == self._origMaxIntercept - # noinspection PyUnresolvedReferences - assert maxM == self._origMaxSlope return ( self.fraction, self.startTimeIsRelative, self.endTimeIsRelative, self.minIsBarrier, self.maxIsBarrier, self.marketOrder, - False, False, False, 0, 0, # reserved + self.minIsRatio, self.maxIsRatio, + False, # _reserved7 + self.rateLimitFraction, self.rateLimitPeriod, self.startTime, self.endTime, minB, minM, maxB, maxM, ) @@ -246,6 +245,8 @@ class Tranche: msg += f' <{self.maxIntercept:.5g}' if self.maxSlope: msg += f'{self.maxSlope:+.5g}' + if self.rateLimitPeriod: + msg += f' {self.rateLimitFraction/MAX_FRACTION:.1%} every {self.rateLimitPeriod/60:.0} minutes' return msg @dataclass diff --git a/src/dexorder/bin/main.py b/src/dexorder/bin/main.py index 68a5222..efa8eb6 100644 --- a/src/dexorder/bin/main.py +++ b/src/dexorder/bin/main.py @@ -13,7 +13,7 @@ from dexorder.contract.dexorder import get_factory_contract, get_dexorder_contra from dexorder.event_handler import init, dump_log, handle_vault_created, handle_order_placed, \ handle_transfer, handle_swap_filled, handle_order_canceled, handle_order_cancel_all, handle_dexorderexecutions, \ activate_time_triggers, activate_price_triggers, \ - process_active_tranches, process_execution_requests, check_ohlc_rollover, handle_uniswap_swaps + process_active_tranches, process_execution_requests, check_ohlc_rollover, handle_uniswap_swaps, handle_vault_logic_changed from dexorder.memcache import memcache from dexorder.memcache.memcache_state import RedisState, publish_all from dexorder.order.triggers import activate_orders @@ -37,13 +37,6 @@ def setup_logevent_triggers(runner): log.debug('all events:') runner.add_event_trigger(dump_log, None, {}) - factory = get_factory_contract() - if factory is None: - log.warning(f'No Factory for {current_chain.get()}') - vault_created = get_contract_event('Factory', 'VaultCreated') - else: - vault_created = factory.events.VaultCreated() - dexorder = get_dexorder_contract() if dexorder is None: log.warning(f'No Dexorder for {current_chain.get()}') @@ -52,7 +45,8 @@ def setup_logevent_triggers(runner): executions = dexorder.events.DexorderExecutions() runner.add_event_trigger(init) - runner.add_event_trigger(handle_vault_created, vault_created) + runner.add_event_trigger(handle_vault_created, get_contract_event('Vault', 'VaultCreated')) + runner.add_event_trigger(handle_vault_logic_changed, get_contract_event('Vault', 'VaultLogicChanged')) runner.add_event_trigger(handle_order_placed, get_contract_event('OrderLib', 'DexorderSwapPlaced')) runner.add_event_trigger(handle_transfer, get_contract_event('ERC20', 'Transfer')) runner.add_event_trigger(handle_uniswap_swaps, get_contract_event('IUniswapV3PoolEvents', 'Swap'), multi=True) diff --git a/src/dexorder/contract/__init__.py b/src/dexorder/contract/__init__.py index 6c36422..3c96274 100644 --- a/src/dexorder/contract/__init__.py +++ b/src/dexorder/contract/__init__.py @@ -12,13 +12,19 @@ from ..base.chain import current_chain CONTRACT_ERRORS = (InsufficientDataBytes, ContractLogicError, BadFunctionCallOutput) -def get_contract_data(name): - if name in abis: +def get_abi(name, filename=None): + return get_contract_data(name, filename)['abi'] + + +def get_contract_data(name, filename=None): + if filename is None and name in abis: return {'abi':abis[name]} - if name == "Vault" and os.path.exists(f'../contract/out/I{name}.sol/I{name}.json') : + if filename is None and name == "Vault" and os.path.exists(f'../contract/out/IVault.sol/IVault.json') : # logging.debug("getting abi from IVault.json instead of Vault.json") name = "IVault" # Special case for proxy Vault - with open(f'../contract/out/{name}.sol/{name}.json', 'rt') as file: + if filename is None: + filename = name + with open(f'../contract/out/{filename}.sol/{name}.json', 'rt') as file: return json.load(file) diff --git a/src/dexorder/contract/dexorder.py b/src/dexorder/contract/dexorder.py index 1d099c0..d20ef48 100644 --- a/src/dexorder/contract/dexorder.py +++ b/src/dexorder/contract/dexorder.py @@ -21,10 +21,10 @@ def _load_chain(chain_id: int): with open(f'../contract/broadcast/Deploy.sol/{chain_id}/run-latest.json', 'rt') as file: deployment = json.load(file) for tx in deployment['transactions']: - if tx['contractName'] == 'Factory': + if tx['contractName'] == 'VaultFactory': addr = tx['contractAddress'] - _factory[chain_id] = ContractProxy(addr, 'Factory') - log.info(f'Factory {addr}') + _factory[chain_id] = ContractProxy(addr, 'VaultFactory') + log.info(f'VaultFactory {addr}') elif tx['contractName'] == 'Dexorder': addr = tx['contractAddress'] _dexorder[chain_id] = DexorderContract(addr) diff --git a/src/dexorder/event_handler.py b/src/dexorder/event_handler.py index 7fe9641..1dd224d 100644 --- a/src/dexorder/event_handler.py +++ b/src/dexorder/event_handler.py @@ -3,18 +3,20 @@ import itertools import logging from uuid import UUID +from web3.exceptions import LogTopicError, MismatchedABI from web3.types import EventData from dexorder import current_pub, db, from_timestamp, minutely -from dexorder.base.chain import current_chain, current_clock +from dexorder.base.chain import current_chain, current_clock, Mockchain from dexorder.base.order import TrancheExecutionRequest, TrancheKey, ExecutionRequest, new_tranche_execution_request, \ OrderKey from dexorder.base.orderlib import SwapOrderState from dexorder.blocks import get_block_timestamp from dexorder.blockstate.fork import current_fork -from dexorder.contract import ERC20 -from dexorder.contract.dexorder import vault_address, VaultContract +from dexorder.contract import ERC20, ContractProxy, get_contract_event +from dexorder.contract.dexorder import vault_address, VaultContract, get_factory_contract from dexorder.database.model.transaction import TransactionJob +from dexorder.logics import logics, get_logic_version from dexorder.ohlc import ohlcs, recent_ohlcs from dexorder.order.orderstate import Order from dexorder.order.triggers import OrderTriggers, price_triggers, time_triggers, \ @@ -142,7 +144,6 @@ async def handle_uniswap_swaps(swaps: list[EventData]): async def handle_uniswap_swap(swap: EventData): - # todo gather prices first then apply only the value at the end of the block data = await get_uniswap_data(swap) if data is None: return @@ -157,10 +158,14 @@ async def handle_vault_created(created: EventData): try: owner = created['args']['owner'] num = created['args']['num'] + addr = created['address'] except KeyError: log.debug('couldnt parse event data for VaultCreated', created) return - addr = vault_address(owner, num) + # Verify the authenticity of the vault. We are permissive on Mockchain due to irregular restarts of various components + if not await verify_vault(addr, owner, num): + log.warning(f'Discarding rogue vault {addr}') + return vault_owners[addr] = owner log.debug(f'VaultCreated {owner} #{num} => {addr}') vaults = [] @@ -173,6 +178,25 @@ async def handle_vault_created(created: EventData): current_pub.get()(f'{current_chain.get().id}|{owner}', 'vaults', vaults) +async def handle_vault_logic_changed(upgrade: EventData): + addr = upgrade['address'] + # this event could come from the VaultFactory + if addr == get_factory_contract().address: + log.info(f'Default VaultLogic changed for VaultFactory {addr} to implementation {addr}') + return + # otherwise it's from a Vault + try: + logic = upgrade['args']['logic'] + except KeyError: + log.debug('Could not parse VaultLogicChanged', upgrade) + return + if not await verify_vault(addr): + log.warning(f'Ignoring "upgrade" of rogue vault {addr}') + return + version = await get_logic_version(logic) + log.debug(f'Vault {addr} upgraded to logic version {version}') + + async def activate_time_triggers(): now = current_clock.get().timestamp # log.debug(f'activating time triggers at {now}') diff --git a/src/dexorder/logics.py b/src/dexorder/logics.py new file mode 100644 index 0000000..b0e8fc3 --- /dev/null +++ b/src/dexorder/logics.py @@ -0,0 +1,17 @@ +import logging + +from dexorder.base import Address +from dexorder.blockstate import BlockDict +from dexorder.contract import ContractProxy, get_abi + +log = logging.getLogger(__name__) + +logics: BlockDict[Address, int] = BlockDict('logics', db=True, redis=True) # int is a version/feature identifier + +async def get_logic_version(addr): + try: + return logics[addr] + except KeyError: + version = await ContractProxy(addr, abi=get_abi('IVaultLogic', 'IVault')).version() + logics[addr] = version + return version diff --git a/src/dexorder/progressor.py b/src/dexorder/progressor.py index 0c4f74b..c562a42 100644 --- a/src/dexorder/progressor.py +++ b/src/dexorder/progressor.py @@ -101,9 +101,9 @@ class BlockProgressor(metaclass=ABCMeta): for log_event in log_events: try: parsed = event.process_log(log_event) if event is not None else log_event + parsed_events.append(parsed) except (LogTopicError, MismatchedABI): # this happens for Swap events from non-Uniswap pools - parsed = NARG # need a placeholder - parsed_events.append(parsed) + pass # todo try/except for known retryable errors await maywait(callback(parsed_events)) diff --git a/src/dexorder/uniswap.py b/src/dexorder/uniswap.py index fef94ae..7820d30 100644 --- a/src/dexorder/uniswap.py +++ b/src/dexorder/uniswap.py @@ -1,7 +1,7 @@ from charset_normalizer.md import getLogger from eth_utils import keccak, to_bytes, to_checksum_address -from dexorder.base.chain import Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mock, Alpha +from dexorder.base.chain import Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mockchain, Alpha from dexorder.blockchain import ByBlockchainDict from dexorder.contract import ContractProxy from dexorder.util.abiencode import abi_encoder @@ -54,7 +54,7 @@ class _UniswapContracts (ByBlockchainDict[ContractProxy]): 'quoter': ContractProxy('0xb27308f9F90D607463bb33eA1BeBb41C27CE5AB6', 'IQuoter'), 'swap_router': ContractProxy('0xE592427A0AEce92De3Edee1F18E0157C05861564', 'ISwapRouter'), } - super().__init__({chain.id:std for chain in (Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mock, Alpha)}) + super().__init__({chain.id:std for chain in (Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mockchain, Alpha)}) uniswapV3 = _UniswapContracts() diff --git a/src/dexorder/vault_blockdata.py b/src/dexorder/vault_blockdata.py index 2c7eac2..90954c1 100644 --- a/src/dexorder/vault_blockdata.py +++ b/src/dexorder/vault_blockdata.py @@ -26,6 +26,7 @@ async def verify_vault(addr: str, owner: str = None, num: int = None) -> bool: except CONTRACT_ERRORS: log.warning(f'vault owner for {addr} could not be found.') return False + log.debug(f'verify_vault {addr} {owner} {num}') if num is not None: if vault_address(owner, num) == addr: vault_owners[addr] = owner diff --git a/test/test_blockstate.py b/test/test_blockstate.py index 617ee6f..bf5c7ff 100644 --- a/test/test_blockstate.py +++ b/test/test_blockstate.py @@ -2,7 +2,7 @@ import logging import sys from dexorder import DELETE, NARG -from dexorder.base.chain import current_chain, Mock +from dexorder.base.chain import current_chain, Mockchain from dexorder.blockstate import BlockState, BlockDict, current_blockstate from dexorder.blockstate.branch import Branch from dexorder.blockstate.fork import current_fork, Fork @@ -10,7 +10,7 @@ from dexorder.blockstate.fork import current_fork, Fork logging.basicConfig(level=logging.INFO, stream=sys.stdout) logging.getLogger('dexorder').setLevel(logging.DEBUG) -current_chain.set(Mock) +current_chain.set(Mockchain) b0 = bytes([0]) # genesis block hash root_branch = Branch(0, 0, bytes(), [b0])