vault upgrades; fees; refactoring

dexorder-alpha.toml

@@ -1,25 +0,0 @@
-account='test1' # Dev Account #1
-rpc_url='http://alpharpc:8545'
-ws_url='ws://alpharpc:8545'
-db_url='postgresql://dexorder:redroxed@postgres/dexorder'
-redis_url='redis://redis:6379'
-
-mirror_pools=[
-  '0xC6962004f452bE9203591991D15f6b388e09E8D0',  # WETH/USDC 0.05%
-  '0x2f5e87C9312fa29aed5c179E456625D79015299c',  # WBTC/WETH 0.05%
-  '0x0d94947374cbc779a0FB4D1bfF795C0Af6Dfae25',  # USDC/UNI  1.00%
-  '0x689C96ceAb93f5E131631D225D75DeA3fD37747E',  # WBTC/ARB  0.30%
-  '0x0E4831319A50228B9e450861297aB92dee15B44F',  # WBTC/USDC 0.05%
-  '0x2038eEAa7100E08739352a37Ed67852E8529E8ED',  # ARB/UNI   1.00%
-  '0x468b88941e7Cc0B88c1869d68ab6b570bCEF62Ff',  # WETH/LINK 0.30%
-  '0xC24f7d8E51A64dc1238880BD00bb961D54cbeb29',  # WETH/UNI  0.30%
-  '0xbBe36e6f0331C6a36AB44Bc8421E28E1a1871C1e',  # USDC/LINK 0.30%
-  '0xa79fD76cA2b24631Ec3151f10c0660a30Bc946E7',  # WBTC/LINK 0.30%
-  '0xb0f6cA40411360c03d41C5fFc5F179b8403CdcF8',  # ARB/USDC  0.05%
-  '0xC6F780497A95e246EB9449f5e4770916DCd6396A',  # WETH/ARB  0.05%
-  '0x8b6149aF984140BD3F8e158CcDCD05984a4ad0f5',  # ARB/LINK  0.30%
-  '0xEd701Ba0cec723d85B7d96c80C21148E49D2Bf05',  # LINK/UNI  1.00%
-]
-
-[deployments]
-1337='alpha'

dexorder-separb.toml

@@ -0,0 +1,31 @@
+account = 'test1' # Dev Account #1
+rpc_url = 'http://alpharpc:8545'
+ws_url = 'ws://alpharpc:8545'
+db_url = 'postgresql://dexorder:redroxed@postgres/dexorder'
+redis_url = 'redis://redis:6379'
+
+mirror_pools = [
+    # Arbitrum Pools
+    '0xC6962004f452bE9203591991D15f6b388e09E8D0',  # WETH/USDC 0.05%
+    #  '0x2f5e87C9312fa29aed5c179E456625D79015299c',  # WBTC/WETH 0.05%
+    #  '0x0d94947374cbc779a0FB4D1bfF795C0Af6Dfae25',  # USDC/UNI  1.00%
+    #  '0x689C96ceAb93f5E131631D225D75DeA3fD37747E',  # WBTC/ARB  0.30%
+    #  '0x0E4831319A50228B9e450861297aB92dee15B44F',  # WBTC/USDC 0.05%
+    #  '0x2038eEAa7100E08739352a37Ed67852E8529E8ED',  # ARB/UNI   1.00%
+    #  '0x468b88941e7Cc0B88c1869d68ab6b570bCEF62Ff',  # WETH/LINK 0.30%
+    #  '0xC24f7d8E51A64dc1238880BD00bb961D54cbeb29',  # WETH/UNI  0.30%
+    #  '0xbBe36e6f0331C6a36AB44Bc8421E28E1a1871C1e',  # USDC/LINK 0.30%
+    #  '0xa79fD76cA2b24631Ec3151f10c0660a30Bc946E7',  # WBTC/LINK 0.30%
+    #  '0xb0f6cA40411360c03d41C5fFc5F179b8403CdcF8',  # ARB/USDC  0.05%
+    #  '0xC6F780497A95e246EB9449f5e4770916DCd6396A',  # WETH/ARB  0.05%
+    #  '0x8b6149aF984140BD3F8e158CcDCD05984a4ad0f5',  # ARB/LINK  0.30%
+    #  '0xEd701Ba0cec723d85B7d96c80C21148E49D2Bf05',  # LINK/UNI  1.00%
+
+    # Base Pools
+    #  '0xd0b53D9277642d899DF5C87A3966A349A798F224', # WETH/USDC
+    #  '0xb4CB800910B228ED3d0834cF79D697127BBB00e5', # WETH/USDC
+
+]
+
+[deployments]
+1337 = 'alpha'

src/dexorder/base/chain.py

@@ -49,10 +49,10 @@ Polygon = Blockchain(137, 'Polygon') # POS not zkEVM
 Mumbai = Blockchain(80001, 'Mumbai')
 BSC = Blockchain(56, 'BSC')
 Arbitrum = Blockchain(42161, 'Arbitrum', 3, batch_size=2000)
-Mock = Blockchain(31337, 'Mock', 3, batch_size=2000)
+Mockchain = Blockchain(31337, 'Mockchain', 3, batch_size=2000)
 Alpha = Blockchain(1337, 'Dexorder Alpha', 3, batch_size=1000)
 
-current_chain = ContextVar[Blockchain]('current_chain', default=Mock)
+current_chain = ContextVar[Blockchain]('current_chain', default=Mockchain)
 
 
 class Clock:

src/dexorder/base/orderlib.py

@@ -54,7 +54,7 @@ class SwapOrder:
     minFillAmount: int
     amountIsInput: bool
     outputDirectlyToOwner: bool
-    chainOrder: int
+    conditionalOrder: int
     tranches: list['Tranche']
 
     @staticmethod
@@ -64,7 +64,7 @@ class SwapOrder:
 
     def dump(self):
         return (self.tokenIn, self.tokenOut, self.route.dump(), str(self.amount), str(self.minFillAmount), self.amountIsInput,
-                self.outputDirectlyToOwner, self.chainOrder, [t.dump() for t in self.tranches])
+                self.outputDirectlyToOwner, self.conditionalOrder, [t.dump() for t in self.tranches])
 
     def __str__(self):
         msg = f'''
@@ -83,7 +83,7 @@ SwapOrder
 @dataclass
 class SwapStatus:
     # this is an elaborated version of the on-chain status
-    fillFeeBP: int
+    fillFeeHalfBps: int
     state: SwapOrderState
     start: int
     ocoGroup: Optional[int]
@@ -91,6 +91,7 @@ class SwapStatus:
     filledOut: Optional[int]  # if None then look in the order_filled blockstate
     trancheFilledIn: Optional[list[int]]  # if None then look in the tranche_filled blockstate
     trancheFilledOut: Optional[list[int]]  # if None then look in the tranche_filled blockstate
+    trancheActivationTime: list[int]
 
 
 @dataclass
@@ -105,7 +106,7 @@ class SwapOrderStatus(SwapStatus):
     @staticmethod
     def load(obj):
         order = SwapOrder.load(obj[0])
-        fillFeeBP = int(obj[1])
+        fillFeeHalfBps = int(obj[1])
         state = SwapOrderState(obj[2])
         start = obj[3]
         ocoGroup = None if obj[4] == NO_OCO else obj[4]
@@ -113,20 +114,23 @@ class SwapOrderStatus(SwapStatus):
         filledOut = int(obj[6])
         trancheFilledIn = [int(f) for f in obj[7]]
         trancheFilledOut = [int(f) for f in obj[8]]
-        return SwapOrderStatus(order, fillFeeBP, state, start, ocoGroup, filledIn, filledOut, trancheFilledIn, trancheFilledOut)
+        trancheActivationTime = [int(f) for f in obj[9]]
+        return SwapOrderStatus(order, fillFeeHalfBps, state, start, ocoGroup,
+                               filledIn, filledOut, trancheFilledIn, trancheFilledOut, trancheActivationTime)
 
     @staticmethod
     def load_from_chain(obj):
         # 0  SwapOrder order;
-        # 1  int fillFeeBP
+        # 1  int fillFeeHalfBps
         # 2  bool canceled;
         # 3  uint32 start;
         # 4  uint64 ocoGroup;
         # 5  uint256 filled;  // total
         # 6  uint256[] trancheFilled;  // sum(trancheFilled) == filled
+        # 7  uint32[] trancheActivationTime;
 
         order = SwapOrder.load(obj[0])
-        fillFeeBP = obj[1]
+        fillFeeHalfBps = obj[1]
         state = SwapOrderState.Canceled if obj[2] else SwapOrderState.Open
         start = obj[3]
         ocoGroup = None if obj[4] == NO_OCO else obj[4]
@@ -135,12 +139,17 @@ class SwapOrderStatus(SwapStatus):
         filledOut = 0
         trancheFilledIn = [0 for _ in range(len(obj[6]))]
         trancheFilledOut = [0 for _ in range(len(obj[6]))]
-        return SwapOrderStatus(order, fillFeeBP, state, start, ocoGroup, filledIn, filledOut, trancheFilledIn, trancheFilledOut)
+        trancheActivationTime = [int(i) for i in obj[7]]
+        return SwapOrderStatus(order, fillFeeHalfBps, state, start, ocoGroup,
+                               filledIn, filledOut, trancheFilledIn, trancheFilledOut, trancheActivationTime)
 
     def dump(self):
-        return (self.order.dump(), self.fillFeeBP, self.state.value, self.start, self.ocoGroup,
+        return (
-                str(self.filledIn), str(self.filledOut),
+            self.order.dump(), self.fillFeeHalfBps, self.state.value, self.start, self.ocoGroup,
-                [str(f) for f in self.trancheFilledIn], [str(f) for f in self.trancheFilledOut])
+            str(self.filledIn), str(self.filledOut),
+            [str(f) for f in self.trancheFilledIn], [str(f) for f in self.trancheFilledOut],
+            [int(i) for i in self.trancheActivationTime]
+        )
 
     def copy(self):
         return copy.deepcopy(self)
@@ -168,11 +177,11 @@ class Tranche:
     maxIsBarrier: bool
     marketOrder: bool  
 
-    _reserved5: bool
+    minIsRatio: bool
-    _reserved6: bool
+    maxIsRatio: bool
     _reserved7: bool
-    _reserved8: int
+    rateLimitFraction: int
-    _reserved16: int
+    rateLimitPeriod: int
 
     startTime: int
     endTime: int
@@ -196,11 +205,11 @@ class Tranche:
             obj[3], # minIsBarrier
             obj[4], # maxIsBarrier
             obj[5], # marketOrder
-            obj[6], # reserved
+            obj[6], # minIsRatio
-            obj[7], # reserved
+            obj[7], # maxIsRatio
-            obj[8], # reserved
+            obj[8], # _reserved7
-            obj[9], # reserved
+            obj[9], # rateLimitFraction
-            obj[10], # reserved
+            obj[10], # rateLimitPeriod
             obj[11], # startTime
             obj[12], # endTime
             decode_IEEE754(obj[13]), # minIntercept
@@ -208,10 +217,6 @@ class Tranche:
             decode_IEEE754(obj[15]), # maxIntercept
             decode_IEEE754(obj[16]), # maxSlope
         )
-        result._origMinIntercept = obj[13]
-        result._origMinSlope = obj[14]
-        result._origMaxIntercept = obj[15]
-        result._origMaxSlope = obj[16]
         return result
 
     def dump(self):
@@ -219,17 +224,11 @@ class Tranche:
         minM = encode_IEEE754(self.minSlope)
         maxB = encode_IEEE754(self.maxIntercept)
         maxM = encode_IEEE754(self.maxSlope)
-        if hasattr(self, '_origMinIntercept'):
-            assert minB == self._origMinIntercept
-            # noinspection PyUnresolvedReferences
-            assert minM == self._origMinSlope
-            # noinspection PyUnresolvedReferences
-            assert maxB == self._origMaxIntercept
-            # noinspection PyUnresolvedReferences
-            assert maxM == self._origMaxSlope
         return (
             self.fraction, self.startTimeIsRelative, self.endTimeIsRelative, self.minIsBarrier, self.maxIsBarrier, self.marketOrder,
-            False, False, False, 0, 0, # reserved
+            self.minIsRatio, self.maxIsRatio,
+            False,  # _reserved7
+            self.rateLimitFraction, self.rateLimitPeriod,
             self.startTime, self.endTime, minB, minM, maxB, maxM,
         )
 
@@ -246,6 +245,8 @@ class Tranche:
                 msg += f' <{self.maxIntercept:.5g}'
                 if self.maxSlope:
                     msg += f'{self.maxSlope:+.5g}'
+        if self.rateLimitPeriod:
+            msg += f' {self.rateLimitFraction/MAX_FRACTION:.1%} every {self.rateLimitPeriod/60:.0} minutes'
         return msg
 
 @dataclass

src/dexorder/bin/main.py

@@ -13,7 +13,7 @@ from dexorder.contract.dexorder import get_factory_contract, get_dexorder_contra
 from dexorder.event_handler import init, dump_log, handle_vault_created, handle_order_placed, \
     handle_transfer, handle_swap_filled, handle_order_canceled, handle_order_cancel_all, handle_dexorderexecutions, \
     activate_time_triggers, activate_price_triggers, \
-    process_active_tranches, process_execution_requests, check_ohlc_rollover, handle_uniswap_swaps
+    process_active_tranches, process_execution_requests, check_ohlc_rollover, handle_uniswap_swaps, handle_vault_logic_changed
 from dexorder.memcache import memcache
 from dexorder.memcache.memcache_state import RedisState, publish_all
 from dexorder.order.triggers import activate_orders
@@ -37,13 +37,6 @@ def setup_logevent_triggers(runner):
         log.debug('all events:')
         runner.add_event_trigger(dump_log, None, {})
 
-    factory = get_factory_contract()
-    if factory is None:
-        log.warning(f'No Factory for {current_chain.get()}')
-        vault_created = get_contract_event('Factory', 'VaultCreated')
-    else:
-        vault_created = factory.events.VaultCreated()
-
     dexorder = get_dexorder_contract()
     if dexorder is None:
         log.warning(f'No Dexorder for {current_chain.get()}')
@@ -52,7 +45,8 @@ def setup_logevent_triggers(runner):
         executions = dexorder.events.DexorderExecutions()
 
     runner.add_event_trigger(init)
-    runner.add_event_trigger(handle_vault_created, vault_created)
+    runner.add_event_trigger(handle_vault_created, get_contract_event('Vault', 'VaultCreated'))
+    runner.add_event_trigger(handle_vault_logic_changed, get_contract_event('Vault', 'VaultLogicChanged'))
     runner.add_event_trigger(handle_order_placed, get_contract_event('OrderLib', 'DexorderSwapPlaced'))
     runner.add_event_trigger(handle_transfer, get_contract_event('ERC20', 'Transfer'))
     runner.add_event_trigger(handle_uniswap_swaps, get_contract_event('IUniswapV3PoolEvents', 'Swap'), multi=True)

src/dexorder/contract/__init__.py

@@ -12,13 +12,19 @@ from ..base.chain import current_chain
 CONTRACT_ERRORS = (InsufficientDataBytes, ContractLogicError, BadFunctionCallOutput)
 
 
-def get_contract_data(name):
+def get_abi(name, filename=None):
-    if name in abis:
+    return get_contract_data(name, filename)['abi']
+
+
+def get_contract_data(name, filename=None):
+    if filename is None and name in abis:
         return {'abi':abis[name]}
-    if name == "Vault" and os.path.exists(f'../contract/out/I{name}.sol/I{name}.json') :
+    if filename is None and name == "Vault" and os.path.exists(f'../contract/out/IVault.sol/IVault.json') :
         # logging.debug("getting abi from IVault.json instead of Vault.json")
         name = "IVault" # Special case for proxy Vault
-    with open(f'../contract/out/{name}.sol/{name}.json', 'rt') as file:
+    if filename is None:
+        filename = name
+    with open(f'../contract/out/{filename}.sol/{name}.json', 'rt') as file:
         return json.load(file)
 
 

src/dexorder/contract/dexorder.py

@@ -21,10 +21,10 @@ def _load_chain(chain_id: int):
         with open(f'../contract/broadcast/Deploy.sol/{chain_id}/run-latest.json', 'rt') as file:
             deployment = json.load(file)
             for tx in deployment['transactions']:
-                if tx['contractName'] == 'Factory':
+                if tx['contractName'] == 'VaultFactory':
                     addr = tx['contractAddress']
-                    _factory[chain_id] = ContractProxy(addr, 'Factory')
+                    _factory[chain_id] = ContractProxy(addr, 'VaultFactory')
-                    log.info(f'Factory {addr}')
+                    log.info(f'VaultFactory {addr}')
                 elif tx['contractName'] == 'Dexorder':
                     addr = tx['contractAddress']
                     _dexorder[chain_id] = DexorderContract(addr)

src/dexorder/event_handler.py

@@ -3,18 +3,20 @@ import itertools
 import logging
 from uuid import UUID
 
+from web3.exceptions import LogTopicError, MismatchedABI
 from web3.types import EventData
 
 from dexorder import current_pub, db, from_timestamp, minutely
-from dexorder.base.chain import current_chain, current_clock
+from dexorder.base.chain import current_chain, current_clock, Mockchain
 from dexorder.base.order import TrancheExecutionRequest, TrancheKey, ExecutionRequest, new_tranche_execution_request, \
     OrderKey
 from dexorder.base.orderlib import SwapOrderState
 from dexorder.blocks import get_block_timestamp
 from dexorder.blockstate.fork import current_fork
-from dexorder.contract import ERC20
+from dexorder.contract import ERC20, ContractProxy, get_contract_event
-from dexorder.contract.dexorder import vault_address, VaultContract
+from dexorder.contract.dexorder import vault_address, VaultContract, get_factory_contract
 from dexorder.database.model.transaction import TransactionJob
+from dexorder.logics import logics, get_logic_version
 from dexorder.ohlc import ohlcs, recent_ohlcs
 from dexorder.order.orderstate import Order
 from dexorder.order.triggers import OrderTriggers, price_triggers, time_triggers, \
@@ -142,7 +144,6 @@ async def handle_uniswap_swaps(swaps: list[EventData]):
 
 
 async def handle_uniswap_swap(swap: EventData):
-    # todo gather prices first then apply only the value at the end of the block
     data = await get_uniswap_data(swap)
     if data is None:
         return
@@ -157,10 +158,14 @@ async def handle_vault_created(created: EventData):
     try:
         owner = created['args']['owner']
         num = created['args']['num']
+        addr = created['address']
     except KeyError:
         log.debug('couldnt parse event data for VaultCreated', created)
         return
-    addr = vault_address(owner, num)
+    # Verify the authenticity of the vault. We are permissive on Mockchain due to irregular restarts of various components
+    if not await verify_vault(addr, owner, num):
+        log.warning(f'Discarding rogue vault {addr}')
+        return
     vault_owners[addr] = owner
     log.debug(f'VaultCreated {owner} #{num} => {addr}')
     vaults = []
@@ -173,6 +178,25 @@ async def handle_vault_created(created: EventData):
     current_pub.get()(f'{current_chain.get().id}|{owner}', 'vaults', vaults)
 
 
+async def handle_vault_logic_changed(upgrade: EventData):
+    addr = upgrade['address']
+    # this event could come from the VaultFactory
+    if addr == get_factory_contract().address:
+        log.info(f'Default VaultLogic changed for VaultFactory {addr} to implementation {addr}')
+        return
+    # otherwise it's from a Vault
+    try:
+        logic = upgrade['args']['logic']
+    except KeyError:
+        log.debug('Could not parse VaultLogicChanged', upgrade)
+        return
+    if not await verify_vault(addr):
+        log.warning(f'Ignoring "upgrade" of rogue vault {addr}')
+        return
+    version = await get_logic_version(logic)
+    log.debug(f'Vault {addr} upgraded to logic version {version}')
+
+
 async def activate_time_triggers():
     now = current_clock.get().timestamp
     # log.debug(f'activating time triggers at {now}')

src/dexorder/logics.py

@@ -0,0 +1,17 @@
+import logging
+
+from dexorder.base import Address
+from dexorder.blockstate import BlockDict
+from dexorder.contract import ContractProxy, get_abi
+
+log = logging.getLogger(__name__)
+
+logics: BlockDict[Address, int] = BlockDict('logics', db=True, redis=True)  # int is a version/feature identifier
+
+async def get_logic_version(addr):
+    try:
+        return logics[addr]
+    except KeyError:
+        version = await ContractProxy(addr, abi=get_abi('IVaultLogic', 'IVault')).version()
+        logics[addr] = version
+        return version

src/dexorder/progressor.py

@@ -101,9 +101,9 @@ class BlockProgressor(metaclass=ABCMeta):
                     for log_event in log_events:
                         try:
                             parsed = event.process_log(log_event) if event is not None else log_event
+                            parsed_events.append(parsed)
                         except (LogTopicError, MismatchedABI):
                             # this happens for Swap events from non-Uniswap pools
-                            parsed = NARG  # need a placeholder
+                            pass
-                        parsed_events.append(parsed)
                 # todo try/except for known retryable errors
                 await maywait(callback(parsed_events))

src/dexorder/uniswap.py

@@ -1,7 +1,7 @@
 from charset_normalizer.md import getLogger
 from eth_utils import keccak, to_bytes, to_checksum_address
 
-from dexorder.base.chain import Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mock, Alpha
+from dexorder.base.chain import Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mockchain, Alpha
 from dexorder.blockchain import ByBlockchainDict
 from dexorder.contract import ContractProxy
 from dexorder.util.abiencode import abi_encoder
@@ -54,7 +54,7 @@ class _UniswapContracts (ByBlockchainDict[ContractProxy]):
             'quoter': ContractProxy('0xb27308f9F90D607463bb33eA1BeBb41C27CE5AB6', 'IQuoter'),
             'swap_router': ContractProxy('0xE592427A0AEce92De3Edee1F18E0157C05861564', 'ISwapRouter'),
         }
-        super().__init__({chain.id:std for chain in (Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mock, Alpha)})
+        super().__init__({chain.id:std for chain in (Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mockchain, Alpha)})
 
 
 uniswapV3 = _UniswapContracts()

src/dexorder/vault_blockdata.py

@@ -26,6 +26,7 @@ async def verify_vault(addr: str, owner: str = None, num: int = None) -> bool:
         except CONTRACT_ERRORS:
             log.warning(f'vault owner for {addr} could not be found.')
             return False
+    log.debug(f'verify_vault {addr} {owner} {num}')
     if num is not None:
         if vault_address(owner, num) == addr:
             vault_owners[addr] = owner

test/test_blockstate.py

@@ -2,7 +2,7 @@ import logging
 import sys
 
 from dexorder import DELETE, NARG
-from dexorder.base.chain import current_chain, Mock
+from dexorder.base.chain import current_chain, Mockchain
 from dexorder.blockstate import BlockState, BlockDict, current_blockstate
 from dexorder.blockstate.branch import Branch
 from dexorder.blockstate.fork import current_fork, Fork
@@ -10,7 +10,7 @@ from dexorder.blockstate.fork import current_fork, Fork
 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
 logging.getLogger('dexorder').setLevel(logging.DEBUG)
 
-current_chain.set(Mock)
+current_chain.set(Mockchain)
 
 b0 = bytes([0])  # genesis block hash
 root_branch = Branch(0, 0, bytes(), [b0])