order completion debug; refactor UNLOAD as a magic attr

src/dexorder/__init__.py

@@ -7,11 +7,18 @@ from web3 import AsyncWeb3
 dec = Decimal
 
 # NARG is used in argument defaults to mean "not specified" rather than "specified as None"
-class _NARG:
+class _Token:
+    def __init__(self, name):
+        self.__token_name = name
+    def __repr__(self): return self.__token_name
+    def __str__(self): return self.__token_name
+
+class _FalseToken (_Token):
     def __bool__(self): return False
-NARG = _NARG()
+
-DELETE = object() # used as a value token to indicate removal of the key
+
-UNLOAD = object() # used as a value token to indicate the key is no longer needed in memory
+NARG = _FalseToken('NARG')
+DELETE = _FalseToken('DELETE') # used as a value token to indicate removal of the key
 ADDRESS_0 = '0x0000000000000000000000000000000000000000'
 WEI  = 1
 GWEI = 1_000_000_000

src/dexorder/blockstate/blockdata.py

@@ -1,9 +1,10 @@
+import copy
 import json
 import logging
 from enum import Enum
 from typing import TypeVar, Generic, Iterable, Union, Any, Iterator, Callable
 
-from dexorder import NARG, DELETE, UNLOAD
+from dexorder import NARG, DELETE
 from dexorder.base.fork import current_fork
 from .state import current_blockstate
 from dexorder.util import key2str as util_key2str, str2key as util_str2key
@@ -77,7 +78,14 @@ class BlockData:
         used when lazy_getitem is set
         """
         assert self.lazy_getitem is not None
-        self.setitem(item, UNLOAD)
+        try:
+            value = self.getitem(item)
+        except KeyError:
+            pass  # no key exists to unload
+        else:
+            value = copy.deepcopy(value)
+            value.__dexorder_unload = True  # mark value with magic attr
+            self.setitem(item, value)
 
     def contains(self, item):
         try:

src/dexorder/blockstate/db_state.py

@@ -4,7 +4,7 @@ from typing import Iterable, Optional, Union, Any
 from . import BlockSet, BlockDict, BlockState, current_blockstate, DataType
 from .blockdata import BlockData, SeriesCollection
 from .diff import DiffItem, DiffEntryItem
-from .. import db, UNLOAD, DELETE
+from .. import db, DELETE
 from ..base.chain import current_chain
 from ..base.fork import current_fork, Fork
 from ..database.model import SeriesSet, SeriesDict, Block
@@ -52,8 +52,6 @@ class DbState(SeriesCollection):
             if diff.value is DELETE:
                 Entity = SeriesSet if t == DataType.SET else SeriesDict if t == DataType.DICT else None
                 db.session.query(Entity).filter(Entity.chain==chain_id, Entity.series==diffseries, Entity.key==diffkey).delete()
-            elif diff.value is UNLOAD:
-                pass
             else:
                 # upsert
                 if t == DataType.SET:

src/dexorder/blockstate/diff.py

@@ -1,12 +1,12 @@
 from dataclasses import dataclass
 from typing import Union, Any
 
-from dexorder import DELETE, UNLOAD
+from dexorder import DELETE
 
 
 @dataclass
 class DiffEntry:
-    value: Union[Any, DELETE, UNLOAD]
+    value: Union[Any, DELETE]
     height: int
     hash: bytes
 
@@ -18,7 +18,7 @@ class DiffItem:
     value: Any
 
     def __str__(self):
-        return f'{self.series}.{self.key}={"[DEL]" if self.value is DELETE else "[UNL]" if self.value is UNLOAD else self.value}'
+        return f'{self.series}.{self.key}={"[DEL]" if self.value is DELETE else self.value}'
 
 @dataclass
 class DiffEntryItem:
@@ -32,4 +32,4 @@ class DiffEntryItem:
 
     def __str__(self):
         return (f'{self.entry.hash.hex()} {self.series}.{self.key}='
-                f'{"[DEL]" if self.entry.value is DELETE else "[UNL]" if self.value is UNLOAD else self.entry.value}')
+                f'{"[DEL]" if self.entry.value is DELETE else self.entry.value}')

src/dexorder/blockstate/state.py

@@ -6,7 +6,7 @@ from typing import Any, Optional, Union, Reversible
 
 from sortedcontainers import SortedList
 
-from dexorder import NARG, UNLOAD
+from dexorder import NARG
 from dexorder.base.fork import Fork, DisjointFork
 from dexorder.database.model import Block
 from dexorder.util import hexstr
@@ -125,7 +125,7 @@ class BlockState:
 
     def _get_from_diffs(self, fork, diffs):
         for diff in reversed(diffs):
-            if diff.height <= self.root_block.height or fork is not None and diff in fork and diff.value is not UNLOAD:
+            if diff.height <= self.root_block.height or fork is not None and diff in fork and not hasattr(diff.value, '__dexorder_unload'):
                 if diff.value is DELETE:
                     break
                 else:
@@ -191,7 +191,7 @@ class BlockState:
                 for d in block_diffs:
                     if d.key == BlockState._DELETE_SERIES_KEY and dead.hash in new_root_fork:
                         series_deletions.append(d.series)
-                    elif d.value is UNLOAD and dead in new_root_fork:
+                    elif hasattr(d.value, '__dexorder_unload') and dead in new_root_fork:
                         key_unloads.append((d.series, d.key))
                     else:
                         updated_keys.add((d.series, d.key))

src/dexorder/event_handler.py

@@ -6,7 +6,7 @@ from web3.types import EventData
 
 from dexorder import current_pub, db, dec
 from dexorder.base.chain import current_chain
-from dexorder.base.order import TrancheExecutionRequest, TrancheKey, ExecutionRequest, new_tranche_execution_request
+from dexorder.base.order import TrancheExecutionRequest, TrancheKey, ExecutionRequest, new_tranche_execution_request, OrderKey
 from dexorder.transaction import create_transactions, submit_transaction_request, handle_transaction_receipts, send_transactions
 from dexorder.uniswap import UniswapV3Pool, uniswap_price
 from dexorder.contract.dexorder import get_factory_contract, vault_address, VaultContract, get_dexorder_contract
@@ -128,11 +128,11 @@ def handle_swap_filled(event: EventData):
     except KeyError:
         log.warning(f'DexorderSwapFilled IGNORED due to missing order {vault} {order_index}')
         return
+    try:
         triggers = OrderTriggers.instances[order.key]
         triggers.fill(tranche_index, amount_in, amount_out)
-    # check for fill
+    except KeyError:
-    if order.remaining <= 0:
+        log.warning(f'No order triggers for fill of {TrancheKey(*order.key,tranche_index)}')
-        close_order_and_disable_triggers(order, SwapOrderState.Filled)
 
 async def handle_order_completed(event: EventData):
     # event DexorderCompleted (uint64 orderIndex); // todo remove?
@@ -294,7 +294,11 @@ def handle_dexorderexecutions(event: EventData):
 
 
 def finish_execution_request(req: TrancheExecutionRequest, error: str):
+    try:
         order: Order = Order.of(req.vault, req.order_index)
+    except KeyError:
+        log.error(f'Could not get order {OrderKey(req.vault, req.order_index)}')
+        return
     tk = TrancheKey(req.vault, req.order_index, req.tranche_index)
     if error != '':
         log.debug(f'execution request for tranche {tk} had error "{error}"')

src/dexorder/memcache/memcache_state.py

@@ -62,12 +62,12 @@ class RedisState (SeriesCollection):
                 continue
             t = d.type
             series = f'{chain_id}|{d.series2str(diff.series)}'
-            key = d.key2str(diff.key)
-            value = d.value2str(diff.value)
 
             # pub/sub socketio/redis
             pub_era = d.opts.get('pub') # event, room, args
             if pub_era is True:
+                key = d.key2str(diff.key)
+                value = d.value2str(diff.value)
                 pub_era = series, key, [value]
             elif callable(pub_era):
                 pub_era = await maywait(pub_era(diff.key, diff.value))
@@ -75,7 +75,8 @@ class RedisState (SeriesCollection):
                 e, r, a = pub_era
                 # noinspection PyTypeChecker
                 pubs.append((e,r,a))
-            if diff.value is DELETE:
+            if diff.value is DELETE or hasattr(diff.value, '__dexorder_unload'):
+                key = d.key2str(diff.key)
                 if t == DataType.SET:
                     sdels[series].add(key)
                 elif t == DataType.DICT:
@@ -83,6 +84,8 @@ class RedisState (SeriesCollection):
                 else:
                     raise NotImplementedError
             else:
+                key = d.key2str(diff.key)
+                value = d.value2str(diff.value)
                 if t == DataType.SET:
                     sadds[series].add(key)
                 elif t == DataType.DICT:

src/dexorder/order/orderstate.py

@@ -3,6 +3,7 @@ import logging
 from dataclasses import dataclass
 from typing import overload
 
+from dexorder import DELETE
 from dexorder.base.chain import current_chain
 from dexorder.base.order import OrderKey, TrancheKey
 from dexorder.blockstate import BlockDict, BlockSet
@@ -61,10 +62,7 @@ class Order:
     @staticmethod
     def of(a, b=None) -> 'Order':
         key = a if b is None else OrderKey(a, b)
-        try:
         return Order.instances[key]
-        except KeyError:
-            log.error(f'Could not find {key} among:\n{", ".join(str(k) for k in Order.instances.keys())}')
 
 
     @staticmethod
@@ -182,10 +180,14 @@ class Order:
         except KeyError:
             log.warning(f'No vault owner for {k}')
             return None
+        except AttributeError:
+            log.error(f'could not dump {v}')
 
     @staticmethod
     async def pub_order_fills(k, v):
         # publish status updates (on placing and completion) to web clients
+        if v is DELETE:
+            return None
         try:
             chain_id = current_chain.get().chain_id
             return (f'{chain_id}|{vault_owners[k.vault]}',  # publish on the vault owner's channel