From aeb7a41e7da2fdcb6f0aa5669a74db0d09522635 Mon Sep 17 00:00:00 2001
From: Tim <tim@dexorder.trade>
Date: Fri, 29 Mar 2024 19:05:32 -0400
Subject: [PATCH] orderstate unload() bugfix

---
 logging-default.toml                 |  2 +-
 src/dexorder/__init__.py             |  2 +-
 src/dexorder/blockstate/blockdata.py | 11 +++--------
 src/dexorder/blockstate/state.py     | 21 ++++++++++++---------
 src/dexorder/order/orderstate.py     |  1 +
 5 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/logging-default.toml b/logging-default.toml
index 3264f8c..da6870b 100644
--- a/logging-default.toml
+++ b/logging-default.toml
@@ -30,7 +30,7 @@ datefmt='%Y-%m-%d %H:%M:%S'
 # Uncomment this block to enable an orders.log file that records only dexorder.order logs
 #
 
-#[loggers.'dexorder.order']
+#[loggers.'dexorder.order.log']
 #level='DEBUG'
 #handlers=['ordersfile',]
 #propagate=true
diff --git a/src/dexorder/__init__.py b/src/dexorder/__init__.py
index 32055b3..19b5eab 100644
--- a/src/dexorder/__init__.py
+++ b/src/dexorder/__init__.py
@@ -7,7 +7,7 @@ from typing import Callable, Any
 
 from web3 import AsyncWeb3
 
-order_log = logging.getLogger('dexorder.order')
+order_log = logging.getLogger('dexorder.order.log')
 
 dec = Decimal
 def now():
diff --git a/src/dexorder/blockstate/blockdata.py b/src/dexorder/blockstate/blockdata.py
index 84b50c3..6b7ad7a 100644
--- a/src/dexorder/blockstate/blockdata.py
+++ b/src/dexorder/blockstate/blockdata.py
@@ -80,14 +80,9 @@ class BlockData (Generic[T]):
         used when lazy_getitem is set
         """
         assert self.lazy_getitem is not None
-        try:
-            value = self.getitem(item)
-        except KeyError:
-            pass  # no key exists to unload
-        else:
-            value = copy.deepcopy(value)
-            value.__dexorder_unload = True  # mark value with magic attr
-            self.setitem(item, value)
+        state = current_blockstate.get()
+        fork = current_fork.get()
+        state.unload(fork, self.series, item)
 
     def contains(self, item):
         try:
diff --git a/src/dexorder/blockstate/state.py b/src/dexorder/blockstate/state.py
index 9e09d32..6e7cce4 100644
--- a/src/dexorder/blockstate/state.py
+++ b/src/dexorder/blockstate/state.py
@@ -50,6 +50,7 @@ class BlockState:
         # diffs_by_hash holds the diff items generated by each block
         self.diffs_by_hash: dict[bytes, list[DiffEntryItem]] = defaultdict(list)
         self.ancestors: dict[bytes, Block] = {}
+        self.unloads: dict[bytes, list] = defaultdict(list)
         BlockState.by_chain[root_block.chain] = self
 
     def add_block(self, block: Block) -> Optional[Fork]:
@@ -127,7 +128,7 @@ class BlockState:
 
     def _get_from_diffs(self, fork, diffs):
         for diff in reversed(diffs):
-            if diff.height <= self.root_block.height or fork is not None and diff in fork and not hasattr(diff.value, '__dexorder_unload'):
+            if diff.height <= self.root_block.height or fork is not None and diff in fork:
                 if diff.value is DELETE:
                     break
                 else:
@@ -146,6 +147,9 @@ class BlockState:
                 self.diffs_by_hash[fork.hash].append(DiffEntryItem(series, key, diff))
             diffs.add(diff)
 
+    def unload(self, fork: Optional[Fork], series, key):
+        self.unloads[fork.hash].append((series, key))
+
     def iteritems(self, fork: Optional[Fork], series):
         for k, difflist in self.diffs_by_series.get(series, {}).items():
             for diff in reversed(difflist):
@@ -179,7 +183,6 @@ class BlockState:
         # walk the by_height list to delete any aged-out block data
         # in order to prune diffs_by_series, updated_keys remembers all the keys that were touched by any aged-out block
         series_deletions = []
-        key_unloads: list[tuple[Any,Any]] = []
         updated_keys = set()
         while self.by_height and self.by_height[0].height <= block.height:
             dead = self.by_height.pop(0)
@@ -193,8 +196,6 @@ class BlockState:
                 for d in block_diffs:
                     if d.key == BlockState._DELETE_SERIES_KEY and dead.hash in new_root_fork:
                         series_deletions.append(d.series)
-                    elif hasattr(d.value, '__dexorder_unload') and dead in new_root_fork:
-                        key_unloads.append((d.series, d.key))
                     else:
                         updated_keys.add((d.series, d.key))
                 del self.diffs_by_hash[dead.hash]
@@ -223,11 +224,13 @@ class BlockState:
             # if only one diff remains, and it's old, and it's a delete, then we can actually delete the diff list
             if not difflist or len(difflist) == 1 and difflist[0].value == DELETE and difflist[0].height <= new_root_fork.height:
                 del self.diffs_by_series[s][k]
-        for s,k in key_unloads:
-            try:
-                del self.diffs_by_series[s][k]
-            except KeyError:
-                pass
+        if block.hash in self.unloads:
+            key_unloads = self.unloads.pop(block.hash)
+            for s,k in key_unloads:
+                try:
+                    del self.diffs_by_series[s][k]
+                except KeyError:
+                    pass
         for s in series_deletions:
             del self.diffs_by_series[s]
         self.root_block = block
diff --git a/src/dexorder/order/orderstate.py b/src/dexorder/order/orderstate.py
index 43779f2..a94a795 100644
--- a/src/dexorder/order/orderstate.py
+++ b/src/dexorder/order/orderstate.py
@@ -188,6 +188,7 @@ class Order:
 
     @staticmethod
     def pub_order_status(_s, k, v):
+        log.debug(f'pub order status {_s} {k} {v}')
         # publish status updates (on placing and completion) to web clients
         try:
             chain_id = current_chain.get().chain_id