finalize_cb bugfix; vault recent orders; transaction handling for backfill branches

src/dexorder/addrmeta.py

@@ -28,4 +28,4 @@ def save_addrmeta(address: str, meta: AddressMetadata):
         log.warning(f'Address {address} had unknown metadata type {meta["type"]}')
 
 
-address_metadata: BlockDict[str,AddressMetadata] = BlockDict('a', redis=True, db=True, savecb=save_addrmeta)
+address_metadata: BlockDict[str,AddressMetadata] = BlockDict('a', redis=True, db=True, finalize_cb=save_addrmeta)

src/dexorder/base/__init__.py

@@ -12,3 +12,20 @@ TransactionDict = TypedDict( 'TransactionDict', {
     'data': Union[bytes,str],
     'nonce': Quantity,
 })
+
+TransactionReceiptDict = TypedDict( 'TransactionReceiptDict', {
+    'transactionHash': bytes,
+    'transactionIndex': Quantity,
+    'blockHash': bytes,
+    'blockNumber': Quantity,
+    'from': str,
+    'to': str,
+    'cumulativeGasUsed': Quantity,
+    'effectiveGasPrice': Quantity,
+    'gasUsed': Quantity,
+    'contractAddress': str,
+    'logs': list,
+    'logsBloom': bytes,
+    'type': Quantity,
+    'status': Quantity,
+})

src/dexorder/base/orderlib.py

@@ -11,13 +11,22 @@ log = logging.getLogger(__name__)
 
 class SwapOrderState (Enum):
     Unknown = -1
-    Signing = 0 # only used by the web but here for completeness
+    Signing = 0 # only used by the web but here for completeness  todo rework OrderLib.sol to remove offchain statuses
     Underfunded = 1
     Open = 2
     Canceled = 3
     Expired = 4
     Filled = 5
 
+    @property
+    def is_open(self):
+        return self in (SwapOrderState.Underfunded, SwapOrderState.Open)
+
+    @property
+    def is_closed(self):
+        return self in (SwapOrderState.Canceled, SwapOrderState.Expired, SwapOrderState.Filled)
+
+
 class Exchange (Enum):
     Unknown = -1
     UniswapV2 = 0

src/dexorder/order/orderstate.py

@@ -152,7 +152,7 @@ class Order:
 
     @property
     def is_open(self):
-        return self.state is SwapOrderState.Open
+        return self.state.is_open
 
 
     def add_fill(self, tranche_index: int, filled_in: int, filled_out: int):
@@ -222,21 +222,23 @@ class Order:
             return None
 
     @staticmethod
-    def save_order_index(key, status):
+    def save_order_index(key: OrderKey, status: SwapOrderStatus):
-        key: OrderKey
-        status: SwapOrderStatus
-        sess = db.session
-        oi = sess.get(OrderIndex, (current_chain.get(), key.vault, key.order_index))
         if status is DELETE:
+            sess = db.session
+            oi = sess.get(OrderIndex, (current_chain.get(), key.vault, key.order_index))
             if oi:
                 oi.delete()
-        else:
+        elif status.state.is_closed:
+            sess = db.session
+            oi = sess.get(OrderIndex, (current_chain.get(), key.vault, key.order_index))
             if oi:
                 oi.state = status.state
             else:
                 order_log.debug(f'saving OrderIndex {key} {status.state}')
                 oi = OrderIndex(chain=current_chain.get(), vault=key.vault, order_index=key.order_index, state=status.state)
                 sess.add(oi)
+            # garbage collect recently closed orders
+            Order.vault_recently_closed_orders.listremove(key.vault, key.order_index)
 
 
     # ORDER STATE
@@ -247,8 +249,9 @@ class Order:
     # it holds "everything" about an order in the canonical format specified by the contract orderlib, except that
     # the filled amount fields for active orders are maintained in the order_remainings and tranche_remainings series.
     order_statuses: BlockDict[OrderKey, SwapOrderStatus] = BlockDict(
-        'o', db='lazy', redis=True, pub=pub_order_status, savecb=save_order_index,
+        'o', db='lazy', redis=True, pub=pub_order_status, finalize_cb=save_order_index,
-        str2key=OrderKey.str2key, value2str=lambda v: json.dumps(v.dump()), str2value=lambda s:SwapOrderStatus.load(json.loads(s)),
+        str2key=OrderKey.str2key, value2str=lambda v: json.dumps(v.dump()),
+        str2value=lambda s:SwapOrderStatus.load(json.loads(s)),
     )
 
     # open orders = the set of unfilled, not-canceled orders
@@ -257,6 +260,10 @@ class Order:
     # open orders organized by vault
     vault_open_orders: BlockDict[str, list[int]] = BlockDict('voo', db=True, redis=True)
 
+    # we need to keep closed orders around until their closure is finalized. this data structure is garbage collected
+    # when a closed order status gets finalized in order_statuses. see Order.save_order_index()
+    vault_recently_closed_orders: BlockDict[str, list[int]] = BlockDict('vrco', db=True, redis=True)
+
     # fill amounts for open orders are stored here so any updates and publishes do not have to work with the
     # entire order structure, much of which is static. so any open orders must load the order_status entry first
     # and then overide the fill values with the data from the order_filled table. once the order completes and

src/dexorder/transaction.py

@@ -6,9 +6,9 @@ from sqlalchemy import select
 from web3.exceptions import TransactionNotFound
 
 from dexorder import db, current_w3
+from dexorder.base import TransactionReceiptDict
 from dexorder.base.chain import current_chain
 from dexorder.base.order import TransactionRequest
-from dexorder.blocks import current_block
 from dexorder.blockstate import BlockDict
 from dexorder.blockstate.diff import DiffEntryItem
 from dexorder.blockstate.fork import current_fork, Fork
@@ -97,13 +97,15 @@ async def handle_transaction_receipts():
             TransactionJob.state == TransactionJobState.Sent,
     ):
         assert job.tx_id and not job.receipt
-        block = current_block.get()
+        try:
-        if job.tx_id in block.data['transactions']:
+            receipt: TransactionReceiptDict = await w3.eth.get_transaction_receipt(job.tx_id)
-            try:
+        except TransactionNotFound:
-                receipt = await w3.eth.get_transaction_receipt(job.tx_id)
+            pass
-            except TransactionNotFound:
+        else:
-                pass
+            fork = current_fork.get()
-            else:
+            assert fork is not None
+            if fork.branch.contiguous and receipt['blockHash'] in fork.branch.path or \
+                    fork.branch.disjoint and receipt['blockNumber'] <= fork.height:
                 # don't set the database yet because we could get reorged
                 completed_transactions[job.tx_id] = receipt
                 try: