From ced92dbefda616d4a6fd131d7a5f16d67b6f2e2b Mon Sep 17 00:00:00 2001 From: Tim Date: Thu, 11 Apr 2024 16:26:31 -0400 Subject: [PATCH] finalize_cb bugfix; vault recent orders; transaction handling for backfill branches --- src/dexorder/addrmeta.py | 2 +- src/dexorder/base/__init__.py | 17 +++++++++++++++++ src/dexorder/base/orderlib.py | 11 ++++++++++- src/dexorder/order/orderstate.py | 25 ++++++++++++++++--------- src/dexorder/transaction.py | 18 ++++++++++-------- 5 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/dexorder/addrmeta.py b/src/dexorder/addrmeta.py index e34bc4b..f44973a 100644 --- a/src/dexorder/addrmeta.py +++ b/src/dexorder/addrmeta.py @@ -28,4 +28,4 @@ def save_addrmeta(address: str, meta: AddressMetadata): log.warning(f'Address {address} had unknown metadata type {meta["type"]}') -address_metadata: BlockDict[str,AddressMetadata] = BlockDict('a', redis=True, db=True, savecb=save_addrmeta) +address_metadata: BlockDict[str,AddressMetadata] = BlockDict('a', redis=True, db=True, finalize_cb=save_addrmeta) diff --git a/src/dexorder/base/__init__.py b/src/dexorder/base/__init__.py index 3d32c0e..2356a13 100644 --- a/src/dexorder/base/__init__.py +++ b/src/dexorder/base/__init__.py @@ -12,3 +12,20 @@ TransactionDict = TypedDict( 'TransactionDict', { 'data': Union[bytes,str], 'nonce': Quantity, }) + +TransactionReceiptDict = TypedDict( 'TransactionReceiptDict', { + 'transactionHash': bytes, + 'transactionIndex': Quantity, + 'blockHash': bytes, + 'blockNumber': Quantity, + 'from': str, + 'to': str, + 'cumulativeGasUsed': Quantity, + 'effectiveGasPrice': Quantity, + 'gasUsed': Quantity, + 'contractAddress': str, + 'logs': list, + 'logsBloom': bytes, + 'type': Quantity, + 'status': Quantity, +}) diff --git a/src/dexorder/base/orderlib.py b/src/dexorder/base/orderlib.py index fb1fc59..e02958e 100644 --- a/src/dexorder/base/orderlib.py +++ b/src/dexorder/base/orderlib.py @@ -11,13 +11,22 @@ log = logging.getLogger(__name__) class SwapOrderState (Enum): Unknown = -1 - Signing = 0 # only used by the web but here for completeness + Signing = 0 # only used by the web but here for completeness todo rework OrderLib.sol to remove offchain statuses Underfunded = 1 Open = 2 Canceled = 3 Expired = 4 Filled = 5 + @property + def is_open(self): + return self in (SwapOrderState.Underfunded, SwapOrderState.Open) + + @property + def is_closed(self): + return self in (SwapOrderState.Canceled, SwapOrderState.Expired, SwapOrderState.Filled) + + class Exchange (Enum): Unknown = -1 UniswapV2 = 0 diff --git a/src/dexorder/order/orderstate.py b/src/dexorder/order/orderstate.py index a8c34b2..4b71b61 100644 --- a/src/dexorder/order/orderstate.py +++ b/src/dexorder/order/orderstate.py @@ -152,7 +152,7 @@ class Order: @property def is_open(self): - return self.state is SwapOrderState.Open + return self.state.is_open def add_fill(self, tranche_index: int, filled_in: int, filled_out: int): @@ -222,21 +222,23 @@ class Order: return None @staticmethod - def save_order_index(key, status): - key: OrderKey - status: SwapOrderStatus - sess = db.session - oi = sess.get(OrderIndex, (current_chain.get(), key.vault, key.order_index)) + def save_order_index(key: OrderKey, status: SwapOrderStatus): if status is DELETE: + sess = db.session + oi = sess.get(OrderIndex, (current_chain.get(), key.vault, key.order_index)) if oi: oi.delete() - else: + elif status.state.is_closed: + sess = db.session + oi = sess.get(OrderIndex, (current_chain.get(), key.vault, key.order_index)) if oi: oi.state = status.state else: order_log.debug(f'saving OrderIndex {key} {status.state}') oi = OrderIndex(chain=current_chain.get(), vault=key.vault, order_index=key.order_index, state=status.state) sess.add(oi) + # garbage collect recently closed orders + Order.vault_recently_closed_orders.listremove(key.vault, key.order_index) # ORDER STATE @@ -247,8 +249,9 @@ class Order: # it holds "everything" about an order in the canonical format specified by the contract orderlib, except that # the filled amount fields for active orders are maintained in the order_remainings and tranche_remainings series. order_statuses: BlockDict[OrderKey, SwapOrderStatus] = BlockDict( - 'o', db='lazy', redis=True, pub=pub_order_status, savecb=save_order_index, - str2key=OrderKey.str2key, value2str=lambda v: json.dumps(v.dump()), str2value=lambda s:SwapOrderStatus.load(json.loads(s)), + 'o', db='lazy', redis=True, pub=pub_order_status, finalize_cb=save_order_index, + str2key=OrderKey.str2key, value2str=lambda v: json.dumps(v.dump()), + str2value=lambda s:SwapOrderStatus.load(json.loads(s)), ) # open orders = the set of unfilled, not-canceled orders @@ -257,6 +260,10 @@ class Order: # open orders organized by vault vault_open_orders: BlockDict[str, list[int]] = BlockDict('voo', db=True, redis=True) + # we need to keep closed orders around until their closure is finalized. this data structure is garbage collected + # when a closed order status gets finalized in order_statuses. see Order.save_order_index() + vault_recently_closed_orders: BlockDict[str, list[int]] = BlockDict('vrco', db=True, redis=True) + # fill amounts for open orders are stored here so any updates and publishes do not have to work with the # entire order structure, much of which is static. so any open orders must load the order_status entry first # and then overide the fill values with the data from the order_filled table. once the order completes and diff --git a/src/dexorder/transaction.py b/src/dexorder/transaction.py index 53af11f..5dcdc7a 100644 --- a/src/dexorder/transaction.py +++ b/src/dexorder/transaction.py @@ -6,9 +6,9 @@ from sqlalchemy import select from web3.exceptions import TransactionNotFound from dexorder import db, current_w3 +from dexorder.base import TransactionReceiptDict from dexorder.base.chain import current_chain from dexorder.base.order import TransactionRequest -from dexorder.blocks import current_block from dexorder.blockstate import BlockDict from dexorder.blockstate.diff import DiffEntryItem from dexorder.blockstate.fork import current_fork, Fork @@ -97,13 +97,15 @@ async def handle_transaction_receipts(): TransactionJob.state == TransactionJobState.Sent, ): assert job.tx_id and not job.receipt - block = current_block.get() - if job.tx_id in block.data['transactions']: - try: - receipt = await w3.eth.get_transaction_receipt(job.tx_id) - except TransactionNotFound: - pass - else: + try: + receipt: TransactionReceiptDict = await w3.eth.get_transaction_receipt(job.tx_id) + except TransactionNotFound: + pass + else: + fork = current_fork.get() + assert fork is not None + if fork.branch.contiguous and receipt['blockHash'] in fork.branch.path or \ + fork.branch.disjoint and receipt['blockNumber'] <= fork.height: # don't set the database yet because we could get reorged completed_transactions[job.tx_id] = receipt try: