From 1851c866e50e8c1ad427c12dbea62c326dffee9d Mon Sep 17 00:00:00 2001
From: Tim Olson <>
Date: Mon, 13 Nov 2023 20:55:32 -0400
Subject: [PATCH] orderindex

---
 .../versions/db62e7db828d_initial_schema.py   | 11 +++++++-
 src/dexorder/base/order.py                    |  2 +-
 src/dexorder/{order => base}/orderlib.py      | 17 ++----------
 src/dexorder/blockstate/blockdata.py          |  2 ++
 src/dexorder/blockstate/db_state.py           |  3 +++
 src/dexorder/contract/__init__.py             |  6 -----
 src/dexorder/database/model/__init__.py       |  2 ++
 src/dexorder/database/model/orderindex.py     | 23 ++++++++++++++++
 src/dexorder/event_handler.py                 |  2 +-
 src/dexorder/order/orderstate.py              | 27 ++++++++++++++++---
 src/dexorder/order/triggers.py                | 19 ++++++++++---
 src/dexorder/routing.py                       | 14 ++++++++++
 src/dexorder/uniswap.py                       | 14 +++++++---
 src/dexorder/util/abiencode.py                |  8 ++++++
 14 files changed, 114 insertions(+), 36 deletions(-)
 rename src/dexorder/{order => base}/orderlib.py (88%)
 create mode 100644 src/dexorder/database/model/orderindex.py
 create mode 100644 src/dexorder/routing.py
 create mode 100644 src/dexorder/util/abiencode.py

diff --git a/alembic/versions/db62e7db828d_initial_schema.py b/alembic/versions/db62e7db828d_initial_schema.py
index 04f37bc..b47c7f3 100644
--- a/alembic/versions/db62e7db828d_initial_schema.py
+++ b/alembic/versions/db62e7db828d_initial_schema.py
@@ -66,13 +66,22 @@ def upgrade() -> None:
     sa.ForeignKeyConstraint(['job_id'], ['transactionjob.id'], ),
     sa.PrimaryKeyConstraint('id')
     )
+    op.create_table('orderindex',
+    sa.Column('chain', dexorder.database.column_types.Blockchain(), nullable=False),
+    sa.Column('vault', sa.String(), nullable=False),
+    sa.Column('order_index', sa.Integer(), nullable=False),
+    sa.Column('state', sa.Enum('Open', 'Canceled', 'Filled', 'Expired', 'Underfunded', name='swaporderstate'), nullable=False),
+    sa.PrimaryKeyConstraint('chain', 'vault', 'order_index')
+    )
 
 
 def downgrade() -> None:
+    op.drop_table('orderindex')
     op.drop_table('seriesset')
     op.drop_table('seriesdict')
     op.drop_table('keyvalue')
     op.drop_table('block')
     op.drop_table('tx')
     op.drop_table('transactionjob')
-    op.execute('drop type transactionjobstate')  # enum type
\ No newline at end of file
+    op.execute('drop type swaporderstate')  # enum type
+    op.execute('drop type transactionjobstate')  # enum type
diff --git a/src/dexorder/base/order.py b/src/dexorder/base/order.py
index 9b85a92..9d74b7b 100644
--- a/src/dexorder/base/order.py
+++ b/src/dexorder/base/order.py
@@ -15,7 +15,7 @@ class OrderKey:
         return OrderKey(vault, int(order_index))
 
     def __str__(self):
-        return f'{self.vault}|{self.order_index}'
+        return f'{self.vault}|{self.order_index:05}'
 
 
 @dataclass(frozen=True, eq=True)
diff --git a/src/dexorder/order/orderlib.py b/src/dexorder/base/orderlib.py
similarity index 88%
rename from src/dexorder/order/orderlib.py
rename to src/dexorder/base/orderlib.py
index 30a413b..5701773 100644
--- a/src/dexorder/order/orderlib.py
+++ b/src/dexorder/base/orderlib.py
@@ -6,8 +6,7 @@ from enum import Enum
 from typing import Optional, Union
 
 from dexorder import dec
-from dexorder.uniswap import uniswapV3_pool_address, uniswap_price
-from dexorder.contract import abi_decoder, abi_encoder
+from dexorder.util.abiencode import abi_decoder, abi_encoder
 from dexorder.util import hexbytes
 
 log = logging.getLogger(__name__)
@@ -22,6 +21,7 @@ class SwapOrderState (Enum):
     Canceled = 1
     Filled = 2
     Expired = 3
+    Underfunded = 14 # todo this is a pseudostate...
 
 class Exchange (Enum):
     UniswapV2 = 0
@@ -58,12 +58,6 @@ class SwapOrder:
         return (self.tokenIn, self.tokenOut, self.route.dump(), str(self.amount), self.amountIsInput,
                 self.outputDirectlyToOwner, self.chainOrder, [t.dump() for t in self.tranches])
 
-    @property
-    def pool_address(self):
-        if self.route.exchange == Exchange.UniswapV3:
-            return uniswapV3_pool_address( self.tokenIn, self.tokenOut, self.route.fee )
-        else:
-            raise NotImplementedError
 
 @dataclass
 class SwapStatus:
@@ -184,13 +178,6 @@ class LineConstraint (Constraint):
     def dump(self):
         return self._dump(LineConstraint.TYPES, (self.isAbove, self.isRatio, self.time, self.valueSqrtX96, self.slopeSqrtX96))
 
-    async def passes(self, pool_addr: str, price: dec) -> bool:
-        limit = await uniswap_price(pool_addr, self.valueSqrtX96)
-        # todo slopes
-        # todo ratios
-        # prices AT the limit get zero volume, so we only trigger on >, not >=
-        return self.isAbove and price > limit or not self.isAbove and price < limit
-
 
 @dataclass
 class Tranche:
diff --git a/src/dexorder/blockstate/blockdata.py b/src/dexorder/blockstate/blockdata.py
index 9f58f43..eaf8a62 100644
--- a/src/dexorder/blockstate/blockdata.py
+++ b/src/dexorder/blockstate/blockdata.py
@@ -29,6 +29,7 @@ class BlockData:
                  series2str=None, series2key=None, # defaults to key2str and str2key
                  key2str=util_key2str, str2key=util_str2key,
                  value2str=json.dumps, str2value=json.loads, # serialize/deserialize value to something JSON-able
+                 savecb:Callable[[Any,Any],None]=None,  # callback(key, value) where value may be DELETE
                  **opts):
         assert series not in BlockData.registry
         BlockData.registry[series] = self
@@ -41,6 +42,7 @@ class BlockData:
         self.series2key = series2key or self.str2key
         self.value2str = value2str
         self.str2value = str2value
+        self.savecb = savecb
         self.lazy_getitem = None
 
     @property
diff --git a/src/dexorder/blockstate/db_state.py b/src/dexorder/blockstate/db_state.py
index e729d7e..9c1f5a6 100644
--- a/src/dexorder/blockstate/db_state.py
+++ b/src/dexorder/blockstate/db_state.py
@@ -67,6 +67,9 @@ class DbState(SeriesCollection):
                         found.value = value
                 else:
                     raise NotImplementedError
+            if d.savecb:
+                d.savecb(diff.key, diff.value)
+        # save root block info
         db.kv[f'root_block|{root_block.chain}'] = [root_block.height, root_block.hash]
 
     # noinspection PyShadowingBuiltins
diff --git a/src/dexorder/contract/__init__.py b/src/dexorder/contract/__init__.py
index d3bbfc3..0924b33 100644
--- a/src/dexorder/contract/__init__.py
+++ b/src/dexorder/contract/__init__.py
@@ -1,15 +1,9 @@
 import json
 
-from eth_abi.codec import ABIDecoder, ABIEncoder
-from eth_abi.registry import registry as default_registry
-
 from .. import current_w3 as _current_w3
 from .abi import abis
 from .contract_proxy import ContractProxy
 
-abi_decoder = ABIDecoder(default_registry)
-abi_encoder = ABIEncoder(default_registry)
-
 
 def get_contract_data(name):
     with open(f'../contract/out/{name}.sol/{name}.json', 'rt') as file:
diff --git a/src/dexorder/database/model/__init__.py b/src/dexorder/database/model/__init__.py
index e75a79f..6581392 100644
--- a/src/dexorder/database/model/__init__.py
+++ b/src/dexorder/database/model/__init__.py
@@ -1,4 +1,6 @@
 from .base import Base
+from .kv import KeyValue
 from .block import Block
 from .series import SeriesSet, SeriesDict
 from .transaction import Transaction, TransactionJob
+from .orderindex import OrderIndex
diff --git a/src/dexorder/database/model/orderindex.py b/src/dexorder/database/model/orderindex.py
new file mode 100644
index 0000000..35fa33d
--- /dev/null
+++ b/src/dexorder/database/model/orderindex.py
@@ -0,0 +1,23 @@
+import logging
+
+from sqlalchemy import SMALLINT
+from sqlalchemy.orm import Mapped, mapped_column
+
+from dexorder.database.column import Blockchain
+from dexorder.database.model import Base
+from dexorder.base.orderlib import SwapOrderState
+
+log = logging.getLogger(__name__)
+
+class OrderIndex (Base):
+    chain: Mapped[Blockchain] = mapped_column(primary_key=True)
+    vault: Mapped[str] = mapped_column(primary_key=True)
+    order_index: Mapped[int] = mapped_column(primary_key=True)
+    state: Mapped[SwapOrderState]
+
+
+# class FillIndex (Base):
+#     chain: Mapped[Blockchain] = mapped_column(index=True)
+#     vault: Mapped[str] = mapped_column(index=True)
+#     order_index: Mapped[SMALLINT] = mapped_column(index=True)
+#     tranche_index:
diff --git a/src/dexorder/event_handler.py b/src/dexorder/event_handler.py
index c01b424..51cd260 100644
--- a/src/dexorder/event_handler.py
+++ b/src/dexorder/event_handler.py
@@ -15,7 +15,7 @@ from dexorder.contract import get_contract_event, ERC20
 from dexorder.data import pool_prices, vault_owners, vault_balances, new_pool_prices
 from dexorder.database.model.block import current_block
 from dexorder.database.model.transaction import TransactionJob
-from dexorder.order.orderlib import SwapOrderStatus, SwapOrderState
+from dexorder.base.orderlib import SwapOrderStatus, SwapOrderState
 from dexorder.order.orderstate import Order
 from dexorder.order.triggers import OrderTriggers, price_triggers, time_triggers, \
     unconstrained_price_triggers, execution_requests, inflight_execution_requests, TrancheStatus, active_tranches, new_price_triggers, activate_order
diff --git a/src/dexorder/order/orderstate.py b/src/dexorder/order/orderstate.py
index 07a2f20..05d0844 100644
--- a/src/dexorder/order/orderstate.py
+++ b/src/dexorder/order/orderstate.py
@@ -3,12 +3,15 @@ import logging
 from dataclasses import dataclass
 from typing import overload
 
-from dexorder import DELETE
+from dexorder import DELETE, db
 from dexorder.base.chain import current_chain
 from dexorder.base.order import OrderKey, TrancheKey
 from dexorder.blockstate import BlockDict, BlockSet
 from dexorder.data import vault_owners
-from dexorder.order.orderlib import SwapOrderStatus, SwapOrderState
+from dexorder.database.model.orderindex import OrderIndex
+from dexorder.base.orderlib import SwapOrderStatus, SwapOrderState
+
+from dexorder.routing import pool_address
 from dexorder.util import json
 
 log = logging.getLogger(__name__)
@@ -96,7 +99,7 @@ class Order:
         assert key not in Order.instances
         self.key = key
         self.status: SwapOrderStatus = Order.order_statuses[key].copy()
-        self.pool_address: str = self.status.order.pool_address
+        self.pool_address: str = pool_address(self.status.order)
         self.tranche_keys = [TrancheKey(key.vault, key.order_index, i) for i in range(len(self.status.trancheFilledIn))]
         # flattenings of various static data
         self.order = self.status.order
@@ -203,6 +206,22 @@ class Order:
             log.warning(f'No vault owner for {k}')
             return None
 
+    @staticmethod
+    def save_order_index(key, status):
+        key: OrderKey
+        status: SwapOrderStatus
+        sess = db.session
+        oi = sess.get(OrderIndex, (current_chain.get(), key.vault, key.order_index))
+        if status is DELETE:
+            if oi:
+                oi.delete()
+        else:
+            if oi:
+                oi.state = status.state
+            else:
+                oi = OrderIndex(chain=current_chain.get(), vault=key.vault, order_index=key.order_index, state=status.state)
+                sess.add(oi)
+
 
     # ORDER STATE
     # various blockstate fields hold different aspects of an order's state.
@@ -212,7 +231,7 @@ class Order:
     # it holds "everything" about an order in the canonical format specified by the contract orderlib, except that
     # the filled amount fields for active orders are maintained in the order_remainings and tranche_remainings series.
     order_statuses: BlockDict[OrderKey, SwapOrderStatus] = BlockDict(
-        'o', db='lazy', redis=True, pub=pub_order_status,
+        'o', db='lazy', redis=True, pub=pub_order_status, savecb=save_order_index,
         str2key=OrderKey.str2key, value2str=lambda v: json.dumps(v.dump()), str2value=lambda s:SwapOrderStatus.load(json.loads(s)),
     )
 
diff --git a/src/dexorder/order/triggers.py b/src/dexorder/order/triggers.py
index 36af8f9..a33d8a1 100644
--- a/src/dexorder/order/triggers.py
+++ b/src/dexorder/order/triggers.py
@@ -5,13 +5,15 @@ from enum import Enum, auto
 from typing import Callable, Optional, Union, Awaitable
 
 from dexorder.blockstate import BlockSet, BlockDict
-from .orderlib import TimeConstraint, LineConstraint, ConstraintMode, SwapOrderState, PriceProof
+from dexorder.base.orderlib import TimeConstraint, LineConstraint, ConstraintMode, SwapOrderState, PriceProof
 from dexorder.util import defaultdictk
 from .orderstate import Order
 from .. import dec
 from ..base.order import OrderKey, TrancheKey, ExecutionRequest
 from ..data import ensure_pool_price
 from ..database.model.block import current_block
+from ..routing import pool_address
+from ..uniswap import uniswap_price
 
 log = logging.getLogger(__name__)
 
@@ -30,11 +32,11 @@ execution_requests:BlockDict[TrancheKey, ExecutionRequest] = BlockDict('e')  # g
 inflight_execution_requests:BlockDict[TrancheKey, int] = BlockDict('ei') # value is block height when the request was sent
 
 
-async def activate_order(order):
+async def activate_order(order: Order):
     """
     Call this to enable triggers on an order which is already in the state.
     """
-    await ensure_pool_price(order.pool_address)
+    await ensure_pool_price(pool_address(order.status.order))
     triggers = OrderTriggers(order)
     if triggers.closed:
         log.debug(f'order {order.key} was immediately closed')
@@ -47,6 +49,15 @@ def intersect_ranges( a_low, a_high, b_low, b_high):
         low, high = None, None
     return low, high
 
+
+async def line_passes(lc: LineConstraint, pool_addr: str, price: dec) -> bool:
+    limit = await uniswap_price(pool_addr, lc.valueSqrtX96)
+    # todo slopes
+    # todo ratios
+    # prices AT the limit get zero volume, so we only trigger on >, not >=
+    return lc.isAbove and price > limit or not lc.isAbove and price < limit
+
+
 class TrancheStatus (Enum):
     Early = auto() # first time trigger hasnt happened yet
     Pricing = auto() # we are inside the time window and checking prices
@@ -137,7 +148,7 @@ class TrancheTrigger:
         if self.closed:
             log.debug(f'price trigger ignored because trigger status is {self.status}')
             return
-        if not self.line_constraints or all(await asyncio.gather(*[pc.passes(self.order.pool_address, cur) for pc in self.line_constraints])):
+        if not self.line_constraints or all(await asyncio.gather(*[line_passes(lc, self.order.pool_address, cur) for lc in self.line_constraints])):
             active_tranches[self.tk] = None #  or PriceProof(...)
 
     def fill(self, _amount_in, _amount_out ):
diff --git a/src/dexorder/routing.py b/src/dexorder/routing.py
new file mode 100644
index 0000000..e85821c
--- /dev/null
+++ b/src/dexorder/routing.py
@@ -0,0 +1,14 @@
+import logging
+
+from dexorder.base.orderlib import Exchange, SwapOrder
+from dexorder.uniswap import uniswapV3_pool_address
+
+log = logging.getLogger(__name__)
+
+
+def pool_address(so: SwapOrder):
+    if so.route.exchange == Exchange.UniswapV3:
+        return uniswapV3_pool_address(so.tokenIn, so.tokenOut, so.route.fee)
+    else:
+        raise NotImplementedError
+
diff --git a/src/dexorder/uniswap.py b/src/dexorder/uniswap.py
index b323223..127e18d 100644
--- a/src/dexorder/uniswap.py
+++ b/src/dexorder/uniswap.py
@@ -4,7 +4,8 @@ from eth_utils import keccak, to_bytes, to_checksum_address
 from dexorder import dec, db
 from dexorder.base.chain import Ethereum, Polygon, Goerli, Mumbai, Arbitrum, Mock
 from dexorder.blockchain import ByBlockchainDict
-from dexorder.contract import abi_encoder, ContractProxy
+from dexorder.contract import ContractProxy
+from dexorder.util.abiencode import abi_encoder
 from dexorder.contract.decimals import token_decimals
 from dexorder.util import hexbytes
 
@@ -50,13 +51,16 @@ def uniswap_pool_address(factory_addr: str, addr_a: str, addr_b: str, fee: int)
 async def uniswap_price(addr, sqrt_price) -> dec:
     price = dec(sqrt_price*sqrt_price) / 2 ** (96 * 2)
     decimals = await pool_decimals(addr)
-    return price * dec(10) ** dec(decimals)
+    result = price * dec(10) ** dec(decimals)
+    log.debug(f'pool sqrtX96 {sqrt_price} with {decimals} decimals = {result}')
+    return result
 
 
 async def pool_decimals(addr):
     key = f'pd|{addr}'
     try:
-        return db.kv[key]
+        decimals = db.kv[key]
+        log.debug('got decimals from db')
     except KeyError:
         pool = UniswapV3Pool(addr)
         token0 = await pool.token0()
@@ -65,7 +69,9 @@ async def pool_decimals(addr):
         decimals1 = await token_decimals(token1)
         decimals = decimals0 - decimals1
         db.kv[key] = decimals
-        return decimals
+        log.debug(f'pool decimals: {decimals0} - {decimals1}')
+    log.debug(f'pool decimals {addr} {decimals}')
+    return decimals
 
 
 class _UniswapContracts (ByBlockchainDict[ContractProxy]):
diff --git a/src/dexorder/util/abiencode.py b/src/dexorder/util/abiencode.py
new file mode 100644
index 0000000..ff1b685
--- /dev/null
+++ b/src/dexorder/util/abiencode.py
@@ -0,0 +1,8 @@
+import logging
+
+from eth_abi.codec import ABIDecoder, ABIEncoder
+from eth_abi.registry import registry as default_registry
+
+log = logging.getLogger(__name__)
+abi_decoder = ABIDecoder(default_registry)
+abi_encoder = ABIEncoder(default_registry)