dexorder/backend
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

python code updated for new orderspec but not debugged

Browse Source
This commit is contained in:
Tim Olson
2023-12-04 23:11:13 -04:00
parent ecc9e2eb98
commit feea8160ce
11 changed files with 255 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.idea/backend.iml generated
View File

@@ -9,6 +9,9 @@
<orderEntry type="jdk" jdkName="Python 3.11 (backend)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PackageRequirementsSettings">
<option name="versionSpecifier" value="Don't specify version" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />

2
requirements.txt
View File

@@ -8,3 +8,5 @@ sortedcontainers
defaultlist
redis[hiredis]
socket.io-emitter
hexbytes
websockets

240
src/dexorder/base/orderlib.py
View File

@@ -1,27 +1,21 @@
import copy
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
from enum import Enum
from typing import Optional, Union
from typing import Optional
from dexorder import dec
from dexorder.util.abiencode import abi_decoder, abi_encoder
from dexorder.util import hexbytes
from dexorder.util.convert import decode_IEEE754, encode_IEEE754
log = logging.getLogger(__name__)
# enum SwapOrderState {
# Open, Canceled, Filled, Expired
# }
class SwapOrderState (Enum):
Open = 0
Canceled = 1
Filled = 2
Expired = 3
Underfunded = 14 # todo this is a pseudostate...
Unknown = 0
Open = 1
Canceled = 2
Filled = 3
Expired = 4
Underfunded = 5
class Exchange (Enum):
UniswapV2 = 0
@@ -45,22 +39,27 @@ class SwapOrder:
tokenOut: str
route: Route
amount: int
minFillAmount: int
amountIsInput: bool
outputDirectlyToOwner: bool
chainOrder: int
tranches: list['Tranche']
state: SwapOrderState # this is not in the blockchain orderstatus: it's a computed and cached field.
@staticmethod
def load(obj):
return SwapOrder(obj[0], obj[1], Route.load(obj[2]), int(obj[3]), obj[4], obj[5], obj[6], [Tranche.load(t) for t in obj[7]])
return SwapOrder(obj[0], obj[1], Route.load(obj[2]), int(obj[3]), int(obj[4]), obj[5], obj[6], obj[7],
[Tranche.load(t) for t in obj[8]], SwapOrderState.Unknown)
def dump(self):
return (self.tokenIn, self.tokenOut, self.route.dump(), str(self.amount), self.amountIsInput,
return (self.tokenIn, self.tokenOut, self.route.dump(), str(self.amount), str(self.minFillAmount), self.amountIsInput,
self.outputDirectlyToOwner, self.chainOrder, [t.dump() for t in self.tranches])
@dataclass
class SwapStatus:
# this is an elaborated version of the on-chain status
state: SwapOrderState
start: int
ocoGroup: Optional[int]
@@ -71,128 +70,141 @@ class SwapStatus:
@dataclass
class SwapOrderStatus (SwapStatus):
order: SwapOrder
class SwapOrderStatus(SwapStatus):
order: SwapOrder
def __init__(self, order: SwapOrder, *swapstatus_args):
""" init with order object first follewed by the swap status args"""
super().__init__(*swapstatus_args)
self.order = order
@staticmethod
def load(obj):
order = SwapOrder.load(obj[0])
state = SwapOrderState(obj[1])
start = obj[2]
ocoGroup = None if obj[3] == NO_OCO else obj[3]
filledIn = int(obj[4])
filledOut = int(obj[5])
trancheFilledIn = [int(f) for f in obj[6]]
trancheFilledOut = [int(f) for f in obj[7]]
return SwapOrderStatus(order, state, start, ocoGroup, filledIn, filledOut, trancheFilledIn, trancheFilledOut)
def dump(self):
return (self.order.dump(), self.state.value, self.start, self.ocoGroup,
str(self.filledIn), str(self.filledOut),
[str(f) for f in self.trancheFilledIn], [str(f) for f in self.trancheFilledOut])
def copy(self):
return copy.deepcopy(self)
NO_OCO = 18446744073709551615 # max uint64
class ConstraintMode (Enum):
Time = 0
Line = 1
Barrier = 2
@dataclass
class Constraint (ABC):
mode: ConstraintMode
def __init__(self, order: SwapOrder, *swapstatus_args):
""" init with order object first followed by the swap status args"""
super().__init__(*swapstatus_args)
self.order = order
@staticmethod
def load(obj):
mode = ConstraintMode(obj[0])
if mode == ConstraintMode.Time:
return TimeConstraint.load(obj[1])
elif mode == ConstraintMode.Line:
return LineConstraint.load(obj[1])
else:
raise ValueError(f'Unknown constraint mode {mode}')
order = SwapOrder.load(obj[0])
state = SwapOrderState(obj[1])
start = obj[2]
ocoGroup = None if obj[3] == NO_OCO else obj[3]
filledIn = int(obj[4])
filledOut = int(obj[5])
trancheFilledIn = [int(f) for f in obj[6]]
trancheFilledOut = [int(f) for f in obj[7]]
return SwapOrderStatus(order, state, start, ocoGroup, filledIn, filledOut, trancheFilledIn, trancheFilledOut)
@abstractmethod
def dump(self): ...
@staticmethod
def load_from_chain(obj):
# 0 SwapOrder order;
# 1 bool canceled;
# 2 uint32 start;
# 3 uint64 ocoGroup;
# 4 uint256 filled; // total
# 5 uint256[] trancheFilled; // sum(trancheFilled) == filled
def _dump(self, types, values):
return self.mode.value, abi_encoder.encode(types, values)
order = SwapOrder.load(obj[0])
state = SwapOrderState.Canceled if obj[1] else SwapOrderState.Open
start = obj[2]
ocoGroup = None if obj[3] == NO_OCO else obj[3]
# we ignore any fill values from the on-chain struct, because we will subsequently detect the DexorderFilled events and add them in
filledIn = 0
filledOut = 0
trancheFilledIn = [0 for _ in range(len(obj[5]))]
trancheFilledOut = [0 for _ in range(len(obj[5]))]
return SwapOrderStatus(order, state, start, ocoGroup, filledIn, filledOut, trancheFilledIn, trancheFilledOut)
def dump(self):
return (self.order.dump(), self.state.value, self.start, self.ocoGroup,
str(self.filledIn), str(self.filledOut),
[str(f) for f in self.trancheFilledIn], [str(f) for f in self.trancheFilledOut])
def copy(self):
return copy.deepcopy(self)
NO_OCO = 18446744073709551615 # max uint64
class TimeMode (Enum):
Timestamp = 0
SinceOrderStart = 1
@dataclass
class Time:
mode: TimeMode
time: int
def timestamp(self, order_start: int):
return self.time if self.mode is TimeMode.Timestamp else order_start + self.time
# def timestamp(self, order_start: int):
# return self.time if self.mode is TimeMode.Timestamp else order_start + self.time
DISTANT_PAST = 0
DISTANT_FUTURE = 4294967295 # max uint32
@dataclass
class TimeConstraint (Constraint):
earliest: Time
latest: Time
TYPES = ['uint8', 'uint32', 'uint8', 'uint32']
@staticmethod
def load(obj: Union[bytes|str]):
earliest_mode, earliest_time, latest_mode, latest_time = abi_decoder.decode(TimeConstraint.TYPES, hexbytes(obj))
return TimeConstraint(ConstraintMode.Time, Time(TimeMode(earliest_mode),earliest_time), Time(TimeMode(latest_mode),latest_time))
def dump(self):
return self._dump(TimeConstraint.TYPES, (self.earliest.mode.value, self.earliest.time, self.latest.mode.value, self.latest.time))
@dataclass
class LineConstraint (Constraint):
isAbove: bool
isRatio: bool
time: int
valueSqrtX96: int
slopeSqrtX96: int
TYPES = 'bool','bool','uint32','uint160','int160'
@staticmethod
def load(obj):
return LineConstraint(ConstraintMode.Line, *abi_decoder.decode(LineConstraint.TYPES, hexbytes(obj)))
def dump(self):
return self._dump(LineConstraint.TYPES, (self.isAbove, self.isRatio, self.time, self.valueSqrtX96, self.slopeSqrtX96))
MAX_FRACTION = 65535 # max uint16
@dataclass
class Tranche:
fraction: int # 18-decimal fraction of the order amount which is available to this tranche. must be <= 1
constraints: list[Constraint]
fraction: int # fraction of the order amount which is available to this tranche, as a uint16: a value of MAX_FRACTION (65535) represents 100%
startTimeIsRelative: bool
endTimeIsRelative: bool
minIsBarrier: bool
maxIsBarrier: bool
marketOrder: bool
_reserved5: bool
_reserved6: bool
_reserved7: bool
_reserved8: int
_reserved16: int
startTime: int
endTime: int
minIntercept: float
minSlope: float
maxIntercept: float
maxSlope: float
def fraction_of(self, amount):
return amount * self.fraction // 65535
return amount * self.fraction // MAX_FRACTION
@staticmethod
def load(obj):
return Tranche(obj[0], [Constraint.load(c) for c in obj[1]])
result = Tranche(
# none of these ints need to be strings because fraction is only 16 bits and the timestamps are only 32 bits
obj[0], # fraction
obj[1], # startTimeIsRelative
obj[2], # endTimeIsRelative
obj[3], # minIsBarrier
obj[4], # maxIsBarrier
obj[5], # marketOrder
obj[6], # reserved
obj[7], # reserved
obj[8], # reserved
obj[9], # reserved
obj[10], # reserved
obj[11], # startTime
obj[12], # endTime
decode_IEEE754(obj[13]), # minIntercept
decode_IEEE754(obj[14]), # minSlope
decode_IEEE754(obj[15]), # maxIntercept
decode_IEEE754(obj[16]), # maxSlope
)
result._origMinIntercept = obj[13]
result._origMinSlope = obj[14]
result._origMaxIntercept = obj[15]
result._origMaxSlope = obj[16]
return result
def dump(self):
return self.fraction, [c.dump() for c in self.constraints]
minB = encode_IEEE754(self.minIntercept)
minM = encode_IEEE754(self.minSlope)
maxB = encode_IEEE754(self.maxIntercept)
maxM = encode_IEEE754(self.maxSlope)
if hasattr(self, '_origMinIntercept'):
assert minB == self._origMinIntercept
# noinspection PyUnresolvedReferences
assert minM == self._origMinSlope
# noinspection PyUnresolvedReferences
assert maxB == self._origMaxIntercept
# noinspection PyUnresolvedReferences
assert maxM == self._origMaxSlope
return (
self.fraction, self.startTimeIsRelative, self.endTimeIsRelative, self.minIsBarrier, self.maxIsBarrier, self.marketOrder,
False, False, False, 0, 0, # reserved
self.startTime, self.endTime, minB, minM, maxB, maxM,
)
@dataclass

22
src/dexorder/data.py
View File

@@ -1,10 +1,7 @@
import logging
from dexorder import dec
from dexorder.base.chain import current_chain
from dexorder.blockstate import BlockDict
from dexorder.blockstate.blockdata import K, V
from dexorder.uniswap import UniswapV3Pool
from dexorder.util import json
log = logging.getLogger(__name__)
@@ -30,22 +27,3 @@ vault_balances: BlockDict[str, dict[str, int]] = BlockDict(
pub=pub_vault_balances
)
class PoolPrices (BlockDict[str, dec]):
def __setitem__(self, item: K, value: V) -> None:
super().__setitem__(item, value)
new_pool_prices[item] = value
def pub_pool_price(k,v):
chain_id = current_chain.get().chain_id
return f'{chain_id}|{k}', 'p', (chain_id, k, str(v))
new_pool_prices: dict[str, dec] = {} # tracks which prices were set during the current block. cleared every block.
pool_prices: PoolPrices = PoolPrices('p', db=True, redis=True, pub=pub_pool_price, value2str=lambda d: f'{d:f}', str2value=dec)
async def ensure_pool_price(pool_addr):
if pool_addr not in pool_prices:
log.debug(f'querying price for pool {pool_addr}')
pool_prices[pool_addr] = await UniswapV3Pool(pool_addr).price()

19
src/dexorder/database/model/pool.py Normal file
View File

@@ -0,0 +1,19 @@
import logging
from sqlalchemy.orm import Mapped, mapped_column
from dexorder.base.orderlib import Exchange
from dexorder.database.column import Address, Blockchain
from dexorder.database.model import Base
log = logging.getLogger(__name__)
class PoolModel (Base):
__tablename__ = 'pool'
chain: Mapped[Blockchain] = mapped_column(primary_key=True)
address: Mapped[Address] = mapped_column(primary_key=True)
exchange: Mapped[Exchange]
base: Mapped[Address]
quote: Mapped[Address]
fee: int # in millionths aka 100ths of a bip

8
src/dexorder/event_handler.py
View File

@@ -12,7 +12,8 @@ from dexorder.transaction import create_transactions, submit_transaction_request
from dexorder.uniswap import uniswap_price
from dexorder.contract.dexorder import get_factory_contract, vault_address, VaultContract, get_dexorder_contract
from dexorder.contract import get_contract_event, ERC20
from dexorder.data import pool_prices, vault_owners, vault_balances, new_pool_prices
from dexorder.data import vault_owners, vault_balances
from dexorder.pool import new_pool_prices, pool_prices
from dexorder.database.model.block import current_block
from dexorder.database.model.transaction import TransactionJob
from dexorder.base.orderlib import SwapOrderStatus, SwapOrderState
@@ -109,10 +110,9 @@ async def handle_order_placed(event: EventData):
for index in range(start_index, start_index+num_orders):
obj = await vault.swapOrderStatus(index)
log.debug(f'raw order status {obj}')
order_status = SwapOrderStatus.load(obj)
order = Order.create(vault.address, index, order_status)
order = Order.create(vault.address, index, obj)
await activate_order(order)
log.debug(f'new order {order_status}')
log.debug(f'new order {order}')
def handle_swap_filled(event: EventData):

3
src/dexorder/order/orderstate.py
View File

@@ -75,8 +75,9 @@ class Order:
@staticmethod
def create(vault: str, order_index: int, status: SwapOrderStatus):
def create(vault: str, order_index: int, obj):
""" use when a brand new order is detected by the system """
status = SwapOrderStatus.load_from_chain(obj)
key = OrderKey(vault, order_index)
Order.order_statuses[key] = status.copy() # always copy the struct when setting. values in BlockData must be immutable
order = Order(key)

74
src/dexorder/order/triggers.py
View File

@@ -5,15 +5,14 @@ from enum import Enum, auto
from typing import Callable, Optional, Union, Awaitable
from dexorder.blockstate import BlockSet, BlockDict
from dexorder.base.orderlib import TimeConstraint, LineConstraint, ConstraintMode, SwapOrderState, PriceProof
from dexorder.base.orderlib import SwapOrderState, PriceProof
from dexorder.util import defaultdictk
from .orderstate import Order
from .. import dec
from ..base.order import OrderKey, TrancheKey, ExecutionRequest
from ..data import ensure_pool_price
from ..pool import ensure_pool_price, Pool
from ..database.model.block import current_block
from ..routing import pool_address
from ..uniswap import uniswap_price
log = logging.getLogger(__name__)
@@ -36,8 +35,13 @@ async def activate_order(order: Order):
"""
Call this to enable triggers on an order which is already in the state.
"""
await ensure_pool_price(pool_address(order.status.order))
triggers = OrderTriggers(order)
address = pool_address(order.status.order)
await ensure_pool_price(address)
pool = await Pool.get(address)
inverted = pool.base != order.order.tokenIn
if inverted:
assert pool.base == order.order.tokenOut
triggers = OrderTriggers(order, inverted)
if triggers.closed:
log.debug(f'order {order.key} was immediately closed')
close_order_and_disable_triggers(order, SwapOrderState.Filled if order.remaining <= 0 else SwapOrderState.Expired)
@@ -50,12 +54,12 @@ def intersect_ranges( a_low, a_high, b_low, b_high):
return low, high
async def line_passes(lc: LineConstraint, pool_addr: str, price: dec) -> bool:
limit = await uniswap_price(pool_addr, lc.valueSqrtX96)
# todo slopes
async def line_passes(lc: tuple[float,float], is_min: bool, price: dec) -> bool:
b, m = lc
limit = m * current_block.get().timestamp + b
# todo ratios
# prices AT the limit get zero volume, so we only trigger on >, not >=
return lc.isAbove and price > limit or not lc.isAbove and price < limit
return is_min and price > limit or not is_min and price < limit
class TrancheStatus (Enum):
@@ -65,42 +69,36 @@ class TrancheStatus (Enum):
Expired = auto() # time deadline has past and this tranche cannot be filled
class TrancheTrigger:
def __init__(self, order: Order, tranche_key: TrancheKey):
def __init__(self, order: Order, tranche_key: TrancheKey, inverted: bool):
assert order.key.vault == tranche_key.vault and order.key.order_index == tranche_key.order_index
self.order = order
self.tk = tranche_key
self.status = TrancheStatus.Early
self.time_constraint = None
self.line_constraints: list[LineConstraint] = []
start = self.order.status.start
tranche = order.order.tranches[self.tk.tranche_index]
tranche_amount = tranche.fraction_of(order.amount)
tranche_filled = order.tranche_filled(self.tk.tranche_index)
tranche_remaining = tranche_amount - tranche_filled
# time and price constraints
self.time_constraint = tranche.startTime, tranche.endTime
if tranche.startTimeIsRelative:
self.time_constraint[0] += self.order.status.start
if tranche.endTimeIsRelative:
self.time_constraint[1] += self.order.status.start
self.min_line_constraint = (0.,0.) if tranche.marketOrder else (tranche.minIntercept, tranche.minSlope)
self.max_line_constraint = (0.,0.) if tranche.marketOrder else (tranche.maxIntercept, tranche.maxSlope)
self.has_line_constraint = any( a or b for a,b in (self.min_line_constraint, self.max_line_constraint))
if not tranche.marketOrder and inverted:
self.min_line_constraint, self.max_line_constraint = self.max_line_constraint, self.min_line_constraint
self.slippage = tranche.minIntercept if tranche.marketOrder else 0
# compute status and set relevant triggers
if tranche_remaining <= 0:
self.status = TrancheStatus.Filled
return
for c in tranche.constraints:
if c.mode == ConstraintMode.Time:
c: TimeConstraint
earliest = c.earliest.timestamp(start)
latest = c.latest.timestamp(start)
self.time_constraint = (earliest, latest) if self.time_constraint is None else intersect_ranges(*self.time_constraint, earliest, latest)
elif c.mode == ConstraintMode.Line:
c: LineConstraint
self.line_constraints.append(c)
else:
raise NotImplementedError
if self.time_constraint is None:
self.status = TrancheStatus.Pricing
else:
timestamp = current_block.get().timestamp
earliest, latest = self.time_constraint
self.status = TrancheStatus.Early if timestamp < earliest else TrancheStatus.Expired if timestamp > latest else TrancheStatus.Pricing
timestamp = current_block.get().timestamp
self.status = TrancheStatus.Early if timestamp < tranche.startTime else TrancheStatus.Expired if timestamp > tranche.endTime else TrancheStatus.Pricing
self.enable_time_trigger()
if self.status == TrancheStatus.Pricing:
self.enable_price_trigger()
@@ -131,14 +129,14 @@ class TrancheTrigger:
self.enable_price_trigger()
def enable_price_trigger(self):
if self.line_constraints:
if self.has_line_constraint:
price_triggers[self.order.pool_address].add(self.price_trigger)
new_price_triggers[self.order.pool_address].add(self.price_trigger)
else:
unconstrained_price_triggers.add(self.price_trigger)
def disable_price_trigger(self):
if self.line_constraints:
if self.has_line_constraint:
price_triggers[self.order.pool_address].remove(self.price_trigger)
else:
unconstrained_price_triggers.remove(self.price_trigger)
@@ -148,7 +146,9 @@ class TrancheTrigger:
if self.closed:
log.debug(f'price trigger ignored because trigger status is {self.status}')
return
if not self.line_constraints or all(await asyncio.gather(*[line_passes(lc, self.order.pool_address, cur) for lc in self.line_constraints])):
if not self.has_line_constraint or all(await asyncio.gather(
line_passes(self.min_line_constraint, True, cur),
line_passes(self.max_line_constraint, False, cur))):
active_tranches[self.tk] = None # or PriceProof(...)
def fill(self, _amount_in, _amount_out ):
@@ -180,10 +180,10 @@ class TrancheTrigger:
class OrderTriggers:
instances: dict[OrderKey, 'OrderTriggers'] = {}
def __init__(self, order: Order):
def __init__(self, order: Order, inverted: bool):
assert order.key not in OrderTriggers.instances
self.order = order
self.triggers = [TrancheTrigger(order, tk) for tk in self.order.tranche_keys]
self.triggers = [TrancheTrigger(order, tk, inverted) for tk in self.order.tranche_keys]
OrderTriggers.instances[order.key] = self
log.debug(f'created OrderTriggers for {order.key}')

54
src/dexorder/pool.py Normal file
View File

@@ -0,0 +1,54 @@
import asyncio
import logging
from typing import Optional
from dexorder import dec, db, current_w3
from dexorder.base.chain import current_chain
from dexorder.base.orderlib import Exchange
from dexorder.blockstate import BlockDict
from dexorder.blockstate.blockdata import K, V
from dexorder.database.model.pool import PoolModel
from dexorder.uniswap import UniswapV3Pool
log = logging.getLogger(__name__)
class Pool:
instances: dict[tuple[int,str], PoolModel] = {}
@staticmethod
async def get(address, chain=None) -> Optional[PoolModel]:
if not chain:
chain = current_chain.get()
key = (chain.chain_id, address)
found = Pool.instances.get(key)
if not found:
found = db.session.get(key)
if not found:
# todo other exchanges
t0, t1, fee = await asyncio.gather(UniswapV3Pool(address).token0(), UniswapV3Pool(address).token1(), UniswapV3Pool(address).fee())
found = PoolModel(chain=chain, address=address, exchange=Exchange.UniswapV3, base=t0, quote=t1, fee=fee)
db.session.add(found)
Pool.instances[key] = found
return found
class PoolPrices (BlockDict[str, dec]):
def __setitem__(self, item: K, value: V) -> None:
super().__setitem__(item, value)
new_pool_prices[item] = value
def pub_pool_price(k,v):
chain_id = current_chain.get().chain_id
return f'{chain_id}|{k}', 'p', (chain_id, k, str(v))
new_pool_prices: dict[str, dec] = {} # tracks which prices were set during the current block. cleared every block.
pool_prices: PoolPrices = PoolPrices('p', db=True, redis=True, pub=pub_pool_price, value2str=lambda d: f'{d:f}', str2value=dec)
async def ensure_pool_price(pool_addr):
if pool_addr not in pool_prices:
log.debug(f'querying price for pool {pool_addr}')
pool_prices[pool_addr] = await UniswapV3Pool(pool_addr).price()

7
src/dexorder/util/convert.py
View File

@@ -1,4 +1,5 @@
import math
import struct
def price_to_tick(p):
@@ -18,3 +19,9 @@ def from_fixed(value, decimals):
def tick_to_sqrtPriceX96(tick):
return round(math.sqrt(tick_to_price(tick)) * 2**96)
def encode_IEEE754(value: float) -> int:
return struct.pack('>I', struct.pack('>f', value))[0]
def decode_IEEE754(value: int) -> float:
return struct.unpack('>f', struct.pack('>I', value))[0]

2
test/test_blockstate.py
View File

@@ -1,4 +1,4 @@
from dexorder.base.blockstate import BlockState, BlockDict
from dexorder.blockstate import BlockState, BlockDict
from dexorder.database.model.block import Block
block_10 = Block(chain=1, height=10, hash=bytes.fromhex('10'), parent=bytes.fromhex('09'), data=None)