gas fees handler (incomplete)

src/dexorder/base/__init__.py

@@ -15,6 +15,9 @@ class TransactionRequest:
     type: str
 
     def __init__(self, type: str, key: Any):
+        """
+        key is used to deduplicate requests
+        """
         self.type = type
         self.key = key
 

src/dexorder/bin/main.py

@@ -13,7 +13,8 @@ from dexorder.contract import get_contract_event
 from dexorder.contract.dexorder import get_dexorder_contract
 from dexorder.event_handler import (init, dump_log, handle_vault_created, handle_order_placed,
                                     handle_transfer, handle_swap_filled, handle_order_canceled, handle_order_cancel_all,
-                                    handle_uniswap_swaps, handle_vault_impl_changed, update_metrics)
+                                    handle_uniswap_swaps, handle_vault_impl_changed, initialize_metrics)
+from dexorder.gas_fees import adjust_gas, handle_fees_changed, handle_fee_limits_changed
 from dexorder.memcache import memcache
 from dexorder.memcache.memcache_state import RedisState, publish_all
 from dexorder.order.executionhandler import handle_dexorderexecutions, execute_tranches
@@ -75,7 +76,11 @@ def setup_logevent_triggers(runner):
 
     runner.add_callback(end_trigger_updates)
     runner.add_callback(execute_tranches)
-    runner.add_callback(update_metrics)
+
+    # fee adjustments are handled offline by batch jobs
+    # runner.add_event_trigger(handle_fee_limits_changed, get_contract_event('IFeeManager', 'FeeLimitsChanged'))
+    # runner.add_event_trigger(handle_fees_changed, get_contract_event('IFeeManager', 'FeesChanged'))
+    # runner.add_callback(adjust_gas)
 
 
 # noinspection DuplicatedCode
@@ -110,6 +115,7 @@ async def main():
                 await redis_state.save(state.root_fork, state.diffs_by_branch[state.root_branch.id])
 
     await initialize_accounting()
+    initialize_metrics()
 
     runner = BlockStateRunner(state, publish_all=publish_all if redis_state else None)
     setup_logevent_triggers(runner)

src/dexorder/configuration/schema.py

@@ -33,6 +33,12 @@ class Config:
     backfill: int = 0  # if not 0, then runner will initialize an empty database by backfilling from the given block height. Use negative numbers to indicate a number of blocks before the present.
 
     accounts: list[str] = field(default_factory=list) # the pool of accounts is used round-robin
+    adjuster: Optional[str] = None # special account allowed to adjust fees. must NOT be listed in accounts.
+    order_gas: int = 425000  # cost to place a conditional order
+    execution_gas: int = 275000  # cost to perform a successful execution
+    order_gas_multiplier: float = 2.0  # multiply the gas amount by this to get the fee
+    exeution_gas_multiplier: float = 2.0  # multiply the gas amount by this to get the fee
+    fee_leeway = 0.1  # do not adjust fees if they are within this proportion
     min_gas: str = '0'
 
     # Order slashing

src/dexorder/database/model/accounting.py

@@ -32,6 +32,7 @@ class AccountingSubcategory (Enum):
     # Expense
     VaultCreation = 3
     Execution = 4
+    FeeAdjustment = 5  # includes adjusting fee limits
 
     # Transfer
     # Transfers have no subcategories, but the note field will be the address of the other account. Both a debit and a

src/dexorder/event_handler.py

@@ -3,7 +3,7 @@ import logging
 
 from web3.types import EventData
 
-from dexorder import db, metric
+from dexorder import db, metric, current_w3
 from dexorder.accounting import accounting_fill, accounting_placement, accounting_transfer, is_tracked_address, \
     accounting_lock
 from dexorder.base.chain import current_chain
@@ -84,7 +84,7 @@ async def handle_swap_filled(event: EventData):
         log.warning(f'DexorderSwapFilled IGNORED due to missing order {vault} {order_index}')
         return
     value = await accounting_fill(event, order.order.tokenOut)
-    metric.volume.inc(value)
+    metric.volume.inc(float(value))
     order.status.trancheStatus[tranche_index].activationTime = next_execution_time  # update rate limit
     try:
         triggers = OrderTriggers.instances[order.key]
@@ -213,8 +213,13 @@ async def handle_vault_impl_changed(upgrade: EventData):
     log.debug(f'Vault {addr} upgraded to impl version {version}')
 
 
-def update_metrics():
+async def get_gas_price():
+    return await current_w3.get().eth.gas_price
+
+
+def initialize_metrics():
     metric.vaults.set_function(vault_owners.upper_len)
     metric.open_orders.set_function(Order.open_orders.upper_len)
     metric.triggers_time.set_function(lambda: len(TimeTrigger.all))
     metric.triggers_line.set_function(lambda: len(PriceLineTrigger.triggers_set))
+    metric.gas_price.set_function(get_gas_price)

src/dexorder/gas_fees.py

@@ -0,0 +1,163 @@
+import logging
+import math
+from dataclasses import dataclass
+from typing import Optional
+
+import eth_account
+from web3.types import EventData
+
+from dexorder import current_w3, config, Account
+from dexorder.accounting import accounting_transaction_gas
+from dexorder.alert import warningAlert
+from dexorder.base import TransactionReceiptDict, TransactionRequest
+from dexorder.contract.contract_proxy import ContractTransaction
+from dexorder.contract.dexorder import get_fee_manager_contract
+from dexorder.database.model import TransactionJob
+from dexorder.database.model.accounting import AccountingSubcategory
+from dexorder.transactions import TransactionHandler, submit_transaction_request
+from dexorder.util.convert import to_base_exp
+
+log = logging.getLogger(__name__)
+
+order_fee: Optional[int] = None
+gas_fee: Optional[int] = None
+fill_fee_half_bps: Optional[int] = None
+
+order_fee_limit: Optional[int] = None
+gas_fee_limit: Optional[int] = None
+fill_fee_half_bps_limit: Optional[int] = None
+
+adjuster_account: Optional[Account] = None
+adjuster_locked = False
+
+
+@dataclass
+class AdjustFeeTransactionRequest (TransactionRequest):
+    TYPE = 'adjust'
+
+    order_fee: int
+    order_exp: int
+    gas_fee: int
+    gas_exp:int
+    fill_fee_half_bps: int
+
+    # noinspection PyShadowingNames
+    def __init__(self, order_fee: int, order_exp: int, gas_fee: int, gas_exp:int, fill_fee_half_bps: int):
+        super().__init__(AdjustFeeTransactionRequest.TYPE, (order_fee, order_exp, gas_fee, gas_exp, fill_fee_half_bps))
+        self.order_fee = order_fee
+        self.order_exp = order_exp
+        self.gas_fee = gas_fee
+        self.gas_exp = gas_exp
+        self.fill_fee_half_bps = fill_fee_half_bps
+
+    @property
+    def schedule(self):
+        return self.order_fee, self.order_exp, self.gas_fee, self.gas_exp, self.fill_fee_half_bps
+
+
+class AdjustFeeTransactionHandler (TransactionHandler):
+
+    async def build_transaction(self, job_id: int, tr: TransactionRequest) -> Optional[ContractTransaction]:
+        tr: AdjustFeeTransactionRequest
+        fee_manager = await get_fee_manager_contract()
+        return await fee_manager.build.setFees(tr.schedule)
+
+    async def complete_transaction(self, job: TransactionJob, receipt: TransactionReceiptDict) -> None:
+        await accounting_transaction_gas(receipt, AccountingSubcategory.FeeAdjustment)  # vault creation gas
+
+    async def transaction_exception(self, job: TransactionJob, e: Exception) -> None:
+        pass
+
+    async def acquire_account(self) -> Optional[Account]:
+        global adjuster_account, adjuster_locked
+        if adjuster_locked:
+            return None
+        if config.adjuster is None:
+            return None
+        if adjuster_account is None:
+            local_account = eth_account.Account.from_key(config.adjuster)
+            adjuster_account = Account(local_account)
+        adjuster_locked = True
+        return adjuster_account
+
+    async def release_account(self, account: Account):
+        global adjuster_locked
+        adjuster_locked = False
+
+
+async def ensure_gas_fee_data():
+    global order_fee, gas_fee, fill_fee_half_bps, order_fee_limit, gas_fee_limit, fill_fee_half_bps_limit
+    if order_fee is None or gas_fee is None or order_fee_limit is None or gas_fee_limit is None:
+        fee_manager = await get_fee_manager_contract()
+        order_fee_base, order_fee_exp, gas_fee_base, gas_fee_exp, fill_fee_half_bps = await fee_manager.fees()
+        order_fee = order_fee_base << order_fee_exp
+        gas_fee = gas_fee_base << gas_fee_exp
+        order_fee_base_limit, order_fee_exp_limit, gas_fee_base_limit, gas_fee_exp_limit, fill_fee_half_bps_limit = await fee_manager.fee_limits()
+        order_fee_limit = order_fee_base_limit << order_fee_exp_limit
+        gas_fee_limit = gas_fee_base_limit << gas_fee_exp_limit
+        return fee_manager
+    return None
+
+
+async def adjust_gas():
+    if not config.adjuster:
+        return
+    w3 = current_w3.get()
+    price = await w3.eth.gas_price
+    new_order_fee = round(config.order_gas * config.order_gas_multiplier * price)
+    new_gas_fee = round(config.order_gas * config.order_gas_multiplier * price)
+    log.debug(f'avg gas price: {price/10**18}')
+    await ensure_gas_fee_data()
+    global order_fee, gas_fee
+    if abs(1 - new_order_fee / order_fee) >= config.fee_leeway or abs(1 - new_gas_fee / gas_fee) >= config.fee_leeway:
+        if new_order_fee > order_fee_limit or new_gas_fee > gas_fee_limit:
+            warningAlert('Fees Hit Limits', 'Adjusting fees would exceed existing fee limits.')
+            new_order_fee = min(order_fee_limit, new_order_fee)
+            new_gas_fee = min(gas_fee_limit, new_gas_fee)
+        # TODO check if the new fee is adjusting upwards too fast and cap it
+        # TODO check if the new fees are already proposed and pending
+        # if new_order_fee/order_fee - 1 >
+        if new_order_fee != order_fee or new_gas_fee != gas_fee:
+            log.info(f'adjusting gas fees: orderFee={new_order_fee/10**18}, gasFee={new_gas_fee/10**18}')
+            new_order_fee_base, new_order_fee_exp = to_base_exp(new_order_fee, math.floor)
+            new_gas_fee_base, new_gas_fee_exp = to_base_exp(new_gas_fee, math.floor)
+            req = AdjustFeeTransactionRequest(new_order_fee_base, new_order_fee_exp,
+                                              new_gas_fee_base, new_gas_fee_exp, fill_fee_half_bps)
+            submit_transaction_request(req)
+
+
+# noinspection DuplicatedCode
+async def handle_fee_limits_changed(event: EventData):
+    try:
+        fees = event['args']['fees']
+        new_order_fee_limit = fees['orderFee']
+        new_order_exp_limit = fees['orderExp']
+        new_gas_fee_limit = fees['gasFee']
+        new_gas_exp_limit = fees['gasExp']
+        new_fill_fee_half_bps_limit = fees['fillFeeHalfBps']
+    except KeyError:
+        return
+    global order_fee_limit, gas_fee_limit
+    order_fee_limit = new_order_fee_limit << new_order_exp_limit
+    gas_fee_limit = new_gas_fee_limit << new_gas_exp_limit
+    fill_fee_limit = new_fill_fee_half_bps_limit / 200
+    log.info(f'gas fee limits updated: orderFeeLimit={new_order_fee_limit/10**18}, gasFeeLimit={new_gas_fee_limit/10**18}, fillFeeLimit={fill_fee_limit:.3%}')
+
+
+# noinspection DuplicatedCode
+async def handle_fees_changed(event: EventData):
+    try:
+        fees = event['args']['fees']
+        new_order_fee = fees['orderFee']
+        new_order_exp = fees['orderExp']
+        new_gas_fee = fees['gasFee']
+        new_gas_exp = fees['gasExp']
+        new_fill_fee_half_bps = fees['fillFeeHalfBps']
+    except KeyError:
+        return
+    global order_fee, gas_fee
+    order_fee = new_order_fee << new_order_exp
+    gas_fee = new_gas_fee << new_gas_exp
+    fill_fee = new_fill_fee_half_bps / 200
+    log.info(f'gas fees updated: orderFee={new_order_fee/10**18}, gasFee={new_gas_fee/10**18}, fillFee={fill_fee:.3%}')
+

src/dexorder/metric/metrics.py

@@ -21,3 +21,5 @@ volume = Counter("volume", "Total volume of successful executions in USD")
 
 account_total = Gauge('account_total', 'Total number of accounts configured')
 account_available = Gauge('account_available', 'Number of accounts that do not have any pending transactions')
+
+gas_price = Gauge('gas_price', 'Gas price in wei')

src/dexorder/order/executionhandler.py

@@ -219,7 +219,11 @@ def create_execution_request(tk: TrancheKey, proof: PriceProof):
 def handle_dexorderexecutions(event: EventData):
     log.debug(f'executions {event}')
     exe_id = UUID(bytes=event['args']['id'])
-    errors = event['args']['errors']
+    try:
+        errors = event['args']['errors']
+    except KeyError:
+        log.warning(f'Rogue DexorderExecutions event {event}')
+        return
     if len(errors) == 0:
         log.warning(f'No errors found in DexorderExecutions event: {event}')
         return

src/dexorder/transactions.py

@@ -39,6 +39,18 @@ class TransactionHandler:
     @abstractmethod
     async def transaction_exception(self, job: TransactionJob, e: Exception) -> None: ...
 
+    # noinspection PyMethodMayBeStatic
+    async def acquire_account(self) -> Optional[Account]:
+        try:
+            async with asyncio.timeout(1):
+                return await Account.acquire()
+        except asyncio.TimeoutError:
+            return None
+
+    # noinspection PyMethodMayBeStatic
+    async def release_account(self, account: Account):
+        account.release()
+
 
 in_flight = set()
 accounts_in_flight: dict[bytes, Account] = {} # tx_id_bytes: account
@@ -95,11 +107,7 @@ async def create_and_send_transactions():
             in_flight.discard((job.request.type, job.request.key))
             return
         w3 = current_w3.get()
-        try:
-            async with asyncio.timeout(1):
-                account = await Account.acquire()
-        except asyncio.TimeoutError:
-            account = None
+        account = await handler.acquire_account()
         if account is None:
             log.warning(f'No account available for job {job.id} type "{handler.tag}"')
             continue
@@ -115,12 +123,12 @@ async def create_and_send_transactions():
                 account.reset_nonce()
             else:
                 log.exception(f'Failure sending transaction for job {job.id}')
-            account.release()
+            await handler.release_account(account)
         except:
             log.exception(f'Failure sending transaction for job {job.id}')
             # todo pager
             # todo send state unknown!
-            account.release()
+            await handler.release_account(account)
         else:
             account.tx_id = hexstr(ctx.id_bytes)
             accounts_in_flight[ctx.id_bytes] = account
@@ -154,11 +162,11 @@ async def handle_transaction_receipts():
                 log.warning(f'ignoring transaction request with bad type "{job.request.type}"')
             else:
                 await handler.complete_transaction(job, receipt)
+                try:
+                    await handler.release_account(accounts_in_flight.pop(job.tx_id))
+                except KeyError:
+                    pass
         in_flight.discard((job.request.type, job.request.key))
-        try:
-            accounts_in_flight.pop(job.tx_id).release()
-        except KeyError:
-            pass
 
 
 def finalize_transactions(_fork: Fork, diffs: list[DiffEntryItem]):

src/dexorder/util/convert.py

@@ -25,3 +25,22 @@ def encode_IEEE754(value: float) -> int:
 
 def decode_IEEE754(value: int) -> float:
     return struct.unpack('>f', struct.pack('>I', value))[0]
+
+
+def to_base_exp(value, precision=8, roundingFunc=round) -> tuple[int, int]:
+    """
+    Convert a value to base-2 exponent form.
+    Precision is the number of bits available to the base component
+    """
+    if value <= 0:
+        raise ValueError("Value must be greater than zero")
+    max_base = 2 ** precision
+    exp = int(math.log2(value)) - precision
+    base = roundingFunc(value / (2 ** exp))
+    if base >= max_base:
+        base //= 2
+        exp += 1
+    return base, exp
+
+def from_base_exp(base, exp):
+    return base << exp