From 44d1c4a920e087ca308119859b650e0b23ba184a Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 21 Feb 2025 23:40:14 -0400 Subject: [PATCH] gas fees handler (incomplete) --- ...010f13e8b_accounting_vaultcreation_ofac.py | 2 +- conf/arb1/dexorder-arb1.toml | 8 +- src/dexorder/base/__init__.py | 3 + src/dexorder/bin/main.py | 10 +- src/dexorder/configuration/schema.py | 6 + src/dexorder/database/model/accounting.py | 1 + src/dexorder/event_handler.py | 11 +- src/dexorder/gas_fees.py | 163 ++++++++++++++++++ src/dexorder/metric/metrics.py | 2 + src/dexorder/order/executionhandler.py | 6 +- src/dexorder/transactions.py | 30 ++-- src/dexorder/util/convert.py | 19 ++ 12 files changed, 239 insertions(+), 22 deletions(-) create mode 100644 src/dexorder/gas_fees.py diff --git a/alembic/versions/509010f13e8b_accounting_vaultcreation_ofac.py b/alembic/versions/509010f13e8b_accounting_vaultcreation_ofac.py index 0791584..c05e96f 100644 --- a/alembic/versions/509010f13e8b_accounting_vaultcreation_ofac.py +++ b/alembic/versions/509010f13e8b_accounting_vaultcreation_ofac.py @@ -28,7 +28,7 @@ def upgrade() -> None: sa.Column('time', sa.DateTime(), nullable=False), sa.Column('account', sa.String(), nullable=False), sa.Column('category', sa.Enum('Transfer', 'Income', 'Expense', 'Trade', 'Special', name='accountingcategory'), nullable=False), - sa.Column('subcategory', sa.Enum('OrderFee', 'GasFee', 'FillFee', 'VaultCreation', 'Execution', 'InitialBalance', name='accountingsubcategory'), nullable=True), + sa.Column('subcategory', sa.Enum('OrderFee', 'GasFee', 'FillFee', 'VaultCreation', 'Execution', 'FeeAdjustment', 'InitialBalance', name='accountingsubcategory'), nullable=True), sa.Column('token', sa.String(), nullable=False), sa.Column('amount', dexorder.database.column_types.DecimalNumeric(), nullable=False), sa.Column('value', dexorder.database.column_types.DecimalNumeric(), nullable=True), diff --git a/conf/arb1/dexorder-arb1.toml b/conf/arb1/dexorder-arb1.toml index c102cfb..b5a62d9 100644 --- a/conf/arb1/dexorder-arb1.toml +++ b/conf/arb1/dexorder-arb1.toml @@ -1,5 +1,5 @@ metadata = '' # this setting approves no tokens -account = '${accounts.gas}' -rpc_url = '${rpc_urls.arbitrum_alchemy}' -ws_url = '${rpc_urls.arbitrum_alchemy_ws}' -concurrent_rpc_connections=100 \ No newline at end of file +rpc_url = 'arbitrum_dxod' +archive_url = 'arbitrum_alchemy' +ws_url = 'arbitrum_dxod_ws' +concurrent_rpc_connections=20 diff --git a/src/dexorder/base/__init__.py b/src/dexorder/base/__init__.py index cd62662..74a0e17 100644 --- a/src/dexorder/base/__init__.py +++ b/src/dexorder/base/__init__.py @@ -15,6 +15,9 @@ class TransactionRequest: type: str def __init__(self, type: str, key: Any): + """ + key is used to deduplicate requests + """ self.type = type self.key = key diff --git a/src/dexorder/bin/main.py b/src/dexorder/bin/main.py index 5fcb283..70aa62d 100644 --- a/src/dexorder/bin/main.py +++ b/src/dexorder/bin/main.py @@ -13,7 +13,8 @@ from dexorder.contract import get_contract_event from dexorder.contract.dexorder import get_dexorder_contract from dexorder.event_handler import (init, dump_log, handle_vault_created, handle_order_placed, handle_transfer, handle_swap_filled, handle_order_canceled, handle_order_cancel_all, - handle_uniswap_swaps, handle_vault_impl_changed, update_metrics) + handle_uniswap_swaps, handle_vault_impl_changed, initialize_metrics) +from dexorder.gas_fees import adjust_gas, handle_fees_changed, handle_fee_limits_changed from dexorder.memcache import memcache from dexorder.memcache.memcache_state import RedisState, publish_all from dexorder.order.executionhandler import handle_dexorderexecutions, execute_tranches @@ -75,7 +76,11 @@ def setup_logevent_triggers(runner): runner.add_callback(end_trigger_updates) runner.add_callback(execute_tranches) - runner.add_callback(update_metrics) + + # fee adjustments are handled offline by batch jobs + # runner.add_event_trigger(handle_fee_limits_changed, get_contract_event('IFeeManager', 'FeeLimitsChanged')) + # runner.add_event_trigger(handle_fees_changed, get_contract_event('IFeeManager', 'FeesChanged')) + # runner.add_callback(adjust_gas) # noinspection DuplicatedCode @@ -110,6 +115,7 @@ async def main(): await redis_state.save(state.root_fork, state.diffs_by_branch[state.root_branch.id]) await initialize_accounting() + initialize_metrics() runner = BlockStateRunner(state, publish_all=publish_all if redis_state else None) setup_logevent_triggers(runner) diff --git a/src/dexorder/configuration/schema.py b/src/dexorder/configuration/schema.py index 3c3bc3c..8d389e5 100644 --- a/src/dexorder/configuration/schema.py +++ b/src/dexorder/configuration/schema.py @@ -33,6 +33,12 @@ class Config: backfill: int = 0 # if not 0, then runner will initialize an empty database by backfilling from the given block height. Use negative numbers to indicate a number of blocks before the present. accounts: list[str] = field(default_factory=list) # the pool of accounts is used round-robin + adjuster: Optional[str] = None # special account allowed to adjust fees. must NOT be listed in accounts. + order_gas: int = 425000 # cost to place a conditional order + execution_gas: int = 275000 # cost to perform a successful execution + order_gas_multiplier: float = 2.0 # multiply the gas amount by this to get the fee + exeution_gas_multiplier: float = 2.0 # multiply the gas amount by this to get the fee + fee_leeway = 0.1 # do not adjust fees if they are within this proportion min_gas: str = '0' # Order slashing diff --git a/src/dexorder/database/model/accounting.py b/src/dexorder/database/model/accounting.py index 68b8891..8630f61 100644 --- a/src/dexorder/database/model/accounting.py +++ b/src/dexorder/database/model/accounting.py @@ -32,6 +32,7 @@ class AccountingSubcategory (Enum): # Expense VaultCreation = 3 Execution = 4 + FeeAdjustment = 5 # includes adjusting fee limits # Transfer # Transfers have no subcategories, but the note field will be the address of the other account. Both a debit and a diff --git a/src/dexorder/event_handler.py b/src/dexorder/event_handler.py index d62cbfa..05e8af0 100644 --- a/src/dexorder/event_handler.py +++ b/src/dexorder/event_handler.py @@ -3,7 +3,7 @@ import logging from web3.types import EventData -from dexorder import db, metric +from dexorder import db, metric, current_w3 from dexorder.accounting import accounting_fill, accounting_placement, accounting_transfer, is_tracked_address, \ accounting_lock from dexorder.base.chain import current_chain @@ -84,7 +84,7 @@ async def handle_swap_filled(event: EventData): log.warning(f'DexorderSwapFilled IGNORED due to missing order {vault} {order_index}') return value = await accounting_fill(event, order.order.tokenOut) - metric.volume.inc(value) + metric.volume.inc(float(value)) order.status.trancheStatus[tranche_index].activationTime = next_execution_time # update rate limit try: triggers = OrderTriggers.instances[order.key] @@ -213,8 +213,13 @@ async def handle_vault_impl_changed(upgrade: EventData): log.debug(f'Vault {addr} upgraded to impl version {version}') -def update_metrics(): +async def get_gas_price(): + return await current_w3.get().eth.gas_price + + +def initialize_metrics(): metric.vaults.set_function(vault_owners.upper_len) metric.open_orders.set_function(Order.open_orders.upper_len) metric.triggers_time.set_function(lambda: len(TimeTrigger.all)) metric.triggers_line.set_function(lambda: len(PriceLineTrigger.triggers_set)) + metric.gas_price.set_function(get_gas_price) diff --git a/src/dexorder/gas_fees.py b/src/dexorder/gas_fees.py new file mode 100644 index 0000000..2e45418 --- /dev/null +++ b/src/dexorder/gas_fees.py @@ -0,0 +1,163 @@ +import logging +import math +from dataclasses import dataclass +from typing import Optional + +import eth_account +from web3.types import EventData + +from dexorder import current_w3, config, Account +from dexorder.accounting import accounting_transaction_gas +from dexorder.alert import warningAlert +from dexorder.base import TransactionReceiptDict, TransactionRequest +from dexorder.contract.contract_proxy import ContractTransaction +from dexorder.contract.dexorder import get_fee_manager_contract +from dexorder.database.model import TransactionJob +from dexorder.database.model.accounting import AccountingSubcategory +from dexorder.transactions import TransactionHandler, submit_transaction_request +from dexorder.util.convert import to_base_exp + +log = logging.getLogger(__name__) + +order_fee: Optional[int] = None +gas_fee: Optional[int] = None +fill_fee_half_bps: Optional[int] = None + +order_fee_limit: Optional[int] = None +gas_fee_limit: Optional[int] = None +fill_fee_half_bps_limit: Optional[int] = None + +adjuster_account: Optional[Account] = None +adjuster_locked = False + + +@dataclass +class AdjustFeeTransactionRequest (TransactionRequest): + TYPE = 'adjust' + + order_fee: int + order_exp: int + gas_fee: int + gas_exp:int + fill_fee_half_bps: int + + # noinspection PyShadowingNames + def __init__(self, order_fee: int, order_exp: int, gas_fee: int, gas_exp:int, fill_fee_half_bps: int): + super().__init__(AdjustFeeTransactionRequest.TYPE, (order_fee, order_exp, gas_fee, gas_exp, fill_fee_half_bps)) + self.order_fee = order_fee + self.order_exp = order_exp + self.gas_fee = gas_fee + self.gas_exp = gas_exp + self.fill_fee_half_bps = fill_fee_half_bps + + @property + def schedule(self): + return self.order_fee, self.order_exp, self.gas_fee, self.gas_exp, self.fill_fee_half_bps + + +class AdjustFeeTransactionHandler (TransactionHandler): + + async def build_transaction(self, job_id: int, tr: TransactionRequest) -> Optional[ContractTransaction]: + tr: AdjustFeeTransactionRequest + fee_manager = await get_fee_manager_contract() + return await fee_manager.build.setFees(tr.schedule) + + async def complete_transaction(self, job: TransactionJob, receipt: TransactionReceiptDict) -> None: + await accounting_transaction_gas(receipt, AccountingSubcategory.FeeAdjustment) # vault creation gas + + async def transaction_exception(self, job: TransactionJob, e: Exception) -> None: + pass + + async def acquire_account(self) -> Optional[Account]: + global adjuster_account, adjuster_locked + if adjuster_locked: + return None + if config.adjuster is None: + return None + if adjuster_account is None: + local_account = eth_account.Account.from_key(config.adjuster) + adjuster_account = Account(local_account) + adjuster_locked = True + return adjuster_account + + async def release_account(self, account: Account): + global adjuster_locked + adjuster_locked = False + + +async def ensure_gas_fee_data(): + global order_fee, gas_fee, fill_fee_half_bps, order_fee_limit, gas_fee_limit, fill_fee_half_bps_limit + if order_fee is None or gas_fee is None or order_fee_limit is None or gas_fee_limit is None: + fee_manager = await get_fee_manager_contract() + order_fee_base, order_fee_exp, gas_fee_base, gas_fee_exp, fill_fee_half_bps = await fee_manager.fees() + order_fee = order_fee_base << order_fee_exp + gas_fee = gas_fee_base << gas_fee_exp + order_fee_base_limit, order_fee_exp_limit, gas_fee_base_limit, gas_fee_exp_limit, fill_fee_half_bps_limit = await fee_manager.fee_limits() + order_fee_limit = order_fee_base_limit << order_fee_exp_limit + gas_fee_limit = gas_fee_base_limit << gas_fee_exp_limit + return fee_manager + return None + + +async def adjust_gas(): + if not config.adjuster: + return + w3 = current_w3.get() + price = await w3.eth.gas_price + new_order_fee = round(config.order_gas * config.order_gas_multiplier * price) + new_gas_fee = round(config.order_gas * config.order_gas_multiplier * price) + log.debug(f'avg gas price: {price/10**18}') + await ensure_gas_fee_data() + global order_fee, gas_fee + if abs(1 - new_order_fee / order_fee) >= config.fee_leeway or abs(1 - new_gas_fee / gas_fee) >= config.fee_leeway: + if new_order_fee > order_fee_limit or new_gas_fee > gas_fee_limit: + warningAlert('Fees Hit Limits', 'Adjusting fees would exceed existing fee limits.') + new_order_fee = min(order_fee_limit, new_order_fee) + new_gas_fee = min(gas_fee_limit, new_gas_fee) + # TODO check if the new fee is adjusting upwards too fast and cap it + # TODO check if the new fees are already proposed and pending + # if new_order_fee/order_fee - 1 > + if new_order_fee != order_fee or new_gas_fee != gas_fee: + log.info(f'adjusting gas fees: orderFee={new_order_fee/10**18}, gasFee={new_gas_fee/10**18}') + new_order_fee_base, new_order_fee_exp = to_base_exp(new_order_fee, math.floor) + new_gas_fee_base, new_gas_fee_exp = to_base_exp(new_gas_fee, math.floor) + req = AdjustFeeTransactionRequest(new_order_fee_base, new_order_fee_exp, + new_gas_fee_base, new_gas_fee_exp, fill_fee_half_bps) + submit_transaction_request(req) + + +# noinspection DuplicatedCode +async def handle_fee_limits_changed(event: EventData): + try: + fees = event['args']['fees'] + new_order_fee_limit = fees['orderFee'] + new_order_exp_limit = fees['orderExp'] + new_gas_fee_limit = fees['gasFee'] + new_gas_exp_limit = fees['gasExp'] + new_fill_fee_half_bps_limit = fees['fillFeeHalfBps'] + except KeyError: + return + global order_fee_limit, gas_fee_limit + order_fee_limit = new_order_fee_limit << new_order_exp_limit + gas_fee_limit = new_gas_fee_limit << new_gas_exp_limit + fill_fee_limit = new_fill_fee_half_bps_limit / 200 + log.info(f'gas fee limits updated: orderFeeLimit={new_order_fee_limit/10**18}, gasFeeLimit={new_gas_fee_limit/10**18}, fillFeeLimit={fill_fee_limit:.3%}') + + +# noinspection DuplicatedCode +async def handle_fees_changed(event: EventData): + try: + fees = event['args']['fees'] + new_order_fee = fees['orderFee'] + new_order_exp = fees['orderExp'] + new_gas_fee = fees['gasFee'] + new_gas_exp = fees['gasExp'] + new_fill_fee_half_bps = fees['fillFeeHalfBps'] + except KeyError: + return + global order_fee, gas_fee + order_fee = new_order_fee << new_order_exp + gas_fee = new_gas_fee << new_gas_exp + fill_fee = new_fill_fee_half_bps / 200 + log.info(f'gas fees updated: orderFee={new_order_fee/10**18}, gasFee={new_gas_fee/10**18}, fillFee={fill_fee:.3%}') + diff --git a/src/dexorder/metric/metrics.py b/src/dexorder/metric/metrics.py index 96714d6..bdf5664 100644 --- a/src/dexorder/metric/metrics.py +++ b/src/dexorder/metric/metrics.py @@ -21,3 +21,5 @@ volume = Counter("volume", "Total volume of successful executions in USD") account_total = Gauge('account_total', 'Total number of accounts configured') account_available = Gauge('account_available', 'Number of accounts that do not have any pending transactions') + +gas_price = Gauge('gas_price', 'Gas price in wei') diff --git a/src/dexorder/order/executionhandler.py b/src/dexorder/order/executionhandler.py index 190c4ae..76913e3 100644 --- a/src/dexorder/order/executionhandler.py +++ b/src/dexorder/order/executionhandler.py @@ -219,7 +219,11 @@ def create_execution_request(tk: TrancheKey, proof: PriceProof): def handle_dexorderexecutions(event: EventData): log.debug(f'executions {event}') exe_id = UUID(bytes=event['args']['id']) - errors = event['args']['errors'] + try: + errors = event['args']['errors'] + except KeyError: + log.warning(f'Rogue DexorderExecutions event {event}') + return if len(errors) == 0: log.warning(f'No errors found in DexorderExecutions event: {event}') return diff --git a/src/dexorder/transactions.py b/src/dexorder/transactions.py index 63eb07d..0a5c966 100644 --- a/src/dexorder/transactions.py +++ b/src/dexorder/transactions.py @@ -39,6 +39,18 @@ class TransactionHandler: @abstractmethod async def transaction_exception(self, job: TransactionJob, e: Exception) -> None: ... + # noinspection PyMethodMayBeStatic + async def acquire_account(self) -> Optional[Account]: + try: + async with asyncio.timeout(1): + return await Account.acquire() + except asyncio.TimeoutError: + return None + + # noinspection PyMethodMayBeStatic + async def release_account(self, account: Account): + account.release() + in_flight = set() accounts_in_flight: dict[bytes, Account] = {} # tx_id_bytes: account @@ -95,11 +107,7 @@ async def create_and_send_transactions(): in_flight.discard((job.request.type, job.request.key)) return w3 = current_w3.get() - try: - async with asyncio.timeout(1): - account = await Account.acquire() - except asyncio.TimeoutError: - account = None + account = await handler.acquire_account() if account is None: log.warning(f'No account available for job {job.id} type "{handler.tag}"') continue @@ -115,12 +123,12 @@ async def create_and_send_transactions(): account.reset_nonce() else: log.exception(f'Failure sending transaction for job {job.id}') - account.release() + await handler.release_account(account) except: log.exception(f'Failure sending transaction for job {job.id}') # todo pager # todo send state unknown! - account.release() + await handler.release_account(account) else: account.tx_id = hexstr(ctx.id_bytes) accounts_in_flight[ctx.id_bytes] = account @@ -154,11 +162,11 @@ async def handle_transaction_receipts(): log.warning(f'ignoring transaction request with bad type "{job.request.type}"') else: await handler.complete_transaction(job, receipt) + try: + await handler.release_account(accounts_in_flight.pop(job.tx_id)) + except KeyError: + pass in_flight.discard((job.request.type, job.request.key)) - try: - accounts_in_flight.pop(job.tx_id).release() - except KeyError: - pass def finalize_transactions(_fork: Fork, diffs: list[DiffEntryItem]): diff --git a/src/dexorder/util/convert.py b/src/dexorder/util/convert.py index 4bbe183..e1e328b 100644 --- a/src/dexorder/util/convert.py +++ b/src/dexorder/util/convert.py @@ -25,3 +25,22 @@ def encode_IEEE754(value: float) -> int: def decode_IEEE754(value: int) -> float: return struct.unpack('>f', struct.pack('>I', value))[0] + + +def to_base_exp(value, precision=8, roundingFunc=round) -> tuple[int, int]: + """ + Convert a value to base-2 exponent form. + Precision is the number of bits available to the base component + """ + if value <= 0: + raise ValueError("Value must be greater than zero") + max_base = 2 ** precision + exp = int(math.log2(value)) - precision + base = roundingFunc(value / (2 ** exp)) + if base >= max_base: + base //= 2 + exp += 1 + return base, exp + +def from_base_exp(base, exp): + return base << exp