From a6cff51bf648e64e90b23d156d4e5848f3e0bc2c Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Tue, 30 Jul 2024 12:18:45 +0200 Subject: [PATCH] feat: SDK improvements Add a way to pull stateless contracts code from node, add more settings to test_assets.yaml, add logic to allow dynamic stateless contract by calling another contract --- substreams/ethereum-template/test_assets.yaml | 4 ++ testing/src/runner/runner.py | 10 +++- testing/src/runner/tycho.py | 13 +++-- testing/tycho-client/tycho_client/decoders.py | 56 ++++++++++++++----- .../tycho-client/tycho_client/pool_state.py | 2 +- testing/tycho-client/tycho_client/utils.py | 14 +++++ 6 files changed, 77 insertions(+), 22 deletions(-) diff --git a/substreams/ethereum-template/test_assets.yaml b/substreams/ethereum-template/test_assets.yaml index 0896e57..c8600bf 100644 --- a/substreams/ethereum-template/test_assets.yaml +++ b/substreams/ethereum-template/test_assets.yaml @@ -8,6 +8,8 @@ tests: - name: test_pool_creation start_block: 123 stop_block: 456 + initialized_accounts: + - "0x0c0e5f2fF0ff18a3be9b835635039256dC4B4963" # Needed for .... expected_state: protocol_components: - id: "0xbebc44782c7db0a1a60cb6fe97d0b483032ff1c7" @@ -17,6 +19,7 @@ tests: - "0x6b175474e89094c44da98b954eedeac495271d0f" static_attributes: creation_tx: "0x20793bbf260912aae189d5d261ff003c9b9166da8191d8f9d63ff1c7722f3ac6" + skip_simulation: false - name: test_something_else start_block: 123 stop_block: 456 @@ -28,3 +31,4 @@ tests: - "0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84" static_attributes: creation_tx: "0xfac67ecbd423a5b915deff06045ec9343568edaec34ae95c43d35f2c018afdaa" + skip_simulation: true # If true, always add a reason diff --git a/testing/src/runner/runner.py b/testing/src/runner/runner.py index 2e68dbb..262909e 100644 --- a/testing/src/runner/runner.py +++ b/testing/src/runner/runner.py @@ -61,6 +61,7 @@ class TestRunner: """Run all tests specified in the configuration.""" print(f"Running tests ...") for test in self.config["tests"]: + self.tycho_runner.empty_database(self.db_url) spkg_path = self.build_spkg( os.path.join(self.spkg_src, self.config["substreams_yaml_path"]), @@ -71,6 +72,7 @@ class TestRunner: test["start_block"], test["stop_block"], self.config["protocol_type_names"], + test.get("initialized_accounts", []), ) result = self.tycho_runner.run_with_rpc_server( @@ -83,7 +85,6 @@ class TestRunner: else: print(f"❗️ {test['name']} failed: {result.message}") - self.tycho_runner.empty_database(self.db_url) def validate_state(self, expected_state: dict, stop_block: int) -> TestResult: """Validate the current protocol state against the expected state.""" @@ -104,6 +105,8 @@ class TestRunner: component = components[comp_id] for key, value in expected_component.items(): + if key not in ["tokens", "static_attributes", "creation_tx"]: + continue if key not in component: return TestResult.Failed( f"Missing '{key}' in component '{comp_id}'." @@ -148,10 +151,13 @@ class TestRunner: f"from rpc call and {tycho_balance} from Substreams" ) contract_states = self.tycho_rpc_client.get_contract_state() + filtered_components = {'protocol_components': [pc for pc in protocol_components["protocol_components"] if + pc["id"] in [c["id"].lower() for c in + expected_state["protocol_components"] if c["skip_simulation"] is False]]} simulation_failures = self.simulate_get_amount_out( stop_block, protocol_states, - protocol_components, + filtered_components, contract_states, ) if len(simulation_failures): diff --git a/testing/src/runner/tycho.py b/testing/src/runner/tycho.py index d6e46cf..ec5331e 100644 --- a/testing/src/runner/tycho.py +++ b/testing/src/runner/tycho.py @@ -83,12 +83,15 @@ class TychoRunner: start_block: int, end_block: int, protocol_type_names: list, + initialized_accounts: list, ) -> None: """Run the Tycho indexer with the specified SPKG and block range.""" env = os.environ.copy() env["RUST_LOG"] = "tycho_indexer=info" + all_accounts = self._initialized_accounts + initialized_accounts + try: process = subprocess.Popen( [ @@ -106,16 +109,14 @@ class TychoRunner: str(start_block), "--stop-block", # +2 is to make up for the cache in the index side. - str(end_block + 2), - "--initialized-accounts", - ",".join(self._initialized_accounts) - ], + str(end_block + 2) + ] + (["--initialized-accounts", ",".join(all_accounts)] if all_accounts else []), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1, env=env, - ) + ) with process.stdout: for line in iter(process.stdout.readline, ""): @@ -203,7 +204,7 @@ class TychoRunner: def empty_database(db_url: str) -> None: """Drop and recreate the Tycho indexer database.""" try: - conn = psycopg2.connect(db_url) + conn = psycopg2.connect(db_url[:db_url.rfind('/')]) conn.autocommit = True cursor = conn.cursor() diff --git a/testing/tycho-client/tycho_client/decoders.py b/testing/tycho-client/tycho_client/decoders.py index c5de40a..079364a 100644 --- a/testing/tycho-client/tycho_client/decoders.py +++ b/testing/tycho-client/tycho_client/decoders.py @@ -1,11 +1,18 @@ +import time from decimal import Decimal from logging import getLogger from typing import Any +import eth_abi +from eth_utils import keccak +from protosim_py import SimulationEngine, SimulationParameters, AccountInfo + +from .constants import EXTERNAL_ACCOUNT from .exceptions import TychoDecodeError from .models import EVMBlock, EthereumToken from .pool_state import ThirdPartyPool -from .utils import decode_tycho_exchange +from .tycho_db import TychoDBSingleton +from .utils import decode_tycho_exchange, get_code_for_address log = getLogger(__name__) @@ -49,7 +56,7 @@ class ThirdPartyPoolTychoDecoder: raise TychoDecodeError("Unsupported token", pool_id=component["id"]) balances = self.decode_balances(snap, tokens) - optional_attributes = self.decode_optional_attributes(component, snap) + optional_attributes = self.decode_optional_attributes(component, snap, block.id) return ThirdPartyPool( id_=optional_attributes.pop("pool_id", component["id"]), @@ -62,44 +69,67 @@ class ThirdPartyPoolTychoDecoder: adapter_contract_name=self.adapter_contract, minimum_gas=self.minimum_gas, hard_sell_limit=self.hard_limit, - trace=False, + trace=True, **optional_attributes, ) @staticmethod - def decode_optional_attributes(component, snap): + def decode_optional_attributes(component, snap, block_number): # Handle optional state attributes attributes = snap["state"]["attributes"] + pool_id = attributes.get("pool_id") or component["id"] balance_owner = attributes.get("balance_owner") - balance_owner = bytes.fromhex(balance_owner[2:] if balance_owner.startswith('0x') else balance_owner).decode( - 'utf-8').lower() stateless_contracts = {} static_attributes = snap["component"]["static_attributes"] - pool_id = static_attributes.get("pool_id") or component["id"] - pool_id = bytes.fromhex(pool_id[2:]).decode().lower() - + index = 0 while f"stateless_contract_addr_{index}" in static_attributes: encoded_address = static_attributes[f"stateless_contract_addr_{index}"] - address = bytes.fromhex( - encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8') + decoded = bytes.fromhex(encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8') + if decoded.startswith("call"): + address = ThirdPartyPoolTychoDecoder.get_address_from_call(block_number, decoded) + else: + address = decoded code = static_attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address) stateless_contracts[address] = code index += 1 - + index = 0 while f"stateless_contract_addr_{index}" in attributes: address = attributes[f"stateless_contract_addr_{index}"] code = attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address) stateless_contracts[address] = code - index += 1 + index += 1 return { "balance_owner": balance_owner, "pool_id": pool_id, "stateless_contracts": stateless_contracts, } + @staticmethod + def get_address_from_call(block_number, decoded): + db = TychoDBSingleton.get_instance() + engine = SimulationEngine.new_with_tycho_db(db=db) + engine.init_account( + address="0x0000000000000000000000000000000000000000", + account=AccountInfo(balance=0, nonce=0), + mocked=False, + permanent_storage=None, + ) + selector = keccak(text=decoded.split(":")[-1])[:4] + sim_result = engine.run_sim(SimulationParameters( + data=bytearray(selector), + to=decoded.split(':')[1], + block_number=block_number, + timestamp=int(time.time()), + overrides={}, + caller=EXTERNAL_ACCOUNT, + value=0, + )) + address = eth_abi.decode(["address"], bytearray(sim_result.result)) + return address[0] + @staticmethod def decode_balances(snap, tokens): balances = {} diff --git a/testing/tycho-client/tycho_client/pool_state.py b/testing/tycho-client/tycho_client/pool_state.py index ac690d9..082cb34 100644 --- a/testing/tycho-client/tycho_client/pool_state.py +++ b/testing/tycho-client/tycho_client/pool_state.py @@ -105,7 +105,7 @@ class ThirdPartyPool(BaseModel): engine.init_account( address=ADAPTER_ADDRESS, account=AccountInfo( - balance=0, + balance=MAX_BALANCE, nonce=0, code=get_contract_bytecode(self.adapter_contract_name), ), diff --git a/testing/tycho-client/tycho_client/utils.py b/testing/tycho-client/tycho_client/utils.py index 325d716..df6a8fa 100644 --- a/testing/tycho-client/tycho_client/utils.py +++ b/testing/tycho-client/tycho_client/utils.py @@ -74,6 +74,7 @@ class ERC20OverwriteFactory: self._overwrites = dict() self._balance_slot: Final[int] = 0 self._allowance_slot: Final[int] = 1 + self._total_supply_slot: Final[int] = 2 def set_balance(self, balance: int, owner: Address): """ @@ -111,6 +112,19 @@ class ERC20OverwriteFactory: f"spender={spender} value={allowance} slot={storage_index}", ) + def set_total_supply(self, supply: int): + """ + Set the total supply of the token. + + Parameters: + supply: The total supply value. + """ + self._overwrites[self._total_supply_slot] = supply + log.log( + 5, + f"Override total supply: token={self._token.address} supply={supply}" + ) + def get_protosim_overwrites(self) -> dict[Address, dict[int, int]]: """ Get the overwrites dictionary of previously collected values.