feat: SDK improvements

Add a way to pull stateless contracts code from node, add more settings to test_assets.yaml, add logic to allow dynamic stateless contract by calling another contract

substreams/ethereum-template/test_assets.yaml

@@ -8,6 +8,8 @@ tests:
   - name: test_pool_creation
     start_block: 123
     stop_block: 456
+    initialized_accounts:
+      - "0x0c0e5f2fF0ff18a3be9b835635039256dC4B4963" # Needed for ....
     expected_state:
       protocol_components:
         - id: "0xbebc44782c7db0a1a60cb6fe97d0b483032ff1c7"
@@ -17,6 +19,7 @@ tests:
             - "0x6b175474e89094c44da98b954eedeac495271d0f"
           static_attributes:
           creation_tx: "0x20793bbf260912aae189d5d261ff003c9b9166da8191d8f9d63ff1c7722f3ac6"
+          skip_simulation: false
   - name: test_something_else
     start_block: 123
     stop_block: 456
@@ -28,3 +31,4 @@ tests:
             - "0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84"
           static_attributes:
           creation_tx: "0xfac67ecbd423a5b915deff06045ec9343568edaec34ae95c43d35f2c018afdaa"
+          skip_simulation: true # If true, always add a reason

testing/src/runner/runner.py

@@ -61,6 +61,7 @@ class TestRunner:
         """Run all tests specified in the configuration."""
         print(f"Running tests ...")
         for test in self.config["tests"]:
+            self.tycho_runner.empty_database(self.db_url)
 
             spkg_path = self.build_spkg(
                 os.path.join(self.spkg_src, self.config["substreams_yaml_path"]),
@@ -71,6 +72,7 @@ class TestRunner:
                 test["start_block"],
                 test["stop_block"],
                 self.config["protocol_type_names"],
+                test.get("initialized_accounts", []),
             )
 
             result = self.tycho_runner.run_with_rpc_server(
@@ -83,7 +85,6 @@ class TestRunner:
             else:
                 print(f"❗️ {test['name']} failed: {result.message}")
 
-            self.tycho_runner.empty_database(self.db_url)
 
     def validate_state(self, expected_state: dict, stop_block: int) -> TestResult:
         """Validate the current protocol state against the expected state."""
@@ -104,6 +105,8 @@ class TestRunner:
 
                 component = components[comp_id]
                 for key, value in expected_component.items():
+                    if key not in ["tokens", "static_attributes", "creation_tx"]:
+                        continue
                     if key not in component:
                         return TestResult.Failed(
                             f"Missing '{key}' in component '{comp_id}'."
@@ -148,10 +151,13 @@ class TestRunner:
                                 f"from rpc call and {tycho_balance} from Substreams"
                             )
             contract_states = self.tycho_rpc_client.get_contract_state()
+            filtered_components = {'protocol_components': [pc for pc in protocol_components["protocol_components"] if
+                                                           pc["id"] in [c["id"].lower() for c in
+                                                                        expected_state["protocol_components"] if c["skip_simulation"] is False]]}
             simulation_failures = self.simulate_get_amount_out(
                 stop_block,
                 protocol_states,
-                protocol_components,
+                filtered_components,
                 contract_states,
             )
             if len(simulation_failures):

testing/src/runner/tycho.py

@@ -83,12 +83,15 @@ class TychoRunner:
             start_block: int,
             end_block: int,
             protocol_type_names: list,
+            initialized_accounts: list,
     ) -> None:
         """Run the Tycho indexer with the specified SPKG and block range."""
 
         env = os.environ.copy()
         env["RUST_LOG"] = "tycho_indexer=info"
 
+        all_accounts = self._initialized_accounts + initialized_accounts
+
         try:
             process = subprocess.Popen(
                 [
@@ -106,16 +109,14 @@ class TychoRunner:
                     str(start_block),
                     "--stop-block",
                     # +2 is to make up for the cache in the index side.
-                    str(end_block + 2),
+                    str(end_block + 2)
-                    "--initialized-accounts",
+                ] + (["--initialized-accounts", ",".join(all_accounts)] if all_accounts else []),
-                    ",".join(self._initialized_accounts)
-                ],
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 text=True,
                 bufsize=1,
                 env=env,
-            )
+                )
 
             with process.stdout:
                 for line in iter(process.stdout.readline, ""):
@@ -203,7 +204,7 @@ class TychoRunner:
     def empty_database(db_url: str) -> None:
         """Drop and recreate the Tycho indexer database."""
         try:
-            conn = psycopg2.connect(db_url)
+            conn = psycopg2.connect(db_url[:db_url.rfind('/')])
             conn.autocommit = True
             cursor = conn.cursor()
 

testing/tycho-client/tycho_client/decoders.py

@@ -1,11 +1,18 @@
+import time
 from decimal import Decimal
 from logging import getLogger
 from typing import Any
 
+import eth_abi
+from eth_utils import keccak
+from protosim_py import SimulationEngine, SimulationParameters, AccountInfo
+
+from .constants import EXTERNAL_ACCOUNT
 from .exceptions import TychoDecodeError
 from .models import EVMBlock, EthereumToken
 from .pool_state import ThirdPartyPool
-from .utils import decode_tycho_exchange
+from .tycho_db import TychoDBSingleton
+from .utils import decode_tycho_exchange, get_code_for_address
 
 log = getLogger(__name__)
 
@@ -49,7 +56,7 @@ class ThirdPartyPoolTychoDecoder:
             raise TychoDecodeError("Unsupported token", pool_id=component["id"])
 
         balances = self.decode_balances(snap, tokens)
-        optional_attributes = self.decode_optional_attributes(component, snap)
+        optional_attributes = self.decode_optional_attributes(component, snap, block.id)
 
         return ThirdPartyPool(
             id_=optional_attributes.pop("pool_id", component["id"]),
@@ -62,44 +69,67 @@ class ThirdPartyPoolTychoDecoder:
             adapter_contract_name=self.adapter_contract,
             minimum_gas=self.minimum_gas,
             hard_sell_limit=self.hard_limit,
-            trace=False,
+            trace=True,
             **optional_attributes,
         )
 
     @staticmethod
-    def decode_optional_attributes(component, snap):
+    def decode_optional_attributes(component, snap, block_number):
         # Handle optional state attributes
         attributes = snap["state"]["attributes"]
+        pool_id = attributes.get("pool_id") or component["id"]
         balance_owner = attributes.get("balance_owner")
-        balance_owner = bytes.fromhex(balance_owner[2:] if balance_owner.startswith('0x') else balance_owner).decode(
-            'utf-8').lower()
         stateless_contracts = {}
         static_attributes = snap["component"]["static_attributes"]
-        pool_id = static_attributes.get("pool_id") or component["id"]
+        
-        pool_id = bytes.fromhex(pool_id[2:]).decode().lower()
-
         index = 0
         while f"stateless_contract_addr_{index}" in static_attributes:
             encoded_address = static_attributes[f"stateless_contract_addr_{index}"]
-            address = bytes.fromhex(
+            decoded = bytes.fromhex(encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8')
-                encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8')
+            if decoded.startswith("call"):
+                address = ThirdPartyPoolTychoDecoder.get_address_from_call(block_number, decoded)
+            else:
+                address = decoded
 
             code = static_attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address)
             stateless_contracts[address] = code
             index += 1
-
+            
         index = 0
         while f"stateless_contract_addr_{index}" in attributes:
             address = attributes[f"stateless_contract_addr_{index}"]
             code = attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address)
             stateless_contracts[address] = code
-            index += 1
+            index += 1      
         return {
             "balance_owner": balance_owner,
             "pool_id": pool_id,
             "stateless_contracts": stateless_contracts,
         }
 
+    @staticmethod
+    def get_address_from_call(block_number, decoded):
+        db = TychoDBSingleton.get_instance()
+        engine = SimulationEngine.new_with_tycho_db(db=db)
+        engine.init_account(
+            address="0x0000000000000000000000000000000000000000",
+            account=AccountInfo(balance=0, nonce=0),
+            mocked=False,
+            permanent_storage=None,
+        )
+        selector = keccak(text=decoded.split(":")[-1])[:4]
+        sim_result = engine.run_sim(SimulationParameters(
+            data=bytearray(selector),
+            to=decoded.split(':')[1],
+            block_number=block_number,
+            timestamp=int(time.time()),
+            overrides={},
+            caller=EXTERNAL_ACCOUNT,
+            value=0,
+        ))
+        address = eth_abi.decode(["address"], bytearray(sim_result.result))
+        return address[0]
+
     @staticmethod
     def decode_balances(snap, tokens):
         balances = {}

testing/tycho-client/tycho_client/pool_state.py

@@ -105,7 +105,7 @@ class ThirdPartyPool(BaseModel):
             engine.init_account(
                 address=ADAPTER_ADDRESS,
                 account=AccountInfo(
-                    balance=0,
+                    balance=MAX_BALANCE,
                     nonce=0,
                     code=get_contract_bytecode(self.adapter_contract_name),
                 ),

testing/tycho-client/tycho_client/utils.py

@@ -74,6 +74,7 @@ class ERC20OverwriteFactory:
         self._overwrites = dict()
         self._balance_slot: Final[int] = 0
         self._allowance_slot: Final[int] = 1
+        self._total_supply_slot: Final[int] = 2
 
     def set_balance(self, balance: int, owner: Address):
         """
@@ -111,6 +112,19 @@ class ERC20OverwriteFactory:
             f"spender={spender} value={allowance} slot={storage_index}",
         )
 
+    def set_total_supply(self, supply: int):
+        """
+        Set the total supply of the token.
+
+        Parameters:
+            supply: The total supply value.
+        """
+        self._overwrites[self._total_supply_slot] = supply
+        log.log(
+            5,
+            f"Override total supply: token={self._token.address} supply={supply}"
+        )
+
     def get_protosim_overwrites(self) -> dict[Address, dict[int, int]]:
         """
         Get the overwrites dictionary of previously collected values.