feat(runner): Support initialized accounts + misc fixes.

Simplifies a lot the setup of testing:

- Looks up tycho-indexer under the usual paths no OS specific naming necessary.
- Simply assumes that protosim can be pulled from our private PyPi
- Navigates the foundry out folder to find solidity runtime binaries

Includes some additional fixes to deal with some attribtues that may have to be reflected to defibot later on.

testing/docker-compose.yaml

@@ -2,6 +2,7 @@ version: '3.1'
 services:
   db:
     build:
+      context: .
       dockerfile: postgres.Dockerfile
     restart: "always"
     environment:

testing/src/requirements.txt

@@ -2,4 +2,4 @@ psycopg2==2.9.9
 PyYAML==6.0.1
 Requests==2.32.2
 web3==5.31.3
-./tycho-client
+-e ./tycho-client

testing/src/runner/cli.py

@@ -7,7 +7,7 @@ def main() -> None:
         description="Run indexer within a specified range of blocks"
     )
     parser.add_argument(
-        "--test_yaml_path", type=str, help="Path to the test configuration YAML file."
+        "--package", type=str, help="Name of the package to test."
     )
     parser.add_argument(
         "--with_binary_logs",
@@ -20,7 +20,7 @@ def main() -> None:
     args = parser.parse_args()
 
     test_runner = TestRunner(
-        args.test_yaml_path, args.with_binary_logs, db_url=args.db_url
+        args.package, args.with_binary_logs, db_url=args.db_url
     )
     test_runner.run_tests()
 

testing/src/runner/runner.py

@@ -46,10 +46,13 @@ class SimulationFailure(BaseModel):
 
 
 class TestRunner:
-    def __init__(self, config_path: str, with_binary_logs: bool, db_url: str):
+    def __init__(self, package: str, with_binary_logs: bool, db_url: str):
+        self.repo_root = os.getcwd()
+        config_path = os.path.join(self.repo_root, "substreams", package, "test_assets.yaml")
         self.config = load_config(config_path)
-        self.base_dir = os.path.dirname(config_path)
-        self.tycho_runner = TychoRunner(with_binary_logs)
+        self.spkg_src = os.path.join(self.repo_root, "substreams", package)
+        self.adapters_src = os.path.join(self.repo_root, "evm")
+        self.tycho_runner = TychoRunner(db_url, with_binary_logs, self.config["initialized_accounts"])
         self.tycho_rpc_client = TychoRPCClient()
         self.db_url = db_url
         self._chain = Blockchain.ethereum
@@ -60,7 +63,7 @@ class TestRunner:
         for test in self.config["tests"]:
 
             spkg_path = self.build_spkg(
-                os.path.join(self.base_dir, self.config["substreams_yaml_path"]),
+                os.path.join(self.spkg_src, self.config["substreams_yaml_path"]),
                 lambda data: self.update_initial_block(data, test["start_block"]),
             )
             self.tycho_runner.run_tycho(
@@ -107,7 +110,7 @@ class TestRunner:
                         )
                     if isinstance(value, list):
                         if set(map(str.lower, value)) != set(
-                            map(str.lower, component[key])
+                                map(str.lower, component[key])
                         ):
                             return TestResult.Failed(
                                 f"List mismatch for key '{key}': {value} != {component[key]}"
@@ -146,7 +149,6 @@ class TestRunner:
                             )
             contract_states = self.tycho_rpc_client.get_contract_state()
             simulation_failures = self.simulate_get_amount_out(
-                token_balances,
                 stop_block,
                 protocol_states,
                 protocol_components,
@@ -169,12 +171,11 @@ class TestRunner:
             return TestResult.Failed(error_message)
 
     def simulate_get_amount_out(
-        self,
-        token_balances: dict[str, dict[str, int]],
-        block_number: int,
-        protocol_states: dict,
-        protocol_components: dict,
-        contract_state: dict,
+            self,
+            block_number: int,
+            protocol_states: dict,
+            protocol_components: dict,
+            contract_state: dict,
     ) -> dict[str, list[SimulationFailure]]:
         protocol_type_names = self.config["protocol_type_names"]
 
@@ -188,7 +189,8 @@ class TestRunner:
         failed_simulations: dict[str, list[SimulationFailure]] = dict()
         for protocol in protocol_type_names:
             adapter_contract = os.path.join(
-                self.base_dir, "evm", self.config["adapter_contract"]
+                self.adapters_src, "out", f"{self.config['adapter_contract']}.sol",
+                f"{self.config['adapter_contract']}.evm.runtime"
             )
             decoder = ThirdPartyPoolTychoDecoder(adapter_contract, 0, False)
             stream_adapter = TychoPoolStateStreamAdapter(
@@ -204,21 +206,17 @@ class TestRunner:
 
             for pool_state in decoded.pool_states.values():
                 pool_id = pool_state.id_
-                protocol_balances = token_balances.get(pool_id)
-                if not protocol_balances:
+                if not pool_state.balances:
                     raise ValueError(f"Missing balances for pool {pool_id}")
                 for sell_token, buy_token in itertools.permutations(
-                    pool_state.tokens, 2
+                        pool_state.tokens, 2
                 ):
+                    # Try to sell 0.1% of the protocol balance
+                    sell_amount = Decimal("0.001") * pool_state.balances[sell_token.address]
                     try:
-                        # Try to sell 0.1% of the protocol balance
-                        sell_amount = Decimal("0.001") * sell_token.from_onchain_amount(
-                            protocol_balances[sell_token.address]
-                        )
                         amount_out, gas_used, _ = pool_state.get_amount_out(
                             sell_token, sell_amount, buy_token
                         )
-                        # TODO: Should we validate this with an archive node or RPC reader?
                         print(
                             f"Amount out for {pool_id}: {sell_amount} {sell_token} -> {amount_out} {buy_token} - "
                             f"Gas used: {gas_used}"
@@ -233,8 +231,8 @@ class TestRunner:
                         failed_simulations[pool_id].append(
                             SimulationFailure(
                                 pool_id=pool_id,
-                                sell_token=sell_token,
-                                buy_token=buy_token,
+                                sell_token=str(sell_token),
+                                buy_token=str(buy_token),
                                 error=str(e),
                             )
                         )

testing/src/runner/tycho.py

@@ -1,29 +1,42 @@
-import os
-import platform
 import signal
 import subprocess
-import sys
 import threading
 import time
-from pathlib import Path
 
 import psycopg2
 import requests
 from psycopg2 import sql
 
-
-def get_binary_path():
-    path = Path(__file__).parent
-    if sys.platform.startswith("darwin") and platform.machine() == "arm64":
-        return Path(__file__).parent / "tycho-indexer-mac-arm64"
-    elif sys.platform.startswith("linux") and platform.machine() == "x86_64":
-        return Path(__file__).parent / "tycho-indexer-linux-x64"
-
-    else:
-        raise RuntimeError("Unsupported platform or architecture")
+import os
 
 
-binary_path = get_binary_path()
+def find_binary_file(file_name):
+    # Define usual locations for binary files in Unix-based systems
+    locations = [
+        "/bin",
+        "/sbin",
+        "/usr/bin",
+        "/usr/sbin",
+        "/usr/local/bin",
+        "/usr/local/sbin",
+    ]
+
+    # Add user's local bin directory if it exists
+    home = os.path.expanduser("~")
+    if os.path.exists(home + "/.local/bin"):
+        locations.append(home + "/.local/bin")
+
+    # Check each location
+    for location in locations:
+        potential_path = location + "/" + file_name
+        if os.path.exists(potential_path):
+            return potential_path
+
+    # If binary is not found in the usual locations, return None
+    raise RuntimeError("Unable to locate tycho-indexer binary")
+
+
+binary_path = find_binary_file("tycho-indexer")
 
 
 class TychoRPCClient:
@@ -59,25 +72,29 @@ class TychoRPCClient:
 
 
 class TychoRunner:
-    def __init__(self, with_binary_logs: bool = False):
+    def __init__(self, db_url: str, with_binary_logs: bool = False, initialized_accounts: list[str] = None):
         self.with_binary_logs = with_binary_logs
+        self._db_url = db_url
+        self._initialized_accounts = initialized_accounts or []
 
     def run_tycho(
-        self,
-        spkg_path: str,
-        start_block: int,
-        end_block: int,
-        protocol_type_names: list,
+            self,
+            spkg_path: str,
+            start_block: int,
+            end_block: int,
+            protocol_type_names: list,
     ) -> None:
         """Run the Tycho indexer with the specified SPKG and block range."""
 
         env = os.environ.copy()
-        env["RUST_LOG"] = "info"
+        env["RUST_LOG"] = "tycho_indexer=info"
 
         try:
             process = subprocess.Popen(
                 [
                     binary_path,
+                    "--database-url",
+                    self._db_url,
                     "run",
                     "--spkg",
                     spkg_path,
@@ -88,8 +105,11 @@ class TychoRunner:
                     "--start-block",
                     str(start_block),
                     "--stop-block",
+                    # +2 is to make up for the cache in the index side.
                     str(end_block + 2),
-                ],  # +2 is to make up for the cache in the index side.
+                    "--initialized-accounts",
+                    ",".join(self._initialized_accounts)
+                ],
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 text=True,
@@ -128,7 +148,12 @@ class TychoRunner:
                 env["RUST_LOG"] = "info"
 
                 process = subprocess.Popen(
-                    [binary_path, "rpc"],
+                    [
+                        binary_path,
+                        "--database-url",
+                        self._db_url,
+                        "rpc"
+                    ],
                     stdout=subprocess.PIPE,
                     stderr=subprocess.PIPE,
                     text=True,

testing/tycho-client/setup.py

@@ -1,7 +1,4 @@
 from setuptools import setup, find_packages
-import sys
-import platform
-from pathlib import Path
 
 
 def read_requirements():
@@ -11,25 +8,6 @@ def read_requirements():
     return [req for req in requirements if req and not req.startswith("#")]
 
 
-# Determine the correct wheel file based on the platform and Python version
-def get_wheel_file():
-    path = Path(__file__).parent
-    if sys.platform.startswith("darwin") and platform.machine() == "arm64":
-        return str(
-            path / "wheels" / f"protosim_py-0.4.9-cp39-cp39-macosx_11_0_arm64.whl"
-        )
-    elif sys.platform.startswith("linux") and platform.machine() == "x86_64":
-        return str(
-            path
-            / "wheels"
-            / f"protosim_py-0.4.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
-        )
-    else:
-        raise RuntimeError("Unsupported platform or architecture")
-
-
-wheel_file = get_wheel_file()
-
 setup(
     name="tycho-client",
     version="0.1.0",
@@ -51,7 +29,7 @@ setup(
         "eth-utils==1.9.5",
         "hexbytes==0.3.1",
         "pydantic==2.8.2",
-        f"protosim_py @ file://{wheel_file}",
+        "protosim_py==0.4.11",
     ],
     package_data={"tycho-client": ["../wheels/*", "./assets/*", "./bins/*"]},
     include_package_data=True,

testing/tycho-client/tycho_client/decoders.py

@@ -19,10 +19,10 @@ class ThirdPartyPoolTychoDecoder:
         self.hard_limit = hard_limit
 
     def decode_snapshot(
-        self,
-        snapshot: dict[str, Any],
-        block: EVMBlock,
-        tokens: dict[str, EthereumToken],
+            self,
+            snapshot: dict[str, Any],
+            block: EVMBlock,
+            tokens: dict[str, EthereumToken],
     ) -> tuple[dict[str, ThirdPartyPool], list[str]]:
         pools = {}
         failed_pools = []
@@ -38,7 +38,7 @@ class ThirdPartyPoolTychoDecoder:
         return pools, failed_pools
 
     def decode_pool_state(
-        self, snap: dict, block: EVMBlock, tokens: dict[str, EthereumToken]
+            self, snap: dict, block: EVMBlock, tokens: dict[str, EthereumToken]
     ) -> ThirdPartyPool:
         component = snap["component"]
         exchange, _ = decode_tycho_exchange(component["protocol_system"])
@@ -70,15 +70,19 @@ class ThirdPartyPoolTychoDecoder:
     def decode_optional_attributes(component, snap):
         # Handle optional state attributes
         attributes = snap["state"]["attributes"]
-        pool_id = attributes.get("pool_id") or component["id"]
         balance_owner = attributes.get("balance_owner")
+        balance_owner = bytes.fromhex(balance_owner[2:] if balance_owner.startswith('0x') else balance_owner).decode(
+            'utf-8').lower()
         stateless_contracts = {}
         static_attributes = snap["component"]["static_attributes"]
+        pool_id = static_attributes.get("pool_id") or component["id"]
+        pool_id = bytes.fromhex(pool_id[2:]).decode().lower()
 
         index = 0
         while f"stateless_contract_addr_{index}" in static_attributes:
             encoded_address = static_attributes[f"stateless_contract_addr_{index}"]
-            address = bytes.fromhex(encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8')
+            address = bytes.fromhex(
+                encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8')
 
             code = static_attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address)
             stateless_contracts[address] = code
@@ -109,10 +113,10 @@ class ThirdPartyPoolTychoDecoder:
 
     @staticmethod
     def apply_update(
-        pool: ThirdPartyPool,
-        pool_update: dict[str, Any],
-        balance_updates: dict[str, Any],
-        block: EVMBlock,
+            pool: ThirdPartyPool,
+            pool_update: dict[str, Any],
+            balance_updates: dict[str, Any],
+            block: EVMBlock,
     ) -> ThirdPartyPool:
         # check for and apply optional state attributes
         attributes = pool_update.get("updated_attributes")