Make tycho_client a python package, small bugfixes

substreams/ethereum-curve/test_assets.yaml

@@ -1,6 +1,7 @@
 substreams_yaml_path: ./substreams.yaml
 protocol_type_names:
   - "curve_pool"
+adapter_contract: "CurveSwapAdapter.evm.runtime"
 tests:
   - name: test_3pool_creation
     start_block: 10809470

testing/Dockerfile

@@ -5,8 +5,7 @@ FROM --platform=linux/amd64 continuumio/miniconda3:24.4.0-0
 WORKDIR /app
 
 # Add current directory code to /app in container
-ADD ./testing /app/testing
+ADD ./ /app/testing
-ADD ./tycho_client /app/tycho_client
 
 RUN chmod +x /app/testing/tycho-indexer
 
@@ -22,7 +21,7 @@ RUN apt-get update \
     && pip install psycopg2 \
     && apt-get clean
 
-RUN /bin/bash -c "source activate myenv && pip install --no-cache-dir -r testing/requirements.txt && cd /app/tycho_client && pip install -r requirements-docker.txt && cd -"
+RUN /bin/bash -c "source activate myenv && cd testing && pip install --no-cache-dir -r requirements.txt && cd -"
 
 # Make port 80 available to the world outside this container
 EXPOSE 80

testing/README.md

@@ -21,6 +21,10 @@ Tests are defined in a `yaml` file. A template can be found at `substreams/ether
 
 Each test will index all blocks between `start-block` and `stop-block` and verify that the indexed state matches the expected state.
 
+You will also need the EVM Runtime file for the adapter contract. 
+The script to generate this file is available under `/evm/scripts/buildRuntime.sh`.
+Please place this Runtime file under the respective `substream` directory inside the `evm` folder.
+
 ## Running Tests
 
 ### Step 1: Export Environment Variables

testing/cli.py

@@ -15,13 +15,13 @@ def main() -> None:
         help="Flag to activate logs from Tycho.",
     )
     parser.add_argument(
-        "--db_url",
+        "--db_url", type=str, help="Postgres database URL for the Tycho indexer."
-        type=str,
-        help="Postgres database URL for the Tycho indexer.",
     )
     args = parser.parse_args()
 
-    test_runner = TestRunner(args.test_yaml_path, args.with_binary_logs, db_url=args.db_url)
+    test_runner = TestRunner(
+        args.test_yaml_path, args.with_binary_logs, db_url=args.db_url
+    )
     test_runner.run_tests()
 
 

testing/docker-compose.yaml

@@ -15,8 +15,8 @@ services:
       - postgres_data:/var/lib/postgresql/data
   app:
     build:
-      context: ..
+      context: .
-      dockerfile: testing/Dockerfile
+      dockerfile: Dockerfile
     volumes:
       - ${SUBSTREAMS_PATH}:/app/substreams/my_substream
       - ../substreams:/app/substreams

testing/requirements.txt

@@ -2,3 +2,4 @@ psycopg2==2.9.9
 PyYAML==6.0.1
 Requests==2.32.2
 web3==5.31.3
+./tycho-client

testing/runner.py

@@ -1,5 +1,4 @@
 import itertools
-import itertools
 import os
 import shutil
 import subprocess
@@ -10,12 +9,12 @@ from pathlib import Path
 
 import yaml
 from pydantic import BaseModel
+from tycho_client.decoders import ThirdPartyPoolTychoDecoder
+from tycho_client.models import Blockchain, EVMBlock
+from tycho_client.tycho_adapter import TychoPoolStateStreamAdapter
 
 from evm import get_token_balance, get_block_header
 from tycho import TychoRunner
-from tycho_client.tycho.decoders import ThirdPartyPoolTychoDecoder
-from tycho_client.tycho.models import Blockchain, EVMBlock
-from tycho_client.tycho.tycho_adapter import TychoPoolStateStreamAdapter
 
 
 class TestResult:
@@ -139,16 +138,27 @@ class TestRunner:
                     node_balance = get_token_balance(token, comp_id, stop_block)
                     if node_balance != tycho_balance:
                         return TestResult.Failed(
-                            f"Balance mismatch for {comp_id}:{token} at block {stop_block}: got {node_balance} from rpc call and {tycho_balance} from Substreams"
+                            f"Balance mismatch for {comp_id}:{token} at block {stop_block}: got {node_balance} "
+                            f"from rpc call and {tycho_balance} from Substreams"
                         )
             contract_states = self.tycho_runner.get_contract_state()
-            self.simulate_get_amount_out(
+            simulation_failures = self.simulate_get_amount_out(
                 token_balances,
                 stop_block,
                 protocol_states,
                 protocol_components,
                 contract_states,
             )
+            if len(simulation_failures):
+                error_msgs = []
+                for pool_id, failures in simulation_failures.items():
+                    failures_ = [
+                        f"{f.sell_token} -> {f.buy_token}: {f.error}" for f in failures
+                    ]
+                    error_msgs.append(
+                        f"Pool {pool_id} failed simulations: {', '.join(failures_)}"
+                    )
+                raise ValueError(". ".join(error_msgs))
 
             return TestResult.Passed()
         except Exception as e:
@@ -161,7 +171,7 @@ class TestRunner:
         protocol_states: dict,
         protocol_components: dict,
         contract_state: dict,
-    ) -> TestResult:
+    ) -> dict[str, list[SimulationFailure]]:
         protocol_type_names = self.config["protocol_type_names"]
 
         block_header = get_block_header(block_number)
@@ -171,12 +181,12 @@ class TestRunner:
             hash_=block_header.hash.hex(),
         )
 
-        failed_simulations = dict[str, list[SimulationFailure]]
+        failed_simulations: dict[str, list[SimulationFailure]] = dict()
         for protocol in protocol_type_names:
-            # TODO: Parametrize this
+            adapter_contract = os.path.join(
-            decoder = ThirdPartyPoolTychoDecoder(
+                self.base_dir, "evm", self.config["adapter_contract"]
-                "CurveSwapAdapter.evm.runtime", 0, False
             )
+            decoder = ThirdPartyPoolTychoDecoder(adapter_contract, 0, False)
             stream_adapter = TychoPoolStateStreamAdapter(
                 tycho_url="0.0.0.0:4242",
                 protocol=protocol,
@@ -225,6 +235,7 @@ class TestRunner:
                             )
                         )
                         continue
+        return failed_simulations
 
     @staticmethod
     def build_spkg(yaml_file_path: str, modify_func: callable) -> str:

testing/tycho-client/MANIFEST.in

@@ -0,0 +1,3 @@
+include wheels/*.whl
+include tycho_client/assets/*
+include tycho_client/bins/*

testing/tycho-client/requirements.txt

@@ -1,4 +1,3 @@
-./tycho/bins/protosim_py-0.4.9-cp39-cp39-macosx_11_0_arm64.whl
 requests==2.32.2
 eth-abi==2.2.0
 eth-typing==2.3.0

testing/tycho-client/setup.py

@@ -0,0 +1,58 @@
+from setuptools import setup, find_packages
+import sys
+import platform
+from pathlib import Path
+
+
+def read_requirements():
+    with open("requirements.txt") as req:
+        content = req.read()
+        requirements = content.split("\n")
+    return [req for req in requirements if req and not req.startswith("#")]
+
+
+# Determine the correct wheel file based on the platform and Python version
+def get_wheel_file():
+    path = Path(__file__).parent
+    if sys.platform.startswith("darwin") and platform.machine() == "arm64":
+        return str(
+            path / "wheels" / f"protosim_py-0.4.9-cp39-cp39-macosx_11_0_arm64.whl"
+        )
+    elif sys.platform.startswith("linux") and platform.machine() == "x86_64":
+        return str(
+            path
+            / "wheels"
+            / f"protosim_py-0.4.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
+        )
+    else:
+        raise RuntimeError("Unsupported platform or architecture")
+
+
+wheel_file = get_wheel_file()
+
+setup(
+    name="tycho-client",
+    version="0.1.0",
+    author="Propeller Heads",
+    description="A package for interacting with the Tycho API.",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    packages=find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires="~=3.9",
+    install_requires=[
+        "requests==2.32.2",
+        "eth-abi==2.2.0",
+        "eth-typing==2.3.0",
+        "eth-utils==1.9.5",
+        "hexbytes==0.3.1",
+        "pydantic==2.8.2",
+        f"protosim_py @ file://{wheel_file}",
+    ],
+    package_data={"tycho-client": ["../wheels/*", "./assets/*", "./bins/*"]},
+    include_package_data=True,
+)

testing/tycho-client/tycho_client/constants.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from typing import Final
 
+ASSETS_FOLDER = Path(__file__).parent / "assets"
 TYCHO_CLIENT_FOLDER = Path(__file__).parent / "bins"
 TYCHO_CLIENT_LOG_FOLDER = TYCHO_CLIENT_FOLDER / "logs"
 

testing/tycho-client/tycho_client/tycho_adapter.py

@@ -343,4 +343,3 @@ class TychoPoolStateStreamAdapter:
 
         block_header = BlockHeader(block.id, block.hash_, int(block.ts.timestamp()))
         TychoDBSingleton.get_instance().update(vm_updates, block_header)
-

testing/tycho-client/tycho_client/utils.py

@@ -13,7 +13,7 @@ from hexbytes import HexBytes
 from protosim_py import SimulationEngine, AccountInfo
 from web3 import Web3
 
-from .constants import EXTERNAL_ACCOUNT, MAX_BALANCE
+from .constants import EXTERNAL_ACCOUNT, MAX_BALANCE, ASSETS_FOLDER
 from .exceptions import OutOfGas
 from .models import Address, EthereumToken
 from .tycho_db import TychoDBSingleton
@@ -45,7 +45,9 @@ def create_engine(
     engine = SimulationEngine.new_with_tycho_db(db=db, trace=trace)
 
     for t in mocked_tokens:
-        info = AccountInfo(balance=0, nonce=0, code=get_contract_bytecode("ERC20.bin"))
+        info = AccountInfo(
+            balance=0, nonce=0, code=get_contract_bytecode(ASSETS_FOLDER / "ERC20.bin")
+        )
         engine.init_account(
             address=t, account=info, mocked=True, permanent_storage=None
         )
@@ -132,7 +134,8 @@ class ERC20OverwriteFactory:
             HexBytes(key).hex(): "0x" + HexBytes(val).hex().lstrip("0x").zfill(64)
             for key, val in self._overwrites.items()
         }
-        code = "0x" + get_contract_bytecode("ERC20.bin").hex()
+
+        code = "0x" + get_contract_bytecode(ASSETS_FOLDER / "ERC20.bin").hex()
         return {self._token.address: {"stateDiff": formatted_overwrites, "code": code}}
 
 
@@ -181,10 +184,9 @@ def get_storage_slot_at_key(key: Address, mapping_slot: int) -> int:
 
 
 @lru_cache
-def get_contract_bytecode(name: str) -> bytes:
+def get_contract_bytecode(path: str) -> bytes:
-    """Load contract bytecode from a file in the assets directory"""
+    """Load contract bytecode from a file given an absolute path"""
-    # TODO: Check if this locaation is correct
+    with open(path, "rb") as fh:
-    with open(Path(__file__).parent / "assets" / name, "rb") as fh:
         code = fh.read()
     return code
 

testing/tycho.py

@@ -1,6 +1,8 @@
 import os
+import platform
 import signal
 import subprocess
+import sys
 import threading
 import time
 from pathlib import Path
@@ -9,7 +11,19 @@ import psycopg2
 import requests
 from psycopg2 import sql
 
-binary_path = Path(__file__).parent / "tycho-indexer"
+
+def get_binary_path():
+    path = Path(__file__).parent
+    if sys.platform.startswith("darwin") and platform.machine() == "arm64":
+        return Path(__file__).parent / "tycho-indexer-mac-arm64"
+    elif sys.platform.startswith("linux") and platform.machine() == "x86_64":
+        return Path(__file__).parent / "tycho-indexer-linux-x64"
+
+    else:
+        raise RuntimeError("Unsupported platform or architecture")
+
+
+binary_path = get_binary_path()
 
 
 class TychoRunner:

tycho_client/requirements-docker.txt

@@ -1,7 +0,0 @@
-./tycho/bins/protosim_py-0.4.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
-requests==2.32.2
-eth-abi==2.2.0
-eth-typing==2.3.0
-eth-utils==1.9.5
-hexbytes==0.3.1
-pydantic==2.8.2