From e0c1ba3b5014695018676dae2a32d55591d1cfae Mon Sep 17 00:00:00 2001 From: Thales Lima Date: Fri, 19 Jul 2024 04:19:34 +0200 Subject: [PATCH] Make tycho_client a python package, small bugfixes --- .../evm}/CurveSwapAdapter.evm.runtime | Bin substreams/ethereum-curve/test_assets.yaml | 1 + testing/Dockerfile | 5 +- testing/README.md | 4 ++ testing/cli.py | 8 +-- testing/docker-compose.yaml | 4 +- testing/requirements.txt | 1 + testing/runner.py | 33 ++++++---- testing/tycho-client/MANIFEST.in | 3 + .../tycho-client}/README.md | 0 .../tycho-client}/__init__.py | 0 .../tycho-client}/requirements.txt | 1 - testing/tycho-client/setup.py | 58 ++++++++++++++++++ .../tycho-client}/tycho_client/__init__.py | 0 .../tycho_client/adapter_contract.py | 0 .../tycho_client/assets/ERC20.bin | Bin .../tycho_client/assets/IERC20.sol | 0 .../tycho_client/assets/ISwapAdapter.abi | 0 .../tycho_client/assets/mocked_ERC20.sol | 0 .../tycho-client}/tycho_client/constants.py | 1 + .../tycho-client}/tycho_client/decoders.py | 0 .../tycho-client}/tycho_client/exceptions.py | 0 .../tycho-client}/tycho_client/models.py | 0 .../tycho-client}/tycho_client/pool_state.py | 0 .../tycho_client/tycho_adapter.py | 1 - .../tycho-client}/tycho_client/tycho_db.py | 0 .../tycho-client}/tycho_client/utils.py | 16 ++--- testing/tycho.py | 16 ++++- tycho_client/requirements-docker.txt | 7 --- 29 files changed, 122 insertions(+), 37 deletions(-) rename {tycho_client/tycho_client/assets => substreams/ethereum-curve/evm}/CurveSwapAdapter.evm.runtime (100%) create mode 100644 testing/tycho-client/MANIFEST.in rename {tycho_client => testing/tycho-client}/README.md (100%) rename {tycho_client => testing/tycho-client}/__init__.py (100%) rename {tycho_client => testing/tycho-client}/requirements.txt (60%) create mode 100644 testing/tycho-client/setup.py rename {tycho_client => testing/tycho-client}/tycho_client/__init__.py (100%) rename {tycho_client => testing/tycho-client}/tycho_client/adapter_contract.py (100%) rename {tycho_client => testing/tycho-client}/tycho_client/assets/ERC20.bin (100%) rename {tycho_client => testing/tycho-client}/tycho_client/assets/IERC20.sol (100%) rename {tycho_client => testing/tycho-client}/tycho_client/assets/ISwapAdapter.abi (100%) rename {tycho_client => testing/tycho-client}/tycho_client/assets/mocked_ERC20.sol (100%) rename {tycho_client => testing/tycho-client}/tycho_client/constants.py (90%) rename {tycho_client => testing/tycho-client}/tycho_client/decoders.py (100%) rename {tycho_client => testing/tycho-client}/tycho_client/exceptions.py (100%) rename {tycho_client => testing/tycho-client}/tycho_client/models.py (100%) rename {tycho_client => testing/tycho-client}/tycho_client/pool_state.py (100%) rename {tycho_client => testing/tycho-client}/tycho_client/tycho_adapter.py (99%) rename {tycho_client => testing/tycho-client}/tycho_client/tycho_db.py (100%) rename {tycho_client => testing/tycho-client}/tycho_client/utils.py (95%) delete mode 100644 tycho_client/requirements-docker.txt diff --git a/tycho_client/tycho_client/assets/CurveSwapAdapter.evm.runtime b/substreams/ethereum-curve/evm/CurveSwapAdapter.evm.runtime similarity index 100% rename from tycho_client/tycho_client/assets/CurveSwapAdapter.evm.runtime rename to substreams/ethereum-curve/evm/CurveSwapAdapter.evm.runtime diff --git a/substreams/ethereum-curve/test_assets.yaml b/substreams/ethereum-curve/test_assets.yaml index 73e6a9b..dff8257 100644 --- a/substreams/ethereum-curve/test_assets.yaml +++ b/substreams/ethereum-curve/test_assets.yaml @@ -1,6 +1,7 @@ substreams_yaml_path: ./substreams.yaml protocol_type_names: - "curve_pool" +adapter_contract: "CurveSwapAdapter.evm.runtime" tests: - name: test_3pool_creation start_block: 10809470 diff --git a/testing/Dockerfile b/testing/Dockerfile index 0489832..42272a4 100644 --- a/testing/Dockerfile +++ b/testing/Dockerfile @@ -5,8 +5,7 @@ FROM --platform=linux/amd64 continuumio/miniconda3:24.4.0-0 WORKDIR /app # Add current directory code to /app in container -ADD ./testing /app/testing -ADD ./tycho_client /app/tycho_client +ADD ./ /app/testing RUN chmod +x /app/testing/tycho-indexer @@ -22,7 +21,7 @@ RUN apt-get update \ && pip install psycopg2 \ && apt-get clean -RUN /bin/bash -c "source activate myenv && pip install --no-cache-dir -r testing/requirements.txt && cd /app/tycho_client && pip install -r requirements-docker.txt && cd -" +RUN /bin/bash -c "source activate myenv && cd testing && pip install --no-cache-dir -r requirements.txt && cd -" # Make port 80 available to the world outside this container EXPOSE 80 diff --git a/testing/README.md b/testing/README.md index 7355128..7810b34 100644 --- a/testing/README.md +++ b/testing/README.md @@ -21,6 +21,10 @@ Tests are defined in a `yaml` file. A template can be found at `substreams/ether Each test will index all blocks between `start-block` and `stop-block` and verify that the indexed state matches the expected state. +You will also need the EVM Runtime file for the adapter contract. +The script to generate this file is available under `/evm/scripts/buildRuntime.sh`. +Please place this Runtime file under the respective `substream` directory inside the `evm` folder. + ## Running Tests ### Step 1: Export Environment Variables diff --git a/testing/cli.py b/testing/cli.py index 0243c8f..109e23e 100644 --- a/testing/cli.py +++ b/testing/cli.py @@ -15,13 +15,13 @@ def main() -> None: help="Flag to activate logs from Tycho.", ) parser.add_argument( - "--db_url", - type=str, - help="Postgres database URL for the Tycho indexer.", + "--db_url", type=str, help="Postgres database URL for the Tycho indexer." ) args = parser.parse_args() - test_runner = TestRunner(args.test_yaml_path, args.with_binary_logs, db_url=args.db_url) + test_runner = TestRunner( + args.test_yaml_path, args.with_binary_logs, db_url=args.db_url + ) test_runner.run_tests() diff --git a/testing/docker-compose.yaml b/testing/docker-compose.yaml index bcc2360..a8a3b91 100644 --- a/testing/docker-compose.yaml +++ b/testing/docker-compose.yaml @@ -15,8 +15,8 @@ services: - postgres_data:/var/lib/postgresql/data app: build: - context: .. - dockerfile: testing/Dockerfile + context: . + dockerfile: Dockerfile volumes: - ${SUBSTREAMS_PATH}:/app/substreams/my_substream - ../substreams:/app/substreams diff --git a/testing/requirements.txt b/testing/requirements.txt index d141296..cf5321c 100644 --- a/testing/requirements.txt +++ b/testing/requirements.txt @@ -2,3 +2,4 @@ psycopg2==2.9.9 PyYAML==6.0.1 Requests==2.32.2 web3==5.31.3 +./tycho-client \ No newline at end of file diff --git a/testing/runner.py b/testing/runner.py index 733688a..f85dc22 100644 --- a/testing/runner.py +++ b/testing/runner.py @@ -1,5 +1,4 @@ import itertools -import itertools import os import shutil import subprocess @@ -10,12 +9,12 @@ from pathlib import Path import yaml from pydantic import BaseModel +from tycho_client.decoders import ThirdPartyPoolTychoDecoder +from tycho_client.models import Blockchain, EVMBlock +from tycho_client.tycho_adapter import TychoPoolStateStreamAdapter from evm import get_token_balance, get_block_header from tycho import TychoRunner -from tycho_client.tycho.decoders import ThirdPartyPoolTychoDecoder -from tycho_client.tycho.models import Blockchain, EVMBlock -from tycho_client.tycho.tycho_adapter import TychoPoolStateStreamAdapter class TestResult: @@ -139,16 +138,27 @@ class TestRunner: node_balance = get_token_balance(token, comp_id, stop_block) if node_balance != tycho_balance: return TestResult.Failed( - f"Balance mismatch for {comp_id}:{token} at block {stop_block}: got {node_balance} from rpc call and {tycho_balance} from Substreams" + f"Balance mismatch for {comp_id}:{token} at block {stop_block}: got {node_balance} " + f"from rpc call and {tycho_balance} from Substreams" ) contract_states = self.tycho_runner.get_contract_state() - self.simulate_get_amount_out( + simulation_failures = self.simulate_get_amount_out( token_balances, stop_block, protocol_states, protocol_components, contract_states, ) + if len(simulation_failures): + error_msgs = [] + for pool_id, failures in simulation_failures.items(): + failures_ = [ + f"{f.sell_token} -> {f.buy_token}: {f.error}" for f in failures + ] + error_msgs.append( + f"Pool {pool_id} failed simulations: {', '.join(failures_)}" + ) + raise ValueError(". ".join(error_msgs)) return TestResult.Passed() except Exception as e: @@ -161,7 +171,7 @@ class TestRunner: protocol_states: dict, protocol_components: dict, contract_state: dict, - ) -> TestResult: + ) -> dict[str, list[SimulationFailure]]: protocol_type_names = self.config["protocol_type_names"] block_header = get_block_header(block_number) @@ -171,12 +181,12 @@ class TestRunner: hash_=block_header.hash.hex(), ) - failed_simulations = dict[str, list[SimulationFailure]] + failed_simulations: dict[str, list[SimulationFailure]] = dict() for protocol in protocol_type_names: - # TODO: Parametrize this - decoder = ThirdPartyPoolTychoDecoder( - "CurveSwapAdapter.evm.runtime", 0, False + adapter_contract = os.path.join( + self.base_dir, "evm", self.config["adapter_contract"] ) + decoder = ThirdPartyPoolTychoDecoder(adapter_contract, 0, False) stream_adapter = TychoPoolStateStreamAdapter( tycho_url="0.0.0.0:4242", protocol=protocol, @@ -225,6 +235,7 @@ class TestRunner: ) ) continue + return failed_simulations @staticmethod def build_spkg(yaml_file_path: str, modify_func: callable) -> str: diff --git a/testing/tycho-client/MANIFEST.in b/testing/tycho-client/MANIFEST.in new file mode 100644 index 0000000..b990f17 --- /dev/null +++ b/testing/tycho-client/MANIFEST.in @@ -0,0 +1,3 @@ +include wheels/*.whl +include tycho_client/assets/* +include tycho_client/bins/* \ No newline at end of file diff --git a/tycho_client/README.md b/testing/tycho-client/README.md similarity index 100% rename from tycho_client/README.md rename to testing/tycho-client/README.md diff --git a/tycho_client/__init__.py b/testing/tycho-client/__init__.py similarity index 100% rename from tycho_client/__init__.py rename to testing/tycho-client/__init__.py diff --git a/tycho_client/requirements.txt b/testing/tycho-client/requirements.txt similarity index 60% rename from tycho_client/requirements.txt rename to testing/tycho-client/requirements.txt index 6fb950e..21fd342 100644 --- a/tycho_client/requirements.txt +++ b/testing/tycho-client/requirements.txt @@ -1,4 +1,3 @@ -./tycho/bins/protosim_py-0.4.9-cp39-cp39-macosx_11_0_arm64.whl requests==2.32.2 eth-abi==2.2.0 eth-typing==2.3.0 diff --git a/testing/tycho-client/setup.py b/testing/tycho-client/setup.py new file mode 100644 index 0000000..0053b58 --- /dev/null +++ b/testing/tycho-client/setup.py @@ -0,0 +1,58 @@ +from setuptools import setup, find_packages +import sys +import platform +from pathlib import Path + + +def read_requirements(): + with open("requirements.txt") as req: + content = req.read() + requirements = content.split("\n") + return [req for req in requirements if req and not req.startswith("#")] + + +# Determine the correct wheel file based on the platform and Python version +def get_wheel_file(): + path = Path(__file__).parent + if sys.platform.startswith("darwin") and platform.machine() == "arm64": + return str( + path / "wheels" / f"protosim_py-0.4.9-cp39-cp39-macosx_11_0_arm64.whl" + ) + elif sys.platform.startswith("linux") and platform.machine() == "x86_64": + return str( + path + / "wheels" + / f"protosim_py-0.4.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + ) + else: + raise RuntimeError("Unsupported platform or architecture") + + +wheel_file = get_wheel_file() + +setup( + name="tycho-client", + version="0.1.0", + author="Propeller Heads", + description="A package for interacting with the Tycho API.", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + packages=find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires="~=3.9", + install_requires=[ + "requests==2.32.2", + "eth-abi==2.2.0", + "eth-typing==2.3.0", + "eth-utils==1.9.5", + "hexbytes==0.3.1", + "pydantic==2.8.2", + f"protosim_py @ file://{wheel_file}", + ], + package_data={"tycho-client": ["../wheels/*", "./assets/*", "./bins/*"]}, + include_package_data=True, +) diff --git a/tycho_client/tycho_client/__init__.py b/testing/tycho-client/tycho_client/__init__.py similarity index 100% rename from tycho_client/tycho_client/__init__.py rename to testing/tycho-client/tycho_client/__init__.py diff --git a/tycho_client/tycho_client/adapter_contract.py b/testing/tycho-client/tycho_client/adapter_contract.py similarity index 100% rename from tycho_client/tycho_client/adapter_contract.py rename to testing/tycho-client/tycho_client/adapter_contract.py diff --git a/tycho_client/tycho_client/assets/ERC20.bin b/testing/tycho-client/tycho_client/assets/ERC20.bin similarity index 100% rename from tycho_client/tycho_client/assets/ERC20.bin rename to testing/tycho-client/tycho_client/assets/ERC20.bin diff --git a/tycho_client/tycho_client/assets/IERC20.sol b/testing/tycho-client/tycho_client/assets/IERC20.sol similarity index 100% rename from tycho_client/tycho_client/assets/IERC20.sol rename to testing/tycho-client/tycho_client/assets/IERC20.sol diff --git a/tycho_client/tycho_client/assets/ISwapAdapter.abi b/testing/tycho-client/tycho_client/assets/ISwapAdapter.abi similarity index 100% rename from tycho_client/tycho_client/assets/ISwapAdapter.abi rename to testing/tycho-client/tycho_client/assets/ISwapAdapter.abi diff --git a/tycho_client/tycho_client/assets/mocked_ERC20.sol b/testing/tycho-client/tycho_client/assets/mocked_ERC20.sol similarity index 100% rename from tycho_client/tycho_client/assets/mocked_ERC20.sol rename to testing/tycho-client/tycho_client/assets/mocked_ERC20.sol diff --git a/tycho_client/tycho_client/constants.py b/testing/tycho-client/tycho_client/constants.py similarity index 90% rename from tycho_client/tycho_client/constants.py rename to testing/tycho-client/tycho_client/constants.py index 5ec8c6d..293482a 100644 --- a/tycho_client/tycho_client/constants.py +++ b/testing/tycho-client/tycho_client/constants.py @@ -1,6 +1,7 @@ from pathlib import Path from typing import Final +ASSETS_FOLDER = Path(__file__).parent / "assets" TYCHO_CLIENT_FOLDER = Path(__file__).parent / "bins" TYCHO_CLIENT_LOG_FOLDER = TYCHO_CLIENT_FOLDER / "logs" diff --git a/tycho_client/tycho_client/decoders.py b/testing/tycho-client/tycho_client/decoders.py similarity index 100% rename from tycho_client/tycho_client/decoders.py rename to testing/tycho-client/tycho_client/decoders.py diff --git a/tycho_client/tycho_client/exceptions.py b/testing/tycho-client/tycho_client/exceptions.py similarity index 100% rename from tycho_client/tycho_client/exceptions.py rename to testing/tycho-client/tycho_client/exceptions.py diff --git a/tycho_client/tycho_client/models.py b/testing/tycho-client/tycho_client/models.py similarity index 100% rename from tycho_client/tycho_client/models.py rename to testing/tycho-client/tycho_client/models.py diff --git a/tycho_client/tycho_client/pool_state.py b/testing/tycho-client/tycho_client/pool_state.py similarity index 100% rename from tycho_client/tycho_client/pool_state.py rename to testing/tycho-client/tycho_client/pool_state.py diff --git a/tycho_client/tycho_client/tycho_adapter.py b/testing/tycho-client/tycho_client/tycho_adapter.py similarity index 99% rename from tycho_client/tycho_client/tycho_adapter.py rename to testing/tycho-client/tycho_client/tycho_adapter.py index 9b2cd9e..1afcbde 100644 --- a/tycho_client/tycho_client/tycho_adapter.py +++ b/testing/tycho-client/tycho_client/tycho_adapter.py @@ -343,4 +343,3 @@ class TychoPoolStateStreamAdapter: block_header = BlockHeader(block.id, block.hash_, int(block.ts.timestamp())) TychoDBSingleton.get_instance().update(vm_updates, block_header) - diff --git a/tycho_client/tycho_client/tycho_db.py b/testing/tycho-client/tycho_client/tycho_db.py similarity index 100% rename from tycho_client/tycho_client/tycho_db.py rename to testing/tycho-client/tycho_client/tycho_db.py diff --git a/tycho_client/tycho_client/utils.py b/testing/tycho-client/tycho_client/utils.py similarity index 95% rename from tycho_client/tycho_client/utils.py rename to testing/tycho-client/tycho_client/utils.py index e1ecab2..f743728 100644 --- a/tycho_client/tycho_client/utils.py +++ b/testing/tycho-client/tycho_client/utils.py @@ -13,7 +13,7 @@ from hexbytes import HexBytes from protosim_py import SimulationEngine, AccountInfo from web3 import Web3 -from .constants import EXTERNAL_ACCOUNT, MAX_BALANCE +from .constants import EXTERNAL_ACCOUNT, MAX_BALANCE, ASSETS_FOLDER from .exceptions import OutOfGas from .models import Address, EthereumToken from .tycho_db import TychoDBSingleton @@ -45,7 +45,9 @@ def create_engine( engine = SimulationEngine.new_with_tycho_db(db=db, trace=trace) for t in mocked_tokens: - info = AccountInfo(balance=0, nonce=0, code=get_contract_bytecode("ERC20.bin")) + info = AccountInfo( + balance=0, nonce=0, code=get_contract_bytecode(ASSETS_FOLDER / "ERC20.bin") + ) engine.init_account( address=t, account=info, mocked=True, permanent_storage=None ) @@ -132,7 +134,8 @@ class ERC20OverwriteFactory: HexBytes(key).hex(): "0x" + HexBytes(val).hex().lstrip("0x").zfill(64) for key, val in self._overwrites.items() } - code = "0x" + get_contract_bytecode("ERC20.bin").hex() + + code = "0x" + get_contract_bytecode(ASSETS_FOLDER / "ERC20.bin").hex() return {self._token.address: {"stateDiff": formatted_overwrites, "code": code}} @@ -181,10 +184,9 @@ def get_storage_slot_at_key(key: Address, mapping_slot: int) -> int: @lru_cache -def get_contract_bytecode(name: str) -> bytes: - """Load contract bytecode from a file in the assets directory""" - # TODO: Check if this locaation is correct - with open(Path(__file__).parent / "assets" / name, "rb") as fh: +def get_contract_bytecode(path: str) -> bytes: + """Load contract bytecode from a file given an absolute path""" + with open(path, "rb") as fh: code = fh.read() return code diff --git a/testing/tycho.py b/testing/tycho.py index acec759..1f7039d 100644 --- a/testing/tycho.py +++ b/testing/tycho.py @@ -1,6 +1,8 @@ import os +import platform import signal import subprocess +import sys import threading import time from pathlib import Path @@ -9,7 +11,19 @@ import psycopg2 import requests from psycopg2 import sql -binary_path = Path(__file__).parent / "tycho-indexer" + +def get_binary_path(): + path = Path(__file__).parent + if sys.platform.startswith("darwin") and platform.machine() == "arm64": + return Path(__file__).parent / "tycho-indexer-mac-arm64" + elif sys.platform.startswith("linux") and platform.machine() == "x86_64": + return Path(__file__).parent / "tycho-indexer-linux-x64" + + else: + raise RuntimeError("Unsupported platform or architecture") + + +binary_path = get_binary_path() class TychoRunner: diff --git a/tycho_client/requirements-docker.txt b/tycho_client/requirements-docker.txt deleted file mode 100644 index 2efee6f..0000000 --- a/tycho_client/requirements-docker.txt +++ /dev/null @@ -1,7 +0,0 @@ -./tycho/bins/protosim_py-0.4.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -requests==2.32.2 -eth-abi==2.2.0 -eth-typing==2.3.0 -eth-utils==1.9.5 -hexbytes==0.3.1 -pydantic==2.8.2 \ No newline at end of file