diff --git a/testing/docker-compose.yaml b/testing/docker-compose.yaml index 9254d47..e8e399e 100644 --- a/testing/docker-compose.yaml +++ b/testing/docker-compose.yaml @@ -2,6 +2,7 @@ version: '3.1' services: db: build: + context: . dockerfile: postgres.Dockerfile restart: "always" environment: diff --git a/testing/src/requirements.txt b/testing/src/requirements.txt index cf5321c..c81212f 100644 --- a/testing/src/requirements.txt +++ b/testing/src/requirements.txt @@ -2,4 +2,4 @@ psycopg2==2.9.9 PyYAML==6.0.1 Requests==2.32.2 web3==5.31.3 -./tycho-client \ No newline at end of file +-e ./tycho-client \ No newline at end of file diff --git a/testing/src/runner/cli.py b/testing/src/runner/cli.py index 109e23e..512408a 100644 --- a/testing/src/runner/cli.py +++ b/testing/src/runner/cli.py @@ -7,7 +7,7 @@ def main() -> None: description="Run indexer within a specified range of blocks" ) parser.add_argument( - "--test_yaml_path", type=str, help="Path to the test configuration YAML file." + "--package", type=str, help="Name of the package to test." ) parser.add_argument( "--with_binary_logs", @@ -20,7 +20,7 @@ def main() -> None: args = parser.parse_args() test_runner = TestRunner( - args.test_yaml_path, args.with_binary_logs, db_url=args.db_url + args.package, args.with_binary_logs, db_url=args.db_url ) test_runner.run_tests() diff --git a/testing/src/runner/runner.py b/testing/src/runner/runner.py index cf684ff..2e68dbb 100644 --- a/testing/src/runner/runner.py +++ b/testing/src/runner/runner.py @@ -46,10 +46,13 @@ class SimulationFailure(BaseModel): class TestRunner: - def __init__(self, config_path: str, with_binary_logs: bool, db_url: str): + def __init__(self, package: str, with_binary_logs: bool, db_url: str): + self.repo_root = os.getcwd() + config_path = os.path.join(self.repo_root, "substreams", package, "test_assets.yaml") self.config = load_config(config_path) - self.base_dir = os.path.dirname(config_path) - self.tycho_runner = TychoRunner(with_binary_logs) + self.spkg_src = os.path.join(self.repo_root, "substreams", package) + self.adapters_src = os.path.join(self.repo_root, "evm") + self.tycho_runner = TychoRunner(db_url, with_binary_logs, self.config["initialized_accounts"]) self.tycho_rpc_client = TychoRPCClient() self.db_url = db_url self._chain = Blockchain.ethereum @@ -60,7 +63,7 @@ class TestRunner: for test in self.config["tests"]: spkg_path = self.build_spkg( - os.path.join(self.base_dir, self.config["substreams_yaml_path"]), + os.path.join(self.spkg_src, self.config["substreams_yaml_path"]), lambda data: self.update_initial_block(data, test["start_block"]), ) self.tycho_runner.run_tycho( @@ -107,7 +110,7 @@ class TestRunner: ) if isinstance(value, list): if set(map(str.lower, value)) != set( - map(str.lower, component[key]) + map(str.lower, component[key]) ): return TestResult.Failed( f"List mismatch for key '{key}': {value} != {component[key]}" @@ -146,7 +149,6 @@ class TestRunner: ) contract_states = self.tycho_rpc_client.get_contract_state() simulation_failures = self.simulate_get_amount_out( - token_balances, stop_block, protocol_states, protocol_components, @@ -169,12 +171,11 @@ class TestRunner: return TestResult.Failed(error_message) def simulate_get_amount_out( - self, - token_balances: dict[str, dict[str, int]], - block_number: int, - protocol_states: dict, - protocol_components: dict, - contract_state: dict, + self, + block_number: int, + protocol_states: dict, + protocol_components: dict, + contract_state: dict, ) -> dict[str, list[SimulationFailure]]: protocol_type_names = self.config["protocol_type_names"] @@ -188,7 +189,8 @@ class TestRunner: failed_simulations: dict[str, list[SimulationFailure]] = dict() for protocol in protocol_type_names: adapter_contract = os.path.join( - self.base_dir, "evm", self.config["adapter_contract"] + self.adapters_src, "out", f"{self.config['adapter_contract']}.sol", + f"{self.config['adapter_contract']}.evm.runtime" ) decoder = ThirdPartyPoolTychoDecoder(adapter_contract, 0, False) stream_adapter = TychoPoolStateStreamAdapter( @@ -204,21 +206,17 @@ class TestRunner: for pool_state in decoded.pool_states.values(): pool_id = pool_state.id_ - protocol_balances = token_balances.get(pool_id) - if not protocol_balances: + if not pool_state.balances: raise ValueError(f"Missing balances for pool {pool_id}") for sell_token, buy_token in itertools.permutations( - pool_state.tokens, 2 + pool_state.tokens, 2 ): + # Try to sell 0.1% of the protocol balance + sell_amount = Decimal("0.001") * pool_state.balances[sell_token.address] try: - # Try to sell 0.1% of the protocol balance - sell_amount = Decimal("0.001") * sell_token.from_onchain_amount( - protocol_balances[sell_token.address] - ) amount_out, gas_used, _ = pool_state.get_amount_out( sell_token, sell_amount, buy_token ) - # TODO: Should we validate this with an archive node or RPC reader? print( f"Amount out for {pool_id}: {sell_amount} {sell_token} -> {amount_out} {buy_token} - " f"Gas used: {gas_used}" @@ -233,8 +231,8 @@ class TestRunner: failed_simulations[pool_id].append( SimulationFailure( pool_id=pool_id, - sell_token=sell_token, - buy_token=buy_token, + sell_token=str(sell_token), + buy_token=str(buy_token), error=str(e), ) ) diff --git a/testing/src/runner/tycho.py b/testing/src/runner/tycho.py index fc917d0..31fcafb 100644 --- a/testing/src/runner/tycho.py +++ b/testing/src/runner/tycho.py @@ -1,29 +1,42 @@ -import os -import platform import signal import subprocess -import sys import threading import time -from pathlib import Path import psycopg2 import requests from psycopg2 import sql - -def get_binary_path(): - path = Path(__file__).parent - if sys.platform.startswith("darwin") and platform.machine() == "arm64": - return Path(__file__).parent / "tycho-indexer-mac-arm64" - elif sys.platform.startswith("linux") and platform.machine() == "x86_64": - return Path(__file__).parent / "tycho-indexer-linux-x64" - - else: - raise RuntimeError("Unsupported platform or architecture") +import os -binary_path = get_binary_path() +def find_binary_file(file_name): + # Define usual locations for binary files in Unix-based systems + locations = [ + "/bin", + "/sbin", + "/usr/bin", + "/usr/sbin", + "/usr/local/bin", + "/usr/local/sbin", + ] + + # Add user's local bin directory if it exists + home = os.path.expanduser("~") + if os.path.exists(home + "/.local/bin"): + locations.append(home + "/.local/bin") + + # Check each location + for location in locations: + potential_path = location + "/" + file_name + if os.path.exists(potential_path): + return potential_path + + # If binary is not found in the usual locations, return None + raise RuntimeError("Unable to locate tycho-indexer binary") + + +binary_path = find_binary_file("tycho-indexer") class TychoRPCClient: @@ -59,25 +72,29 @@ class TychoRPCClient: class TychoRunner: - def __init__(self, with_binary_logs: bool = False): + def __init__(self, db_url: str, with_binary_logs: bool = False, initialized_accounts: list[str] = None): self.with_binary_logs = with_binary_logs + self._db_url = db_url + self._initialized_accounts = initialized_accounts or [] def run_tycho( - self, - spkg_path: str, - start_block: int, - end_block: int, - protocol_type_names: list, + self, + spkg_path: str, + start_block: int, + end_block: int, + protocol_type_names: list, ) -> None: """Run the Tycho indexer with the specified SPKG and block range.""" env = os.environ.copy() - env["RUST_LOG"] = "info" + env["RUST_LOG"] = "tycho_indexer=info" try: process = subprocess.Popen( [ binary_path, + "--database-url", + self._db_url, "run", "--spkg", spkg_path, @@ -88,8 +105,11 @@ class TychoRunner: "--start-block", str(start_block), "--stop-block", + # +2 is to make up for the cache in the index side. str(end_block + 2), - ], # +2 is to make up for the cache in the index side. + "--initialized-accounts", + ",".join(self._initialized_accounts) + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -128,7 +148,12 @@ class TychoRunner: env["RUST_LOG"] = "info" process = subprocess.Popen( - [binary_path, "rpc"], + [ + binary_path, + "--database-url", + self._db_url, + "rpc" + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, diff --git a/testing/tycho-client/setup.py b/testing/tycho-client/setup.py index a47dfe8..9692a44 100644 --- a/testing/tycho-client/setup.py +++ b/testing/tycho-client/setup.py @@ -1,7 +1,4 @@ from setuptools import setup, find_packages -import sys -import platform -from pathlib import Path def read_requirements(): @@ -11,25 +8,6 @@ def read_requirements(): return [req for req in requirements if req and not req.startswith("#")] -# Determine the correct wheel file based on the platform and Python version -def get_wheel_file(): - path = Path(__file__).parent - if sys.platform.startswith("darwin") and platform.machine() == "arm64": - return str( - path / "wheels" / f"protosim_py-0.4.9-cp39-cp39-macosx_11_0_arm64.whl" - ) - elif sys.platform.startswith("linux") and platform.machine() == "x86_64": - return str( - path - / "wheels" - / f"protosim_py-0.4.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - ) - else: - raise RuntimeError("Unsupported platform or architecture") - - -wheel_file = get_wheel_file() - setup( name="tycho-client", version="0.1.0", @@ -51,7 +29,7 @@ setup( "eth-utils==1.9.5", "hexbytes==0.3.1", "pydantic==2.8.2", - f"protosim_py @ file://{wheel_file}", + "protosim_py==0.4.11", ], package_data={"tycho-client": ["../wheels/*", "./assets/*", "./bins/*"]}, include_package_data=True, diff --git a/testing/tycho-client/tycho_client/decoders.py b/testing/tycho-client/tycho_client/decoders.py index 52b6e01..c5de40a 100644 --- a/testing/tycho-client/tycho_client/decoders.py +++ b/testing/tycho-client/tycho_client/decoders.py @@ -19,10 +19,10 @@ class ThirdPartyPoolTychoDecoder: self.hard_limit = hard_limit def decode_snapshot( - self, - snapshot: dict[str, Any], - block: EVMBlock, - tokens: dict[str, EthereumToken], + self, + snapshot: dict[str, Any], + block: EVMBlock, + tokens: dict[str, EthereumToken], ) -> tuple[dict[str, ThirdPartyPool], list[str]]: pools = {} failed_pools = [] @@ -38,7 +38,7 @@ class ThirdPartyPoolTychoDecoder: return pools, failed_pools def decode_pool_state( - self, snap: dict, block: EVMBlock, tokens: dict[str, EthereumToken] + self, snap: dict, block: EVMBlock, tokens: dict[str, EthereumToken] ) -> ThirdPartyPool: component = snap["component"] exchange, _ = decode_tycho_exchange(component["protocol_system"]) @@ -70,26 +70,30 @@ class ThirdPartyPoolTychoDecoder: def decode_optional_attributes(component, snap): # Handle optional state attributes attributes = snap["state"]["attributes"] - pool_id = attributes.get("pool_id") or component["id"] balance_owner = attributes.get("balance_owner") + balance_owner = bytes.fromhex(balance_owner[2:] if balance_owner.startswith('0x') else balance_owner).decode( + 'utf-8').lower() stateless_contracts = {} static_attributes = snap["component"]["static_attributes"] - + pool_id = static_attributes.get("pool_id") or component["id"] + pool_id = bytes.fromhex(pool_id[2:]).decode().lower() + index = 0 while f"stateless_contract_addr_{index}" in static_attributes: encoded_address = static_attributes[f"stateless_contract_addr_{index}"] - address = bytes.fromhex(encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8') + address = bytes.fromhex( + encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8') code = static_attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address) stateless_contracts[address] = code index += 1 - + index = 0 while f"stateless_contract_addr_{index}" in attributes: address = attributes[f"stateless_contract_addr_{index}"] code = attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address) stateless_contracts[address] = code - index += 1 + index += 1 return { "balance_owner": balance_owner, "pool_id": pool_id, @@ -109,10 +113,10 @@ class ThirdPartyPoolTychoDecoder: @staticmethod def apply_update( - pool: ThirdPartyPool, - pool_update: dict[str, Any], - balance_updates: dict[str, Any], - block: EVMBlock, + pool: ThirdPartyPool, + pool_update: dict[str, Any], + balance_updates: dict[str, Any], + block: EVMBlock, ) -> ThirdPartyPool: # check for and apply optional state attributes attributes = pool_update.get("updated_attributes")