Liquidity.Party/tycho-protocol-sdk
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(runner): Support initialized accounts + misc fixes.

Browse Source 
Simplifies a lot the setup of testing:

- Looks up tycho-indexer under the usual paths no OS specific naming necessary.
- Simply assumes that protosim can be pulled from our private PyPi
- Navigates the foundry out folder to find solidity runtime binaries

Includes some additional fixes to deal with some attribtues that may have to be reflected to defibot later on.
This commit is contained in:
kayibal
2024-07-25 19:31:47 +01:00
parent fcaae2f643
commit 4c337a36d1
7 changed files with 93 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
testing/docker-compose.yaml
View File

@@ -2,6 +2,7 @@ version: '3.1'
services: services:
db: db:
build: build:
context: .
dockerfile: postgres.Dockerfile dockerfile: postgres.Dockerfile
restart: "always" restart: "always"
environment: environment:

2
testing/src/requirements.txt
View File

@@ -2,4 +2,4 @@ psycopg2==2.9.9
PyYAML==6.0.1 PyYAML==6.0.1
Requests==2.32.2 Requests==2.32.2
web3==5.31.3 web3==5.31.3
./tycho-client -e ./tycho-client

4
testing/src/runner/cli.py
View File

@@ -7,7 +7,7 @@ def main() -> None:
description="Run indexer within a specified range of blocks" description="Run indexer within a specified range of blocks"
) )
parser.add_argument( parser.add_argument(
"--test_yaml_path", type=str, help="Path to the test configuration YAML file." "--package", type=str, help="Name of the package to test."
) )
parser.add_argument( parser.add_argument(
"--with_binary_logs", "--with_binary_logs",
@@ -20,7 +20,7 @@ def main() -> None:
args = parser.parse_args() args = parser.parse_args()
test_runner = TestRunner( test_runner = TestRunner(
args.test_yaml_path, args.with_binary_logs, db_url=args.db_url args.package, args.with_binary_logs, db_url=args.db_url
) )
test_runner.run_tests() test_runner.run_tests()

44
testing/src/runner/runner.py
View File

@@ -46,10 +46,13 @@ class SimulationFailure(BaseModel):
class TestRunner: class TestRunner:
def __init__(self, config_path: str, with_binary_logs: bool, db_url: str): def __init__(self, package: str, with_binary_logs: bool, db_url: str):
self.repo_root = os.getcwd()
config_path = os.path.join(self.repo_root, "substreams", package, "test_assets.yaml")
self.config = load_config(config_path) self.config = load_config(config_path)
self.base_dir = os.path.dirname(config_path) self.spkg_src = os.path.join(self.repo_root, "substreams", package)
self.tycho_runner = TychoRunner(with_binary_logs) self.adapters_src = os.path.join(self.repo_root, "evm")
self.tycho_runner = TychoRunner(db_url, with_binary_logs, self.config["initialized_accounts"])
self.tycho_rpc_client = TychoRPCClient() self.tycho_rpc_client = TychoRPCClient()
self.db_url = db_url self.db_url = db_url
self._chain = Blockchain.ethereum self._chain = Blockchain.ethereum
@@ -60,7 +63,7 @@ class TestRunner:
for test in self.config["tests"]: for test in self.config["tests"]:
spkg_path = self.build_spkg( spkg_path = self.build_spkg(
os.path.join(self.base_dir, self.config["substreams_yaml_path"]), os.path.join(self.spkg_src, self.config["substreams_yaml_path"]),
lambda data: self.update_initial_block(data, test["start_block"]), lambda data: self.update_initial_block(data, test["start_block"]),
) )
self.tycho_runner.run_tycho( self.tycho_runner.run_tycho(
@@ -107,7 +110,7 @@ class TestRunner:
) )
if isinstance(value, list): if isinstance(value, list):
if set(map(str.lower, value)) != set( if set(map(str.lower, value)) != set(
map(str.lower, component[key]) map(str.lower, component[key])
): ):
return TestResult.Failed( return TestResult.Failed(
f"List mismatch for key '{key}': {value} != {component[key]}" f"List mismatch for key '{key}': {value} != {component[key]}"
@@ -146,7 +149,6 @@ class TestRunner:
) )
contract_states = self.tycho_rpc_client.get_contract_state() contract_states = self.tycho_rpc_client.get_contract_state()
simulation_failures = self.simulate_get_amount_out( simulation_failures = self.simulate_get_amount_out(
token_balances,
stop_block, stop_block,
protocol_states, protocol_states,
protocol_components, protocol_components,
@@ -169,12 +171,11 @@ class TestRunner:
return TestResult.Failed(error_message) return TestResult.Failed(error_message)
def simulate_get_amount_out( def simulate_get_amount_out(
self, self,
token_balances: dict[str, dict[str, int]], block_number: int,
block_number: int, protocol_states: dict,
protocol_states: dict, protocol_components: dict,
protocol_components: dict, contract_state: dict,
contract_state: dict,
) -> dict[str, list[SimulationFailure]]: ) -> dict[str, list[SimulationFailure]]:
protocol_type_names = self.config["protocol_type_names"] protocol_type_names = self.config["protocol_type_names"]
@@ -188,7 +189,8 @@ class TestRunner:
failed_simulations: dict[str, list[SimulationFailure]] = dict() failed_simulations: dict[str, list[SimulationFailure]] = dict()
for protocol in protocol_type_names: for protocol in protocol_type_names:
adapter_contract = os.path.join( adapter_contract = os.path.join(
self.base_dir, "evm", self.config["adapter_contract"] self.adapters_src, "out", f"{self.config['adapter_contract']}.sol",
f"{self.config['adapter_contract']}.evm.runtime"
) )
decoder = ThirdPartyPoolTychoDecoder(adapter_contract, 0, False) decoder = ThirdPartyPoolTychoDecoder(adapter_contract, 0, False)
stream_adapter = TychoPoolStateStreamAdapter( stream_adapter = TychoPoolStateStreamAdapter(
@@ -204,21 +206,17 @@ class TestRunner:
for pool_state in decoded.pool_states.values(): for pool_state in decoded.pool_states.values():
pool_id = pool_state.id_ pool_id = pool_state.id_
protocol_balances = token_balances.get(pool_id) if not pool_state.balances:
if not protocol_balances:
raise ValueError(f"Missing balances for pool {pool_id}") raise ValueError(f"Missing balances for pool {pool_id}")
for sell_token, buy_token in itertools.permutations( for sell_token, buy_token in itertools.permutations(
pool_state.tokens, 2 pool_state.tokens, 2
): ):
# Try to sell 0.1% of the protocol balance
sell_amount = Decimal("0.001") * pool_state.balances[sell_token.address]
try: try:
# Try to sell 0.1% of the protocol balance
sell_amount = Decimal("0.001") * sell_token.from_onchain_amount(
protocol_balances[sell_token.address]
)
amount_out, gas_used, _ = pool_state.get_amount_out( amount_out, gas_used, _ = pool_state.get_amount_out(
sell_token, sell_amount, buy_token sell_token, sell_amount, buy_token
) )
# TODO: Should we validate this with an archive node or RPC reader?
print( print(
f"Amount out for {pool_id}: {sell_amount} {sell_token} -> {amount_out} {buy_token} - " f"Amount out for {pool_id}: {sell_amount} {sell_token} -> {amount_out} {buy_token} - "
f"Gas used: {gas_used}" f"Gas used: {gas_used}"
@@ -233,8 +231,8 @@ class TestRunner:
failed_simulations[pool_id].append( failed_simulations[pool_id].append(
SimulationFailure( SimulationFailure(
pool_id=pool_id, pool_id=pool_id,
sell_token=sell_token, sell_token=str(sell_token),
buy_token=buy_token, buy_token=str(buy_token),
error=str(e), error=str(e),
) )
) )

73
testing/src/runner/tycho.py
View File

@@ -1,29 +1,42 @@
import os
import platform
import signal import signal
import subprocess import subprocess
import sys
import threading import threading
import time import time
from pathlib import Path
import psycopg2 import psycopg2
import requests import requests
from psycopg2 import sql from psycopg2 import sql
import os
def get_binary_path():
path = Path(__file__).parent
if sys.platform.startswith("darwin") and platform.machine() == "arm64":
return Path(__file__).parent / "tycho-indexer-mac-arm64"
elif sys.platform.startswith("linux") and platform.machine() == "x86_64":
return Path(__file__).parent / "tycho-indexer-linux-x64"
else:
raise RuntimeError("Unsupported platform or architecture")
binary_path = get_binary_path() def find_binary_file(file_name):
# Define usual locations for binary files in Unix-based systems
locations = [
"/bin",
"/sbin",
"/usr/bin",
"/usr/sbin",
"/usr/local/bin",
"/usr/local/sbin",
]
# Add user's local bin directory if it exists
home = os.path.expanduser("~")
if os.path.exists(home + "/.local/bin"):
locations.append(home + "/.local/bin")
# Check each location
for location in locations:
potential_path = location + "/" + file_name
if os.path.exists(potential_path):
return potential_path
# If binary is not found in the usual locations, return None
raise RuntimeError("Unable to locate tycho-indexer binary")
binary_path = find_binary_file("tycho-indexer")
class TychoRPCClient: class TychoRPCClient:
@@ -59,25 +72,29 @@ class TychoRPCClient:
class TychoRunner: class TychoRunner:
def __init__(self, with_binary_logs: bool = False): def __init__(self, db_url: str, with_binary_logs: bool = False, initialized_accounts: list[str] = None):
self.with_binary_logs = with_binary_logs self.with_binary_logs = with_binary_logs
self._db_url = db_url
self._initialized_accounts = initialized_accounts or []
def run_tycho( def run_tycho(
self, self,
spkg_path: str, spkg_path: str,
start_block: int, start_block: int,
end_block: int, end_block: int,
protocol_type_names: list, protocol_type_names: list,
) -> None: ) -> None:
"""Run the Tycho indexer with the specified SPKG and block range.""" """Run the Tycho indexer with the specified SPKG and block range."""
env = os.environ.copy() env = os.environ.copy()
env["RUST_LOG"] = "info" env["RUST_LOG"] = "tycho_indexer=info"
try: try:
process = subprocess.Popen( process = subprocess.Popen(
[ [
binary_path, binary_path,
"--database-url",
self._db_url,
"run", "run",
"--spkg", "--spkg",
spkg_path, spkg_path,
@@ -88,8 +105,11 @@ class TychoRunner:
"--start-block", "--start-block",
str(start_block), str(start_block),
"--stop-block", "--stop-block",
# +2 is to make up for the cache in the index side.
str(end_block + 2), str(end_block + 2),
], # +2 is to make up for the cache in the index side. "--initialized-accounts",
",".join(self._initialized_accounts)
],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, text=True,
@@ -128,7 +148,12 @@ class TychoRunner:
env["RUST_LOG"] = "info" env["RUST_LOG"] = "info"
process = subprocess.Popen( process = subprocess.Popen(
[binary_path, "rpc"], [
binary_path,
"--database-url",
self._db_url,
"rpc"
],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, text=True,

24
testing/tycho-client/setup.py
View File

@@ -1,7 +1,4 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
import sys
import platform
from pathlib import Path
def read_requirements(): def read_requirements():
@@ -11,25 +8,6 @@ def read_requirements():
return [req for req in requirements if req and not req.startswith("#")] return [req for req in requirements if req and not req.startswith("#")]
# Determine the correct wheel file based on the platform and Python version
def get_wheel_file():
path = Path(__file__).parent
if sys.platform.startswith("darwin") and platform.machine() == "arm64":
return str(
path / "wheels" / f"protosim_py-0.4.9-cp39-cp39-macosx_11_0_arm64.whl"
)
elif sys.platform.startswith("linux") and platform.machine() == "x86_64":
return str(
path
/ "wheels"
/ f"protosim_py-0.4.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
)
else:
raise RuntimeError("Unsupported platform or architecture")
wheel_file = get_wheel_file()
setup( setup(
name="tycho-client", name="tycho-client",
version="0.1.0", version="0.1.0",
@@ -51,7 +29,7 @@ setup(
"eth-utils==1.9.5", "eth-utils==1.9.5",
"hexbytes==0.3.1", "hexbytes==0.3.1",
"pydantic==2.8.2", "pydantic==2.8.2",
f"protosim_py @ file://{wheel_file}", "protosim_py==0.4.11",
], ],
package_data={"tycho-client": ["../wheels/*", "./assets/*", "./bins/*"]}, package_data={"tycho-client": ["../wheels/*", "./assets/*", "./bins/*"]},
include_package_data=True, include_package_data=True,

32
testing/tycho-client/tycho_client/decoders.py
View File

@@ -19,10 +19,10 @@ class ThirdPartyPoolTychoDecoder:
self.hard_limit = hard_limit self.hard_limit = hard_limit
def decode_snapshot( def decode_snapshot(
self, self,
snapshot: dict[str, Any], snapshot: dict[str, Any],
block: EVMBlock, block: EVMBlock,
tokens: dict[str, EthereumToken], tokens: dict[str, EthereumToken],
) -> tuple[dict[str, ThirdPartyPool], list[str]]: ) -> tuple[dict[str, ThirdPartyPool], list[str]]:
pools = {} pools = {}
failed_pools = [] failed_pools = []
@@ -38,7 +38,7 @@ class ThirdPartyPoolTychoDecoder:
return pools, failed_pools return pools, failed_pools
def decode_pool_state( def decode_pool_state(
self, snap: dict, block: EVMBlock, tokens: dict[str, EthereumToken] self, snap: dict, block: EVMBlock, tokens: dict[str, EthereumToken]
) -> ThirdPartyPool: ) -> ThirdPartyPool:
component = snap["component"] component = snap["component"]
exchange, _ = decode_tycho_exchange(component["protocol_system"]) exchange, _ = decode_tycho_exchange(component["protocol_system"])
@@ -70,26 +70,30 @@ class ThirdPartyPoolTychoDecoder:
def decode_optional_attributes(component, snap): def decode_optional_attributes(component, snap):
# Handle optional state attributes # Handle optional state attributes
attributes = snap["state"]["attributes"] attributes = snap["state"]["attributes"]
pool_id = attributes.get("pool_id") or component["id"]
balance_owner = attributes.get("balance_owner") balance_owner = attributes.get("balance_owner")
balance_owner = bytes.fromhex(balance_owner[2:] if balance_owner.startswith('0x') else balance_owner).decode(
'utf-8').lower()
stateless_contracts = {} stateless_contracts = {}
static_attributes = snap["component"]["static_attributes"] static_attributes = snap["component"]["static_attributes"]
pool_id = static_attributes.get("pool_id") or component["id"]
pool_id = bytes.fromhex(pool_id[2:]).decode().lower()
index = 0 index = 0
while f"stateless_contract_addr_{index}" in static_attributes: while f"stateless_contract_addr_{index}" in static_attributes:
encoded_address = static_attributes[f"stateless_contract_addr_{index}"] encoded_address = static_attributes[f"stateless_contract_addr_{index}"]
address = bytes.fromhex(encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8') address = bytes.fromhex(
encoded_address[2:] if encoded_address.startswith('0x') else encoded_address).decode('utf-8')
code = static_attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address) code = static_attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address)
stateless_contracts[address] = code stateless_contracts[address] = code
index += 1 index += 1
index = 0 index = 0
while f"stateless_contract_addr_{index}" in attributes: while f"stateless_contract_addr_{index}" in attributes:
address = attributes[f"stateless_contract_addr_{index}"] address = attributes[f"stateless_contract_addr_{index}"]
code = attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address) code = attributes.get(f"stateless_contract_code_{index}") or get_code_for_address(address)
stateless_contracts[address] = code stateless_contracts[address] = code
index += 1 index += 1
return { return {
"balance_owner": balance_owner, "balance_owner": balance_owner,
"pool_id": pool_id, "pool_id": pool_id,
@@ -109,10 +113,10 @@ class ThirdPartyPoolTychoDecoder:
@staticmethod @staticmethod
def apply_update( def apply_update(
pool: ThirdPartyPool, pool: ThirdPartyPool,
pool_update: dict[str, Any], pool_update: dict[str, Any],
balance_updates: dict[str, Any], balance_updates: dict[str, Any],
block: EVMBlock, block: EVMBlock,
) -> ThirdPartyPool: ) -> ThirdPartyPool:
# check for and apply optional state attributes # check for and apply optional state attributes
attributes = pool_update.get("updated_attributes") attributes = pool_update.get("updated_attributes")