Merge pull request #67 from propeller-heads/zz/sdk/improve-onboarding-process

refactor: make onboarding easier

testing/.env.default

@@ -1,5 +1,3 @@
-export SUBSTREAMS_PACKAGE=ethereum-curve
 export RPC_URL=https://mainnet.infura.io/v3/your-infura-key
-export DATABASE_URL: "postgres://postgres:mypassword@db:5432/tycho_indexer_0"
 export SUBSTREAMS_API_TOKEN="changeme"
 export DOMAIN_OWNER="AWSAccountId"

testing/README.md

@@ -1,66 +1,116 @@
 # Substreams Testing
 
-This package provides a comprehensive testing suite for Substreams modules. The testing suite is designed to facilitate end-to-end testing, ensuring that your Substreams modules function as expected.
+This package provides a comprehensive testing suite for Substreams modules. The testing suite is designed to facilitate
+end-to-end testing, ensuring that your Substreams modules function as expected.
 
 ## Overview
 
-The testing suite builds the `.spkg` for your Substreams module, indexes a specified block range, and verifies that the expected state has been correctly indexed in PostgreSQL.
+The testing suite builds the `.spkg` for your Substreams module, indexes a specified block range, and verifies that the
+expected state has been correctly indexed in PostgreSQL.
+Additionally, it will also try to simulate some transactions using the `SwapAdapter` interface.
 
 ## Prerequisites
 
-- Latest version of our indexer, Tycho. Please contact us to obtain the latest version. Once acquired, place it in the `/testing/` directory.
+- Latest version of our indexer, Tycho. Please contact us to obtain the latest version. Once acquired, place it in a directory that is included in your system’s PATH.
 - Access to PropellerHeads' private PyPI repository. Please contact us to obtain access.
 - Docker installed on your machine.
+- [Conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html)
+  and [AWS cli](https://aws.amazon.com/cli/) installed
 
 ## Test Configuration
 
-Tests are defined in a `yaml` file. A template can be found at
+Tests are defined in a `yaml` file. A documented template can be found at
 `substreams/ethereum-template/integration_test.tycho.yaml`. The configuration file should include:
 
 - The target Substreams config file.
+- The corresponding SwapAdapter and args to build it.
 - The expected protocol types.
 - The tests to be run.
 
-Each test will index all blocks between `start-block` and `stop-block` and verify that the indexed state matches the expected state.
+Each test will index all blocks between `start-block` and `stop-block`, verify that the indexed state matches the
+expected state and optionally simulate transactions using `SwapAdapter` interface.
 
-You will also need the EVM Runtime file for the adapter contract. 
-The script to generate this file is available under `evm/scripts/buildRuntime.sh`.
-Please place this Runtime file under the respective `substream` directory inside the `evm` folder.
+You will also need the VM Runtime file for the adapter contract.
+Our testing script should be able to build it using your test config.
+The script to generate this file manually is available under `evm/scripts/buildRuntime.sh`.
 
-## Running Tests
+## Setup testing environment
 
 ### Step 1: Export Environment Variables
 
-Export the required environment variables for the execution. You can find the available environment variables in the `.env.default` file.
+**DOMAIN_OWNER**
+
+- **Description**: The domain owner identifier for Propellerhead's AWS account, used for authenticating on the private
+  PyPI repository.
+- **Example**: `export DOMAIN_OWNER=123456789`
+
+### Step 2: Create python virtual environment for testing
+
+Run setup env script. It will create a conda virtual env and install all dependencies.
+This script must be run from within the `propeller-protocol-lib/testing` directory.
+
+Please note that some dependencies require access to our private PyPI repository.
+
+```
+setup_env.sh
+```
+
+## Running Tests
+
+### Prerequisites
+
+This section requires a testing environment setup. If you don’t have it yet, please refer to the [setup testing
+environment section](#setup-testing-environment)
+
+### Step 1: Export Environment Variables
+
+Export the required environment variables for the execution. You can find the available environment variables in the
+`.env.default` file.
 Please create a `.env` file in the `testing` directory and set the required environment variables.
 
 #### Environment Variables
 
-**SUBSTREAMS_PACKAGE**
-- **Description**: Specifies the Substreams module that you want to test
-- **Example**: `export SUBSTREAMS_PACKAGE=ethereum-balancer`
-
-**DATABASE_URL**
-- **Description**: The connection string for the PostgreSQL database. It includes the username, password, host, port, and database name. It's already set to the default for the Docker container.
-- **Example**: `export DATABASE_URL="postgres://postgres:mypassword@localhost:5431/tycho_indexer_0`
-
 **RPC_URL**
-- **Description**: The URL for the Ethereum RPC endpoint. This is used to fetch the storage data. The node needs to be an archive node, and support [debug_storageRangeAt](https://www.quicknode.com/docs/ethereum/debug_storageRangeAt).
+
+- **Description**: The URL for the Ethereum RPC endpoint. This is used to fetch the storage data. The node needs to be
+  an archive node, and support [debug_storageRangeAt](https://www.quicknode.com/docs/ethereum/debug_storageRangeAt).
 - **Example**: `export RPC_URL="https://ethereum-mainnet.core.chainstack.com/123123123123"`
 
 **SUBSTREAMS_API_TOKEN**
+
 - **Description**: The API token for accessing Substreams services. This token is required for authentication.
 - **Example**: `export SUBSTREAMS_API_TOKEN=eyJhbGci...`
 
-**DOMAIN_OWNER**
-- **Description**: The domain owner identifier for Propellerhead's AWS account, used for authenticating on the private PyPI repository.
-- **Example**: `export DOMAIN_OWNER=123456789`
+### Step 2: Run tests
 
-### Step 2: Build and the Testing Script
+Run local postgres database using docker compose
 
-To build the testing script, run the following commands:
 ```bash
-source pre_build.sh
-docker compose build
-docker compose run app
-```
+docker compose up -d db
+```
+
+Run tests for your package.
+
+```bash
+python ./testing/src/runner/cli.py --package "your-package-name"
+```
+
+#### Example
+
+If you want to run tests for `ethereum-balancer`, use:
+
+```bash
+conda activate propeller-protocol-lib-testing
+export RPC_URL="https://ethereum-mainnet.core.chainstack.com/123123123123"
+export SUBSTREAMS_API_TOKEN=eyJhbGci...
+docker compose up -d db
+python ./testing/src/runner/cli.py --package "ethereum-balancer"
+```
+
+#### Testing CLI args
+
+A list and description of all available CLI args can be found using:
+
+```
+python ./testing/src/runner/cli.py --help
+```

testing/requirements.txt

@@ -2,5 +2,5 @@ psycopg2==2.9.9
 PyYAML==6.0.1
 Requests==2.32.2
 web3==5.31.3
-tycho-indexer-client>=0.7.0
-protosim_py>=0.5.0
+tycho-indexer-client>=0.7.2
+protosim_py>=0.6.3

testing/setup_env.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Variables
+ENV_NAME="propeller-protocol-lib-testing"
+PYTHON_VERSION="3.9"
+REQUIREMENTS_FILE="requirements.txt"
+
+# Create conda environment
+echo "Creating conda environment ${ENV_NAME} with Python ${PYTHON_VERSION}..."
+conda create --name $ENV_NAME python=$PYTHON_VERSION -y
+
+# Activate the environment
+echo "Activating the environment..."
+source activate $ENV_NAME
+
+# Install the requirements
+echo "Installing the requirements from ${REQUIREMENTS_FILE}..."
+./pre_build.sh
+pip install -r $REQUIREMENTS_FILE
+conda activate $ENV_NAME
+
+echo "Setup complete."

testing/src/runner/cli.py

@@ -8,10 +8,13 @@ def main() -> None:
     )
     parser.add_argument("--package", type=str, help="Name of the package to test.")
     parser.add_argument(
-        "--tycho-logs", action="store_true", help="Flag to activate logs from Tycho."
+        "--tycho-logs", action="store_true", help="Enable Tycho logs."
     )
     parser.add_argument(
-        "--db-url", type=str, help="Postgres database URL for the Tycho indexer."
+        "--db-url",
+        default="postgres://postgres:mypassword@localhost:5431/tycho_indexer_0",
+        type=str,
+        help="Postgres database URL for the Tycho indexer. Default: postgres://postgres:mypassword@localhost:5431/tycho_indexer_0",
     )
     parser.add_argument(
         "--vm-traces", action="store_true", help="Enable tracing during vm simulations."

testing/src/runner/models.py

@@ -4,8 +4,6 @@ from hexbytes import HexBytes
 from pydantic import BaseModel, Field, validator
 from typing import List, Dict, Optional
 
-from tycho_client.dto import ProtocolComponent
-
 
 class ProtocolComponentExpectation(BaseModel):
     """Represents a ProtocolComponent with its main attributes."""

testing/src/runner/runner.py

@@ -14,7 +14,7 @@ from protosim_py.evm.decoders import ThirdPartyPoolTychoDecoder
 from protosim_py.evm.storage import TychoDBSingleton
 from protosim_py.models import EVMBlock
 from pydantic import BaseModel
-from tycho_client.dto import (
+from tycho_indexer_client.dto import (
     Chain,
     ProtocolComponentsParams,
     ProtocolStateParams,
@@ -26,7 +26,7 @@ from tycho_client.dto import (
     Snapshot,
     ContractId,
 )
-from tycho_client.rpc_client import TychoRPCClient
+from tycho_indexer_client.rpc_client import TychoRPCClient
 
 from models import (
     IntegrationTestsConfig,

testing/src/runner/tycho.py

@@ -27,12 +27,15 @@ def find_binary_file(file_name):
 
     # Check each location
     for location in locations:
-        potential_path = location + "/" + file_name
+        potential_path = os.path.join(location, file_name)
         if os.path.exists(potential_path):
             return potential_path
 
     # If binary is not found in the usual locations, return None
-    raise RuntimeError("Unable to locate tycho-indexer binary")
+    searched_paths = "\n".join(locations)
+    raise RuntimeError(
+        f"Unable to locate {file_name} binary. Searched paths:\n{searched_paths}"
+    )
 
 
 binary_path = find_binary_file("tycho-indexer")

testing/src/runner/utils.py

@@ -3,7 +3,7 @@ from typing import Union
 
 from eth_utils import to_checksum_address
 from protosim_py.models import EthereumToken
-from tycho_client.dto import (
+from tycho_indexer_client.dto import (
     ResponseProtocolState,
     ProtocolComponent,
     ResponseAccount,
@@ -13,7 +13,7 @@ from tycho_client.dto import (
     TokensParams,
     PaginationParams,
 )
-from tycho_client.rpc_client import TychoRPCClient
+from tycho_indexer_client.rpc_client import TychoRPCClient
 
 log = getLogger(__name__)