From d12834cfc5b7def25034cf2fa92ae54d354245fc Mon Sep 17 00:00:00 2001 From: Louise Poole Date: Thu, 31 Oct 2024 13:00:30 +0200 Subject: [PATCH] fix: improve handling of forked protocols on substreams release workflow (#95) * fix: improve pre-release naming and handling of forked protocols * fix: build packages for all forked protocols on a new release --- .github/workflows/substream.cd.yaml | 4 +- substreams/Readme.md | 4 +- .../arbitrum-uniswap-v2.yaml | 2 +- substreams/release.sh | 68 +++++++++++++------ 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/.github/workflows/substream.cd.yaml b/.github/workflows/substream.cd.yaml index cafe987..b068596 100644 --- a/.github/workflows/substream.cd.yaml +++ b/.github/workflows/substream.cd.yaml @@ -11,8 +11,8 @@ on: description: "Package to build" config_file: required: false - description: "Path to the substreams configuration file" - default: "substreams.yaml" + description: "Name of the substreams configuration file" + default: "substreams" jobs: Release: diff --git a/substreams/Readme.md b/substreams/Readme.md index 9dc583e..391c27d 100644 --- a/substreams/Readme.md +++ b/substreams/Readme.md @@ -19,9 +19,11 @@ since this will error. To create a pre release for testing in dev you can start CD pipeline manually supplying the package you'd like to pre release. This will create a -`[package]-[semver].pre-[commit-sha]` release in our spkg repository which you can use +`[package].pre-[commit-sha]` release in our spkg repository which you can use to run the substream´. +For forked protocols you'll need to also supply the config file name, e.g. `ethereum-pancakeswap`. + ## Test your implementation To run a full end-to-end integration test you can refer to the [testing script documentation](../testing/README.md) diff --git a/substreams/ethereum-uniswap-v2/arbitrum-uniswap-v2.yaml b/substreams/ethereum-uniswap-v2/arbitrum-uniswap-v2.yaml index 22b4ee3..a4ff3b8 100644 --- a/substreams/ethereum-uniswap-v2/arbitrum-uniswap-v2.yaml +++ b/substreams/ethereum-uniswap-v2/arbitrum-uniswap-v2.yaml @@ -12,7 +12,7 @@ protobuf: binaries: default: type: wasm/rust-v1 - file: ../../target/wasm32-unknown-unknown/substreams/ethereum_uniswap_v2.wasm + file: ../target/wasm32-unknown-unknown/release/ethereum_uniswap_v2.wasm modules: - name: map_pools_created diff --git a/substreams/release.sh b/substreams/release.sh index 0f1c5be..db6470d 100755 --- a/substreams/release.sh +++ b/substreams/release.sh @@ -14,11 +14,11 @@ if [ -n "$current_tag" ]; then # Prefix without the trailing hyphen (if any) package="${BASH_REMATCH[1]%?}" # Semantic version - version="${BASH_REMATCH[2]}" + version="v${BASH_REMATCH[2]}" cargo_version=$(cargo pkgid -p "$package" | cut -d# -f2 | cut -d: -f2) - if [[ "$cargo_version" != "$version" ]]; then - echo "Error: Cargo version: ${cargo_version} does not match tag version: ${version}!" + if [[ "v$cargo_version" != "$version" ]]; then + echo "Error: Cargo version: v${cargo_version} does not match tag version: ${version}!" exit 1 fi # Check if the Git repository is dirty @@ -33,37 +33,61 @@ if [ -n "$current_tag" ]; then else # If the HEAD is not at a tag, construct the tag name with the pre-release postfix if [ -z "$1" ]; then - echo "Error: package argument is required to create a pre release!" + echo "Error: package argument is required to create a pre-release!" exit 1 fi package=$1 - version_prefix=$(git describe --tags --match "$package-*" --abbrev=0 2>/dev/null) - if [ -z "$version_prefix" ]; then - # If no tags are found in the history, default to version 0.0.1 - version_prefix="0.0.1" - fi - # Get the short commit hash of the current HEAD commit_hash=$(git rev-parse --short HEAD) - version="${version_prefix}-pre.${commit_hash}" + version="pre.${commit_hash}" fi -REPOSITORY=${REPOSITORY:-"s3://repo.propellerheads/substreams"} -repository_path="$REPOSITORY/$package/$package-$version.spkg" +chain_name=$(echo "$package" | cut -d'-' -f1) -# Optional input for yaml file; defaults to substreams.yaml if not provided -yaml_file=${2:-"substreams.yaml"} - -if [[ ! -f "$package/$yaml_file" ]]; then - echo "Error: manifest reader: unable to stat input file $yaml_file: file does not exist." - exit 1 +# Find all YAML files in the specified package directory if no YAML file input is provided +yaml_files=() +if [ -z "$2" ]; then + # Check for YAML files in the package directory and filter by chain name + yaml_files=($(ls "$package"/*.yaml 2>/dev/null | grep "^$package/$chain_name")) + if [ ${#yaml_files[@]} -eq 0 ]; then + echo "Error: No YAML files found in the package directory that match the chain name: $chain_name." + exit 1 + fi +else + yaml_files=("$package/$2.yaml") fi set -e # Exit the script if any command fails cargo build --target wasm32-unknown-unknown --release -p "$package" mkdir -p ./target/spkg/ -substreams pack "$package/$yaml_file" -o ./target/spkg/$package-$version.spkg -aws s3 cp ./target/spkg/$package-$version.spkg $repository_path -echo "Released substreams package: '$repository_path'" +# Loop through each YAML file and build the substreams package +for yaml_file in "${yaml_files[@]}"; do + # Determine the version prefix based on the YAML file name + yaml_name=$(basename "$yaml_file" .yaml) + if [ "$yaml_name" = "buf.gen" ]; then + continue + fi + if [ "$yaml_name" = "substreams" ] && [ ${#yaml_files[@]} -eq 1 ]; then + version_prefix="$package" + else + version_prefix="${yaml_name}" + fi + + echo "------------------------------------------------------" + echo "Building substreams package with config: $yaml_file" + + if [[ ! -f "$yaml_file" ]]; then + echo "Error: manifest reader: unable to stat input file $yaml_file: file does not exist." + exit 1 + fi + + REPOSITORY=${REPOSITORY:-"s3://repo.propellerheads/substreams"} + repository_path="$REPOSITORY/$package/$version_prefix-$version.spkg" + + substreams pack "$yaml_file" -o ./target/spkg/$version_prefix-$version.spkg + aws s3 cp ./target/spkg/$version_prefix-$version.spkg $repository_path + + echo "RELEASED SUBSTREAMS PACKAGE: '$repository_path'" +done