fix: improve handling of forked protocols on substreams release workflow (#95)

* fix: improve pre-release naming and handling of forked protocols

* fix: build packages for all forked protocols on a new release

.github/workflows/substream.cd.yaml

@@ -11,8 +11,8 @@ on:
         description: "Package to build"
       config_file:
         required: false
-        description: "Path to the substreams configuration file"
+        description: "Name of the substreams configuration file"
-        default: "substreams.yaml"
+        default: "substreams"
 
 jobs:
   Release:

substreams/Readme.md

@@ -19,9 +19,11 @@ since this will error.
 
 To create a pre release for testing in dev you can start CD pipeline manually supplying 
 the package you'd like to pre release. This will create a 
-`[package]-[semver].pre-[commit-sha]` release in our spkg repository which you can use 
+`[package].pre-[commit-sha]` release in our spkg repository which you can use 
 to run the substream´.
 
+For forked protocols you'll need to also supply the config file name, e.g. `ethereum-pancakeswap`.
+
 ## Test your implementation
 
 To run a full end-to-end integration test you can refer to the [testing script documentation](../testing/README.md)

substreams/ethereum-uniswap-v2/arbitrum-uniswap-v2.yaml

@@ -12,7 +12,7 @@ protobuf:
 binaries:
   default:
     type: wasm/rust-v1
-    file: ../../target/wasm32-unknown-unknown/substreams/ethereum_uniswap_v2.wasm
+    file: ../target/wasm32-unknown-unknown/release/ethereum_uniswap_v2.wasm
 
 modules:
   - name: map_pools_created

substreams/release.sh

@@ -14,11 +14,11 @@ if [ -n "$current_tag" ]; then
         # Prefix without the trailing hyphen (if any)
         package="${BASH_REMATCH[1]%?}"
         # Semantic version
-        version="${BASH_REMATCH[2]}"
+        version="v${BASH_REMATCH[2]}"
 
         cargo_version=$(cargo pkgid -p "$package" | cut -d# -f2 | cut -d: -f2)
-        if [[ "$cargo_version" != "$version" ]]; then
+        if [[ "v$cargo_version" != "$version" ]]; then
-          echo "Error: Cargo version: ${cargo_version} does not match tag version: ${version}!"
+          echo "Error: Cargo version: v${cargo_version} does not match tag version: ${version}!"
           exit 1
         fi
         # Check if the Git repository is dirty
@@ -33,37 +33,61 @@ if [ -n "$current_tag" ]; then
 else
     # If the HEAD is not at a tag, construct the tag name with the pre-release postfix
     if [ -z "$1" ]; then
-        echo "Error: package argument is required to create a pre release!"
+        echo "Error: package argument is required to create a pre-release!"
         exit 1
     fi
     package=$1
 
-    version_prefix=$(git describe --tags --match "$package-*" --abbrev=0 2>/dev/null)
-    if [ -z "$version_prefix" ]; then
-        # If no tags are found in the history, default to version 0.0.1
-        version_prefix="0.0.1"
-    fi
-
     # Get the short commit hash of the current HEAD
     commit_hash=$(git rev-parse --short HEAD)
-    version="${version_prefix}-pre.${commit_hash}"
+    version="pre.${commit_hash}"
 fi
 
-REPOSITORY=${REPOSITORY:-"s3://repo.propellerheads/substreams"}
+chain_name=$(echo "$package" | cut -d'-' -f1)
-repository_path="$REPOSITORY/$package/$package-$version.spkg"
 
-# Optional input for yaml file; defaults to substreams.yaml if not provided
+# Find all YAML files in the specified package directory if no YAML file input is provided
-yaml_file=${2:-"substreams.yaml"}
+yaml_files=()
-
+if [ -z "$2" ]; then
-if [[ ! -f "$package/$yaml_file" ]]; then
+    # Check for YAML files in the package directory and filter by chain name
-    echo "Error: manifest reader: unable to stat input file $yaml_file: file does not exist."
+    yaml_files=($(ls "$package"/*.yaml 2>/dev/null | grep "^$package/$chain_name"))
-    exit 1
+    if [ ${#yaml_files[@]} -eq 0 ]; then
+        echo "Error: No YAML files found in the package directory that match the chain name: $chain_name."
+        exit 1
+    fi
+else
+    yaml_files=("$package/$2.yaml")
 fi
 
 set -e  # Exit the script if any command fails
 cargo build --target wasm32-unknown-unknown --release -p "$package"
 mkdir -p ./target/spkg/
-substreams pack "$package/$yaml_file" -o ./target/spkg/$package-$version.spkg
-aws s3 cp ./target/spkg/$package-$version.spkg $repository_path
 
-echo "Released substreams package: '$repository_path'"
+# Loop through each YAML file and build the substreams package
+for yaml_file in "${yaml_files[@]}"; do
+    # Determine the version prefix based on the YAML file name
+    yaml_name=$(basename "$yaml_file" .yaml)
+    if [ "$yaml_name" = "buf.gen" ]; then
+        continue
+    fi
+    if [ "$yaml_name" = "substreams" ] && [ ${#yaml_files[@]} -eq 1 ]; then
+        version_prefix="$package"
+    else
+        version_prefix="${yaml_name}"
+    fi
+
+    echo "------------------------------------------------------"
+    echo "Building substreams package with config: $yaml_file"
+
+    if [[ ! -f "$yaml_file" ]]; then
+        echo "Error: manifest reader: unable to stat input file $yaml_file: file does not exist."
+        exit 1
+    fi
+
+    REPOSITORY=${REPOSITORY:-"s3://repo.propellerheads/substreams"}
+    repository_path="$REPOSITORY/$package/$version_prefix-$version.spkg"
+
+    substreams pack "$yaml_file" -o ./target/spkg/$version_prefix-$version.spkg
+    aws s3 cp ./target/spkg/$version_prefix-$version.spkg $repository_path
+
+    echo "RELEASED SUBSTREAMS PACKAGE: '$repository_path'"
+done