Merge pull request #16 from propeller-heads/ah/ci-and-cd

CI & CD for Substreams
This commit is contained in:
Alan Höng
2024-03-14 10:54:58 +00:00
committed by GitHub
24 changed files with 497 additions and 1449 deletions

View File

@@ -1,6 +1,9 @@
name: test name: test evm
on: workflow_dispatch on:
push:
paths:
- "evm/**"
env: env:
FOUNDRY_PROFILE: ci FOUNDRY_PROFILE: ci

45
.github/workflows/substream.cd.yaml vendored Normal file
View File

@@ -0,0 +1,45 @@
name: Substreams CD
on:
push:
tags:
- "ethereum-*"
workflow_dispatch:
inputs:
package:
required: true
description: "Package to build"
jobs:
Release:
name: Release ${{ inputs.package }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Setup toolchain
uses: dtolnay/rust-toolchain@v1
with:
toolchain: stable
targets: wasm32-unknown-unknown
- name: Setup Rust Cache
uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install Substreams CLI
run: |
# Use correct binary for your platform
LINK=$(curl -s https://api.github.com/repos/streamingfast/substreams/releases/latest | awk "/download.url.*linux_$(uname -m)/ {print \$2}" | sed 's/"//g')
curl -L "$LINK" | tar zxf - -C /usr/local/bin
chmod +x /usr/local/bin/substreams
substreams --version
- name: Run checks
run: |
cd substreams
./release.sh ${{ inputs.package }}

59
.github/workflows/substream.ci.yaml vendored Normal file
View File

@@ -0,0 +1,59 @@
name: Substreams CI
on:
push:
paths:
- "substreams/**"
jobs:
lint:
name: Substreams Lint
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Setup toolchain
uses: dtolnay/rust-toolchain@v1
with:
toolchain: nightly
components: clippy, rustfmt
- name: Setup Rust Cache
uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run checks
run: |
cd substreams
cargo +nightly fmt -- --check
cargo +nightly clippy --all --all-features --all-targets -- -D warnings
test:
name: Substreams Test
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Setup toolchain
uses: dtolnay/rust-toolchain@v1
with:
toolchain: stable
targets: wasm32-unknown-unknown
- name: Setup Rust Cache
uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run checks
run: |
cd substreams
cargo build --target wasm32-unknown-unknown --all-targets --all-features
cargo test --workspace --all-targets --all-features

View File

@@ -1,11 +0,0 @@
[package]
name = "tycho-substreams"
version = "0.1.0"
edition = "2021"
[dependencies]
substreams-ethereum = "0.9.9"
substreams = "0.5"
prost = "0.11"
hex = "0.4.3"
itertools = "0.12.0"

View File

@@ -13,9 +13,9 @@ dependencies = [
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.80" version = "1.0.81"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
[[package]] [[package]]
name = "arrayvec" name = "arrayvec"
@@ -154,7 +154,24 @@ version = "17.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b"
dependencies = [ dependencies = [
"ethereum-types", "ethereum-types 0.13.1",
"hex",
"once_cell",
"regex",
"serde",
"serde_json",
"sha3",
"thiserror",
"uint",
]
[[package]]
name = "ethabi"
version = "18.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7413c5f74cc903ea37386a8965a936cbeb334bd270862fdece542c1b2dcbc898"
dependencies = [
"ethereum-types 0.14.1",
"hex", "hex",
"once_cell", "once_cell",
"regex", "regex",
@@ -172,23 +189,69 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef"
dependencies = [ dependencies = [
"crunchy", "crunchy",
"fixed-hash", "fixed-hash 0.7.0",
"impl-rlp", "impl-rlp",
"impl-serde", "impl-serde 0.3.2",
"tiny-keccak", "tiny-keccak",
] ]
[[package]]
name = "ethbloom"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60"
dependencies = [
"crunchy",
"fixed-hash 0.8.0",
"impl-rlp",
"impl-serde 0.4.0",
"tiny-keccak",
]
[[package]]
name = "ethereum-balancer"
version = "0.1.0"
dependencies = [
"anyhow",
"bytes",
"ethabi 18.0.0",
"getrandom",
"hex",
"hex-literal 0.4.1",
"itertools 0.12.1",
"num-bigint",
"prost 0.11.9",
"prost-types 0.12.3",
"substreams",
"substreams-ethereum",
"tycho-substreams",
]
[[package]] [[package]]
name = "ethereum-types" name = "ethereum-types"
version = "0.13.1" version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6"
dependencies = [ dependencies = [
"ethbloom", "ethbloom 0.12.1",
"fixed-hash", "fixed-hash 0.7.0",
"impl-rlp", "impl-rlp",
"impl-serde", "impl-serde 0.3.2",
"primitive-types", "primitive-types 0.11.1",
"uint",
]
[[package]]
name = "ethereum-types"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02d215cbf040552efcbe99a38372fe80ab9d00268e20012b79fcd0f073edd8ee"
dependencies = [
"ethbloom 0.13.0",
"fixed-hash 0.8.0",
"impl-rlp",
"impl-serde 0.4.0",
"primitive-types 0.12.2",
"uint", "uint",
] ]
@@ -210,6 +273,18 @@ dependencies = [
"static_assertions", "static_assertions",
] ]
[[package]]
name = "fixed-hash"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534"
dependencies = [
"byteorder",
"rand",
"rustc-hex",
"static_assertions",
]
[[package]] [[package]]
name = "fixedbitset" name = "fixedbitset"
version = "0.4.2" version = "0.4.2"
@@ -267,6 +342,12 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0"
[[package]]
name = "hex-literal"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46"
[[package]] [[package]]
name = "home" name = "home"
version = "0.5.9" version = "0.5.9"
@@ -303,6 +384,15 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "impl-serde"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "impl-trait-for-tuples" name = "impl-trait-for-tuples"
version = "0.2.2" version = "0.2.2"
@@ -495,10 +585,23 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a"
dependencies = [ dependencies = [
"fixed-hash", "fixed-hash 0.7.0",
"impl-codec", "impl-codec",
"impl-rlp", "impl-rlp",
"impl-serde", "impl-serde 0.3.2",
"uint",
]
[[package]]
name = "primitive-types"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2"
dependencies = [
"fixed-hash 0.8.0",
"impl-codec",
"impl-rlp",
"impl-serde 0.4.0",
"uint", "uint",
] ]
@@ -514,9 +617,9 @@ dependencies = [
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.78" version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
dependencies = [ dependencies = [
"unicode-ident", "unicode-ident",
] ]
@@ -528,7 +631,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
dependencies = [ dependencies = [
"bytes", "bytes",
"prost-derive", "prost-derive 0.11.9",
]
[[package]]
name = "prost"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
dependencies = [
"bytes",
"prost-derive 0.12.3",
] ]
[[package]] [[package]]
@@ -545,8 +658,8 @@ dependencies = [
"multimap", "multimap",
"petgraph", "petgraph",
"prettyplease", "prettyplease",
"prost", "prost 0.11.9",
"prost-types", "prost-types 0.11.9",
"regex", "regex",
"syn 1.0.109", "syn 1.0.109",
"tempfile", "tempfile",
@@ -566,13 +679,35 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "prost-derive"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"
dependencies = [
"anyhow",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 2.0.52",
]
[[package]] [[package]]
name = "prost-types" name = "prost-types"
version = "0.11.9" version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13"
dependencies = [ dependencies = [
"prost", "prost 0.11.9",
]
[[package]]
name = "prost-types"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e"
dependencies = [
"prost 0.12.3",
] ]
[[package]] [[package]]
@@ -740,14 +875,14 @@ dependencies = [
"anyhow", "anyhow",
"bigdecimal", "bigdecimal",
"hex", "hex",
"hex-literal", "hex-literal 0.3.4",
"num-bigint", "num-bigint",
"num-integer", "num-integer",
"num-traits", "num-traits",
"pad", "pad",
"prost", "prost 0.11.9",
"prost-build", "prost-build",
"prost-types", "prost-types 0.11.9",
"substreams-macro", "substreams-macro",
"thiserror", "thiserror",
] ]
@@ -773,7 +908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c04307913a355aaf2a1bb7186d4bc7e36875f3d4aff77b47e83f1b63b24da55" checksum = "9c04307913a355aaf2a1bb7186d4bc7e36875f3d4aff77b47e83f1b63b24da55"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"ethabi", "ethabi 17.2.0",
"heck", "heck",
"hex", "hex",
"prettyplease", "prettyplease",
@@ -790,12 +925,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db9048cc9a66873ab7069ef958c2684994e6ee323da49c186b19156fdb4ca131" checksum = "db9048cc9a66873ab7069ef958c2684994e6ee323da49c186b19156fdb4ca131"
dependencies = [ dependencies = [
"bigdecimal", "bigdecimal",
"ethabi", "ethabi 17.2.0",
"getrandom", "getrandom",
"num-bigint", "num-bigint",
"prost", "prost 0.11.9",
"prost-build", "prost-build",
"prost-types", "prost-types 0.11.9",
"substreams", "substreams",
] ]
@@ -805,7 +940,7 @@ version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e862928bee8653f5c9291ac619c8dc0da14ca61d8cd8d89b3acdbbde4d0bf304" checksum = "e862928bee8653f5c9291ac619c8dc0da14ca61d8cd8d89b3acdbbde4d0bf304"
dependencies = [ dependencies = [
"ethabi", "ethabi 17.2.0",
"heck", "heck",
"hex", "hex",
"num-bigint", "num-bigint",
@@ -869,18 +1004,18 @@ dependencies = [
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.57" version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.57" version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@@ -919,7 +1054,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"hex", "hex",
"itertools 0.12.1", "itertools 0.12.1",
"prost", "prost 0.11.9",
"substreams", "substreams",
"substreams-ethereum", "substreams-ethereum",
] ]

22
substreams/Cargo.toml Normal file
View File

@@ -0,0 +1,22 @@
[workspace]
members = [
"ethereum-balancer",
"crates/tycho-substreams",
]
resolver = "2"
[workspace.dependencies]
substreams-ethereum = "0.9.9"
substreams = "0.5"
prost = "0.11"
prost-types = "0.12.3"
hex-literal = "0.4.1"
hex = "0.4.3"
ethabi = "18.0.0"
tycho-substreams = {path ="crates/tycho-substreams"}
[profile.release]
lto = true
opt-level = 's'
strip = "debuginfo"

6
substreams/check.sh Executable file
View File

@@ -0,0 +1,6 @@
set -e
cargo +nightly fmt -- --check
cargo +nightly clippy --all --all-features --all-targets -- -D warnings
cargo build --target wasm32-unknown-unknown --all-targets --all-features
cargo test --workspace --all-targets --all-features

View File

@@ -0,0 +1,11 @@
[package]
name = "tycho-substreams"
version = "0.1.0"
edition = "2021"
[dependencies]
substreams-ethereum.workspace = true
substreams.workspace = true
prost.workspace = true
hex.workspace = true
itertools = "0.12.0"

View File

@@ -13,6 +13,6 @@ directory:
```bash ```bash
buf generate \ buf generate \
--path tycho \ --path tycho \
--template ../crates/tycho-substreams/buf.gen.yaml \ --template ../substreams/crates/tycho-substreams/buf.gen.yaml \
--output ../crates/tycho-substreams/ --output ../substreams/crates/tycho-substreams/
``` ```

View File

@@ -3,21 +3,21 @@
//! //!
//! To aggregate relative balances changes to absolute balances the general approach is: //! To aggregate relative balances changes to absolute balances the general approach is:
//! //!
//! 1. Use a map function that will extract a `BlockBalanceDeltas` message. BalanceDeltas //! 1. Use a map function that will extract a `BlockBalanceDeltas` message. BalanceDeltas within
//! within this message are required to have increasing ordinals so that //! this message are required to have increasing ordinals so that the order of relative balance
//! the order of relative balance changes is unambiguous. //! changes is unambiguous.
//! 2. Store the balances changes with a store handler. You can use the //! 2. Store the balances changes with a store handler. You can use the `store_balance_changes`
//! `store_balance_changes` library method directly for this. //! library method directly for this.
//! 3. In the output module, use aggregate_balance_changes to receive an //! 3. In the output module, use aggregate_balance_changes to receive an aggregated map of absolute
//! aggregated map of absolute balances. //! balances.
//!
use crate::pb::tycho::evm::v1::{BalanceChange, BlockBalanceDeltas, Transaction}; use crate::pb::tycho::evm::v1::{BalanceChange, BlockBalanceDeltas, Transaction};
use itertools::Itertools; use itertools::Itertools;
use std::collections::HashMap; use std::{collections::HashMap, str::FromStr};
use std::str::FromStr; use substreams::{
use substreams::key; key,
use substreams::pb::substreams::StoreDeltas; pb::substreams::StoreDeltas,
use substreams::prelude::{BigInt, StoreAdd}; prelude::{BigInt, StoreAdd},
};
/// Store relative balances changes in a additive manner. /// Store relative balances changes in a additive manner.
/// ///
@@ -25,9 +25,9 @@ use substreams::prelude::{BigInt, StoreAdd};
/// ///
/// ## Arguments /// ## Arguments
/// ///
/// * `deltas` - A `BlockBalanceDeltas` message containing the relative balances changes. /// * `deltas` - A `BlockBalanceDeltas` message containing the relative balances changes. Note:
/// Note: relative balance deltas must have strictly increasing ordinals per token /// relative balance deltas must have strictly increasing ordinals per token address, will panic
/// address, will panic otherwise. /// otherwise.
/// * `store` - An AddStore that will add relative balance changes. /// * `store` - An AddStore that will add relative balance changes.
/// ///
/// This method is meant to be used in combination with `aggregate_balances_changes` /// This method is meant to be used in combination with `aggregate_balances_changes`
@@ -63,11 +63,12 @@ pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: impl StoreAdd<Bi
}); });
} }
type TxAggregatedBalances = HashMap<Vec<u8>, (Transaction, HashMap<Vec<u8>, BalanceChange>)>;
/// Aggregates absolute balances per transaction and token. /// Aggregates absolute balances per transaction and token.
/// ///
/// ## Arguments /// ## Arguments
/// * `balance_store` - A `StoreDeltas` with all changes that occured in the source /// * `balance_store` - A `StoreDeltas` with all changes that occured in the source store module.
/// store module.
/// * `deltas` - A `BlockBalanceDeltas` message containing the relative balances changes. /// * `deltas` - A `BlockBalanceDeltas` message containing the relative balances changes.
/// ///
/// Reads absolute balance values from the additive store (see `store_balance_changes` /// Reads absolute balance values from the additive store (see `store_balance_changes`
@@ -82,7 +83,7 @@ pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: impl StoreAdd<Bi
pub fn aggregate_balances_changes( pub fn aggregate_balances_changes(
balance_store: StoreDeltas, balance_store: StoreDeltas,
deltas: BlockBalanceDeltas, deltas: BlockBalanceDeltas,
) -> HashMap<Vec<u8>, (Transaction, HashMap<Vec<u8>, BalanceChange>)> { ) -> TxAggregatedBalances {
balance_store balance_store
.deltas .deltas
.into_iter() .into_iter()
@@ -125,10 +126,11 @@ pub fn aggregate_balances_changes(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::mock_store::MockStore; use crate::{mock_store::MockStore, pb::tycho::evm::v1::BalanceDelta};
use crate::pb::tycho::evm::v1::{BalanceDelta, Transaction}; use substreams::{
use substreams::pb::substreams::StoreDelta; pb::substreams::StoreDelta,
use substreams::prelude::{StoreGet, StoreNew}; prelude::{StoreGet, StoreNew},
};
fn block_balance_deltas() -> BlockBalanceDeltas { fn block_balance_deltas() -> BlockBalanceDeltas {
let comp_id = "0x42c0ffee" let comp_id = "0x42c0ffee"
@@ -207,9 +209,9 @@ mod tests {
let token_1 = hex::decode("babe00").unwrap(); let token_1 = hex::decode("babe00").unwrap();
let t0_key = let t0_key =
format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(&token_0)); format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(token_0));
let t1_key = let t1_key =
format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(&token_1)); format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(token_1));
StoreDeltas { StoreDeltas {
deltas: vec![ deltas: vec![
StoreDelta { StoreDelta {
@@ -283,12 +285,12 @@ mod tests {
let res_0 = store.get_last(format!( let res_0 = store.get_last(format!(
"{}:{}", "{}:{}",
String::from_utf8(comp_id.clone()).unwrap(), String::from_utf8(comp_id.clone()).unwrap(),
hex::encode(&token_0) hex::encode(token_0)
)); ));
let res_1 = store.get_last(format!( let res_1 = store.get_last(format!(
"{}:{}", "{}:{}",
String::from_utf8(comp_id.clone()).unwrap(), String::from_utf8(comp_id.clone()).unwrap(),
hex::encode(&token_1) hex::encode(token_1)
)); ));
assert_eq!(res_0, Some(BigInt::from_str("+999").unwrap())); assert_eq!(res_0, Some(BigInt::from_str("+999").unwrap()));

View File

@@ -10,9 +10,10 @@
/// more [here](https://streamingfastio.medium.com/new-block-model-to-accelerate-chain-integration-9f65126e5425) /// more [here](https://streamingfastio.medium.com/new-block-model-to-accelerate-chain-integration-9f65126e5425)
use std::collections::HashMap; use std::collections::HashMap;
use substreams_ethereum::pb::eth; use substreams_ethereum::pb::{
use substreams_ethereum::pb::eth::v2::block::DetailLevel; eth,
use substreams_ethereum::pb::eth::v2::StorageChange; eth::v2::{block::DetailLevel, StorageChange},
};
use crate::pb::tycho::evm::v1::{self as tycho}; use crate::pb::tycho::evm::v1::{self as tycho};
@@ -49,11 +50,7 @@ impl InterimContractChange {
balance: vec![], balance: vec![],
code: vec![], code: vec![],
slots: Default::default(), slots: Default::default(),
change: if creation { change: if creation { tycho::ChangeType::Creation } else { tycho::ChangeType::Update },
tycho::ChangeType::Creation.into()
} else {
tycho::ChangeType::Update.into()
},
} }
} }
} }
@@ -82,7 +79,8 @@ impl From<InterimContractChange> for tycho::ContractChange {
/// ## Arguments /// ## Arguments
/// ///
/// * `block` - The block to extract changes from. Must be the extended block model. /// * `block` - The block to extract changes from. Must be the extended block model.
/// * `inclusion_predicate` - A predicate function that determines if a contract address is relevant. /// * `inclusion_predicate` - A predicate function that determines if a contract address is
/// relevant.
/// * `transaction_contract_changes` - A mutable map to store the contract changes in. /// * `transaction_contract_changes` - A mutable map to store the contract changes in.
/// ///
/// ## Panics /// ## Panics
@@ -193,9 +191,9 @@ pub fn extract_contract_changes<F: Fn(&[u8]) -> bool>(
.extend_from_slice(&code_change.new_code); .extend_from_slice(&code_change.new_code);
}); });
if !storage_changes.is_empty() if !storage_changes.is_empty() ||
|| !balance_changes.is_empty() !balance_changes.is_empty() ||
|| !code_changes.is_empty() !code_changes.is_empty()
{ {
transaction_contract_changes transaction_contract_changes
.entry(block_tx.index.into()) .entry(block_tx.index.into())

View File

@@ -1,15 +1,17 @@
//! Contains a mock store for internal testing. //! Contains a mock store for internal testing.
//! //!
//! Might make this public alter to users can test their store handlers. //! Might make this public alter to users can test their store handlers.
use std::cell::RefCell; use std::{cell::RefCell, collections::HashMap, rc::Rc};
use std::collections::HashMap; use substreams::{
use std::rc::Rc; prelude::{BigInt, StoreDelete, StoreGet, StoreNew},
use substreams::prelude::{BigInt, StoreDelete, StoreGet, StoreNew}; store::StoreAdd,
use substreams::store::StoreAdd; };
type BigIntStore = HashMap<String, Vec<(u64, BigInt)>>;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct MockStore { pub struct MockStore {
data: Rc<RefCell<HashMap<String, Vec<(u64, BigInt)>>>>, data: Rc<RefCell<BigIntStore>>,
} }
impl StoreDelete for MockStore { impl StoreDelete for MockStore {

File diff suppressed because it is too large Load Diff

View File

@@ -1,25 +1,25 @@
[package] [package]
name = "substreams-ethereum-balancer" name = "ethereum-balancer"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
[lib] [lib]
name = "substreams_ethereum_balancer" name = "ethereum_balancer"
crate-type = ["cdylib"] crate-type = ["cdylib"]
[dependencies] [dependencies]
substreams = "0.5" substreams.workspace = true
substreams-ethereum = "0.9.9" substreams-ethereum.workspace = true
prost = "0.11" prost.workspace = true
hex-literal = "0.4.1" prost-types.workspace = true
ethabi = "18.0.0" hex-literal.workspace = true
hex = "0.4.2" ethabi.workspace = true
hex.workspace = true
bytes = "1.5.0" bytes = "1.5.0"
anyhow = "1.0.75" anyhow = "1.0.75"
prost-types = "0.12.3"
num-bigint = "0.4.4" num-bigint = "0.4.4"
itertools = "0.12.0" itertools = "0.12.0"
tycho-substreams = {path ="../../crates/tycho-substreams"} tycho-substreams.workspace = true
[build-dependencies] [build-dependencies]
anyhow = "1" anyhow = "1"

View File

@@ -2,16 +2,15 @@ use crate::{abi, pool_factories};
use anyhow::Result; use anyhow::Result;
use itertools::Itertools; use itertools::Itertools;
use std::collections::HashMap; use std::collections::HashMap;
use substreams::hex; use substreams::{
use substreams::pb::substreams::StoreDeltas; hex,
use substreams::store::{ pb::substreams::StoreDeltas,
StoreAdd, StoreAddBigInt, StoreAddInt64, StoreGet, StoreGetInt64, StoreNew, store::{StoreAdd, StoreAddBigInt, StoreAddInt64, StoreGet, StoreGetInt64, StoreNew},
};
use substreams_ethereum::{pb::eth, Event};
use tycho_substreams::{
balances::aggregate_balances_changes, contract::extract_contract_changes, prelude::*,
}; };
use substreams_ethereum::pb::eth;
use substreams_ethereum::Event;
use tycho_substreams::balances::aggregate_balances_changes;
use tycho_substreams::contract::extract_contract_changes;
use tycho_substreams::prelude::*;
const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8");
@@ -37,10 +36,7 @@ pub fn map_pools_created(block: eth::v2::Block) -> Result<BlockTransactionProtoc
.collect::<Vec<_>>(); .collect::<Vec<_>>();
if !components.is_empty() { if !components.is_empty() {
Some(TransactionProtocolComponents { Some(TransactionProtocolComponents { tx: Some(tx.into()), components })
tx: Some(tx.into()),
components,
})
} else { } else {
None None
} }
@@ -63,8 +59,8 @@ pub fn store_pools_created(map: BlockTransactionProtocolComponents, store: Store
); );
} }
/// Since the `PoolBalanceChanged` and `Swap` events administer only deltas, we need to leverage a map and a /// Since the `PoolBalanceChanged` and `Swap` events administer only deltas, we need to leverage a
/// store to be able to tally up final balances for tokens in a pool. /// map and a store to be able to tally up final balances for tokens in a pool.
#[substreams::handlers::map] #[substreams::handlers::map]
pub fn map_balance_deltas( pub fn map_balance_deltas(
block: eth::v2::Block, block: eth::v2::Block,
@@ -79,12 +75,10 @@ pub fn map_balance_deltas(
if let Some(ev) = if let Some(ev) =
abi::vault::events::PoolBalanceChanged::match_and_decode(vault_log.log) abi::vault::events::PoolBalanceChanged::match_and_decode(vault_log.log)
{ {
let component_id = format!( let component_id =
"0x{}", format!("0x{}", String::from_utf8(ev.pool_id[..20].to_vec()).unwrap())
String::from_utf8(ev.pool_id[..20].to_vec()).unwrap() .as_bytes()
) .to_vec();
.as_bytes()
.to_vec();
if store if store
.get_last(format!("pool:{}", hex::encode(&component_id))) .get_last(format!("pool:{}", hex::encode(&component_id)))
@@ -101,12 +95,10 @@ pub fn map_balance_deltas(
} }
} }
} else if let Some(ev) = abi::vault::events::Swap::match_and_decode(vault_log.log) { } else if let Some(ev) = abi::vault::events::Swap::match_and_decode(vault_log.log) {
let component_id = format!( let component_id =
"0x{}", format!("0x{}", String::from_utf8(ev.pool_id[..20].to_vec()).unwrap())
String::from_utf8(ev.pool_id[..20].to_vec()).unwrap() .as_bytes()
) .to_vec();
.as_bytes()
.to_vec();
if store if store
.get_last(format!("pool:{}", hex::encode(&component_id))) .get_last(format!("pool:{}", hex::encode(&component_id)))
@@ -149,8 +141,8 @@ pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: StoreAddBigInt)
/// Every contract change is grouped by transaction index via the `transaction_contract_changes` /// Every contract change is grouped by transaction index via the `transaction_contract_changes`
/// map. Each block of code will extend the `TransactionContractChanges` struct with the /// map. Each block of code will extend the `TransactionContractChanges` struct with the
/// cooresponding changes (balance, component, contract), inserting a new one if it doesn't exist. /// cooresponding changes (balance, component, contract), inserting a new one if it doesn't exist.
/// At the very end, the map can easily be sorted by index to ensure the final `BlockContractChanges` /// At the very end, the map can easily be sorted by index to ensure the final
/// is ordered by transactions properly. /// `BlockContractChanges` is ordered by transactions properly.
#[substreams::handlers::map] #[substreams::handlers::map]
pub fn map_changes( pub fn map_changes(
block: eth::v2::Block, block: eth::v2::Block,
@@ -172,15 +164,15 @@ pub fn map_changes(
let tx = tx_component.tx.as_ref().unwrap(); let tx = tx_component.tx.as_ref().unwrap();
transaction_contract_changes transaction_contract_changes
.entry(tx.index) .entry(tx.index)
.or_insert_with(|| TransactionContractChanges::new(&tx)) .or_insert_with(|| TransactionContractChanges::new(tx))
.component_changes .component_changes
.extend_from_slice(&tx_component.components); .extend_from_slice(&tx_component.components);
}); });
// Balance changes are gathered by the `StoreDelta` based on `PoolBalanceChanged` creating // Balance changes are gathered by the `StoreDelta` based on `PoolBalanceChanged` creating
// `BlockBalanceDeltas`. We essentially just process the changes that occurred to the `store` this // `BlockBalanceDeltas`. We essentially just process the changes that occurred to the `store`
// block. Then, these balance changes are merged onto the existing map of tx contract changes, // this block. Then, these balance changes are merged onto the existing map of tx contract
// inserting a new one if it doesn't exist. // changes, inserting a new one if it doesn't exist.
aggregate_balances_changes(balance_store, deltas) aggregate_balances_changes(balance_store, deltas)
.into_iter() .into_iter()
.for_each(|(_, (tx, balances))| { .for_each(|(_, (tx, balances))| {
@@ -188,7 +180,7 @@ pub fn map_changes(
.entry(tx.index) .entry(tx.index)
.or_insert_with(|| TransactionContractChanges::new(&tx)) .or_insert_with(|| TransactionContractChanges::new(&tx))
.balance_changes .balance_changes
.extend(balances.into_iter().map(|(_, change)| change)); .extend(balances.into_values());
}); });
// Extract and insert any storage changes that happened for any of the components. // Extract and insert any storage changes that happened for any of the components.
@@ -196,7 +188,7 @@ pub fn map_changes(
&block, &block,
|addr| { |addr| {
components_store components_store
.get_last(format!("pool:{0}", hex::encode(&addr))) .get_last(format!("pool:{0}", hex::encode(addr)))
.is_some() .is_some()
}, },
&mut transaction_contract_changes, &mut transaction_contract_changes,
@@ -210,9 +202,9 @@ pub fn map_changes(
.drain() .drain()
.sorted_unstable_by_key(|(index, _)| *index) .sorted_unstable_by_key(|(index, _)| *index)
.filter_map(|(_, change)| { .filter_map(|(_, change)| {
if change.contract_changes.is_empty() if change.contract_changes.is_empty() &&
&& change.component_changes.is_empty() change.component_changes.is_empty() &&
&& change.balance_changes.is_empty() change.balance_changes.is_empty()
{ {
None None
} else { } else {

View File

@@ -1,8 +1,9 @@
use crate::abi; use crate::abi;
use substreams::hex; use substreams::{hex, scalar::BigInt};
use substreams::scalar::BigInt; use substreams_ethereum::{
use substreams_ethereum::pb::eth::v2::{Call, Log}; pb::eth::v2::{Call, Log},
use substreams_ethereum::{Event, Function}; Event, Function,
};
use tycho_substreams::prelude::*; use tycho_substreams::prelude::*;
/// This trait defines some helpers for serializing and deserializing `Vec<BigInt` which is needed /// This trait defines some helpers for serializing and deserializing `Vec<BigInt` which is needed
@@ -53,13 +54,15 @@ pub fn address_map(
abi::weighted_pool_factory::events::PoolCreated::match_and_decode(log)?; abi::weighted_pool_factory::events::PoolCreated::match_and_decode(log)?;
Some( Some(
ProtocolComponent::at_contract(&pool_created.pool, &tx) ProtocolComponent::at_contract(&pool_created.pool, tx)
.with_tokens(&create_call.tokens) .with_tokens(&create_call.tokens)
.with_attributes(&[ .with_attributes(&[
("pool_type", "WeightedPoolFactory".as_bytes()), ("pool_type", "WeightedPoolFactory".as_bytes()),
( (
"normalized_weights", "normalized_weights",
&create_call.normalized_weights.serialize_bytes(), &create_call
.normalized_weights
.serialize_bytes(),
), ),
]) ])
.as_swap_type("balancer_pool", ImplementationType::Vm), .as_swap_type("balancer_pool", ImplementationType::Vm),
@@ -72,7 +75,7 @@ pub fn address_map(
abi::composable_stable_pool_factory::events::PoolCreated::match_and_decode(log)?; abi::composable_stable_pool_factory::events::PoolCreated::match_and_decode(log)?;
Some( Some(
ProtocolComponent::at_contract(&pool_created.pool, &tx) ProtocolComponent::at_contract(&pool_created.pool, tx)
.with_tokens(&create_call.tokens) .with_tokens(&create_call.tokens)
.with_attributes(&[("pool_type", "ComposableStablePoolFactory".as_bytes())]) .with_attributes(&[("pool_type", "ComposableStablePoolFactory".as_bytes())])
.as_swap_type("balancer_pool", ImplementationType::Vm), .as_swap_type("balancer_pool", ImplementationType::Vm),
@@ -85,13 +88,15 @@ pub fn address_map(
abi::erc_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; abi::erc_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
Some( Some(
ProtocolComponent::at_contract(&pool_created.pool, &tx) ProtocolComponent::at_contract(&pool_created.pool, tx)
.with_tokens(&[create_call.main_token, create_call.wrapped_token]) .with_tokens(&[create_call.main_token, create_call.wrapped_token])
.with_attributes(&[ .with_attributes(&[
("pool_type", "ERC4626LinearPoolFactory".as_bytes()), ("pool_type", "ERC4626LinearPoolFactory".as_bytes()),
( (
"upper_target", "upper_target",
&create_call.upper_target.to_signed_bytes_be(), &create_call
.upper_target
.to_signed_bytes_be(),
), ),
]) ])
.as_swap_type("balancer_pool", ImplementationType::Vm), .as_swap_type("balancer_pool", ImplementationType::Vm),
@@ -104,13 +109,15 @@ pub fn address_map(
abi::euler_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; abi::euler_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
Some( Some(
ProtocolComponent::at_contract(&pool_created.pool, &tx) ProtocolComponent::at_contract(&pool_created.pool, tx)
.with_tokens(&[create_call.main_token, create_call.wrapped_token]) .with_tokens(&[create_call.main_token, create_call.wrapped_token])
.with_attributes(&[ .with_attributes(&[
("pool_type", "EulerLinearPoolFactory".as_bytes()), ("pool_type", "EulerLinearPoolFactory".as_bytes()),
( (
"upper_target", "upper_target",
&create_call.upper_target.to_signed_bytes_be(), &create_call
.upper_target
.to_signed_bytes_be(),
), ),
]) ])
.as_swap_type("balancer_pool", ImplementationType::Vm), .as_swap_type("balancer_pool", ImplementationType::Vm),
@@ -142,10 +149,11 @@ pub fn address_map(
// change: tycho::ChangeType::Creation.into(), // change: tycho::ChangeType::Creation.into(),
// }) // })
// } // }
// ❌ The `ManagedPoolFactory` is a bit ✨ unique ✨, so we'll leave it commented out for now // ❌ The `ManagedPoolFactory` is a bit ✨ unique ✨, so we'll leave it commented out for
// Take a look at it's `Create` call to see how the params are structured. // now Take a look at it's `Create` call to see how the params are structured.
// hex!("BF904F9F340745B4f0c4702c7B6Ab1e808eA6b93") => { // hex!("BF904F9F340745B4f0c4702c7B6Ab1e808eA6b93") => {
// let create_call = abi::managed_pool_factory::functions::Create::match_and_decode(call)?; // let create_call =
// abi::managed_pool_factory::functions::Create::match_and_decode(call)?;
// let pool_created = // let pool_created =
// abi::managed_pool_factory::events::PoolCreated::match_and_decode(log)?; // abi::managed_pool_factory::events::PoolCreated::match_and_decode(log)?;
@@ -170,13 +178,15 @@ pub fn address_map(
abi::silo_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; abi::silo_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
Some( Some(
ProtocolComponent::at_contract(&pool_created.pool, &tx) ProtocolComponent::at_contract(&pool_created.pool, tx)
.with_tokens(&[create_call.main_token, create_call.wrapped_token]) .with_tokens(&[create_call.main_token, create_call.wrapped_token])
.with_attributes(&[ .with_attributes(&[
("pool_type", "SiloLinearPoolFactory".as_bytes()), ("pool_type", "SiloLinearPoolFactory".as_bytes()),
( (
"upper_target", "upper_target",
&create_call.upper_target.to_signed_bytes_be(), &create_call
.upper_target
.to_signed_bytes_be(),
), ),
]) ])
.as_swap_type("balancer_pool", ImplementationType::Vm), .as_swap_type("balancer_pool", ImplementationType::Vm),
@@ -189,13 +199,15 @@ pub fn address_map(
abi::yearn_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; abi::yearn_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
Some( Some(
ProtocolComponent::at_contract(&pool_created.pool, &tx) ProtocolComponent::at_contract(&pool_created.pool, tx)
.with_tokens(&[create_call.main_token, create_call.wrapped_token]) .with_tokens(&[create_call.main_token, create_call.wrapped_token])
.with_attributes(&[ .with_attributes(&[
("pool_type", "YearnLinearPoolFactory".as_bytes()), ("pool_type", "YearnLinearPoolFactory".as_bytes()),
( (
"upper_target", "upper_target",
&create_call.upper_target.to_signed_bytes_be(), &create_call
.upper_target
.to_signed_bytes_be(),
), ),
]) ])
.as_swap_type("balancer_pool", ImplementationType::Vm), .as_swap_type("balancer_pool", ImplementationType::Vm),
@@ -210,7 +222,7 @@ pub fn address_map(
abi::weighted_pool_tokens_factory::events::PoolCreated::match_and_decode(log)?; abi::weighted_pool_tokens_factory::events::PoolCreated::match_and_decode(log)?;
Some( Some(
ProtocolComponent::at_contract(&pool_created.pool, &tx) ProtocolComponent::at_contract(&pool_created.pool, tx)
.with_tokens(&create_call.tokens) .with_tokens(&create_call.tokens)
.with_attributes(&[ .with_attributes(&[
("pool_type", "WeightedPool2TokensFactory".as_bytes()), ("pool_type", "WeightedPool2TokensFactory".as_bytes()),

View File

@@ -1,20 +1,12 @@
specVersion: v0.1.0 specVersion: v0.1.0
package: package:
name: "substreams_ethereum_balancer" name: "ethereum_balancer"
version: v0.1.0 version: v0.1.0
protobuf:
files:
- tycho/evm/v1/vm.proto
- tycho/evm/v1/common.proto
- tycho/evm/v1/utils.proto
importPaths:
- ../../proto
binaries: binaries:
default: default:
type: wasm/rust-v1 type: wasm/rust-v1
file: target/wasm32-unknown-unknown/release/substreams_ethereum_balancer.wasm file: ../target/wasm32-unknown-unknown/release/ethereum_balancer.wasm
modules: modules:
- name: map_pools_created - name: map_pools_created

60
substreams/release.sh Executable file
View File

@@ -0,0 +1,60 @@
#!/bin/bash
# Allows releasing multiple substream packages within the same repo.
# To trigger a release simply create a tag with [package-name]-[semver].
# The script will look for these tags, then infer which package needs to be built and
# released.
# Try to get the tag name associated with the current HEAD commit
current_tag=$(git describe --tags --exact-match HEAD 2>/dev/null)
if [ -n "$current_tag" ]; then
# If the HEAD is at a tag, extract the prefix and version
if [[ $current_tag =~ ^([a-zA-Z-]*-)?([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
# Prefix without the trailing hyphen (if any)
package="${BASH_REMATCH[1]%?}"
# Semantic version
version="${BASH_REMATCH[2]}"
cargo_version=$(cargo pkgid -p ethereum-balancer | cut -d# -f2 | cut -d: -f2)
if [[ "$cargo_version" != "$version" ]]; then
echo "Error: Cargo version: ${cargo_version} does not match tag version: ${version}!"
exit 1
fi
# Check if the Git repository is dirty
if [ -n "$(git status --porcelain)" ]; then
echo "Error: The repository is dirty. Please commit or stash your changes."
exit 1
fi
else
echo "Error: Current tag ($current_tag) does not match the expected format."
exit 1
fi
else
# If the HEAD is not at a tag, construct the tag name with the pre-release postfix
if [ -z "$1" ]; then
echo "Error: package argument is required to create a pre release!"
exit 1
fi
package=$1
version_prefix=$(git describe --tags --match "$package-*" --abbrev=0 2>/dev/null)
if [ -z "$version_prefix" ]; then
# If no tags are found in the history, default to version 0.0.1
version_prefix="0.0.1"
fi
# Get the short commit hash of the current HEAD
commit_hash=$(git rev-parse --short HEAD)
version="${version_prefix}-pre.${commit_hash}"
fi
REPOSITORY=${REPOSITORY:-"s3://repo.propellerheads/substreams"}
repository_path="$REPOSITORY/$package/$package-$version.spkg"
cargo build --target wasm32-unknown-unknown --release -p "$package"
mkdir -p ./target/spkg/
substreams pack $package/substreams.yaml -o ./target/spkg/$package-$version.spkg
aws s3 cp ./target/spkg/$package-$version.spkg $repository_path
echo "Released substreams package: '$repository_path'"

View File

@@ -9,5 +9,6 @@ trailing_semicolon = false
use_field_init_shorthand = true use_field_init_shorthand = true
chain_width = 40 chain_width = 40
ignore = [ ignore = [
"src/pb", "crates/tycho-substreams/src/pb",
"ethereum-balancer/src/abi",
] ]