Merge branch 'main' of https://github.com/propeller-heads/propeller-protocol-lib into feat/curve-maverick

.github/workflows/evm.yml

@@ -1,6 +1,9 @@
-name: test
+name: test evm
 
-on: workflow_dispatch
+on:
+  push:
+    paths:
+      - "evm/**"
 
 env:
   FOUNDRY_PROFILE: ci
@@ -29,10 +32,16 @@ jobs:
           forge build --sizes
         id: build
 
+      - name: Run Forge format check
+        run:  |
+              forge --version
+              forge fmt --check
+        id:   format
+
       - name: Run Forge tests
         run: |
           cd evm
           forge test -vvv
         id: test
         env:
-          ETH_RPC_URL: ${{ secrets.ETH_RPC_URL }} 
+          ETH_RPC_URL: ${{ secrets.ETH_RPC_URL }}

.github/workflows/substream.cd.yaml

@@ -0,0 +1,45 @@
+name: Substreams CD
+
+on:
+  push:
+    tags:
+      - "ethereum-*"
+  workflow_dispatch:
+    inputs:
+      package:
+        required: true
+        description: "Package to build"
+
+jobs:
+  Release:
+    name: Release ${{ inputs.package }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: stable
+          targets: wasm32-unknown-unknown
+
+      - name: Setup Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Install Substreams CLI
+        run: |
+          # Use correct binary for your platform
+          LINK=$(curl -s https://api.github.com/repos/streamingfast/substreams/releases/latest | awk "/download.url.*linux_$(uname -m)/ {print \$2}" | sed 's/"//g')
+          curl -L "$LINK" | tar zxf - -C /usr/local/bin
+          chmod +x /usr/local/bin/substreams
+          substreams --version
+
+      - name: Run checks
+        run: |
+          cd substreams
+          ./release.sh ${{ inputs.package }}

.github/workflows/substream.ci.yaml

@@ -0,0 +1,59 @@
+name: Substreams CI
+
+on:
+  push:
+    paths:
+      - "substreams/**"
+
+jobs:
+  lint:
+    name: Substreams Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: nightly
+          components: clippy, rustfmt
+
+      - name: Setup Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Run checks
+        run: |
+          cd substreams
+          cargo +nightly fmt -- --check
+          cargo +nightly clippy --all --all-features --all-targets -- -D warnings
+
+  test:
+    name: Substreams Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: stable
+          targets: wasm32-unknown-unknown
+
+      - name: Setup Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Run checks
+        run: |
+          cd substreams
+          cargo build --target wasm32-unknown-unknown --all-targets --all-features
+          cargo test --workspace --all-targets --all-features

.gitignore

@@ -11,3 +11,5 @@ target/
 .vscode
 .idea
 *.log
+
+substreams/ethereum-template/Cargo.lock

docs/indexing/substreams-integration.md

@@ -42,6 +42,18 @@ The models below are very similar to the vm integration models but have a few mo
 
 Once again changes must be aggregated on a transaction level, emitting these models with duplicated transaction as the final output would be considered an error.
 
+#### Integer Byte encoding
+
+Many of the types above are variable length bytes. This allows for flexibility across blockchains but require agreeing on an informal interface, so later applications know how to interpret these bytes.
+
+**Integers:** especially integers used to communicate balances, should always be encoded as unsigned big-endian integer. This is simply because balances serve multiple purposes within the system and need to be decoded in multiple location of the messages journey.
+
+**Strings**: If you need to store strings, please use utf-8 encoding to store them as bytes.
+
+**Attributes:** the value encoding for attributes in the native implementation messages is variable. It depends on the use case. Since the attributes are highly dynamic they are only used by the corresponding logic components, so the encoding can be tailored to the logic implementation: E.g. since Rust uses little endian one may choose to use little endian encoding for integers if the native logic module is written in Rust.
+
+
+
 ### Changes of interest
 
 PropellerHeads integration should at least communicate the following changes:

docs/logic/vm-integration/ethereum-solidity.md

@@ -43,7 +43,7 @@ contract:
 instances:
   - chain:
       name: mainnet
-      id: 0
+      id: 1
     # Arguments passed to the constructor when building the contract
     arguments:
       - "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f"
@@ -57,8 +57,8 @@ tests:
       buy_token: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"
       block: 17000000
       chain:
-        id: 0
         name: mainnet
+        id: 1
 ```
 
 #### Price (optional)

evm/src/balancer-v2/BalancerV2SwapAdapter.sol

@@ -1,5 +1,4 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-pragma experimental ABIEncoderV2;
 pragma solidity ^0.8.13;
 
 import {IERC20, ISwapAdapter} from "src/interfaces/ISwapAdapter.sol";
@@ -478,9 +477,8 @@ interface IVault {
             uint256 lastChangeBlock
         );
 
-    enum SwapKind
+    enum SwapKind {
-    /// The number of tokens to send to the Pool is known
+        /// The number of tokens to send to the Pool is known
-    {
         GIVEN_IN,
         /// The number of tokens to take from the Pool is known
         GIVEN_OUT

evm/src/balancer-v2/manifest.yaml

@@ -1,6 +1,6 @@
 # information about the author helps us reach out in case of issues.
 author:
-  name: Propellerheads.xyz
+  name: Matt
   email: pistomat@propellerheads.xyz
 
 # Protocol Constants
@@ -19,7 +19,7 @@ contract: BalancerV2SwapAdapter.sol
 instances:
   - chain:
       name: mainnet
-      id: 0
+      id: 1
     arguments:
       - "0xBA12222222228d8Ba445958a75a0704d566BF2C8"
 
@@ -27,10 +27,10 @@ instances:
 # getTokens are not implemented.
 tests:
   instances:
-    - pool_id: "0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc"
+    - pool_id: "0x5c6ee304399dbdb9c8ef030ab642b10820db8f56000200000000000000000014"
-      sell_token: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"
+      sell_token: "0xba100000625a3754423978a60c9317c58a424e3D"
-      buy_token: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"
+      buy_token: "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"
       block: 17000000
       chain:
-        id: 0
         name: mainnet
+        id: 1

evm/src/integral/IntegralSwapAdapter.sol

@@ -3,12 +3,15 @@ pragma solidity ^0.8.13;
 
 import {IERC20, ISwapAdapter} from "src/interfaces/ISwapAdapter.sol";
 import {ERC20} from "openzeppelin-contracts/contracts/token/ERC20/ERC20.sol";
-import {SafeERC20} from "openzeppelin-contracts/contracts/token/ERC20/utils/SafeERC20.sol";
+import {SafeERC20} from
+    "openzeppelin-contracts/contracts/token/ERC20/utils/SafeERC20.sol";
 
-/// @dev Integral submitted deadline of 3600 seconds (1 hour) to Paraswap, but it is not strictly necessary to be this long
+/// @dev Integral submitted deadline of 3600 seconds (1 hour) to Paraswap, but
-/// as the contract allows less durations, we use 1000 seconds (15 minutes) as a deadline
+/// it is not strictly necessary to be this long
+/// as the contract allows less durations, we use 1000 seconds (15 minutes) as a
+/// deadline
 uint256 constant SWAP_DEADLINE_SEC = 1000;
-uint256 constant STANDARD_TOKEN_DECIMALS = 10**18;
+uint256 constant STANDARD_TOKEN_DECIMALS = 10 ** 18;
 
 /// @title Integral Swap Adapter
 contract IntegralSwapAdapter is ISwapAdapter {
@@ -21,12 +24,18 @@ contract IntegralSwapAdapter is ISwapAdapter {
     }
 
     /// @inheritdoc ISwapAdapter
-    /// @dev Integral always relies on a single pool linked to the factory to map two pairs, and does not use routing
+    /// @dev Integral always relies on a single pool linked to the factory to
-    /// we can then use getPriceByTokenAddresses() instead of getPriceByPairAddresses() 
+    /// map two pairs, and does not use routing
-    /// as they both return the same value and the first also handles the order of tokens inside.
+    /// we can then use getPriceByTokenAddresses() instead of
-    /// @dev Since the price of a token is determined externally by Integral Oracles and not by reserves
+    /// getPriceByPairAddresses()
-    /// it will always be the same (pre and post trade) and independent of the amounts swapped,
+    /// as they both return the same value and the first also handles the order
-    /// but we still return an array of length=specifiedAmounts.length with same values to make sure the return value is the expected from caller.
+    /// of tokens inside.
+    /// @dev Since the price of a token is determined externally by Integral
+    /// Oracles and not by reserves
+    /// it will always be the same (pre and post trade) and independent of the
+    /// amounts swapped,
+    /// but we still return an array of length=specifiedAmounts.length with same
+    /// values to make sure the return value is the expected from caller.
     function price(
         bytes32,
         IERC20 _sellToken,
@@ -34,8 +43,9 @@ contract IntegralSwapAdapter is ISwapAdapter {
         uint256[] memory _specifiedAmounts
     ) external view override returns (Fraction[] memory _prices) {
         _prices = new Fraction[](_specifiedAmounts.length);
-        Fraction memory price = getPriceAt(address(_sellToken), address(_buyToken));
+        Fraction memory price =
-        
+            getPriceAt(address(_sellToken), address(_buyToken));
+
         for (uint256 i = 0; i < _specifiedAmounts.length; i++) {
             _prices[i] = price;
         }
@@ -54,12 +64,12 @@ contract IntegralSwapAdapter is ISwapAdapter {
         }
 
         uint256 gasBefore = gasleft();
-        if (side == OrderSide.Sell) { // sell
+        if (side == OrderSide.Sell) {
-            trade.calculatedAmount =
+            // sell
-                sell(sellToken, buyToken, specifiedAmount);
+            trade.calculatedAmount = sell(sellToken, buyToken, specifiedAmount);
-        } else { // buy
+        } else {
-            trade.calculatedAmount =
+            // buy
-                buy(sellToken, buyToken, specifiedAmount);
+            trade.calculatedAmount = buy(sellToken, buyToken, specifiedAmount);
         }
         trade.gasUsed = gasBefore - gasleft();
         trade.price = getPriceAt(address(sellToken), address(buyToken));
@@ -72,21 +82,17 @@ contract IntegralSwapAdapter is ISwapAdapter {
         override
         returns (uint256[] memory limits)
     {
-        (
+        (,,, uint256 limitMax0,, uint256 limitMax1) =
-            ,
+            relayer.getPoolState(address(sellToken), address(buyToken));
-            ,
-            ,
-            uint256 limitMax0,
-            ,
-            uint256 limitMax1
-        ) = relayer.getPoolState(address(sellToken), address(buyToken));
 
         limits = new uint256[](2);
         limits[0] = limitMax0;
         limits[1] = limitMax1;
-        /** 
+        /**
-         * @dev minLimits in integral are the args: 2(for sellToken, the one before limitMax0)
+         * @dev minLimits in integral are the args: 2(for sellToken, the one
-         * and 4(for buyToken, the one before limitMax1) of the function relayer.getPoolState(sellToken, buyToken);
+         * before limitMax0)
+         * and 4(for buyToken, the one before limitMax1) of the function
+         * relayer.getPoolState(sellToken, buyToken);
          * an implementation of them can be found in the test of this adapter
          */
     }
@@ -141,12 +147,12 @@ contract IntegralSwapAdapter is ISwapAdapter {
     /// @param buyToken The address of the token being bought.
     /// @param amount The amount to be traded.
     /// @return uint256 The amount of tokens received.
-    function sell(
+    function sell(IERC20 sellToken, IERC20 buyToken, uint256 amount)
-        IERC20 sellToken,
+        internal
-        IERC20 buyToken,
+        returns (uint256)
-        uint256 amount
+    {
-    ) internal returns (uint256) {
+        uint256 amountOut =
-        uint256 amountOut = relayer.quoteSell(address(sellToken), address(buyToken), amount);
+            relayer.quoteSell(address(sellToken), address(buyToken), amount);
         if (amountOut == 0) {
             revert Unavailable("AmountOut is zero!");
         }
@@ -154,15 +160,17 @@ contract IntegralSwapAdapter is ISwapAdapter {
         sellToken.safeTransferFrom(msg.sender, address(this), amount);
         sellToken.safeIncreaseAllowance(address(relayer), amount);
 
-        relayer.sell(ITwapRelayer.SellParams({
+        relayer.sell(
-            tokenIn: address(sellToken),
+            ITwapRelayer.SellParams({
-            tokenOut: address(buyToken),
+                tokenIn: address(sellToken),
-            wrapUnwrap: false,
+                tokenOut: address(buyToken),
-            to: msg.sender,
+                wrapUnwrap: false,
-            submitDeadline: uint32(block.timestamp + SWAP_DEADLINE_SEC),
+                to: msg.sender,
-            amountIn: amount,
+                submitDeadline: uint32(block.timestamp + SWAP_DEADLINE_SEC),
-            amountOutMin: amountOut
+                amountIn: amount,
-        }));
+                amountOutMin: amountOut
+            })
+        );
 
         return amountOut;
     }
@@ -172,12 +180,13 @@ contract IntegralSwapAdapter is ISwapAdapter {
     /// @param buyToken The address of the token being bought.
     /// @param amountBought The amount of buyToken tokens to buy.
     /// @return uint256 The amount of tokens received.
-    function buy(
+    function buy(IERC20 sellToken, IERC20 buyToken, uint256 amountBought)
-        IERC20 sellToken,
+        internal
-        IERC20 buyToken,
+        returns (uint256)
-        uint256 amountBought
+    {
-    ) internal returns (uint256) {
+        uint256 amountIn = relayer.quoteBuy(
-        uint256 amountIn = relayer.quoteBuy(address(sellToken), address(buyToken), amountBought);
+            address(sellToken), address(buyToken), amountBought
+        );
         if (amountIn == 0) {
             revert Unavailable("AmountIn is zero!");
         }
@@ -185,15 +194,17 @@ contract IntegralSwapAdapter is ISwapAdapter {
         sellToken.safeTransferFrom(msg.sender, address(this), amountIn);
         sellToken.safeIncreaseAllowance(address(relayer), amountIn);
 
-        relayer.buy(ITwapRelayer.BuyParams({
+        relayer.buy(
-            tokenIn: address(sellToken),
+            ITwapRelayer.BuyParams({
-            tokenOut: address(buyToken),
+                tokenIn: address(sellToken),
-            wrapUnwrap: false,
+                tokenOut: address(buyToken),
-            to: msg.sender,
+                wrapUnwrap: false,
-            submitDeadline: uint32(block.timestamp + SWAP_DEADLINE_SEC),
+                to: msg.sender,
-            amountInMax: amountIn,
+                submitDeadline: uint32(block.timestamp + SWAP_DEADLINE_SEC),
-            amountOut: amountBought
+                amountInMax: amountIn,
-        }));
+                amountOut: amountBought
+            })
+        );
 
         return amountIn;
     }
@@ -201,30 +212,43 @@ contract IntegralSwapAdapter is ISwapAdapter {
     /// @notice Get swap price including fee
     /// @param sellToken token to sell
     /// @param buyToken token to buy
-    function getPriceAt(address sellToken, address buyToken) internal view returns(Fraction memory) {
+    function getPriceAt(address sellToken, address buyToken)
-        uint256 priceWithoutFee = relayer.getPriceByTokenAddresses(address(sellToken), address(buyToken));
+        internal
+        view
+        returns (Fraction memory)
+    {
+        uint256 priceWithoutFee = relayer.getPriceByTokenAddresses(
+            address(sellToken), address(buyToken)
+        );
         ITwapFactory factory = ITwapFactory(relayer.factory());
-        address pairAddress = factory.getPair(address(sellToken), address(buyToken));
+        address pairAddress =
+            factory.getPair(address(sellToken), address(buyToken));
 
         // get swapFee formatted; swapFee is a constant
-        uint256 swapFeeFormatted = (STANDARD_TOKEN_DECIMALS - relayer.swapFee(pairAddress));
+        uint256 swapFeeFormatted =
+            (STANDARD_TOKEN_DECIMALS - relayer.swapFee(pairAddress));
 
         // get token decimals
-        uint256 sellTokenDecimals = 10**ERC20(sellToken).decimals();
+        uint256 sellTokenDecimals = 10 ** ERC20(sellToken).decimals();
-        uint256 buyTokenDecimals = 10**ERC20(buyToken).decimals();
+        uint256 buyTokenDecimals = 10 ** ERC20(buyToken).decimals();
 
         /**
          * @dev
-         * Denominator works as a "standardizer" for the price rather than a reserve value
+         * Denominator works as a "standardizer" for the price rather than a
-         * as Integral takes prices from oracles and do not operate with reserves;
+         * reserve value
+         * as Integral takes prices from oracles and do not operate with
+         * reserves;
          * it is therefore used to maintain integrity for the Fraction division,
-         * as numerator and denominator could have different token decimals(es. ETH(18)-USDC(6)).
+         * as numerator and denominator could have different token decimals(es.
-         * Both numerator and denominator are also multiplied by STANDARD_TOKEN_DECIMALS
+         * ETH(18)-USDC(6)).
+         * Both numerator and denominator are also multiplied by
+         * STANDARD_TOKEN_DECIMALS
          * to ensure that precision losses are minimized or null.
          */
         return Fraction(
             priceWithoutFee * STANDARD_TOKEN_DECIMALS,
-            STANDARD_TOKEN_DECIMALS * sellTokenDecimals * swapFeeFormatted / buyTokenDecimals
+            STANDARD_TOKEN_DECIMALS * sellTokenDecimals * swapFeeFormatted
+                / buyTokenDecimals
         );
     }
 }
@@ -276,8 +300,12 @@ interface ITwapRelayer {
         uint256 amountIn,
         uint256 indexed delayOrderId
     );
-    event RebalanceSellWithOneInch(address indexed oneInchRouter, uint256 gas, bytes data);
+    event RebalanceSellWithOneInch(
-    event OneInchRouterWhitelisted(address indexed oneInchRouter, bool whitelisted);
+        address indexed oneInchRouter, uint256 gas, bytes data
+    );
+    event OneInchRouterWhitelisted(
+        address indexed oneInchRouter, bool whitelisted
+    );
 
     function factory() external pure returns (address);
 
@@ -289,7 +317,10 @@ interface ITwapRelayer {
 
     function rebalancer() external view returns (address);
 
-    function isOneInchRouterWhitelisted(address oneInchRouter) external view returns (bool);
+    function isOneInchRouterWhitelisted(address oneInchRouter)
+        external
+        view
+        returns (bool);
 
     function setOwner(address _owner) external;
 
@@ -309,19 +340,26 @@ interface ITwapRelayer {
 
     function tokenLimitMin(address token) external pure returns (uint256);
 
-    function tokenLimitMaxMultiplier(address token) external pure returns (uint256);
+    function tokenLimitMaxMultiplier(address token)
+        external
+        pure
+        returns (uint256);
 
     function tolerance(address pair) external pure returns (uint16);
 
     function setRebalancer(address _rebalancer) external;
 
-    function whitelistOneInchRouter(address oneInchRouter, bool whitelisted) external;
+    function whitelistOneInchRouter(address oneInchRouter, bool whitelisted)
+        external;
 
     function getTolerance(address pair) external pure returns (uint16);
 
     function getTokenLimitMin(address token) external pure returns (uint256);
 
-    function getTokenLimitMaxMultiplier(address token) external pure returns (uint256);
+    function getTokenLimitMaxMultiplier(address token)
+        external
+        pure
+        returns (uint256);
 
     function getTwapInterval(address pair) external pure returns (uint32);
 
@@ -335,7 +373,10 @@ interface ITwapRelayer {
         uint32 submitDeadline;
     }
 
-    function sell(SellParams memory sellParams) external payable returns (uint256 orderId);
+    function sell(SellParams memory sellParams)
+        external
+        payable
+        returns (uint256 orderId);
 
     struct BuyParams {
         address tokenIn;
@@ -347,18 +388,20 @@ interface ITwapRelayer {
         uint32 submitDeadline;
     }
 
-    function buy(BuyParams memory buyParams) external payable returns (uint256 orderId);
+    function buy(BuyParams memory buyParams)
+        external
+        payable
+        returns (uint256 orderId);
 
     function getPriceByPairAddress(address pair, bool inverted)
         external
         view
-        returns (
+        returns (uint8 xDecimals, uint8 yDecimals, uint256 price);
-            uint8 xDecimals,
-            uint8 yDecimals,
-            uint256 price
-        );
 
-    function getPriceByTokenAddresses(address tokenIn, address tokenOut) external view returns (uint256 price);
+    function getPriceByTokenAddresses(address tokenIn, address tokenOut)
+        external
+        view
+        returns (uint256 price);
 
     function getPoolState(address token0, address token1)
         external
@@ -372,29 +415,19 @@ interface ITwapRelayer {
             uint256 limitMax1
         );
 
-    function quoteSell(
+    function quoteSell(address tokenIn, address tokenOut, uint256 amountIn)
-        address tokenIn,
+        external
-        address tokenOut,
+        view
-        uint256 amountIn
+        returns (uint256 amountOut);
-    ) external view returns (uint256 amountOut);
 
-    function quoteBuy(
+    function quoteBuy(address tokenIn, address tokenOut, uint256 amountOut)
-        address tokenIn,
+        external
-        address tokenOut,
+        view
-        uint256 amountOut
+        returns (uint256 amountIn);
-    ) external view returns (uint256 amountIn);
 
-    function approve(
+    function approve(address token, uint256 amount, address to) external;
-        address token,
-        uint256 amount,
-        address to
-    ) external;
 
-    function withdraw(
+    function withdraw(address token, uint256 amount, address to) external;
-        address token,
-        uint256 amount,
-        address to
-    ) external;
 
     function rebalanceSellWithDelay(
         address tokenIn,
@@ -412,12 +445,17 @@ interface ITwapRelayer {
 }
 
 interface ITwapFactory {
-    event PairCreated(address indexed token0, address indexed token1, address pair, uint256);
+    event PairCreated(
+        address indexed token0, address indexed token1, address pair, uint256
+    );
     event OwnerSet(address owner);
 
     function owner() external view returns (address);
 
-    function getPair(address tokenA, address tokenB) external view returns (address pair);
+    function getPair(address tokenA, address tokenB)
+        external
+        view
+        returns (address pair);
 
     function allPairs(uint256) external view returns (address pair);
 
@@ -432,41 +470,19 @@ interface ITwapFactory {
 
     function setOwner(address) external;
 
-    function setMintFee(
+    function setMintFee(address tokenA, address tokenB, uint256 fee) external;
-        address tokenA,
-        address tokenB,
-        uint256 fee
-    ) external;
 
-    function setBurnFee(
+    function setBurnFee(address tokenA, address tokenB, uint256 fee) external;
-        address tokenA,
-        address tokenB,
-        uint256 fee
-    ) external;
 
-    function setSwapFee(
+    function setSwapFee(address tokenA, address tokenB, uint256 fee) external;
-        address tokenA,
-        address tokenB,
-        uint256 fee
-    ) external;
 
-    function setOracle(
+    function setOracle(address tokenA, address tokenB, address oracle)
-        address tokenA,
+        external;
-        address tokenB,
-        address oracle
-    ) external;
 
-    function setTrader(
+    function setTrader(address tokenA, address tokenB, address trader)
-        address tokenA,
+        external;
-        address tokenB,
-        address trader
-    ) external;
 
-    function collect(
+    function collect(address tokenA, address tokenB, address to) external;
-        address tokenA,
-        address tokenB,
-        address to
-    ) external;
 
     function withdraw(
         address tokenA,
@@ -491,20 +507,39 @@ interface ITwapERC20 is IERC20 {
         bytes32 s
     ) external;
 
-    function increaseAllowance(address spender, uint256 addedValue) external returns (bool);
+    function increaseAllowance(address spender, uint256 addedValue)
+        external
+        returns (bool);
 
-    function decreaseAllowance(address spender, uint256 subtractedValue) external returns (bool);
+    function decreaseAllowance(address spender, uint256 subtractedValue)
+        external
+        returns (bool);
 }
 
 interface IReserves {
-    function getReserves() external view returns (uint112 reserve0, uint112 reserve1);
+    function getReserves()
+        external
+        view
+        returns (uint112 reserve0, uint112 reserve1);
 
     function getFees() external view returns (uint256 fee0, uint256 fee1);
 }
 
 interface ITwapPair is ITwapERC20, IReserves {
-    event Mint(address indexed sender, uint256 amount0In, uint256 amount1In, uint256 liquidityOut, address indexed to);
+    event Mint(
-    event Burn(address indexed sender, uint256 amount0Out, uint256 amount1Out, uint256 liquidityIn, address indexed to);
+        address indexed sender,
+        uint256 amount0In,
+        uint256 amount1In,
+        uint256 liquidityOut,
+        address indexed to
+    );
+    event Burn(
+        address indexed sender,
+        uint256 amount0Out,
+        uint256 amount1Out,
+        uint256 liquidityIn,
+        address indexed to
+    );
     event Swap(
         address indexed sender,
         uint256 amount0In,
@@ -541,7 +576,9 @@ interface ITwapPair is ITwapERC20, IReserves {
 
     function setBurnFee(uint256 fee) external;
 
-    function burn(address to) external returns (uint256 amount0, uint256 amount1);
+    function burn(address to)
+        external
+        returns (uint256 amount0, uint256 amount1);
 
     function swapFee() external view returns (uint256);
 
@@ -569,15 +606,33 @@ interface ITwapPair is ITwapERC20, IReserves {
         address _trader
     ) external;
 
-    function getSwapAmount0In(uint256 amount1Out, bytes calldata data) external view returns (uint256 swapAmount0In);
+    function getSwapAmount0In(uint256 amount1Out, bytes calldata data)
+        external
+        view
+        returns (uint256 swapAmount0In);
 
-    function getSwapAmount1In(uint256 amount0Out, bytes calldata data) external view returns (uint256 swapAmount1In);
+    function getSwapAmount1In(uint256 amount0Out, bytes calldata data)
+        external
+        view
+        returns (uint256 swapAmount1In);
 
-    function getSwapAmount0Out(uint256 amount1In, bytes calldata data) external view returns (uint256 swapAmount0Out);
+    function getSwapAmount0Out(uint256 amount1In, bytes calldata data)
+        external
+        view
+        returns (uint256 swapAmount0Out);
 
-    function getSwapAmount1Out(uint256 amount0In, bytes calldata data) external view returns (uint256 swapAmount1Out);
+    function getSwapAmount1Out(uint256 amount0In, bytes calldata data)
+        external
+        view
+        returns (uint256 swapAmount1Out);
 
-    function getDepositAmount0In(uint256 amount0, bytes calldata data) external view returns (uint256 depositAmount0In);
+    function getDepositAmount0In(uint256 amount0, bytes calldata data)
+        external
+        view
+        returns (uint256 depositAmount0In);
 
-    function getDepositAmount1In(uint256 amount1, bytes calldata data) external view returns (uint256 depositAmount1In);
+    function getDepositAmount1In(uint256 amount1, bytes calldata data)
+        external
+        view
+        returns (uint256 depositAmount1In);
 }

evm/src/template/TemplateSwapAdapter.sol

@@ -1,5 +1,4 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-pragma experimental ABIEncoderV2;
 pragma solidity ^0.8.13;
 
 import {IERC20, ISwapAdapter} from "src/interfaces/ISwapAdapter.sol";

evm/src/template/manifest.yaml

@@ -19,7 +19,7 @@ contract: TemplateSwapAdapter.sol
 instances:
   - chain:
       name: mainnet
-      id: 0
+      id: 1
     arguments:
       - "0xBA12222222228d8Ba445958a75a0704d566BF2C8"
 
@@ -32,5 +32,5 @@ tests:
       buy_token: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"
       block: 17000000
       chain:
-        id: 0
         name: mainnet
+        id: 1

evm/src/uniswap-v2/UniswapV2SwapAdapter.sol

@@ -87,10 +87,9 @@ contract UniswapV2SwapAdapter is ISwapAdapter {
                 buy(pair, sellToken, zero2one, r0, r1, specifiedAmount);
         }
         trade.gasUsed = gasBefore - gasleft();
-        if(side == OrderSide.Sell) {
+        if (side == OrderSide.Sell) {
             trade.price = getPriceAt(specifiedAmount, r0, r1);
-        }
+        } else {
-        else {
             trade.price = getPriceAt(trade.calculatedAmount, r0, r1);
         }
     }

evm/src/uniswap-v2/manifest.yaml

@@ -1,6 +1,6 @@
 # information about the author helps us reach out in case of issues.
 author:
-  name: Propellerheads.xyz
+  name: Alan
   email: alan@propellerheads.xyz
 
 # Protocol Constants
@@ -19,18 +19,6 @@ contract: UniswapV2SwapAdapter.sol
 instances:
   - chain:
       name: mainnet
-      id: 0
+      id: 1
     arguments:
       - "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f"
-
-# Specify some automatic test cases in case getPoolIds and
-# getTokens are not implemented.
-tests:
-  instances:
-    - pool_id: "0xB4e16d0168e52d35CaCD2c6185b44281Ec28C9Dc"
-      sell_token: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"
-      buy_token: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"
-      block: 17000000
-      chain:
-        id: 0
-        name: mainnet

evm/test/IntegralSwapAdapter.t.sol

@@ -1,103 +1,97 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 pragma solidity ^0.8.13;
- 
+
 import "forge-std/Test.sol";
 import "openzeppelin-contracts/contracts/interfaces/IERC20.sol";
 import "src/interfaces/ISwapAdapterTypes.sol";
 import "src/libraries/FractionMath.sol";
 import "src/integral/IntegralSwapAdapter.sol";
- 
+
 contract IntegralSwapAdapterTest is Test, ISwapAdapterTypes {
     using FractionMath for Fraction;
- 
+
     IntegralSwapAdapter adapter;
     ITwapRelayer relayer;
     IERC20 constant WETH = IERC20(0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2);
     IERC20 constant USDC = IERC20(0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48);
-    address constant USDC_WETH_PAIR =
+    address constant USDC_WETH_PAIR = 0x2fe16Dd18bba26e457B7dD2080d5674312b026a2;
-        0x2fe16Dd18bba26e457B7dD2080d5674312b026a2;
+    address constant relayerAddress = 0xd17b3c9784510E33cD5B87b490E79253BcD81e2E;
-    address constant relayerAddress =
+
-        0xd17b3c9784510E33cD5B87b490E79253BcD81e2E;
- 
     uint256 constant TEST_ITERATIONS = 100;
- 
+
     function setUp() public {
         uint256 forkBlock = 18835309;
         vm.createSelectFork(vm.rpcUrl("mainnet"), forkBlock);
         adapter = new IntegralSwapAdapter(relayerAddress);
         relayer = ITwapRelayer(relayerAddress);
- 
+
         vm.label(address(WETH), "WETH");
         vm.label(address(USDC), "USDC");
         vm.label(address(USDC_WETH_PAIR), "USDC_WETH_PAIR");
     }
- 
+
     function testPriceFuzzIntegral(uint256 amount0, uint256 amount1) public {
         bytes32 pair = bytes32(bytes20(USDC_WETH_PAIR));
         uint256[] memory limits = adapter.getLimits(pair, USDC, WETH);
         vm.assume(amount0 < limits[0]);
         vm.assume(amount1 < limits[1]);
- 
+
         uint256[] memory amounts = new uint256[](2);
         amounts[0] = amount0;
         amounts[1] = amount1;
- 
+
         Fraction[] memory prices = adapter.price(pair, WETH, USDC, amounts);
- 
+
         for (uint256 i = 0; i < prices.length; i++) {
             assertGt(prices[i].numerator, 0);
             assertGt(prices[i].denominator, 0);
         }
     }
- 
+
-    /// @dev Since TwapRelayer's calculateAmountOut function is internal, and using quoteSell would
+    /// @dev Since TwapRelayer's calculateAmountOut function is internal, and
+    /// using quoteSell would
     /// revert the transaction if calculateAmountOut is not enough,
-    /// we need a threshold to cover this internal amount, applied to 
+    /// we need a threshold to cover this internal amount, applied to
     function testSwapFuzzIntegral(uint256 specifiedAmount, bool isBuy) public {
         OrderSide side = isBuy ? OrderSide.Buy : OrderSide.Sell;
- 
+
         bytes32 pair = bytes32(bytes20(USDC_WETH_PAIR));
         uint256[] memory limits = new uint256[](2);
         uint256[] memory limitsMin = new uint256[](2);
- 
+
         if (side == OrderSide.Buy) {
             limits = adapter.getLimits(pair, USDC, WETH);
             vm.assume(specifiedAmount < limits[1]);
- 
+
             limitsMin = getMinLimits(USDC, WETH);
             vm.assume(specifiedAmount > limitsMin[1] * 115 / 100);
- 
+
             deal(address(USDC), address(this), type(uint256).max);
             USDC.approve(address(adapter), type(uint256).max);
         } else {
             limits = adapter.getLimits(pair, USDC, WETH);
             vm.assume(specifiedAmount < limits[0]);
- 
+
             limitsMin = getMinLimits(USDC, WETH);
             vm.assume(specifiedAmount > limitsMin[0] * 115 / 100);
- 
+
             deal(address(USDC), address(this), type(uint256).max);
             USDC.approve(address(adapter), specifiedAmount);
         }
- 
+
         uint256 usdc_balance_before = USDC.balanceOf(address(this));
         uint256 weth_balance_before = WETH.balanceOf(address(this));
- 
+
-        Trade memory trade = adapter.swap(
+        Trade memory trade =
-            pair,
+            adapter.swap(pair, USDC, WETH, side, specifiedAmount);
-            USDC,
+
-            WETH,
-            side,
-            specifiedAmount
-        );
- 
         if (trade.calculatedAmount > 0) {
             if (side == OrderSide.Buy) {
                 assertEq(
                     specifiedAmount,
                     WETH.balanceOf(address(this)) - weth_balance_before
                 );
- 
+
                 assertEq(
                     trade.calculatedAmount,
                     usdc_balance_before - USDC.balanceOf(address(this))
@@ -107,7 +101,7 @@ contract IntegralSwapAdapterTest is Test, ISwapAdapterTypes {
                     specifiedAmount,
                     usdc_balance_before - USDC.balanceOf(address(this))
                 );
- 
+
                 assertEq(
                     trade.calculatedAmount,
                     WETH.balanceOf(address(this)) - weth_balance_before
@@ -127,76 +121,67 @@ contract IntegralSwapAdapterTest is Test, ISwapAdapterTypes {
     function executeIncreasingSwapsIntegral(OrderSide side) internal {
         bytes32 pair = bytes32(bytes20(USDC_WETH_PAIR));
 
-        uint256 amountConstant_ = side == OrderSide.Sell ? 1000 * 10**6 : 10**17;
+        uint256 amountConstant_ =
- 
+            side == OrderSide.Sell ? 1000 * 10 ** 6 : 10 ** 17;
+
         uint256[] memory amounts = new uint256[](TEST_ITERATIONS);
         amounts[0] = amountConstant_;
         for (uint256 i = 1; i < TEST_ITERATIONS; i++) {
             amounts[i] = amountConstant_ * i;
         }
- 
+
         Trade[] memory trades = new Trade[](TEST_ITERATIONS);
         uint256 beforeSwap;
         for (uint256 i = 1; i < TEST_ITERATIONS; i++) {
             beforeSwap = vm.snapshot();
- 
+
             deal(address(USDC), address(this), amounts[i]);
             USDC.approve(address(adapter), amounts[i]);
- 
+
             trades[i] = adapter.swap(pair, USDC, WETH, side, amounts[i]);
             vm.revertTo(beforeSwap);
         }
- 
+
         for (uint256 i = 1; i < TEST_ITERATIONS - 1; i++) {
-            assertLe(
+            assertLe(trades[i].calculatedAmount, trades[i + 1].calculatedAmount);
-                trades[i].calculatedAmount,
-                trades[i + 1].calculatedAmount
-            );
             assertLe(trades[i].gasUsed, trades[i + 1].gasUsed);
             assertEq(trades[i].price.compareFractions(trades[i + 1].price), 0);
         }
     }
- 
+
-    function testGetCapabilitiesIntegral(
+    function testGetCapabilitiesIntegral(bytes32 pair, address t0, address t1)
-        bytes32 pair,
+        public
-        address t0,
+    {
-        address t1
+        Capability[] memory res =
-    ) public {
+            adapter.getCapabilities(pair, IERC20(t0), IERC20(t1));
-        Capability[] memory res = adapter.getCapabilities(
+
-            pair,
-            IERC20(t0),
-            IERC20(t1)
-        );
- 
         assertEq(res.length, 4);
     }
- 
+
     function testGetTokensIntegral() public {
         bytes32 pair = bytes32(bytes20(USDC_WETH_PAIR));
         IERC20[] memory tokens = adapter.getTokens(pair);
- 
+
         assertEq(tokens.length, 2);
     }
- 
+
     function testGetLimitsIntegral() public {
         bytes32 pair = bytes32(bytes20(USDC_WETH_PAIR));
         uint256[] memory limits = adapter.getLimits(pair, USDC, WETH);
- 
+
         assertEq(limits.length, 2);
     }
 
-    function getMinLimits(IERC20 sellToken, IERC20 buyToken) public view returns (uint256[] memory limits) {
+    function getMinLimits(IERC20 sellToken, IERC20 buyToken)
-        (
+        public
-            ,
+        view
-            ,
+        returns (uint256[] memory limits)
-            uint256 limitMin0,
+    {
-            ,
+        (,, uint256 limitMin0,, uint256 limitMin1,) =
-            uint256 limitMin1
+            relayer.getPoolState(address(sellToken), address(buyToken));
-            ,
+
-        ) = relayer.getPoolState(address(sellToken), address(buyToken));
- 
         limits = new uint256[](2);
         limits[0] = limitMin0;
         limits[1] = limitMin1;
     }
-}
+}

evm/test/TemplateSwapAdapter.t.sol

@@ -7,12 +7,14 @@ import "src/libraries/FractionMath.sol";
 
 /// @title TemplateSwapAdapterTest
 /// @dev This is a template for a swap adapter test.
-/// Test all functions that are implemented in your swap adapter, the two test included here are just an example.
+/// Test all functions that are implemented in your swap adapter, the two test
-/// Feel free to use UniswapV2SwapAdapterTest and BalancerV2SwapAdapterTest as a reference.
+/// included here are just an example.
+/// Feel free to use UniswapV2SwapAdapterTest and BalancerV2SwapAdapterTest as a
+/// reference.
 contract TemplateSwapAdapterTest is Test, ISwapAdapterTypes {
     using FractionMath for Fraction;
 
     function testPriceFuzz(uint256 amount0, uint256 amount1) public {}
 
     function testSwapFuzz(uint256 specifiedAmount) public {}
-}
+}

evm/test/UniswapV2SwapAdapter.t.sol

@@ -20,8 +20,8 @@ contract UniswapV2PairFunctionTest is Test, ISwapAdapterTypes {
     function setUp() public {
         uint256 forkBlock = 17000000;
         vm.createSelectFork(vm.rpcUrl("mainnet"), forkBlock);
-        adapter = new
+        adapter =
-            UniswapV2SwapAdapter(0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f);
+            new UniswapV2SwapAdapter(0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f);
 
         vm.label(address(adapter), "UniswapV2SwapAdapter");
         vm.label(address(WETH), "WETH");

proto/tycho/evm/v1/common.proto

@@ -25,6 +25,7 @@ message Transaction {
   // The receiver of the transaction.
   bytes to = 3;
   // The transactions index within the block.
+  // TODO: should this be uint32? to match the type from the native substream type?
   uint64 index = 4;
 }
 
@@ -47,6 +48,26 @@ message Attribute {
   ChangeType change = 3;
 }
 
+enum FinancialType{
+  SWAP = 0;
+  LEND = 1;
+  LEVERAGE = 2;
+  PSM = 3;
+}
+
+
+enum ImplementationType {
+  VM = 0;
+  CUSTOM = 1;
+}
+
+message ProtocolType{
+  string name = 1;
+  FinancialType financial_type = 2;
+  repeated Attribute attribute_schema = 3;
+  ImplementationType implementation_type = 4;
+}
+
 // A struct describing a part of the protocol.
 // Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, 
 // the component would represent a single contract. In case of VM integration, such component would 
@@ -67,6 +88,10 @@ message ProtocolComponent {
   repeated Attribute static_att = 4;
   // Type of change the component underwent.
   ChangeType change = 5;
+  /// Represents the functionality of the component.
+  ProtocolType protocol_type = 6;
+  // Transaction where this component was created
+  Transaction tx = 7;
 }
 
 // A struct for following the changes of Total Value Locked (TVL) of a protocol component.
@@ -75,9 +100,9 @@ message ProtocolComponent {
 message BalanceChange {
   // The address of the ERC20 token whose balance changed.
   bytes token = 1;
-  // The new balance of the token.
+  // The new balance of the token. Note: it must be a big endian encoded int.
   bytes balance = 2;
-  // The id of the component whose TVL is tracked.
+  // The id of the component whose TVL is tracked.  Note: This MUST be utf8 encoded.
   // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded.
   bytes component_id = 3;
 }

proto/tycho/evm/v1/utils.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package tycho.evm.v1;
+
+import "tycho/evm/v1/common.proto";
+
+// A message containing relative balance changes.
+//
+// Used to track token balances of protocol components in case they are only
+// available as relative values within a block.
+message BalanceDelta {
+  // The ordinal of the balance change. Must be unique & deterministic over all balances
+  // changes within a block.
+  uint64 ord = 1;
+  // The tx hash of the transaction that caused the balance change.
+  Transaction tx = 2;
+  // The address of the ERC20 token whose balance changed.
+  bytes token = 3;
+  // The delta balance of the token.
+  bytes delta = 4;
+  // The id of the component whose TVL is tracked.
+  // If the protocol component includes multiple contracts, the balance change must be
+  // aggregated to reflect how much tokens can be traded.
+  bytes component_id = 5;
+}
+
+// A set of balances deltas, usually a group of changes within a single block.
+message BlockBalanceDeltas {
+  repeated BalanceDelta balance_deltas = 1;
+}
+
+// A message containing protocol components that were created by a single tx.
+message TransactionProtocolComponents {
+  Transaction tx = 1;
+  repeated ProtocolComponent components = 2;
+}
+
+// All protocol components that were created within a block with their corresponding tx.
+message BlockTransactionProtocolComponents {
+  repeated TransactionProtocolComponents tx_components = 1;
+}

substreams/Cargo.lock

@@ -13,9 +13,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.75"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
 
 [[package]]
 name = "arrayvec"
@@ -42,15 +42,9 @@ dependencies = [
 
 [[package]]
 name = "bitflags"
-version = "1.3.2"
+version = "2.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
-
-[[package]]
-name = "bitflags"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
 
 [[package]]
 name = "bitvec"
@@ -99,9 +93,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
+checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
 dependencies = [
  "libc",
 ]
@@ -134,9 +128,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
 
 [[package]]
 name = "equivalent"
@@ -151,7 +145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -214,6 +208,25 @@ dependencies = [
  "tiny-keccak",
 ]
 
+[[package]]
+name = "ethereum-balancer"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "bytes",
+ "ethabi 18.0.0",
+ "getrandom",
+ "hex",
+ "hex-literal 0.4.1",
+ "itertools 0.12.1",
+ "num-bigint",
+ "prost 0.11.9",
+ "prost-types 0.12.3",
+ "substreams",
+ "substreams-ethereum",
+ "tycho-substreams",
+]
+
 [[package]]
 name = "ethereum-types"
 version = "0.13.1"
@@ -296,9 +309,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
 dependencies = [
  "cfg-if",
  "libc",
@@ -341,7 +354,7 @@ version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys",
 ]
 
 [[package]]
@@ -393,9 +406,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.1.0"
+version = "2.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
+checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4"
 dependencies = [
  "equivalent",
  "hashbrown",
@@ -412,9 +425,9 @@ dependencies = [
 
 [[package]]
 name = "itertools"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
 dependencies = [
  "either",
 ]
@@ -427,9 +440,9 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
 
 [[package]]
 name = "keccak"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940"
+checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654"
 dependencies = [
  "cpufeatures",
 ]
@@ -442,27 +455,27 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.151"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.12"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
+checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
 
 [[package]]
 name = "memchr"
-version = "2.6.4"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
 
 [[package]]
 name = "multimap"
@@ -483,19 +496,18 @@ dependencies = [
 
 [[package]]
 name = "num-integer"
-version = "0.1.45"
+version = "0.1.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
 dependencies = [
- "autocfg",
  "num-traits",
 ]
 
 [[package]]
 name = "num-traits"
-version = "0.2.17"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
+checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
 dependencies = [
  "autocfg",
 ]
@@ -595,9 +607,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-crate"
-version = "2.0.1"
+version = "2.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97dc5fea232fc28d2f597b37c4876b348a40e33f3b02cc975c8d006d78d94b1a"
+checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24"
 dependencies = [
  "toml_datetime",
  "toml_edit",
@@ -605,9 +617,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.70"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b"
+checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
 dependencies = [
  "unicode-ident",
 ]
@@ -677,7 +689,7 @@ dependencies = [
  "itertools 0.10.5",
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -700,9 +712,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
 dependencies = [
  "proc-macro2",
 ]
@@ -743,20 +755,11 @@ dependencies = [
  "getrandom",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
-dependencies = [
- "bitflags 1.3.2",
-]
-
 [[package]]
 name = "regex"
-version = "1.10.2"
+version = "1.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -766,9 +769,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.3"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -799,48 +802,48 @@ checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6"
 
 [[package]]
 name = "rustix"
-version = "0.38.28"
+version = "0.38.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
+checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949"
 dependencies = [
- "bitflags 2.4.1",
+ "bitflags",
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys",
 ]
 
 [[package]]
 name = "ryu"
-version = "1.0.16"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
+checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
 
 [[package]]
 name = "serde"
-version = "1.0.193"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
+checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.193"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
+checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.52",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.108"
+version = "1.0.114"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
+checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
 dependencies = [
  "itoa",
  "ryu",
@@ -865,9 +868,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
 [[package]]
 name = "substreams"
-version = "0.5.12"
+version = "0.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e3524a4e2931ff6cd58783e62adbd7e44f461752eca0c423793cfb462351f24"
+checksum = "3520661f782c338f0e3c6cfc001ac790ed5e68d8f28515139e2aa674f8bb54da"
 dependencies = [
  "anyhow",
  "bigdecimal",
@@ -884,24 +887,6 @@ dependencies = [
  "thiserror",
 ]
 
-[[package]]
-name = "substreams-balancer"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "bytes",
- "ethabi 18.0.0",
- "getrandom",
- "hex",
- "hex-literal 0.4.1",
- "itertools 0.12.0",
- "num-bigint",
- "prost 0.11.9",
- "prost-types 0.12.3",
- "substreams",
- "substreams-ethereum",
-]
-
 [[package]]
 name = "substreams-ethereum"
 version = "0.9.9"
@@ -967,9 +952,9 @@ dependencies = [
 
 [[package]]
 name = "substreams-macro"
-version = "0.5.12"
+version = "0.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63c2b15adf5b4d7a6d1a73c73df951a6b2df6fbb4f0b41304dc28c5550ce0ed0"
+checksum = "c15595ceab80fece579e462d4823048fe85d67922584c681f5e94305727ad9ee"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -990,9 +975,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.41"
+version = "2.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269"
+checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1007,35 +992,34 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tempfile"
-version = "3.8.1"
+version = "3.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
+checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
 dependencies = [
  "cfg-if",
  "fastrand",
- "redox_syscall",
  "rustix",
- "windows-sys 0.48.0",
+ "windows-sys",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.51"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7"
+checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.51"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df"
+checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.41",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1064,6 +1048,17 @@ dependencies = [
  "winnow",
 ]
 
+[[package]]
+name = "tycho-substreams"
+version = "0.1.0"
+dependencies = [
+ "hex",
+ "itertools 0.12.1",
+ "prost 0.11.9",
+ "substreams",
+ "substreams-ethereum",
+]
+
 [[package]]
 name = "typenum"
 version = "1.17.0"
@@ -1118,143 +1113,77 @@ dependencies = [
  "rustix",
 ]
 
-[[package]]
-name = "windows-sys"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
-dependencies = [
- "windows-targets 0.48.5",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.0",
+ "windows-targets",
 ]
 
 [[package]]
 name = "windows-targets"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b"
 dependencies = [
- "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_gnullvm",
- "windows_aarch64_msvc 0.48.5",
+ "windows_aarch64_msvc",
- "windows_i686_gnu 0.48.5",
+ "windows_i686_gnu",
- "windows_i686_msvc 0.48.5",
+ "windows_i686_msvc",
- "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnu",
- "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_gnullvm",
- "windows_x86_64_msvc 0.48.5",
+ "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
-dependencies = [
- "windows_aarch64_gnullvm 0.52.0",
- "windows_aarch64_msvc 0.52.0",
- "windows_i686_gnu 0.52.0",
- "windows_i686_msvc 0.52.0",
- "windows_x86_64_gnu 0.52.0",
- "windows_x86_64_gnullvm 0.52.0",
- "windows_x86_64_msvc 0.52.0",
 ]
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9"
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.48.5"
+version = "0.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
 
 [[package]]
 name = "winnow"
-version = "0.5.30"
+version = "0.5.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5"
+checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
 dependencies = [
  "memchr",
 ]

substreams/Cargo.toml

@@ -0,0 +1,22 @@
+[workspace]
+members = [
+    "ethereum-balancer",
+    "crates/tycho-substreams",
+]
+resolver = "2"
+
+
+[workspace.dependencies]
+substreams-ethereum = "0.9.9"
+substreams = "0.5"
+prost = "0.11"
+prost-types = "0.12.3"
+hex-literal = "0.4.1"
+hex = "0.4.3"
+ethabi = "18.0.0"
+tycho-substreams = {path  ="crates/tycho-substreams"}
+
+[profile.release]
+lto = true
+opt-level = 's'
+strip = "debuginfo"

substreams/Readme.md

@@ -1,99 +1,24 @@
-# Subtreams packages
+# Substreams Indexing Integrations
 
-This directory contains all substream packages that are used to index integrated protocols across different blockchains.
+Please refer to the official [Substreams Indexing](https://app.gitbook.com/o/9wMvRDQIhk1xOsIZ0Zde/s/Yx9kvxtpT2xWdzvFiB3t/indexing/substreams-integration) docs.
 
-## Adding a new package
+## Release
 
-To add a new package add a folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. 
+To release a package simply tag a commit with the package name and its version: 
+e.g. `ethereum-balancer-0.1.0`. This will create a release and automatically build 
+and push the spkg into our registry.
 
-### Manifest
+### Note
-In this new folder add a manifest file `substreams.yaml`. You can use the template below to get started:
+The CD pipeline will error if the Cargo version is not the same as the version in 
+the tag.
 
-```yaml
+Releases are immutable so do no try to delete tags or build the same release twice 
-specVersion: v0.1.0
+since this will error.
-package:
-  name: 'substreams_[CHAIN]_[PROTOCOL_SYSTEM]'
-  version: v0.1.0
 
-protobuf:
+### Pre release
-  files:
-    - vm.proto
-    - common.proto
-    # You can specify any internal proto files here
-  importPaths:
-    - ../../proto/tycho/evm/v1/
-    # Any private message types only used in internal modules 
-    # can remain local to the folder.
-    - ./proto
 
-binaries:
+To create a pre release for testing in dev you can start CD pipeline manually supplying 
-  default:
+the package you'd like to pre release. This will create a 
-    type: wasm/rust-v1
+`[package]-[semver].pre-[commit-sha]` release in our spkg repository which you can use 
-    # this points to the workspace target directory we use a special 
+to run the substream´.
-    # substreams build profile to optimise wasm binaries
-    file: ../../target/wasm32-unknown-unknown/substreams/substreams_[CHAIN]_[PROTOCOL_SYSTEM].wasm
 
-modules:
-  - name: map_changes
-    kind: map
-    inputs:
-      - source: sf.ethereum.type.v2.Block
-    output:
-      type: proto:tycho.evm.state.v1.BlockContractChanges
-```
-
-Substreams packages are Rust crates so we also need a `cargo.toml`.
-The example from the official docs will serve us just well:
-
-```toml
-[package]
-name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]"
-crate-type = ["cdylib"]
-
-[dependencies]
-substreams = "0.5"
-substreams-ethereum = "0.9"
-prost = "0.11"
-
-```
-
-There are already some generated rust files in the `src/pb` directory. These are generated 
-from the protobuf files in the `/proto/tycho/evm/v1` directory. They specify the output protobuf messages
-we want to generate. The input Block is specified by the subtreams crate, specifically the [sf.ethereum.type.v2.Block](https://github.com/streamingfast/substreams-ethereum/blob/develop/core/src/pb/sf.ethereum.type.v2.rs) message.
-
-You can define your own protobuf messages, make a new directory `/substreams/[CHAIN]-[PROTOCOL]/proto` for them.
-
-
-Now we can generate the Rust protobuf code:
-
-```
-substreams protogen substreams.yaml --exclude-paths="sf/substreams,google"
-```
-
-The command above should put the generate rust files under `/src/pb`. You
-can start using these now in your module handlers: See
-the [official substreams documentation](https://thegraph.com/docs/en/substreams/getting-started/quickstart/#create-substreams-module-handlers)
-on
-how to implement module handlers.
-
-You can also look into already existing substreams packages to see how it
-is done. E.g. [ethereum-ambient](./ethereum-ambient/) provides a pretty good
-example of how to get access to raw contract storage.
-
-# Tests
-
-To create a block test asset for ethereum do the following:
-
-- Follow [this tutorial](https://substreams.streamingfast.io/tutorials/overview/map_block_meta_module). Make sure you
-  set up the substreams-explorer repo in the same directory as this repo.
-    - Comment out `image: ./ethereum.png` in `ethereum-explorer/substreams.yaml`
-    - Add `prost-types = "0.11.0"` to `ethereum-explorer/Cargo.toml`
-- Make sure you set up your key env vars.
-- Run `sh scripts/download-ethereum-block-to-s3 BLOCK_NUMBER`
-
-Do not commit the block files (they are quite big).

substreams/check.sh

@@ -0,0 +1,6 @@
+set -e
+
+cargo +nightly fmt -- --check
+cargo +nightly clippy --all --all-features --all-targets -- -D warnings
+cargo build --target wasm32-unknown-unknown --all-targets --all-features
+cargo test --workspace --all-targets --all-features

substreams/crates/tycho-substreams/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "tycho-substreams"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+substreams-ethereum.workspace = true
+substreams.workspace = true
+prost.workspace = true
+hex.workspace = true
+itertools = "0.12.0"

substreams/crates/tycho-substreams/Readme.md

@@ -0,0 +1,18 @@
+# Tycho Substreams SDK
+
+Some shared functionality that is used to create tycho substream packages.
+
+## Protobuf Models
+
+Protobuf models are manually synced from the `tycho-indexer` repository whenever they 
+changed.
+
+To generate the rust structs run the following command from within the `./proto` 
+directory:
+
+```bash
+buf generate \
+  --path tycho \
+  --template ../substreams/crates/tycho-substreams/buf.gen.yaml \
+  --output ../substreams/crates/tycho-substreams/ 
+```

substreams/crates/tycho-substreams/buf.gen.yaml

@@ -0,0 +1,12 @@
+version: v1
+plugins:
+  - plugin: buf.build/community/neoeinstein-prost:v0.2.2
+    out: src/pb
+    opt:
+      - file_descriptor_set=false
+      - type_attribute=.tycho.evm.v1.Transaction=#[derive(Eq\, Hash)]
+
+  - plugin: buf.build/community/neoeinstein-prost-crate:v0.3.1
+    out: src/pb
+    opt:
+      - no_features

substreams/crates/tycho-substreams/src/balances.rs

@@ -0,0 +1,380 @@
+//! Module for Handling Relative Balance Changes.
+//!
+//! This module facilitates the conversion of relative balance changes into absolute balances,
+//! employing a structured approach to ensure the accurate representation of balance data.
+//!
+//! Process Overview:
+//!
+//! 1. **Mapping (User-Implemented)**: The initial step requires the user to implement a mapping
+//!    function that extracts `BlockBalanceDeltas` messages. It's crucial that `BalanceDelta`
+//!    messages within these messages have strictly increasing ordinals, which guarantees the order
+//!    of balance changes is preserved and unambiguous. This step is not provided by the SDK and
+//!    must be custom-implemented to suit the specific protocol.
+//!
+//! 2. **Storing Changes**: Utilize the `store_balance_changes` function to apply relative balance
+//!    changes. This function handles changes additively, preparing them for final aggregation.
+//!
+//! 3. **Aggregation**: Use the `aggregate_balance_changes` function to compile the processed
+//!    changes into a detailed map of absolute balances. This final step produces the comprehensive
+//!    balance data ready for output modules or further analysis.
+//!
+//! Through this sequence, the module ensures the transformation from relative to absolute
+//! balances is conducted with high fidelity, upholding the integrity of transactional data.
+
+use crate::pb::tycho::evm::v1::{BalanceChange, BlockBalanceDeltas, Transaction};
+use itertools::Itertools;
+use std::{collections::HashMap, str::FromStr};
+use substreams::{
+    key,
+    pb::substreams::StoreDeltas,
+    prelude::{BigInt, StoreAdd},
+};
+
+/// Stores relative balance changes in an additive manner.
+///
+/// Aggregates the relative balance changes from a `BlockBalanceDeltas` message into the store
+/// in an additive way. This function ensures that balance changes are applied correctly
+/// according to the order specified by their ordinal values. Each token's balance changes
+/// must have strictly increasing ordinals; otherwise, the function will panic.
+///
+/// This method is designed to work in conjunction with `aggregate_balances_changes`,
+/// which consumes the data stored by this function. The stored data is intended for use
+/// in a "deltas mode" processing pattern, as described in the
+/// [Substreams documentation](https://substreams.streamingfast.io/documentation/develop/manifest-modules/types#deltas-mode).
+///
+/// ## Arguments
+/// * `deltas` - A `BlockBalanceDeltas` message containing the relative balance changes. It is
+///   crucial that the relative balance deltas for each token address have strictly increasing
+///   ordinals; the function will panic otherwise.
+/// * `store` - An implementation of the `StoreAdd` trait that will be used to add relative balance
+///   changes. This store should support the addition of `BigInt` values.
+///
+/// ## Panics
+/// This function will panic if:
+/// - The `component_id` of any delta is not valid UTF-8.
+/// - The ordinals for any given token address are not strictly increasing.
+pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: impl StoreAdd<BigInt>) {
+    let mut previous_ordinal = HashMap::<String, u64>::new();
+    deltas
+        .balance_deltas
+        .iter()
+        .for_each(|delta| {
+            let balance_key = format!(
+                "{0}:{1}",
+                String::from_utf8(delta.component_id.clone())
+                    .expect("delta.component_id is not valid utf-8!"),
+                hex::encode(&delta.token)
+            );
+            let current_ord = delta.ord;
+            previous_ordinal
+                .entry(balance_key.clone())
+                .and_modify(|ord| {
+                    // ordinals must arrive in increasing order
+                    if *ord >= current_ord {
+                        panic!(
+                            "Invalid ordinal sequence for {}: {} >= {}",
+                            balance_key, *ord, current_ord
+                        );
+                    }
+                    *ord = current_ord;
+                })
+                .or_insert(delta.ord);
+            store.add(delta.ord, balance_key, BigInt::from_signed_bytes_be(&delta.delta));
+        });
+}
+
+type TxAggregatedBalances = HashMap<Vec<u8>, (Transaction, HashMap<Vec<u8>, BalanceChange>)>;
+
+/// Aggregates absolute balances per transaction and token.
+///
+/// ## Arguments
+/// * `balance_store` - A `StoreDeltas` with all changes that occured in the source store module.
+/// * `deltas` - A `BlockBalanceDeltas` message containing the relative balances changes.
+///
+/// This function reads absolute balance values from an additive store (see `store_balance_changes`
+/// for how to create such a store). It zips these values with the relative balance deltas to
+/// associate balance values with tokens and components, ensuring the last balance change per token
+/// per transaction is kept if there are multiple changes. Negative balances are set to 0, adhering
+/// to the expectation that absolute balances must be non-negative.
+///
+/// Will keep the last balance change per token per transaction if there are multiple
+/// changes. In case a balance ends up being negative, it will be clipped to 0 since
+/// absolute balances are expected to be either zero or positive.
+///
+/// ## Panics
+/// May panic if the store deltas values are not in the correct format. Values are
+/// expected to be utf-8 encoded string integers, which is the default behaviour
+/// for substreams stores.
+///
+/// ## Returns
+/// A map of transactions hashes to a tuple of `Transaction` and aggregated
+/// absolute balance changes.
+pub fn aggregate_balances_changes(
+    balance_store: StoreDeltas,
+    deltas: BlockBalanceDeltas,
+) -> TxAggregatedBalances {
+    balance_store
+        .deltas
+        .into_iter()
+        .zip(deltas.balance_deltas)
+        .map(|(store_delta, balance_delta)| {
+            let component_id = key::segment_at(&store_delta.key, 0);
+            let token_id = key::segment_at(&store_delta.key, 1);
+            // store_delta.new_value is an ASCII string representing an integer
+            let ascii_string =
+                String::from_utf8(store_delta.new_value.clone()).expect("Invalid UTF-8 sequence");
+            let balance = BigInt::from_str(&ascii_string).expect("Failed to parse integer");
+
+            // If the absolute balance is negative, we set it to zero.
+            let big_endian_bytes_balance = if balance < BigInt::zero() {
+                BigInt::zero().to_bytes_be().1
+            } else {
+                balance.to_bytes_be().1
+            };
+
+            (
+                balance_delta
+                    .tx
+                    .expect("Missing transaction on delta"),
+                BalanceChange {
+                    token: hex::decode(token_id).expect("Token ID not valid hex"),
+                    balance: big_endian_bytes_balance,
+                    component_id: component_id.as_bytes().to_vec(),
+                },
+            )
+        })
+        // We need to group the balance changes by tx hash for the `TransactionContractChanges` agg
+        .group_by(|(tx, _)| tx.hash.clone())
+        .into_iter()
+        .map(|(txh, group)| {
+            let (mut transactions, balance_changes): (Vec<_>, Vec<_>) = group.into_iter().unzip();
+
+            let balances = balance_changes
+                .into_iter()
+                .map(|balance_change| (balance_change.token.clone(), balance_change))
+                .collect();
+            (txh, (transactions.pop().unwrap(), balances))
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{mock_store::MockStore, pb::tycho::evm::v1::BalanceDelta};
+    use substreams::{
+        pb::substreams::StoreDelta,
+        prelude::{StoreGet, StoreNew},
+    };
+
+    fn block_balance_deltas() -> BlockBalanceDeltas {
+        let comp_id = "0x42c0ffee"
+            .to_string()
+            .as_bytes()
+            .to_vec();
+        let token_0 = hex::decode("bad999").unwrap();
+        let token_1 = hex::decode("babe00").unwrap();
+        BlockBalanceDeltas {
+            balance_deltas: vec![
+                BalanceDelta {
+                    ord: 0,
+                    tx: Some(Transaction {
+                        hash: vec![0, 1],
+                        from: vec![9, 9],
+                        to: vec![8, 8],
+                        index: 0,
+                    }),
+                    token: token_0.clone(),
+                    delta: BigInt::from_str("+1000")
+                        .unwrap()
+                        .to_signed_bytes_be(),
+                    component_id: comp_id.clone(),
+                },
+                BalanceDelta {
+                    ord: 2,
+                    tx: Some(Transaction {
+                        hash: vec![0, 1],
+                        from: vec![9, 9],
+                        to: vec![8, 8],
+                        index: 0,
+                    }),
+                    token: token_1.clone(),
+                    delta: BigInt::from_str("+100")
+                        .unwrap()
+                        .to_signed_bytes_be(),
+                    component_id: comp_id.clone(),
+                },
+                BalanceDelta {
+                    ord: 3,
+                    tx: Some(Transaction {
+                        hash: vec![0, 1],
+                        from: vec![9, 9],
+                        to: vec![8, 8],
+                        index: 0,
+                    }),
+                    token: token_1.clone(),
+                    delta: BigInt::from_str("50")
+                        .unwrap()
+                        .to_signed_bytes_be(),
+                    component_id: comp_id.clone(),
+                },
+                BalanceDelta {
+                    ord: 10,
+                    tx: Some(Transaction {
+                        hash: vec![0, 1],
+                        from: vec![9, 9],
+                        to: vec![8, 8],
+                        index: 0,
+                    }),
+                    token: token_0.clone(),
+                    delta: BigInt::from_str("-1")
+                        .unwrap()
+                        .to_signed_bytes_be(),
+                    component_id: comp_id.clone(),
+                },
+            ],
+        }
+    }
+    fn store_deltas() -> StoreDeltas {
+        let comp_id = "0x42c0ffee"
+            .to_string()
+            .as_bytes()
+            .to_vec();
+        let token_0 = hex::decode("bad999").unwrap();
+        let token_1 = hex::decode("babe00").unwrap();
+
+        let t0_key =
+            format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(token_0));
+        let t1_key =
+            format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(token_1));
+        StoreDeltas {
+            deltas: vec![
+                StoreDelta {
+                    operation: 0,
+                    ordinal: 0,
+                    key: t0_key.clone(),
+                    old_value: BigInt::from(0)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                    new_value: BigInt::from(1000)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                },
+                StoreDelta {
+                    operation: 0,
+                    ordinal: 2,
+                    key: t1_key.clone(),
+                    old_value: BigInt::from(0)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                    new_value: BigInt::from(100)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                },
+                StoreDelta {
+                    operation: 0,
+                    ordinal: 3,
+                    key: t1_key.clone(),
+                    old_value: BigInt::from(100)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                    new_value: BigInt::from(150)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                },
+                StoreDelta {
+                    operation: 0,
+                    ordinal: 10,
+                    key: t0_key.clone(),
+                    old_value: BigInt::from(1000)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                    new_value: BigInt::from(999)
+                        .to_string()
+                        .as_bytes()
+                        .to_vec(),
+                },
+            ],
+        }
+    }
+
+    #[test]
+    fn test_store_balances() {
+        let comp_id = "0x42c0ffee"
+            .to_string()
+            .as_bytes()
+            .to_vec();
+        let token_0 = hex::decode("bad999").unwrap();
+        let token_1 = hex::decode("babe00").unwrap();
+        let deltas = block_balance_deltas();
+        let store = <MockStore as StoreNew>::new();
+
+        store_balance_changes(deltas, store.clone());
+        let res_0 = store.get_last(format!(
+            "{}:{}",
+            String::from_utf8(comp_id.clone()).unwrap(),
+            hex::encode(token_0)
+        ));
+        let res_1 = store.get_last(format!(
+            "{}:{}",
+            String::from_utf8(comp_id.clone()).unwrap(),
+            hex::encode(token_1)
+        ));
+
+        assert_eq!(res_0, Some(BigInt::from_str("+999").unwrap()));
+        assert_eq!(res_1, Some(BigInt::from_str("+150").unwrap()));
+    }
+
+    #[test]
+    fn test_aggregate_balances_changes() {
+        let store_deltas = store_deltas();
+        let balance_deltas = block_balance_deltas();
+        let comp_id = "0x42c0ffee"
+            .to_string()
+            .as_bytes()
+            .to_vec();
+        let token_0 = hex::decode("bad999").unwrap();
+        let token_1 = hex::decode("babe00").unwrap();
+
+        let exp = [(
+            vec![0, 1],
+            (
+                Transaction { hash: vec![0, 1], from: vec![9, 9], to: vec![8, 8], index: 0 },
+                [
+                    (
+                        token_0.clone(),
+                        BalanceChange {
+                            token: token_0,
+                            balance: BigInt::from(999)
+                                .to_signed_bytes_be()
+                                .to_vec(),
+                            component_id: comp_id.clone(),
+                        },
+                    ),
+                    (
+                        token_1.clone(),
+                        BalanceChange {
+                            token: token_1,
+                            balance: vec![150],
+                            component_id: comp_id.clone(),
+                        },
+                    ),
+                ]
+                .into_iter()
+                .collect::<HashMap<_, _>>(),
+            ),
+        )]
+        .into_iter()
+        .collect::<HashMap<_, _>>();
+
+        let res = aggregate_balances_changes(store_deltas, balance_deltas);
+        assert_eq!(res, exp);
+    }
+}

substreams/crates/tycho-substreams/src/contract.rs

@@ -0,0 +1,223 @@
+/// Helpers to extract relevant contract storage.
+///
+/// This file contains helpers to capture contract changes from the expanded block
+/// model. These leverage the `code_changes`, `balance_changes`, and `storage_changes`
+/// fields available on the `Call` type provided by block model in a substream
+/// (i.e. `logs_and_calls`, etc).
+///
+/// ## Warning
+/// ⚠️ These helpers *only* work if the **extended block model** is available,
+/// more [here](https://streamingfastio.medium.com/new-block-model-to-accelerate-chain-integration-9f65126e5425)
+use std::collections::HashMap;
+
+use substreams_ethereum::pb::{
+    eth,
+    eth::v2::{block::DetailLevel, StorageChange},
+};
+
+use crate::pb::tycho::evm::v1::{self as tycho};
+
+struct SlotValue {
+    new_value: Vec<u8>,
+    start_value: Vec<u8>,
+}
+
+impl From<&StorageChange> for SlotValue {
+    fn from(change: &StorageChange) -> Self {
+        Self { new_value: change.new_value.clone(), start_value: change.old_value.clone() }
+    }
+}
+
+impl SlotValue {
+    fn has_changed(&self) -> bool {
+        self.start_value != self.new_value
+    }
+}
+
+// Uses a map for slots, protobuf does not allow bytes in hashmap keys
+struct InterimContractChange {
+    address: Vec<u8>,
+    balance: Vec<u8>,
+    code: Vec<u8>,
+    slots: HashMap<Vec<u8>, SlotValue>,
+    change: tycho::ChangeType,
+}
+
+impl InterimContractChange {
+    fn new(address: &[u8], creation: bool) -> Self {
+        Self {
+            address: address.to_vec(),
+            balance: vec![],
+            code: vec![],
+            slots: Default::default(),
+            change: if creation { tycho::ChangeType::Creation } else { tycho::ChangeType::Update },
+        }
+    }
+}
+
+impl From<InterimContractChange> for tycho::ContractChange {
+    fn from(value: InterimContractChange) -> Self {
+        tycho::ContractChange {
+            address: value.address,
+            balance: value.balance,
+            code: value.code,
+            slots: value
+                .slots
+                .into_iter()
+                .filter(|(_, value)| value.has_changed())
+                .map(|(slot, value)| tycho::ContractSlot { slot, value: value.new_value })
+                .collect(),
+            change: value.change.into(),
+        }
+    }
+}
+
+/// Extracts and aggregates contract changes from a block.
+///
+/// This function identifies and collects changes to contract storage, code, and native balance for
+/// contracts of interest within a given block. It filters contracts based on a user-defined
+/// predicate and aggregates changes into a provided mutable map.
+///
+/// ## Arguments
+/// * `block` - The block from which to extract contract changes, expected to be an extended block
+///   model.
+/// * `inclusion_predicate` - A closure that determines if a contract's address is of interest for
+///   the collection of changes. Only contracts satisfying this predicate are included.
+/// * `transaction_contract_changes` - A mutable reference to a map where extracted contract changes
+///   are stored. Keyed by transaction index, it aggregates changes into
+///   `tycho::TransactionContractChanges`.
+///
+/// ## Panics
+/// Panics if the provided block is not an extended block model, as indicated by its detail level.
+///
+/// ## Operation
+/// The function iterates over transactions and their calls within the block, collecting contract
+/// changes (storage, balance, code) that pass the inclusion predicate. Changes are then sorted by
+/// their ordinals to maintain the correct sequence of events. Aggregated changes for each contract
+/// are stored in `transaction_contract_changes`, categorized by transaction index.
+///
+/// Contracts created within the block are tracked to differentiate between new and existing
+/// contracts. The aggregation process respects transaction boundaries, ensuring that changes are
+/// mapped accurately to their originating transactions.
+pub fn extract_contract_changes<F: Fn(&[u8]) -> bool>(
+    block: &eth::v2::Block,
+    inclusion_predicate: F,
+    transaction_contract_changes: &mut HashMap<u64, tycho::TransactionContractChanges>,
+) {
+    if block.detail_level != Into::<i32>::into(DetailLevel::DetaillevelExtended) {
+        panic!("Only extended blocks are supported");
+    }
+    let mut changed_contracts: HashMap<Vec<u8>, InterimContractChange> = HashMap::new();
+
+    // Collect all accounts created in this block
+    let created_accounts: HashMap<_, _> = block
+        .transactions()
+        .flat_map(|tx| {
+            tx.calls.iter().flat_map(|call| {
+                call.account_creations
+                    .iter()
+                    .map(|ac| (&ac.account, ac.ordinal))
+            })
+        })
+        .collect();
+
+    block
+        .transactions()
+        .for_each(|block_tx| {
+            let mut storage_changes = Vec::new();
+            let mut balance_changes = Vec::new();
+            let mut code_changes = Vec::new();
+
+            block_tx
+                .calls
+                .iter()
+                .filter(|call| !call.state_reverted && inclusion_predicate(&call.address))
+                .for_each(|call| {
+                    storage_changes.extend(call.storage_changes.iter());
+                    balance_changes.extend(call.balance_changes.iter());
+                    code_changes.extend(call.code_changes.iter());
+                });
+
+            storage_changes.sort_unstable_by_key(|change| change.ordinal);
+            balance_changes.sort_unstable_by_key(|change| change.ordinal);
+            code_changes.sort_unstable_by_key(|change| change.ordinal);
+
+            storage_changes
+                .iter()
+                .filter(|changes| inclusion_predicate(&changes.address))
+                .for_each(|&storage_change| {
+                    let contract_change = changed_contracts
+                        .entry(storage_change.address.clone())
+                        .or_insert_with(|| {
+                            InterimContractChange::new(
+                                &storage_change.address,
+                                created_accounts.contains_key(&storage_change.address),
+                            )
+                        });
+
+                    let slot_value = contract_change
+                        .slots
+                        .entry(storage_change.key.clone())
+                        .or_insert_with(|| storage_change.into());
+
+                    slot_value
+                        .new_value
+                        .copy_from_slice(&storage_change.new_value);
+                });
+
+            balance_changes
+                .iter()
+                .filter(|changes| inclusion_predicate(&changes.address))
+                .for_each(|balance_change| {
+                    let contract_change = changed_contracts
+                        .entry(balance_change.address.clone())
+                        .or_insert_with(|| {
+                            InterimContractChange::new(
+                                &balance_change.address,
+                                created_accounts.contains_key(&balance_change.address),
+                            )
+                        });
+
+                    if let Some(new_balance) = &balance_change.new_value {
+                        contract_change.balance.clear();
+                        contract_change
+                            .balance
+                            .extend_from_slice(&new_balance.bytes);
+                    }
+                });
+
+            code_changes
+                .iter()
+                .filter(|changes| inclusion_predicate(&changes.address))
+                .for_each(|code_change| {
+                    let contract_change = changed_contracts
+                        .entry(code_change.address.clone())
+                        .or_insert_with(|| {
+                            InterimContractChange::new(
+                                &code_change.address,
+                                created_accounts.contains_key(&code_change.address),
+                            )
+                        });
+
+                    contract_change.code.clear();
+                    contract_change
+                        .code
+                        .extend_from_slice(&code_change.new_code);
+                });
+
+            if !storage_changes.is_empty() ||
+                !balance_changes.is_empty() ||
+                !code_changes.is_empty()
+            {
+                transaction_contract_changes
+                    .entry(block_tx.index.into())
+                    .or_insert_with(|| tycho::TransactionContractChanges::new(&(block_tx.into())))
+                    .contract_changes
+                    .extend(
+                        changed_contracts
+                            .drain()
+                            .map(|(_, change)| change.into()),
+                    );
+            }
+        });
+}

substreams/crates/tycho-substreams/src/lib.rs

@@ -0,0 +1,9 @@
+pub mod balances;
+pub mod contract;
+mod mock_store;
+pub mod models;
+mod pb;
+
+pub mod prelude {
+    pub use super::models::*;
+}

substreams/crates/tycho-substreams/src/mock_store.rs

@@ -0,0 +1,95 @@
+//! Contains a mock store for internal testing.
+//!
+//! Might make this public alter to users can test their store handlers.
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
+use substreams::{
+    prelude::{BigInt, StoreDelete, StoreGet, StoreNew},
+    store::StoreAdd,
+};
+
+type BigIntStore = HashMap<String, Vec<(u64, BigInt)>>;
+
+#[derive(Debug, Clone)]
+pub struct MockStore {
+    data: Rc<RefCell<BigIntStore>>,
+}
+
+impl StoreDelete for MockStore {
+    fn delete_prefix(&self, _ord: i64, prefix: &String) {
+        self.data
+            .borrow_mut()
+            .retain(|k, _| !k.starts_with(prefix));
+    }
+}
+
+impl StoreNew for MockStore {
+    fn new() -> Self {
+        Self { data: Rc::new(RefCell::new(HashMap::new())) }
+    }
+}
+
+impl StoreAdd<BigInt> for MockStore {
+    fn add<K: AsRef<str>>(&self, ord: u64, key: K, value: BigInt) {
+        let mut guard = self.data.borrow_mut();
+        guard
+            .entry(key.as_ref().to_string())
+            .and_modify(|v| {
+                let prev_value = v.last().unwrap().1.clone();
+                v.push((ord, prev_value + value.clone()));
+            })
+            .or_insert(vec![(ord, value)]);
+    }
+
+    fn add_many<K: AsRef<str>>(&self, _ord: u64, _keys: &Vec<K>, _value: BigInt) {
+        todo!()
+    }
+}
+
+impl StoreGet<BigInt> for MockStore {
+    fn new(_idx: u32) -> Self {
+        Self { data: Rc::new(RefCell::new(HashMap::new())) }
+    }
+
+    fn get_at<K: AsRef<str>>(&self, ord: u64, key: K) -> Option<BigInt> {
+        self.data
+            .borrow()
+            .get(&key.as_ref().to_string())
+            .map(|v| {
+                v.iter()
+                    .find(|(current_ord, _)| *current_ord == ord)
+                    .unwrap()
+                    .1
+                    .clone()
+            })
+    }
+
+    fn get_last<K: AsRef<str>>(&self, key: K) -> Option<BigInt> {
+        self.data
+            .borrow()
+            .get(&key.as_ref().to_string())
+            .map(|v| v.last().unwrap().1.clone())
+    }
+
+    fn get_first<K: AsRef<str>>(&self, key: K) -> Option<BigInt> {
+        self.data
+            .borrow()
+            .get(&key.as_ref().to_string())
+            .map(|v| v.first().unwrap().1.clone())
+    }
+
+    fn has_at<K: AsRef<str>>(&self, ord: u64, key: K) -> bool {
+        self.data
+            .borrow()
+            .get(&key.as_ref().to_string())
+            .map(|v| v.iter().any(|(v, _)| *v == ord))
+            .unwrap_or(false)
+    }
+
+    fn has_last<K: AsRef<str>>(&self, _key: K) -> bool {
+        todo!()
+    }
+
+    fn has_first<K: AsRef<str>>(&self, _key: K) -> bool {
+        todo!()
+    }
+}

substreams/crates/tycho-substreams/src/models.rs

@@ -0,0 +1,145 @@
+use substreams_ethereum::pb::eth::v2::{self as sf};
+
+// re-export the protobuf types here.
+pub use crate::pb::tycho::evm::v1::*;
+
+impl TransactionContractChanges {
+    /// Creates a new empty `TransactionContractChanges` instance.
+    pub fn new(tx: &Transaction) -> Self {
+        Self {
+            tx: Some(tx.clone()),
+            contract_changes: vec![],
+            component_changes: vec![],
+            balance_changes: vec![],
+        }
+    }
+}
+
+impl From<&sf::TransactionTrace> for Transaction {
+    fn from(tx: &sf::TransactionTrace) -> Self {
+        Self {
+            hash: tx.hash.clone(),
+            from: tx.from.clone(),
+            to: tx.to.clone(),
+            index: tx.index.into(),
+        }
+    }
+}
+
+impl From<&sf::Block> for Block {
+    fn from(block: &sf::Block) -> Self {
+        Self {
+            number: block.number,
+            hash: block.hash.clone(),
+            parent_hash: block
+                .header
+                .as_ref()
+                .expect("Block header not present")
+                .parent_hash
+                .clone(),
+            ts: block.timestamp_seconds(),
+        }
+    }
+}
+
+impl ProtocolComponent {
+    /// Constructs a new, empty `ProtocolComponent`.
+    ///
+    /// Initializes an instance with default values. Use `with_*` methods to populate fields
+    /// conveniently.
+    ///
+    /// ## Parameters
+    /// - `id`: Identifier for the component.
+    /// - `tx`: Reference to the associated transaction.
+    pub fn new(id: &str, tx: &Transaction) -> Self {
+        Self {
+            id: id.to_string(),
+            tokens: Vec::new(),
+            contracts: Vec::new(),
+            static_att: Vec::new(),
+            change: ChangeType::Creation.into(),
+            protocol_type: None,
+            tx: Some(tx.clone()),
+        }
+    }
+
+    /// Initializes a `ProtocolComponent` with a direct association to a contract.
+    ///
+    /// Sets the component's ID to the hex-encoded address with a `0x` prefix and includes the
+    /// contract in the contracts list.
+    ///
+    /// ## Parameters
+    /// - `id`: Contract address to be encoded and set as the component's ID.
+    /// - `tx`: Reference to the associated transaction.
+    pub fn at_contract(id: &[u8], tx: &Transaction) -> Self {
+        Self {
+            id: format!("0x{}", hex::encode(id)),
+            tokens: Vec::new(),
+            contracts: vec![id.to_vec()],
+            static_att: Vec::new(),
+            change: ChangeType::Creation.into(),
+            protocol_type: None,
+            tx: Some(tx.clone()),
+        }
+    }
+
+    /// Updates the tokens associated with this component.
+    ///
+    /// ## Parameters
+    /// - `tokens`: Slice of byte slices representing the tokens to associate.
+    pub fn with_tokens<B: AsRef<[u8]>>(mut self, tokens: &[B]) -> Self {
+        self.tokens = tokens
+            .iter()
+            .map(|e| e.as_ref().to_vec())
+            .collect::<Vec<Vec<u8>>>();
+        self
+    }
+
+    /// Updates the contracts associated with this component.
+    ///
+    /// ## Parameters
+    /// - `contracts`: Slice of byte slices representing the contracts to associate.
+    pub fn with_contracts<B: AsRef<[u8]>>(mut self, contracts: &[B]) -> Self {
+        self.contracts = contracts
+            .iter()
+            .map(|e| e.as_ref().to_vec())
+            .collect::<Vec<Vec<u8>>>();
+        self
+    }
+
+    /// Updates the static attributes of this component.
+    ///
+    /// Sets the change type to `Creation` for all attributes.
+    ///
+    /// ## Parameters
+    /// - `attributes`: Slice of key-value pairs representing the attributes to set.
+    pub fn with_attributes<K: AsRef<str>, V: AsRef<[u8]>>(mut self, attributes: &[(K, V)]) -> Self {
+        self.static_att = attributes
+            .iter()
+            .map(|(k, v)| Attribute {
+                name: k.as_ref().to_string(),
+                value: v.as_ref().to_vec(),
+                change: ChangeType::Creation.into(),
+            })
+            .collect::<Vec<Attribute>>();
+        self
+    }
+
+    /// Designates this component as a swap type within the protocol.
+    ///
+    /// Sets the `protocol_type` accordingly, including `financial_type` as `Swap` and leaving
+    /// `attribute_schema` empty.
+    ///
+    /// ## Parameters
+    /// - `name`: The name of the swap protocol.
+    /// - `implementation_type`: The implementation type of the protocol.
+    pub fn as_swap_type(mut self, name: &str, implementation_type: ImplementationType) -> Self {
+        self.protocol_type = Some(ProtocolType {
+            name: name.to_string(),
+            financial_type: FinancialType::Swap.into(),
+            attribute_schema: Vec::new(),
+            implementation_type: implementation_type.into(),
+        });
+        self
+    }
+}

substreams/crates/tycho-substreams/src/pb/tycho.evm.v1.rs

@@ -19,6 +19,7 @@ pub struct Block {
     pub ts: u64,
 }
 /// A struct describing a transaction.
+#[derive(Eq, Hash)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct Transaction {
@@ -51,6 +52,18 @@ pub struct Attribute {
     #[prost(enumeration="ChangeType", tag="3")]
     pub change: i32,
 }
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtocolType {
+    #[prost(string, tag="1")]
+    pub name: ::prost::alloc::string::String,
+    #[prost(enumeration="FinancialType", tag="2")]
+    pub financial_type: i32,
+    #[prost(message, repeated, tag="3")]
+    pub attribute_schema: ::prost::alloc::vec::Vec<Attribute>,
+    #[prost(enumeration="ImplementationType", tag="4")]
+    pub implementation_type: i32,
+}
 /// A struct describing a part of the protocol.
 /// Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, 
 /// the component would represent a single contract. In case of VM integration, such component would 
@@ -78,20 +91,12 @@ pub struct ProtocolComponent {
     /// Type of change the component underwent.
     #[prost(enumeration="ChangeType", tag="5")]
     pub change: i32,
-}
+    /// / Represents the functionality of the component.
-#[allow(clippy::derive_partial_eq_without_eq)]
+    #[prost(message, optional, tag="6")]
-#[derive(Clone, PartialEq, ::prost::Message)]
+    pub protocol_type: ::core::option::Option<ProtocolType>,
-pub struct TransactionProtocolComponents {
+    /// Transaction where this component was created
-    #[prost(message, optional, tag="1")]
+    #[prost(message, optional, tag="7")]
     pub tx: ::core::option::Option<Transaction>,
-    #[prost(message, repeated, tag="2")]
-    pub components: ::prost::alloc::vec::Vec<ProtocolComponent>,
-}
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct GroupedTransactionProtocolComponents {
-    #[prost(message, repeated, tag="1")]
-    pub tx_components: ::prost::alloc::vec::Vec<TransactionProtocolComponents>,
 }
 /// A struct for following the changes of Total Value Locked (TVL) of a protocol component.
 /// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole.
@@ -102,42 +107,14 @@ pub struct BalanceChange {
     /// The address of the ERC20 token whose balance changed.
     #[prost(bytes="vec", tag="1")]
     pub token: ::prost::alloc::vec::Vec<u8>,
-    /// The new balance of the token.
+    /// The new balance of the token. Note: it must be a big endian encoded int.
     #[prost(bytes="vec", tag="2")]
     pub balance: ::prost::alloc::vec::Vec<u8>,
-    /// The id of the component whose TVL is tracked.
+    /// The id of the component whose TVL is tracked.  Note: This MUST be utf8 encoded.
     /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded.
     #[prost(bytes="vec", tag="3")]
     pub component_id: ::prost::alloc::vec::Vec<u8>,
 }
-/// A struct for following the changes of Total Value Locked (TVL) of a protocol component.
-/// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole.
-/// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct BalanceDelta {
-    #[prost(uint64, tag="1")]
-    pub ord: u64,
-    /// The tx hash of the transaction that caused the balance change.
-    #[prost(message, optional, tag="2")]
-    pub tx: ::core::option::Option<Transaction>,
-    /// The address of the ERC20 token whose balance changed.
-    #[prost(bytes="vec", tag="3")]
-    pub token: ::prost::alloc::vec::Vec<u8>,
-    /// The delta balance of the token.
-    #[prost(bytes="vec", tag="4")]
-    pub delta: ::prost::alloc::vec::Vec<u8>,
-    /// The id of the component whose TVL is tracked.
-    /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded.
-    #[prost(bytes="vec", tag="5")]
-    pub component_id: ::prost::alloc::vec::Vec<u8>,
-}
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct BalanceDeltas {
-    #[prost(message, repeated, tag="1")]
-    pub balance_deltas: ::prost::alloc::vec::Vec<BalanceDelta>,
-}
 /// Enum to specify the type of a change.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
@@ -171,6 +148,151 @@ impl ChangeType {
         }
     }
 }
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum FinancialType {
+    Swap = 0,
+    Lend = 1,
+    Leverage = 2,
+    Psm = 3,
+}
+impl FinancialType {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            FinancialType::Swap => "SWAP",
+            FinancialType::Lend => "LEND",
+            FinancialType::Leverage => "LEVERAGE",
+            FinancialType::Psm => "PSM",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "SWAP" => Some(Self::Swap),
+            "LEND" => Some(Self::Lend),
+            "LEVERAGE" => Some(Self::Leverage),
+            "PSM" => Some(Self::Psm),
+            _ => None,
+        }
+    }
+}
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum ImplementationType {
+    Vm = 0,
+    Custom = 1,
+}
+impl ImplementationType {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            ImplementationType::Vm => "VM",
+            ImplementationType::Custom => "CUSTOM",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "VM" => Some(Self::Vm),
+            "CUSTOM" => Some(Self::Custom),
+            _ => None,
+        }
+    }
+}
+// This file contains the definition for the native integration of Substreams.
+
+/// A component is a set of attributes that are associated with a custom entity.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct EntityChanges {
+    /// A unique identifier of the entity within the protocol.
+    #[prost(string, tag="1")]
+    pub component_id: ::prost::alloc::string::String,
+    /// The set of attributes that are associated with the entity.
+    #[prost(message, repeated, tag="2")]
+    pub attributes: ::prost::alloc::vec::Vec<Attribute>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TransactionEntityChanges {
+    #[prost(message, optional, tag="1")]
+    pub tx: ::core::option::Option<Transaction>,
+    #[prost(message, repeated, tag="2")]
+    pub entity_changes: ::prost::alloc::vec::Vec<EntityChanges>,
+    /// An array of newly added components.
+    #[prost(message, repeated, tag="3")]
+    pub component_changes: ::prost::alloc::vec::Vec<ProtocolComponent>,
+    /// An array of balance changes to components.
+    #[prost(message, repeated, tag="4")]
+    pub balance_changes: ::prost::alloc::vec::Vec<BalanceChange>,
+}
+/// A set of transaction changes within a single block.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BlockEntityChanges {
+    /// The block for which these changes are collectively computed.
+    #[prost(message, optional, tag="1")]
+    pub block: ::core::option::Option<Block>,
+    /// The set of transaction changes observed in the specified block.
+    #[prost(message, repeated, tag="2")]
+    pub changes: ::prost::alloc::vec::Vec<TransactionEntityChanges>,
+}
+/// A message containing relative balance changes.
+///
+/// Used to track token balances of protocol components in case they are only
+/// available as relative values within a block.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BalanceDelta {
+    /// The ordinal of the balance change. Must be unique & deterministic over all balances
+    /// changes within a block.
+    #[prost(uint64, tag="1")]
+    pub ord: u64,
+    /// The tx hash of the transaction that caused the balance change.
+    #[prost(message, optional, tag="2")]
+    pub tx: ::core::option::Option<Transaction>,
+    /// The address of the ERC20 token whose balance changed.
+    #[prost(bytes="vec", tag="3")]
+    pub token: ::prost::alloc::vec::Vec<u8>,
+    /// The delta balance of the token.
+    #[prost(bytes="vec", tag="4")]
+    pub delta: ::prost::alloc::vec::Vec<u8>,
+    /// The id of the component whose TVL is tracked.
+    /// If the protocol component includes multiple contracts, the balance change must be
+    /// aggregated to reflect how much tokens can be traded.
+    #[prost(bytes="vec", tag="5")]
+    pub component_id: ::prost::alloc::vec::Vec<u8>,
+}
+/// A set of balances deltas, usually a group of changes within a single block.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BlockBalanceDeltas {
+    #[prost(message, repeated, tag="1")]
+    pub balance_deltas: ::prost::alloc::vec::Vec<BalanceDelta>,
+}
+/// A message containing protocol components that were created by a single tx.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TransactionProtocolComponents {
+    #[prost(message, optional, tag="1")]
+    pub tx: ::core::option::Option<Transaction>,
+    #[prost(message, repeated, tag="2")]
+    pub components: ::prost::alloc::vec::Vec<ProtocolComponent>,
+}
+/// All protocol components that were created within a block with their corresponding tx.
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BlockTransactionProtocolComponents {
+    #[prost(message, repeated, tag="1")]
+    pub tx_components: ::prost::alloc::vec::Vec<TransactionProtocolComponents>,
+}
 // This file contains proto definitions specific to the VM integration.
 
 /// A key value entry into contract storage.

substreams/ethereum-balancer/Cargo.toml

@@ -1,24 +1,25 @@
 [package]
-name = "substreams-balancer"
+name = "ethereum-balancer"
 version = "0.1.0"
 edition = "2021"
 
 [lib]
-name = "substreams_balancer"
+name = "ethereum_balancer"
 crate-type = ["cdylib"]
 
 [dependencies]
-substreams = "0.5"
+substreams.workspace =  true
-substreams-ethereum = "0.9.9"
+substreams-ethereum.workspace =  true
-prost = "0.11"
+prost.workspace = true
-hex-literal = "0.4.1"
+prost-types.workspace = true
-ethabi = "18.0.0"
+hex-literal.workspace = true
-hex = "0.4.2"
+ethabi.workspace = true
+hex.workspace = true
 bytes = "1.5.0"
 anyhow = "1.0.75"
-prost-types = "0.12.3"
 num-bigint = "0.4.4"
 itertools = "0.12.0"
+tycho-substreams.workspace = true
 
 [build-dependencies]
 anyhow = "1"

substreams/ethereum-balancer/build.rs

@@ -8,6 +8,7 @@ fn main() -> Result<()> {
 
     let files = fs::read_dir(abi_folder)?;
     let mut mod_rs_content = String::new();
+    mod_rs_content.push_str("#![allow(clippy::all)]\n");
 
     for file in files {
         let file = file?;

substreams/ethereum-balancer/proto/factory.proto

@@ -1,19 +0,0 @@
-syntax = "proto3";
-
-package eth.factory.v1;
-
-message Pools {
-  repeated Pool pools = 1;
-}
-
-message Pool {
-  bytes pool_id = 1;
-  fixed64 log_ordinal = 2;
-}
-
-message Transfer {
-  bytes from = 1;
-  bytes to = 2;
-  string token = 3;
-  string amount = 4;
-}

substreams/ethereum-balancer/proto/pool.proto

@@ -1,15 +0,0 @@
-syntax = "proto3";
-
-package eth.pool.v1;
-
-message Transfers {
-  repeated Transfer transfers = 1;
-}
-
-message Transfer {
-  string from = 1;
-  string to = 2;
-  uint64 token_id = 3;
-  string trx_hash = 4;
-  uint64 ordinal = 5;
-}

substreams/ethereum-balancer/proto/tycho/evm/v1/common.proto

@@ -1,113 +0,0 @@
-syntax = "proto3";
-
-package tycho.evm.v1;
-
-// This file contains the proto definitions for Substreams common to all integrations.
-
-// A struct describing a block.
-message Block {
-  // The blocks hash.
-  bytes hash = 1;
-  // The parent blocks hash.
-  bytes parent_hash = 2;
-  // The block number.
-  uint64 number = 3;
-  // The block timestamp.
-  uint64 ts = 4;
-}
-
-// A struct describing a transaction.
-message Transaction {
-  // The transaction hash.
-  bytes hash = 1;
-  // The sender of the transaction.
-  bytes from = 2;
-  // The receiver of the transaction.
-  bytes to = 3;
-  // The transactions index within the block.
-  // TODO: should this be uint32? to match the type from the native substream type?
-  uint64 index = 4;
-}
-
-// Enum to specify the type of a change.
-enum ChangeType {
-  CHANGE_TYPE_UNSPECIFIED = 0;
-  CHANGE_TYPE_UPDATE = 1;
-  CHANGE_TYPE_CREATION = 2;
-  CHANGE_TYPE_DELETION = 3;
-}
-
-// A custom struct representing an arbitrary attribute of a protocol component.
-// This is mainly used by the native integration to track the necessary information about the protocol.
-message Attribute {
-  // The name of the attribute.
-  string name = 1;
-  // The value of the attribute.
-  bytes value = 2;
-  // The type of change the attribute underwent.
-  ChangeType change = 3;
-}
-
-// A struct describing a part of the protocol.
-// Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, 
-// the component would represent a single contract. In case of VM integration, such component would 
-// not need any attributes, because all the relevant info would be tracked via storage slots and balance changes.
-// It can also be a wrapping contract, like WETH, that has a constant price, but it allows swapping tokens. 
-// This is why the name ProtocolComponent is used instead of "Pool" or "Pair".
-message ProtocolComponent {
-  // A unique identifier for the component within the protocol.
-  // Can be e.g. a stringified address or a string describing the trading pair.
-  string id = 1;
-  // Addresses of the ERC20 tokens used by the component.
-  repeated bytes tokens = 2;
-  // Addresses of the contracts used by the component.
-  // Usually it is a single contract, but some protocols use multiple contracts.
-  repeated bytes contracts = 3;
-  // Attributes of the component. Used mainly be the native integration.
-  // The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent.
-  repeated Attribute static_att = 4;
-  // Type of change the component underwent.
-  ChangeType change = 5;
-}
-
-message TransactionProtocolComponents {
-  Transaction tx = 1;
-  repeated ProtocolComponent components = 2;
-}
-
-message GroupedTransactionProtocolComponents {
-  repeated TransactionProtocolComponents tx_components = 1;
-}
-
-// A struct for following the changes of Total Value Locked (TVL) of a protocol component.
-// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole.
-// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract.
-message BalanceChange {
-  // The address of the ERC20 token whose balance changed.
-  bytes token = 1;
-  // The new balance of the token.
-  bytes balance = 2;
-  // The id of the component whose TVL is tracked.
-  // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded.
-  bytes component_id = 3;
-}
-
-// A struct for following the changes of Total Value Locked (TVL) of a protocol component.
-// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole.
-// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract.
-message BalanceDelta {
-  uint64 ord = 1;
-  // The tx hash of the transaction that caused the balance change.
-  Transaction tx = 2;
-  // The address of the ERC20 token whose balance changed.
-  bytes token = 3;
-  // The delta balance of the token.
-  bytes delta = 4;
-  // The id of the component whose TVL is tracked.
-  // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded.
-  bytes component_id = 5;
-}
-
-message BalanceDeltas {
-  repeated BalanceDelta balance_deltas = 1;
-}

substreams/ethereum-balancer/proto/tycho/evm/v1/entity.proto

@@ -1,32 +0,0 @@
-syntax = "proto3";
-
-package tycho.evm.v1;
-
-import "tycho/evm/v1/common.proto";
-
-// This file contains the definition for the native integration of Substreams.
-
-// A component is a set of attributes that are associated with a custom entity.
-message EntityChanges {
-  // A unique identifier of the entity within the protocol.
-  string component_id = 1;
-  // The set of attributes that are associated with the entity.
-  repeated Attribute attributes = 2;
-}
-
-message TransactionEntityChanges {
-  Transaction tx = 1;
-  repeated EntityChanges entity_changes = 2;
-  // An array of newly added components.
-  repeated ProtocolComponent component_changes = 3;
-  // An array of balance changes to components.
-  repeated BalanceChange balance_changes = 4;
-}
-
-// A set of transaction changes within a single block.
-message BlockEntityChanges {
-  // The block for which these changes are collectively computed.
-  Block block = 1;
-  // The set of transaction changes observed in the specified block.
-  repeated TransactionEntityChanges changes = 2;
-}

substreams/ethereum-balancer/proto/tycho/evm/v1/vm.proto

@@ -1,50 +0,0 @@
-syntax = "proto3";
-
-package tycho.evm.v1;
-
-import "tycho/evm/v1/common.proto";
-
-// This file contains proto definitions specific to the VM integration.
-
-// A key value entry into contract storage.
-message ContractSlot {
-  // A contract's storage slot.
-  bytes slot = 2;
-  // The new value for this storage slot.
-  bytes value = 3;
-}
-
-// Changes made to a single contract's state.
-message ContractChange {
-  // The contract's address
-  bytes address = 1;
-  // The new native balance of the contract, empty bytes indicates no change.
-  bytes balance = 2;
-  // The new code of the contract, empty bytes indicates no change.
-  bytes code = 3;
-  // The changes to this contract's slots, empty sequence indicates no change.
-  repeated ContractSlot slots = 4;
-  // Whether this is an update, a creation or a deletion.
-  ChangeType change = 5;
-}
-
-// A set of changes aggregated by transaction.
-message TransactionContractChanges {
-  // The transaction instance that results in the changes.
-  Transaction tx = 1;
-  // Contains the changes induced by the above transaction, aggregated on a per-contract basis.
-  // Must include changes to every contract that is tracked by all ProtocolComponents.
-  repeated ContractChange contract_changes = 2;
-  // An array of any component changes.
-  repeated ProtocolComponent component_changes = 3;
-  // An array of balance changes to components.
-  repeated BalanceChange balance_changes = 4;
-}
-
-// A set of transaction changes within a single block.
-message BlockContractChanges {
-  // The block for which these changes are collectively computed.
-  Block block = 1;
-  // The set of transaction changes observed in the specified block.
-  repeated TransactionContractChanges changes = 2;
-}

substreams/ethereum-balancer/src/abi/mod.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::all)]
 pub mod yearn_linear_pool_factory;
 pub mod composable_stable_pool_factory;
 pub mod vault;

substreams/ethereum-balancer/src/contract_changes.rs

@@ -1,190 +0,0 @@
-/// This file contains helpers to capture contract changes from the expanded block model. These
-///  leverage the `code_changes`, `balance_changes`, and `storage_changes` fields available on the
-///  `Call` type provided by block model in a substream (i.e. `logs_and_calls`, etc).
-///
-/// ⚠️ These helpers *only* work if the **expanded block model** is available, more info blow.
-/// https://streamingfastio.medium.com/new-block-model-to-accelerate-chain-integration-9f65126e5425
-use std::collections::HashMap;
-
-use substreams_ethereum::pb::eth;
-
-use pb::tycho::evm::v1::{self as tycho};
-
-use substreams::store::{StoreGet, StoreGetInt64};
-
-use crate::pb;
-
-struct SlotValue {
-    new_value: Vec<u8>,
-    start_value: Vec<u8>,
-}
-
-impl SlotValue {
-    fn has_changed(&self) -> bool {
-        self.start_value != self.new_value
-    }
-}
-
-// Uses a map for slots, protobuf does not allow bytes in hashmap keys
-pub struct InterimContractChange {
-    address: Vec<u8>,
-    balance: Vec<u8>,
-    code: Vec<u8>,
-    slots: HashMap<Vec<u8>, SlotValue>,
-    change: tycho::ChangeType,
-}
-
-impl From<InterimContractChange> for tycho::ContractChange {
-    fn from(value: InterimContractChange) -> Self {
-        tycho::ContractChange {
-            address: value.address,
-            balance: value.balance,
-            code: value.code,
-            slots: value
-                .slots
-                .into_iter()
-                .filter(|(_, value)| value.has_changed())
-                .map(|(slot, value)| tycho::ContractSlot {
-                    slot,
-                    value: value.new_value,
-                })
-                .collect(),
-            change: value.change.into(),
-        }
-    }
-}
-
-pub fn extract_contract_changes(
-    block: &eth::v2::Block,
-    contracts: StoreGetInt64,
-    transaction_contract_changes: &mut HashMap<u64, tycho::TransactionContractChanges>,
-) {
-    let mut changed_contracts: HashMap<Vec<u8>, InterimContractChange> = HashMap::new();
-
-    // Collect all accounts created in this block
-    let created_accounts: HashMap<_, _> = block
-        .transactions()
-        .flat_map(|tx| {
-            tx.calls.iter().flat_map(|call| {
-                call.account_creations
-                    .iter()
-                    .map(|ac| (&ac.account, ac.ordinal))
-            })
-        })
-        .collect();
-
-    block.transactions().for_each(|block_tx| {
-        let mut storage_changes = Vec::new();
-        let mut balance_changes = Vec::new();
-        let mut code_changes = Vec::new();
-
-        block_tx
-            .calls
-            .iter()
-            .filter(|call| {
-                !call.state_reverted
-                    && contracts
-                        .get_last(format!("pool:{0}", hex::encode(&call.address)))
-                        .is_some()
-            })
-            .for_each(|call| {
-                storage_changes.extend(call.storage_changes.iter());
-                balance_changes.extend(call.balance_changes.iter());
-                code_changes.extend(call.code_changes.iter());
-            });
-
-        storage_changes.sort_unstable_by_key(|change| change.ordinal);
-        balance_changes.sort_unstable_by_key(|change| change.ordinal);
-        code_changes.sort_unstable_by_key(|change| change.ordinal);
-
-        storage_changes.iter().for_each(|storage_change| {
-            let contract_change = changed_contracts
-                .entry(storage_change.address.clone())
-                .or_insert_with(|| InterimContractChange {
-                    address: storage_change.address.clone(),
-                    balance: Vec::new(),
-                    code: Vec::new(),
-                    slots: HashMap::new(),
-                    change: if created_accounts.contains_key(&storage_change.address) {
-                        tycho::ChangeType::Creation
-                    } else {
-                        tycho::ChangeType::Update
-                    },
-                });
-
-            let slot_value = contract_change
-                .slots
-                .entry(storage_change.key.clone())
-                .or_insert_with(|| SlotValue {
-                    new_value: storage_change.new_value.clone(),
-                    start_value: storage_change.old_value.clone(),
-                });
-
-            slot_value
-                .new_value
-                .copy_from_slice(&storage_change.new_value);
-        });
-
-        balance_changes.iter().for_each(|balance_change| {
-            let contract_change = changed_contracts
-                .entry(balance_change.address.clone())
-                .or_insert_with(|| InterimContractChange {
-                    address: balance_change.address.clone(),
-                    balance: Vec::new(),
-                    code: Vec::new(),
-                    slots: HashMap::new(),
-                    change: if created_accounts.contains_key(&balance_change.address) {
-                        tycho::ChangeType::Creation
-                    } else {
-                        tycho::ChangeType::Update
-                    },
-                });
-
-            if let Some(new_balance) = &balance_change.new_value {
-                contract_change.balance.clear();
-                contract_change
-                    .balance
-                    .extend_from_slice(&new_balance.bytes);
-            }
-        });
-
-        code_changes.iter().for_each(|code_change| {
-            let contract_change = changed_contracts
-                .entry(code_change.address.clone())
-                .or_insert_with(|| InterimContractChange {
-                    address: code_change.address.clone(),
-                    balance: Vec::new(),
-                    code: Vec::new(),
-                    slots: HashMap::new(),
-                    change: if created_accounts.contains_key(&code_change.address) {
-                        tycho::ChangeType::Creation
-                    } else {
-                        tycho::ChangeType::Update
-                    },
-                });
-
-            contract_change.code.clear();
-            contract_change
-                .code
-                .extend_from_slice(&code_change.new_code);
-        });
-
-        if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() {
-            transaction_contract_changes
-                .entry(block_tx.index.into())
-                .or_insert_with(|| tycho::TransactionContractChanges {
-                    tx: Some(tycho::Transaction {
-                        hash: block_tx.hash.clone(),
-                        from: block_tx.from.clone(),
-                        to: block_tx.to.clone(),
-                        index: block_tx.index as u64,
-                    }),
-                    contract_changes: vec![],
-                    component_changes: vec![],
-                    balance_changes: vec![],
-                })
-                .contract_changes
-                .extend(changed_contracts.drain().map(|(_, change)| change.into()));
-        }
-    });
-}

substreams/ethereum-balancer/src/lib.rs

@@ -1,5 +1,3 @@
 mod abi;
-mod contract_changes;
 mod modules;
-mod pb;
 mod pool_factories;

substreams/ethereum-balancer/src/modules.rs

@@ -1,44 +1,24 @@
-use std::collections::HashMap;
+use crate::{abi, pool_factories};
-
 use anyhow::Result;
-use substreams::hex;
-use substreams::pb::substreams::StoreDeltas;
-use substreams::store::{
-    StoreAdd, StoreAddBigInt, StoreAddInt64, StoreGet, StoreGetInt64, StoreNew,
-};
-
-use substreams::key;
-use substreams::scalar::BigInt;
-
-use substreams_ethereum::pb::eth;
-
 use itertools::Itertools;
-use pb::tycho::evm::v1::{self as tycho};
+use std::collections::HashMap;
-
+use substreams::{
-use contract_changes::extract_contract_changes;
+    hex,
-
+    pb::substreams::StoreDeltas,
-use crate::{abi, contract_changes, pb, pool_factories};
+    store::{StoreAdd, StoreAddBigInt, StoreAddInt64, StoreGet, StoreGetInt64, StoreNew},
+};
+use substreams_ethereum::{pb::eth, Event};
+use tycho_substreams::{
+    balances::aggregate_balances_changes, contract::extract_contract_changes, prelude::*,
+};
 
 const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8");
 
-/// This struct purely exists to spoof the `PartialEq` trait for `Transaction` so we can use it in
-///  a later groupby operation.
-#[derive(Debug)]
-struct TransactionWrapper(tycho::Transaction);
-
-impl PartialEq for TransactionWrapper {
-    fn eq(&self, other: &Self) -> bool {
-        self.0.hash == other.0.hash
-    }
-}
-
 #[substreams::handlers::map]
-pub fn map_pools_created(
+pub fn map_components(block: eth::v2::Block) -> Result<BlockTransactionProtocolComponents> {
-    block: eth::v2::Block,
-) -> Result<tycho::GroupedTransactionProtocolComponents> {
     // Gather contract changes by indexing `PoolCreated` events and analysing the `Create` call
     // We store these as a hashmap by tx hash since we need to agg by tx hash later
-    Ok(tycho::GroupedTransactionProtocolComponents {
+    Ok(BlockTransactionProtocolComponents {
         tx_components: block
             .transactions()
             .filter_map(|tx| {
@@ -46,24 +26,17 @@ pub fn map_pools_created(
                     .logs_with_calls()
                     .filter(|(_, call)| !call.call.state_reverted)
                     .filter_map(|(log, call)| {
-                        Some(pool_factories::address_map(
+                        pool_factories::address_map(
                             call.call.address.as_slice(),
                             log,
                             call.call,
-                        )?)
+                            &(tx.into()),
+                        )
                     })
                     .collect::<Vec<_>>();
 
                 if !components.is_empty() {
-                    Some(tycho::TransactionProtocolComponents {
+                    Some(TransactionProtocolComponents { tx: Some(tx.into()), components })
-                        tx: Some(tycho::Transaction {
-                            hash: tx.hash.clone(),
-                            from: tx.from.clone(),
-                            to: tx.to.clone(),
-                            index: Into::<u64>::into(tx.index),
-                        }),
-                        components,
-                    })
                 } else {
                     None
                 }
@@ -74,7 +47,7 @@ pub fn map_pools_created(
 
 /// Simply stores the `ProtocolComponent`s with the pool id as the key
 #[substreams::handlers::store]
-pub fn store_pools_created(map: tycho::GroupedTransactionProtocolComponents, store: StoreAddInt64) {
+pub fn store_components(map: BlockTransactionProtocolComponents, store: StoreAddInt64) {
     store.add_many(
         0,
         &map.tx_components
@@ -86,85 +59,95 @@ pub fn store_pools_created(map: tycho::GroupedTransactionProtocolComponents, sto
     );
 }
 
-/// Since the `PoolBalanceChanged` events administer only deltas, we need to leverage a map and a
+/// Since the `PoolBalanceChanged` and `Swap` events administer only deltas, we need to leverage a
-///  store to be able to tally up final balances for tokens in a pool.
+/// map and a  store to be able to tally up final balances for tokens in a pool.
 #[substreams::handlers::map]
-pub fn map_balance_deltas(
+pub fn map_relative_balances(
     block: eth::v2::Block,
     store: StoreGetInt64,
-) -> Result<tycho::BalanceDeltas, anyhow::Error> {
+) -> Result<BlockBalanceDeltas, anyhow::Error> {
-    Ok(tycho::BalanceDeltas {
+    let balance_deltas = block
-        balance_deltas: block
+        .logs()
-            .events::<abi::vault::events::PoolBalanceChanged>(&[&VAULT_ADDRESS])
+        .filter(|log| log.address() == VAULT_ADDRESS)
-            .flat_map(|(event, log)| {
+        .flat_map(|vault_log| {
-                event
+            let mut deltas = Vec::new();
-                    .tokens
-                    .iter()
-                    .zip(event.deltas.iter())
-                    .filter_map(|(token, delta)| {
-                        let component_id: Vec<_> = event.pool_id.into();
 
-                        if store
+            if let Some(ev) =
-                            .get_last(format!("pool:{0}", hex::encode(&component_id)))
+                abi::vault::events::PoolBalanceChanged::match_and_decode(vault_log.log)
-                            .is_none()
+            {
-                        {
+                let component_id = format!("0x{}", hex::encode(&ev.pool_id[..20]));
-                            return None;
-                        }
 
-                        Some(tycho::BalanceDelta {
+                if store
-                            ord: log.log.ordinal,
+                    .get_last(format!("pool:{}", component_id))
-                            tx: Some(tycho::Transaction {
+                    .is_some()
-                                hash: log.receipt.transaction.hash.clone(),
+                {
-                                from: log.receipt.transaction.from.clone(),
+                    for (token, delta) in ev.tokens.iter().zip(ev.deltas.iter()) {
-                                to: log.receipt.transaction.to.clone(),
+                        deltas.push(BalanceDelta {
-                                index: Into::<u64>::into(log.receipt.transaction.index),
+                            ord: vault_log.ordinal(),
-                            }),
+                            tx: Some(vault_log.receipt.transaction.into()),
-                            token: token.clone(),
+                            token: token.to_vec(),
                             delta: delta.to_signed_bytes_be(),
-                            component_id: component_id.clone(),
+                            component_id: component_id.as_bytes().to_vec(),
-                        })
+                        });
-                    })
+                    }
-                    .collect::<Vec<_>>()
+                }
-            })
+            } else if let Some(ev) = abi::vault::events::Swap::match_and_decode(vault_log.log) {
-            .collect::<Vec<_>>(),
+                let component_id = format!("0x{}", hex::encode(&ev.pool_id[..20]));
-    })
+
+                if store
+                    .get_last(format!("pool:{}", component_id))
+                    .is_some()
+                {
+                    deltas.extend_from_slice(&[
+                        BalanceDelta {
+                            ord: vault_log.ordinal(),
+                            tx: Some(vault_log.receipt.transaction.into()),
+                            token: ev.token_in.to_vec(),
+                            delta: ev.amount_in.to_signed_bytes_be(),
+                            component_id: component_id.as_bytes().to_vec(),
+                        },
+                        BalanceDelta {
+                            ord: vault_log.ordinal(),
+                            tx: Some(vault_log.receipt.transaction.into()),
+                            token: ev.token_out.to_vec(),
+                            delta: ev.amount_out.neg().to_signed_bytes_be(),
+                            component_id: component_id.as_bytes().to_vec(),
+                        },
+                    ]);
+                }
+            }
+
+            deltas
+        })
+        .collect::<Vec<_>>();
+
+    Ok(BlockBalanceDeltas { balance_deltas })
 }
 
 /// It's significant to include both the `pool_id` and the `token_id` for each balance delta as the
 ///  store key to ensure that there's a unique balance being tallied for each.
 #[substreams::handlers::store]
-pub fn store_balance_changes(deltas: tycho::BalanceDeltas, store: StoreAddBigInt) {
+pub fn store_balances(deltas: BlockBalanceDeltas, store: StoreAddBigInt) {
-    deltas.balance_deltas.iter().for_each(|delta| {
+    tycho_substreams::balances::store_balance_changes(deltas, store);
-        store.add(
-            delta.ord,
-            format!(
-                "pool:{0}:token:{1}",
-                hex::encode(&delta.component_id),
-                hex::encode(&delta.token)
-            ),
-            BigInt::from_signed_bytes_be(&delta.delta),
-        );
-    });
 }
 
 /// This is the main map that handles most of the indexing of this substream.
 /// Every contract change is grouped by transaction index via the `transaction_contract_changes`
 ///  map. Each block of code will extend the `TransactionContractChanges` struct with the
 ///  cooresponding changes (balance, component, contract), inserting a new one if it doesn't exist.
-///  At the very end, the map can easily be sorted by index to ensure the final `BlockContractChanges`
+///  At the very end, the map can easily be sorted by index to ensure the final
-///  is ordered by transactions properly.
+/// `BlockContractChanges`  is ordered by transactions properly.
 #[substreams::handlers::map]
-pub fn map_changes(
+pub fn map_protocol_changes(
     block: eth::v2::Block,
-    grouped_components: tycho::GroupedTransactionProtocolComponents,
+    grouped_components: BlockTransactionProtocolComponents,
-    deltas: tycho::BalanceDeltas,
+    deltas: BlockBalanceDeltas,
     components_store: StoreGetInt64,
     balance_store: StoreDeltas, // Note, this map module is using the `deltas` mode for the store.
-) -> Result<tycho::BlockContractChanges> {
+) -> Result<BlockContractChanges> {
     // We merge contract changes by transaction (identified by transaction index) making it easy to
     //  sort them at the very end.
-    let mut transaction_contract_changes: HashMap<_, tycho::TransactionContractChanges> =
+    let mut transaction_contract_changes: HashMap<_, TransactionContractChanges> = HashMap::new();
-        HashMap::new();
 
     // `ProtocolComponents` are gathered from `map_pools_created` which just need a bit of work to
     //   convert into `TransactionContractChanges`
@@ -173,83 +156,49 @@ pub fn map_changes(
         .iter()
         .for_each(|tx_component| {
             let tx = tx_component.tx.as_ref().unwrap();
-
             transaction_contract_changes
                 .entry(tx.index)
-                .or_insert_with(|| tycho::TransactionContractChanges {
+                .or_insert_with(|| TransactionContractChanges::new(tx))
-                    tx: Some(tx.clone()),
-                    contract_changes: vec![],
-                    component_changes: vec![],
-                    balance_changes: vec![],
-                })
                 .component_changes
                 .extend_from_slice(&tx_component.components);
         });
 
     // Balance changes are gathered by the `StoreDelta` based on `PoolBalanceChanged` creating
-    //  `BalanceDeltas`. We essentially just process the changes that occured to the `store` this
+    //  `BlockBalanceDeltas`. We essentially just process the changes that occurred to the `store`
-    //  block. Then, these balance changes are merged onto the existing map of tx contract changes,
+    // this  block. Then, these balance changes are merged onto the existing map of tx contract
-    //  inserting a new one if it doesn't exist.
+    // changes,  inserting a new one if it doesn't exist.
-    balance_store
+    aggregate_balances_changes(balance_store, deltas)
-        .deltas
         .into_iter()
-        .zip(deltas.balance_deltas)
+        .for_each(|(_, (tx, balances))| {
-        .map(|(store_delta, balance_delta)| {
-            let pool_id = key::segment_at(&store_delta.key, 1);
-            let token_id = key::segment_at(&store_delta.key, 3);
-            (
-                balance_delta.tx.unwrap(),
-                tycho::BalanceChange {
-                    token: hex::decode(token_id).expect("Token ID not valid hex"),
-                    balance: store_delta.new_value,
-                    component_id: hex::decode(pool_id).expect("Token ID not valid hex"),
-                },
-            )
-        })
-        // We need to group the balance changes by tx hash for the `TransactionContractChanges` agg
-        .group_by(|(tx, _)| TransactionWrapper(tx.clone()))
-        .into_iter()
-        .for_each(|(tx_wrapped, group)| {
-            let tx = tx_wrapped.0;
-
             transaction_contract_changes
                 .entry(tx.index)
-                .or_insert_with(|| tycho::TransactionContractChanges {
+                .or_insert_with(|| TransactionContractChanges::new(&tx))
-                    tx: Some(tx.clone()),
-                    contract_changes: vec![],
-                    component_changes: vec![],
-                    balance_changes: vec![],
-                })
                 .balance_changes
-                .extend(group.map(|(_, change)| change));
+                .extend(balances.into_values());
         });
 
-    // General helper for extracting contract changes. Uses block, our component store which holds
+    // Extract and insert any storage changes that happened for any of the components.
-    //  all of our tracked deployed pool addresses, and the map of tx contract changes which we
+    extract_contract_changes(
-    //  output into for final processing later.
+        &block,
-    extract_contract_changes(&block, components_store, &mut transaction_contract_changes);
+        |addr| {
+            components_store
+                .get_last(format!("pool:0x{0}", hex::encode(addr)))
+                .is_some()
+        },
+        &mut transaction_contract_changes,
+    );
 
     // Process all `transaction_contract_changes` for final output in the `BlockContractChanges`,
     //  sorted by transaction index (the key).
-    Ok(tycho::BlockContractChanges {
+    Ok(BlockContractChanges {
-        block: Some(tycho::Block {
+        block: Some((&block).into()),
-            number: block.number,
-            hash: block.hash.clone(),
-            parent_hash: block
-                .header
-                .as_ref()
-                .expect("Block header not present")
-                .parent_hash
-                .clone(),
-            ts: block.timestamp_seconds(),
-        }),
         changes: transaction_contract_changes
             .drain()
-            .sorted_unstable_by_key(|(index, _)| index.clone())
+            .sorted_unstable_by_key(|(index, _)| *index)
             .filter_map(|(_, change)| {
-                if change.contract_changes.is_empty()
+                if change.contract_changes.is_empty() &&
-                    && change.component_changes.is_empty()
+                    change.component_changes.is_empty() &&
-                    && change.balance_changes.is_empty()
+                    change.balance_changes.is_empty()
                 {
                     None
                 } else {

substreams/ethereum-balancer/src/pb/eth.balancer.v1.rs

@@ -1,28 +0,0 @@
-// @generated
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Pools {
-    #[prost(message, repeated, tag="1")]
-    pub pools: ::prost::alloc::vec::Vec<Pool>,
-}
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Pool {
-    #[prost(bytes="vec", tag="1")]
-    pub pool_id: ::prost::alloc::vec::Vec<u8>,
-    #[prost(fixed64, tag="2")]
-    pub log_ordinal: u64,
-}
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Transfer {
-    #[prost(bytes="vec", tag="1")]
-    pub from: ::prost::alloc::vec::Vec<u8>,
-    #[prost(bytes="vec", tag="2")]
-    pub to: ::prost::alloc::vec::Vec<u8>,
-    #[prost(string, tag="3")]
-    pub token: ::prost::alloc::string::String,
-    #[prost(string, tag="4")]
-    pub amount: ::prost::alloc::string::String,
-}
-// @@protoc_insertion_point(module)

substreams/ethereum-balancer/src/pb/eth.factory.v1.rs

@@ -1,28 +0,0 @@
-// @generated
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Pools {
-    #[prost(message, repeated, tag="1")]
-    pub pools: ::prost::alloc::vec::Vec<Pool>,
-}
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Pool {
-    #[prost(bytes="vec", tag="1")]
-    pub pool_id: ::prost::alloc::vec::Vec<u8>,
-    #[prost(fixed64, tag="2")]
-    pub log_ordinal: u64,
-}
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Transfer {
-    #[prost(bytes="vec", tag="1")]
-    pub from: ::prost::alloc::vec::Vec<u8>,
-    #[prost(bytes="vec", tag="2")]
-    pub to: ::prost::alloc::vec::Vec<u8>,
-    #[prost(string, tag="3")]
-    pub token: ::prost::alloc::string::String,
-    #[prost(string, tag="4")]
-    pub amount: ::prost::alloc::string::String,
-}
-// @@protoc_insertion_point(module)

substreams/ethereum-balancer/src/pb/eth.pool.v1.rs

@@ -1,22 +0,0 @@
-// @generated
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Transfers {
-    #[prost(message, repeated, tag="1")]
-    pub transfers: ::prost::alloc::vec::Vec<Transfer>,
-}
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Transfer {
-    #[prost(string, tag="1")]
-    pub from: ::prost::alloc::string::String,
-    #[prost(string, tag="2")]
-    pub to: ::prost::alloc::string::String,
-    #[prost(uint64, tag="3")]
-    pub token_id: u64,
-    #[prost(string, tag="4")]
-    pub trx_hash: ::prost::alloc::string::String,
-    #[prost(uint64, tag="5")]
-    pub ordinal: u64,
-}
-// @@protoc_insertion_point(module)

substreams/ethereum-balancer/src/pool_factories.rs

@@ -1,18 +1,17 @@
-use substreams_ethereum::pb::eth::v2::{Call, Log};
-use substreams_ethereum::{Event, Function};
-
 use crate::abi;
-use crate::pb;
+use substreams::{hex, scalar::BigInt};
-use pb::tycho::evm::v1::{self as tycho};
+use substreams_ethereum::{
-use substreams::hex;
+    pb::eth::v2::{Call, Log},
-
+    Event, Function,
-use substreams::scalar::BigInt;
+};
+use tycho_substreams::prelude::*;
 
 /// This trait defines some helpers for serializing and deserializing `Vec<BigInt` which is needed
 ///  to be able to encode the `normalized_weights` and `weights` `Attribute`s. This should also be
 ///  handled by any downstream application.
 trait SerializableVecBigInt {
     fn serialize_bytes(&self) -> Vec<u8>;
+    #[allow(dead_code)]
     fn deserialize_bytes(bytes: &[u8]) -> Vec<BigInt>;
 }
 
@@ -25,7 +24,7 @@ impl SerializableVecBigInt for Vec<BigInt> {
     fn deserialize_bytes(bytes: &[u8]) -> Vec<BigInt> {
         bytes
             .chunks_exact(32)
-            .map(|chunk| BigInt::from_signed_bytes_be(chunk))
+            .map(BigInt::from_signed_bytes_be)
             .collect::<Vec<BigInt>>()
     }
 }
@@ -37,15 +36,16 @@ impl SerializableVecBigInt for Vec<BigInt> {
 ///  - Stable Pool Factories
 /// (Balancer does have a bit more (esp. in the deprecated section) that could be implemented as
 ///  desired.)
-/// We use the specific ABIs to decode both the log event and cooresponding call to gather
+/// We use the specific ABIs to decode both the log event and corresponding call to gather
-///  `PoolCreated` event information alongside the `Create` calldata that provide us details to
+///  `PoolCreated` event information alongside the `Create` call data that provide us details to
-///  fufill both the required details + any extra `Attributes`
+///  fulfill both the required details + any extra `Attributes`
 /// Ref: https://docs.balancer.fi/reference/contracts/deployment-addresses/mainnet.html
 pub fn address_map(
     pool_factory_address: &[u8],
     log: &Log,
     call: &Call,
-) -> Option<tycho::ProtocolComponent> {
+    tx: &Transaction,
+) -> Option<ProtocolComponent> {
     match *pool_factory_address {
         hex!("897888115Ada5773E02aA29F775430BFB5F34c51") => {
             let create_call =
@@ -53,24 +53,20 @@ pub fn address_map(
             let pool_created =
                 abi::weighted_pool_factory::events::PoolCreated::match_and_decode(log)?;
 
-            Some(tycho::ProtocolComponent {
+            Some(
-                id: hex::encode(&pool_created.pool),
+                ProtocolComponent::at_contract(&pool_created.pool, tx)
-                tokens: create_call.tokens,
+                    .with_tokens(&create_call.tokens)
-                contracts: vec![pool_factory_address.into(), pool_created.pool],
+                    .with_attributes(&[
-                static_att: vec![
+                        ("pool_type", "WeightedPoolFactory".as_bytes()),
-                    tycho::Attribute {
+                        (
-                        name: "pool_type".into(),
+                            "normalized_weights",
-                        value: "WeightedPoolFactory".into(),
+                            &create_call
-                        change: tycho::ChangeType::Creation.into(),
+                                .normalized_weights
-                    },
+                                .serialize_bytes(),
-                    tycho::Attribute {
+                        ),
-                        name: "normalized_weights".into(),
+                    ])
-                        value: create_call.normalized_weights.serialize_bytes(),
+                    .as_swap_type("balancer_pool", ImplementationType::Vm),
-                        change: tycho::ChangeType::Creation.into(),
+            )
-                    },
-                ],
-                change: tycho::ChangeType::Creation.into(),
-            })
         }
         hex!("DB8d758BCb971e482B2C45f7F8a7740283A1bd3A") => {
             let create_call =
@@ -78,19 +74,12 @@ pub fn address_map(
             let pool_created =
                 abi::composable_stable_pool_factory::events::PoolCreated::match_and_decode(log)?;
 
-            Some(tycho::ProtocolComponent {
+            Some(
-                id: hex::encode(&pool_created.pool),
+                ProtocolComponent::at_contract(&pool_created.pool, tx)
-                tokens: create_call.tokens,
+                    .with_tokens(&create_call.tokens)
-                contracts: vec![pool_factory_address.into(), pool_created.pool],
+                    .with_attributes(&[("pool_type", "ComposableStablePoolFactory".as_bytes())])
-                static_att: vec![
+                    .as_swap_type("balancer_pool", ImplementationType::Vm),
-                    tycho::Attribute {
+            )
-                        name: "pool_type".into(),
-                        value: "ComposableStablePoolFactory".into(),
-                        change: tycho::ChangeType::Creation.into(),
-                    },
-                ],
-                change: tycho::ChangeType::Creation.into(),
-            })
         }
         hex!("813EE7a840CE909E7Fea2117A44a90b8063bd4fd") => {
             let create_call =
@@ -98,26 +87,20 @@ pub fn address_map(
             let pool_created =
                 abi::erc_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
 
-            Some(tycho::ProtocolComponent {
+            Some(
-                id: hex::encode(&pool_created.pool),
+                ProtocolComponent::at_contract(&pool_created.pool, tx)
-                tokens: vec![create_call.main_token, create_call.wrapped_token],
+                    .with_tokens(&[create_call.main_token, create_call.wrapped_token])
-                contracts: vec![pool_factory_address.into(), pool_created.pool],
+                    .with_attributes(&[
-                static_att: vec![
+                        ("pool_type", "ERC4626LinearPoolFactory".as_bytes()),
-                    tycho::Attribute {
+                        (
-                        name: "pool_type".into(),
+                            "upper_target",
-                        value: "ERC4626LinearPoolFactory".into(),
+                            &create_call
-                        change: tycho::ChangeType::Creation.into(),
+                                .upper_target
-                    },
+                                .to_signed_bytes_be(),
-                    tycho::Attribute {
+                        ),
-                        name: "upper_target".into(),
+                    ])
-                        value: create_call.upper_target.to_signed_bytes_be(),
+                    .as_swap_type("balancer_pool", ImplementationType::Vm),
-                        change: tycho::ChangeType::Creation.into(),
+            )
-                    },
-                    // Note, `lower_target` is generally hardcoded for all pools, not located in call data
-                    // Note, rate provider might be provided as `create.protocol_id`, but as a BigInt. needs investigation
-                ],
-                change: tycho::ChangeType::Creation.into(),
-            })
         }
         hex!("5F43FBa61f63Fa6bFF101a0A0458cEA917f6B347") => {
             let create_call =
@@ -125,24 +108,20 @@ pub fn address_map(
             let pool_created =
                 abi::euler_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
 
-            Some(tycho::ProtocolComponent {
+            Some(
-                id: hex::encode(&pool_created.pool),
+                ProtocolComponent::at_contract(&pool_created.pool, tx)
-                tokens: vec![create_call.main_token, create_call.wrapped_token],
+                    .with_tokens(&[create_call.main_token, create_call.wrapped_token])
-                contracts: vec![pool_factory_address.into(), pool_created.pool],
+                    .with_attributes(&[
-                static_att: vec![
+                        ("pool_type", "EulerLinearPoolFactory".as_bytes()),
-                    tycho::Attribute {
+                        (
-                        name: "pool_type".into(),
+                            "upper_target",
-                        value: "EulerLinearPoolFactory".into(),
+                            &create_call
-                        change: tycho::ChangeType::Creation.into(),
+                                .upper_target
-                    },
+                                .to_signed_bytes_be(),
-                    tycho::Attribute {
+                        ),
-                        name: "upper_target".into(),
+                    ])
-                        value: create_call.upper_target.to_signed_bytes_be(),
+                    .as_swap_type("balancer_pool", ImplementationType::Vm),
-                        change: tycho::ChangeType::Creation.into(),
+            )
-                    },
-                ],
-                change: tycho::ChangeType::Creation.into(),
-            })
         }
         // ❌ Reading the deployed factory for Gearbox showcases that it's currently disabled
         // hex!("39A79EB449Fc05C92c39aA6f0e9BfaC03BE8dE5B") => {
@@ -170,10 +149,11 @@ pub fn address_map(
         //         change: tycho::ChangeType::Creation.into(),
         //     })
         // }
-        // ❌ The `ManagedPoolFactory` is a bit ✨ unique ✨, so we'll leave it commented out for now
+        // ❌ The `ManagedPoolFactory` is a bit ✨ unique ✨, so we'll leave it commented out for
-        // Take a look at it's `Create` call to see how the params are structured.
+        // now Take a look at it's `Create` call to see how the params are structured.
         // hex!("BF904F9F340745B4f0c4702c7B6Ab1e808eA6b93") => {
-        //     let create_call = abi::managed_pool_factory::functions::Create::match_and_decode(call)?;
+        //     let create_call =
+        // abi::managed_pool_factory::functions::Create::match_and_decode(call)?;
         //     let pool_created =
         //         abi::managed_pool_factory::events::PoolCreated::match_and_decode(log)?;
 
@@ -197,24 +177,20 @@ pub fn address_map(
             let pool_created =
                 abi::silo_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
 
-            Some(tycho::ProtocolComponent {
+            Some(
-                id: hex::encode(&pool_created.pool),
+                ProtocolComponent::at_contract(&pool_created.pool, tx)
-                tokens: vec![create_call.main_token, create_call.wrapped_token],
+                    .with_tokens(&[create_call.main_token, create_call.wrapped_token])
-                contracts: vec![pool_factory_address.into(), pool_created.pool],
+                    .with_attributes(&[
-                static_att: vec![
+                        ("pool_type", "SiloLinearPoolFactory".as_bytes()),
-                    tycho::Attribute {
+                        (
-                        name: "pool_type".into(),
+                            "upper_target",
-                        value: "SiloLinearPoolFactory".into(),
+                            &create_call
-                        change: tycho::ChangeType::Creation.into(),
+                                .upper_target
-                    },
+                                .to_signed_bytes_be(),
-                    tycho::Attribute {
+                        ),
-                        name: "upper_target".into(),
+                    ])
-                        value: create_call.upper_target.to_signed_bytes_be(),
+                    .as_swap_type("balancer_pool", ImplementationType::Vm),
-                        change: tycho::ChangeType::Creation.into(),
+            )
-                    },
-                ],
-                change: tycho::ChangeType::Creation.into(),
-            })
         }
         hex!("5F5222Ffa40F2AEd6380D022184D6ea67C776eE0") => {
             let create_call =
@@ -222,51 +198,38 @@ pub fn address_map(
             let pool_created =
                 abi::yearn_linear_pool_factory::events::PoolCreated::match_and_decode(log)?;
 
-            Some(tycho::ProtocolComponent {
+            Some(
-                id: hex::encode(&pool_created.pool),
+                ProtocolComponent::at_contract(&pool_created.pool, tx)
-                tokens: vec![create_call.main_token, create_call.wrapped_token],
+                    .with_tokens(&[create_call.main_token, create_call.wrapped_token])
-                contracts: vec![pool_factory_address.into(), pool_created.pool],
+                    .with_attributes(&[
-                static_att: vec![
+                        ("pool_type", "YearnLinearPoolFactory".as_bytes()),
-                    tycho::Attribute {
+                        (
-                        name: "pool_type".into(),
+                            "upper_target",
-                        value: "YearnLinearPoolFactory".into(),
+                            &create_call
-                        change: tycho::ChangeType::Creation.into(),
+                                .upper_target
-                    },
+                                .to_signed_bytes_be(),
-                    tycho::Attribute {
+                        ),
-                        name: "upper_target".into(),
+                    ])
-                        value: create_call.upper_target.to_signed_bytes_be(),
+                    .as_swap_type("balancer_pool", ImplementationType::Vm),
-                        change: tycho::ChangeType::Creation.into(),
+            )
-                    },
-                ],
-                change: tycho::ChangeType::Creation.into(),
-            })
         }
-        // The `WeightedPool2TokenFactory` is a deprecated contract but we've included it since one
+        // The `WeightedPool2TokenFactory` is a deprecated contract, but we've included
-        //  of the highest TVL pools, 80BAL-20WETH, is able to be tracked.
+        // it to be able to track one of the highest TVL pools: 80BAL-20WETH.
         hex!("A5bf2ddF098bb0Ef6d120C98217dD6B141c74EE0") => {
             let create_call =
                 abi::weighted_pool_tokens_factory::functions::Create::match_and_decode(call)?;
             let pool_created =
                 abi::weighted_pool_tokens_factory::events::PoolCreated::match_and_decode(log)?;
 
-            Some(tycho::ProtocolComponent {
+            Some(
-                id: hex::encode(&pool_created.pool),
+                ProtocolComponent::at_contract(&pool_created.pool, tx)
-                tokens: create_call.tokens,
+                    .with_tokens(&create_call.tokens)
-                contracts: vec![pool_factory_address.into(), pool_created.pool],
+                    .with_attributes(&[
-                static_att: vec![
+                        ("pool_type", "WeightedPool2TokensFactory".as_bytes()),
-                    tycho::Attribute {
+                        ("weights", &create_call.weights.serialize_bytes()),
-                        name: "pool_type".into(),
+                    ])
-                        value: "WeightedPool2TokensFactory".into(),
+                    .as_swap_type("balancer_pool", ImplementationType::Vm),
-                        change: tycho::ChangeType::Creation.into(),
+            )
-                    },
-                    tycho::Attribute {
-                        name: "weights".into(),
-                        value: create_call.weights.serialize_bytes(),
-                        change: tycho::ChangeType::Creation.into(),
-                    },
-                ],
-                change: tycho::ChangeType::Creation.into(),
-            })
         }
         _ => None,
     }

substreams/ethereum-balancer/substreams.yaml

@@ -1,23 +1,23 @@
 specVersion: v0.1.0
 package:
-  name: "substreams_balancer"
+  name: "ethereum_balancer"
   version: v0.1.0
 
 protobuf:
   files:
     - tycho/evm/v1/vm.proto
     - tycho/evm/v1/common.proto
+    - tycho/evm/v1/utils.proto
   importPaths:
-    - ../../proto/tycho/evm/v1/
+    - ../../proto
-    - ./proto
 
 binaries:
   default:
     type: wasm/rust-v1
-    file: target/wasm32-unknown-unknown/release/substreams_balancer.wasm
+    file: ../target/wasm32-unknown-unknown/release/ethereum_balancer.wasm
 
 modules:
-  - name: map_pools_created
+  - name: map_components
     kind: map
     initialBlock: 12369300
     inputs:
@@ -25,40 +25,40 @@ modules:
     output:
       type: proto:tycho.evm.v1.GroupedTransactionProtocolComponents
 
-  - name: store_pools_created
+  - name: store_components
     kind: store
     initialBlock: 12369300
     updatePolicy: add
     valueType: int64
     inputs:
-      - map: map_pools_created
+      - map: map_components
 
-  - name: map_balance_deltas
+  - name: map_relative_balances
     kind: map
-    initialBlock: 12369300  # An arbitrary block that should change based on your requirements
+    initialBlock: 12369300 # An arbitrary block that should change based on your requirements
     inputs:
       - source: sf.ethereum.type.v2.Block
-      - store: store_pools_created
+      - store: store_components
     output:
       type: proto:tycho.evm.v1.BalanceDeltas
 
-  - name: store_balance_changes
+  - name: store_balances
     kind: store
     initialBlock: 12369300
     updatePolicy: add
     valueType: bigint
     inputs:
-      - map: map_balance_deltas
+      - map: map_relative_balances
 
-  - name: map_changes
+  - name: map_protocol_changes
     kind: map
     initialBlock: 12369300
     inputs:
       - source: sf.ethereum.type.v2.Block
-      - map: map_pools_created
+      - map: map_components
-      - map: map_balance_deltas
+      - map: map_relative_balances
-      - store: store_pools_created
+      - store: store_components
-      - store: store_balance_changes
+      - store: store_balances
-        mode: deltas  # This is the key property that simplifies `BalanceChange` handling
+        mode: deltas # This is the key property that simplifies `BalanceChange` handling
     output:
       type: proto:tycho.evm.v1.BlockContractChanges

substreams/ethereum-template/Cargo.toml

@@ -1,13 +1,14 @@
 [package]
-name = "substreams-ethereum-template"
+name = "ethereum-template"
 version = "0.1.0"
 edition = "2021"
 
 [lib]
-name = "substreams_ethereum_template"
+name = "ethereum_template"
 crate-type = ["cdylib"]
 
 [dependencies]
-substreams = "0.5"
+substreams.workspace = true
-substreams-ethereum = "0.9"
+substreams-ethereum.workspace = true
-prost = "0.11"
+prost.workspace = true
+tycho-substreams.workspace = true

substreams/ethereum-template/src/lib.rs

@@ -1,13 +1,9 @@
-
 use substreams_ethereum::pb::eth;
-
+mod modules;
-use pb::tycho::evm::v1::{self as tycho};
-
-mod pb;
 
 #[substreams::handlers::map]
 fn map_changes(
     block: eth::v2::Block,
 ) -> Result<tycho::BlockContractChanges, substreams::errors::Error> {
     todo!("Not implemented")
-}
+}

substreams/ethereum-template/src/modules.rs

@@ -0,0 +1,12 @@
+use std::collections::HashMap;
+use substreams_ethereum::pb::eth;
+use tycho_substreams::prelude::*;
+
+#[substreams::handlers::map]
+fn map_protocol_changes(
+    block: eth::v2::Block,
+) -> Result<BlockContractChanges, substreams::errors::Error> {
+    let mut transaction_contract_changes = Vec::<TransactionContractChanges>::new();
+    // TODO: protocol specific logic goes here
+    Ok(BlockContractChanges { block: Some((&block).into()), changes: transaction_contract_changes })
+}

substreams/ethereum-template/src/pb/mod.rs

@@ -1,10 +0,0 @@
-// @generated
-pub mod tycho {
-    pub mod evm {
-        // @@protoc_insertion_point(attribute:tycho.evm.v1)
-        pub mod v1 {
-            include!("tycho.evm.v1.rs");
-            // @@protoc_insertion_point(tycho.evm.v1)
-        }
-    }
-}

substreams/ethereum-template/src/pb/tycho.evm.v1.rs

@@ -1,183 +0,0 @@
-// @generated
-// This file contains the proto definitions for Substreams common to all integrations.
-
-/// A struct describing a block.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Block {
-    /// The blocks hash.
-    #[prost(bytes="vec", tag="1")]
-    pub hash: ::prost::alloc::vec::Vec<u8>,
-    /// The parent blocks hash.
-    #[prost(bytes="vec", tag="2")]
-    pub parent_hash: ::prost::alloc::vec::Vec<u8>,
-    /// The block number.
-    #[prost(uint64, tag="3")]
-    pub number: u64,
-    /// The block timestamp.
-    #[prost(uint64, tag="4")]
-    pub ts: u64,
-}
-/// A struct describing a transaction.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Transaction {
-    /// The transaction hash.
-    #[prost(bytes="vec", tag="1")]
-    pub hash: ::prost::alloc::vec::Vec<u8>,
-    /// The sender of the transaction.
-    #[prost(bytes="vec", tag="2")]
-    pub from: ::prost::alloc::vec::Vec<u8>,
-    /// The receiver of the transaction.
-    #[prost(bytes="vec", tag="3")]
-    pub to: ::prost::alloc::vec::Vec<u8>,
-    /// The transactions index within the block.
-    #[prost(uint64, tag="4")]
-    pub index: u64,
-}
-/// A custom struct representing an arbitrary attribute of a protocol component.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Attribute {
-    /// The name of the attribute.
-    #[prost(string, tag="1")]
-    pub name: ::prost::alloc::string::String,
-    /// The value of the attribute.
-    #[prost(bytes="vec", tag="2")]
-    pub value: ::prost::alloc::vec::Vec<u8>,
-    /// The type of change the attribute underwent.
-    #[prost(enumeration="ChangeType", tag="3")]
-    pub change: i32,
-}
-/// A struct describing a part of the protocol.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ProtocolComponent {
-    /// A unique identifier for the component within the protocol.
-    /// Can be a stringified address or a string describing the trading pair.
-    #[prost(string, tag="1")]
-    pub id: ::prost::alloc::string::String,
-    /// Addresses of the ERC20 tokens used by the component.
-    #[prost(bytes="vec", repeated, tag="2")]
-    pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
-    /// Addresses of the contracts used by the component.
-    #[prost(bytes="vec", repeated, tag="3")]
-    pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
-    /// Attributes of the component.
-    /// The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent.
-    #[prost(message, repeated, tag="4")]
-    pub static_att: ::prost::alloc::vec::Vec<Attribute>,
-    /// Type of change the component underwent.
-    #[prost(enumeration="ChangeType", tag="5")]
-    pub change: i32,
-}
-/// A struct for following the changes of Total Value Locked (TVL) of a protocol component.
-/// Note that if the ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct BalanceChange {
-    /// The address of the ERC20 token whose balance changed.
-    #[prost(bytes="vec", tag="1")]
-    pub token: ::prost::alloc::vec::Vec<u8>,
-    /// The new balance of the token.
-    #[prost(bytes="vec", tag="2")]
-    pub balance: ::prost::alloc::vec::Vec<u8>,
-    /// The id of the component whose TVL is tracked.
-    #[prost(bytes="vec", tag="3")]
-    pub component_id: ::prost::alloc::vec::Vec<u8>,
-}
-/// Enum to specify the type of a change.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
-#[repr(i32)]
-pub enum ChangeType {
-    Unspecified = 0,
-    Update = 1,
-    Creation = 2,
-    Deletion = 3,
-}
-impl ChangeType {
-    /// String value of the enum field names used in the ProtoBuf definition.
-    ///
-    /// The values are not transformed in any way and thus are considered stable
-    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
-    pub fn as_str_name(&self) -> &'static str {
-        match self {
-            ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED",
-            ChangeType::Update => "CHANGE_TYPE_UPDATE",
-            ChangeType::Creation => "CHANGE_TYPE_CREATION",
-            ChangeType::Deletion => "CHANGE_TYPE_DELETION",
-        }
-    }
-    /// Creates an enum from field names used in the ProtoBuf definition.
-    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
-        match value {
-            "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified),
-            "CHANGE_TYPE_UPDATE" => Some(Self::Update),
-            "CHANGE_TYPE_CREATION" => Some(Self::Creation),
-            "CHANGE_TYPE_DELETION" => Some(Self::Deletion),
-            _ => None,
-        }
-    }
-}
-// This file contains proto definitions specific to the VM integration.
-
-/// A key value entry into contract storage.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ContractSlot {
-    /// A contract's storage slot.
-    #[prost(bytes="vec", tag="2")]
-    pub slot: ::prost::alloc::vec::Vec<u8>,
-    /// The new value for this storage slot.
-    #[prost(bytes="vec", tag="3")]
-    pub value: ::prost::alloc::vec::Vec<u8>,
-}
-/// Changes made to a single contract's state.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ContractChange {
-    /// The contract's address
-    #[prost(bytes="vec", tag="1")]
-    pub address: ::prost::alloc::vec::Vec<u8>,
-    /// The new balance of the contract, empty bytes indicates no change.
-    #[prost(bytes="vec", tag="2")]
-    pub balance: ::prost::alloc::vec::Vec<u8>,
-    /// The new code of the contract, empty bytes indicates no change.
-    #[prost(bytes="vec", tag="3")]
-    pub code: ::prost::alloc::vec::Vec<u8>,
-    /// The changes to this contract's slots, empty sequence indicates no change.
-    #[prost(message, repeated, tag="4")]
-    pub slots: ::prost::alloc::vec::Vec<ContractSlot>,
-    /// Whether this is an update, a creation or a deletion.
-    #[prost(enumeration="ChangeType", tag="5")]
-    pub change: i32,
-}
-/// A set of changes aggregated by transaction.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct TransactionContractChanges {
-    /// The transaction instance that results in the changes.
-    #[prost(message, optional, tag="1")]
-    pub tx: ::core::option::Option<Transaction>,
-    /// Contains the changes induced by the above transaction, aggregated on a per-contract basis.
-    #[prost(message, repeated, tag="2")]
-    pub contract_changes: ::prost::alloc::vec::Vec<ContractChange>,
-    /// An array of newly added components.
-    #[prost(message, repeated, tag="3")]
-    pub component_changes: ::prost::alloc::vec::Vec<ProtocolComponent>,
-    /// An array of balance changes to components.
-    #[prost(message, repeated, tag="4")]
-    pub balance_changes: ::prost::alloc::vec::Vec<BalanceChange>,
-}
-/// A set of transaction changes within a single block.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct BlockContractChanges {
-    /// The block for which these changes are collectively computed.
-    #[prost(message, optional, tag="1")]
-    pub block: ::core::option::Option<Block>,
-    /// The set of transaction changes observed in the specified block.
-    #[prost(message, repeated, tag="2")]
-    pub changes: ::prost::alloc::vec::Vec<TransactionContractChanges>,
-}
-// @@protoc_insertion_point(module)

substreams/ethereum-template/substreams.yaml

@@ -1,22 +1,23 @@
 specVersion: v0.1.0
 package:
-  name: "substreams_ethereum_template"
+  name: "ethereum_template"
   version: v0.1.0
 
 protobuf:
   files:
-    - vm.proto
+    - tycho/evm/v1/vm.proto
-    - common.proto
+    - tycho/evm/v1/common.proto
+    - tycho/evm/v1/utils.proto
   importPaths:
-    - ../../proto/tycho/evm/v1/
+    - ../../proto
 
 binaries:
   default:
     type: wasm/rust-v1
-    file: ../../target/wasm32-unknown-unknown/substreams/substreams_ethereum_template.wasm
+    file: ../target/wasm32-unknown-unknown/release/ethereum_template.wasm
 
 modules:
-  - name: map_changes
+  - name: map_protocol_changes
     kind: map
     inputs:
       - source: sf.ethereum.type.v2.Block

substreams/release.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Allows releasing multiple substream packages within the same repo.
+# To trigger a release simply create a tag with [package-name]-[semver].
+# The script will look for these tags, then infer which package needs to be built and
+# released.
+
+# Try to get the tag name associated with the current HEAD commit
+current_tag=$(git describe --tags --exact-match HEAD 2>/dev/null)
+
+if [ -n "$current_tag" ]; then
+    # If the HEAD is at a tag, extract the prefix and version
+    if [[ $current_tag =~ ^([a-zA-Z-]*-)?([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
+        # Prefix without the trailing hyphen (if any)
+        package="${BASH_REMATCH[1]%?}"
+        # Semantic version
+        version="${BASH_REMATCH[2]}"
+
+        cargo_version=$(cargo pkgid -p ethereum-balancer | cut -d# -f2 | cut -d: -f2)
+        if [[ "$cargo_version" != "$version" ]]; then
+          echo "Error: Cargo version: ${cargo_version} does not match tag version: ${version}!"
+          exit 1
+        fi
+        # Check if the Git repository is dirty
+        if [ -n "$(git status --porcelain)" ]; then
+          echo "Error: The repository is dirty. Please commit or stash your changes."
+          exit 1
+        fi
+    else
+        echo "Error: Current tag ($current_tag) does not match the expected format."
+        exit 1
+    fi
+else
+    # If the HEAD is not at a tag, construct the tag name with the pre-release postfix
+    if [ -z "$1" ]; then
+        echo "Error: package argument is required to create a pre release!"
+        exit 1
+    fi
+    package=$1
+
+    version_prefix=$(git describe --tags --match "$package-*" --abbrev=0 2>/dev/null)
+    if [ -z "$version_prefix" ]; then
+        # If no tags are found in the history, default to version 0.0.1
+        version_prefix="0.0.1"
+    fi
+
+    # Get the short commit hash of the current HEAD
+    commit_hash=$(git rev-parse --short HEAD)
+    version="${version_prefix}-pre.${commit_hash}"
+fi
+
+REPOSITORY=${REPOSITORY:-"s3://repo.propellerheads/substreams"}
+repository_path="$REPOSITORY/$package/$package-$version.spkg"
+
+cargo build --target wasm32-unknown-unknown --release -p "$package"
+mkdir -p ./target/spkg/
+substreams pack $package/substreams.yaml -o ./target/spkg/$package-$version.spkg
+aws s3 cp ./target/spkg/$package-$version.spkg $repository_path
+
+echo "Released substreams package: '$repository_path'"

substreams/rustfmt.toml

@@ -0,0 +1,14 @@
+reorder_imports = true
+imports_granularity = "Crate"
+use_small_heuristics = "Max"
+comment_width = 100
+wrap_comments = true
+binop_separator = "Back"
+trailing_comma = "Vertical"
+trailing_semicolon = false
+use_field_init_shorthand = true
+chain_width = 40
+ignore = [
+    "crates/tycho-substreams/src/pb",
+    "ethereum-balancer/src/abi",
+]