From b8d7f6686c7e63d85378bbdec3abdde53f39311d Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:39:07 +0100 Subject: [PATCH 01/12] Fix balancer substream naming --- substreams/Readme.md | 2 +- substreams/ethereum-balancer/Cargo.toml | 4 ++-- substreams/ethereum-balancer/substreams.yaml | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/substreams/Readme.md b/substreams/Readme.md index f4f9aa3..530d858 100644 --- a/substreams/Readme.md +++ b/substreams/Readme.md @@ -47,7 +47,7 @@ The example from the official docs will serve us just well: ```toml [package] -name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]" +name = "substreams-[CHAIN]-[PROTOCOL_SYSTEM]" version = "0.1.0" edition = "2021" diff --git a/substreams/ethereum-balancer/Cargo.toml b/substreams/ethereum-balancer/Cargo.toml index 795c565..ace53d3 100644 --- a/substreams/ethereum-balancer/Cargo.toml +++ b/substreams/ethereum-balancer/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "substreams-balancer" +name = "substreams-ethereum-balancer" version = "0.1.0" edition = "2021" [lib] -name = "substreams_balancer" +name = "substreams_ethereum_balancer" crate-type = ["cdylib"] [dependencies] diff --git a/substreams/ethereum-balancer/substreams.yaml b/substreams/ethereum-balancer/substreams.yaml index 6324d2b..f1bc3af 100644 --- a/substreams/ethereum-balancer/substreams.yaml +++ b/substreams/ethereum-balancer/substreams.yaml @@ -1,6 +1,6 @@ specVersion: v0.1.0 package: - name: "substreams_balancer" + name: "substreams_ethereum_balancer" version: v0.1.0 protobuf: @@ -14,7 +14,7 @@ protobuf: binaries: default: type: wasm/rust-v1 - file: target/wasm32-unknown-unknown/release/substreams_balancer.wasm + file: target/wasm32-unknown-unknown/release/substreams_ethereum_balancer.wasm modules: - name: map_pools_created @@ -35,7 +35,7 @@ modules: - name: map_balance_deltas kind: map - initialBlock: 12369300 # An arbitrary block that should change based on your requirements + initialBlock: 12369300 # An arbitrary block that should change based on your requirements inputs: - source: sf.ethereum.type.v2.Block - store: store_pools_created @@ -59,6 +59,6 @@ modules: - map: map_balance_deltas - store: store_pools_created - store: store_balance_changes - mode: deltas # This is the key property that simplifies `BalanceChange` handling + mode: deltas # This is the key property that simplifies `BalanceChange` handling output: type: proto:tycho.evm.v1.BlockContractChanges From 66a048fbe2333c3cdcf17ae4c07914bc565fa07b Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:45:58 +0100 Subject: [PATCH 02/12] refactor: update protobuf messages --- proto/tycho/evm/v1/balancer.proto | 32 ++++ proto/tycho/evm/v1/common.proto | 27 ++- substreams/ethereum-balancer/Cargo.lock | 36 ++-- .../ethereum-balancer/proto/factory.proto | 19 --- substreams/ethereum-balancer/proto/pool.proto | 15 -- .../proto/tycho/evm/v1/common.proto | 113 ------------- .../proto/tycho/evm/v1/entity.proto | 32 ---- .../proto/tycho/evm/v1/vm.proto | 50 ------ substreams/ethereum-balancer/src/modules.rs | 11 +- .../src/pb/eth.balancer.v1.rs | 28 ---- .../src/pb/eth.factory.v1.rs | 28 ---- .../ethereum-balancer/src/pb/eth.pool.v1.rs | 22 --- .../ethereum-balancer/src/pb/tycho.evm.v1.rs | 158 +++++++++++++----- .../ethereum-balancer/src/pool_factories.rs | 66 +++++++- substreams/ethereum-balancer/substreams.yaml | 4 +- 15 files changed, 261 insertions(+), 380 deletions(-) create mode 100644 proto/tycho/evm/v1/balancer.proto delete mode 100644 substreams/ethereum-balancer/proto/factory.proto delete mode 100644 substreams/ethereum-balancer/proto/pool.proto delete mode 100644 substreams/ethereum-balancer/proto/tycho/evm/v1/common.proto delete mode 100644 substreams/ethereum-balancer/proto/tycho/evm/v1/entity.proto delete mode 100644 substreams/ethereum-balancer/proto/tycho/evm/v1/vm.proto delete mode 100644 substreams/ethereum-balancer/src/pb/eth.balancer.v1.rs delete mode 100644 substreams/ethereum-balancer/src/pb/eth.factory.v1.rs delete mode 100644 substreams/ethereum-balancer/src/pb/eth.pool.v1.rs diff --git a/proto/tycho/evm/v1/balancer.proto b/proto/tycho/evm/v1/balancer.proto new file mode 100644 index 0000000..f5a4c63 --- /dev/null +++ b/proto/tycho/evm/v1/balancer.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package tycho.evm.v1; + +import "tycho/evm/v1/common.proto"; + +// A struct for following the changes of Total Value Locked (TVL). +message BalanceDelta { + uint64 ord = 1; + // The tx hash of the transaction that caused the balance change. + Transaction tx = 2; + // The address of the ERC20 token whose balance changed. + bytes token = 3; + // The delta balance of the token. + bytes delta = 4; + // The id of the component whose TVL is tracked. + // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + bytes component_id = 5; +} + +message BalanceDeltas { + repeated BalanceDelta balance_deltas = 1; +} + +message TransactionProtocolComponents { + Transaction tx = 1; + repeated ProtocolComponent components = 2; +} + +message GroupedTransactionProtocolComponents { + repeated TransactionProtocolComponents tx_components = 1; +} \ No newline at end of file diff --git a/proto/tycho/evm/v1/common.proto b/proto/tycho/evm/v1/common.proto index d52dbed..545cad0 100644 --- a/proto/tycho/evm/v1/common.proto +++ b/proto/tycho/evm/v1/common.proto @@ -25,6 +25,7 @@ message Transaction { // The receiver of the transaction. bytes to = 3; // The transactions index within the block. + // TODO: should this be uint32? to match the type from the native substream type? uint64 index = 4; } @@ -47,6 +48,26 @@ message Attribute { ChangeType change = 3; } +enum FinancialType{ + SWAP = 0; + LEND = 1; + LEVERAGE = 2; + PSM = 3; +} + + +enum ImplementationType { + VM = 0; + CUSTOM = 1; +} + +message ProtocolType{ + string name = 1; + FinancialType financial_type = 2; + repeated Attribute attribute_schema = 3; + ImplementationType implementation_type = 4; +} + // A struct describing a part of the protocol. // Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, // the component would represent a single contract. In case of VM integration, such component would @@ -67,6 +88,10 @@ message ProtocolComponent { repeated Attribute static_att = 4; // Type of change the component underwent. ChangeType change = 5; + /// Represents the functionality of the component. + ProtocolType protocol_type = 6; + // Transaction where this component was created + Transaction tx = 7; } // A struct for following the changes of Total Value Locked (TVL) of a protocol component. @@ -77,7 +102,7 @@ message BalanceChange { bytes token = 1; // The new balance of the token. bytes balance = 2; - // The id of the component whose TVL is tracked. + // The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. bytes component_id = 3; } diff --git a/substreams/ethereum-balancer/Cargo.lock b/substreams/ethereum-balancer/Cargo.lock index c96455e..b05dce5 100644 --- a/substreams/ethereum-balancer/Cargo.lock +++ b/substreams/ethereum-balancer/Cargo.lock @@ -884,24 +884,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "substreams-balancer" -version = "0.1.0" -dependencies = [ - "anyhow", - "bytes", - "ethabi 18.0.0", - "getrandom", - "hex", - "hex-literal 0.4.1", - "itertools 0.12.0", - "num-bigint", - "prost 0.11.9", - "prost-types 0.12.3", - "substreams", - "substreams-ethereum", -] - [[package]] name = "substreams-ethereum" version = "0.9.9" @@ -933,6 +915,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "substreams-ethereum-balancer" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "ethabi 18.0.0", + "getrandom", + "hex", + "hex-literal 0.4.1", + "itertools 0.12.0", + "num-bigint", + "prost 0.11.9", + "prost-types 0.12.3", + "substreams", + "substreams-ethereum", +] + [[package]] name = "substreams-ethereum-core" version = "0.9.9" diff --git a/substreams/ethereum-balancer/proto/factory.proto b/substreams/ethereum-balancer/proto/factory.proto deleted file mode 100644 index f191ea8..0000000 --- a/substreams/ethereum-balancer/proto/factory.proto +++ /dev/null @@ -1,19 +0,0 @@ -syntax = "proto3"; - -package eth.factory.v1; - -message Pools { - repeated Pool pools = 1; -} - -message Pool { - bytes pool_id = 1; - fixed64 log_ordinal = 2; -} - -message Transfer { - bytes from = 1; - bytes to = 2; - string token = 3; - string amount = 4; -} diff --git a/substreams/ethereum-balancer/proto/pool.proto b/substreams/ethereum-balancer/proto/pool.proto deleted file mode 100644 index c4dd94c..0000000 --- a/substreams/ethereum-balancer/proto/pool.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -package eth.pool.v1; - -message Transfers { - repeated Transfer transfers = 1; -} - -message Transfer { - string from = 1; - string to = 2; - uint64 token_id = 3; - string trx_hash = 4; - uint64 ordinal = 5; -} diff --git a/substreams/ethereum-balancer/proto/tycho/evm/v1/common.proto b/substreams/ethereum-balancer/proto/tycho/evm/v1/common.proto deleted file mode 100644 index 1a5caf4..0000000 --- a/substreams/ethereum-balancer/proto/tycho/evm/v1/common.proto +++ /dev/null @@ -1,113 +0,0 @@ -syntax = "proto3"; - -package tycho.evm.v1; - -// This file contains the proto definitions for Substreams common to all integrations. - -// A struct describing a block. -message Block { - // The blocks hash. - bytes hash = 1; - // The parent blocks hash. - bytes parent_hash = 2; - // The block number. - uint64 number = 3; - // The block timestamp. - uint64 ts = 4; -} - -// A struct describing a transaction. -message Transaction { - // The transaction hash. - bytes hash = 1; - // The sender of the transaction. - bytes from = 2; - // The receiver of the transaction. - bytes to = 3; - // The transactions index within the block. - // TODO: should this be uint32? to match the type from the native substream type? - uint64 index = 4; -} - -// Enum to specify the type of a change. -enum ChangeType { - CHANGE_TYPE_UNSPECIFIED = 0; - CHANGE_TYPE_UPDATE = 1; - CHANGE_TYPE_CREATION = 2; - CHANGE_TYPE_DELETION = 3; -} - -// A custom struct representing an arbitrary attribute of a protocol component. -// This is mainly used by the native integration to track the necessary information about the protocol. -message Attribute { - // The name of the attribute. - string name = 1; - // The value of the attribute. - bytes value = 2; - // The type of change the attribute underwent. - ChangeType change = 3; -} - -// A struct describing a part of the protocol. -// Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, -// the component would represent a single contract. In case of VM integration, such component would -// not need any attributes, because all the relevant info would be tracked via storage slots and balance changes. -// It can also be a wrapping contract, like WETH, that has a constant price, but it allows swapping tokens. -// This is why the name ProtocolComponent is used instead of "Pool" or "Pair". -message ProtocolComponent { - // A unique identifier for the component within the protocol. - // Can be e.g. a stringified address or a string describing the trading pair. - string id = 1; - // Addresses of the ERC20 tokens used by the component. - repeated bytes tokens = 2; - // Addresses of the contracts used by the component. - // Usually it is a single contract, but some protocols use multiple contracts. - repeated bytes contracts = 3; - // Attributes of the component. Used mainly be the native integration. - // The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent. - repeated Attribute static_att = 4; - // Type of change the component underwent. - ChangeType change = 5; -} - -message TransactionProtocolComponents { - Transaction tx = 1; - repeated ProtocolComponent components = 2; -} - -message GroupedTransactionProtocolComponents { - repeated TransactionProtocolComponents tx_components = 1; -} - -// A struct for following the changes of Total Value Locked (TVL) of a protocol component. -// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. -// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract. -message BalanceChange { - // The address of the ERC20 token whose balance changed. - bytes token = 1; - // The new balance of the token. - bytes balance = 2; - // The id of the component whose TVL is tracked. - // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. - bytes component_id = 3; -} - -// A struct for following the changes of Total Value Locked (TVL) of a protocol component. -// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. -// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract. -message BalanceDelta { - uint64 ord = 1; - // The tx hash of the transaction that caused the balance change. - Transaction tx = 2; - // The address of the ERC20 token whose balance changed. - bytes token = 3; - // The delta balance of the token. - bytes delta = 4; - // The id of the component whose TVL is tracked. - // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. - bytes component_id = 5; -} - -message BalanceDeltas { - repeated BalanceDelta balance_deltas = 1; -} diff --git a/substreams/ethereum-balancer/proto/tycho/evm/v1/entity.proto b/substreams/ethereum-balancer/proto/tycho/evm/v1/entity.proto deleted file mode 100644 index 14539e4..0000000 --- a/substreams/ethereum-balancer/proto/tycho/evm/v1/entity.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -package tycho.evm.v1; - -import "tycho/evm/v1/common.proto"; - -// This file contains the definition for the native integration of Substreams. - -// A component is a set of attributes that are associated with a custom entity. -message EntityChanges { - // A unique identifier of the entity within the protocol. - string component_id = 1; - // The set of attributes that are associated with the entity. - repeated Attribute attributes = 2; -} - -message TransactionEntityChanges { - Transaction tx = 1; - repeated EntityChanges entity_changes = 2; - // An array of newly added components. - repeated ProtocolComponent component_changes = 3; - // An array of balance changes to components. - repeated BalanceChange balance_changes = 4; -} - -// A set of transaction changes within a single block. -message BlockEntityChanges { - // The block for which these changes are collectively computed. - Block block = 1; - // The set of transaction changes observed in the specified block. - repeated TransactionEntityChanges changes = 2; -} diff --git a/substreams/ethereum-balancer/proto/tycho/evm/v1/vm.proto b/substreams/ethereum-balancer/proto/tycho/evm/v1/vm.proto deleted file mode 100644 index a49dcf0..0000000 --- a/substreams/ethereum-balancer/proto/tycho/evm/v1/vm.proto +++ /dev/null @@ -1,50 +0,0 @@ -syntax = "proto3"; - -package tycho.evm.v1; - -import "tycho/evm/v1/common.proto"; - -// This file contains proto definitions specific to the VM integration. - -// A key value entry into contract storage. -message ContractSlot { - // A contract's storage slot. - bytes slot = 2; - // The new value for this storage slot. - bytes value = 3; -} - -// Changes made to a single contract's state. -message ContractChange { - // The contract's address - bytes address = 1; - // The new native balance of the contract, empty bytes indicates no change. - bytes balance = 2; - // The new code of the contract, empty bytes indicates no change. - bytes code = 3; - // The changes to this contract's slots, empty sequence indicates no change. - repeated ContractSlot slots = 4; - // Whether this is an update, a creation or a deletion. - ChangeType change = 5; -} - -// A set of changes aggregated by transaction. -message TransactionContractChanges { - // The transaction instance that results in the changes. - Transaction tx = 1; - // Contains the changes induced by the above transaction, aggregated on a per-contract basis. - // Must include changes to every contract that is tracked by all ProtocolComponents. - repeated ContractChange contract_changes = 2; - // An array of any component changes. - repeated ProtocolComponent component_changes = 3; - // An array of balance changes to components. - repeated BalanceChange balance_changes = 4; -} - -// A set of transaction changes within a single block. -message BlockContractChanges { - // The block for which these changes are collectively computed. - Block block = 1; - // The set of transaction changes observed in the specified block. - repeated TransactionContractChanges changes = 2; -} diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 2ac0f0e..d121c2d 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -17,6 +17,7 @@ use pb::tycho::evm::v1::{self as tycho}; use contract_changes::extract_contract_changes; +use crate::pb::tycho::evm::v1::{BalanceDeltas, GroupedTransactionProtocolComponents}; use crate::{abi, contract_changes, pb, pool_factories}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); @@ -50,6 +51,12 @@ pub fn map_pools_created( call.call.address.as_slice(), log, call.call, + &tycho::Transaction { + hash: tx.hash.clone(), + from: tx.from.clone(), + to: tx.to.clone(), + index: tx.index.into(), + }, )?) }) .collect::>(); @@ -156,8 +163,8 @@ pub fn store_balance_changes(deltas: tycho::BalanceDeltas, store: StoreAddBigInt #[substreams::handlers::map] pub fn map_changes( block: eth::v2::Block, - grouped_components: tycho::GroupedTransactionProtocolComponents, - deltas: tycho::BalanceDeltas, + grouped_components: GroupedTransactionProtocolComponents, + deltas: BalanceDeltas, components_store: StoreGetInt64, balance_store: StoreDeltas, // Note, this map module is using the `deltas` mode for the store. ) -> Result { diff --git a/substreams/ethereum-balancer/src/pb/eth.balancer.v1.rs b/substreams/ethereum-balancer/src/pb/eth.balancer.v1.rs deleted file mode 100644 index 47065d4..0000000 --- a/substreams/ethereum-balancer/src/pb/eth.balancer.v1.rs +++ /dev/null @@ -1,28 +0,0 @@ -// @generated -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Pools { - #[prost(message, repeated, tag="1")] - pub pools: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Pool { - #[prost(bytes="vec", tag="1")] - pub pool_id: ::prost::alloc::vec::Vec, - #[prost(fixed64, tag="2")] - pub log_ordinal: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transfer { - #[prost(bytes="vec", tag="1")] - pub from: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="2")] - pub to: ::prost::alloc::vec::Vec, - #[prost(string, tag="3")] - pub token: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub amount: ::prost::alloc::string::String, -} -// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pb/eth.factory.v1.rs b/substreams/ethereum-balancer/src/pb/eth.factory.v1.rs deleted file mode 100644 index 47065d4..0000000 --- a/substreams/ethereum-balancer/src/pb/eth.factory.v1.rs +++ /dev/null @@ -1,28 +0,0 @@ -// @generated -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Pools { - #[prost(message, repeated, tag="1")] - pub pools: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Pool { - #[prost(bytes="vec", tag="1")] - pub pool_id: ::prost::alloc::vec::Vec, - #[prost(fixed64, tag="2")] - pub log_ordinal: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transfer { - #[prost(bytes="vec", tag="1")] - pub from: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="2")] - pub to: ::prost::alloc::vec::Vec, - #[prost(string, tag="3")] - pub token: ::prost::alloc::string::String, - #[prost(string, tag="4")] - pub amount: ::prost::alloc::string::String, -} -// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pb/eth.pool.v1.rs b/substreams/ethereum-balancer/src/pb/eth.pool.v1.rs deleted file mode 100644 index 16ad6e5..0000000 --- a/substreams/ethereum-balancer/src/pb/eth.pool.v1.rs +++ /dev/null @@ -1,22 +0,0 @@ -// @generated -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transfers { - #[prost(message, repeated, tag="1")] - pub transfers: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transfer { - #[prost(string, tag="1")] - pub from: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub to: ::prost::alloc::string::String, - #[prost(uint64, tag="3")] - pub token_id: u64, - #[prost(string, tag="4")] - pub trx_hash: ::prost::alloc::string::String, - #[prost(uint64, tag="5")] - pub ordinal: u64, -} -// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs index affc411..21d371d 100644 --- a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs @@ -51,6 +51,18 @@ pub struct Attribute { #[prost(enumeration="ChangeType", tag="3")] pub change: i32, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtocolType { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(enumeration="FinancialType", tag="2")] + pub financial_type: i32, + #[prost(message, repeated, tag="3")] + pub attribute_schema: ::prost::alloc::vec::Vec, + #[prost(enumeration="ImplementationType", tag="4")] + pub implementation_type: i32, +} /// A struct describing a part of the protocol. /// Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, /// the component would represent a single contract. In case of VM integration, such component would @@ -78,20 +90,12 @@ pub struct ProtocolComponent { /// Type of change the component underwent. #[prost(enumeration="ChangeType", tag="5")] pub change: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionProtocolComponents { - #[prost(message, optional, tag="1")] + /// / Represents the functionality of the component. + #[prost(message, optional, tag="6")] + pub protocol_type: ::core::option::Option, + /// Transaction where this component was created + #[prost(message, optional, tag="7")] pub tx: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub components: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct GroupedTransactionProtocolComponents { - #[prost(message, repeated, tag="1")] - pub tx_components: ::prost::alloc::vec::Vec, } /// A struct for following the changes of Total Value Locked (TVL) of a protocol component. /// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. @@ -105,39 +109,11 @@ pub struct BalanceChange { /// The new balance of the token. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, - /// The id of the component whose TVL is tracked. + /// The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. #[prost(bytes="vec", tag="3")] pub component_id: ::prost::alloc::vec::Vec, } -/// A struct for following the changes of Total Value Locked (TVL) of a protocol component. -/// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. -/// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDelta { - #[prost(uint64, tag="1")] - pub ord: u64, - /// The tx hash of the transaction that caused the balance change. - #[prost(message, optional, tag="2")] - pub tx: ::core::option::Option, - /// The address of the ERC20 token whose balance changed. - #[prost(bytes="vec", tag="3")] - pub token: ::prost::alloc::vec::Vec, - /// The delta balance of the token. - #[prost(bytes="vec", tag="4")] - pub delta: ::prost::alloc::vec::Vec, - /// The id of the component whose TVL is tracked. - /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. - #[prost(bytes="vec", tag="5")] - pub component_id: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDeltas { - #[prost(message, repeated, tag="1")] - pub balance_deltas: ::prost::alloc::vec::Vec, -} /// Enum to specify the type of a change. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -171,6 +147,64 @@ impl ChangeType { } } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FinancialType { + Swap = 0, + Lend = 1, + Leverage = 2, + Psm = 3, +} +impl FinancialType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + FinancialType::Swap => "SWAP", + FinancialType::Lend => "LEND", + FinancialType::Leverage => "LEVERAGE", + FinancialType::Psm => "PSM", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SWAP" => Some(Self::Swap), + "LEND" => Some(Self::Lend), + "LEVERAGE" => Some(Self::Leverage), + "PSM" => Some(Self::Psm), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ImplementationType { + Vm = 0, + Custom = 1, +} +impl ImplementationType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ImplementationType::Vm => "VM", + ImplementationType::Custom => "CUSTOM", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "VM" => Some(Self::Vm), + "CUSTOM" => Some(Self::Custom), + _ => None, + } + } +} // This file contains proto definitions specific to the VM integration. /// A key value entry into contract storage. @@ -233,4 +267,44 @@ pub struct BlockContractChanges { #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } +/// A struct for following the changes of Total Value Locked (TVL). +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDelta { + #[prost(uint64, tag="1")] + pub ord: u64, + /// The tx hash of the transaction that caused the balance change. + #[prost(message, optional, tag="2")] + pub tx: ::core::option::Option, + /// The address of the ERC20 token whose balance changed. + #[prost(bytes="vec", tag="3")] + pub token: ::prost::alloc::vec::Vec, + /// The delta balance of the token. + #[prost(bytes="vec", tag="4")] + pub delta: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. + /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + #[prost(bytes="vec", tag="5")] + pub component_id: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDeltas { + #[prost(message, repeated, tag="1")] + pub balance_deltas: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionProtocolComponents { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub components: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GroupedTransactionProtocolComponents { + #[prost(message, repeated, tag="1")] + pub tx_components: ::prost::alloc::vec::Vec, +} // @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pool_factories.rs b/substreams/ethereum-balancer/src/pool_factories.rs index 62f1601..fb99696 100644 --- a/substreams/ethereum-balancer/src/pool_factories.rs +++ b/substreams/ethereum-balancer/src/pool_factories.rs @@ -3,6 +3,7 @@ use substreams_ethereum::{Event, Function}; use crate::abi; use crate::pb; +use crate::pb::tycho::evm::v1::{FinancialType, ImplementationType, ProtocolType, Transaction}; use pb::tycho::evm::v1::{self as tycho}; use substreams::hex; @@ -13,6 +14,7 @@ use substreams::scalar::BigInt; /// handled by any downstream application. trait SerializableVecBigInt { fn serialize_bytes(&self) -> Vec; + #[allow(dead_code)] fn deserialize_bytes(bytes: &[u8]) -> Vec; } @@ -25,7 +27,7 @@ impl SerializableVecBigInt for Vec { fn deserialize_bytes(bytes: &[u8]) -> Vec { bytes .chunks_exact(32) - .map(|chunk| BigInt::from_signed_bytes_be(chunk)) + .map(BigInt::from_signed_bytes_be) .collect::>() } } @@ -45,6 +47,7 @@ pub fn address_map( pool_factory_address: &[u8], log: &Log, call: &Call, + tx: &Transaction, ) -> Option { match *pool_factory_address { hex!("897888115Ada5773E02aA29F775430BFB5F34c51") => { @@ -70,6 +73,13 @@ pub fn address_map( }, ], change: tycho::ChangeType::Creation.into(), + protocol_type: Some(ProtocolType { + name: "balancer".to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: ImplementationType::Vm.into(), + }), + tx: Some(tx.clone()), }) } hex!("DB8d758BCb971e482B2C45f7F8a7740283A1bd3A") => { @@ -82,14 +92,19 @@ pub fn address_map( id: hex::encode(&pool_created.pool), tokens: create_call.tokens, contracts: vec![pool_factory_address.into(), pool_created.pool], - static_att: vec![ - tycho::Attribute { - name: "pool_type".into(), - value: "ComposableStablePoolFactory".into(), - change: tycho::ChangeType::Creation.into(), - }, - ], + static_att: vec![tycho::Attribute { + name: "pool_type".into(), + value: "ComposableStablePoolFactory".into(), + change: tycho::ChangeType::Creation.into(), + }], change: tycho::ChangeType::Creation.into(), + protocol_type: Some(ProtocolType { + name: "balancer".to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: ImplementationType::Vm.into(), + }), + tx: Some(tx.clone()), }) } hex!("813EE7a840CE909E7Fea2117A44a90b8063bd4fd") => { @@ -117,6 +132,13 @@ pub fn address_map( // Note, rate provider might be provided as `create.protocol_id`, but as a BigInt. needs investigation ], change: tycho::ChangeType::Creation.into(), + protocol_type: Some(ProtocolType { + name: "balancer".to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: ImplementationType::Vm.into(), + }), + tx: Some(tx.clone()), }) } hex!("5F43FBa61f63Fa6bFF101a0A0458cEA917f6B347") => { @@ -142,6 +164,13 @@ pub fn address_map( }, ], change: tycho::ChangeType::Creation.into(), + protocol_type: Some(ProtocolType { + name: "balancer".to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: ImplementationType::Vm.into(), + }), + tx: Some(tx.clone()), }) } // ❌ Reading the deployed factory for Gearbox showcases that it's currently disabled @@ -214,6 +243,13 @@ pub fn address_map( }, ], change: tycho::ChangeType::Creation.into(), + protocol_type: Some(ProtocolType { + name: "balancer".to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: ImplementationType::Vm.into(), + }), + tx: Some(tx.clone()), }) } hex!("5F5222Ffa40F2AEd6380D022184D6ea67C776eE0") => { @@ -239,6 +275,13 @@ pub fn address_map( }, ], change: tycho::ChangeType::Creation.into(), + protocol_type: Some(ProtocolType { + name: "balancer".to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: ImplementationType::Vm.into(), + }), + tx: Some(tx.clone()), }) } // The `WeightedPool2TokenFactory` is a deprecated contract but we've included it since one @@ -266,6 +309,13 @@ pub fn address_map( }, ], change: tycho::ChangeType::Creation.into(), + protocol_type: Some(ProtocolType { + name: "balancer".to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: ImplementationType::Vm.into(), + }), + tx: Some(tx.clone()), }) } _ => None, diff --git a/substreams/ethereum-balancer/substreams.yaml b/substreams/ethereum-balancer/substreams.yaml index f1bc3af..9919f23 100644 --- a/substreams/ethereum-balancer/substreams.yaml +++ b/substreams/ethereum-balancer/substreams.yaml @@ -7,9 +7,9 @@ protobuf: files: - tycho/evm/v1/vm.proto - tycho/evm/v1/common.proto + - tycho/evm/v1/balancer.proto importPaths: - - ../../proto/tycho/evm/v1/ - - ./proto + - ../../proto binaries: default: From 51275f0ccfba5caa43ca72c7d90ad4a193e1edc5 Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:46:52 +0100 Subject: [PATCH 03/12] fix: remove balancer factory from components' contracts --- substreams/ethereum-balancer/src/modules.rs | 5 ++-- .../ethereum-balancer/src/pool_factories.rs | 28 +++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index d121c2d..e741fb3 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -47,7 +47,7 @@ pub fn map_pools_created( .logs_with_calls() .filter(|(_, call)| !call.call.state_reverted) .filter_map(|(log, call)| { - Some(pool_factories::address_map( + pool_factories::address_map( call.call.address.as_slice(), log, call.call, @@ -57,7 +57,7 @@ pub fn map_pools_created( to: tx.to.clone(), index: tx.index.into(), }, - )?) + ) }) .collect::>(); @@ -204,6 +204,7 @@ pub fn map_changes( .map(|(store_delta, balance_delta)| { let pool_id = key::segment_at(&store_delta.key, 1); let token_id = key::segment_at(&store_delta.key, 3); + ( balance_delta.tx.unwrap(), tycho::BalanceChange { diff --git a/substreams/ethereum-balancer/src/pool_factories.rs b/substreams/ethereum-balancer/src/pool_factories.rs index fb99696..e19933c 100644 --- a/substreams/ethereum-balancer/src/pool_factories.rs +++ b/substreams/ethereum-balancer/src/pool_factories.rs @@ -59,7 +59,7 @@ pub fn address_map( Some(tycho::ProtocolComponent { id: hex::encode(&pool_created.pool), tokens: create_call.tokens, - contracts: vec![pool_factory_address.into(), pool_created.pool], + contracts: vec![pool_created.pool], static_att: vec![ tycho::Attribute { name: "pool_type".into(), @@ -74,7 +74,7 @@ pub fn address_map( ], change: tycho::ChangeType::Creation.into(), protocol_type: Some(ProtocolType { - name: "balancer".to_string(), + name: "balancer_pool".to_string(), financial_type: FinancialType::Swap.into(), attribute_schema: vec![], implementation_type: ImplementationType::Vm.into(), @@ -91,7 +91,7 @@ pub fn address_map( Some(tycho::ProtocolComponent { id: hex::encode(&pool_created.pool), tokens: create_call.tokens, - contracts: vec![pool_factory_address.into(), pool_created.pool], + contracts: vec![pool_created.pool], static_att: vec![tycho::Attribute { name: "pool_type".into(), value: "ComposableStablePoolFactory".into(), @@ -99,7 +99,7 @@ pub fn address_map( }], change: tycho::ChangeType::Creation.into(), protocol_type: Some(ProtocolType { - name: "balancer".to_string(), + name: "balancer_pool".to_string(), financial_type: FinancialType::Swap.into(), attribute_schema: vec![], implementation_type: ImplementationType::Vm.into(), @@ -116,7 +116,7 @@ pub fn address_map( Some(tycho::ProtocolComponent { id: hex::encode(&pool_created.pool), tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_factory_address.into(), pool_created.pool], + contracts: vec![pool_created.pool], static_att: vec![ tycho::Attribute { name: "pool_type".into(), @@ -133,7 +133,7 @@ pub fn address_map( ], change: tycho::ChangeType::Creation.into(), protocol_type: Some(ProtocolType { - name: "balancer".to_string(), + name: "balancer_pool".to_string(), financial_type: FinancialType::Swap.into(), attribute_schema: vec![], implementation_type: ImplementationType::Vm.into(), @@ -150,7 +150,7 @@ pub fn address_map( Some(tycho::ProtocolComponent { id: hex::encode(&pool_created.pool), tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_factory_address.into(), pool_created.pool], + contracts: vec![pool_created.pool], static_att: vec![ tycho::Attribute { name: "pool_type".into(), @@ -165,7 +165,7 @@ pub fn address_map( ], change: tycho::ChangeType::Creation.into(), protocol_type: Some(ProtocolType { - name: "balancer".to_string(), + name: "balancer_pool".to_string(), financial_type: FinancialType::Swap.into(), attribute_schema: vec![], implementation_type: ImplementationType::Vm.into(), @@ -229,7 +229,7 @@ pub fn address_map( Some(tycho::ProtocolComponent { id: hex::encode(&pool_created.pool), tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_factory_address.into(), pool_created.pool], + contracts: vec![pool_created.pool], static_att: vec![ tycho::Attribute { name: "pool_type".into(), @@ -244,7 +244,7 @@ pub fn address_map( ], change: tycho::ChangeType::Creation.into(), protocol_type: Some(ProtocolType { - name: "balancer".to_string(), + name: "balancer_pool".to_string(), financial_type: FinancialType::Swap.into(), attribute_schema: vec![], implementation_type: ImplementationType::Vm.into(), @@ -261,7 +261,7 @@ pub fn address_map( Some(tycho::ProtocolComponent { id: hex::encode(&pool_created.pool), tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_factory_address.into(), pool_created.pool], + contracts: vec![pool_created.pool], static_att: vec![ tycho::Attribute { name: "pool_type".into(), @@ -276,7 +276,7 @@ pub fn address_map( ], change: tycho::ChangeType::Creation.into(), protocol_type: Some(ProtocolType { - name: "balancer".to_string(), + name: "balancer_pool".to_string(), financial_type: FinancialType::Swap.into(), attribute_schema: vec![], implementation_type: ImplementationType::Vm.into(), @@ -295,7 +295,7 @@ pub fn address_map( Some(tycho::ProtocolComponent { id: hex::encode(&pool_created.pool), tokens: create_call.tokens, - contracts: vec![pool_factory_address.into(), pool_created.pool], + contracts: vec![pool_created.pool], static_att: vec![ tycho::Attribute { name: "pool_type".into(), @@ -310,7 +310,7 @@ pub fn address_map( ], change: tycho::ChangeType::Creation.into(), protocol_type: Some(ProtocolType { - name: "balancer".to_string(), + name: "balancer_pool".to_string(), financial_type: FinancialType::Swap.into(), attribute_schema: vec![], implementation_type: ImplementationType::Vm.into(), From accfa074fd9ccfaedf387891eb16c38b400889a9 Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:50:12 +0100 Subject: [PATCH 04/12] fix: filter out changes that aren't balancer related --- .../ethereum-balancer/src/contract_changes.rs | 153 ++++++++++-------- 1 file changed, 87 insertions(+), 66 deletions(-) diff --git a/substreams/ethereum-balancer/src/contract_changes.rs b/substreams/ethereum-balancer/src/contract_changes.rs index 4ab49fe..664b0e7 100644 --- a/substreams/ethereum-balancer/src/contract_changes.rs +++ b/substreams/ethereum-balancer/src/contract_changes.rs @@ -97,77 +97,98 @@ pub fn extract_contract_changes( balance_changes.sort_unstable_by_key(|change| change.ordinal); code_changes.sort_unstable_by_key(|change| change.ordinal); - storage_changes.iter().for_each(|storage_change| { - let contract_change = changed_contracts - .entry(storage_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: storage_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&storage_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); + storage_changes + .iter() + .filter(|changes| { + contracts + .get_last(format!("pool:{0}", hex::encode(&changes.address))) + .is_some() + }) + .for_each(|storage_change| { + let contract_change = changed_contracts + .entry(storage_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: storage_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&storage_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); - let slot_value = contract_change - .slots - .entry(storage_change.key.clone()) - .or_insert_with(|| SlotValue { - new_value: storage_change.new_value.clone(), - start_value: storage_change.old_value.clone(), - }); + let slot_value = contract_change + .slots + .entry(storage_change.key.clone()) + .or_insert_with(|| SlotValue { + new_value: storage_change.new_value.clone(), + start_value: storage_change.old_value.clone(), + }); - slot_value - .new_value - .copy_from_slice(&storage_change.new_value); - }); + slot_value + .new_value + .copy_from_slice(&storage_change.new_value); + }); - balance_changes.iter().for_each(|balance_change| { - let contract_change = changed_contracts - .entry(balance_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: balance_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&balance_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); + balance_changes + .iter() + .filter(|changes| { + contracts + .get_last(format!("pool:{0}", hex::encode(&changes.address))) + .is_some() + }) + .for_each(|balance_change| { + let contract_change = changed_contracts + .entry(balance_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: balance_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&balance_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); - if let Some(new_balance) = &balance_change.new_value { - contract_change.balance.clear(); + if let Some(new_balance) = &balance_change.new_value { + contract_change.balance.clear(); + contract_change + .balance + .extend_from_slice(&new_balance.bytes); + } + }); + + code_changes + .iter() + .filter(|changes| { + contracts + .get_last(format!("pool:{0}", hex::encode(&changes.address))) + .is_some() + }) + .for_each(|code_change| { + let contract_change = changed_contracts + .entry(code_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: code_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&code_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); + + contract_change.code.clear(); contract_change - .balance - .extend_from_slice(&new_balance.bytes); - } - }); - - code_changes.iter().for_each(|code_change| { - let contract_change = changed_contracts - .entry(code_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: code_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&code_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); - - contract_change.code.clear(); - contract_change - .code - .extend_from_slice(&code_change.new_code); - }); + .code + .extend_from_slice(&code_change.new_code); + }); if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() { transaction_contract_changes From c283a8134112b502a874e8c76bf63999bdc1e7ce Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:32:42 +0100 Subject: [PATCH 05/12] Make clippy happy --- substreams/ethereum-balancer/build.rs | 1 + substreams/ethereum-balancer/src/abi/mod.rs | 1 + substreams/ethereum-balancer/src/modules.rs | 11 +++-------- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/substreams/ethereum-balancer/build.rs b/substreams/ethereum-balancer/build.rs index ceabf48..90e1f1e 100644 --- a/substreams/ethereum-balancer/build.rs +++ b/substreams/ethereum-balancer/build.rs @@ -8,6 +8,7 @@ fn main() -> Result<()> { let files = fs::read_dir(abi_folder)?; let mut mod_rs_content = String::new(); + mod_rs_content.push_str("#![allow(clippy::all)]\n"); for file in files { let file = file?; diff --git a/substreams/ethereum-balancer/src/abi/mod.rs b/substreams/ethereum-balancer/src/abi/mod.rs index ef6622d..0f60dc0 100644 --- a/substreams/ethereum-balancer/src/abi/mod.rs +++ b/substreams/ethereum-balancer/src/abi/mod.rs @@ -1,3 +1,4 @@ +#![allow(clippy::all)] pub mod yearn_linear_pool_factory; pub mod composable_stable_pool_factory; pub mod vault; diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index e741fb3..7d666cc 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -102,7 +102,7 @@ pub fn map_balance_deltas( ) -> Result { Ok(tycho::BalanceDeltas { balance_deltas: block - .events::(&[&VAULT_ADDRESS]) + .events::(&[VAULT_ADDRESS]) .flat_map(|(event, log)| { event .tokens @@ -111,12 +111,7 @@ pub fn map_balance_deltas( .filter_map(|(token, delta)| { let component_id: Vec<_> = event.pool_id.into(); - if store - .get_last(format!("pool:{0}", hex::encode(&component_id))) - .is_none() - { - return None; - } + store.get_last(format!("pool:{0}", hex::encode(&component_id)))?; Some(tycho::BalanceDelta { ord: log.log.ordinal, @@ -253,7 +248,7 @@ pub fn map_changes( }), changes: transaction_contract_changes .drain() - .sorted_unstable_by_key(|(index, _)| index.clone()) + .sorted_unstable_by_key(|(index, _)| *index) .filter_map(|(_, change)| { if change.contract_changes.is_empty() && change.component_changes.is_empty() From a1864e3cd4bb09dfda2f25d4ce2887e9646e8cb5 Mon Sep 17 00:00:00 2001 From: kayibal Date: Mon, 11 Mar 2024 19:10:28 +0000 Subject: [PATCH 06/12] Don't include balancer specific pb in tycho pbs. --- .../ethereum-balancer/proto}/balancer.proto | 8 ++-- substreams/ethereum-balancer/src/modules.rs | 18 ++++---- .../ethereum-balancer/src/pb/balancer.rs | 42 +++++++++++++++++++ substreams/ethereum-balancer/src/pb/mod.rs | 5 +++ .../ethereum-balancer/src/pb/tycho.evm.v1.rs | 40 ------------------ substreams/ethereum-balancer/substreams.yaml | 3 +- 6 files changed, 62 insertions(+), 54 deletions(-) rename {proto/tycho/evm/v1 => substreams/ethereum-balancer/proto}/balancer.proto (84%) create mode 100644 substreams/ethereum-balancer/src/pb/balancer.rs diff --git a/proto/tycho/evm/v1/balancer.proto b/substreams/ethereum-balancer/proto/balancer.proto similarity index 84% rename from proto/tycho/evm/v1/balancer.proto rename to substreams/ethereum-balancer/proto/balancer.proto index f5a4c63..6efe1c9 100644 --- a/proto/tycho/evm/v1/balancer.proto +++ b/substreams/ethereum-balancer/proto/balancer.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package tycho.evm.v1; +package balancer; import "tycho/evm/v1/common.proto"; @@ -8,7 +8,7 @@ import "tycho/evm/v1/common.proto"; message BalanceDelta { uint64 ord = 1; // The tx hash of the transaction that caused the balance change. - Transaction tx = 2; + tycho.evm.v1.Transaction tx = 2; // The address of the ERC20 token whose balance changed. bytes token = 3; // The delta balance of the token. @@ -23,8 +23,8 @@ message BalanceDeltas { } message TransactionProtocolComponents { - Transaction tx = 1; - repeated ProtocolComponent components = 2; + tycho.evm.v1.Transaction tx = 1; + repeated tycho.evm.v1.ProtocolComponent components = 2; } message GroupedTransactionProtocolComponents { diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 7d666cc..96554ca 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -17,7 +17,7 @@ use pb::tycho::evm::v1::{self as tycho}; use contract_changes::extract_contract_changes; -use crate::pb::tycho::evm::v1::{BalanceDeltas, GroupedTransactionProtocolComponents}; +use crate::pb::balancer::{BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, TransactionProtocolComponents}; use crate::{abi, contract_changes, pb, pool_factories}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); @@ -36,10 +36,10 @@ impl PartialEq for TransactionWrapper { #[substreams::handlers::map] pub fn map_pools_created( block: eth::v2::Block, -) -> Result { +) -> Result { // Gather contract changes by indexing `PoolCreated` events and analysing the `Create` call // We store these as a hashmap by tx hash since we need to agg by tx hash later - Ok(tycho::GroupedTransactionProtocolComponents { + Ok(GroupedTransactionProtocolComponents { tx_components: block .transactions() .filter_map(|tx| { @@ -62,7 +62,7 @@ pub fn map_pools_created( .collect::>(); if !components.is_empty() { - Some(tycho::TransactionProtocolComponents { + Some(TransactionProtocolComponents { tx: Some(tycho::Transaction { hash: tx.hash.clone(), from: tx.from.clone(), @@ -81,7 +81,7 @@ pub fn map_pools_created( /// Simply stores the `ProtocolComponent`s with the pool id as the key #[substreams::handlers::store] -pub fn store_pools_created(map: tycho::GroupedTransactionProtocolComponents, store: StoreAddInt64) { +pub fn store_pools_created(map: GroupedTransactionProtocolComponents, store: StoreAddInt64) { store.add_many( 0, &map.tx_components @@ -99,8 +99,8 @@ pub fn store_pools_created(map: tycho::GroupedTransactionProtocolComponents, sto pub fn map_balance_deltas( block: eth::v2::Block, store: StoreGetInt64, -) -> Result { - Ok(tycho::BalanceDeltas { +) -> Result { + Ok(BalanceDeltas { balance_deltas: block .events::(&[VAULT_ADDRESS]) .flat_map(|(event, log)| { @@ -113,7 +113,7 @@ pub fn map_balance_deltas( store.get_last(format!("pool:{0}", hex::encode(&component_id)))?; - Some(tycho::BalanceDelta { + Some(BalanceDelta { ord: log.log.ordinal, tx: Some(tycho::Transaction { hash: log.receipt.transaction.hash.clone(), @@ -135,7 +135,7 @@ pub fn map_balance_deltas( /// It's significant to include both the `pool_id` and the `token_id` for each balance delta as the /// store key to ensure that there's a unique balance being tallied for each. #[substreams::handlers::store] -pub fn store_balance_changes(deltas: tycho::BalanceDeltas, store: StoreAddBigInt) { +pub fn store_balance_changes(deltas: BalanceDeltas, store: StoreAddBigInt) { deltas.balance_deltas.iter().for_each(|delta| { store.add( delta.ord, diff --git a/substreams/ethereum-balancer/src/pb/balancer.rs b/substreams/ethereum-balancer/src/pb/balancer.rs new file mode 100644 index 0000000..b2f52d2 --- /dev/null +++ b/substreams/ethereum-balancer/src/pb/balancer.rs @@ -0,0 +1,42 @@ +// @generated +/// A struct for following the changes of Total Value Locked (TVL). +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDelta { + #[prost(uint64, tag="1")] + pub ord: u64, + /// The tx hash of the transaction that caused the balance change. + #[prost(message, optional, tag="2")] + pub tx: ::core::option::Option, + /// The address of the ERC20 token whose balance changed. + #[prost(bytes="vec", tag="3")] + pub token: ::prost::alloc::vec::Vec, + /// The delta balance of the token. + #[prost(bytes="vec", tag="4")] + pub delta: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. + /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + #[prost(bytes="vec", tag="5")] + pub component_id: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDeltas { + #[prost(message, repeated, tag="1")] + pub balance_deltas: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionProtocolComponents { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub components: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GroupedTransactionProtocolComponents { + #[prost(message, repeated, tag="1")] + pub tx_components: ::prost::alloc::vec::Vec, +} +// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pb/mod.rs b/substreams/ethereum-balancer/src/pb/mod.rs index 43d8838..1d5f1e8 100644 --- a/substreams/ethereum-balancer/src/pb/mod.rs +++ b/substreams/ethereum-balancer/src/pb/mod.rs @@ -1,4 +1,9 @@ // @generated +// @@protoc_insertion_point(attribute:balancer) +pub mod balancer { + include!("balancer.rs"); + // @@protoc_insertion_point(balancer) +} pub mod tycho { pub mod evm { // @@protoc_insertion_point(attribute:tycho.evm.v1) diff --git a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs index 21d371d..cf81ed2 100644 --- a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs @@ -267,44 +267,4 @@ pub struct BlockContractChanges { #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } -/// A struct for following the changes of Total Value Locked (TVL). -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDelta { - #[prost(uint64, tag="1")] - pub ord: u64, - /// The tx hash of the transaction that caused the balance change. - #[prost(message, optional, tag="2")] - pub tx: ::core::option::Option, - /// The address of the ERC20 token whose balance changed. - #[prost(bytes="vec", tag="3")] - pub token: ::prost::alloc::vec::Vec, - /// The delta balance of the token. - #[prost(bytes="vec", tag="4")] - pub delta: ::prost::alloc::vec::Vec, - /// The id of the component whose TVL is tracked. - /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. - #[prost(bytes="vec", tag="5")] - pub component_id: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDeltas { - #[prost(message, repeated, tag="1")] - pub balance_deltas: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionProtocolComponents { - #[prost(message, optional, tag="1")] - pub tx: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub components: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct GroupedTransactionProtocolComponents { - #[prost(message, repeated, tag="1")] - pub tx_components: ::prost::alloc::vec::Vec, -} // @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/substreams.yaml b/substreams/ethereum-balancer/substreams.yaml index 9919f23..c5b56d0 100644 --- a/substreams/ethereum-balancer/substreams.yaml +++ b/substreams/ethereum-balancer/substreams.yaml @@ -7,9 +7,10 @@ protobuf: files: - tycho/evm/v1/vm.proto - tycho/evm/v1/common.proto - - tycho/evm/v1/balancer.proto + - balancer.proto importPaths: - ../../proto + - ./proto binaries: default: From 1e546230ded47e52096b6a7ec84fa1f175c7a9b9 Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Tue, 12 Mar 2024 12:49:09 +0100 Subject: [PATCH 07/12] fix: map_balance_deltas pool_id check --- substreams/ethereum-balancer/src/modules.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 96554ca..8448772 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -17,7 +17,10 @@ use pb::tycho::evm::v1::{self as tycho}; use contract_changes::extract_contract_changes; -use crate::pb::balancer::{BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, TransactionProtocolComponents}; +use crate::pb::balancer::{ + BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, + TransactionProtocolComponents, +}; use crate::{abi, contract_changes, pb, pool_factories}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); @@ -34,9 +37,7 @@ impl PartialEq for TransactionWrapper { } #[substreams::handlers::map] -pub fn map_pools_created( - block: eth::v2::Block, -) -> Result { +pub fn map_pools_created(block: eth::v2::Block) -> Result { // Gather contract changes by indexing `PoolCreated` events and analysing the `Create` call // We store these as a hashmap by tx hash since we need to agg by tx hash later Ok(GroupedTransactionProtocolComponents { @@ -109,7 +110,7 @@ pub fn map_balance_deltas( .iter() .zip(event.deltas.iter()) .filter_map(|(token, delta)| { - let component_id: Vec<_> = event.pool_id.into(); + let component_id = event.pool_id[..20].to_vec(); store.get_last(format!("pool:{0}", hex::encode(&component_id)))?; @@ -119,11 +120,11 @@ pub fn map_balance_deltas( hash: log.receipt.transaction.hash.clone(), from: log.receipt.transaction.from.clone(), to: log.receipt.transaction.to.clone(), - index: Into::::into(log.receipt.transaction.index), + index: log.receipt.transaction.index.into(), }), - token: token.clone(), + token: token.to_vec(), delta: delta.to_signed_bytes_be(), - component_id: component_id.clone(), + component_id, }) }) .collect::>() From 38dcd9d843354bfc8fdb9e461da4594d102837e1 Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Tue, 12 Mar 2024 12:51:26 +0100 Subject: [PATCH 08/12] fix: BalanceChange encoding --- proto/tycho/evm/v1/common.proto | 2 +- substreams/ethereum-balancer/src/modules.rs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/proto/tycho/evm/v1/common.proto b/proto/tycho/evm/v1/common.proto index 545cad0..7f6d369 100644 --- a/proto/tycho/evm/v1/common.proto +++ b/proto/tycho/evm/v1/common.proto @@ -100,7 +100,7 @@ message ProtocolComponent { message BalanceChange { // The address of the ERC20 token whose balance changed. bytes token = 1; - // The new balance of the token. + // The new balance of the token. Note: it must be a big endian encoded int. bytes balance = 2; // The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 8448772..f84a496 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::str::FromStr; use anyhow::Result; use substreams::hex; @@ -200,12 +201,17 @@ pub fn map_changes( .map(|(store_delta, balance_delta)| { let pool_id = key::segment_at(&store_delta.key, 1); let token_id = key::segment_at(&store_delta.key, 3); + // store_delta.new_value is an ASCII string representing an integer + let ascii_string = + String::from_utf8(store_delta.new_value.clone()).expect("Invalid UTF-8 sequence"); + let balance = BigInt::from_str(&ascii_string).expect("Failed to parse integer"); + let big_endian_bytes_balance = balance.to_bytes_be().1; ( balance_delta.tx.unwrap(), tycho::BalanceChange { token: hex::decode(token_id).expect("Token ID not valid hex"), - balance: store_delta.new_value, + balance: big_endian_bytes_balance, component_id: hex::decode(pool_id).expect("Token ID not valid hex"), }, ) From 2a1b83f6c5c317ac5e2f270904e9f944c9668f09 Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:53:30 +0100 Subject: [PATCH 09/12] fix: index balance changes from `Swap` events --- substreams/ethereum-balancer/src/modules.rs | 90 +++++++++++++++------ 1 file changed, 65 insertions(+), 25 deletions(-) diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index f84a496..71f0e83 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -17,6 +17,7 @@ use itertools::Itertools; use pb::tycho::evm::v1::{self as tycho}; use contract_changes::extract_contract_changes; +use substreams_ethereum::Event; use crate::pb::balancer::{ BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, @@ -95,43 +96,82 @@ pub fn store_pools_created(map: GroupedTransactionProtocolComponents, store: Sto ); } -/// Since the `PoolBalanceChanged` events administer only deltas, we need to leverage a map and a +/// Since the `PoolBalanceChanged` and `Swap` events administer only deltas, we need to leverage a map and a /// store to be able to tally up final balances for tokens in a pool. #[substreams::handlers::map] pub fn map_balance_deltas( block: eth::v2::Block, store: StoreGetInt64, ) -> Result { - Ok(BalanceDeltas { - balance_deltas: block - .events::(&[VAULT_ADDRESS]) - .flat_map(|(event, log)| { - event - .tokens - .iter() - .zip(event.deltas.iter()) - .filter_map(|(token, delta)| { - let component_id = event.pool_id[..20].to_vec(); + let balance_deltas = block + .logs() + .filter(|log| log.address() == VAULT_ADDRESS) + .flat_map(|vault_log| { + let mut deltas = Vec::new(); - store.get_last(format!("pool:{0}", hex::encode(&component_id)))?; - - Some(BalanceDelta { - ord: log.log.ordinal, + if let Some(ev) = + abi::vault::events::PoolBalanceChanged::match_and_decode(vault_log.log) + { + let component_id = ev.pool_id[..20].to_vec(); + if store + .get_last(format!("pool:{}", hex::encode(&component_id))) + .is_some() + { + for (token, delta) in ev.tokens.iter().zip(ev.deltas.iter()) { + deltas.push(BalanceDelta { + ord: vault_log.ordinal(), tx: Some(tycho::Transaction { - hash: log.receipt.transaction.hash.clone(), - from: log.receipt.transaction.from.clone(), - to: log.receipt.transaction.to.clone(), - index: log.receipt.transaction.index.into(), + hash: vault_log.receipt.transaction.hash.clone(), + from: vault_log.receipt.transaction.from.clone(), + to: vault_log.receipt.transaction.to.clone(), + index: vault_log.receipt.transaction.index.into(), }), token: token.to_vec(), delta: delta.to_signed_bytes_be(), + component_id: component_id.clone(), + }); + } + } + } else if let Some(ev) = abi::vault::events::Swap::match_and_decode(vault_log.log) { + let component_id = ev.pool_id[..20].to_vec(); + if store + .get_last(format!("pool:{}", hex::encode(&component_id))) + .is_some() + { + deltas.extend_from_slice(&[ + BalanceDelta { + ord: vault_log.ordinal(), + tx: Some(tycho::Transaction { + hash: vault_log.receipt.transaction.hash.clone(), + from: vault_log.receipt.transaction.from.clone(), + to: vault_log.receipt.transaction.to.clone(), + index: vault_log.receipt.transaction.index.into(), + }), + token: ev.token_in.to_vec(), + delta: ev.amount_in.to_signed_bytes_be(), + component_id: component_id.clone(), + }, + BalanceDelta { + ord: vault_log.ordinal(), + tx: Some(tycho::Transaction { + hash: vault_log.receipt.transaction.hash.clone(), + from: vault_log.receipt.transaction.from.clone(), + to: vault_log.receipt.transaction.to.clone(), + index: vault_log.receipt.transaction.index.into(), + }), + token: ev.token_out.to_vec(), + delta: ev.amount_out.neg().to_signed_bytes_be(), component_id, - }) - }) - .collect::>() - }) - .collect::>(), - }) + }, + ]); + } + } + + deltas + }) + .collect::>(); + + Ok(BalanceDeltas { balance_deltas }) } /// It's significant to include both the `pool_id` and the `token_id` for each balance delta as the From 7601072e873b21738c8238bb23016365ca4fc882 Mon Sep 17 00:00:00 2001 From: Florian Pellissier <111426680+flopell@users.noreply.github.com> Date: Wed, 13 Mar 2024 10:19:59 +0100 Subject: [PATCH 10/12] fix: encode component_id as UTF8 --- substreams/ethereum-balancer/src/modules.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 71f0e83..2b47bd1 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -252,7 +252,7 @@ pub fn map_changes( tycho::BalanceChange { token: hex::decode(token_id).expect("Token ID not valid hex"), balance: big_endian_bytes_balance, - component_id: hex::decode(pool_id).expect("Token ID not valid hex"), + component_id: pool_id.as_bytes().to_vec(), }, ) }) From 4b2437dab63d320daa09711375d0cc879f0b3fad Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 12:11:38 +0000 Subject: [PATCH 11/12] Revert "Don't include balancer specific pb in tycho pbs." This reverts commit a1864e3c. We will actually make the balance delta messages part of the tycho packages since this makes a few things simpler when trying to factor out an sdk. --- .../tycho/evm/v1}/balancer.proto | 8 ++-- substreams/ethereum-balancer/src/modules.rs | 10 ++--- .../ethereum-balancer/src/pb/balancer.rs | 42 ------------------- substreams/ethereum-balancer/src/pb/mod.rs | 5 --- .../ethereum-balancer/src/pb/tycho.evm.v1.rs | 40 ++++++++++++++++++ substreams/ethereum-balancer/substreams.yaml | 3 +- 6 files changed, 49 insertions(+), 59 deletions(-) rename {substreams/ethereum-balancer/proto => proto/tycho/evm/v1}/balancer.proto (84%) delete mode 100644 substreams/ethereum-balancer/src/pb/balancer.rs diff --git a/substreams/ethereum-balancer/proto/balancer.proto b/proto/tycho/evm/v1/balancer.proto similarity index 84% rename from substreams/ethereum-balancer/proto/balancer.proto rename to proto/tycho/evm/v1/balancer.proto index 6efe1c9..f5a4c63 100644 --- a/substreams/ethereum-balancer/proto/balancer.proto +++ b/proto/tycho/evm/v1/balancer.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package balancer; +package tycho.evm.v1; import "tycho/evm/v1/common.proto"; @@ -8,7 +8,7 @@ import "tycho/evm/v1/common.proto"; message BalanceDelta { uint64 ord = 1; // The tx hash of the transaction that caused the balance change. - tycho.evm.v1.Transaction tx = 2; + Transaction tx = 2; // The address of the ERC20 token whose balance changed. bytes token = 3; // The delta balance of the token. @@ -23,8 +23,8 @@ message BalanceDeltas { } message TransactionProtocolComponents { - tycho.evm.v1.Transaction tx = 1; - repeated tycho.evm.v1.ProtocolComponent components = 2; + Transaction tx = 1; + repeated ProtocolComponent components = 2; } message GroupedTransactionProtocolComponents { diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 2b47bd1..75f61a3 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -14,16 +14,14 @@ use substreams::scalar::BigInt; use substreams_ethereum::pb::eth; use itertools::Itertools; -use pb::tycho::evm::v1::{self as tycho}; + use contract_changes::extract_contract_changes; use substreams_ethereum::Event; -use crate::pb::balancer::{ - BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, - TransactionProtocolComponents, -}; -use crate::{abi, contract_changes, pb, pool_factories}; +use crate::pb::tycho::evm::v1::{self as tycho}; +use crate::pb::tycho::evm::v1::{BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, TransactionProtocolComponents}; +use crate::{abi, contract_changes, pool_factories}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); diff --git a/substreams/ethereum-balancer/src/pb/balancer.rs b/substreams/ethereum-balancer/src/pb/balancer.rs deleted file mode 100644 index b2f52d2..0000000 --- a/substreams/ethereum-balancer/src/pb/balancer.rs +++ /dev/null @@ -1,42 +0,0 @@ -// @generated -/// A struct for following the changes of Total Value Locked (TVL). -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDelta { - #[prost(uint64, tag="1")] - pub ord: u64, - /// The tx hash of the transaction that caused the balance change. - #[prost(message, optional, tag="2")] - pub tx: ::core::option::Option, - /// The address of the ERC20 token whose balance changed. - #[prost(bytes="vec", tag="3")] - pub token: ::prost::alloc::vec::Vec, - /// The delta balance of the token. - #[prost(bytes="vec", tag="4")] - pub delta: ::prost::alloc::vec::Vec, - /// The id of the component whose TVL is tracked. - /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. - #[prost(bytes="vec", tag="5")] - pub component_id: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDeltas { - #[prost(message, repeated, tag="1")] - pub balance_deltas: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionProtocolComponents { - #[prost(message, optional, tag="1")] - pub tx: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub components: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct GroupedTransactionProtocolComponents { - #[prost(message, repeated, tag="1")] - pub tx_components: ::prost::alloc::vec::Vec, -} -// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pb/mod.rs b/substreams/ethereum-balancer/src/pb/mod.rs index 1d5f1e8..43d8838 100644 --- a/substreams/ethereum-balancer/src/pb/mod.rs +++ b/substreams/ethereum-balancer/src/pb/mod.rs @@ -1,9 +1,4 @@ // @generated -// @@protoc_insertion_point(attribute:balancer) -pub mod balancer { - include!("balancer.rs"); - // @@protoc_insertion_point(balancer) -} pub mod tycho { pub mod evm { // @@protoc_insertion_point(attribute:tycho.evm.v1) diff --git a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs index cf81ed2..21d371d 100644 --- a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs @@ -267,4 +267,44 @@ pub struct BlockContractChanges { #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } +/// A struct for following the changes of Total Value Locked (TVL). +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDelta { + #[prost(uint64, tag="1")] + pub ord: u64, + /// The tx hash of the transaction that caused the balance change. + #[prost(message, optional, tag="2")] + pub tx: ::core::option::Option, + /// The address of the ERC20 token whose balance changed. + #[prost(bytes="vec", tag="3")] + pub token: ::prost::alloc::vec::Vec, + /// The delta balance of the token. + #[prost(bytes="vec", tag="4")] + pub delta: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. + /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + #[prost(bytes="vec", tag="5")] + pub component_id: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDeltas { + #[prost(message, repeated, tag="1")] + pub balance_deltas: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionProtocolComponents { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub components: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GroupedTransactionProtocolComponents { + #[prost(message, repeated, tag="1")] + pub tx_components: ::prost::alloc::vec::Vec, +} // @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/substreams.yaml b/substreams/ethereum-balancer/substreams.yaml index c5b56d0..9919f23 100644 --- a/substreams/ethereum-balancer/substreams.yaml +++ b/substreams/ethereum-balancer/substreams.yaml @@ -7,10 +7,9 @@ protobuf: files: - tycho/evm/v1/vm.proto - tycho/evm/v1/common.proto - - balancer.proto + - tycho/evm/v1/balancer.proto importPaths: - ../../proto - - ./proto binaries: default: From 0d621a4545c9a00f6ce6942d127b4af259b33dec Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 12:28:10 +0000 Subject: [PATCH 12/12] Improve the naming of utils. --- .../evm/v1/{balancer.proto => utils.proto} | 17 ++++++++++++---- substreams/ethereum-balancer/src/modules.rs | 20 +++++++++---------- .../ethereum-balancer/src/pb/tycho.evm.v1.rs | 19 +++++++++++++----- substreams/ethereum-balancer/substreams.yaml | 2 +- 4 files changed, 38 insertions(+), 20 deletions(-) rename proto/tycho/evm/v1/{balancer.proto => utils.proto} (50%) diff --git a/proto/tycho/evm/v1/balancer.proto b/proto/tycho/evm/v1/utils.proto similarity index 50% rename from proto/tycho/evm/v1/balancer.proto rename to proto/tycho/evm/v1/utils.proto index f5a4c63..ea3f5d5 100644 --- a/proto/tycho/evm/v1/balancer.proto +++ b/proto/tycho/evm/v1/utils.proto @@ -4,8 +4,13 @@ package tycho.evm.v1; import "tycho/evm/v1/common.proto"; -// A struct for following the changes of Total Value Locked (TVL). +// A message containing relative balance changes. +// +// Used to track token balances of protocol components in case they are only +// available as relative values within a block. message BalanceDelta { + // The ordinal of the balance change. Must be unique & deterministic over all balances + // changes within a block. uint64 ord = 1; // The tx hash of the transaction that caused the balance change. Transaction tx = 2; @@ -14,19 +19,23 @@ message BalanceDelta { // The delta balance of the token. bytes delta = 4; // The id of the component whose TVL is tracked. - // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + // If the protocol component includes multiple contracts, the balance change must be + // aggregated to reflect how much tokens can be traded. bytes component_id = 5; } -message BalanceDeltas { +// A set of balances deltas, usually a group of changes within a single block. +message BlockBalanceDeltas { repeated BalanceDelta balance_deltas = 1; } +// A message containing protocol components that were created by a single tx. message TransactionProtocolComponents { Transaction tx = 1; repeated ProtocolComponent components = 2; } -message GroupedTransactionProtocolComponents { +// All protocol components that were created within a block with their corresponding tx. +message BlockTransactionProtocolComponents { repeated TransactionProtocolComponents tx_components = 1; } \ No newline at end of file diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 75f61a3..c4fcb4a 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -20,7 +20,7 @@ use contract_changes::extract_contract_changes; use substreams_ethereum::Event; use crate::pb::tycho::evm::v1::{self as tycho}; -use crate::pb::tycho::evm::v1::{BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, TransactionProtocolComponents}; +use crate::pb::tycho::evm::v1::{BalanceDelta, BlockBalanceDeltas, BlockTransactionProtocolComponents, TransactionProtocolComponents}; use crate::{abi, contract_changes, pool_factories}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); @@ -37,10 +37,10 @@ impl PartialEq for TransactionWrapper { } #[substreams::handlers::map] -pub fn map_pools_created(block: eth::v2::Block) -> Result { +pub fn map_pools_created(block: eth::v2::Block) -> Result { // Gather contract changes by indexing `PoolCreated` events and analysing the `Create` call // We store these as a hashmap by tx hash since we need to agg by tx hash later - Ok(GroupedTransactionProtocolComponents { + Ok(BlockTransactionProtocolComponents { tx_components: block .transactions() .filter_map(|tx| { @@ -82,7 +82,7 @@ pub fn map_pools_created(block: eth::v2::Block) -> Result Result { +) -> Result { let balance_deltas = block .logs() .filter(|log| log.address() == VAULT_ADDRESS) @@ -169,13 +169,13 @@ pub fn map_balance_deltas( }) .collect::>(); - Ok(BalanceDeltas { balance_deltas }) + Ok(BlockBalanceDeltas { balance_deltas }) } /// It's significant to include both the `pool_id` and the `token_id` for each balance delta as the /// store key to ensure that there's a unique balance being tallied for each. #[substreams::handlers::store] -pub fn store_balance_changes(deltas: BalanceDeltas, store: StoreAddBigInt) { +pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: StoreAddBigInt) { deltas.balance_deltas.iter().for_each(|delta| { store.add( delta.ord, @@ -198,8 +198,8 @@ pub fn store_balance_changes(deltas: BalanceDeltas, store: StoreAddBigInt) { #[substreams::handlers::map] pub fn map_changes( block: eth::v2::Block, - grouped_components: GroupedTransactionProtocolComponents, - deltas: BalanceDeltas, + grouped_components: BlockTransactionProtocolComponents, + deltas: BlockBalanceDeltas, components_store: StoreGetInt64, balance_store: StoreDeltas, // Note, this map module is using the `deltas` mode for the store. ) -> Result { @@ -229,7 +229,7 @@ pub fn map_changes( }); // Balance changes are gathered by the `StoreDelta` based on `PoolBalanceChanged` creating - // `BalanceDeltas`. We essentially just process the changes that occured to the `store` this + // `BlockBalanceDeltas`. We essentially just process the changes that occured to the `store` this // block. Then, these balance changes are merged onto the existing map of tx contract changes, // inserting a new one if it doesn't exist. balance_store diff --git a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs index 21d371d..387ff7b 100644 --- a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs @@ -106,7 +106,7 @@ pub struct BalanceChange { /// The address of the ERC20 token whose balance changed. #[prost(bytes="vec", tag="1")] pub token: ::prost::alloc::vec::Vec, - /// The new balance of the token. + /// The new balance of the token. Note: it must be a big endian encoded int. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, /// The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. @@ -267,10 +267,15 @@ pub struct BlockContractChanges { #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } -/// A struct for following the changes of Total Value Locked (TVL). +/// A message containing relative balance changes. +/// +/// Used to track token balances of protocol components in case they are only +/// available as relative values within a block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BalanceDelta { + /// The ordinal of the balance change. Must be unique & deterministic over all balances + /// changes within a block. #[prost(uint64, tag="1")] pub ord: u64, /// The tx hash of the transaction that caused the balance change. @@ -283,16 +288,19 @@ pub struct BalanceDelta { #[prost(bytes="vec", tag="4")] pub delta: ::prost::alloc::vec::Vec, /// The id of the component whose TVL is tracked. - /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + /// If the protocol component includes multiple contracts, the balance change must be + /// aggregated to reflect how much tokens can be traded. #[prost(bytes="vec", tag="5")] pub component_id: ::prost::alloc::vec::Vec, } +/// A set of balances deltas, usually a group of changes within a single block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDeltas { +pub struct BlockBalanceDeltas { #[prost(message, repeated, tag="1")] pub balance_deltas: ::prost::alloc::vec::Vec, } +/// A message containing protocol components that were created by a single tx. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionProtocolComponents { @@ -301,9 +309,10 @@ pub struct TransactionProtocolComponents { #[prost(message, repeated, tag="2")] pub components: ::prost::alloc::vec::Vec, } +/// All protocol components that were created within a block with their corresponding tx. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct GroupedTransactionProtocolComponents { +pub struct BlockTransactionProtocolComponents { #[prost(message, repeated, tag="1")] pub tx_components: ::prost::alloc::vec::Vec, } diff --git a/substreams/ethereum-balancer/substreams.yaml b/substreams/ethereum-balancer/substreams.yaml index 9919f23..66da244 100644 --- a/substreams/ethereum-balancer/substreams.yaml +++ b/substreams/ethereum-balancer/substreams.yaml @@ -7,7 +7,7 @@ protobuf: files: - tycho/evm/v1/vm.proto - tycho/evm/v1/common.proto - - tycho/evm/v1/balancer.proto + - tycho/evm/v1/utils.proto importPaths: - ../../proto