From 4b2437dab63d320daa09711375d0cc879f0b3fad Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 12:11:38 +0000 Subject: [PATCH 1/2] Revert "Don't include balancer specific pb in tycho pbs." This reverts commit a1864e3c. We will actually make the balance delta messages part of the tycho packages since this makes a few things simpler when trying to factor out an sdk. --- .../tycho/evm/v1}/balancer.proto | 8 ++-- substreams/ethereum-balancer/src/modules.rs | 10 ++--- .../ethereum-balancer/src/pb/balancer.rs | 42 ------------------- substreams/ethereum-balancer/src/pb/mod.rs | 5 --- .../ethereum-balancer/src/pb/tycho.evm.v1.rs | 40 ++++++++++++++++++ substreams/ethereum-balancer/substreams.yaml | 3 +- 6 files changed, 49 insertions(+), 59 deletions(-) rename {substreams/ethereum-balancer/proto => proto/tycho/evm/v1}/balancer.proto (84%) delete mode 100644 substreams/ethereum-balancer/src/pb/balancer.rs diff --git a/substreams/ethereum-balancer/proto/balancer.proto b/proto/tycho/evm/v1/balancer.proto similarity index 84% rename from substreams/ethereum-balancer/proto/balancer.proto rename to proto/tycho/evm/v1/balancer.proto index 6efe1c9..f5a4c63 100644 --- a/substreams/ethereum-balancer/proto/balancer.proto +++ b/proto/tycho/evm/v1/balancer.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package balancer; +package tycho.evm.v1; import "tycho/evm/v1/common.proto"; @@ -8,7 +8,7 @@ import "tycho/evm/v1/common.proto"; message BalanceDelta { uint64 ord = 1; // The tx hash of the transaction that caused the balance change. - tycho.evm.v1.Transaction tx = 2; + Transaction tx = 2; // The address of the ERC20 token whose balance changed. bytes token = 3; // The delta balance of the token. @@ -23,8 +23,8 @@ message BalanceDeltas { } message TransactionProtocolComponents { - tycho.evm.v1.Transaction tx = 1; - repeated tycho.evm.v1.ProtocolComponent components = 2; + Transaction tx = 1; + repeated ProtocolComponent components = 2; } message GroupedTransactionProtocolComponents { diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 2b47bd1..75f61a3 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -14,16 +14,14 @@ use substreams::scalar::BigInt; use substreams_ethereum::pb::eth; use itertools::Itertools; -use pb::tycho::evm::v1::{self as tycho}; + use contract_changes::extract_contract_changes; use substreams_ethereum::Event; -use crate::pb::balancer::{ - BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, - TransactionProtocolComponents, -}; -use crate::{abi, contract_changes, pb, pool_factories}; +use crate::pb::tycho::evm::v1::{self as tycho}; +use crate::pb::tycho::evm::v1::{BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, TransactionProtocolComponents}; +use crate::{abi, contract_changes, pool_factories}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); diff --git a/substreams/ethereum-balancer/src/pb/balancer.rs b/substreams/ethereum-balancer/src/pb/balancer.rs deleted file mode 100644 index b2f52d2..0000000 --- a/substreams/ethereum-balancer/src/pb/balancer.rs +++ /dev/null @@ -1,42 +0,0 @@ -// @generated -/// A struct for following the changes of Total Value Locked (TVL). -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDelta { - #[prost(uint64, tag="1")] - pub ord: u64, - /// The tx hash of the transaction that caused the balance change. - #[prost(message, optional, tag="2")] - pub tx: ::core::option::Option, - /// The address of the ERC20 token whose balance changed. - #[prost(bytes="vec", tag="3")] - pub token: ::prost::alloc::vec::Vec, - /// The delta balance of the token. - #[prost(bytes="vec", tag="4")] - pub delta: ::prost::alloc::vec::Vec, - /// The id of the component whose TVL is tracked. - /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. - #[prost(bytes="vec", tag="5")] - pub component_id: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDeltas { - #[prost(message, repeated, tag="1")] - pub balance_deltas: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionProtocolComponents { - #[prost(message, optional, tag="1")] - pub tx: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub components: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct GroupedTransactionProtocolComponents { - #[prost(message, repeated, tag="1")] - pub tx_components: ::prost::alloc::vec::Vec, -} -// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pb/mod.rs b/substreams/ethereum-balancer/src/pb/mod.rs index 1d5f1e8..43d8838 100644 --- a/substreams/ethereum-balancer/src/pb/mod.rs +++ b/substreams/ethereum-balancer/src/pb/mod.rs @@ -1,9 +1,4 @@ // @generated -// @@protoc_insertion_point(attribute:balancer) -pub mod balancer { - include!("balancer.rs"); - // @@protoc_insertion_point(balancer) -} pub mod tycho { pub mod evm { // @@protoc_insertion_point(attribute:tycho.evm.v1) diff --git a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs index cf81ed2..21d371d 100644 --- a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs @@ -267,4 +267,44 @@ pub struct BlockContractChanges { #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } +/// A struct for following the changes of Total Value Locked (TVL). +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDelta { + #[prost(uint64, tag="1")] + pub ord: u64, + /// The tx hash of the transaction that caused the balance change. + #[prost(message, optional, tag="2")] + pub tx: ::core::option::Option, + /// The address of the ERC20 token whose balance changed. + #[prost(bytes="vec", tag="3")] + pub token: ::prost::alloc::vec::Vec, + /// The delta balance of the token. + #[prost(bytes="vec", tag="4")] + pub delta: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. + /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + #[prost(bytes="vec", tag="5")] + pub component_id: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDeltas { + #[prost(message, repeated, tag="1")] + pub balance_deltas: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionProtocolComponents { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub components: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GroupedTransactionProtocolComponents { + #[prost(message, repeated, tag="1")] + pub tx_components: ::prost::alloc::vec::Vec, +} // @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/substreams.yaml b/substreams/ethereum-balancer/substreams.yaml index c5b56d0..9919f23 100644 --- a/substreams/ethereum-balancer/substreams.yaml +++ b/substreams/ethereum-balancer/substreams.yaml @@ -7,10 +7,9 @@ protobuf: files: - tycho/evm/v1/vm.proto - tycho/evm/v1/common.proto - - balancer.proto + - tycho/evm/v1/balancer.proto importPaths: - ../../proto - - ./proto binaries: default: From 0d621a4545c9a00f6ce6942d127b4af259b33dec Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 12:28:10 +0000 Subject: [PATCH 2/2] Improve the naming of utils. --- .../evm/v1/{balancer.proto => utils.proto} | 17 ++++++++++++---- substreams/ethereum-balancer/src/modules.rs | 20 +++++++++---------- .../ethereum-balancer/src/pb/tycho.evm.v1.rs | 19 +++++++++++++----- substreams/ethereum-balancer/substreams.yaml | 2 +- 4 files changed, 38 insertions(+), 20 deletions(-) rename proto/tycho/evm/v1/{balancer.proto => utils.proto} (50%) diff --git a/proto/tycho/evm/v1/balancer.proto b/proto/tycho/evm/v1/utils.proto similarity index 50% rename from proto/tycho/evm/v1/balancer.proto rename to proto/tycho/evm/v1/utils.proto index f5a4c63..ea3f5d5 100644 --- a/proto/tycho/evm/v1/balancer.proto +++ b/proto/tycho/evm/v1/utils.proto @@ -4,8 +4,13 @@ package tycho.evm.v1; import "tycho/evm/v1/common.proto"; -// A struct for following the changes of Total Value Locked (TVL). +// A message containing relative balance changes. +// +// Used to track token balances of protocol components in case they are only +// available as relative values within a block. message BalanceDelta { + // The ordinal of the balance change. Must be unique & deterministic over all balances + // changes within a block. uint64 ord = 1; // The tx hash of the transaction that caused the balance change. Transaction tx = 2; @@ -14,19 +19,23 @@ message BalanceDelta { // The delta balance of the token. bytes delta = 4; // The id of the component whose TVL is tracked. - // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + // If the protocol component includes multiple contracts, the balance change must be + // aggregated to reflect how much tokens can be traded. bytes component_id = 5; } -message BalanceDeltas { +// A set of balances deltas, usually a group of changes within a single block. +message BlockBalanceDeltas { repeated BalanceDelta balance_deltas = 1; } +// A message containing protocol components that were created by a single tx. message TransactionProtocolComponents { Transaction tx = 1; repeated ProtocolComponent components = 2; } -message GroupedTransactionProtocolComponents { +// All protocol components that were created within a block with their corresponding tx. +message BlockTransactionProtocolComponents { repeated TransactionProtocolComponents tx_components = 1; } \ No newline at end of file diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 75f61a3..c4fcb4a 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -20,7 +20,7 @@ use contract_changes::extract_contract_changes; use substreams_ethereum::Event; use crate::pb::tycho::evm::v1::{self as tycho}; -use crate::pb::tycho::evm::v1::{BalanceDelta, BalanceDeltas, GroupedTransactionProtocolComponents, TransactionProtocolComponents}; +use crate::pb::tycho::evm::v1::{BalanceDelta, BlockBalanceDeltas, BlockTransactionProtocolComponents, TransactionProtocolComponents}; use crate::{abi, contract_changes, pool_factories}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); @@ -37,10 +37,10 @@ impl PartialEq for TransactionWrapper { } #[substreams::handlers::map] -pub fn map_pools_created(block: eth::v2::Block) -> Result { +pub fn map_pools_created(block: eth::v2::Block) -> Result { // Gather contract changes by indexing `PoolCreated` events and analysing the `Create` call // We store these as a hashmap by tx hash since we need to agg by tx hash later - Ok(GroupedTransactionProtocolComponents { + Ok(BlockTransactionProtocolComponents { tx_components: block .transactions() .filter_map(|tx| { @@ -82,7 +82,7 @@ pub fn map_pools_created(block: eth::v2::Block) -> Result Result { +) -> Result { let balance_deltas = block .logs() .filter(|log| log.address() == VAULT_ADDRESS) @@ -169,13 +169,13 @@ pub fn map_balance_deltas( }) .collect::>(); - Ok(BalanceDeltas { balance_deltas }) + Ok(BlockBalanceDeltas { balance_deltas }) } /// It's significant to include both the `pool_id` and the `token_id` for each balance delta as the /// store key to ensure that there's a unique balance being tallied for each. #[substreams::handlers::store] -pub fn store_balance_changes(deltas: BalanceDeltas, store: StoreAddBigInt) { +pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: StoreAddBigInt) { deltas.balance_deltas.iter().for_each(|delta| { store.add( delta.ord, @@ -198,8 +198,8 @@ pub fn store_balance_changes(deltas: BalanceDeltas, store: StoreAddBigInt) { #[substreams::handlers::map] pub fn map_changes( block: eth::v2::Block, - grouped_components: GroupedTransactionProtocolComponents, - deltas: BalanceDeltas, + grouped_components: BlockTransactionProtocolComponents, + deltas: BlockBalanceDeltas, components_store: StoreGetInt64, balance_store: StoreDeltas, // Note, this map module is using the `deltas` mode for the store. ) -> Result { @@ -229,7 +229,7 @@ pub fn map_changes( }); // Balance changes are gathered by the `StoreDelta` based on `PoolBalanceChanged` creating - // `BalanceDeltas`. We essentially just process the changes that occured to the `store` this + // `BlockBalanceDeltas`. We essentially just process the changes that occured to the `store` this // block. Then, these balance changes are merged onto the existing map of tx contract changes, // inserting a new one if it doesn't exist. balance_store diff --git a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs index 21d371d..387ff7b 100644 --- a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs @@ -106,7 +106,7 @@ pub struct BalanceChange { /// The address of the ERC20 token whose balance changed. #[prost(bytes="vec", tag="1")] pub token: ::prost::alloc::vec::Vec, - /// The new balance of the token. + /// The new balance of the token. Note: it must be a big endian encoded int. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, /// The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. @@ -267,10 +267,15 @@ pub struct BlockContractChanges { #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } -/// A struct for following the changes of Total Value Locked (TVL). +/// A message containing relative balance changes. +/// +/// Used to track token balances of protocol components in case they are only +/// available as relative values within a block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BalanceDelta { + /// The ordinal of the balance change. Must be unique & deterministic over all balances + /// changes within a block. #[prost(uint64, tag="1")] pub ord: u64, /// The tx hash of the transaction that caused the balance change. @@ -283,16 +288,19 @@ pub struct BalanceDelta { #[prost(bytes="vec", tag="4")] pub delta: ::prost::alloc::vec::Vec, /// The id of the component whose TVL is tracked. - /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + /// If the protocol component includes multiple contracts, the balance change must be + /// aggregated to reflect how much tokens can be traded. #[prost(bytes="vec", tag="5")] pub component_id: ::prost::alloc::vec::Vec, } +/// A set of balances deltas, usually a group of changes within a single block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDeltas { +pub struct BlockBalanceDeltas { #[prost(message, repeated, tag="1")] pub balance_deltas: ::prost::alloc::vec::Vec, } +/// A message containing protocol components that were created by a single tx. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionProtocolComponents { @@ -301,9 +309,10 @@ pub struct TransactionProtocolComponents { #[prost(message, repeated, tag="2")] pub components: ::prost::alloc::vec::Vec, } +/// All protocol components that were created within a block with their corresponding tx. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct GroupedTransactionProtocolComponents { +pub struct BlockTransactionProtocolComponents { #[prost(message, repeated, tag="1")] pub tx_components: ::prost::alloc::vec::Vec, } diff --git a/substreams/ethereum-balancer/substreams.yaml b/substreams/ethereum-balancer/substreams.yaml index 9919f23..66da244 100644 --- a/substreams/ethereum-balancer/substreams.yaml +++ b/substreams/ethereum-balancer/substreams.yaml @@ -7,7 +7,7 @@ protobuf: files: - tycho/evm/v1/vm.proto - tycho/evm/v1/common.proto - - tycho/evm/v1/balancer.proto + - tycho/evm/v1/utils.proto importPaths: - ../../proto