Merge pull request #18 from propeller-heads/ah/template-and-docs

Update template and docs

.gitignore

@@ -11,3 +11,5 @@ target/
 .vscode
 .idea
 *.log
+
+substreams/ethereum-template/Cargo.lock

substreams/Readme.md

@@ -1,99 +1,24 @@
-# Subtreams packages
+# Substreams Indexing Integrations
 
-This directory contains all substream packages that are used to index integrated protocols across different blockchains.
+Please refer to the official [Substreams Indexing](https://app.gitbook.com/o/9wMvRDQIhk1xOsIZ0Zde/s/Yx9kvxtpT2xWdzvFiB3t/indexing/substreams-integration) docs.
 
-## Adding a new package
+## Release
 
-To add a new package add a folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. 
+To release a package simply tag a commit with the package name and its version: 
+e.g. `ethereum-balancer-0.1.0`. This will create a release and automatically build 
+and push the spkg into our registry.
 
-### Manifest
+### Note
-In this new folder add a manifest file `substreams.yaml`. You can use the template below to get started:
+The CD pipeline will error if the Cargo version is not the same as the version in 
+the tag.
 
-```yaml
+Releases are immutable so do no try to delete tags or build the same release twice 
-specVersion: v0.1.0
+since this will error.
-package:
-  name: 'substreams_[CHAIN]_[PROTOCOL_SYSTEM]'
-  version: v0.1.0
 
-protobuf:
+### Pre release
-  files:
-    - vm.proto
-    - common.proto
-    # You can specify any internal proto files here
-  importPaths:
-    - ../../proto/tycho/evm/v1/
-    # Any private message types only used in internal modules 
-    # can remain local to the folder.
-    - ./proto
 
-binaries:
+To create a pre release for testing in dev you can start CD pipeline manually supplying 
-  default:
+the package you'd like to pre release. This will create a 
-    type: wasm/rust-v1
+`[package]-[semver].pre-[commit-sha]` release in our spkg repository which you can use 
-    # this points to the workspace target directory we use a special 
+to run the substream´.
-    # substreams build profile to optimise wasm binaries
-    file: ../../target/wasm32-unknown-unknown/substreams/substreams_[CHAIN]_[PROTOCOL_SYSTEM].wasm
 
-modules:
-  - name: map_changes
-    kind: map
-    inputs:
-      - source: sf.ethereum.type.v2.Block
-    output:
-      type: proto:tycho.evm.state.v1.BlockContractChanges
-```
-
-Substreams packages are Rust crates so we also need a `cargo.toml`.
-The example from the official docs will serve us just well:
-
-```toml
-[package]
-name = "substreams-[CHAIN]-[PROTOCOL_SYSTEM]"
-version = "0.1.0"
-edition = "2021"
-
-[lib]
-name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]"
-crate-type = ["cdylib"]
-
-[dependencies]
-substreams = "0.5"
-substreams-ethereum = "0.9"
-prost = "0.11"
-
-```
-
-There are already some generated rust files in the `src/pb` directory. These are generated 
-from the protobuf files in the `/proto/tycho/evm/v1` directory. They specify the output protobuf messages
-we want to generate. The input Block is specified by the subtreams crate, specifically the [sf.ethereum.type.v2.Block](https://github.com/streamingfast/substreams-ethereum/blob/develop/core/src/pb/sf.ethereum.type.v2.rs) message.
-
-You can define your own protobuf messages, make a new directory `/substreams/[CHAIN]-[PROTOCOL]/proto` for them.
-
-
-Now we can generate the Rust protobuf code:
-
-```
-substreams protogen substreams.yaml --exclude-paths="sf/substreams,google"
-```
-
-The command above should put the generate rust files under `/src/pb`. You
-can start using these now in your module handlers: See
-the [official substreams documentation](https://thegraph.com/docs/en/substreams/getting-started/quickstart/#create-substreams-module-handlers)
-on
-how to implement module handlers.
-
-You can also look into already existing substreams packages to see how it
-is done. E.g. [ethereum-ambient](./ethereum-ambient/) provides a pretty good
-example of how to get access to raw contract storage.
-
-# Tests
-
-To create a block test asset for ethereum do the following:
-
-- Follow [this tutorial](https://substreams.streamingfast.io/tutorials/overview/map_block_meta_module). Make sure you
-  set up the substreams-explorer repo in the same directory as this repo.
-    - Comment out `image: ./ethereum.png` in `ethereum-explorer/substreams.yaml`
-    - Add `prost-types = "0.11.0"` to `ethereum-explorer/Cargo.toml`
-- Make sure you set up your key env vars.
-- Run `sh scripts/download-ethereum-block-to-s3 BLOCK_NUMBER`
-
-Do not commit the block files (they are quite big).

substreams/ethereum-balancer/src/modules.rs

@@ -15,7 +15,7 @@ use tycho_substreams::{
 const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8");
 
 #[substreams::handlers::map]
-pub fn map_pools_created(block: eth::v2::Block) -> Result<BlockTransactionProtocolComponents> {
+pub fn map_components(block: eth::v2::Block) -> Result<BlockTransactionProtocolComponents> {
     // Gather contract changes by indexing `PoolCreated` events and analysing the `Create` call
     // We store these as a hashmap by tx hash since we need to agg by tx hash later
     Ok(BlockTransactionProtocolComponents {
@@ -47,7 +47,7 @@ pub fn map_pools_created(block: eth::v2::Block) -> Result<BlockTransactionProtoc
 
 /// Simply stores the `ProtocolComponent`s with the pool id as the key
 #[substreams::handlers::store]
-pub fn store_pools_created(map: BlockTransactionProtocolComponents, store: StoreAddInt64) {
+pub fn store_components(map: BlockTransactionProtocolComponents, store: StoreAddInt64) {
     store.add_many(
         0,
         &map.tx_components
@@ -62,7 +62,7 @@ pub fn store_pools_created(map: BlockTransactionProtocolComponents, store: Store
 /// Since the `PoolBalanceChanged` and `Swap` events administer only deltas, we need to leverage a
 /// map and a  store to be able to tally up final balances for tokens in a pool.
 #[substreams::handlers::map]
-pub fn map_balance_deltas(
+pub fn map_relative_balances(
     block: eth::v2::Block,
     store: StoreGetInt64,
 ) -> Result<BlockBalanceDeltas, anyhow::Error> {
@@ -133,7 +133,7 @@ pub fn map_balance_deltas(
 /// It's significant to include both the `pool_id` and the `token_id` for each balance delta as the
 ///  store key to ensure that there's a unique balance being tallied for each.
 #[substreams::handlers::store]
-pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: StoreAddBigInt) {
+pub fn store_balances(deltas: BlockBalanceDeltas, store: StoreAddBigInt) {
     tycho_substreams::balances::store_balance_changes(deltas, store);
 }
 
@@ -144,7 +144,7 @@ pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: StoreAddBigInt)
 ///  At the very end, the map can easily be sorted by index to ensure the final
 /// `BlockContractChanges`  is ordered by transactions properly.
 #[substreams::handlers::map]
-pub fn map_changes(
+pub fn map_protocol_changes(
     block: eth::v2::Block,
     grouped_components: BlockTransactionProtocolComponents,
     deltas: BlockBalanceDeltas,

substreams/ethereum-balancer/substreams.yaml

@@ -9,7 +9,7 @@ binaries:
     file: ../target/wasm32-unknown-unknown/release/ethereum_balancer.wasm
 
 modules:
-  - name: map_pools_created
+  - name: map_components
     kind: map
     initialBlock: 12369300
     inputs:
@@ -17,40 +17,40 @@ modules:
     output:
       type: proto:tycho.evm.v1.GroupedTransactionProtocolComponents
 
-  - name: store_pools_created
+  - name: store_components
     kind: store
     initialBlock: 12369300
     updatePolicy: add
     valueType: int64
     inputs:
-      - map: map_pools_created
+      - map: map_components
 
-  - name: map_balance_deltas
+  - name: map_relative_balances
     kind: map
     initialBlock: 12369300 # An arbitrary block that should change based on your requirements
     inputs:
       - source: sf.ethereum.type.v2.Block
-      - store: store_pools_created
+      - store: store_components
     output:
       type: proto:tycho.evm.v1.BalanceDeltas
 
-  - name: store_balance_changes
+  - name: store_balances
     kind: store
     initialBlock: 12369300
     updatePolicy: add
     valueType: bigint
     inputs:
-      - map: map_balance_deltas
+      - map: map_relative_balances
 
   - name: map_changes
     kind: map
     initialBlock: 12369300
     inputs:
       - source: sf.ethereum.type.v2.Block
-      - map: map_pools_created
+      - map: map_components
-      - map: map_balance_deltas
+      - map: map_relative_balances
-      - store: store_pools_created
+      - store: store_components
-      - store: store_balance_changes
+      - store: store_balances
         mode: deltas # This is the key property that simplifies `BalanceChange` handling
     output:
       type: proto:tycho.evm.v1.BlockContractChanges

substreams/ethereum-template/Cargo.toml

@@ -1,13 +1,14 @@
 [package]
-name = "substreams-ethereum-template"
+name = "ethereum-template"
 version = "0.1.0"
 edition = "2021"
 
 [lib]
-name = "substreams_ethereum_template"
+name = "ethereum_template"
 crate-type = ["cdylib"]
 
 [dependencies]
-substreams = "0.5"
+substreams.workspace = true
-substreams-ethereum = "0.9"
+substreams-ethereum.workspace = true
-prost = "0.11"
+prost.workspace = true
+tycho-substreams.workspace = true

substreams/ethereum-template/src/lib.rs

@@ -1,13 +1,9 @@
-
 use substreams_ethereum::pb::eth;
-
+mod modules;
-use pb::tycho::evm::v1::{self as tycho};
-
-mod pb;
 
 #[substreams::handlers::map]
 fn map_changes(
     block: eth::v2::Block,
 ) -> Result<tycho::BlockContractChanges, substreams::errors::Error> {
     todo!("Not implemented")
-}
+}

substreams/ethereum-template/src/modules.rs

@@ -0,0 +1,12 @@
+use std::collections::HashMap;
+use substreams_ethereum::pb::eth;
+use tycho_substreams::prelude::*;
+
+#[substreams::handlers::map]
+fn map_protocol_changes(
+    block: eth::v2::Block,
+) -> Result<BlockContractChanges, substreams::errors::Error> {
+    let mut transaction_contract_changes = Vec::<TransactionContractChanges>::new();
+    // TODO: protocol specific logic goes here
+    Ok(BlockContractChanges { block: Some((&block).into()), changes: transaction_contract_changes })
+}

substreams/ethereum-template/src/pb/mod.rs

@@ -1,10 +0,0 @@
-// @generated
-pub mod tycho {
-    pub mod evm {
-        // @@protoc_insertion_point(attribute:tycho.evm.v1)
-        pub mod v1 {
-            include!("tycho.evm.v1.rs");
-            // @@protoc_insertion_point(tycho.evm.v1)
-        }
-    }
-}

substreams/ethereum-template/src/pb/tycho.evm.v1.rs

@@ -1,183 +0,0 @@
-// @generated
-// This file contains the proto definitions for Substreams common to all integrations.
-
-/// A struct describing a block.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Block {
-    /// The blocks hash.
-    #[prost(bytes="vec", tag="1")]
-    pub hash: ::prost::alloc::vec::Vec<u8>,
-    /// The parent blocks hash.
-    #[prost(bytes="vec", tag="2")]
-    pub parent_hash: ::prost::alloc::vec::Vec<u8>,
-    /// The block number.
-    #[prost(uint64, tag="3")]
-    pub number: u64,
-    /// The block timestamp.
-    #[prost(uint64, tag="4")]
-    pub ts: u64,
-}
-/// A struct describing a transaction.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Transaction {
-    /// The transaction hash.
-    #[prost(bytes="vec", tag="1")]
-    pub hash: ::prost::alloc::vec::Vec<u8>,
-    /// The sender of the transaction.
-    #[prost(bytes="vec", tag="2")]
-    pub from: ::prost::alloc::vec::Vec<u8>,
-    /// The receiver of the transaction.
-    #[prost(bytes="vec", tag="3")]
-    pub to: ::prost::alloc::vec::Vec<u8>,
-    /// The transactions index within the block.
-    #[prost(uint64, tag="4")]
-    pub index: u64,
-}
-/// A custom struct representing an arbitrary attribute of a protocol component.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct Attribute {
-    /// The name of the attribute.
-    #[prost(string, tag="1")]
-    pub name: ::prost::alloc::string::String,
-    /// The value of the attribute.
-    #[prost(bytes="vec", tag="2")]
-    pub value: ::prost::alloc::vec::Vec<u8>,
-    /// The type of change the attribute underwent.
-    #[prost(enumeration="ChangeType", tag="3")]
-    pub change: i32,
-}
-/// A struct describing a part of the protocol.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ProtocolComponent {
-    /// A unique identifier for the component within the protocol.
-    /// Can be a stringified address or a string describing the trading pair.
-    #[prost(string, tag="1")]
-    pub id: ::prost::alloc::string::String,
-    /// Addresses of the ERC20 tokens used by the component.
-    #[prost(bytes="vec", repeated, tag="2")]
-    pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
-    /// Addresses of the contracts used by the component.
-    #[prost(bytes="vec", repeated, tag="3")]
-    pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
-    /// Attributes of the component.
-    /// The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent.
-    #[prost(message, repeated, tag="4")]
-    pub static_att: ::prost::alloc::vec::Vec<Attribute>,
-    /// Type of change the component underwent.
-    #[prost(enumeration="ChangeType", tag="5")]
-    pub change: i32,
-}
-/// A struct for following the changes of Total Value Locked (TVL) of a protocol component.
-/// Note that if the ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct BalanceChange {
-    /// The address of the ERC20 token whose balance changed.
-    #[prost(bytes="vec", tag="1")]
-    pub token: ::prost::alloc::vec::Vec<u8>,
-    /// The new balance of the token.
-    #[prost(bytes="vec", tag="2")]
-    pub balance: ::prost::alloc::vec::Vec<u8>,
-    /// The id of the component whose TVL is tracked.
-    #[prost(bytes="vec", tag="3")]
-    pub component_id: ::prost::alloc::vec::Vec<u8>,
-}
-/// Enum to specify the type of a change.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
-#[repr(i32)]
-pub enum ChangeType {
-    Unspecified = 0,
-    Update = 1,
-    Creation = 2,
-    Deletion = 3,
-}
-impl ChangeType {
-    /// String value of the enum field names used in the ProtoBuf definition.
-    ///
-    /// The values are not transformed in any way and thus are considered stable
-    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
-    pub fn as_str_name(&self) -> &'static str {
-        match self {
-            ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED",
-            ChangeType::Update => "CHANGE_TYPE_UPDATE",
-            ChangeType::Creation => "CHANGE_TYPE_CREATION",
-            ChangeType::Deletion => "CHANGE_TYPE_DELETION",
-        }
-    }
-    /// Creates an enum from field names used in the ProtoBuf definition.
-    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
-        match value {
-            "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified),
-            "CHANGE_TYPE_UPDATE" => Some(Self::Update),
-            "CHANGE_TYPE_CREATION" => Some(Self::Creation),
-            "CHANGE_TYPE_DELETION" => Some(Self::Deletion),
-            _ => None,
-        }
-    }
-}
-// This file contains proto definitions specific to the VM integration.
-
-/// A key value entry into contract storage.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ContractSlot {
-    /// A contract's storage slot.
-    #[prost(bytes="vec", tag="2")]
-    pub slot: ::prost::alloc::vec::Vec<u8>,
-    /// The new value for this storage slot.
-    #[prost(bytes="vec", tag="3")]
-    pub value: ::prost::alloc::vec::Vec<u8>,
-}
-/// Changes made to a single contract's state.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct ContractChange {
-    /// The contract's address
-    #[prost(bytes="vec", tag="1")]
-    pub address: ::prost::alloc::vec::Vec<u8>,
-    /// The new balance of the contract, empty bytes indicates no change.
-    #[prost(bytes="vec", tag="2")]
-    pub balance: ::prost::alloc::vec::Vec<u8>,
-    /// The new code of the contract, empty bytes indicates no change.
-    #[prost(bytes="vec", tag="3")]
-    pub code: ::prost::alloc::vec::Vec<u8>,
-    /// The changes to this contract's slots, empty sequence indicates no change.
-    #[prost(message, repeated, tag="4")]
-    pub slots: ::prost::alloc::vec::Vec<ContractSlot>,
-    /// Whether this is an update, a creation or a deletion.
-    #[prost(enumeration="ChangeType", tag="5")]
-    pub change: i32,
-}
-/// A set of changes aggregated by transaction.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct TransactionContractChanges {
-    /// The transaction instance that results in the changes.
-    #[prost(message, optional, tag="1")]
-    pub tx: ::core::option::Option<Transaction>,
-    /// Contains the changes induced by the above transaction, aggregated on a per-contract basis.
-    #[prost(message, repeated, tag="2")]
-    pub contract_changes: ::prost::alloc::vec::Vec<ContractChange>,
-    /// An array of newly added components.
-    #[prost(message, repeated, tag="3")]
-    pub component_changes: ::prost::alloc::vec::Vec<ProtocolComponent>,
-    /// An array of balance changes to components.
-    #[prost(message, repeated, tag="4")]
-    pub balance_changes: ::prost::alloc::vec::Vec<BalanceChange>,
-}
-/// A set of transaction changes within a single block.
-#[allow(clippy::derive_partial_eq_without_eq)]
-#[derive(Clone, PartialEq, ::prost::Message)]
-pub struct BlockContractChanges {
-    /// The block for which these changes are collectively computed.
-    #[prost(message, optional, tag="1")]
-    pub block: ::core::option::Option<Block>,
-    /// The set of transaction changes observed in the specified block.
-    #[prost(message, repeated, tag="2")]
-    pub changes: ::prost::alloc::vec::Vec<TransactionContractChanges>,
-}
-// @@protoc_insertion_point(module)

substreams/ethereum-template/substreams.yaml

@@ -1,22 +1,15 @@
 specVersion: v0.1.0
 package:
-  name: "substreams_ethereum_template"
+  name: "ethereum_template"
   version: v0.1.0
 
-protobuf:
-  files:
-    - vm.proto
-    - common.proto
-  importPaths:
-    - ../../proto/tycho/evm/v1/
-
 binaries:
   default:
     type: wasm/rust-v1
-    file: ../../target/wasm32-unknown-unknown/substreams/substreams_ethereum_template.wasm
+    file: ../target/wasm32-unknown-unknown/release/ethereum_template.wasm
 
 modules:
-  - name: map_changes
+  - name: map_protocol_changes
     kind: map
     inputs:
       - source: sf.ethereum.type.v2.Block