Add substreams integration template, readmes

2023-12-20 18:12:41 +01:00
parent 5f87b1d0af
commit 0f3a5b819c
14 changed files with 1549 additions and 53 deletions
--- a/substreams/Readme.md
+++ b/substreams/Readme.md
@@ -5,8 +5,10 @@ blockchains.

 ## Adding a new package

-To add a new package add folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. In this new folder add a manifest
-file `substreams.yaml`. You can use the template below to get started:
+To add a new package add folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. 
+
+### Manifest
+In this new folder add a manifest file `substreams.yaml`. You can use the template below to get started:

 ```yaml
 specVersion: v0.1.0
@@ -36,13 +38,12 @@ binaries:
    file: ../../target/wasm32-unknown-unknown/substreams/substreams_[CHAIN]_[PROTOCOL_SYSTEM].wasm

 modules:
-  # sample module provides access to blocks.
-  - name: map_block
+  - name: map_changes
    kind: map
    inputs:
      - source: sf.ethereum.type.v2.Block
    output:
-      type: proto:acme.block_meta.v1.BlockMeta
+      type: proto:tycho.evm.state.v1.BlockContractChanges
 ```

 Substreams packages are Rust crates so we also need a `cargo.toml`.
@@ -65,6 +66,9 @@ prost = "0.11"

 ```

+There are already some generated rust files in the `src/pb` directory. These are generated from the protobuf files in the 
+
+
 Now we can generate the Rust protobuf code:

 ```
--- a/substreams/ethereum-ambient/Cargo.lock
+++ b/substreams/ethereum-ambient/Cargo.lock
@@ -180,7 +180,24 @@ version = "17.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b"
 dependencies = [
- "ethereum-types",
+ "ethereum-types 0.13.1",
+ "hex",
+ "once_cell",
+ "regex",
+ "serde",
+ "serde_json",
+ "sha3",
+ "thiserror",
+ "uint",
+]
+
+[[package]]
+name = "ethabi"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7413c5f74cc903ea37386a8965a936cbeb334bd270862fdece542c1b2dcbc898"
+dependencies = [
+ "ethereum-types 0.14.1",
 "hex",
 "once_cell",
 "regex",
@@ -198,9 +215,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef"
 dependencies = [
 "crunchy",
- "fixed-hash",
+ "fixed-hash 0.7.0",
 "impl-rlp",
- "impl-serde",
+ "impl-serde 0.3.2",
+ "tiny-keccak",
+]
+
+[[package]]
+name = "ethbloom"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60"
+dependencies = [
+ "crunchy",
+ "fixed-hash 0.8.0",
+ "impl-rlp",
+ "impl-serde 0.4.0",
 "tiny-keccak",
 ]

@@ -210,11 +240,25 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6"
 dependencies = [
- "ethbloom",
- "fixed-hash",
+ "ethbloom 0.12.1",
+ "fixed-hash 0.7.0",
 "impl-rlp",
- "impl-serde",
- "primitive-types",
+ "impl-serde 0.3.2",
+ "primitive-types 0.11.1",
+ "uint",
+]
+
+[[package]]
+name = "ethereum-types"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02d215cbf040552efcbe99a38372fe80ab9d00268e20012b79fcd0f073edd8ee"
+dependencies = [
+ "ethbloom 0.13.0",
+ "fixed-hash 0.8.0",
+ "impl-rlp",
+ "impl-serde 0.4.0",
+ "primitive-types 0.12.2",
 "uint",
 ]

@@ -236,6 +280,18 @@ dependencies = [
 "static_assertions",
 ]

+[[package]]
+name = "fixed-hash"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534"
+dependencies = [
+ "byteorder",
+ "rand",
+ "rustc-hex",
+ "static_assertions",
+]
+
 [[package]]
 name = "fixedbitset"
 version = "0.4.2"
@@ -335,6 +391,15 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "impl-serde"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "impl-trait-for-tuples"
 version = "0.2.2"
@@ -519,10 +584,23 @@ version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a"
 dependencies = [
- "fixed-hash",
+ "fixed-hash 0.7.0",
 "impl-codec",
 "impl-rlp",
- "impl-serde",
+ "impl-serde 0.3.2",
+ "uint",
+]
+
+[[package]]
+name = "primitive-types"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2"
+dependencies = [
+ "fixed-hash 0.8.0",
+ "impl-codec",
+ "impl-rlp",
+ "impl-serde 0.4.0",
 "uint",
 ]

@@ -805,7 +883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97a176f39a6e09553c17a287edacd1854e5686fd20ffea3c9655dfc44d94b35e"
 dependencies = [
 "anyhow",
- "ethabi",
+ "ethabi 17.2.0",
 "heck",
 "hex",
 "prettyplease",
@@ -819,6 +897,10 @@ dependencies = [
 name = "substreams-ethereum-ambient"
 version = "0.3.0"
 dependencies = [
+ "anyhow",
+ "bytes",
+ "ethabi 18.0.0",
+ "hex",
 "hex-literal 0.4.1",
 "prost",
 "substreams",
@@ -832,7 +914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "db4700cfe408b75634a3c6b3a0caf7bddba4879601d2085c811485ea54cbde2d"
 dependencies = [
 "bigdecimal",
- "ethabi",
+ "ethabi 17.2.0",
 "getrandom",
 "num-bigint",
 "prost",
@@ -847,7 +929,7 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40d6d278d926fe3f0775d996ee2b5e1dc822c1b4bf4f7bf07c7fbb5bce6c79a9"
 dependencies = [
- "ethabi",
+ "ethabi 17.2.0",
 "heck",
 "hex",
 "num-bigint",
--- a/substreams/ethereum-ambient/src/lib.rs
+++ b/substreams/ethereum-ambient/src/lib.rs
@@ -3,9 +3,9 @@ use std::collections::{hash_map::Entry, HashMap};
 use anyhow::{anyhow, bail};
 use ethabi::{decode, ParamType};
 use hex_literal::hex;
-use substreams_ethereum::pb::eth::{self};
+use substreams_ethereum::pb::eth;

-use pb::tycho::evm::v1::{self as tycho, ChangeType};
+use pb::tycho::evm::v1::{self as tycho};

 mod pb;

@@ -52,24 +52,6 @@ impl From<InterimContractChange> for tycho::ContractChange {
 }

 /// Extracts all contract changes relevant to vm simulations
-///
-/// This implementation has currently two major limitations:
-/// 1. It is hardwired to only care about changes to the ambient main contract, this is ok for this
-///    particular use case but for a more general purpose implementation this is not ideal
-/// 2. Changes are processed separately, this means that if there are any side effects between each
-///    other (e.g. if account is deleted and then created again in ethereum all the storage is set
-///    to 0. So there is a side effect between account creation and contract storage.) these might
-///    not be properly accounted for. Most of the time this should not be a major issue but may lead
-///    to wrong results so consume this implementation with care. See example below for a concrete
-///    case where this is problematic.
-///
-/// ## A very contrived example:
-/// 1. Some existing contract receives a transaction that changes it state, the state is updated
-/// 2. Next, this contract has self destruct called on itself
-/// 3. The contract is created again using CREATE2 at the same address
-/// 4. The contract receives a transaction that changes it state
-/// 5. We would emit this as as contract creation with slots set from 1 and from 4, although we
-///    should only emit the slots changed from 4.
 #[substreams::handlers::map]
 fn map_changes(
    block: eth::v2::Block,
@@ -179,7 +161,7 @@ fn map_changes(
                            let static_attribute = tycho::Attribute {
                                name: String::from("pool_index"),
                                value: pool_index.to_be_bytes().to_vec(),
-                                change: ChangeType::Creation.into(),
+                                change: tycho::ChangeType::Creation.into(),
                            };

                            let mut tokens: Vec<Vec<u8>> = vec![base.clone(), quote.clone()];
@@ -195,7 +177,7 @@ fn map_changes(
                                tokens,
                                contracts: vec![hex::encode(AMBIENT_CONTRACT)],
                                static_att: vec![static_attribute],
-                                change: ChangeType::Creation.into(),
+                                change: tycho::ChangeType::Creation.into(),
                            };
                            tx_change
                                .component_changes
@@ -257,9 +239,9 @@ fn map_changes(
                        code: Vec::new(),
                        slots,
                        change: if created_accounts.contains_key(&storage_change.address) {
-                            ChangeType::Creation
+                            tycho::ChangeType::Creation
                        } else {
-                            ChangeType::Update
+                            tycho::ChangeType::Update
                        },
                    });
                }
@@ -298,9 +280,9 @@ fn map_changes(
                            code: Vec::new(),
                            slots: HashMap::new(),
                            change: if created_accounts.contains_key(&balance_change.address) {
-                                ChangeType::Creation
+                                tycho::ChangeType::Creation
                            } else {
-                                ChangeType::Update
+                                tycho::ChangeType::Update
                            },
                        });
                    }
@@ -337,9 +319,9 @@ fn map_changes(
                        code: code_change.new_code.clone(),
                        slots: HashMap::new(),
                        change: if created_accounts.contains_key(&code_change.address) {
-                            ChangeType::Creation
+                            tycho::ChangeType::Creation
                        } else {
-                            ChangeType::Update
+                            tycho::ChangeType::Update
                        },
                    });
                }
--- a/substreams/ethereum-template/Cargo.lock
+++ b/substreams/ethereum-template/Cargo.lock
--- a/substreams/ethereum-template/Cargo.toml
+++ b/substreams/ethereum-template/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "substreams-ethereum-template"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "substreams_ethereum_template"
+crate-type = ["cdylib"]
+
+[dependencies]
+substreams = "0.5"
+substreams-ethereum = "0.9"
+prost = "0.11"
--- a/substreams/ethereum-template/src/lib.rs
+++ b/substreams/ethereum-template/src/lib.rs
@@ -0,0 +1,13 @@
+
+use substreams_ethereum::pb::eth;
+
+use pb::tycho::evm::v1::{self as tycho};
+
+mod pb;
+
+#[substreams::handlers::map]
+fn map_changes(
+    block: eth::v2::Block,
+) -> Result<tycho::BlockContractChanges, substreams::errors::Error> {
+    todo!("Not implemented")
+}
--- a/substreams/ethereum-template/src/pb/mod.rs
+++ b/substreams/ethereum-template/src/pb/mod.rs
@@ -0,0 +1,10 @@
+// @generated
+pub mod tycho {
+    pub mod evm {
+        // @@protoc_insertion_point(attribute:tycho.evm.v1)
+        pub mod v1 {
+            include!("tycho.evm.v1.rs");
+            // @@protoc_insertion_point(tycho.evm.v1)
+        }
+    }
+}
--- a/substreams/ethereum-template/src/pb/tycho.evm.v1.rs
+++ b/substreams/ethereum-template/src/pb/tycho.evm.v1.rs
@@ -0,0 +1,166 @@
+// @generated
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Block {
+    #[prost(bytes="vec", tag="1")]
+    pub hash: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes="vec", tag="2")]
+    pub parent_hash: ::prost::alloc::vec::Vec<u8>,
+    #[prost(uint64, tag="3")]
+    pub number: u64,
+    #[prost(uint64, tag="4")]
+    pub ts: u64,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Transaction {
+    #[prost(bytes="vec", tag="1")]
+    pub hash: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes="vec", tag="2")]
+    pub from: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes="vec", tag="3")]
+    pub to: ::prost::alloc::vec::Vec<u8>,
+    #[prost(uint64, tag="4")]
+    pub index: u64,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Attribute {
+    #[prost(string, tag="1")]
+    pub name: ::prost::alloc::string::String,
+    #[prost(bytes="vec", tag="2")]
+    pub value: ::prost::alloc::vec::Vec<u8>,
+    #[prost(enumeration="ChangeType", tag="3")]
+    pub change: i32,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProtocolComponent {
+    #[prost(string, tag="1")]
+    pub id: ::prost::alloc::string::String,
+    #[prost(bytes="vec", repeated, tag="2")]
+    pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec<u8>>,
+    #[prost(string, repeated, tag="3")]
+    pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    #[prost(message, repeated, tag="4")]
+    pub static_att: ::prost::alloc::vec::Vec<Attribute>,
+    #[prost(enumeration="ChangeType", tag="5")]
+    pub change: i32,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BalanceChange {
+    #[prost(bytes="vec", tag="1")]
+    pub token: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes="vec", tag="2")]
+    pub balance: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes="vec", tag="3")]
+    pub component_id: ::prost::alloc::vec::Vec<u8>,
+}
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
+#[repr(i32)]
+pub enum ChangeType {
+    Unspecified = 0,
+    Update = 1,
+    Creation = 2,
+    Deletion = 3,
+}
+impl ChangeType {
+    /// String value of the enum field names used in the ProtoBuf definition.
+    ///
+    /// The values are not transformed in any way and thus are considered stable
+    /// (if the ProtoBuf definition does not change) and safe for programmatic use.
+    pub fn as_str_name(&self) -> &'static str {
+        match self {
+            ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED",
+            ChangeType::Update => "CHANGE_TYPE_UPDATE",
+            ChangeType::Creation => "CHANGE_TYPE_CREATION",
+            ChangeType::Deletion => "CHANGE_TYPE_DELETION",
+        }
+    }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified),
+            "CHANGE_TYPE_UPDATE" => Some(Self::Update),
+            "CHANGE_TYPE_CREATION" => Some(Self::Creation),
+            "CHANGE_TYPE_DELETION" => Some(Self::Deletion),
+            _ => None,
+        }
+    }
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct EntityChanges {
+    #[prost(string, tag="1")]
+    pub component_id: ::prost::alloc::string::String,
+    #[prost(message, repeated, tag="2")]
+    pub attributes: ::prost::alloc::vec::Vec<Attribute>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TransactionEntityChanges {
+    #[prost(message, optional, tag="1")]
+    pub tx: ::core::option::Option<Transaction>,
+    #[prost(message, repeated, tag="2")]
+    pub entity_changes: ::prost::alloc::vec::Vec<EntityChanges>,
+    #[prost(message, repeated, tag="3")]
+    pub component_changes: ::prost::alloc::vec::Vec<ProtocolComponent>,
+    #[prost(message, repeated, tag="4")]
+    pub balance_changes: ::prost::alloc::vec::Vec<BalanceChange>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BlockEntityChanges {
+    #[prost(message, optional, tag="1")]
+    pub block: ::core::option::Option<Block>,
+    #[prost(message, repeated, tag="2")]
+    pub changes: ::prost::alloc::vec::Vec<TransactionEntityChanges>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ContractSlot {
+    #[prost(bytes="vec", tag="2")]
+    pub slot: ::prost::alloc::vec::Vec<u8>,
+    #[prost(bytes="vec", tag="3")]
+    pub value: ::prost::alloc::vec::Vec<u8>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ContractChange {
+    #[prost(bytes="vec", tag="1")]
+    pub address: ::prost::alloc::vec::Vec<u8>,
+    /// empty bytes indicates no change
+    #[prost(bytes="vec", tag="2")]
+    pub balance: ::prost::alloc::vec::Vec<u8>,
+    /// empty bytes indicates no change
+    #[prost(bytes="vec", tag="3")]
+    pub code: ::prost::alloc::vec::Vec<u8>,
+    /// empty sequence indicates no change
+    #[prost(message, repeated, tag="4")]
+    pub slots: ::prost::alloc::vec::Vec<ContractSlot>,
+    /// Whether this is an update, creation or deletion
+    #[prost(enumeration="ChangeType", tag="5")]
+    pub change: i32,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TransactionContractChanges {
+    #[prost(message, optional, tag="1")]
+    pub tx: ::core::option::Option<Transaction>,
+    #[prost(message, repeated, tag="2")]
+    pub contract_changes: ::prost::alloc::vec::Vec<ContractChange>,
+    #[prost(message, repeated, tag="3")]
+    pub component_changes: ::prost::alloc::vec::Vec<ProtocolComponent>,
+    #[prost(message, repeated, tag="4")]
+    pub balance_changes: ::prost::alloc::vec::Vec<BalanceChange>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BlockContractChanges {
+    #[prost(message, optional, tag="1")]
+    pub block: ::core::option::Option<Block>,
+    #[prost(message, repeated, tag="2")]
+    pub changes: ::prost::alloc::vec::Vec<TransactionContractChanges>,
+}
+// @@protoc_insertion_point(module)
--- a/substreams/ethereum-template/substreams.yaml
+++ b/substreams/ethereum-template/substreams.yaml
@@ -0,0 +1,24 @@
+specVersion: v0.1.0
+package:
+  name: "substreams_ethereum_template"
+  version: v0.1.0
+
+protobuf:
+  files:
+    - vm.proto
+    - common.proto
+  importPaths:
+    - ../../proto/tycho/evm/v1/
+
+binaries:
+  default:
+    type: wasm/rust-v1
+    file: ../../target/wasm32-unknown-unknown/substreams/substreams_ethereum_template.wasm
+
+modules:
+  - name: map_changes
+    kind: map
+    inputs:
+      - source: sf.ethereum.type.v2.Block
+    output:
+      type: proto:tycho.evm.state.v1.BlockContractChanges