First version of the Ethereum explorer

ethereum-explorer/.gitpod.yml

@@ -0,0 +1,10 @@
+image: ghcr.io/graphprotocol/substreams-gitpod:latest
+tasks:
+  - command: |
+      # Authenticate with the substreams server
+      export SUBSTREAMS_API_TOKEN=$(curl https://auth.dfuse.io/v1/auth/issue -s --data-binary '{"api_key":"'$STREAMINGFAST_KEY'"}' | jq -r .token)
+ports:
+  - port: 6060
+    onOpen: ignore
+  - port: 1065
+    onOpen: ignore

ethereum-explorer/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "substreams-ethereum-explorer"
+version = "0.1.0"
+description = "Substreams showcasing the building blocks of Ethereum"
+edition = "2021"
+repository = "https://github.com/streamingfast/substreams-explorers"
+license = "Apache 2.0"
+
+[lib]
+name = "substreams"
+crate-type = ["cdylib"]
+
+[dependencies]
+ethabi = "17"
+hex-literal = "0.3"
+num-bigint = "0.4"
+prost = "0.11"
+# Use latest from https://crates.io/crates/substreams
+substreams = "0.5"
+# Use latest from https://crates.io/crates/substreams-ethereum
+substreams-ethereum = "0.9"
+
+# Required so that ethabi > ethereum-types build correctly under wasm32-unknown-unknown
+[target.wasm32-unknown-unknown.dependencies]
+getrandom = { version = "0.2", features = ["custom"] }
+
+[build-dependencies]
+anyhow = "1"
+substreams-ethereum = "0.9"
+
+[profile.release]
+lto = true
+opt-level = 's'
+strip = "debuginfo"

ethereum-explorer/Makefile

@@ -0,0 +1,30 @@
+ENDPOINT ?= mainnet.eth.streamingfast.io:443
+START_BLOCK ?= 17717997
+#17712040
+#17717997
+STOP_BLOCK ?= +50
+
+.PHONY: build
+build:
+	cargo build --target wasm32-unknown-unknown --release
+
+.PHONY: stream
+stream: build
+	substreams run -e $(ENDPOINT) substreams.yaml map_transfers -s $(START_BLOCK) -t $(STOP_BLOCK)
+
+.PHONY: stream1
+stream1: build
+	substreams run -e $(ENDPOINT) substreams.yaml map_events -s $(START_BLOCK) -t $(STOP_BLOCK)
+
+.PHONY: stream2
+stream2: build
+	substreams run -e $(ENDPOINT) substreams.yaml map_block_meta -s $(START_BLOCK) -t $(STOP_BLOCK)
+
+
+.PHONY: protogen
+protogen:
+	substreams protogen ./substreams.yaml --exclude-paths="sf/substreams,google"
+
+.PHONY: stream
+package: build
+	substreams package substreams.yaml

ethereum-explorer/README.md

@@ -0,0 +1,174 @@
+# Ethereum Explorer
+
+The Ethereum Explorer consists of several Substreams modules showcasing the most basic operations that you can perform with Substreams on the Ethereum blockchain.
+
+## Before You Begin
+
+Make sure you have the [Substreams CLI installed](https://substreams.streamingfast.io/getting-started/installing-the-cli), and you know the [basic structure of a Substreams module](https://substreams.streamingfast.io/getting-started/quickstart).
+
+## Modules
+
+The modules in this repository answer some interesting questions when developing a blockchain application:
+
+### How Can You Get the Basic Information of a Block?
+
+For every block, the `map_block_meta` module retrieves the most relevant information of the block (number, hash, and parent hash).
+
+### How Can You Retrieve a Specific Transaction By Its Hash?
+
+Given a transaction hash parameter (`tx_hash`), the `map_filter_transaction` filters a transaction among all transactions in the blockchain. This involves:
+
+1. Iterating over all the transactions.
+2. Filter the transactions, where the `hash` field is equal to the hash parameter (`hash == tx_hash`).
+
+### How Can You Retrieve All the Events for a Specific Smart Contract?
+
+Given a smart contract address parameter (`contract_address`), the `map_contract_events` module retrieves all the events related to a specific smart contract. This involves:
+
+1. Iterating over all the smart contract transactions,
+2. Filtering the transactions, where the `to` field is equal to the smart contract address parameter (`to == contract_address`).
+3. For every filtered transaction, retrieve the logs.
+
+## Running the Substreams
+
+First, generate the Protobuf code, which are the outputs of the Substreams:
+
+```
+> make protogen
+```
+
+Then, build the Rust code using the `cargo` command-line tool:
+
+```
+> make build
+```
+
+Now, you are ready to run the Substreams. The Substreams contained in this project are independent of each other, so you must specify which Substreams module you want to run.
+
+### Running the "map_block_meta" Module
+
+In the following command, you retrieve the metadata of the block with number `17712040`. You specify the starting block by using the `--start-block` parameter.
+
+```bash
+> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_block_meta --start-block 17712040 --stop-block +1
+Connected (trace ID e98f04cd7ebb6191befbbf2e6668eafc)
+Progress messages received: 0 (0/sec)
+Backprocessing history up to requested target block 17712040:
+(hit 'm' to switch mode)
+
+----------- BLOCK #17,712,040 (31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b) ---------------
+{
+  "@module": "map_block_meta",
+  "@block": 17712040,
+  "@type": "eth.block_meta.v1.BlockMeta",
+  "@data": {
+    "number": "17712040",
+    "hash": "31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b",
+    "parentHash": "1385f853d28b16ad7ebc5d51b6f2ef6d43df4b57bd4c6fe4ef8ccb6f266d8b91"
+  }
+}
+
+all done
+```
+
+### Running the "map_filter_transaction" Module
+
+To run this module, you must provide a transaction hash, so that Substreams can filter the transactions accordingly. The Substreams manifest (`substreams.yaml`) contains a default transaction hash (`4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7`) in the `params` section, which you can update.
+
+The `4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7` transaction is at block number `17712040`. In order to avoid iterating over the full blockchain, the following command starts searching at block number `17712038`.
+
+```bash
+> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_filter_transaction --start-block 17712038 --stop-block +10
+Connected (trace ID 9dce06621a0ec353213adeaf9f10ef79)
+Progress messages received: 0 (0/sec)
+Backprocessing history up to requested target block 17712038:
+(hit 'm' to switch mode)
+
+----------- BLOCK #17,712,038 (b96fc7e71c0daf69b19211c45fbb5c201f4356fb2b5607500b7d88d298599f5b) ---------------
+----------- BLOCK #17,712,039 (1385f853d28b16ad7ebc5d51b6f2ef6d43df4b57bd4c6fe4ef8ccb6f266d8b91) ---------------
+----------- BLOCK #17,712,040 (31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b) ---------------
+{
+  "@module": "map_filter_transaction",
+  "@block": 17712040,
+  "@type": "eth.transaction.v1.TransactionOption",
+  "@data": {
+    "transaction": {
+      "from": "e93685f3bba03016f02bd1828badd6195988d950",
+      "to": "902f09715b6303d4173037652fa7377e5b98089e",
+      "hash": "4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7"
+    }
+  }
+}
+
+----------- BLOCK #17,712,041 (1af541642176c51580b54de214e955fba8bf1b82af569b81d4038956f2402a41) ---------------
+----------- BLOCK #17,712,042 (43c6ebe5b89dd689a9f07468a04d0faf5274a46d0763056ea53b8b1e5ac32148) ---------------
+----------- BLOCK #17,712,043 (2457f742913dbbdb171a8d8cc3b1ef8a383f8f547982700a646aa97581bfaeb8) ---------------
+----------- BLOCK #17,712,044 (af7abef4f80d7c6f3111e761bb86f305e9847e544fb36b55ba5b64e6103bd5d3) ---------------
+----------- BLOCK #17,712,045 (db4664944f7ca8aed5de798bf74ebbdbbeda60e58316b4291bfec61c7287fb17) ---------------
+----------- BLOCK #17,712,046 (d9bce9b9210b7deb746720435d1eca99b87fe17aaf7d5055fcd54959e0c9932e) ---------------
+----------- BLOCK #17,712,047 (3a5ffab8dacbaf89e10b66e9d2f7ebe65bddae4dcb5e5e8739f8b938f16f98ec) ---------------
+all done
+```
+
+As you can see, the Substreams does not provide an output for the blocks where the transaction is not present.
+You can check out the transaction at [Etherscan](https://etherscan.io/tx/0x4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7).
+
+### Running the "map_filter_transaction" Module
+
+To run this module, you must provide the address of a smart contract. By default, the `params` section of the Substreams manifest contains the `bc4ca0eda7647a8ab7c2061c2e118a18a936f13d` address, corresponding to the Bore Yacht Club smart contract. You can change this address to track any smart contract.
+
+The logs of a smart contract might be split across thousands of Ethereum blocks, so for testing purposes, and to avoid iterating over the full blockchain, the following command limits starts searching at block number `17717995`.
+
+```bash
+> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_contract_events --start-block 17717995 --stop-block +10
+Connected (trace ID 834915f19a2b07a8a3aa0159cbbf56da)
+Progress messages received: 0 (0/sec)
+Backprocessing history up to requested target block 17717995:
+(hit 'm' to switch mode)
+
+----------- BLOCK #17,717,995 (bfecb26963a2cd77700754612185e0074fc9589d2d73abb90e362fe9e7969451) ---------------
+----------- BLOCK #17,717,996 (7bf431a4f9df67e1d7e385d9a6cba41c658e66a77f0eb926163a7bbf6619ce20) ---------------
+----------- BLOCK #17,717,997 (fa5a57231348f1f138cb71207f0cdcc4a0a267e2688aa63ebff14265b8dae275) ---------------
+map_contract_events: log: //////////////////
+{
+  "@module": "map_contract_events",
+  "@block": 17717997,
+  "@type": "eth.event.v1.Events",
+  "@data": {
+    "events": [
+      {
+        "address": "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d",
+        "topics": [
+          "8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925",
+          "000000000000000000000000e2a83b15fc300d8457eb9e176f98d92a8ff40a49",
+          "0000000000000000000000000000000000000000000000000000000000000000",
+          "00000000000000000000000000000000000000000000000000000000000026a7"
+        ],
+        "txHash": "f18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939"
+      },
+      {
+        "address": "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d",
+        "topics": [
+          "ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
+          "000000000000000000000000e2a83b15fc300d8457eb9e176f98d92a8ff40a49",
+          "000000000000000000000000c67db0df922238979da0fd00d46016e8ae14cecb",
+          "00000000000000000000000000000000000000000000000000000000000026a7"
+        ],
+        "txHash": "f18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939"
+      }
+    ]
+  }
+}
+
+----------- BLOCK #17,717,998 (372ff635821a434c81759b3b23e8dac59393fc27a7ebb88b561c1e5da3c4643a) ---------------
+----------- BLOCK #17,717,999 (43f0878e119836cc789ecaf12c3280b82dc49567600cc44f6a042149e2a03779) ---------------
+----------- BLOCK #17,718,000 (439efaf9cc0059890a09d34b4cb5a3fe4b61e8ef96ee67673c060d58ff951d4f) ---------------
+----------- BLOCK #17,718,001 (c97ca5fd26db28128b0ec2483645348bbfe998e9a6e19e3a442221198254c9ea) ---------------
+----------- BLOCK #17,718,002 (9398569e46a954378b16e0e7ce95e49d0f21e6119ed0e3ab84f1c91f16c0c30e) ---------------
+----------- BLOCK #17,718,003 (80bcd4c1131c35a413c32903ffa52a14f8c8fe712492a8f6a0feddbb03b10bba) ---------------
+----------- BLOCK #17,718,004 (d27309ac29fe47f09fa4987a318818c325403863a53eec6a3676c2c2f8c069d9) ---------------
+all done
+```
+
+The block number `17717997` contains a transaction (`f18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939`) that executes a method of the smart contract, so the Substreams retrieves the corresponding logs.
+You can check out the logs at [Etherscan](https://etherscan.io/tx/0xf18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939#eventlog) and verify that topics are correct.

ethereum-explorer/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+cargo build --target wasm32-unknown-unknown --release

ethereum-explorer/proto/block_meta.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package eth.block_meta.v1;
+
+message BlockMeta {
+  uint64 number = 1;
+  string hash = 2;
+  string parent_hash = 3;
+}

ethereum-explorer/proto/event.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package eth.event.v1;
+
+message Events {
+    repeated Event events = 1;
+}
+
+message Event {
+    string address = 1;
+    repeated string topics = 2;
+    string tx_hash = 3;
+}

ethereum-explorer/proto/transaction.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package eth.transaction.v1;
+
+message TransactionOption {
+  Transaction transaction = 1;
+}
+
+message Transaction {
+  string from = 1;
+  string to = 2;
+  string hash = 3;
+}

ethereum-explorer/rust-toolchain.toml

@@ -0,0 +1,4 @@
+[toolchain]
+channel = "1.65"
+components = [ "rustfmt" ]
+targets = [ "wasm32-unknown-unknown" ]

ethereum-explorer/src/lib.rs

@@ -0,0 +1,40 @@
+mod pb;
+
+#[path = "map_block_meta.rs"]
+mod block_meta;
+
+#[path = "map_filter_transaction.rs"]
+mod filter_transaction;
+
+#[path = "map_contract_events.rs"]
+mod contract_events;
+
+
+use pb::eth::transaction::v1::TransactionOption;
+use substreams_ethereum::pb::eth::v2::Block;
+use pb::eth::block_meta::v1::BlockMeta;
+use crate::pb::eth::event::v1::Events;
+
+substreams_ethereum::init!();
+
+#[substreams::handlers::map]
+pub fn map_filter_transaction(transaction_hash: String, blk: Block) -> Result<TransactionOption, substreams::errors::Error> {
+    let filtered_transaction = filter_transaction::filter_by_transaction_hash(transaction_hash, &blk);
+
+    Ok(filtered_transaction)
+}
+
+#[substreams::handlers::map]
+fn map_contract_events(contract_address: String, blk: Block) -> Result<Events, substreams::errors::Error> {
+    let events: Events = contract_events::map_contract_events(contract_address, &blk);
+
+    Ok(events)
+}
+
+#[substreams::handlers::map]
+fn map_block_meta(blk: Block) -> Result<BlockMeta, substreams::errors::Error> {
+    let block_meta = block_meta::map_block_meta(&blk);
+
+    Ok(block_meta)
+}
+

ethereum-explorer/src/map_block_meta.rs

@@ -0,0 +1,16 @@
+use substreams_ethereum::pb::eth::v2::Block;
+use crate::pb::eth::block_meta::v1::BlockMeta;
+use substreams::Hex;
+
+pub fn map_block_meta (blk: &Block) -> BlockMeta {
+    let header = blk.header.as_ref().unwrap();
+
+    let hash_string = Hex(&blk.hash).to_string();
+    let parent_hash_string = Hex(&header.parent_hash).to_string();
+
+    return BlockMeta {
+         number: blk.number, 
+         hash: hash_string, 
+         parent_hash: parent_hash_string 
+    }
+}

ethereum-explorer/src/map_contract_events.rs

@@ -0,0 +1,60 @@
+mod pb;
+use crate::pb::eth::event::v1::Events;
+use crate::pb::eth::event::v1::Event;
+use substreams_ethereum::pb::eth::v2::Block;
+use substreams::Hex;
+use substreams_ethereum::pb::eth::v2::Log;
+use substreams_ethereum::pb::eth::v2::TransactionTrace;
+
+pub fn map_contract_events(contract_address: String, blk: &Block) -> Events {
+    let mut events: Vec<Event> = Vec::new();
+
+    for tr in &blk.transaction_traces {
+        let to = Hex(&tr.to).to_string();
+
+        if to == contract_address {
+            //substreams::log::info!(hash);
+
+            let transaction_events = &mut get_transaction_events(&tr);
+            events.append(transaction_events);
+
+            substreams::log::info!("//////////////////");
+        }
+    }
+
+    return Events { events }
+}
+
+fn get_transaction_events(transaction: &TransactionTrace) -> Vec<Event> {
+    let mut transaction_events: Vec<Event> = Vec::new();
+
+    for log in &transaction.receipt().receipt.logs {
+        let address = Hex(&log.address).to_string();
+        let topics = get_log_topics(&log);
+
+        transaction_events.push(create_event_from(address, topics, &transaction.hash))
+    }
+
+    return transaction_events;
+}
+
+fn get_log_topics(log: &Log) -> Vec<String> {
+    let mut topics: Vec<String> = Vec::new();
+
+    for topic in &log.topics {
+        let topic_string = Hex(topic).to_string();
+        topics.push(topic_string)
+    }
+
+    return topics;
+}
+
+fn create_event_from(address: String, topics: Vec<String>, hash: &Vec<u8>) -> Event {
+    let hash_as_string = Hex(hash).to_string();
+
+    return Event { 
+        address, 
+        topics, 
+        tx_hash: hash_as_string 
+    }
+}

ethereum-explorer/src/map_filter_transaction.rs

@@ -0,0 +1,32 @@
+mod pb;
+use substreams_ethereum::pb::eth::v2::Block;
+use substreams::Hex;
+use crate::pb::eth::transaction::v1::TransactionOption;
+use crate::pb::eth::transaction::v1::Transaction;
+
+
+pub fn filter_by_transaction_hash(transaction_hash: String, blk: &Block) -> TransactionOption {
+    let transaction_traces = &blk.transaction_traces;
+    
+    for transfer in transaction_traces {
+        //let transferValue = transfer;
+        let hash = &transfer.hash;
+        let from = &transfer.from;
+        let to = &transfer.to;
+        
+        if Hex(hash).to_string() == transaction_hash {
+            let trans = Transaction { from: Hex(from).to_string(), to: Hex(to).to_string(), hash: Hex(hash).to_string() };
+            return transfer_option_of(Some(trans));
+        }
+    }
+
+    return empty_transfer_option()
+}
+
+fn transfer_option_of (transaction: Option<Transaction>) -> TransactionOption {
+    return TransactionOption { transaction }
+}
+
+fn empty_transfer_option() -> TransactionOption {
+    return transfer_option_of(None)
+}

ethereum-explorer/src/pb/eth.block_meta.v1.rs

@@ -0,0 +1,12 @@
+// @generated
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BlockMeta {
+    #[prost(uint64, tag="1")]
+    pub number: u64,
+    #[prost(string, tag="2")]
+    pub hash: ::prost::alloc::string::String,
+    #[prost(string, tag="3")]
+    pub parent_hash: ::prost::alloc::string::String,
+}
+// @@protoc_insertion_point(module)

ethereum-explorer/src/pb/eth.event.v1.rs

@@ -0,0 +1,18 @@
+// @generated
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Events {
+    #[prost(message, repeated, tag="1")]
+    pub events: ::prost::alloc::vec::Vec<Event>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Event {
+    #[prost(string, tag="1")]
+    pub address: ::prost::alloc::string::String,
+    #[prost(string, repeated, tag="2")]
+    pub topics: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    #[prost(string, tag="3")]
+    pub tx_hash: ::prost::alloc::string::String,
+}
+// @@protoc_insertion_point(module)

ethereum-explorer/src/pb/eth.transaction.v1.rs

@@ -0,0 +1,18 @@
+// @generated
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct TransactionOption {
+    #[prost(message, optional, tag="1")]
+    pub transaction: ::core::option::Option<Transaction>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Transaction {
+    #[prost(string, tag="1")]
+    pub from: ::prost::alloc::string::String,
+    #[prost(string, tag="2")]
+    pub to: ::prost::alloc::string::String,
+    #[prost(string, tag="3")]
+    pub hash: ::prost::alloc::string::String,
+}
+// @@protoc_insertion_point(module)

ethereum-explorer/src/pb/mod.rs

@@ -0,0 +1,24 @@
+// @generated
+pub mod eth {
+    pub mod block_meta {
+        // @@protoc_insertion_point(attribute:eth.block_meta.v1)
+        pub mod v1 {
+            include!("eth.block_meta.v1.rs");
+            // @@protoc_insertion_point(eth.block_meta.v1)
+        }
+    }
+    pub mod event {
+        // @@protoc_insertion_point(attribute:eth.event.v1)
+        pub mod v1 {
+            include!("eth.event.v1.rs");
+            // @@protoc_insertion_point(eth.event.v1)
+        }
+    }
+    pub mod transaction {
+        // @@protoc_insertion_point(attribute:eth.transaction.v1)
+        pub mod v1 {
+            include!("eth.transaction.v1.rs");
+            // @@protoc_insertion_point(eth.transaction.v1)
+        }
+    }
+}

ethereum-explorer/substreams.yaml

@@ -0,0 +1,43 @@
+specVersion: v0.1.0
+package:
+  name: "substreams_template"
+  version: v0.1.0
+
+protobuf:
+  files:
+    - transaction.proto
+    - event.proto
+    - block_meta.proto
+  importPaths:
+    - ./proto
+
+binaries:
+  default:
+    type: wasm/rust-v1
+    file: ./target/wasm32-unknown-unknown/release/substreams.wasm
+
+modules:
+  - name: map_block_meta
+    kind: map
+    inputs:
+      - source: sf.ethereum.type.v2.Block
+    output:
+      type: proto:eth.block_meta.v1.BlockMeta
+  - name: map_filter_transaction
+    kind: map
+    inputs:
+      - params: string
+      - source: sf.ethereum.type.v2.Block
+    output:
+      type: proto:eth.transaction.v1.TransactionOption
+  - name: map_contract_events
+    kind: map
+    inputs:
+      - params: string
+      - source: sf.ethereum.type.v2.Block
+    output:
+      type: proto:eth.event.v1.Events
+
+params:
+  map_filter_transaction: "4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7"
+  map_contract_events: "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d"