From 8ce2b40d59a94a3f663da00b02d7b68d926b19da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 20 Jul 2023 01:45:17 +0200 Subject: [PATCH 1/8] First version of the Ethereum explorer --- .gitignore => ethereum-explorer/.gitignore | 0 ethereum-explorer/.gitpod.yml | 10 + ethereum-explorer/Cargo.lock | 1121 +++++++++++++++++ ethereum-explorer/Cargo.toml | 34 + ethereum-explorer/Makefile | 30 + ethereum-explorer/README.md | 174 +++ ethereum-explorer/build.sh | 3 + ethereum-explorer/proto/block_meta.proto | 9 + ethereum-explorer/proto/event.proto | 13 + ethereum-explorer/proto/transaction.proto | 13 + ethereum-explorer/rust-toolchain.toml | 4 + ethereum-explorer/src/lib.rs | 40 + ethereum-explorer/src/map_block_meta.rs | 16 + ethereum-explorer/src/map_contract_events.rs | 60 + .../src/map_filter_transaction.rs | 32 + ethereum-explorer/src/pb/eth.block_meta.v1.rs | 12 + ethereum-explorer/src/pb/eth.event.v1.rs | 18 + .../src/pb/eth.transaction.v1.rs | 18 + ethereum-explorer/src/pb/mod.rs | 24 + ethereum-explorer/substreams | 0 ethereum-explorer/substreams.yaml | 43 + 21 files changed, 1674 insertions(+) rename .gitignore => ethereum-explorer/.gitignore (100%) create mode 100644 ethereum-explorer/.gitpod.yml create mode 100644 ethereum-explorer/Cargo.lock create mode 100644 ethereum-explorer/Cargo.toml create mode 100644 ethereum-explorer/Makefile create mode 100644 ethereum-explorer/README.md create mode 100755 ethereum-explorer/build.sh create mode 100644 ethereum-explorer/proto/block_meta.proto create mode 100644 ethereum-explorer/proto/event.proto create mode 100644 ethereum-explorer/proto/transaction.proto create mode 100644 ethereum-explorer/rust-toolchain.toml create mode 100644 ethereum-explorer/src/lib.rs create mode 100644 ethereum-explorer/src/map_block_meta.rs create mode 100644 ethereum-explorer/src/map_contract_events.rs create mode 100644 ethereum-explorer/src/map_filter_transaction.rs create mode 100644 ethereum-explorer/src/pb/eth.block_meta.v1.rs create mode 100644 ethereum-explorer/src/pb/eth.event.v1.rs create mode 100644 ethereum-explorer/src/pb/eth.transaction.v1.rs create mode 100644 ethereum-explorer/src/pb/mod.rs create mode 100644 ethereum-explorer/substreams create mode 100644 ethereum-explorer/substreams.yaml diff --git a/.gitignore b/ethereum-explorer/.gitignore similarity index 100% rename from .gitignore rename to ethereum-explorer/.gitignore diff --git a/ethereum-explorer/.gitpod.yml b/ethereum-explorer/.gitpod.yml new file mode 100644 index 0000000..000c433 --- /dev/null +++ b/ethereum-explorer/.gitpod.yml @@ -0,0 +1,10 @@ +image: ghcr.io/graphprotocol/substreams-gitpod:latest +tasks: + - command: | + # Authenticate with the substreams server + export SUBSTREAMS_API_TOKEN=$(curl https://auth.dfuse.io/v1/auth/issue -s --data-binary '{"api_key":"'$STREAMINGFAST_KEY'"}' | jq -r .token) +ports: + - port: 6060 + onOpen: ignore + - port: 1065 + onOpen: ignore diff --git a/ethereum-explorer/Cargo.lock b/ethereum-explorer/Cargo.lock new file mode 100644 index 0000000..a82d03e --- /dev/null +++ b/ethereum-explorer/Cargo.lock @@ -0,0 +1,1121 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "ethabi" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +dependencies = [ + "ethereum-types", + "hex", + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "thiserror", + "uint", +] + +[[package]] +name = "ethbloom" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +dependencies = [ + "crunchy", + "fixed-hash", + "impl-rlp", + "impl-serde", + "tiny-keccak", +] + +[[package]] +name = "ethereum-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +dependencies = [ + "ethbloom", + "fixed-hash", + "impl-rlp", + "impl-serde", + "primitive-types", + "uint", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-rlp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +dependencies = [ + "rlp", +] + +[[package]] +name = "impl-serde" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "pad" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "parity-scale-codec" +version = "3.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8e946cc0cc711189c0b0249fb8b599cbeeab9784d83c415719368bb8d4ac64" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a296c3079b5fefbc499e1de58dc26c09b1b9a5952d26694ee89f04a43ebbb3e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "petgraph" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +dependencies = [ + "fixedbitset", + "indexmap 1.9.3", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + +[[package]] +name = "rlp" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" +dependencies = [ + "bytes", + "rustc-hex", +] + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustix" +version = "0.37.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.26", +] + +[[package]] +name = "serde_json" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "substreams" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea94f238b54b075ad17894537bdcc20d5fc65cdc199bf1594c9ecfdc6454840" +dependencies = [ + "anyhow", + "bigdecimal", + "hex", + "hex-literal", + "num-bigint", + "num-traits", + "pad", + "prost", + "prost-build", + "prost-types", + "substreams-macro", + "thiserror", +] + +[[package]] +name = "substreams-ethereum" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3a20ea2cf8648b88983efddada20d7f5af7dce834e0a7232cd0c7b02b1aa32" +dependencies = [ + "getrandom", + "num-bigint", + "substreams", + "substreams-ethereum-abigen", + "substreams-ethereum-core", + "substreams-ethereum-derive", +] + +[[package]] +name = "substreams-ethereum-abigen" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37193a8fbbaa8659ee2ca008380d5db00109e60385a825b7f82d5864f054a01e" +dependencies = [ + "anyhow", + "ethabi", + "heck", + "hex", + "prettyplease", + "proc-macro2", + "quote", + "substreams-ethereum-core", + "syn 1.0.109", +] + +[[package]] +name = "substreams-ethereum-core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4caa3329aa29a4584906f2624b836ec81ef61f9ee51da821ac718354e6199d05" +dependencies = [ + "bigdecimal", + "ethabi", + "getrandom", + "num-bigint", + "prost", + "prost-build", + "prost-types", + "substreams", +] + +[[package]] +name = "substreams-ethereum-derive" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8deeda1cdf7dc2b2b66d66b8d2ca1210eb13aac9c62bc4a693e7656c2fd82780" +dependencies = [ + "ethabi", + "heck", + "hex", + "num-bigint", + "proc-macro2", + "quote", + "substreams-ethereum-abigen", + "syn 1.0.109", +] + +[[package]] +name = "substreams-ethereum-explorer" +version = "0.1.0" +dependencies = [ + "anyhow", + "ethabi", + "getrandom", + "hex-literal", + "num-bigint", + "prost", + "substreams", + "substreams-ethereum", +] + +[[package]] +name = "substreams-macro" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9df3ebfeefa8958b1de17f7e9e80f9b1d9a78cbe9114716a872a52b60b8343" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.26", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + +[[package]] +name = "toml_edit" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +dependencies = [ + "indexmap 2.0.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +dependencies = [ + "either", + "libc", + "once_cell", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "winnow" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fac9742fd1ad1bd9643b991319f72dd031016d44b77039a26977eb667141e7" +dependencies = [ + "memchr", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] diff --git a/ethereum-explorer/Cargo.toml b/ethereum-explorer/Cargo.toml new file mode 100644 index 0000000..020e14c --- /dev/null +++ b/ethereum-explorer/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "substreams-ethereum-explorer" +version = "0.1.0" +description = "Substreams showcasing the building blocks of Ethereum" +edition = "2021" +repository = "https://github.com/streamingfast/substreams-explorers" +license = "Apache 2.0" + +[lib] +name = "substreams" +crate-type = ["cdylib"] + +[dependencies] +ethabi = "17" +hex-literal = "0.3" +num-bigint = "0.4" +prost = "0.11" +# Use latest from https://crates.io/crates/substreams +substreams = "0.5" +# Use latest from https://crates.io/crates/substreams-ethereum +substreams-ethereum = "0.9" + +# Required so that ethabi > ethereum-types build correctly under wasm32-unknown-unknown +[target.wasm32-unknown-unknown.dependencies] +getrandom = { version = "0.2", features = ["custom"] } + +[build-dependencies] +anyhow = "1" +substreams-ethereum = "0.9" + +[profile.release] +lto = true +opt-level = 's' +strip = "debuginfo" \ No newline at end of file diff --git a/ethereum-explorer/Makefile b/ethereum-explorer/Makefile new file mode 100644 index 0000000..ed1fafe --- /dev/null +++ b/ethereum-explorer/Makefile @@ -0,0 +1,30 @@ +ENDPOINT ?= mainnet.eth.streamingfast.io:443 +START_BLOCK ?= 17717997 +#17712040 +#17717997 +STOP_BLOCK ?= +50 + +.PHONY: build +build: + cargo build --target wasm32-unknown-unknown --release + +.PHONY: stream +stream: build + substreams run -e $(ENDPOINT) substreams.yaml map_transfers -s $(START_BLOCK) -t $(STOP_BLOCK) + +.PHONY: stream1 +stream1: build + substreams run -e $(ENDPOINT) substreams.yaml map_events -s $(START_BLOCK) -t $(STOP_BLOCK) + +.PHONY: stream2 +stream2: build + substreams run -e $(ENDPOINT) substreams.yaml map_block_meta -s $(START_BLOCK) -t $(STOP_BLOCK) + + +.PHONY: protogen +protogen: + substreams protogen ./substreams.yaml --exclude-paths="sf/substreams,google" + +.PHONY: stream +package: build + substreams package substreams.yaml diff --git a/ethereum-explorer/README.md b/ethereum-explorer/README.md new file mode 100644 index 0000000..d07428f --- /dev/null +++ b/ethereum-explorer/README.md @@ -0,0 +1,174 @@ +# Ethereum Explorer + +The Ethereum Explorer consists of several Substreams modules showcasing the most basic operations that you can perform with Substreams on the Ethereum blockchain. + +## Before You Begin + +Make sure you have the [Substreams CLI installed](https://substreams.streamingfast.io/getting-started/installing-the-cli), and you know the [basic structure of a Substreams module](https://substreams.streamingfast.io/getting-started/quickstart). + +## Modules + +The modules in this repository answer some interesting questions when developing a blockchain application: + +### How Can You Get the Basic Information of a Block? + +For every block, the `map_block_meta` module retrieves the most relevant information of the block (number, hash, and parent hash). + +### How Can You Retrieve a Specific Transaction By Its Hash? + +Given a transaction hash parameter (`tx_hash`), the `map_filter_transaction` filters a transaction among all transactions in the blockchain. This involves: + +1. Iterating over all the transactions. +2. Filter the transactions, where the `hash` field is equal to the hash parameter (`hash == tx_hash`). + +### How Can You Retrieve All the Events for a Specific Smart Contract? + +Given a smart contract address parameter (`contract_address`), the `map_contract_events` module retrieves all the events related to a specific smart contract. This involves: + +1. Iterating over all the smart contract transactions, +2. Filtering the transactions, where the `to` field is equal to the smart contract address parameter (`to == contract_address`). +3. For every filtered transaction, retrieve the logs. + +## Running the Substreams + +First, generate the Protobuf code, which are the outputs of the Substreams: + +``` +> make protogen +``` + +Then, build the Rust code using the `cargo` command-line tool: + +``` +> make build +``` + +Now, you are ready to run the Substreams. The Substreams contained in this project are independent of each other, so you must specify which Substreams module you want to run. + +### Running the "map_block_meta" Module + +In the following command, you retrieve the metadata of the block with number `17712040`. You specify the starting block by using the `--start-block` parameter. + +```bash +> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_block_meta --start-block 17712040 --stop-block +1 +Connected (trace ID e98f04cd7ebb6191befbbf2e6668eafc) +Progress messages received: 0 (0/sec) +Backprocessing history up to requested target block 17712040: +(hit 'm' to switch mode) + +----------- BLOCK #17,712,040 (31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b) --------------- +{ + "@module": "map_block_meta", + "@block": 17712040, + "@type": "eth.block_meta.v1.BlockMeta", + "@data": { + "number": "17712040", + "hash": "31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b", + "parentHash": "1385f853d28b16ad7ebc5d51b6f2ef6d43df4b57bd4c6fe4ef8ccb6f266d8b91" + } +} + +all done +``` + +### Running the "map_filter_transaction" Module + +To run this module, you must provide a transaction hash, so that Substreams can filter the transactions accordingly. The Substreams manifest (`substreams.yaml`) contains a default transaction hash (`4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7`) in the `params` section, which you can update. + +The `4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7` transaction is at block number `17712040`. In order to avoid iterating over the full blockchain, the following command starts searching at block number `17712038`. + +```bash +> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_filter_transaction --start-block 17712038 --stop-block +10 +Connected (trace ID 9dce06621a0ec353213adeaf9f10ef79) +Progress messages received: 0 (0/sec) +Backprocessing history up to requested target block 17712038: +(hit 'm' to switch mode) + +----------- BLOCK #17,712,038 (b96fc7e71c0daf69b19211c45fbb5c201f4356fb2b5607500b7d88d298599f5b) --------------- +----------- BLOCK #17,712,039 (1385f853d28b16ad7ebc5d51b6f2ef6d43df4b57bd4c6fe4ef8ccb6f266d8b91) --------------- +----------- BLOCK #17,712,040 (31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b) --------------- +{ + "@module": "map_filter_transaction", + "@block": 17712040, + "@type": "eth.transaction.v1.TransactionOption", + "@data": { + "transaction": { + "from": "e93685f3bba03016f02bd1828badd6195988d950", + "to": "902f09715b6303d4173037652fa7377e5b98089e", + "hash": "4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7" + } + } +} + +----------- BLOCK #17,712,041 (1af541642176c51580b54de214e955fba8bf1b82af569b81d4038956f2402a41) --------------- +----------- BLOCK #17,712,042 (43c6ebe5b89dd689a9f07468a04d0faf5274a46d0763056ea53b8b1e5ac32148) --------------- +----------- BLOCK #17,712,043 (2457f742913dbbdb171a8d8cc3b1ef8a383f8f547982700a646aa97581bfaeb8) --------------- +----------- BLOCK #17,712,044 (af7abef4f80d7c6f3111e761bb86f305e9847e544fb36b55ba5b64e6103bd5d3) --------------- +----------- BLOCK #17,712,045 (db4664944f7ca8aed5de798bf74ebbdbbeda60e58316b4291bfec61c7287fb17) --------------- +----------- BLOCK #17,712,046 (d9bce9b9210b7deb746720435d1eca99b87fe17aaf7d5055fcd54959e0c9932e) --------------- +----------- BLOCK #17,712,047 (3a5ffab8dacbaf89e10b66e9d2f7ebe65bddae4dcb5e5e8739f8b938f16f98ec) --------------- +all done +``` + +As you can see, the Substreams does not provide an output for the blocks where the transaction is not present. +You can check out the transaction at [Etherscan](https://etherscan.io/tx/0x4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7). + +### Running the "map_filter_transaction" Module + +To run this module, you must provide the address of a smart contract. By default, the `params` section of the Substreams manifest contains the `bc4ca0eda7647a8ab7c2061c2e118a18a936f13d` address, corresponding to the Bore Yacht Club smart contract. You can change this address to track any smart contract. + +The logs of a smart contract might be split across thousands of Ethereum blocks, so for testing purposes, and to avoid iterating over the full blockchain, the following command limits starts searching at block number `17717995`. + +```bash +> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_contract_events --start-block 17717995 --stop-block +10 +Connected (trace ID 834915f19a2b07a8a3aa0159cbbf56da) +Progress messages received: 0 (0/sec) +Backprocessing history up to requested target block 17717995: +(hit 'm' to switch mode) + +----------- BLOCK #17,717,995 (bfecb26963a2cd77700754612185e0074fc9589d2d73abb90e362fe9e7969451) --------------- +----------- BLOCK #17,717,996 (7bf431a4f9df67e1d7e385d9a6cba41c658e66a77f0eb926163a7bbf6619ce20) --------------- +----------- BLOCK #17,717,997 (fa5a57231348f1f138cb71207f0cdcc4a0a267e2688aa63ebff14265b8dae275) --------------- +map_contract_events: log: ////////////////// +{ + "@module": "map_contract_events", + "@block": 17717997, + "@type": "eth.event.v1.Events", + "@data": { + "events": [ + { + "address": "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d", + "topics": [ + "8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925", + "000000000000000000000000e2a83b15fc300d8457eb9e176f98d92a8ff40a49", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000000000000000000000000000000026a7" + ], + "txHash": "f18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939" + }, + { + "address": "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d", + "topics": [ + "ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "000000000000000000000000e2a83b15fc300d8457eb9e176f98d92a8ff40a49", + "000000000000000000000000c67db0df922238979da0fd00d46016e8ae14cecb", + "00000000000000000000000000000000000000000000000000000000000026a7" + ], + "txHash": "f18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939" + } + ] + } +} + +----------- BLOCK #17,717,998 (372ff635821a434c81759b3b23e8dac59393fc27a7ebb88b561c1e5da3c4643a) --------------- +----------- BLOCK #17,717,999 (43f0878e119836cc789ecaf12c3280b82dc49567600cc44f6a042149e2a03779) --------------- +----------- BLOCK #17,718,000 (439efaf9cc0059890a09d34b4cb5a3fe4b61e8ef96ee67673c060d58ff951d4f) --------------- +----------- BLOCK #17,718,001 (c97ca5fd26db28128b0ec2483645348bbfe998e9a6e19e3a442221198254c9ea) --------------- +----------- BLOCK #17,718,002 (9398569e46a954378b16e0e7ce95e49d0f21e6119ed0e3ab84f1c91f16c0c30e) --------------- +----------- BLOCK #17,718,003 (80bcd4c1131c35a413c32903ffa52a14f8c8fe712492a8f6a0feddbb03b10bba) --------------- +----------- BLOCK #17,718,004 (d27309ac29fe47f09fa4987a318818c325403863a53eec6a3676c2c2f8c069d9) --------------- +all done +``` + +The block number `17717997` contains a transaction (`f18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939`) that executes a method of the smart contract, so the Substreams retrieves the corresponding logs. +You can check out the logs at [Etherscan](https://etherscan.io/tx/0xf18291982e955f3c2112de58c1d0a08b79449fb473e58b173de7e0e189d34939#eventlog) and verify that topics are correct. diff --git a/ethereum-explorer/build.sh b/ethereum-explorer/build.sh new file mode 100755 index 0000000..1cdc395 --- /dev/null +++ b/ethereum-explorer/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cargo build --target wasm32-unknown-unknown --release diff --git a/ethereum-explorer/proto/block_meta.proto b/ethereum-explorer/proto/block_meta.proto new file mode 100644 index 0000000..dac4b85 --- /dev/null +++ b/ethereum-explorer/proto/block_meta.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package eth.block_meta.v1; + +message BlockMeta { + uint64 number = 1; + string hash = 2; + string parent_hash = 3; +} \ No newline at end of file diff --git a/ethereum-explorer/proto/event.proto b/ethereum-explorer/proto/event.proto new file mode 100644 index 0000000..e8681c8 --- /dev/null +++ b/ethereum-explorer/proto/event.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package eth.event.v1; + +message Events { + repeated Event events = 1; +} + +message Event { + string address = 1; + repeated string topics = 2; + string tx_hash = 3; +} \ No newline at end of file diff --git a/ethereum-explorer/proto/transaction.proto b/ethereum-explorer/proto/transaction.proto new file mode 100644 index 0000000..3b7ecbd --- /dev/null +++ b/ethereum-explorer/proto/transaction.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package eth.transaction.v1; + +message TransactionOption { + Transaction transaction = 1; +} + +message Transaction { + string from = 1; + string to = 2; + string hash = 3; +} \ No newline at end of file diff --git a/ethereum-explorer/rust-toolchain.toml b/ethereum-explorer/rust-toolchain.toml new file mode 100644 index 0000000..ec334c0 --- /dev/null +++ b/ethereum-explorer/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.65" +components = [ "rustfmt" ] +targets = [ "wasm32-unknown-unknown" ] \ No newline at end of file diff --git a/ethereum-explorer/src/lib.rs b/ethereum-explorer/src/lib.rs new file mode 100644 index 0000000..5dc9cd6 --- /dev/null +++ b/ethereum-explorer/src/lib.rs @@ -0,0 +1,40 @@ +mod pb; + +#[path = "map_block_meta.rs"] +mod block_meta; + +#[path = "map_filter_transaction.rs"] +mod filter_transaction; + +#[path = "map_contract_events.rs"] +mod contract_events; + + +use pb::eth::transaction::v1::TransactionOption; +use substreams_ethereum::pb::eth::v2::Block; +use pb::eth::block_meta::v1::BlockMeta; +use crate::pb::eth::event::v1::Events; + +substreams_ethereum::init!(); + +#[substreams::handlers::map] +pub fn map_filter_transaction(transaction_hash: String, blk: Block) -> Result { + let filtered_transaction = filter_transaction::filter_by_transaction_hash(transaction_hash, &blk); + + Ok(filtered_transaction) +} + +#[substreams::handlers::map] +fn map_contract_events(contract_address: String, blk: Block) -> Result { + let events: Events = contract_events::map_contract_events(contract_address, &blk); + + Ok(events) +} + +#[substreams::handlers::map] +fn map_block_meta(blk: Block) -> Result { + let block_meta = block_meta::map_block_meta(&blk); + + Ok(block_meta) +} + diff --git a/ethereum-explorer/src/map_block_meta.rs b/ethereum-explorer/src/map_block_meta.rs new file mode 100644 index 0000000..41cbb74 --- /dev/null +++ b/ethereum-explorer/src/map_block_meta.rs @@ -0,0 +1,16 @@ +use substreams_ethereum::pb::eth::v2::Block; +use crate::pb::eth::block_meta::v1::BlockMeta; +use substreams::Hex; + +pub fn map_block_meta (blk: &Block) -> BlockMeta { + let header = blk.header.as_ref().unwrap(); + + let hash_string = Hex(&blk.hash).to_string(); + let parent_hash_string = Hex(&header.parent_hash).to_string(); + + return BlockMeta { + number: blk.number, + hash: hash_string, + parent_hash: parent_hash_string + } +} \ No newline at end of file diff --git a/ethereum-explorer/src/map_contract_events.rs b/ethereum-explorer/src/map_contract_events.rs new file mode 100644 index 0000000..75e7533 --- /dev/null +++ b/ethereum-explorer/src/map_contract_events.rs @@ -0,0 +1,60 @@ +mod pb; +use crate::pb::eth::event::v1::Events; +use crate::pb::eth::event::v1::Event; +use substreams_ethereum::pb::eth::v2::Block; +use substreams::Hex; +use substreams_ethereum::pb::eth::v2::Log; +use substreams_ethereum::pb::eth::v2::TransactionTrace; + +pub fn map_contract_events(contract_address: String, blk: &Block) -> Events { + let mut events: Vec = Vec::new(); + + for tr in &blk.transaction_traces { + let to = Hex(&tr.to).to_string(); + + if to == contract_address { + //substreams::log::info!(hash); + + let transaction_events = &mut get_transaction_events(&tr); + events.append(transaction_events); + + substreams::log::info!("//////////////////"); + } + } + + return Events { events } +} + +fn get_transaction_events(transaction: &TransactionTrace) -> Vec { + let mut transaction_events: Vec = Vec::new(); + + for log in &transaction.receipt().receipt.logs { + let address = Hex(&log.address).to_string(); + let topics = get_log_topics(&log); + + transaction_events.push(create_event_from(address, topics, &transaction.hash)) + } + + return transaction_events; +} + +fn get_log_topics(log: &Log) -> Vec { + let mut topics: Vec = Vec::new(); + + for topic in &log.topics { + let topic_string = Hex(topic).to_string(); + topics.push(topic_string) + } + + return topics; +} + +fn create_event_from(address: String, topics: Vec, hash: &Vec) -> Event { + let hash_as_string = Hex(hash).to_string(); + + return Event { + address, + topics, + tx_hash: hash_as_string + } +} \ No newline at end of file diff --git a/ethereum-explorer/src/map_filter_transaction.rs b/ethereum-explorer/src/map_filter_transaction.rs new file mode 100644 index 0000000..75e89d9 --- /dev/null +++ b/ethereum-explorer/src/map_filter_transaction.rs @@ -0,0 +1,32 @@ +mod pb; +use substreams_ethereum::pb::eth::v2::Block; +use substreams::Hex; +use crate::pb::eth::transaction::v1::TransactionOption; +use crate::pb::eth::transaction::v1::Transaction; + + +pub fn filter_by_transaction_hash(transaction_hash: String, blk: &Block) -> TransactionOption { + let transaction_traces = &blk.transaction_traces; + + for transfer in transaction_traces { + //let transferValue = transfer; + let hash = &transfer.hash; + let from = &transfer.from; + let to = &transfer.to; + + if Hex(hash).to_string() == transaction_hash { + let trans = Transaction { from: Hex(from).to_string(), to: Hex(to).to_string(), hash: Hex(hash).to_string() }; + return transfer_option_of(Some(trans)); + } + } + + return empty_transfer_option() +} + +fn transfer_option_of (transaction: Option) -> TransactionOption { + return TransactionOption { transaction } +} + +fn empty_transfer_option() -> TransactionOption { + return transfer_option_of(None) +} \ No newline at end of file diff --git a/ethereum-explorer/src/pb/eth.block_meta.v1.rs b/ethereum-explorer/src/pb/eth.block_meta.v1.rs new file mode 100644 index 0000000..bc151ce --- /dev/null +++ b/ethereum-explorer/src/pb/eth.block_meta.v1.rs @@ -0,0 +1,12 @@ +// @generated +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockMeta { + #[prost(uint64, tag="1")] + pub number: u64, + #[prost(string, tag="2")] + pub hash: ::prost::alloc::string::String, + #[prost(string, tag="3")] + pub parent_hash: ::prost::alloc::string::String, +} +// @@protoc_insertion_point(module) diff --git a/ethereum-explorer/src/pb/eth.event.v1.rs b/ethereum-explorer/src/pb/eth.event.v1.rs new file mode 100644 index 0000000..fe97d9a --- /dev/null +++ b/ethereum-explorer/src/pb/eth.event.v1.rs @@ -0,0 +1,18 @@ +// @generated +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Events { + #[prost(message, repeated, tag="1")] + pub events: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Event { + #[prost(string, tag="1")] + pub address: ::prost::alloc::string::String, + #[prost(string, repeated, tag="2")] + pub topics: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag="3")] + pub tx_hash: ::prost::alloc::string::String, +} +// @@protoc_insertion_point(module) diff --git a/ethereum-explorer/src/pb/eth.transaction.v1.rs b/ethereum-explorer/src/pb/eth.transaction.v1.rs new file mode 100644 index 0000000..28b25b2 --- /dev/null +++ b/ethereum-explorer/src/pb/eth.transaction.v1.rs @@ -0,0 +1,18 @@ +// @generated +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionOption { + #[prost(message, optional, tag="1")] + pub transaction: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(string, tag="1")] + pub from: ::prost::alloc::string::String, + #[prost(string, tag="2")] + pub to: ::prost::alloc::string::String, + #[prost(string, tag="3")] + pub hash: ::prost::alloc::string::String, +} +// @@protoc_insertion_point(module) diff --git a/ethereum-explorer/src/pb/mod.rs b/ethereum-explorer/src/pb/mod.rs new file mode 100644 index 0000000..1c6089d --- /dev/null +++ b/ethereum-explorer/src/pb/mod.rs @@ -0,0 +1,24 @@ +// @generated +pub mod eth { + pub mod block_meta { + // @@protoc_insertion_point(attribute:eth.block_meta.v1) + pub mod v1 { + include!("eth.block_meta.v1.rs"); + // @@protoc_insertion_point(eth.block_meta.v1) + } + } + pub mod event { + // @@protoc_insertion_point(attribute:eth.event.v1) + pub mod v1 { + include!("eth.event.v1.rs"); + // @@protoc_insertion_point(eth.event.v1) + } + } + pub mod transaction { + // @@protoc_insertion_point(attribute:eth.transaction.v1) + pub mod v1 { + include!("eth.transaction.v1.rs"); + // @@protoc_insertion_point(eth.transaction.v1) + } + } +} diff --git a/ethereum-explorer/substreams b/ethereum-explorer/substreams new file mode 100644 index 0000000..e69de29 diff --git a/ethereum-explorer/substreams.yaml b/ethereum-explorer/substreams.yaml new file mode 100644 index 0000000..21e6fe9 --- /dev/null +++ b/ethereum-explorer/substreams.yaml @@ -0,0 +1,43 @@ +specVersion: v0.1.0 +package: + name: "substreams_template" + version: v0.1.0 + +protobuf: + files: + - transaction.proto + - event.proto + - block_meta.proto + importPaths: + - ./proto + +binaries: + default: + type: wasm/rust-v1 + file: ./target/wasm32-unknown-unknown/release/substreams.wasm + +modules: + - name: map_block_meta + kind: map + inputs: + - source: sf.ethereum.type.v2.Block + output: + type: proto:eth.block_meta.v1.BlockMeta + - name: map_filter_transaction + kind: map + inputs: + - params: string + - source: sf.ethereum.type.v2.Block + output: + type: proto:eth.transaction.v1.TransactionOption + - name: map_contract_events + kind: map + inputs: + - params: string + - source: sf.ethereum.type.v2.Block + output: + type: proto:eth.event.v1.Events + +params: + map_filter_transaction: "4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7" + map_contract_events: "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d" From f701e7a818f280990edf0edc4280255650dca375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 20 Jul 2023 01:53:39 +0200 Subject: [PATCH 2/8] Update Makefile --- ethereum-explorer/Makefile | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/ethereum-explorer/Makefile b/ethereum-explorer/Makefile index ed1fafe..c97e5d6 100644 --- a/ethereum-explorer/Makefile +++ b/ethereum-explorer/Makefile @@ -1,26 +1,7 @@ -ENDPOINT ?= mainnet.eth.streamingfast.io:443 -START_BLOCK ?= 17717997 -#17712040 -#17717997 -STOP_BLOCK ?= +50 - .PHONY: build build: cargo build --target wasm32-unknown-unknown --release -.PHONY: stream -stream: build - substreams run -e $(ENDPOINT) substreams.yaml map_transfers -s $(START_BLOCK) -t $(STOP_BLOCK) - -.PHONY: stream1 -stream1: build - substreams run -e $(ENDPOINT) substreams.yaml map_events -s $(START_BLOCK) -t $(STOP_BLOCK) - -.PHONY: stream2 -stream2: build - substreams run -e $(ENDPOINT) substreams.yaml map_block_meta -s $(START_BLOCK) -t $(STOP_BLOCK) - - .PHONY: protogen protogen: substreams protogen ./substreams.yaml --exclude-paths="sf/substreams,google" From 9afafb3421265ec908e431e43e8b5cde32bbb600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 20 Jul 2023 01:54:36 +0200 Subject: [PATCH 3/8] Remove unnecessary file --- ethereum-explorer/substreams | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ethereum-explorer/substreams diff --git a/ethereum-explorer/substreams b/ethereum-explorer/substreams deleted file mode 100644 index e69de29..0000000 From 58e2534e9bd857843aa6f034e8f275e044ee4c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 20 Jul 2023 02:01:14 +0200 Subject: [PATCH 4/8] Minor fixes --- ethereum-explorer/README.md | 2 +- ethereum-explorer/src/lib.rs | 14 +++++++------- ethereum-explorer/src/map_contract_events.rs | 4 ---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/ethereum-explorer/README.md b/ethereum-explorer/README.md index d07428f..cb8a93f 100644 --- a/ethereum-explorer/README.md +++ b/ethereum-explorer/README.md @@ -4,7 +4,7 @@ The Ethereum Explorer consists of several Substreams modules showcasing the most ## Before You Begin -Make sure you have the [Substreams CLI installed](https://substreams.streamingfast.io/getting-started/installing-the-cli), and you know the [basic structure of a Substreams module](https://substreams.streamingfast.io/getting-started/quickstart). +Make sure you have the [Substreams CLI installed](https://substreams.streamingfast.io/getting-started/installing-the-cli), and you know the [basic structure of a Substreams module](https://substreams.streamingfast.io/getting-started/quickstart). You must also provide an [authentication token](https://substreams.streamingfast.io/reference-and-specs/authentication). ## Modules diff --git a/ethereum-explorer/src/lib.rs b/ethereum-explorer/src/lib.rs index 5dc9cd6..70d6aed 100644 --- a/ethereum-explorer/src/lib.rs +++ b/ethereum-explorer/src/lib.rs @@ -17,6 +17,13 @@ use crate::pb::eth::event::v1::Events; substreams_ethereum::init!(); +#[substreams::handlers::map] +fn map_block_meta(blk: Block) -> Result { + let block_meta = block_meta::map_block_meta(&blk); + + Ok(block_meta) +} + #[substreams::handlers::map] pub fn map_filter_transaction(transaction_hash: String, blk: Block) -> Result { let filtered_transaction = filter_transaction::filter_by_transaction_hash(transaction_hash, &blk); @@ -31,10 +38,3 @@ fn map_contract_events(contract_address: String, blk: Block) -> Result Result { - let block_meta = block_meta::map_block_meta(&blk); - - Ok(block_meta) -} - diff --git a/ethereum-explorer/src/map_contract_events.rs b/ethereum-explorer/src/map_contract_events.rs index 75e7533..fc427a6 100644 --- a/ethereum-explorer/src/map_contract_events.rs +++ b/ethereum-explorer/src/map_contract_events.rs @@ -13,12 +13,8 @@ pub fn map_contract_events(contract_address: String, blk: &Block) -> Events { let to = Hex(&tr.to).to_string(); if to == contract_address { - //substreams::log::info!(hash); - let transaction_events = &mut get_transaction_events(&tr); events.append(transaction_events); - - substreams::log::info!("//////////////////"); } } From 3678208dad0dadd71f909274efec29c22bbee243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Fri, 21 Jul 2023 18:33:05 +0200 Subject: [PATCH 5/8] Update transactions and events modules --- ethereum-explorer/Cargo.lock | 19 +++++ ethereum-explorer/Cargo.toml | 2 + ethereum-explorer/Makefile | 4 +- ethereum-explorer/build.sh | 3 - ethereum-explorer/proto/transaction.proto | 4 +- ethereum-explorer/rust-toolchain.toml | 2 +- ethereum-explorer/src/lib.rs | 41 +--------- ethereum-explorer/src/map_block_meta.rs | 7 +- ethereum-explorer/src/map_contract_events.rs | 47 +++++++---- .../src/map_filter_transaction.rs | 32 -------- .../src/map_filter_transactions.rs | 77 +++++++++++++++++++ .../src/pb/eth.transaction.v1.rs | 6 +- ethereum-explorer/src/util.rs | 23 ++++++ ethereum-explorer/substreams.yaml | 6 +- 14 files changed, 172 insertions(+), 101 deletions(-) delete mode 100755 ethereum-explorer/build.sh delete mode 100644 ethereum-explorer/src/map_filter_transaction.rs create mode 100644 ethereum-explorer/src/map_filter_transactions.rs create mode 100644 ethereum-explorer/src/util.rs diff --git a/ethereum-explorer/Cargo.lock b/ethereum-explorer/Cargo.lock index a82d03e..5572aa3 100644 --- a/ethereum-explorer/Cargo.lock +++ b/ethereum-explorer/Cargo.lock @@ -508,6 +508,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + [[package]] name = "petgraph" version = "0.6.3" @@ -770,6 +776,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + [[package]] name = "sha3" version = "0.10.8" @@ -879,6 +896,8 @@ dependencies = [ "hex-literal", "num-bigint", "prost", + "serde", + "serde_qs", "substreams", "substreams-ethereum", ] diff --git a/ethereum-explorer/Cargo.toml b/ethereum-explorer/Cargo.toml index 020e14c..c66fb83 100644 --- a/ethereum-explorer/Cargo.toml +++ b/ethereum-explorer/Cargo.toml @@ -19,6 +19,8 @@ prost = "0.11" substreams = "0.5" # Use latest from https://crates.io/crates/substreams-ethereum substreams-ethereum = "0.9" +serde_qs = "0.12.0" +serde = { version = "1.0", features = ["derive"] } # Required so that ethabi > ethereum-types build correctly under wasm32-unknown-unknown [target.wasm32-unknown-unknown.dependencies] diff --git a/ethereum-explorer/Makefile b/ethereum-explorer/Makefile index c97e5d6..0edf461 100644 --- a/ethereum-explorer/Makefile +++ b/ethereum-explorer/Makefile @@ -6,6 +6,6 @@ build: protogen: substreams protogen ./substreams.yaml --exclude-paths="sf/substreams,google" -.PHONY: stream +.PHONY: package package: build - substreams package substreams.yaml + substreams pack substreams.yaml diff --git a/ethereum-explorer/build.sh b/ethereum-explorer/build.sh deleted file mode 100755 index 1cdc395..0000000 --- a/ethereum-explorer/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -cargo build --target wasm32-unknown-unknown --release diff --git a/ethereum-explorer/proto/transaction.proto b/ethereum-explorer/proto/transaction.proto index 3b7ecbd..9bdf1b0 100644 --- a/ethereum-explorer/proto/transaction.proto +++ b/ethereum-explorer/proto/transaction.proto @@ -2,8 +2,8 @@ syntax = "proto3"; package eth.transaction.v1; -message TransactionOption { - Transaction transaction = 1; +message Transactions { + repeated Transaction transactions = 1; } message Transaction { diff --git a/ethereum-explorer/rust-toolchain.toml b/ethereum-explorer/rust-toolchain.toml index ec334c0..e892501 100644 --- a/ethereum-explorer/rust-toolchain.toml +++ b/ethereum-explorer/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.65" +channel = "1.69" components = [ "rustfmt" ] targets = [ "wasm32-unknown-unknown" ] \ No newline at end of file diff --git a/ethereum-explorer/src/lib.rs b/ethereum-explorer/src/lib.rs index 70d6aed..b1e3689 100644 --- a/ethereum-explorer/src/lib.rs +++ b/ethereum-explorer/src/lib.rs @@ -1,40 +1,7 @@ mod pb; - -#[path = "map_block_meta.rs"] -mod block_meta; - -#[path = "map_filter_transaction.rs"] -mod filter_transaction; - -#[path = "map_contract_events.rs"] -mod contract_events; - - -use pb::eth::transaction::v1::TransactionOption; -use substreams_ethereum::pb::eth::v2::Block; -use pb::eth::block_meta::v1::BlockMeta; -use crate::pb::eth::event::v1::Events; +mod map_block_meta; +mod map_filter_transactions; +mod map_contract_events; +mod util; substreams_ethereum::init!(); - -#[substreams::handlers::map] -fn map_block_meta(blk: Block) -> Result { - let block_meta = block_meta::map_block_meta(&blk); - - Ok(block_meta) -} - -#[substreams::handlers::map] -pub fn map_filter_transaction(transaction_hash: String, blk: Block) -> Result { - let filtered_transaction = filter_transaction::filter_by_transaction_hash(transaction_hash, &blk); - - Ok(filtered_transaction) -} - -#[substreams::handlers::map] -fn map_contract_events(contract_address: String, blk: Block) -> Result { - let events: Events = contract_events::map_contract_events(contract_address, &blk); - - Ok(events) -} - diff --git a/ethereum-explorer/src/map_block_meta.rs b/ethereum-explorer/src/map_block_meta.rs index 41cbb74..e39c183 100644 --- a/ethereum-explorer/src/map_block_meta.rs +++ b/ethereum-explorer/src/map_block_meta.rs @@ -2,15 +2,16 @@ use substreams_ethereum::pb::eth::v2::Block; use crate::pb::eth::block_meta::v1::BlockMeta; use substreams::Hex; -pub fn map_block_meta (blk: &Block) -> BlockMeta { +#[substreams::handlers::map] +fn map_block_meta(blk: Block) -> Result { let header = blk.header.as_ref().unwrap(); let hash_string = Hex(&blk.hash).to_string(); let parent_hash_string = Hex(&header.parent_hash).to_string(); - return BlockMeta { + Ok(BlockMeta { number: blk.number, hash: hash_string, parent_hash: parent_hash_string - } + }) } \ No newline at end of file diff --git a/ethereum-explorer/src/map_contract_events.rs b/ethereum-explorer/src/map_contract_events.rs index fc427a6..3d4bfc9 100644 --- a/ethereum-explorer/src/map_contract_events.rs +++ b/ethereum-explorer/src/map_contract_events.rs @@ -1,34 +1,52 @@ -mod pb; use crate::pb::eth::event::v1::Events; use crate::pb::eth::event::v1::Event; use substreams_ethereum::pb::eth::v2::Block; use substreams::Hex; use substreams_ethereum::pb::eth::v2::Log; use substreams_ethereum::pb::eth::v2::TransactionTrace; +use substreams::errors::Error; +use crate::util; + +#[substreams::handlers::map] +fn map_contract_events(contract_address: String, blk: Block) -> Result { + let error = verify_parameter(&contract_address); + if error.is_some() { + return Err(error.unwrap()); + } -pub fn map_contract_events(contract_address: String, blk: &Block) -> Events { let mut events: Vec = Vec::new(); + let contract_address_as_vec = match Hex::decode(&contract_address) { + Ok(address) => address, + Err(error) => return Err(Error::Unexpected(error.to_string())), + }; - for tr in &blk.transaction_traces { - let to = Hex(&tr.to).to_string(); - - if to == contract_address { - let transaction_events = &mut get_transaction_events(&tr); - events.append(transaction_events); + for transaction in &blk.transaction_traces { + if transaction.to == contract_address_as_vec { + let transaction_events = get_transaction_events(&transaction); + events.extend(transaction_events); } } - return Events { events } + Ok(Events { events }) +} + +fn verify_parameter(contract_address: &String) -> Option { + if !util::is_address_valid(contract_address) { + return Some(Error::Unexpected(String::from("Contract address is not valid"))) + } + + return None } fn get_transaction_events(transaction: &TransactionTrace) -> Vec { let mut transaction_events: Vec = Vec::new(); for log in &transaction.receipt().receipt.logs { - let address = Hex(&log.address).to_string(); + let address = util::hexadecimal_to_string(&log.address); let topics = get_log_topics(&log); - transaction_events.push(create_event_from(address, topics, &transaction.hash)) + let event = create_event_from(address, topics, util::hexadecimal_to_string(&transaction.hash)); + transaction_events.push(event) } return transaction_events; @@ -38,19 +56,18 @@ fn get_log_topics(log: &Log) -> Vec { let mut topics: Vec = Vec::new(); for topic in &log.topics { - let topic_string = Hex(topic).to_string(); + let topic_string = util::hexadecimal_to_string(topic); topics.push(topic_string) } return topics; } -fn create_event_from(address: String, topics: Vec, hash: &Vec) -> Event { - let hash_as_string = Hex(hash).to_string(); +fn create_event_from(address: String, topics: Vec, hash: String) -> Event { return Event { address, topics, - tx_hash: hash_as_string + tx_hash: hash } } \ No newline at end of file diff --git a/ethereum-explorer/src/map_filter_transaction.rs b/ethereum-explorer/src/map_filter_transaction.rs deleted file mode 100644 index 75e89d9..0000000 --- a/ethereum-explorer/src/map_filter_transaction.rs +++ /dev/null @@ -1,32 +0,0 @@ -mod pb; -use substreams_ethereum::pb::eth::v2::Block; -use substreams::Hex; -use crate::pb::eth::transaction::v1::TransactionOption; -use crate::pb::eth::transaction::v1::Transaction; - - -pub fn filter_by_transaction_hash(transaction_hash: String, blk: &Block) -> TransactionOption { - let transaction_traces = &blk.transaction_traces; - - for transfer in transaction_traces { - //let transferValue = transfer; - let hash = &transfer.hash; - let from = &transfer.from; - let to = &transfer.to; - - if Hex(hash).to_string() == transaction_hash { - let trans = Transaction { from: Hex(from).to_string(), to: Hex(to).to_string(), hash: Hex(hash).to_string() }; - return transfer_option_of(Some(trans)); - } - } - - return empty_transfer_option() -} - -fn transfer_option_of (transaction: Option) -> TransactionOption { - return TransactionOption { transaction } -} - -fn empty_transfer_option() -> TransactionOption { - return transfer_option_of(None) -} \ No newline at end of file diff --git a/ethereum-explorer/src/map_filter_transactions.rs b/ethereum-explorer/src/map_filter_transactions.rs new file mode 100644 index 0000000..f2e53b2 --- /dev/null +++ b/ethereum-explorer/src/map_filter_transactions.rs @@ -0,0 +1,77 @@ +use substreams_ethereum::pb::eth::v2::{Block, TransactionTraceStatus}; +use crate::pb::eth::transaction::v1::{Transaction, Transactions}; +use serde::Deserialize; +use substreams::errors::Error; +use crate::util; + +#[derive(Deserialize)] +struct TransactionFilterParams { + hash: Option, + to: Option, + from: Option +} + +#[substreams::handlers::map] +pub fn map_filter_transactions(params: String, blk: Block) -> Result> { + let filters: TransactionFilterParams = serde_qs::from_str(¶ms).unwrap(); + let errors = verify_filter_params(&filters); + if errors.len() > 0 { + return Err(errors) + } + + let mut filtered_transactions: Vec = Vec::new(); + + for transaction in &blk.transaction_traces { + let tx_hash = util::hexadecimal_to_string(&transaction.hash); + let tx_from = util::hexadecimal_to_string(&transaction.from); + let tx_to = util::hexadecimal_to_string(&transaction.to); + let mut current_transaction_filtered = true; + + if !filter_by_parameter(&filters.hash, &tx_hash) || + !filter_by_parameter(&filters.from, &tx_from) || + !filter_by_parameter(&filters.to, &tx_to) || + transaction.status != (TransactionTraceStatus::Succeeded as i32) { + current_transaction_filtered = false + } + + if current_transaction_filtered { + let trans = Transaction { from: tx_from, to: tx_to, hash: tx_hash }; + filtered_transactions.push(trans) + } + } + + Ok(Transactions { transactions: filtered_transactions }) +} + +fn verify_filter_params(params: &TransactionFilterParams) -> Vec { + let mut errors: Vec = Vec::new(); + + if params.hash.is_some() + && !util::is_transaction_hash_valid(¶ms.hash.as_ref().unwrap()) { + errors.push(Error::Unexpected(String::from("Transaction hash is not valid"))); + } + + if params.from.is_some() + && !util::is_address_valid(¶ms.from.as_ref().unwrap()) { + errors.push(Error::Unexpected(String::from("'from' address is not valid"))); + } + + if params.to.is_some() + && !util::is_address_valid(¶ms.to.as_ref().unwrap()) { + errors.push(Error::Unexpected(String::from("'to' address is not valid"))); + } + + return errors; +} + +fn filter_by_parameter(parameter: &Option, transaction_field: &String) -> bool { + if parameter.is_none() { + return true; + } + + if parameter.as_ref().unwrap() == transaction_field { + return true + } + + return false; +} \ No newline at end of file diff --git a/ethereum-explorer/src/pb/eth.transaction.v1.rs b/ethereum-explorer/src/pb/eth.transaction.v1.rs index 28b25b2..f299d0f 100644 --- a/ethereum-explorer/src/pb/eth.transaction.v1.rs +++ b/ethereum-explorer/src/pb/eth.transaction.v1.rs @@ -1,9 +1,9 @@ // @generated #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionOption { - #[prost(message, optional, tag="1")] - pub transaction: ::core::option::Option, +pub struct Transactions { + #[prost(message, repeated, tag="1")] + pub transactions: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/ethereum-explorer/src/util.rs b/ethereum-explorer/src/util.rs new file mode 100644 index 0000000..42e4997 --- /dev/null +++ b/ethereum-explorer/src/util.rs @@ -0,0 +1,23 @@ +use substreams::Hex; + +pub fn hexadecimal_to_string(hex: &Vec) -> String { + return Hex::encode(hex); +} + +pub fn is_transaction_hash_valid(hash: &String) -> bool { + // A transaction hash is always 64 hexadecimal characters + if hash.len() != 64 { + return false; + } + + return true; +} + +pub fn is_address_valid(address: &String) -> bool { + // An address is always 40 hexadecimal characters + if address.len() != 40 { + return false; + } + + return true; +} diff --git a/ethereum-explorer/substreams.yaml b/ethereum-explorer/substreams.yaml index 21e6fe9..51c6f6a 100644 --- a/ethereum-explorer/substreams.yaml +++ b/ethereum-explorer/substreams.yaml @@ -23,13 +23,13 @@ modules: - source: sf.ethereum.type.v2.Block output: type: proto:eth.block_meta.v1.BlockMeta - - name: map_filter_transaction + - name: map_filter_transactions kind: map inputs: - params: string - source: sf.ethereum.type.v2.Block output: - type: proto:eth.transaction.v1.TransactionOption + type: proto:eth.transaction.v1.Transactions - name: map_contract_events kind: map inputs: @@ -39,5 +39,5 @@ modules: type: proto:eth.event.v1.Events params: - map_filter_transaction: "4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7" + map_filter_transactions: "to=dac17f958d2ee523a2206206994597c13d831ec7&from=46340b20830761efd32832a74d7169b29feb9758" map_contract_events: "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d" From 73a819ad713d45e0d0939be080c45153b121885f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Wed, 26 Jul 2023 11:22:44 +0200 Subject: [PATCH 6/8] Using stream-fashion structures --- ethereum-explorer/Cargo.lock | 121 +++++++----------- ethereum-explorer/Cargo.toml | 12 +- ethereum-explorer/Makefile | 4 +- ethereum-explorer/README.md | 2 +- ethereum-explorer/src/lib.rs | 4 +- ethereum-explorer/src/map_block_meta.rs | 13 +- ethereum-explorer/src/map_contract_events.rs | 81 +++--------- .../src/map_filter_transactions.rs | 95 +++++++------- ethereum-explorer/src/util.rs | 21 +-- ethereum-explorer/substreams.yaml | 4 +- 10 files changed, 132 insertions(+), 225 deletions(-) diff --git a/ethereum-explorer/Cargo.lock b/ethereum-explorer/Cargo.lock index 5572aa3..e833191 100644 --- a/ethereum-explorer/Cargo.lock +++ b/ethereum-explorer/Cargo.lock @@ -46,6 +46,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "bitvec" version = "1.0.1" @@ -134,9 +140,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "equivalent" @@ -211,12 +217,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "fixed-hash" @@ -281,12 +284,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" - [[package]] name = "hex" version = "0.4.3" @@ -357,26 +354,6 @@ dependencies = [ "hashbrown 0.14.0", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - [[package]] name = "itertools" version = "0.10.5" @@ -415,9 +392,9 @@ checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" [[package]] name = "log" @@ -460,9 +437,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] @@ -628,9 +605,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -677,7 +654,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -727,13 +704,12 @@ checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" [[package]] name = "rustix" -version = "0.37.23" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" dependencies = [ - "bitflags", + "bitflags 2.3.3", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -747,22 +723,22 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -805,9 +781,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "substreams" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea94f238b54b075ad17894537bdcc20d5fc65cdc199bf1594c9ecfdc6454840" +version = "0.5.9" dependencies = [ "anyhow", "bigdecimal", @@ -825,9 +799,9 @@ dependencies = [ [[package]] name = "substreams-ethereum" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3a20ea2cf8648b88983efddada20d7f5af7dce834e0a7232cd0c7b02b1aa32" +checksum = "c0bd6c3ecb69e76111515d8087080ce7a750f41df79de221d24ab76bc3efe81e" dependencies = [ "getrandom", "num-bigint", @@ -839,9 +813,9 @@ dependencies = [ [[package]] name = "substreams-ethereum-abigen" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37193a8fbbaa8659ee2ca008380d5db00109e60385a825b7f82d5864f054a01e" +checksum = "894803ed2422e63afac798ae1370faf6897d9200b53c03aa53b4945ea28cdc3d" dependencies = [ "anyhow", "ethabi", @@ -856,9 +830,9 @@ dependencies = [ [[package]] name = "substreams-ethereum-core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa3329aa29a4584906f2624b836ec81ef61f9ee51da821ac718354e6199d05" +checksum = "49a7a83144082aafd061f36bc68a20f1a2b3b8774ec58cd0dabd222a3f606a0c" dependencies = [ "bigdecimal", "ethabi", @@ -872,9 +846,9 @@ dependencies = [ [[package]] name = "substreams-ethereum-derive" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8deeda1cdf7dc2b2b66d66b8d2ca1210eb13aac9c62bc4a693e7656c2fd82780" +checksum = "ac19d40bbdf62261b6acca3a7e43f8ad68b5bc76ca41a8822988c757b2b0d9e9" dependencies = [ "ethabi", "heck", @@ -904,9 +878,7 @@ dependencies = [ [[package]] name = "substreams-macro" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9df3ebfeefa8958b1de17f7e9e80f9b1d9a78cbe9114716a872a52b60b8343" +version = "0.5.9" dependencies = [ "proc-macro2", "quote", @@ -927,9 +899,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.26" +version = "2.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" dependencies = [ "proc-macro2", "quote", @@ -944,11 +916,10 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" dependencies = [ - "autocfg", "cfg-if", "fastrand", "redox_syscall", @@ -958,22 +929,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -1123,9 +1094,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fac9742fd1ad1bd9643b991319f72dd031016d44b77039a26977eb667141e7" +checksum = "25b5872fa2e10bd067ae946f927e726d7d603eaeb6e02fa6a350e0722d2b8c11" dependencies = [ "memchr", ] diff --git a/ethereum-explorer/Cargo.toml b/ethereum-explorer/Cargo.toml index c66fb83..b52f752 100644 --- a/ethereum-explorer/Cargo.toml +++ b/ethereum-explorer/Cargo.toml @@ -16,21 +16,21 @@ hex-literal = "0.3" num-bigint = "0.4" prost = "0.11" # Use latest from https://crates.io/crates/substreams -substreams = "0.5" +substreams = "0.5.9" # Use latest from https://crates.io/crates/substreams-ethereum substreams-ethereum = "0.9" serde_qs = "0.12.0" serde = { version = "1.0", features = ["derive"] } +anyhow = "1.0" # Required so that ethabi > ethereum-types build correctly under wasm32-unknown-unknown [target.wasm32-unknown-unknown.dependencies] getrandom = { version = "0.2", features = ["custom"] } -[build-dependencies] -anyhow = "1" -substreams-ethereum = "0.9" - [profile.release] lto = true opt-level = 's' -strip = "debuginfo" \ No newline at end of file +strip = "debuginfo" + +[patch.crates-io] +substreams = { path = "/Users/enolalvarezdeprado/Documents/streamingfast/substreams-rs/substreams" } \ No newline at end of file diff --git a/ethereum-explorer/Makefile b/ethereum-explorer/Makefile index 0edf461..36bca83 100644 --- a/ethereum-explorer/Makefile +++ b/ethereum-explorer/Makefile @@ -6,6 +6,6 @@ build: protogen: substreams protogen ./substreams.yaml --exclude-paths="sf/substreams,google" -.PHONY: package -package: build +.PHONY: pack +pack: build substreams pack substreams.yaml diff --git a/ethereum-explorer/README.md b/ethereum-explorer/README.md index cb8a93f..8d64146 100644 --- a/ethereum-explorer/README.md +++ b/ethereum-explorer/README.md @@ -78,7 +78,7 @@ To run this module, you must provide a transaction hash, so that Substreams can The `4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7` transaction is at block number `17712040`. In order to avoid iterating over the full blockchain, the following command starts searching at block number `17712038`. ```bash -> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_filter_transaction --start-block 17712038 --stop-block +10 +> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_filter_transactions --start-block 17712038 --stop-block +10 Connected (trace ID 9dce06621a0ec353213adeaf9f10ef79) Progress messages received: 0 (0/sec) Backprocessing history up to requested target block 17712038: diff --git a/ethereum-explorer/src/lib.rs b/ethereum-explorer/src/lib.rs index b1e3689..62d27a0 100644 --- a/ethereum-explorer/src/lib.rs +++ b/ethereum-explorer/src/lib.rs @@ -1,7 +1,7 @@ -mod pb; mod map_block_meta; -mod map_filter_transactions; mod map_contract_events; +mod map_filter_transactions; +mod pb; mod util; substreams_ethereum::init!(); diff --git a/ethereum-explorer/src/map_block_meta.rs b/ethereum-explorer/src/map_block_meta.rs index e39c183..9ffd481 100644 --- a/ethereum-explorer/src/map_block_meta.rs +++ b/ethereum-explorer/src/map_block_meta.rs @@ -1,17 +1,14 @@ -use substreams_ethereum::pb::eth::v2::Block; use crate::pb::eth::block_meta::v1::BlockMeta; use substreams::Hex; +use substreams_ethereum::pb::eth::v2::Block; #[substreams::handlers::map] fn map_block_meta(blk: Block) -> Result { let header = blk.header.as_ref().unwrap(); - let hash_string = Hex(&blk.hash).to_string(); - let parent_hash_string = Hex(&header.parent_hash).to_string(); - Ok(BlockMeta { - number: blk.number, - hash: hash_string, - parent_hash: parent_hash_string + number: blk.number, + hash: Hex::encode(&blk.hash), + parent_hash: Hex::encode(&header.parent_hash), }) -} \ No newline at end of file +} diff --git a/ethereum-explorer/src/map_contract_events.rs b/ethereum-explorer/src/map_contract_events.rs index 3d4bfc9..8d208c0 100644 --- a/ethereum-explorer/src/map_contract_events.rs +++ b/ethereum-explorer/src/map_contract_events.rs @@ -1,73 +1,32 @@ -use crate::pb::eth::event::v1::Events; use crate::pb::eth::event::v1::Event; -use substreams_ethereum::pb::eth::v2::Block; -use substreams::Hex; -use substreams_ethereum::pb::eth::v2::Log; -use substreams_ethereum::pb::eth::v2::TransactionTrace; -use substreams::errors::Error; +use crate::pb::eth::event::v1::Events; use crate::util; +use anyhow::Ok; +use substreams::errors::Error; +use substreams::Hex; +use substreams_ethereum::pb::eth::v2::Block; +use anyhow::anyhow; #[substreams::handlers::map] fn map_contract_events(contract_address: String, blk: Block) -> Result { - let error = verify_parameter(&contract_address); - if error.is_some() { - return Err(error.unwrap()); - } - - let mut events: Vec = Vec::new(); - let contract_address_as_vec = match Hex::decode(&contract_address) { - Ok(address) => address, - Err(error) => return Err(Error::Unexpected(error.to_string())), - }; - - for transaction in &blk.transaction_traces { - if transaction.to == contract_address_as_vec { - let transaction_events = get_transaction_events(&transaction); - events.extend(transaction_events); - } - } + verify_parameter(&contract_address)?; + + let events: Vec = blk.logs() + .filter(| log| log.address().to_vec() == Hex::decode(&contract_address).expect("already validated")) + .map(|log| Event { + address: Hex::encode(log.address()), + topics: log.topics().into_iter().map(| topic| Hex::encode(topic)).collect(), + tx_hash: Hex::encode(&log.receipt.transaction.hash) + }) + .collect(); Ok(Events { events }) } -fn verify_parameter(contract_address: &String) -> Option { - if !util::is_address_valid(contract_address) { - return Some(Error::Unexpected(String::from("Contract address is not valid"))) +fn verify_parameter(address: &String) -> Result<(), Error> { + if !util::is_address_valid(&address) { + return Err(anyhow!("Contract address is not valid")); } - return None -} - -fn get_transaction_events(transaction: &TransactionTrace) -> Vec { - let mut transaction_events: Vec = Vec::new(); - - for log in &transaction.receipt().receipt.logs { - let address = util::hexadecimal_to_string(&log.address); - let topics = get_log_topics(&log); - - let event = create_event_from(address, topics, util::hexadecimal_to_string(&transaction.hash)); - transaction_events.push(event) - } - - return transaction_events; -} - -fn get_log_topics(log: &Log) -> Vec { - let mut topics: Vec = Vec::new(); - - for topic in &log.topics { - let topic_string = util::hexadecimal_to_string(topic); - topics.push(topic_string) - } - - return topics; -} - -fn create_event_from(address: String, topics: Vec, hash: String) -> Event { - - return Event { - address, - topics, - tx_hash: hash - } + return Ok(()); } \ No newline at end of file diff --git a/ethereum-explorer/src/map_filter_transactions.rs b/ethereum-explorer/src/map_filter_transactions.rs index f2e53b2..5791692 100644 --- a/ethereum-explorer/src/map_filter_transactions.rs +++ b/ethereum-explorer/src/map_filter_transactions.rs @@ -1,77 +1,72 @@ -use substreams_ethereum::pb::eth::v2::{Block, TransactionTraceStatus}; use crate::pb::eth::transaction::v1::{Transaction, Transactions}; -use serde::Deserialize; -use substreams::errors::Error; use crate::util; +use serde::Deserialize; +use substreams::Hex; +use substreams_ethereum::pb::eth::v2::{Block, TransactionTraceStatus, TransactionTrace}; +use anyhow::anyhow; #[derive(Deserialize)] struct TransactionFilterParams { - hash: Option, to: Option, - from: Option + from: Option, } #[substreams::handlers::map] -pub fn map_filter_transactions(params: String, blk: Block) -> Result> { +fn map_filter_transactions(params: String, blk: Block) -> Result> { let filters: TransactionFilterParams = serde_qs::from_str(¶ms).unwrap(); - let errors = verify_filter_params(&filters); + verify_filter_params(&filters)?; + + let transactions: Vec = blk.transactions() + .filter(|trans| apply_filter(&trans, &filters)) + .map(|trans| Transaction { + from: Hex::encode(&trans.from), + to: Hex::encode(&trans.to), + hash: Hex::encode(&trans.hash), + }).collect(); + + + Ok(Transactions { + transactions, + }) +} + +fn verify_filter_params(params: &TransactionFilterParams) -> Result<(), Vec> { + let mut errors: Vec = Vec::new(); + + if params.from.is_some() && !util::is_address_valid(¶ms.from.as_ref().unwrap()) { + errors.push(anyhow!("'from' address is not valid")); + } + + if params.to.is_some() && !util::is_address_valid(¶ms.to.as_ref().unwrap()) { + errors.push(anyhow!("'to' address is not valid")); + } + if errors.len() > 0 { return Err(errors) } - let mut filtered_transactions: Vec = Vec::new(); - - for transaction in &blk.transaction_traces { - let tx_hash = util::hexadecimal_to_string(&transaction.hash); - let tx_from = util::hexadecimal_to_string(&transaction.from); - let tx_to = util::hexadecimal_to_string(&transaction.to); - let mut current_transaction_filtered = true; - - if !filter_by_parameter(&filters.hash, &tx_hash) || - !filter_by_parameter(&filters.from, &tx_from) || - !filter_by_parameter(&filters.to, &tx_to) || - transaction.status != (TransactionTraceStatus::Succeeded as i32) { - current_transaction_filtered = false - } - - if current_transaction_filtered { - let trans = Transaction { from: tx_from, to: tx_to, hash: tx_hash }; - filtered_transactions.push(trans) - } - } - - Ok(Transactions { transactions: filtered_transactions }) + return Ok(()) } -fn verify_filter_params(params: &TransactionFilterParams) -> Vec { - let mut errors: Vec = Vec::new(); - - if params.hash.is_some() - && !util::is_transaction_hash_valid(¶ms.hash.as_ref().unwrap()) { - errors.push(Error::Unexpected(String::from("Transaction hash is not valid"))); +fn apply_filter(transaction: &TransactionTrace, filters: &TransactionFilterParams) -> bool { + if !filter_by_parameter(&filters.from, &transaction.from) + || !filter_by_parameter(&filters.to, &transaction.to) + || transaction.status != (TransactionTraceStatus::Succeeded as i32) { + return false; } - if params.from.is_some() - && !util::is_address_valid(¶ms.from.as_ref().unwrap()) { - errors.push(Error::Unexpected(String::from("'from' address is not valid"))); - } - - if params.to.is_some() - && !util::is_address_valid(¶ms.to.as_ref().unwrap()) { - errors.push(Error::Unexpected(String::from("'to' address is not valid"))); - } - - return errors; + return true; } -fn filter_by_parameter(parameter: &Option, transaction_field: &String) -> bool { +fn filter_by_parameter(parameter: &Option, transaction_field: &Vec) -> bool { if parameter.is_none() { return true; } - if parameter.as_ref().unwrap() == transaction_field { - return true + let parameter_as_vec = &Hex::decode(parameter.as_ref().unwrap()).expect("already verified"); + if parameter_as_vec == transaction_field { + return true; } return false; -} \ No newline at end of file +} diff --git a/ethereum-explorer/src/util.rs b/ethereum-explorer/src/util.rs index 42e4997..cfe61a0 100644 --- a/ethereum-explorer/src/util.rs +++ b/ethereum-explorer/src/util.rs @@ -1,23 +1,8 @@ -use substreams::Hex; - -pub fn hexadecimal_to_string(hex: &Vec) -> String { - return Hex::encode(hex); -} - -pub fn is_transaction_hash_valid(hash: &String) -> bool { - // A transaction hash is always 64 hexadecimal characters - if hash.len() != 64 { +pub fn is_address_valid(address: &String) -> bool { + // An address is always 40 hexadecimal characters (or 2 more character with 0x prefix) + if address.len() != 40 && address.len() != 42 { return false; } - - return true; -} - -pub fn is_address_valid(address: &String) -> bool { - // An address is always 40 hexadecimal characters - if address.len() != 40 { - return false; - } return true; } diff --git a/ethereum-explorer/substreams.yaml b/ethereum-explorer/substreams.yaml index 51c6f6a..94b3f53 100644 --- a/ethereum-explorer/substreams.yaml +++ b/ethereum-explorer/substreams.yaml @@ -39,5 +39,5 @@ modules: type: proto:eth.event.v1.Events params: - map_filter_transactions: "to=dac17f958d2ee523a2206206994597c13d831ec7&from=46340b20830761efd32832a74d7169b29feb9758" - map_contract_events: "bc4ca0eda7647a8ab7c2061c2e118a18a936f13d" + map_filter_transactions: "to=0xae0Ee0A63A2cE6BaeEFFE56e7714FB4EFE48D419" + map_contract_events: "0x6e60bCdF52078A250932CF9FeC174c5F67348845" \ No newline at end of file From 38c52d9a2a74f3f59c26ba43c7a7d4523effa3ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 27 Jul 2023 11:19:31 +0200 Subject: [PATCH 7/8] Upgrade to new version of substreams library & README --- ethereum-explorer/Cargo.lock | 20 +- ethereum-explorer/Cargo.toml | 5 +- ethereum-explorer/README.md | 196 +++++++++++++++--- ethereum-explorer/src/map_contract_events.rs | 2 +- .../src/map_filter_transactions.rs | 6 +- ethereum-explorer/src/util.rs | 2 +- ethereum-explorer/substreams.yaml | 4 +- 7 files changed, 188 insertions(+), 47 deletions(-) diff --git a/ethereum-explorer/Cargo.lock b/ethereum-explorer/Cargo.lock index e833191..c2a3e7c 100644 --- a/ethereum-explorer/Cargo.lock +++ b/ethereum-explorer/Cargo.lock @@ -723,18 +723,18 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "serde" -version = "1.0.175" +version = "1.0.176" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" +checksum = "76dc28c9523c5d70816e393136b86d48909cfb27cecaa902d338c19ed47164dc" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.175" +version = "1.0.176" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" +checksum = "a4e7b8c5dc823e3b90651ff1d3808419cd14e5ad76de04feaf37da114e7a306f" dependencies = [ "proc-macro2", "quote", @@ -743,9 +743,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ "itoa", "ryu", @@ -781,7 +781,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "substreams" -version = "0.5.9" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af972e374502cdfc9998132f5343848d1c58f27a295dc061a89804371f408a46" dependencies = [ "anyhow", "bigdecimal", @@ -878,7 +880,9 @@ dependencies = [ [[package]] name = "substreams-macro" -version = "0.5.9" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6521ccd011a4c3f52cd3c31fc7400733e4feba2094e0e0e6354adca25b2b3f37" dependencies = [ "proc-macro2", "quote", diff --git a/ethereum-explorer/Cargo.toml b/ethereum-explorer/Cargo.toml index b52f752..1881b16 100644 --- a/ethereum-explorer/Cargo.toml +++ b/ethereum-explorer/Cargo.toml @@ -16,7 +16,7 @@ hex-literal = "0.3" num-bigint = "0.4" prost = "0.11" # Use latest from https://crates.io/crates/substreams -substreams = "0.5.9" +substreams = "0.5" # Use latest from https://crates.io/crates/substreams-ethereum substreams-ethereum = "0.9" serde_qs = "0.12.0" @@ -31,6 +31,3 @@ getrandom = { version = "0.2", features = ["custom"] } lto = true opt-level = 's' strip = "debuginfo" - -[patch.crates-io] -substreams = { path = "/Users/enolalvarezdeprado/Documents/streamingfast/substreams-rs/substreams" } \ No newline at end of file diff --git a/ethereum-explorer/README.md b/ethereum-explorer/README.md index 8d64146..fc86b2a 100644 --- a/ethereum-explorer/README.md +++ b/ethereum-explorer/README.md @@ -14,12 +14,13 @@ The modules in this repository answer some interesting questions when developing For every block, the `map_block_meta` module retrieves the most relevant information of the block (number, hash, and parent hash). -### How Can You Retrieve a Specific Transaction By Its Hash? +### How Can You Retrieve Transactions By Their From or To Fields? -Given a transaction hash parameter (`tx_hash`), the `map_filter_transaction` filters a transaction among all transactions in the blockchain. This involves: +Given any combination of two parameters (`from` and `to`), the `map_filter_transactions` filters a transaction among all transactions in the blockchain. This involves: +1. Providing the filters (only the `from` fields, only the `to` field, both `from` and `to` fields, or none) 1. Iterating over all the transactions. -2. Filter the transactions, where the `hash` field is equal to the hash parameter (`hash == tx_hash`). +2. Filter the transactions, according to the parameters provided. For example, `from == tx_from`, `from == tx_from and to == tx_to`. ### How Can You Retrieve All the Events for a Specific Smart Contract? @@ -71,51 +72,190 @@ Backprocessing history up to requested target block 17712040: all done ``` -### Running the "map_filter_transaction" Module +### Running the "map_filter_transactions" Module To run this module, you must provide a transaction hash, so that Substreams can filter the transactions accordingly. The Substreams manifest (`substreams.yaml`) contains a default transaction hash (`4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7`) in the `params` section, which you can update. +This module allows you to filter transactions by two fields: `from` (the address that created the transaction) and `to` (the address that received the transaction). The Substreams manifest (`substreams.yaml`) contains a sample filter in the `params` section: + +yml +``` +map_filter_transactions: "to=0xdAC17F958D2ee523a2206206994597C13D831ec7" +``` +This filter only returns the transactions received by the USDT contract address (`0xdAC17F958D2ee523a2206206994597C13D831ec7`). Other possibilities for the filter include: + +- Using just `from`: `map_filter_transactions: "from=0xdAC17F958D2ee523a2206206994597C13D831ec7"` +- Using `from` and `to`: `map_filter_transactions: "from=0x1A384fE78e92F3e41e1B21ad79CB93A557Ab4EfD&to=0xdAC17F958D2ee523a2206206994597C13D831ec7"` +- Without filters (returns all the transactions): remove the `map_filter_transactions` entry in the `params` section of the manifest. + The `4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7` transaction is at block number `17712040`. In order to avoid iterating over the full blockchain, the following command starts searching at block number `17712038`. ```bash -> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_filter_transactions --start-block 17712038 --stop-block +10 -Connected (trace ID 9dce06621a0ec353213adeaf9f10ef79) +> substreams run -e mainnet.eth.streamingfast.io:443 substreams.yaml map_filter_transactions --start-block 17712038 --stop-block +3 +Connected (trace ID 5d4ab3248f19fa87052f523259353ff2) Progress messages received: 0 (0/sec) Backprocessing history up to requested target block 17712038: (hit 'm' to switch mode) ----------- BLOCK #17,712,038 (b96fc7e71c0daf69b19211c45fbb5c201f4356fb2b5607500b7d88d298599f5b) --------------- ------------ BLOCK #17,712,039 (1385f853d28b16ad7ebc5d51b6f2ef6d43df4b57bd4c6fe4ef8ccb6f266d8b91) --------------- ------------ BLOCK #17,712,040 (31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b) --------------- { - "@module": "map_filter_transaction", - "@block": 17712040, - "@type": "eth.transaction.v1.TransactionOption", + "@module": "map_filter_transactions", + "@block": 17712038, + "@type": "eth.transaction.v1.Transactions", "@data": { - "transaction": { - "from": "e93685f3bba03016f02bd1828badd6195988d950", - "to": "902f09715b6303d4173037652fa7377e5b98089e", - "hash": "4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7" - } + "transactions": [ + { + "from": "b6692f7ae54e89da0269c1bfd685ccdfd41d2bf7", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "933b74565234ac9ca8389f7a49fad80099abf1be77e4bef5af69ade30127f30e" + }, + { + "from": "0543ba40d4b8b33dc5f7d163f41c6dc54cf1d923", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "51071d7a94fc6ecfec2aba477c26ff5098db3e36a287d43d13b763b3118b160b" + }, + { + "from": "09e52cbb57dce8cd2836effc44686b6008a84914", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "806ef36a1e022d52d00a288bc150676af0cb2bad6b5500378c8fc7253a0434fa" + }, + { + "from": "2f13d388b85e0ecd32e7c3d7f36d1053354ef104", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "0a9a5707b5d4047b1e44de9283f0c88606eac49b4eb132a61df0dffc20668ad0" + }, + { + "from": "4fac9d83ffad797072db8bd72cc544ad5ec45e4f", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "8466f371eed9b742a2ed869213dde10661e3df22366e258e09f68e37ca47b2c1" + }, + { + "from": "48c04ed5691981c42154c6167398f95e8f38a7ff", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "571670afd47e29fe901c1b17ed21fca6088cc9540efd684c5b7b4c1c1e748612" + }, + { + "from": "4c8e30406f5dbedfaa18cb6b9d0484cd5390490a", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "558031630b43c8c61e36d742a779f967f3f0102fa290111f6f6f9c2acaadf3ea" + } + ] + } +} + +----------- BLOCK #17,712,039 (1385f853d28b16ad7ebc5d51b6f2ef6d43df4b57bd4c6fe4ef8ccb6f266d8b91) --------------- +{ + "@module": "map_filter_transactions", + "@block": 17712039, + "@type": "eth.transaction.v1.Transactions", + "@data": { + "transactions": [ + { + "from": "75e89d5979e4f6fba9f97c104c2f0afb3f1dcb88", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "43e0e1b6315c4cc1608d876f98c9bbf09f2a25404aabaeac045b5cc852df0e85" + }, + { + "from": "75e89d5979e4f6fba9f97c104c2f0afb3f1dcb88", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "619d94c33b027df694cbf32659aae51743623b4d1cb11c69d7d0e95cad63b712" + }, + { + "from": "75e89d5979e4f6fba9f97c104c2f0afb3f1dcb88", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "027cccdba1a127bcfb5bb39b5d89e3552e83c8c3c6dd13cf779d7720241e71b9" + }, + { + "from": "3d1d8a1d418220fd53c18744d44c182c46f47468", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "762350dcf3ab62ad515331436ce952ba5b3641bbf87c7d56c1e8a9f21473875c" + }, + { + "from": "a45c27ef3df487525b33a70cb0020de792dc7a3f", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "b9e08dfe7b1f4971ea96d1424c32548028bdeb62b2ee7f6775dd55d05c4d4ad6" + }, + { + "from": "9696f59e4d72e237be84ffd425dcad154bf96976", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "44f36363290969d8b581bb9a856bc9f2ca9a64e4a12e4db054927a45795480fa" + }, + { + "from": "e074f1967080cd7b9352c8cbe2d1d9cd121d4daf", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "8795aa5088fb13a21048c592316ad7da850a8f80f3ce417bc4d7d2bbeca3f596" + }, + { + "from": "fb8131c260749c7835a08ccbdb64728de432858e", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "0a907108aecaf909452f7035070a28f9cad6c51896763e760ea1f544a9b9edf3" + }, + { + "from": "e41febca31f997718d2ddf6b21b9710c5c7a3425", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "45c03fcbefcce9920806dcd7d638cef262ad405f8beae383fbc2695ad4bc9b1b" + } + ] + } +} + +----------- BLOCK #17,712,040 (31ad07fed936990d3c75314589b15cbdec91e4cc53a984a43de622b314c38d0b) --------------- +{ + "@module": "map_filter_transactions", + "@block": 17712040, + "@type": "eth.transaction.v1.Transactions", + "@data": { + "transactions": [ + { + "from": "48c04ed5691981c42154c6167398f95e8f38a7ff", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "137799eea9fa8ae410c913e16ebc5cc8a01352a638f3ce6f3f29a283ad918987" + }, + { + "from": "7c0a7899f69a7034325ffee90355906cf72aeebb", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "132fc93b8a155c614001665a40381c8de9ad7519034352628c075e17a06d884b" + }, + { + "from": "180277c2f8bd489a4e27e261c6fbca079b6fa58f", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "06e74e08b51a0c03219c3aa12a871595516c1d466611ed848ea2ae8cbfb083ea" + }, + { + "from": "1440ec793ae50fa046b95bfeca5af475b6003f9e", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "83862ea45a6f777acd81a3469c54e347d3eb527cbee9fb673c6e312f7ae6fb83" + }, + { + "from": "89e51fa8ca5d66cd220baed62ed01e8951aa7c40", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "1b9e5059181ca90969ee423beea3073cf99faf8a91b73890303531ebd6c197ec" + }, + { + "from": "89e51fa8ca5d66cd220baed62ed01e8951aa7c40", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "ca1750068bee961ccd2e45679c9d9dadc5ba93fd3212c0f31361d39abe3ed36c" + }, + { + "from": "82cbcb64a2eb51622fb847c9c957fdac532712ac", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "284b6359cf66a010798738bb764f5cd015658e8f59273a49e19a855731f22bb8" + }, + { + "from": "f89d7b9c864f589bbf53a82105107622b35eaa40", + "to": "dac17f958d2ee523a2206206994597c13d831ec7", + "hash": "0544143b459969c9ed36741533fba70d6ea7069f156d2019d5362c06bf8d887f" + } + ] } } ------------ BLOCK #17,712,041 (1af541642176c51580b54de214e955fba8bf1b82af569b81d4038956f2402a41) --------------- ------------ BLOCK #17,712,042 (43c6ebe5b89dd689a9f07468a04d0faf5274a46d0763056ea53b8b1e5ac32148) --------------- ------------ BLOCK #17,712,043 (2457f742913dbbdb171a8d8cc3b1ef8a383f8f547982700a646aa97581bfaeb8) --------------- ------------ BLOCK #17,712,044 (af7abef4f80d7c6f3111e761bb86f305e9847e544fb36b55ba5b64e6103bd5d3) --------------- ------------ BLOCK #17,712,045 (db4664944f7ca8aed5de798bf74ebbdbbeda60e58316b4291bfec61c7287fb17) --------------- ------------ BLOCK #17,712,046 (d9bce9b9210b7deb746720435d1eca99b87fe17aaf7d5055fcd54959e0c9932e) --------------- ------------ BLOCK #17,712,047 (3a5ffab8dacbaf89e10b66e9d2f7ebe65bddae4dcb5e5e8739f8b938f16f98ec) --------------- all done ``` -As you can see, the Substreams does not provide an output for the blocks where the transaction is not present. -You can check out the transaction at [Etherscan](https://etherscan.io/tx/0x4faa877df84080a9d98b1e28294c4680bb141ec27a1a5dee009c3e02dfa65ab7). +### Running the "map_contract_events" Module -### Running the "map_filter_transaction" Module - -To run this module, you must provide the address of a smart contract. By default, the `params` section of the Substreams manifest contains the `bc4ca0eda7647a8ab7c2061c2e118a18a936f13d` address, corresponding to the Bore Yacht Club smart contract. You can change this address to track any smart contract. +To run this module, you must provide the address of a smart contract. By default, the `params` section of the Substreams manifest contains the `0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d` address, corresponding to the Bore Yacht Club smart contract. You can change this address to track any smart contract. The logs of a smart contract might be split across thousands of Ethereum blocks, so for testing purposes, and to avoid iterating over the full blockchain, the following command limits starts searching at block number `17717995`. diff --git a/ethereum-explorer/src/map_contract_events.rs b/ethereum-explorer/src/map_contract_events.rs index 8d208c0..16c07ed 100644 --- a/ethereum-explorer/src/map_contract_events.rs +++ b/ethereum-explorer/src/map_contract_events.rs @@ -28,5 +28,5 @@ fn verify_parameter(address: &String) -> Result<(), Error> { return Err(anyhow!("Contract address is not valid")); } - return Ok(()); + Ok(()) } \ No newline at end of file diff --git a/ethereum-explorer/src/map_filter_transactions.rs b/ethereum-explorer/src/map_filter_transactions.rs index 5791692..ad6825d 100644 --- a/ethereum-explorer/src/map_filter_transactions.rs +++ b/ethereum-explorer/src/map_filter_transactions.rs @@ -45,7 +45,7 @@ fn verify_filter_params(params: &TransactionFilterParams) -> Result<(), Vec bool { @@ -55,7 +55,7 @@ fn apply_filter(transaction: &TransactionTrace, filters: &TransactionFilterParam return false; } - return true; + true } fn filter_by_parameter(parameter: &Option, transaction_field: &Vec) -> bool { @@ -68,5 +68,5 @@ fn filter_by_parameter(parameter: &Option, transaction_field: &Vec) return true; } - return false; + false } diff --git a/ethereum-explorer/src/util.rs b/ethereum-explorer/src/util.rs index cfe61a0..4a7e463 100644 --- a/ethereum-explorer/src/util.rs +++ b/ethereum-explorer/src/util.rs @@ -4,5 +4,5 @@ pub fn is_address_valid(address: &String) -> bool { return false; } - return true; + true } diff --git a/ethereum-explorer/substreams.yaml b/ethereum-explorer/substreams.yaml index 94b3f53..88ee0d8 100644 --- a/ethereum-explorer/substreams.yaml +++ b/ethereum-explorer/substreams.yaml @@ -39,5 +39,5 @@ modules: type: proto:eth.event.v1.Events params: - map_filter_transactions: "to=0xae0Ee0A63A2cE6BaeEFFE56e7714FB4EFE48D419" - map_contract_events: "0x6e60bCdF52078A250932CF9FeC174c5F67348845" \ No newline at end of file + map_filter_transactions: "to=0xdAC17F958D2ee523a2206206994597C13D831ec7" + map_contract_events: "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d" \ No newline at end of file From 0618138987ce89a572a3cbe675714b29a59d96d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enol=20=C3=81lvarez?= Date: Thu, 27 Jul 2023 19:44:06 +0200 Subject: [PATCH 8/8] Fixes --- ethereum-explorer/.rustfmt.toml | 1 + ethereum-explorer/src/map_contract_events.rs | 25 ++++----- .../src/map_filter_transactions.rs | 51 +++++++++++++------ ethereum-explorer/substreams.yaml | 4 +- 4 files changed, 52 insertions(+), 29 deletions(-) create mode 100644 ethereum-explorer/.rustfmt.toml diff --git a/ethereum-explorer/.rustfmt.toml b/ethereum-explorer/.rustfmt.toml new file mode 100644 index 0000000..866c756 --- /dev/null +++ b/ethereum-explorer/.rustfmt.toml @@ -0,0 +1 @@ +max_width = 120 \ No newline at end of file diff --git a/ethereum-explorer/src/map_contract_events.rs b/ethereum-explorer/src/map_contract_events.rs index 16c07ed..00da71a 100644 --- a/ethereum-explorer/src/map_contract_events.rs +++ b/ethereum-explorer/src/map_contract_events.rs @@ -1,32 +1,33 @@ use crate::pb::eth::event::v1::Event; use crate::pb::eth::event::v1::Events; use crate::util; +use anyhow::anyhow; use anyhow::Ok; use substreams::errors::Error; use substreams::Hex; use substreams_ethereum::pb::eth::v2::Block; -use anyhow::anyhow; #[substreams::handlers::map] fn map_contract_events(contract_address: String, blk: Block) -> Result { verify_parameter(&contract_address)?; - - let events: Vec = blk.logs() - .filter(| log| log.address().to_vec() == Hex::decode(&contract_address).expect("already validated")) - .map(|log| Event { - address: Hex::encode(log.address()), - topics: log.topics().into_iter().map(| topic| Hex::encode(topic)).collect(), - tx_hash: Hex::encode(&log.receipt.transaction.hash) - }) - .collect(); + + let events: Vec = blk + .logs() + .filter(|log| log.address().to_vec() == Hex::decode(&contract_address).expect("already validated")) + .map(|log| Event { + address: Hex::encode(log.address()), + topics: log.topics().into_iter().map(Hex::encode).collect(), + tx_hash: Hex::encode(&log.receipt.transaction.hash), + }) + .collect(); Ok(Events { events }) } fn verify_parameter(address: &String) -> Result<(), Error> { if !util::is_address_valid(&address) { - return Err(anyhow!("Contract address is not valid")); + return Err(anyhow!("Contract address ({}) is not valid", address)); } Ok(()) -} \ No newline at end of file +} diff --git a/ethereum-explorer/src/map_filter_transactions.rs b/ethereum-explorer/src/map_filter_transactions.rs index ad6825d..aa2e6ad 100644 --- a/ethereum-explorer/src/map_filter_transactions.rs +++ b/ethereum-explorer/src/map_filter_transactions.rs @@ -1,9 +1,9 @@ use crate::pb::eth::transaction::v1::{Transaction, Transactions}; use crate::util; +use anyhow::anyhow; use serde::Deserialize; use substreams::Hex; -use substreams_ethereum::pb::eth::v2::{Block, TransactionTraceStatus, TransactionTrace}; -use anyhow::anyhow; +use substreams_ethereum::pb::eth::v2::{Block, TransactionTrace, TransactionTraceStatus}; #[derive(Deserialize)] struct TransactionFilterParams { @@ -13,36 +13,54 @@ struct TransactionFilterParams { #[substreams::handlers::map] fn map_filter_transactions(params: String, blk: Block) -> Result> { - let filters: TransactionFilterParams = serde_qs::from_str(¶ms).unwrap(); - verify_filter_params(&filters)?; + let filters = parse_filters_from_params(params)?; - let transactions: Vec = blk.transactions() + let transactions: Vec = blk + .transactions() .filter(|trans| apply_filter(&trans, &filters)) .map(|trans| Transaction { from: Hex::encode(&trans.from), to: Hex::encode(&trans.to), hash: Hex::encode(&trans.hash), - }).collect(); - + }) + .collect(); - Ok(Transactions { - transactions, - }) + Ok(Transactions { transactions }) } -fn verify_filter_params(params: &TransactionFilterParams) -> Result<(), Vec> { +fn parse_filters_from_params(params: String) -> Result> { + let parsed_result = serde_qs::from_str(¶ms); + if parsed_result.is_err() { + return Err(Vec::from([anyhow!("Unpexcted error while parsing parameters")])); + } + + let filters = parsed_result.unwrap(); + verify_filters(&filters)?; + + Ok(filters) +} + +fn verify_filters(params: &TransactionFilterParams) -> Result<(), Vec> { let mut errors: Vec = Vec::new(); if params.from.is_some() && !util::is_address_valid(¶ms.from.as_ref().unwrap()) { - errors.push(anyhow!("'from' address is not valid")); + let from = params.from.as_ref().unwrap(); + + if !util::is_address_valid(from) { + errors.push(anyhow!("'from' address ({}) is not valid", from)); + } } if params.to.is_some() && !util::is_address_valid(¶ms.to.as_ref().unwrap()) { - errors.push(anyhow!("'to' address is not valid")); + let to = params.to.as_ref().unwrap(); + + if !util::is_address_valid(to) { + errors.push(anyhow!("'to' address ({}) is not valid", to)); + } } if errors.len() > 0 { - return Err(errors) + return Err(errors); } Ok(()) @@ -51,7 +69,8 @@ fn verify_filter_params(params: &TransactionFilterParams) -> Result<(), Vec bool { if !filter_by_parameter(&filters.from, &transaction.from) || !filter_by_parameter(&filters.to, &transaction.to) - || transaction.status != (TransactionTraceStatus::Succeeded as i32) { + || transaction.status != (TransactionTraceStatus::Succeeded as i32) + { return false; } @@ -64,7 +83,7 @@ fn filter_by_parameter(parameter: &Option, transaction_field: &Vec) } let parameter_as_vec = &Hex::decode(parameter.as_ref().unwrap()).expect("already verified"); - if parameter_as_vec == transaction_field { + if transaction_field == parameter_as_vec { return true; } diff --git a/ethereum-explorer/substreams.yaml b/ethereum-explorer/substreams.yaml index 88ee0d8..e99db20 100644 --- a/ethereum-explorer/substreams.yaml +++ b/ethereum-explorer/substreams.yaml @@ -39,5 +39,7 @@ modules: type: proto:eth.event.v1.Events params: + # Filtering transactions with to = 0xdAC17F958D2ee523a2206206994597C13D831ec7 (USDT contract address) map_filter_transactions: "to=0xdAC17F958D2ee523a2206206994597C13D831ec7" - map_contract_events: "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d" \ No newline at end of file + # Getting the logs of the BoredApeYachtClub smart contract + map_contract_events: "0xbc4ca0eda7647a8ab7c2061c2e118a18a936f13d" \ No newline at end of file