From 92f20d0521dcd16c2ba11fc54d6f51c37873935e Mon Sep 17 00:00:00 2001 From: kayibal Date: Mon, 11 Mar 2024 20:08:10 +0000 Subject: [PATCH 1/8] Add shared tycho-substreams lib. --- crates/tycho-substreams/Cargo.lock | 1050 +++++++++++++++++ crates/tycho-substreams/Cargo.toml | 7 + crates/tycho-substreams/Readme.md | 18 + crates/tycho-substreams/buf.gen.yaml | 12 + crates/tycho-substreams/src/contract.rs | 199 ++++ crates/tycho-substreams/src/lib.rs | 2 + crates/tycho-substreams/src/pb/mod.rs | 10 + .../tycho-substreams/src/pb/tycho.evm.v1.rs | 309 +++++ 8 files changed, 1607 insertions(+) create mode 100644 crates/tycho-substreams/Cargo.lock create mode 100644 crates/tycho-substreams/Cargo.toml create mode 100644 crates/tycho-substreams/Readme.md create mode 100644 crates/tycho-substreams/buf.gen.yaml create mode 100644 crates/tycho-substreams/src/contract.rs create mode 100644 crates/tycho-substreams/src/lib.rs create mode 100644 crates/tycho-substreams/src/pb/mod.rs create mode 100644 crates/tycho-substreams/src/pb/tycho.evm.v1.rs diff --git a/crates/tycho-substreams/Cargo.lock b/crates/tycho-substreams/Cargo.lock new file mode 100644 index 0000000..5e61c4a --- /dev/null +++ b/crates/tycho-substreams/Cargo.lock @@ -0,0 +1,1050 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "ethabi" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +dependencies = [ + "ethereum-types", + "hex", + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "thiserror", + "uint", +] + +[[package]] +name = "ethbloom" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +dependencies = [ + "crunchy", + "fixed-hash", + "impl-rlp", + "impl-serde", + "tiny-keccak", +] + +[[package]] +name = "ethereum-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +dependencies = [ + "ethbloom", + "fixed-hash", + "impl-rlp", + "impl-serde", + "primitive-types", + "uint", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-rlp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +dependencies = [ + "rlp", +] + +[[package]] +name = "impl-serde" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "indexmap" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pad" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "parity-scale-codec" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881331e34fa842a2fb61cc2db9643a8fedc615e47cfcc52597d1af0db9a7e8fe" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be30eaf4b0a9fba5336683b38de57bb86d179a35862ba6bfcf57625d006bde5b" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24" +dependencies = [ + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rlp" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" +dependencies = [ + "bytes", + "rustc-hex", +] + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "substreams" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3520661f782c338f0e3c6cfc001ac790ed5e68d8f28515139e2aa674f8bb54da" +dependencies = [ + "anyhow", + "bigdecimal", + "hex", + "hex-literal", + "num-bigint", + "num-integer", + "num-traits", + "pad", + "prost", + "prost-build", + "prost-types", + "substreams-macro", + "thiserror", +] + +[[package]] +name = "substreams-ethereum" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f45dc04be50b7ca08d6d5c4560ee3eeba16ccaa1c124d0361bb30b5b84e28b" +dependencies = [ + "getrandom", + "num-bigint", + "substreams", + "substreams-ethereum-abigen", + "substreams-ethereum-core", + "substreams-ethereum-derive", +] + +[[package]] +name = "substreams-ethereum-abigen" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c04307913a355aaf2a1bb7186d4bc7e36875f3d4aff77b47e83f1b63b24da55" +dependencies = [ + "anyhow", + "ethabi", + "heck", + "hex", + "prettyplease", + "proc-macro2", + "quote", + "substreams-ethereum-core", + "syn 1.0.109", +] + +[[package]] +name = "substreams-ethereum-core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9048cc9a66873ab7069ef958c2684994e6ee323da49c186b19156fdb4ca131" +dependencies = [ + "bigdecimal", + "ethabi", + "getrandom", + "num-bigint", + "prost", + "prost-build", + "prost-types", + "substreams", +] + +[[package]] +name = "substreams-ethereum-derive" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e862928bee8653f5c9291ac619c8dc0da14ca61d8cd8d89b3acdbbde4d0bf304" +dependencies = [ + "ethabi", + "heck", + "hex", + "num-bigint", + "proc-macro2", + "quote", + "substreams-ethereum-abigen", + "syn 1.0.109", +] + +[[package]] +name = "substreams-macro" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15595ceab80fece579e462d4823048fe85d67922584c681f5e94305727ad9ee" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + +[[package]] +name = "toml_edit" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tycho-substreams" +version = "0.1.0" +dependencies = [ + "substreams-ethereum", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] diff --git a/crates/tycho-substreams/Cargo.toml b/crates/tycho-substreams/Cargo.toml new file mode 100644 index 0000000..269aea3 --- /dev/null +++ b/crates/tycho-substreams/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "tycho-substreams" +version = "0.1.0" +edition = "2021" + +[dependencies] +substreams-ethereum = "0.9.9" \ No newline at end of file diff --git a/crates/tycho-substreams/Readme.md b/crates/tycho-substreams/Readme.md new file mode 100644 index 0000000..56b374b --- /dev/null +++ b/crates/tycho-substreams/Readme.md @@ -0,0 +1,18 @@ +# Tycho Substreams SDK + +Some shared functionality that is used to create tycho substream packages. + +## Protobuf Models + +Protobuf models are manually synced from the `tycho-indexer` repository whenever they +changed. + +To generate the rust structs run the following command from within the `./proto` +directory: + +```bash +buf generate \ + --path tycho \ + --template ../crates/tycho-substreams/buf.gen.yaml \ + --output ../crates/tycho-substreams/ +``` \ No newline at end of file diff --git a/crates/tycho-substreams/buf.gen.yaml b/crates/tycho-substreams/buf.gen.yaml new file mode 100644 index 0000000..07f4f81 --- /dev/null +++ b/crates/tycho-substreams/buf.gen.yaml @@ -0,0 +1,12 @@ +version: v1 +plugins: + - plugin: buf.build/community/neoeinstein-prost:v0.2.2 + out: src/pb + opt: + - file_descriptor_set=false + - type_attribute=.tycho.evm.v1.Transaction=#[derive(Eq\, Hash)] + + - plugin: buf.build/community/neoeinstein-prost-crate:v0.3.1 + out: src/pb + opt: + - no_features diff --git a/crates/tycho-substreams/src/contract.rs b/crates/tycho-substreams/src/contract.rs new file mode 100644 index 0000000..a0391b3 --- /dev/null +++ b/crates/tycho-substreams/src/contract.rs @@ -0,0 +1,199 @@ +/// This file contains helpers to capture contract changes from the expanded block model. These +/// leverage the `code_changes`, `balance_changes`, and `storage_changes` fields available on the +/// `Call` type provided by block model in a substream (i.e. `logs_and_calls`, etc). +/// +/// ⚠️ These helpers *only* work if the **expanded block model** is available, +/// more [here](https://streamingfastio.medium.com/new-block-model-to-accelerate-chain-integration-9f65126e5425) +use std::collections::HashMap; + +use substreams_ethereum::pb::eth; + +use crate::pb::tycho::evm::v1::{self as tycho}; + +struct SlotValue { + new_value: Vec, + start_value: Vec, +} + +impl SlotValue { + fn has_changed(&self) -> bool { + self.start_value != self.new_value + } +} + +// Uses a map for slots, protobuf does not allow bytes in hashmap keys +pub struct InterimContractChange { + address: Vec, + balance: Vec, + code: Vec, + slots: HashMap, SlotValue>, + change: tycho::ChangeType, +} + +impl From for tycho::ContractChange { + fn from(value: InterimContractChange) -> Self { + tycho::ContractChange { + address: value.address, + balance: value.balance, + code: value.code, + slots: value + .slots + .into_iter() + .filter(|(_, value)| value.has_changed()) + .map(|(slot, value)| tycho::ContractSlot { + slot, + value: value.new_value, + }) + .collect(), + change: value.change.into(), + } + } +} + +pub fn extract_contract_changes bool >( + block: ð::v2::Block, + inclusion_predicate: F, + transaction_contract_changes: &mut HashMap, +) { + let mut changed_contracts: HashMap, InterimContractChange> = HashMap::new(); + + // Collect all accounts created in this block + let created_accounts: HashMap<_, _> = block + .transactions() + .flat_map(|tx| { + tx.calls.iter().flat_map(|call| { + call.account_creations + .iter() + .map(|ac| (&ac.account, ac.ordinal)) + }) + }) + .collect(); + + block.transactions().for_each(|block_tx| { + let mut storage_changes = Vec::new(); + let mut balance_changes = Vec::new(); + let mut code_changes = Vec::new(); + + block_tx + .calls + .iter() + .filter(|call| { + !call.state_reverted + && inclusion_predicate(&call.address) + }) + .for_each(|call| { + storage_changes.extend(call.storage_changes.iter()); + balance_changes.extend(call.balance_changes.iter()); + code_changes.extend(call.code_changes.iter()); + }); + + storage_changes.sort_unstable_by_key(|change| change.ordinal); + balance_changes.sort_unstable_by_key(|change| change.ordinal); + code_changes.sort_unstable_by_key(|change| change.ordinal); + + storage_changes + .iter() + .filter(|changes| { + inclusion_predicate(&changes.address) + }) + .for_each(|storage_change| { + let contract_change = changed_contracts + .entry(storage_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: storage_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&storage_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); + + let slot_value = contract_change + .slots + .entry(storage_change.key.clone()) + .or_insert_with(|| SlotValue { + new_value: storage_change.new_value.clone(), + start_value: storage_change.old_value.clone(), + }); + + slot_value + .new_value + .copy_from_slice(&storage_change.new_value); + }); + + balance_changes + .iter() + .filter(|changes| { + inclusion_predicate(&changes.address) + }) + .for_each(|balance_change| { + let contract_change = changed_contracts + .entry(balance_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: balance_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&balance_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); + + if let Some(new_balance) = &balance_change.new_value { + contract_change.balance.clear(); + contract_change + .balance + .extend_from_slice(&new_balance.bytes); + } + }); + + code_changes + .iter() + .filter(|changes| { + inclusion_predicate(&changes.address) + }) + .for_each(|code_change| { + let contract_change = changed_contracts + .entry(code_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: code_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&code_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); + + contract_change.code.clear(); + contract_change + .code + .extend_from_slice(&code_change.new_code); + }); + + if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() { + transaction_contract_changes + .entry(block_tx.index.into()) + .or_insert_with(|| tycho::TransactionContractChanges { + tx: Some(tycho::Transaction { + hash: block_tx.hash.clone(), + from: block_tx.from.clone(), + to: block_tx.to.clone(), + index: block_tx.index as u64, + }), + contract_changes: vec![], + component_changes: vec![], + balance_changes: vec![], + }) + .contract_changes + .extend(changed_contracts.drain().map(|(_, change)| change.into())); + } + }); +} diff --git a/crates/tycho-substreams/src/lib.rs b/crates/tycho-substreams/src/lib.rs new file mode 100644 index 0000000..2f52c0e --- /dev/null +++ b/crates/tycho-substreams/src/lib.rs @@ -0,0 +1,2 @@ +pub mod contract; +pub mod pb; \ No newline at end of file diff --git a/crates/tycho-substreams/src/pb/mod.rs b/crates/tycho-substreams/src/pb/mod.rs new file mode 100644 index 0000000..43d8838 --- /dev/null +++ b/crates/tycho-substreams/src/pb/mod.rs @@ -0,0 +1,10 @@ +// @generated +pub mod tycho { + pub mod evm { + // @@protoc_insertion_point(attribute:tycho.evm.v1) + pub mod v1 { + include!("tycho.evm.v1.rs"); + // @@protoc_insertion_point(tycho.evm.v1) + } + } +} diff --git a/crates/tycho-substreams/src/pb/tycho.evm.v1.rs b/crates/tycho-substreams/src/pb/tycho.evm.v1.rs new file mode 100644 index 0000000..7424ba2 --- /dev/null +++ b/crates/tycho-substreams/src/pb/tycho.evm.v1.rs @@ -0,0 +1,309 @@ +// @generated +// This file contains the proto definitions for Substreams common to all integrations. + +/// A struct describing a block. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + /// The blocks hash. + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + /// The parent blocks hash. + #[prost(bytes="vec", tag="2")] + pub parent_hash: ::prost::alloc::vec::Vec, + /// The block number. + #[prost(uint64, tag="3")] + pub number: u64, + /// The block timestamp. + #[prost(uint64, tag="4")] + pub ts: u64, +} +/// A struct describing a transaction. +#[derive(Eq, Hash)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + /// The transaction hash. + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + /// The sender of the transaction. + #[prost(bytes="vec", tag="2")] + pub from: ::prost::alloc::vec::Vec, + /// The receiver of the transaction. + #[prost(bytes="vec", tag="3")] + pub to: ::prost::alloc::vec::Vec, + /// The transactions index within the block. + /// TODO: should this be uint32? to match the type from the native substream type? + #[prost(uint64, tag="4")] + pub index: u64, +} +/// A custom struct representing an arbitrary attribute of a protocol component. +/// This is mainly used by the native integration to track the necessary information about the protocol. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Attribute { + /// The name of the attribute. + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + /// The value of the attribute. + #[prost(bytes="vec", tag="2")] + pub value: ::prost::alloc::vec::Vec, + /// The type of change the attribute underwent. + #[prost(enumeration="ChangeType", tag="3")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtocolType { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(enumeration="FinancialType", tag="2")] + pub financial_type: i32, + #[prost(message, repeated, tag="3")] + pub attribute_schema: ::prost::alloc::vec::Vec, + #[prost(enumeration="ImplementationType", tag="4")] + pub implementation_type: i32, +} +/// A struct describing a part of the protocol. +/// Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, +/// the component would represent a single contract. In case of VM integration, such component would +/// not need any attributes, because all the relevant info would be tracked via storage slots and balance changes. +/// It can also be a wrapping contract, like WETH, that has a constant price, but it allows swapping tokens. +/// This is why the name ProtocolComponent is used instead of "Pool" or "Pair". +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtocolComponent { + /// A unique identifier for the component within the protocol. + /// Can be e.g. a stringified address or a string describing the trading pair. + #[prost(string, tag="1")] + pub id: ::prost::alloc::string::String, + /// Addresses of the ERC20 tokens used by the component. + #[prost(bytes="vec", repeated, tag="2")] + pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// Addresses of the contracts used by the component. + /// Usually it is a single contract, but some protocols use multiple contracts. + #[prost(bytes="vec", repeated, tag="3")] + pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// Attributes of the component. Used mainly be the native integration. + /// The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent. + #[prost(message, repeated, tag="4")] + pub static_att: ::prost::alloc::vec::Vec, + /// Type of change the component underwent. + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, + /// / Represents the functionality of the component. + #[prost(message, optional, tag="6")] + pub protocol_type: ::core::option::Option, + /// Transaction where this component was created + #[prost(message, optional, tag="7")] + pub tx: ::core::option::Option, +} +/// A struct for following the changes of Total Value Locked (TVL) of a protocol component. +/// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. +/// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceChange { + /// The address of the ERC20 token whose balance changed. + #[prost(bytes="vec", tag="1")] + pub token: ::prost::alloc::vec::Vec, + /// The new balance of the token. + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. + /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. + #[prost(bytes="vec", tag="3")] + pub component_id: ::prost::alloc::vec::Vec, +} +/// Enum to specify the type of a change. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ChangeType { + Unspecified = 0, + Update = 1, + Creation = 2, + Deletion = 3, +} +impl ChangeType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED", + ChangeType::Update => "CHANGE_TYPE_UPDATE", + ChangeType::Creation => "CHANGE_TYPE_CREATION", + ChangeType::Deletion => "CHANGE_TYPE_DELETION", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified), + "CHANGE_TYPE_UPDATE" => Some(Self::Update), + "CHANGE_TYPE_CREATION" => Some(Self::Creation), + "CHANGE_TYPE_DELETION" => Some(Self::Deletion), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FinancialType { + Swap = 0, + Lend = 1, + Leverage = 2, + Psm = 3, +} +impl FinancialType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + FinancialType::Swap => "SWAP", + FinancialType::Lend => "LEND", + FinancialType::Leverage => "LEVERAGE", + FinancialType::Psm => "PSM", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SWAP" => Some(Self::Swap), + "LEND" => Some(Self::Lend), + "LEVERAGE" => Some(Self::Leverage), + "PSM" => Some(Self::Psm), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ImplementationType { + Vm = 0, + Custom = 1, +} +impl ImplementationType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ImplementationType::Vm => "VM", + ImplementationType::Custom => "CUSTOM", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "VM" => Some(Self::Vm), + "CUSTOM" => Some(Self::Custom), + _ => None, + } + } +} +// This file contains the definition for the native integration of Substreams. + +/// A component is a set of attributes that are associated with a custom entity. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EntityChanges { + /// A unique identifier of the entity within the protocol. + #[prost(string, tag="1")] + pub component_id: ::prost::alloc::string::String, + /// The set of attributes that are associated with the entity. + #[prost(message, repeated, tag="2")] + pub attributes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionEntityChanges { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub entity_changes: ::prost::alloc::vec::Vec, + /// An array of newly added components. + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + /// An array of balance changes to components. + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +/// A set of transaction changes within a single block. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockEntityChanges { + /// The block for which these changes are collectively computed. + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + /// The set of transaction changes observed in the specified block. + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +// This file contains proto definitions specific to the VM integration. + +/// A key value entry into contract storage. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractSlot { + /// A contract's storage slot. + #[prost(bytes="vec", tag="2")] + pub slot: ::prost::alloc::vec::Vec, + /// The new value for this storage slot. + #[prost(bytes="vec", tag="3")] + pub value: ::prost::alloc::vec::Vec, +} +/// Changes made to a single contract's state. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractChange { + /// The contract's address + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + /// The new native balance of the contract, empty bytes indicates no change. + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + /// The new code of the contract, empty bytes indicates no change. + #[prost(bytes="vec", tag="3")] + pub code: ::prost::alloc::vec::Vec, + /// The changes to this contract's slots, empty sequence indicates no change. + #[prost(message, repeated, tag="4")] + pub slots: ::prost::alloc::vec::Vec, + /// Whether this is an update, a creation or a deletion. + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, +} +/// A set of changes aggregated by transaction. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionContractChanges { + /// The transaction instance that results in the changes. + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + /// Contains the changes induced by the above transaction, aggregated on a per-contract basis. + /// Must include changes to every contract that is tracked by all ProtocolComponents. + #[prost(message, repeated, tag="2")] + pub contract_changes: ::prost::alloc::vec::Vec, + /// An array of any component changes. + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + /// An array of balance changes to components. + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +/// A set of transaction changes within a single block. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockContractChanges { + /// The block for which these changes are collectively computed. + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + /// The set of transaction changes observed in the specified block. + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +// @@protoc_insertion_point(module) From 3d248f3fa1e18a95efae60237e789d5a6ae6fba8 Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 18:46:10 +0000 Subject: [PATCH 2/8] Implement helpers for relative balances. Adds some helpers that will users help to convert relative balances changes to absolute ones. The goal is to make this as reduce the effort of the user to implementing a simple map handler that extracts relative balances changes. --- crates/tycho-substreams/Cargo.lock | 17 +- crates/tycho-substreams/Cargo.toml | 6 +- crates/tycho-substreams/rustfmt.toml | 13 + crates/tycho-substreams/src/balances.rs | 306 ++++++++++++++++++ crates/tycho-substreams/src/contract.rs | 245 +++++++------- crates/tycho-substreams/src/lib.rs | 6 +- crates/tycho-substreams/src/mock_store.rs | 93 ++++++ .../tycho-substreams/src/pb/tycho.evm.v1.rs | 51 ++- crates/tycho-substreams/src/utils.rs | 24 ++ 9 files changed, 632 insertions(+), 129 deletions(-) create mode 100644 crates/tycho-substreams/rustfmt.toml create mode 100644 crates/tycho-substreams/src/balances.rs create mode 100644 crates/tycho-substreams/src/mock_store.rs create mode 100644 crates/tycho-substreams/src/utils.rs diff --git a/crates/tycho-substreams/Cargo.lock b/crates/tycho-substreams/Cargo.lock index 5e61c4a..cd827d8 100644 --- a/crates/tycho-substreams/Cargo.lock +++ b/crates/tycho-substreams/Cargo.lock @@ -333,6 +333,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -530,7 +539,7 @@ checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "multimap", @@ -551,7 +560,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", @@ -908,6 +917,10 @@ dependencies = [ name = "tycho-substreams" version = "0.1.0" dependencies = [ + "hex", + "itertools 0.12.1", + "prost", + "substreams", "substreams-ethereum", ] diff --git a/crates/tycho-substreams/Cargo.toml b/crates/tycho-substreams/Cargo.toml index 269aea3..cc0f7bc 100644 --- a/crates/tycho-substreams/Cargo.toml +++ b/crates/tycho-substreams/Cargo.toml @@ -4,4 +4,8 @@ version = "0.1.0" edition = "2021" [dependencies] -substreams-ethereum = "0.9.9" \ No newline at end of file +substreams-ethereum = "0.9.9" +substreams = "0.5" +prost = "0.11" +hex = "0.4.3" +itertools = "0.12.0" \ No newline at end of file diff --git a/crates/tycho-substreams/rustfmt.toml b/crates/tycho-substreams/rustfmt.toml new file mode 100644 index 0000000..d0c0193 --- /dev/null +++ b/crates/tycho-substreams/rustfmt.toml @@ -0,0 +1,13 @@ +reorder_imports = true +imports_granularity = "Crate" +use_small_heuristics = "Max" +comment_width = 100 +wrap_comments = true +binop_separator = "Back" +trailing_comma = "Vertical" +trailing_semicolon = false +use_field_init_shorthand = true +chain_width = 40 +ignore = [ + "src/pb", +] \ No newline at end of file diff --git a/crates/tycho-substreams/src/balances.rs b/crates/tycho-substreams/src/balances.rs new file mode 100644 index 0000000..dbc780a --- /dev/null +++ b/crates/tycho-substreams/src/balances.rs @@ -0,0 +1,306 @@ +//! Utilities to handle relative balances. +//! +//! +//! To aggregate relative balances changes to absolute balances the general approach is: +//! +//! 1. Use a map function that will extract a `BlockBalanceDeltas` message. BalanceDeltas +//! within this message are required to have increasing ordinals so that +//! the order of relative balance changes is unambiguous. +//! 2. Store the balances changes with a store handler. You can use the +//! `store_balance_changes` library method directly for this. +//! 3. In the output module, use aggregate_balance_changes to receive an +//! aggregated map of absolute balances. +//! +use crate::pb::tycho::evm::v1::{BalanceChange, BlockBalanceDeltas}; +use itertools::Itertools; +use std::collections::HashMap; +use std::str::FromStr; +use substreams::key; +use substreams::pb::substreams::StoreDeltas; +use substreams::prelude::{BigInt, StoreAdd}; + +pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: impl StoreAdd) { + let mut previous_ordinal = HashMap::::new(); + deltas + .balance_deltas + .iter() + .for_each(|delta| { + let balance_key = format!( + "{0}:{1}", + String::from_utf8(delta.component_id.clone()) + .expect("delta.component_id is not valid utf-8!"), + hex::encode(&delta.token) + ); + let current_ord = delta.ord; + previous_ordinal + .entry(balance_key.clone()) + .and_modify(|ord| { + // ordinals must arrive in increasing order + if *ord >= current_ord { + panic!( + "Invalid ordinal sequence for {}: {} >= {}", + balance_key, *ord, current_ord + ); + } + *ord = current_ord; + }) + .or_insert(delta.ord); + store.add(delta.ord, balance_key, BigInt::from_signed_bytes_be(&delta.delta)); + }); +} + +pub fn aggregate_balances_changes( + balance_store: StoreDeltas, + deltas: BlockBalanceDeltas, +) -> HashMap, HashMap, BalanceChange>> { + balance_store + .deltas + .into_iter() + .zip(deltas.balance_deltas) + .map(|(store_delta, balance_delta)| { + let component_id = key::segment_at(&store_delta.key, 0); + let token_id = key::segment_at(&store_delta.key, 1); + // store_delta.new_value is an ASCII string representing an integer + let ascii_string = + String::from_utf8(store_delta.new_value.clone()).expect("Invalid UTF-8 sequence"); + let balance = BigInt::from_str(&ascii_string).expect("Failed to parse integer"); + let big_endian_bytes_balance = balance.to_bytes_be().1; + + ( + balance_delta.tx.unwrap(), + BalanceChange { + token: hex::decode(token_id).expect("Token ID not valid hex"), + balance: big_endian_bytes_balance, + component_id: component_id.as_bytes().to_vec(), + }, + ) + }) + // We need to group the balance changes by tx hash for the `TransactionContractChanges` agg + .group_by(|(tx, _)| tx.hash.clone()) + .into_iter() + .map(|(txh, group)| { + let balances = group + .into_iter() + .map(|(_, delta)| (delta.token.clone(), delta)) + .collect(); + (txh, balances) + }) + .collect() +} + +mod tests { + use super::*; + use crate::mock_store::MockStore; + use crate::pb::tycho::evm::v1::{BalanceDelta, Transaction}; + use substreams::pb::substreams::StoreDelta; + use substreams::prelude::{StoreGet, StoreNew}; + + fn block_balance_deltas() -> BlockBalanceDeltas { + let comp_id = "0x42c0ffee" + .to_string() + .as_bytes() + .to_vec(); + let token_0 = hex::decode("bad999").unwrap(); + let token_1 = hex::decode("babe00").unwrap(); + BlockBalanceDeltas { + balance_deltas: vec![ + BalanceDelta { + ord: 0, + tx: Some(Transaction { + hash: vec![0, 1], + from: vec![9, 9], + to: vec![8, 8], + index: 0, + }), + token: token_0.clone(), + delta: BigInt::from_str("+1000") + .unwrap() + .to_signed_bytes_be(), + component_id: comp_id.clone(), + }, + BalanceDelta { + ord: 2, + tx: Some(Transaction { + hash: vec![0, 1], + from: vec![9, 9], + to: vec![8, 8], + index: 0, + }), + token: token_1.clone(), + delta: BigInt::from_str("+100") + .unwrap() + .to_signed_bytes_be(), + component_id: comp_id.clone(), + }, + BalanceDelta { + ord: 3, + tx: Some(Transaction { + hash: vec![0, 1], + from: vec![9, 9], + to: vec![8, 8], + index: 0, + }), + token: token_1.clone(), + delta: BigInt::from_str("50") + .unwrap() + .to_signed_bytes_be(), + component_id: comp_id.clone(), + }, + BalanceDelta { + ord: 10, + tx: Some(Transaction { + hash: vec![0, 1], + from: vec![9, 9], + to: vec![8, 8], + index: 0, + }), + token: token_0.clone(), + delta: BigInt::from_str("-1") + .unwrap() + .to_signed_bytes_be(), + component_id: comp_id.clone(), + }, + ], + } + } + fn store_deltas() -> StoreDeltas { + let comp_id = "0x42c0ffee" + .to_string() + .as_bytes() + .to_vec(); + let token_0 = hex::decode("bad999").unwrap(); + let token_1 = hex::decode("babe00").unwrap(); + + let t0_key = + format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(&token_0)); + let t1_key = + format!("{}:{}", String::from_utf8(comp_id.clone()).unwrap(), hex::encode(&token_1)); + StoreDeltas { + deltas: vec![ + StoreDelta { + operation: 0, + ordinal: 0, + key: t0_key.clone(), + old_value: BigInt::from(0) + .to_string() + .as_bytes() + .to_vec(), + new_value: BigInt::from(1000) + .to_string() + .as_bytes() + .to_vec(), + }, + StoreDelta { + operation: 0, + ordinal: 2, + key: t1_key.clone(), + old_value: BigInt::from(0) + .to_string() + .as_bytes() + .to_vec(), + new_value: BigInt::from(100) + .to_string() + .as_bytes() + .to_vec(), + }, + StoreDelta { + operation: 0, + ordinal: 3, + key: t1_key.clone(), + old_value: BigInt::from(100) + .to_string() + .as_bytes() + .to_vec(), + new_value: BigInt::from(150) + .to_string() + .as_bytes() + .to_vec(), + }, + StoreDelta { + operation: 0, + ordinal: 10, + key: t0_key.clone(), + old_value: BigInt::from(1000) + .to_string() + .as_bytes() + .to_vec(), + new_value: BigInt::from(999) + .to_string() + .as_bytes() + .to_vec(), + }, + ], + } + } + + #[test] + fn test_store_balances() { + let comp_id = "0x42c0ffee" + .to_string() + .as_bytes() + .to_vec(); + let token_0 = hex::decode("bad999").unwrap(); + let token_1 = hex::decode("babe00").unwrap(); + let deltas = block_balance_deltas(); + let store = ::new(); + + store_balance_changes(deltas, store.clone()); + let res_0 = store.get_last(format!( + "{}:{}", + String::from_utf8(comp_id.clone()).unwrap(), + hex::encode(&token_0) + )); + let res_1 = store.get_last(format!( + "{}:{}", + String::from_utf8(comp_id.clone()).unwrap(), + hex::encode(&token_1) + )); + + assert_eq!(res_0, Some(BigInt::from_str("+999").unwrap())); + assert_eq!(res_1, Some(BigInt::from_str("+150").unwrap())); + } + + #[test] + fn test_aggregate_balances_changes() { + let store_deltas = store_deltas(); + let balance_deltas = block_balance_deltas(); + let comp_id = "0x42c0ffee" + .to_string() + .as_bytes() + .to_vec(); + let token_0 = hex::decode("bad999").unwrap(); + let token_1 = hex::decode("babe00").unwrap(); + let exp = [( + vec![0, 1], + [ + ( + token_0.clone(), + BalanceChange { + token: token_0, + balance: BigInt::from(999) + .to_signed_bytes_be() + .to_vec(), + component_id: comp_id.clone(), + }, + ), + ( + token_1.clone(), + BalanceChange { + token: token_1, + balance: vec![150], + component_id: comp_id.clone(), + }, + ), + ] + .into_iter() + .collect(), + )] + .into_iter() + .collect(); + + let res = aggregate_balances_changes(store_deltas, balance_deltas); + dbg!(&res); + + assert_eq!(res, exp); + } +} diff --git a/crates/tycho-substreams/src/contract.rs b/crates/tycho-substreams/src/contract.rs index a0391b3..79ad454 100644 --- a/crates/tycho-substreams/src/contract.rs +++ b/crates/tycho-substreams/src/contract.rs @@ -22,7 +22,7 @@ impl SlotValue { } // Uses a map for slots, protobuf does not allow bytes in hashmap keys -pub struct InterimContractChange { +struct InterimContractChange { address: Vec, balance: Vec, code: Vec, @@ -40,17 +40,14 @@ impl From for tycho::ContractChange { .slots .into_iter() .filter(|(_, value)| value.has_changed()) - .map(|(slot, value)| tycho::ContractSlot { - slot, - value: value.new_value, - }) + .map(|(slot, value)| tycho::ContractSlot { slot, value: value.new_value }) .collect(), change: value.change.into(), } } } -pub fn extract_contract_changes bool >( +pub fn extract_contract_changes bool>( block: ð::v2::Block, inclusion_predicate: F, transaction_contract_changes: &mut HashMap, @@ -69,131 +66,131 @@ pub fn extract_contract_changes bool >( }) .collect(); - block.transactions().for_each(|block_tx| { - let mut storage_changes = Vec::new(); - let mut balance_changes = Vec::new(); - let mut code_changes = Vec::new(); + block + .transactions() + .for_each(|block_tx| { + let mut storage_changes = Vec::new(); + let mut balance_changes = Vec::new(); + let mut code_changes = Vec::new(); - block_tx - .calls - .iter() - .filter(|call| { - !call.state_reverted - && inclusion_predicate(&call.address) - }) - .for_each(|call| { - storage_changes.extend(call.storage_changes.iter()); - balance_changes.extend(call.balance_changes.iter()); - code_changes.extend(call.code_changes.iter()); - }); + block_tx + .calls + .iter() + .filter(|call| !call.state_reverted && inclusion_predicate(&call.address)) + .for_each(|call| { + storage_changes.extend(call.storage_changes.iter()); + balance_changes.extend(call.balance_changes.iter()); + code_changes.extend(call.code_changes.iter()); + }); - storage_changes.sort_unstable_by_key(|change| change.ordinal); - balance_changes.sort_unstable_by_key(|change| change.ordinal); - code_changes.sort_unstable_by_key(|change| change.ordinal); + storage_changes.sort_unstable_by_key(|change| change.ordinal); + balance_changes.sort_unstable_by_key(|change| change.ordinal); + code_changes.sort_unstable_by_key(|change| change.ordinal); - storage_changes - .iter() - .filter(|changes| { - inclusion_predicate(&changes.address) - }) - .for_each(|storage_change| { - let contract_change = changed_contracts - .entry(storage_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: storage_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&storage_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); + storage_changes + .iter() + .filter(|changes| inclusion_predicate(&changes.address)) + .for_each(|storage_change| { + let contract_change = changed_contracts + .entry(storage_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: storage_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&storage_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); - let slot_value = contract_change - .slots - .entry(storage_change.key.clone()) - .or_insert_with(|| SlotValue { - new_value: storage_change.new_value.clone(), - start_value: storage_change.old_value.clone(), - }); + let slot_value = contract_change + .slots + .entry(storage_change.key.clone()) + .or_insert_with(|| SlotValue { + new_value: storage_change.new_value.clone(), + start_value: storage_change.old_value.clone(), + }); - slot_value - .new_value - .copy_from_slice(&storage_change.new_value); - }); + slot_value + .new_value + .copy_from_slice(&storage_change.new_value); + }); - balance_changes - .iter() - .filter(|changes| { - inclusion_predicate(&changes.address) - }) - .for_each(|balance_change| { - let contract_change = changed_contracts - .entry(balance_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: balance_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&balance_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); + balance_changes + .iter() + .filter(|changes| inclusion_predicate(&changes.address)) + .for_each(|balance_change| { + let contract_change = changed_contracts + .entry(balance_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: balance_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&balance_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); - if let Some(new_balance) = &balance_change.new_value { - contract_change.balance.clear(); + if let Some(new_balance) = &balance_change.new_value { + contract_change.balance.clear(); + contract_change + .balance + .extend_from_slice(&new_balance.bytes); + } + }); + + code_changes + .iter() + .filter(|changes| inclusion_predicate(&changes.address)) + .for_each(|code_change| { + let contract_change = changed_contracts + .entry(code_change.address.clone()) + .or_insert_with(|| InterimContractChange { + address: code_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&code_change.address) { + tycho::ChangeType::Creation + } else { + tycho::ChangeType::Update + }, + }); + + contract_change.code.clear(); contract_change - .balance - .extend_from_slice(&new_balance.bytes); - } - }); + .code + .extend_from_slice(&code_change.new_code); + }); - code_changes - .iter() - .filter(|changes| { - inclusion_predicate(&changes.address) - }) - .for_each(|code_change| { - let contract_change = changed_contracts - .entry(code_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: code_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&code_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); - - contract_change.code.clear(); - contract_change - .code - .extend_from_slice(&code_change.new_code); - }); - - if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() { - transaction_contract_changes - .entry(block_tx.index.into()) - .or_insert_with(|| tycho::TransactionContractChanges { - tx: Some(tycho::Transaction { - hash: block_tx.hash.clone(), - from: block_tx.from.clone(), - to: block_tx.to.clone(), - index: block_tx.index as u64, - }), - contract_changes: vec![], - component_changes: vec![], - balance_changes: vec![], - }) - .contract_changes - .extend(changed_contracts.drain().map(|(_, change)| change.into())); - } - }); + if !storage_changes.is_empty() + || !balance_changes.is_empty() + || !code_changes.is_empty() + { + transaction_contract_changes + .entry(block_tx.index.into()) + .or_insert_with(|| tycho::TransactionContractChanges { + tx: Some(tycho::Transaction { + hash: block_tx.hash.clone(), + from: block_tx.from.clone(), + to: block_tx.to.clone(), + index: block_tx.index as u64, + }), + contract_changes: vec![], + component_changes: vec![], + balance_changes: vec![], + }) + .contract_changes + .extend( + changed_contracts + .drain() + .map(|(_, change)| change.into()), + ); + } + }); } diff --git a/crates/tycho-substreams/src/lib.rs b/crates/tycho-substreams/src/lib.rs index 2f52c0e..d5b0d0f 100644 --- a/crates/tycho-substreams/src/lib.rs +++ b/crates/tycho-substreams/src/lib.rs @@ -1,2 +1,6 @@ +pub mod balances; pub mod contract; -pub mod pb; \ No newline at end of file +mod mock_store; +pub mod pb; +// TODO: consider removing this module, after integrating with balancer +pub mod utils; diff --git a/crates/tycho-substreams/src/mock_store.rs b/crates/tycho-substreams/src/mock_store.rs new file mode 100644 index 0000000..a1c0cd2 --- /dev/null +++ b/crates/tycho-substreams/src/mock_store.rs @@ -0,0 +1,93 @@ +//! Contains a mock store for internal testing. +//! +//! Might make this public alter to users can test their store handlers. +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; +use substreams::prelude::{BigInt, StoreDelete, StoreGet, StoreNew}; +use substreams::store::StoreAdd; + +#[derive(Debug, Clone)] +pub struct MockStore { + data: Rc>>>, +} + +impl StoreDelete for MockStore { + fn delete_prefix(&self, _ord: i64, prefix: &String) { + self.data + .borrow_mut() + .retain(|k, _| !k.starts_with(prefix)); + } +} + +impl StoreNew for MockStore { + fn new() -> Self { + Self { data: Rc::new(RefCell::new(HashMap::new())) } + } +} + +impl StoreAdd for MockStore { + fn add>(&self, ord: u64, key: K, value: BigInt) { + let mut guard = self.data.borrow_mut(); + guard + .entry(key.as_ref().to_string()) + .and_modify(|v| { + let prev_value = v.last().unwrap().1.clone(); + v.push((ord, prev_value + value.clone())); + }) + .or_insert(vec![(ord, value)]); + } + + fn add_many>(&self, _ord: u64, _keys: &Vec, _value: BigInt) { + todo!() + } +} + +impl StoreGet for MockStore { + fn new(_idx: u32) -> Self { + Self { data: Rc::new(RefCell::new(HashMap::new())) } + } + + fn get_at>(&self, ord: u64, key: K) -> Option { + self.data + .borrow() + .get(&key.as_ref().to_string()) + .map(|v| { + v.iter() + .find(|(current_ord, _)| *current_ord == ord) + .unwrap() + .1 + .clone() + }) + } + + fn get_last>(&self, key: K) -> Option { + self.data + .borrow() + .get(&key.as_ref().to_string()) + .map(|v| v.last().unwrap().1.clone()) + } + + fn get_first>(&self, key: K) -> Option { + self.data + .borrow() + .get(&key.as_ref().to_string()) + .map(|v| v.first().unwrap().1.clone()) + } + + fn has_at>(&self, ord: u64, key: K) -> bool { + self.data + .borrow() + .get(&key.as_ref().to_string()) + .map(|v| v.iter().any(|(v, _)| *v == ord)) + .unwrap_or(false) + } + + fn has_last>(&self, _key: K) -> bool { + todo!() + } + + fn has_first>(&self, _key: K) -> bool { + todo!() + } +} diff --git a/crates/tycho-substreams/src/pb/tycho.evm.v1.rs b/crates/tycho-substreams/src/pb/tycho.evm.v1.rs index 7424ba2..5409eff 100644 --- a/crates/tycho-substreams/src/pb/tycho.evm.v1.rs +++ b/crates/tycho-substreams/src/pb/tycho.evm.v1.rs @@ -107,7 +107,7 @@ pub struct BalanceChange { /// The address of the ERC20 token whose balance changed. #[prost(bytes="vec", tag="1")] pub token: ::prost::alloc::vec::Vec, - /// The new balance of the token. + /// The new balance of the token. Note: it must be a big endian encoded int. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, /// The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. @@ -244,6 +244,55 @@ pub struct BlockEntityChanges { #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } +/// A message containing relative balance changes. +/// +/// Used to track token balances of protocol components in case they are only +/// available as relative values within a block. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceDelta { + /// The ordinal of the balance change. Must be unique & deterministic over all balances + /// changes within a block. + #[prost(uint64, tag="1")] + pub ord: u64, + /// The tx hash of the transaction that caused the balance change. + #[prost(message, optional, tag="2")] + pub tx: ::core::option::Option, + /// The address of the ERC20 token whose balance changed. + #[prost(bytes="vec", tag="3")] + pub token: ::prost::alloc::vec::Vec, + /// The delta balance of the token. + #[prost(bytes="vec", tag="4")] + pub delta: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. + /// If the protocol component includes multiple contracts, the balance change must be + /// aggregated to reflect how much tokens can be traded. + #[prost(bytes="vec", tag="5")] + pub component_id: ::prost::alloc::vec::Vec, +} +/// A set of balances deltas, usually a group of changes within a single block. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockBalanceDeltas { + #[prost(message, repeated, tag="1")] + pub balance_deltas: ::prost::alloc::vec::Vec, +} +/// A message containing protocol components that were created by a single tx. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionProtocolComponents { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub components: ::prost::alloc::vec::Vec, +} +/// All protocol components that were created within a block with their corresponding tx. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockTransactionProtocolComponents { + #[prost(message, repeated, tag="1")] + pub tx_components: ::prost::alloc::vec::Vec, +} // This file contains proto definitions specific to the VM integration. /// A key value entry into contract storage. diff --git a/crates/tycho-substreams/src/utils.rs b/crates/tycho-substreams/src/utils.rs new file mode 100644 index 0000000..e4729db --- /dev/null +++ b/crates/tycho-substreams/src/utils.rs @@ -0,0 +1,24 @@ +use crate::pb::tycho::evm::v1::Transaction; + +/// This struct purely exists to spoof the `PartialEq` trait for `Transaction` so we can use it in +/// a later groupby operation. +#[derive(Debug)] +pub struct TransactionWrapper(Transaction); + +impl TransactionWrapper { + pub fn new(tx: Transaction) -> Self { + Self(tx) + } +} + +impl PartialEq for TransactionWrapper { + fn eq(&self, other: &Self) -> bool { + self.0.hash == other.0.hash + } +} + +impl From for Transaction { + fn from(value: TransactionWrapper) -> Self { + value.0 + } +} From d9fe9b1e1cad391413d2a14bb144554dbb4ab633 Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 21:06:55 +0000 Subject: [PATCH 3/8] Integrate balancer substream with sdk. We defer the contract storage extraction as well as the balance handling (apart from actually extracing relative deltas) to the SDK. --- crates/tycho-substreams/src/balances.rs | 65 ++-- crates/tycho-substreams/src/pb/mod.rs | 11 + substreams/ethereum-balancer/Cargo.lock | 12 + substreams/ethereum-balancer/Cargo.toml | 1 + .../ethereum-balancer/src/contract_changes.rs | 211 ------------ substreams/ethereum-balancer/src/lib.rs | 2 - substreams/ethereum-balancer/src/modules.rs | 115 +++---- substreams/ethereum-balancer/src/pb/mod.rs | 10 - .../ethereum-balancer/src/pb/tycho.evm.v1.rs | 319 ------------------ .../ethereum-balancer/src/pool_factories.rs | 6 +- 10 files changed, 102 insertions(+), 650 deletions(-) delete mode 100644 substreams/ethereum-balancer/src/contract_changes.rs delete mode 100644 substreams/ethereum-balancer/src/pb/mod.rs delete mode 100644 substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs diff --git a/crates/tycho-substreams/src/balances.rs b/crates/tycho-substreams/src/balances.rs index dbc780a..2547723 100644 --- a/crates/tycho-substreams/src/balances.rs +++ b/crates/tycho-substreams/src/balances.rs @@ -11,7 +11,7 @@ //! 3. In the output module, use aggregate_balance_changes to receive an //! aggregated map of absolute balances. //! -use crate::pb::tycho::evm::v1::{BalanceChange, BlockBalanceDeltas}; +use crate::pb::tycho::evm::v1::{BalanceChange, BlockBalanceDeltas, Transaction}; use itertools::Itertools; use std::collections::HashMap; use std::str::FromStr; @@ -52,7 +52,7 @@ pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: impl StoreAdd HashMap, HashMap, BalanceChange>> { +) -> HashMap, (Transaction, HashMap, BalanceChange>)> { balance_store .deltas .into_iter() @@ -79,15 +79,18 @@ pub fn aggregate_balances_changes( .group_by(|(tx, _)| tx.hash.clone()) .into_iter() .map(|(txh, group)| { - let balances = group + let (mut transactions, balance_changes): (Vec<_>, Vec<_>) = group.into_iter().unzip(); + + let balances = balance_changes .into_iter() - .map(|(_, delta)| (delta.token.clone(), delta)) + .map(|balance_change| (balance_change.token.clone(), balance_change)) .collect(); - (txh, balances) + (txh, (transactions.pop().unwrap(), balances)) }) .collect() } +#[cfg(test)] mod tests { use super::*; use crate::mock_store::MockStore; @@ -270,37 +273,39 @@ mod tests { .to_vec(); let token_0 = hex::decode("bad999").unwrap(); let token_1 = hex::decode("babe00").unwrap(); + let exp = [( vec![0, 1], - [ - ( - token_0.clone(), - BalanceChange { - token: token_0, - balance: BigInt::from(999) - .to_signed_bytes_be() - .to_vec(), - component_id: comp_id.clone(), - }, - ), - ( - token_1.clone(), - BalanceChange { - token: token_1, - balance: vec![150], - component_id: comp_id.clone(), - }, - ), - ] - .into_iter() - .collect(), + ( + Transaction { hash: vec![0, 1], from: vec![9, 9], to: vec![8, 8], index: 0 }, + [ + ( + token_0.clone(), + BalanceChange { + token: token_0, + balance: BigInt::from(999) + .to_signed_bytes_be() + .to_vec(), + component_id: comp_id.clone(), + }, + ), + ( + token_1.clone(), + BalanceChange { + token: token_1, + balance: vec![150], + component_id: comp_id.clone(), + }, + ), + ] + .into_iter() + .collect::>(), + ), )] .into_iter() - .collect(); + .collect::>(); let res = aggregate_balances_changes(store_deltas, balance_deltas); - dbg!(&res); - assert_eq!(res, exp); } } diff --git a/crates/tycho-substreams/src/pb/mod.rs b/crates/tycho-substreams/src/pb/mod.rs index 43d8838..f5cd0b0 100644 --- a/crates/tycho-substreams/src/pb/mod.rs +++ b/crates/tycho-substreams/src/pb/mod.rs @@ -5,6 +5,17 @@ pub mod tycho { pub mod v1 { include!("tycho.evm.v1.rs"); // @@protoc_insertion_point(tycho.evm.v1) + + impl TransactionContractChanges { + pub fn new(tx: &Transaction) -> Self { + Self { + tx: Some(tx.clone()), + contract_changes: vec![], + component_changes: vec![], + balance_changes: vec![], + } + } + } } } } diff --git a/substreams/ethereum-balancer/Cargo.lock b/substreams/ethereum-balancer/Cargo.lock index b05dce5..e2d81a0 100644 --- a/substreams/ethereum-balancer/Cargo.lock +++ b/substreams/ethereum-balancer/Cargo.lock @@ -931,6 +931,7 @@ dependencies = [ "prost-types 0.12.3", "substreams", "substreams-ethereum", + "tycho-substreams", ] [[package]] @@ -1064,6 +1065,17 @@ dependencies = [ "winnow", ] +[[package]] +name = "tycho-substreams" +version = "0.1.0" +dependencies = [ + "hex", + "itertools 0.12.0", + "prost 0.11.9", + "substreams", + "substreams-ethereum", +] + [[package]] name = "typenum" version = "1.17.0" diff --git a/substreams/ethereum-balancer/Cargo.toml b/substreams/ethereum-balancer/Cargo.toml index ace53d3..0a964e5 100644 --- a/substreams/ethereum-balancer/Cargo.toml +++ b/substreams/ethereum-balancer/Cargo.toml @@ -19,6 +19,7 @@ anyhow = "1.0.75" prost-types = "0.12.3" num-bigint = "0.4.4" itertools = "0.12.0" +tycho-substreams = {path ="../../crates/tycho-substreams"} [build-dependencies] anyhow = "1" diff --git a/substreams/ethereum-balancer/src/contract_changes.rs b/substreams/ethereum-balancer/src/contract_changes.rs deleted file mode 100644 index 664b0e7..0000000 --- a/substreams/ethereum-balancer/src/contract_changes.rs +++ /dev/null @@ -1,211 +0,0 @@ -/// This file contains helpers to capture contract changes from the expanded block model. These -/// leverage the `code_changes`, `balance_changes`, and `storage_changes` fields available on the -/// `Call` type provided by block model in a substream (i.e. `logs_and_calls`, etc). -/// -/// ⚠️ These helpers *only* work if the **expanded block model** is available, more info blow. -/// https://streamingfastio.medium.com/new-block-model-to-accelerate-chain-integration-9f65126e5425 -use std::collections::HashMap; - -use substreams_ethereum::pb::eth; - -use pb::tycho::evm::v1::{self as tycho}; - -use substreams::store::{StoreGet, StoreGetInt64}; - -use crate::pb; - -struct SlotValue { - new_value: Vec, - start_value: Vec, -} - -impl SlotValue { - fn has_changed(&self) -> bool { - self.start_value != self.new_value - } -} - -// Uses a map for slots, protobuf does not allow bytes in hashmap keys -pub struct InterimContractChange { - address: Vec, - balance: Vec, - code: Vec, - slots: HashMap, SlotValue>, - change: tycho::ChangeType, -} - -impl From for tycho::ContractChange { - fn from(value: InterimContractChange) -> Self { - tycho::ContractChange { - address: value.address, - balance: value.balance, - code: value.code, - slots: value - .slots - .into_iter() - .filter(|(_, value)| value.has_changed()) - .map(|(slot, value)| tycho::ContractSlot { - slot, - value: value.new_value, - }) - .collect(), - change: value.change.into(), - } - } -} - -pub fn extract_contract_changes( - block: ð::v2::Block, - contracts: StoreGetInt64, - transaction_contract_changes: &mut HashMap, -) { - let mut changed_contracts: HashMap, InterimContractChange> = HashMap::new(); - - // Collect all accounts created in this block - let created_accounts: HashMap<_, _> = block - .transactions() - .flat_map(|tx| { - tx.calls.iter().flat_map(|call| { - call.account_creations - .iter() - .map(|ac| (&ac.account, ac.ordinal)) - }) - }) - .collect(); - - block.transactions().for_each(|block_tx| { - let mut storage_changes = Vec::new(); - let mut balance_changes = Vec::new(); - let mut code_changes = Vec::new(); - - block_tx - .calls - .iter() - .filter(|call| { - !call.state_reverted - && contracts - .get_last(format!("pool:{0}", hex::encode(&call.address))) - .is_some() - }) - .for_each(|call| { - storage_changes.extend(call.storage_changes.iter()); - balance_changes.extend(call.balance_changes.iter()); - code_changes.extend(call.code_changes.iter()); - }); - - storage_changes.sort_unstable_by_key(|change| change.ordinal); - balance_changes.sort_unstable_by_key(|change| change.ordinal); - code_changes.sort_unstable_by_key(|change| change.ordinal); - - storage_changes - .iter() - .filter(|changes| { - contracts - .get_last(format!("pool:{0}", hex::encode(&changes.address))) - .is_some() - }) - .for_each(|storage_change| { - let contract_change = changed_contracts - .entry(storage_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: storage_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&storage_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); - - let slot_value = contract_change - .slots - .entry(storage_change.key.clone()) - .or_insert_with(|| SlotValue { - new_value: storage_change.new_value.clone(), - start_value: storage_change.old_value.clone(), - }); - - slot_value - .new_value - .copy_from_slice(&storage_change.new_value); - }); - - balance_changes - .iter() - .filter(|changes| { - contracts - .get_last(format!("pool:{0}", hex::encode(&changes.address))) - .is_some() - }) - .for_each(|balance_change| { - let contract_change = changed_contracts - .entry(balance_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: balance_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&balance_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); - - if let Some(new_balance) = &balance_change.new_value { - contract_change.balance.clear(); - contract_change - .balance - .extend_from_slice(&new_balance.bytes); - } - }); - - code_changes - .iter() - .filter(|changes| { - contracts - .get_last(format!("pool:{0}", hex::encode(&changes.address))) - .is_some() - }) - .for_each(|code_change| { - let contract_change = changed_contracts - .entry(code_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: code_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&code_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, - }); - - contract_change.code.clear(); - contract_change - .code - .extend_from_slice(&code_change.new_code); - }); - - if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() { - transaction_contract_changes - .entry(block_tx.index.into()) - .or_insert_with(|| tycho::TransactionContractChanges { - tx: Some(tycho::Transaction { - hash: block_tx.hash.clone(), - from: block_tx.from.clone(), - to: block_tx.to.clone(), - index: block_tx.index as u64, - }), - contract_changes: vec![], - component_changes: vec![], - balance_changes: vec![], - }) - .contract_changes - .extend(changed_contracts.drain().map(|(_, change)| change.into())); - } - }); -} diff --git a/substreams/ethereum-balancer/src/lib.rs b/substreams/ethereum-balancer/src/lib.rs index 88ecc50..27bd15a 100644 --- a/substreams/ethereum-balancer/src/lib.rs +++ b/substreams/ethereum-balancer/src/lib.rs @@ -1,5 +1,3 @@ mod abi; -mod contract_changes; mod modules; -mod pb; mod pool_factories; diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index c4fcb4a..9923167 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -1,27 +1,20 @@ -use std::collections::HashMap; -use std::str::FromStr; - +use crate::{abi, pool_factories}; use anyhow::Result; +use itertools::Itertools; +use std::collections::HashMap; use substreams::hex; use substreams::pb::substreams::StoreDeltas; use substreams::store::{ StoreAdd, StoreAddBigInt, StoreAddInt64, StoreGet, StoreGetInt64, StoreNew, }; - -use substreams::key; -use substreams::scalar::BigInt; - use substreams_ethereum::pb::eth; - -use itertools::Itertools; - - -use contract_changes::extract_contract_changes; use substreams_ethereum::Event; - -use crate::pb::tycho::evm::v1::{self as tycho}; -use crate::pb::tycho::evm::v1::{BalanceDelta, BlockBalanceDeltas, BlockTransactionProtocolComponents, TransactionProtocolComponents}; -use crate::{abi, contract_changes, pool_factories}; +use tycho_substreams::balances; +use tycho_substreams::contract::extract_contract_changes; +use tycho_substreams::pb::tycho::evm::v1::{ + self as tycho, BalanceDelta, BlockBalanceDeltas, BlockTransactionProtocolComponents, + TransactionProtocolComponents, +}; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); @@ -110,7 +103,13 @@ pub fn map_balance_deltas( if let Some(ev) = abi::vault::events::PoolBalanceChanged::match_and_decode(vault_log.log) { - let component_id = ev.pool_id[..20].to_vec(); + let component_id = format!( + "0x{}", + String::from_utf8(ev.pool_id[..20].to_vec()).unwrap() + ) + .as_bytes() + .to_vec(); + if store .get_last(format!("pool:{}", hex::encode(&component_id))) .is_some() @@ -131,7 +130,13 @@ pub fn map_balance_deltas( } } } else if let Some(ev) = abi::vault::events::Swap::match_and_decode(vault_log.log) { - let component_id = ev.pool_id[..20].to_vec(); + let component_id = format!( + "0x{}", + String::from_utf8(ev.pool_id[..20].to_vec()).unwrap() + ) + .as_bytes() + .to_vec(); + if store .get_last(format!("pool:{}", hex::encode(&component_id))) .is_some() @@ -176,17 +181,7 @@ pub fn map_balance_deltas( /// store key to ensure that there's a unique balance being tallied for each. #[substreams::handlers::store] pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: StoreAddBigInt) { - deltas.balance_deltas.iter().for_each(|delta| { - store.add( - delta.ord, - format!( - "pool:{0}:token:{1}", - hex::encode(&delta.component_id), - hex::encode(&delta.token) - ), - BigInt::from_signed_bytes_be(&delta.delta), - ); - }); + balances::store_balance_changes(deltas, store); } /// This is the main map that handles most of the indexing of this substream. @@ -215,67 +210,37 @@ pub fn map_changes( .iter() .for_each(|tx_component| { let tx = tx_component.tx.as_ref().unwrap(); - transaction_contract_changes .entry(tx.index) - .or_insert_with(|| tycho::TransactionContractChanges { - tx: Some(tx.clone()), - contract_changes: vec![], - component_changes: vec![], - balance_changes: vec![], - }) + .or_insert_with(|| tycho::TransactionContractChanges::new(&tx)) .component_changes .extend_from_slice(&tx_component.components); }); // Balance changes are gathered by the `StoreDelta` based on `PoolBalanceChanged` creating - // `BlockBalanceDeltas`. We essentially just process the changes that occured to the `store` this + // `BlockBalanceDeltas`. We essentially just process the changes that occurred to the `store` this // block. Then, these balance changes are merged onto the existing map of tx contract changes, // inserting a new one if it doesn't exist. - balance_store - .deltas + balances::aggregate_balances_changes(balance_store, deltas) .into_iter() - .zip(deltas.balance_deltas) - .map(|(store_delta, balance_delta)| { - let pool_id = key::segment_at(&store_delta.key, 1); - let token_id = key::segment_at(&store_delta.key, 3); - // store_delta.new_value is an ASCII string representing an integer - let ascii_string = - String::from_utf8(store_delta.new_value.clone()).expect("Invalid UTF-8 sequence"); - let balance = BigInt::from_str(&ascii_string).expect("Failed to parse integer"); - let big_endian_bytes_balance = balance.to_bytes_be().1; - - ( - balance_delta.tx.unwrap(), - tycho::BalanceChange { - token: hex::decode(token_id).expect("Token ID not valid hex"), - balance: big_endian_bytes_balance, - component_id: pool_id.as_bytes().to_vec(), - }, - ) - }) - // We need to group the balance changes by tx hash for the `TransactionContractChanges` agg - .group_by(|(tx, _)| TransactionWrapper(tx.clone())) - .into_iter() - .for_each(|(tx_wrapped, group)| { - let tx = tx_wrapped.0; - + .for_each(|(_, (tx, balances))| { transaction_contract_changes .entry(tx.index) - .or_insert_with(|| tycho::TransactionContractChanges { - tx: Some(tx.clone()), - contract_changes: vec![], - component_changes: vec![], - balance_changes: vec![], - }) + .or_insert_with(|| tycho::TransactionContractChanges::new(&tx)) .balance_changes - .extend(group.map(|(_, change)| change)); + .extend(balances.into_iter().map(|(_, change)| change)); }); - // General helper for extracting contract changes. Uses block, our component store which holds - // all of our tracked deployed pool addresses, and the map of tx contract changes which we - // output into for final processing later. - extract_contract_changes(&block, components_store, &mut transaction_contract_changes); + // Extract and insert any storage changes that happened for any of the components. + extract_contract_changes( + &block, + |addr| { + components_store + .get_last(format!("pool:{0}", hex::encode(&addr))) + .is_some() + }, + &mut transaction_contract_changes, + ); // Process all `transaction_contract_changes` for final output in the `BlockContractChanges`, // sorted by transaction index (the key). diff --git a/substreams/ethereum-balancer/src/pb/mod.rs b/substreams/ethereum-balancer/src/pb/mod.rs deleted file mode 100644 index 43d8838..0000000 --- a/substreams/ethereum-balancer/src/pb/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -// @generated -pub mod tycho { - pub mod evm { - // @@protoc_insertion_point(attribute:tycho.evm.v1) - pub mod v1 { - include!("tycho.evm.v1.rs"); - // @@protoc_insertion_point(tycho.evm.v1) - } - } -} diff --git a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs b/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs deleted file mode 100644 index 387ff7b..0000000 --- a/substreams/ethereum-balancer/src/pb/tycho.evm.v1.rs +++ /dev/null @@ -1,319 +0,0 @@ -// @generated -// This file contains the proto definitions for Substreams common to all integrations. - -/// A struct describing a block. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - /// The blocks hash. - #[prost(bytes="vec", tag="1")] - pub hash: ::prost::alloc::vec::Vec, - /// The parent blocks hash. - #[prost(bytes="vec", tag="2")] - pub parent_hash: ::prost::alloc::vec::Vec, - /// The block number. - #[prost(uint64, tag="3")] - pub number: u64, - /// The block timestamp. - #[prost(uint64, tag="4")] - pub ts: u64, -} -/// A struct describing a transaction. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transaction { - /// The transaction hash. - #[prost(bytes="vec", tag="1")] - pub hash: ::prost::alloc::vec::Vec, - /// The sender of the transaction. - #[prost(bytes="vec", tag="2")] - pub from: ::prost::alloc::vec::Vec, - /// The receiver of the transaction. - #[prost(bytes="vec", tag="3")] - pub to: ::prost::alloc::vec::Vec, - /// The transactions index within the block. - /// TODO: should this be uint32? to match the type from the native substream type? - #[prost(uint64, tag="4")] - pub index: u64, -} -/// A custom struct representing an arbitrary attribute of a protocol component. -/// This is mainly used by the native integration to track the necessary information about the protocol. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Attribute { - /// The name of the attribute. - #[prost(string, tag="1")] - pub name: ::prost::alloc::string::String, - /// The value of the attribute. - #[prost(bytes="vec", tag="2")] - pub value: ::prost::alloc::vec::Vec, - /// The type of change the attribute underwent. - #[prost(enumeration="ChangeType", tag="3")] - pub change: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtocolType { - #[prost(string, tag="1")] - pub name: ::prost::alloc::string::String, - #[prost(enumeration="FinancialType", tag="2")] - pub financial_type: i32, - #[prost(message, repeated, tag="3")] - pub attribute_schema: ::prost::alloc::vec::Vec, - #[prost(enumeration="ImplementationType", tag="4")] - pub implementation_type: i32, -} -/// A struct describing a part of the protocol. -/// Note: For example this can be a UniswapV2 pair, that tracks the two ERC20 tokens used by the pair, -/// the component would represent a single contract. In case of VM integration, such component would -/// not need any attributes, because all the relevant info would be tracked via storage slots and balance changes. -/// It can also be a wrapping contract, like WETH, that has a constant price, but it allows swapping tokens. -/// This is why the name ProtocolComponent is used instead of "Pool" or "Pair". -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProtocolComponent { - /// A unique identifier for the component within the protocol. - /// Can be e.g. a stringified address or a string describing the trading pair. - #[prost(string, tag="1")] - pub id: ::prost::alloc::string::String, - /// Addresses of the ERC20 tokens used by the component. - #[prost(bytes="vec", repeated, tag="2")] - pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, - /// Addresses of the contracts used by the component. - /// Usually it is a single contract, but some protocols use multiple contracts. - #[prost(bytes="vec", repeated, tag="3")] - pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, - /// Attributes of the component. Used mainly be the native integration. - /// The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent. - #[prost(message, repeated, tag="4")] - pub static_att: ::prost::alloc::vec::Vec, - /// Type of change the component underwent. - #[prost(enumeration="ChangeType", tag="5")] - pub change: i32, - /// / Represents the functionality of the component. - #[prost(message, optional, tag="6")] - pub protocol_type: ::core::option::Option, - /// Transaction where this component was created - #[prost(message, optional, tag="7")] - pub tx: ::core::option::Option, -} -/// A struct for following the changes of Total Value Locked (TVL) of a protocol component. -/// Note that if a ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. -/// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceChange { - /// The address of the ERC20 token whose balance changed. - #[prost(bytes="vec", tag="1")] - pub token: ::prost::alloc::vec::Vec, - /// The new balance of the token. Note: it must be a big endian encoded int. - #[prost(bytes="vec", tag="2")] - pub balance: ::prost::alloc::vec::Vec, - /// The id of the component whose TVL is tracked. Note: This MUST be utf8 encoded. - /// If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. - #[prost(bytes="vec", tag="3")] - pub component_id: ::prost::alloc::vec::Vec, -} -/// Enum to specify the type of a change. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum ChangeType { - Unspecified = 0, - Update = 1, - Creation = 2, - Deletion = 3, -} -impl ChangeType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED", - ChangeType::Update => "CHANGE_TYPE_UPDATE", - ChangeType::Creation => "CHANGE_TYPE_CREATION", - ChangeType::Deletion => "CHANGE_TYPE_DELETION", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified), - "CHANGE_TYPE_UPDATE" => Some(Self::Update), - "CHANGE_TYPE_CREATION" => Some(Self::Creation), - "CHANGE_TYPE_DELETION" => Some(Self::Deletion), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum FinancialType { - Swap = 0, - Lend = 1, - Leverage = 2, - Psm = 3, -} -impl FinancialType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - FinancialType::Swap => "SWAP", - FinancialType::Lend => "LEND", - FinancialType::Leverage => "LEVERAGE", - FinancialType::Psm => "PSM", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SWAP" => Some(Self::Swap), - "LEND" => Some(Self::Lend), - "LEVERAGE" => Some(Self::Leverage), - "PSM" => Some(Self::Psm), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum ImplementationType { - Vm = 0, - Custom = 1, -} -impl ImplementationType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - ImplementationType::Vm => "VM", - ImplementationType::Custom => "CUSTOM", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "VM" => Some(Self::Vm), - "CUSTOM" => Some(Self::Custom), - _ => None, - } - } -} -// This file contains proto definitions specific to the VM integration. - -/// A key value entry into contract storage. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ContractSlot { - /// A contract's storage slot. - #[prost(bytes="vec", tag="2")] - pub slot: ::prost::alloc::vec::Vec, - /// The new value for this storage slot. - #[prost(bytes="vec", tag="3")] - pub value: ::prost::alloc::vec::Vec, -} -/// Changes made to a single contract's state. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ContractChange { - /// The contract's address - #[prost(bytes="vec", tag="1")] - pub address: ::prost::alloc::vec::Vec, - /// The new native balance of the contract, empty bytes indicates no change. - #[prost(bytes="vec", tag="2")] - pub balance: ::prost::alloc::vec::Vec, - /// The new code of the contract, empty bytes indicates no change. - #[prost(bytes="vec", tag="3")] - pub code: ::prost::alloc::vec::Vec, - /// The changes to this contract's slots, empty sequence indicates no change. - #[prost(message, repeated, tag="4")] - pub slots: ::prost::alloc::vec::Vec, - /// Whether this is an update, a creation or a deletion. - #[prost(enumeration="ChangeType", tag="5")] - pub change: i32, -} -/// A set of changes aggregated by transaction. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionContractChanges { - /// The transaction instance that results in the changes. - #[prost(message, optional, tag="1")] - pub tx: ::core::option::Option, - /// Contains the changes induced by the above transaction, aggregated on a per-contract basis. - /// Must include changes to every contract that is tracked by all ProtocolComponents. - #[prost(message, repeated, tag="2")] - pub contract_changes: ::prost::alloc::vec::Vec, - /// An array of any component changes. - #[prost(message, repeated, tag="3")] - pub component_changes: ::prost::alloc::vec::Vec, - /// An array of balance changes to components. - #[prost(message, repeated, tag="4")] - pub balance_changes: ::prost::alloc::vec::Vec, -} -/// A set of transaction changes within a single block. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockContractChanges { - /// The block for which these changes are collectively computed. - #[prost(message, optional, tag="1")] - pub block: ::core::option::Option, - /// The set of transaction changes observed in the specified block. - #[prost(message, repeated, tag="2")] - pub changes: ::prost::alloc::vec::Vec, -} -/// A message containing relative balance changes. -/// -/// Used to track token balances of protocol components in case they are only -/// available as relative values within a block. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BalanceDelta { - /// The ordinal of the balance change. Must be unique & deterministic over all balances - /// changes within a block. - #[prost(uint64, tag="1")] - pub ord: u64, - /// The tx hash of the transaction that caused the balance change. - #[prost(message, optional, tag="2")] - pub tx: ::core::option::Option, - /// The address of the ERC20 token whose balance changed. - #[prost(bytes="vec", tag="3")] - pub token: ::prost::alloc::vec::Vec, - /// The delta balance of the token. - #[prost(bytes="vec", tag="4")] - pub delta: ::prost::alloc::vec::Vec, - /// The id of the component whose TVL is tracked. - /// If the protocol component includes multiple contracts, the balance change must be - /// aggregated to reflect how much tokens can be traded. - #[prost(bytes="vec", tag="5")] - pub component_id: ::prost::alloc::vec::Vec, -} -/// A set of balances deltas, usually a group of changes within a single block. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockBalanceDeltas { - #[prost(message, repeated, tag="1")] - pub balance_deltas: ::prost::alloc::vec::Vec, -} -/// A message containing protocol components that were created by a single tx. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionProtocolComponents { - #[prost(message, optional, tag="1")] - pub tx: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub components: ::prost::alloc::vec::Vec, -} -/// All protocol components that were created within a block with their corresponding tx. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockTransactionProtocolComponents { - #[prost(message, repeated, tag="1")] - pub tx_components: ::prost::alloc::vec::Vec, -} -// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-balancer/src/pool_factories.rs b/substreams/ethereum-balancer/src/pool_factories.rs index e19933c..27c5ced 100644 --- a/substreams/ethereum-balancer/src/pool_factories.rs +++ b/substreams/ethereum-balancer/src/pool_factories.rs @@ -2,10 +2,10 @@ use substreams_ethereum::pb::eth::v2::{Call, Log}; use substreams_ethereum::{Event, Function}; use crate::abi; -use crate::pb; -use crate::pb::tycho::evm::v1::{FinancialType, ImplementationType, ProtocolType, Transaction}; -use pb::tycho::evm::v1::{self as tycho}; use substreams::hex; +use tycho_substreams::pb::tycho::evm::v1::{ + self as tycho, FinancialType, ImplementationType, ProtocolType, Transaction, +}; use substreams::scalar::BigInt; From 18e9dfcec497aea9715671e5df65086ca2ca3bb4 Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 21:07:46 +0000 Subject: [PATCH 4/8] Remove TransactionWrapper from SDK & Balancer. This type is not needed anymore. --- crates/tycho-substreams/src/lib.rs | 2 -- crates/tycho-substreams/src/utils.rs | 24 --------------------- substreams/ethereum-balancer/src/modules.rs | 11 ---------- 3 files changed, 37 deletions(-) delete mode 100644 crates/tycho-substreams/src/utils.rs diff --git a/crates/tycho-substreams/src/lib.rs b/crates/tycho-substreams/src/lib.rs index d5b0d0f..78c7e2e 100644 --- a/crates/tycho-substreams/src/lib.rs +++ b/crates/tycho-substreams/src/lib.rs @@ -2,5 +2,3 @@ pub mod balances; pub mod contract; mod mock_store; pub mod pb; -// TODO: consider removing this module, after integrating with balancer -pub mod utils; diff --git a/crates/tycho-substreams/src/utils.rs b/crates/tycho-substreams/src/utils.rs deleted file mode 100644 index e4729db..0000000 --- a/crates/tycho-substreams/src/utils.rs +++ /dev/null @@ -1,24 +0,0 @@ -use crate::pb::tycho::evm::v1::Transaction; - -/// This struct purely exists to spoof the `PartialEq` trait for `Transaction` so we can use it in -/// a later groupby operation. -#[derive(Debug)] -pub struct TransactionWrapper(Transaction); - -impl TransactionWrapper { - pub fn new(tx: Transaction) -> Self { - Self(tx) - } -} - -impl PartialEq for TransactionWrapper { - fn eq(&self, other: &Self) -> bool { - self.0.hash == other.0.hash - } -} - -impl From for Transaction { - fn from(value: TransactionWrapper) -> Self { - value.0 - } -} diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 9923167..a3658ad 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -18,17 +18,6 @@ use tycho_substreams::pb::tycho::evm::v1::{ const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); -/// This struct purely exists to spoof the `PartialEq` trait for `Transaction` so we can use it in -/// a later groupby operation. -#[derive(Debug)] -struct TransactionWrapper(tycho::Transaction); - -impl PartialEq for TransactionWrapper { - fn eq(&self, other: &Self) -> bool { - self.0.hash == other.0.hash - } -} - #[substreams::handlers::map] pub fn map_pools_created(block: eth::v2::Block) -> Result { // Gather contract changes by indexing `PoolCreated` events and analysing the `Create` call From 312d322585f2e2dd91bf2f11e43ed56614f569a1 Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 22:10:54 +0000 Subject: [PATCH 5/8] Add tycho pb type constructors. This allows us to remove a lot of unnecessary boilerplate code. --- crates/tycho-substreams/src/contract.rs | 87 +++--- crates/tycho-substreams/src/pb/mod.rs | 99 +++++++ substreams/ethereum-balancer/src/modules.rs | 47 +--- .../ethereum-balancer/src/pool_factories.rs | 261 ++++++------------ 4 files changed, 228 insertions(+), 266 deletions(-) diff --git a/crates/tycho-substreams/src/contract.rs b/crates/tycho-substreams/src/contract.rs index 79ad454..633ef35 100644 --- a/crates/tycho-substreams/src/contract.rs +++ b/crates/tycho-substreams/src/contract.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use substreams_ethereum::pb::eth; +use substreams_ethereum::pb::eth::v2::StorageChange; use crate::pb::tycho::evm::v1::{self as tycho}; @@ -15,6 +16,12 @@ struct SlotValue { start_value: Vec, } +impl From<&StorageChange> for SlotValue { + fn from(change: &StorageChange) -> Self { + Self { new_value: change.new_value.clone(), start_value: change.old_value.clone() } + } +} + impl SlotValue { fn has_changed(&self) -> bool { self.start_value != self.new_value @@ -30,6 +37,22 @@ struct InterimContractChange { change: tycho::ChangeType, } +impl InterimContractChange { + fn new(address: &[u8], creation: bool) -> Self { + Self { + address: address.to_vec(), + balance: vec![], + code: vec![], + slots: Default::default(), + change: if creation { + tycho::ChangeType::Creation.into() + } else { + tycho::ChangeType::Update.into() + }, + } + } +} + impl From for tycho::ContractChange { fn from(value: InterimContractChange) -> Self { tycho::ContractChange { @@ -90,28 +113,20 @@ pub fn extract_contract_changes bool>( storage_changes .iter() .filter(|changes| inclusion_predicate(&changes.address)) - .for_each(|storage_change| { + .for_each(|&storage_change| { let contract_change = changed_contracts .entry(storage_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: storage_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&storage_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, + .or_insert_with(|| { + InterimContractChange::new( + &storage_change.address, + created_accounts.contains_key(&storage_change.address), + ) }); let slot_value = contract_change .slots .entry(storage_change.key.clone()) - .or_insert_with(|| SlotValue { - new_value: storage_change.new_value.clone(), - start_value: storage_change.old_value.clone(), - }); + .or_insert_with(|| storage_change.into()); slot_value .new_value @@ -124,16 +139,11 @@ pub fn extract_contract_changes bool>( .for_each(|balance_change| { let contract_change = changed_contracts .entry(balance_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: balance_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&balance_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, + .or_insert_with(|| { + InterimContractChange::new( + &balance_change.address, + created_accounts.contains_key(&balance_change.address), + ) }); if let Some(new_balance) = &balance_change.new_value { @@ -150,16 +160,11 @@ pub fn extract_contract_changes bool>( .for_each(|code_change| { let contract_change = changed_contracts .entry(code_change.address.clone()) - .or_insert_with(|| InterimContractChange { - address: code_change.address.clone(), - balance: Vec::new(), - code: Vec::new(), - slots: HashMap::new(), - change: if created_accounts.contains_key(&code_change.address) { - tycho::ChangeType::Creation - } else { - tycho::ChangeType::Update - }, + .or_insert_with(|| { + InterimContractChange::new( + &code_change.address, + created_accounts.contains_key(&code_change.address), + ) }); contract_change.code.clear(); @@ -174,17 +179,7 @@ pub fn extract_contract_changes bool>( { transaction_contract_changes .entry(block_tx.index.into()) - .or_insert_with(|| tycho::TransactionContractChanges { - tx: Some(tycho::Transaction { - hash: block_tx.hash.clone(), - from: block_tx.from.clone(), - to: block_tx.to.clone(), - index: block_tx.index as u64, - }), - contract_changes: vec![], - component_changes: vec![], - balance_changes: vec![], - }) + .or_insert_with(|| tycho::TransactionContractChanges::new(&(block_tx.into()))) .contract_changes .extend( changed_contracts diff --git a/crates/tycho-substreams/src/pb/mod.rs b/crates/tycho-substreams/src/pb/mod.rs index f5cd0b0..ceced59 100644 --- a/crates/tycho-substreams/src/pb/mod.rs +++ b/crates/tycho-substreams/src/pb/mod.rs @@ -3,6 +3,7 @@ pub mod tycho { pub mod evm { // @@protoc_insertion_point(attribute:tycho.evm.v1) pub mod v1 { + use substreams_ethereum::pb::eth::v2::{self as sf}; include!("tycho.evm.v1.rs"); // @@protoc_insertion_point(tycho.evm.v1) @@ -16,6 +17,104 @@ pub mod tycho { } } } + + impl From<&sf::TransactionTrace> for Transaction { + fn from(tx: &sf::TransactionTrace) -> Self { + Self { + hash: tx.hash.clone(), + from: tx.from.clone(), + to: tx.to.clone(), + index: tx.index.into(), + } + } + } + + impl From<&sf::Block> for Block { + fn from(block: &sf::Block) -> Self { + Self { + number: block.number, + hash: block.hash.clone(), + parent_hash: block + .header + .as_ref() + .expect("Block header not present") + .parent_hash + .clone(), + ts: block.timestamp_seconds(), + } + } + } + + impl ProtocolComponent { + pub fn new(id: &str, tx: &Transaction) -> Self { + Self { + id: id.to_string(), + tokens: vec![], + contracts: vec![], + static_att: vec![], + change: ChangeType::Creation.into(), + protocol_type: None, + tx: Some(tx.clone()), + } + } + + pub fn at_contract(id: &[u8], tx: &Transaction) -> Self { + Self { + id: format!("0x{}", hex::encode(id)), + tokens: vec![], + contracts: vec![id.to_vec()], + static_att: vec![], + change: ChangeType::Creation.into(), + protocol_type: None, + tx: Some(tx.clone()), + } + } + + pub fn with_tokens>(mut self, tokens: &[B]) -> Self { + self.tokens = tokens + .iter() + .map(|e| e.as_ref().to_vec()) + .collect::>>(); + self + } + + pub fn with_contracts>(mut self, contracts: &[B]) -> Self { + self.contracts = contracts + .iter() + .map(|e| e.as_ref().to_vec()) + .collect::>>(); + self + } + + pub fn with_attributes, V: AsRef<[u8]>>( + mut self, + attributes: &[(K, V)], + ) -> Self { + self.static_att = attributes + .iter() + .map(|(k, v)| Attribute { + name: k.as_ref().to_string(), + value: v.as_ref().to_vec(), + change: ChangeType::Creation.into(), + }) + .collect::>(); + self + } + + pub fn as_swap_type( + mut self, + name: &str, + implementation_type: ImplementationType, + ) -> Self { + self.protocol_type = Some(ProtocolType { + name: name.to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: implementation_type.into(), + }); + self + } + } } } } diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index a3658ad..81411f4 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -34,24 +34,14 @@ pub fn map_pools_created(block: eth::v2::Block) -> Result>(); if !components.is_empty() { Some(TransactionProtocolComponents { - tx: Some(tycho::Transaction { - hash: tx.hash.clone(), - from: tx.from.clone(), - to: tx.to.clone(), - index: Into::::into(tx.index), - }), + tx: Some(tx.into()), components, }) } else { @@ -106,12 +96,7 @@ pub fn map_balance_deltas( for (token, delta) in ev.tokens.iter().zip(ev.deltas.iter()) { deltas.push(BalanceDelta { ord: vault_log.ordinal(), - tx: Some(tycho::Transaction { - hash: vault_log.receipt.transaction.hash.clone(), - from: vault_log.receipt.transaction.from.clone(), - to: vault_log.receipt.transaction.to.clone(), - index: vault_log.receipt.transaction.index.into(), - }), + tx: Some(vault_log.receipt.transaction.into()), token: token.to_vec(), delta: delta.to_signed_bytes_be(), component_id: component_id.clone(), @@ -133,24 +118,14 @@ pub fn map_balance_deltas( deltas.extend_from_slice(&[ BalanceDelta { ord: vault_log.ordinal(), - tx: Some(tycho::Transaction { - hash: vault_log.receipt.transaction.hash.clone(), - from: vault_log.receipt.transaction.from.clone(), - to: vault_log.receipt.transaction.to.clone(), - index: vault_log.receipt.transaction.index.into(), - }), + tx: Some(vault_log.receipt.transaction.into()), token: ev.token_in.to_vec(), delta: ev.amount_in.to_signed_bytes_be(), component_id: component_id.clone(), }, BalanceDelta { ord: vault_log.ordinal(), - tx: Some(tycho::Transaction { - hash: vault_log.receipt.transaction.hash.clone(), - from: vault_log.receipt.transaction.from.clone(), - to: vault_log.receipt.transaction.to.clone(), - index: vault_log.receipt.transaction.index.into(), - }), + tx: Some(vault_log.receipt.transaction.into()), token: ev.token_out.to_vec(), delta: ev.amount_out.neg().to_signed_bytes_be(), component_id, @@ -234,17 +209,7 @@ pub fn map_changes( // Process all `transaction_contract_changes` for final output in the `BlockContractChanges`, // sorted by transaction index (the key). Ok(tycho::BlockContractChanges { - block: Some(tycho::Block { - number: block.number, - hash: block.hash.clone(), - parent_hash: block - .header - .as_ref() - .expect("Block header not present") - .parent_hash - .clone(), - ts: block.timestamp_seconds(), - }), + block: Some((&block).into()), changes: transaction_contract_changes .drain() .sorted_unstable_by_key(|(index, _)| *index) diff --git a/substreams/ethereum-balancer/src/pool_factories.rs b/substreams/ethereum-balancer/src/pool_factories.rs index 27c5ced..2ef1bc8 100644 --- a/substreams/ethereum-balancer/src/pool_factories.rs +++ b/substreams/ethereum-balancer/src/pool_factories.rs @@ -3,9 +3,7 @@ use substreams_ethereum::{Event, Function}; use crate::abi; use substreams::hex; -use tycho_substreams::pb::tycho::evm::v1::{ - self as tycho, FinancialType, ImplementationType, ProtocolType, Transaction, -}; +use tycho_substreams::pb::tycho::evm::v1::{ImplementationType, ProtocolComponent, Transaction}; use substreams::scalar::BigInt; @@ -39,16 +37,16 @@ impl SerializableVecBigInt for Vec { /// - Stable Pool Factories /// (Balancer does have a bit more (esp. in the deprecated section) that could be implemented as /// desired.) -/// We use the specific ABIs to decode both the log event and cooresponding call to gather -/// `PoolCreated` event information alongside the `Create` calldata that provide us details to -/// fufill both the required details + any extra `Attributes` +/// We use the specific ABIs to decode both the log event and corresponding call to gather +/// `PoolCreated` event information alongside the `Create` call data that provide us details to +/// fulfill both the required details + any extra `Attributes` /// Ref: https://docs.balancer.fi/reference/contracts/deployment-addresses/mainnet.html pub fn address_map( pool_factory_address: &[u8], log: &Log, call: &Call, tx: &Transaction, -) -> Option { +) -> Option { match *pool_factory_address { hex!("897888115Ada5773E02aA29F775430BFB5F34c51") => { let create_call = @@ -56,31 +54,18 @@ pub fn address_map( let pool_created = abi::weighted_pool_factory::events::PoolCreated::match_and_decode(log)?; - Some(tycho::ProtocolComponent { - id: hex::encode(&pool_created.pool), - tokens: create_call.tokens, - contracts: vec![pool_created.pool], - static_att: vec![ - tycho::Attribute { - name: "pool_type".into(), - value: "WeightedPoolFactory".into(), - change: tycho::ChangeType::Creation.into(), - }, - tycho::Attribute { - name: "normalized_weights".into(), - value: create_call.normalized_weights.serialize_bytes(), - change: tycho::ChangeType::Creation.into(), - }, - ], - change: tycho::ChangeType::Creation.into(), - protocol_type: Some(ProtocolType { - name: "balancer_pool".to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: ImplementationType::Vm.into(), - }), - tx: Some(tx.clone()), - }) + Some( + ProtocolComponent::at_contract(&pool_created.pool, &tx) + .with_tokens(&create_call.tokens) + .with_attributes(&[ + ("pool_type", "WeightedPoolFactory".as_bytes()), + ( + "normalized_weights", + &create_call.normalized_weights.serialize_bytes(), + ), + ]) + .as_swap_type("balancer_pool", ImplementationType::Vm), + ) } hex!("DB8d758BCb971e482B2C45f7F8a7740283A1bd3A") => { let create_call = @@ -88,24 +73,12 @@ pub fn address_map( let pool_created = abi::composable_stable_pool_factory::events::PoolCreated::match_and_decode(log)?; - Some(tycho::ProtocolComponent { - id: hex::encode(&pool_created.pool), - tokens: create_call.tokens, - contracts: vec![pool_created.pool], - static_att: vec![tycho::Attribute { - name: "pool_type".into(), - value: "ComposableStablePoolFactory".into(), - change: tycho::ChangeType::Creation.into(), - }], - change: tycho::ChangeType::Creation.into(), - protocol_type: Some(ProtocolType { - name: "balancer_pool".to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: ImplementationType::Vm.into(), - }), - tx: Some(tx.clone()), - }) + Some( + ProtocolComponent::at_contract(&pool_created.pool, &tx) + .with_tokens(&create_call.tokens) + .with_attributes(&[("pool_type", "ComposableStablePoolFactory".as_bytes())]) + .as_swap_type("balancer_pool", ImplementationType::Vm), + ) } hex!("813EE7a840CE909E7Fea2117A44a90b8063bd4fd") => { let create_call = @@ -113,33 +86,18 @@ pub fn address_map( let pool_created = abi::erc_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; - Some(tycho::ProtocolComponent { - id: hex::encode(&pool_created.pool), - tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_created.pool], - static_att: vec![ - tycho::Attribute { - name: "pool_type".into(), - value: "ERC4626LinearPoolFactory".into(), - change: tycho::ChangeType::Creation.into(), - }, - tycho::Attribute { - name: "upper_target".into(), - value: create_call.upper_target.to_signed_bytes_be(), - change: tycho::ChangeType::Creation.into(), - }, - // Note, `lower_target` is generally hardcoded for all pools, not located in call data - // Note, rate provider might be provided as `create.protocol_id`, but as a BigInt. needs investigation - ], - change: tycho::ChangeType::Creation.into(), - protocol_type: Some(ProtocolType { - name: "balancer_pool".to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: ImplementationType::Vm.into(), - }), - tx: Some(tx.clone()), - }) + Some( + ProtocolComponent::at_contract(&pool_created.pool, &tx) + .with_tokens(&[create_call.main_token, create_call.wrapped_token]) + .with_attributes(&[ + ("pool_type", "ERC4626LinearPoolFactory".as_bytes()), + ( + "upper_target", + &create_call.upper_target.to_signed_bytes_be(), + ), + ]) + .as_swap_type("balancer_pool", ImplementationType::Vm), + ) } hex!("5F43FBa61f63Fa6bFF101a0A0458cEA917f6B347") => { let create_call = @@ -147,31 +105,18 @@ pub fn address_map( let pool_created = abi::euler_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; - Some(tycho::ProtocolComponent { - id: hex::encode(&pool_created.pool), - tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_created.pool], - static_att: vec![ - tycho::Attribute { - name: "pool_type".into(), - value: "EulerLinearPoolFactory".into(), - change: tycho::ChangeType::Creation.into(), - }, - tycho::Attribute { - name: "upper_target".into(), - value: create_call.upper_target.to_signed_bytes_be(), - change: tycho::ChangeType::Creation.into(), - }, - ], - change: tycho::ChangeType::Creation.into(), - protocol_type: Some(ProtocolType { - name: "balancer_pool".to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: ImplementationType::Vm.into(), - }), - tx: Some(tx.clone()), - }) + Some( + ProtocolComponent::at_contract(&pool_created.pool, &tx) + .with_tokens(&[create_call.main_token, create_call.wrapped_token]) + .with_attributes(&[ + ("pool_type", "EulerLinearPoolFactory".as_bytes()), + ( + "upper_target", + &create_call.upper_target.to_signed_bytes_be(), + ), + ]) + .as_swap_type("balancer_pool", ImplementationType::Vm), + ) } // ❌ Reading the deployed factory for Gearbox showcases that it's currently disabled // hex!("39A79EB449Fc05C92c39aA6f0e9BfaC03BE8dE5B") => { @@ -226,31 +171,18 @@ pub fn address_map( let pool_created = abi::silo_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; - Some(tycho::ProtocolComponent { - id: hex::encode(&pool_created.pool), - tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_created.pool], - static_att: vec![ - tycho::Attribute { - name: "pool_type".into(), - value: "SiloLinearPoolFactory".into(), - change: tycho::ChangeType::Creation.into(), - }, - tycho::Attribute { - name: "upper_target".into(), - value: create_call.upper_target.to_signed_bytes_be(), - change: tycho::ChangeType::Creation.into(), - }, - ], - change: tycho::ChangeType::Creation.into(), - protocol_type: Some(ProtocolType { - name: "balancer_pool".to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: ImplementationType::Vm.into(), - }), - tx: Some(tx.clone()), - }) + Some( + ProtocolComponent::at_contract(&pool_created.pool, &tx) + .with_tokens(&[create_call.main_token, create_call.wrapped_token]) + .with_attributes(&[ + ("pool_type", "SiloLinearPoolFactory".as_bytes()), + ( + "upper_target", + &create_call.upper_target.to_signed_bytes_be(), + ), + ]) + .as_swap_type("balancer_pool", ImplementationType::Vm), + ) } hex!("5F5222Ffa40F2AEd6380D022184D6ea67C776eE0") => { let create_call = @@ -258,65 +190,36 @@ pub fn address_map( let pool_created = abi::yearn_linear_pool_factory::events::PoolCreated::match_and_decode(log)?; - Some(tycho::ProtocolComponent { - id: hex::encode(&pool_created.pool), - tokens: vec![create_call.main_token, create_call.wrapped_token], - contracts: vec![pool_created.pool], - static_att: vec![ - tycho::Attribute { - name: "pool_type".into(), - value: "YearnLinearPoolFactory".into(), - change: tycho::ChangeType::Creation.into(), - }, - tycho::Attribute { - name: "upper_target".into(), - value: create_call.upper_target.to_signed_bytes_be(), - change: tycho::ChangeType::Creation.into(), - }, - ], - change: tycho::ChangeType::Creation.into(), - protocol_type: Some(ProtocolType { - name: "balancer_pool".to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: ImplementationType::Vm.into(), - }), - tx: Some(tx.clone()), - }) + Some( + ProtocolComponent::at_contract(&pool_created.pool, &tx) + .with_tokens(&[create_call.main_token, create_call.wrapped_token]) + .with_attributes(&[ + ("pool_type", "YearnLinearPoolFactory".as_bytes()), + ( + "upper_target", + &create_call.upper_target.to_signed_bytes_be(), + ), + ]) + .as_swap_type("balancer_pool", ImplementationType::Vm), + ) } - // The `WeightedPool2TokenFactory` is a deprecated contract but we've included it since one - // of the highest TVL pools, 80BAL-20WETH, is able to be tracked. + // The `WeightedPool2TokenFactory` is a deprecated contract, but we've included + // it to be able to track one of the highest TVL pools: 80BAL-20WETH. hex!("A5bf2ddF098bb0Ef6d120C98217dD6B141c74EE0") => { let create_call = abi::weighted_pool_tokens_factory::functions::Create::match_and_decode(call)?; let pool_created = abi::weighted_pool_tokens_factory::events::PoolCreated::match_and_decode(log)?; - Some(tycho::ProtocolComponent { - id: hex::encode(&pool_created.pool), - tokens: create_call.tokens, - contracts: vec![pool_created.pool], - static_att: vec![ - tycho::Attribute { - name: "pool_type".into(), - value: "WeightedPool2TokensFactory".into(), - change: tycho::ChangeType::Creation.into(), - }, - tycho::Attribute { - name: "weights".into(), - value: create_call.weights.serialize_bytes(), - change: tycho::ChangeType::Creation.into(), - }, - ], - change: tycho::ChangeType::Creation.into(), - protocol_type: Some(ProtocolType { - name: "balancer_pool".to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: ImplementationType::Vm.into(), - }), - tx: Some(tx.clone()), - }) + Some( + ProtocolComponent::at_contract(&pool_created.pool, &tx) + .with_tokens(&create_call.tokens) + .with_attributes(&[ + ("pool_type", "WeightedPool2TokensFactory".as_bytes()), + ("weights", &create_call.weights.serialize_bytes()), + ]) + .as_swap_type("balancer_pool", ImplementationType::Vm), + ) } _ => None, } From cfdb31b0c5c470ae665a85a3fe9624c492217e4f Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 23:12:30 +0000 Subject: [PATCH 6/8] Add documentation for the main methods added. --- crates/tycho-substreams/src/balances.rs | 30 +++++++++++++++++++++++++ crates/tycho-substreams/src/contract.rs | 28 +++++++++++++++++++---- crates/tycho-substreams/src/pb/mod.rs | 17 ++++++++++++++ 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/crates/tycho-substreams/src/balances.rs b/crates/tycho-substreams/src/balances.rs index 2547723..66ff2b5 100644 --- a/crates/tycho-substreams/src/balances.rs +++ b/crates/tycho-substreams/src/balances.rs @@ -19,6 +19,20 @@ use substreams::key; use substreams::pb::substreams::StoreDeltas; use substreams::prelude::{BigInt, StoreAdd}; +/// Store relative balances changes in a additive manner. +/// +/// Effectively aggregates the relative balances changes into an absolute balances. +/// +/// ## Arguments +/// +/// * `deltas` - A `BlockBalanceDeltas` message containing the relative balances changes. +/// Note: relative balance deltas must have strictly increasing ordinals per token +/// address, will panic otherwise. +/// * `store` - An AddStore that will add relative balance changes. +/// +/// This method is meant to be used in combination with `aggregate_balances_changes` +/// which consumes the store filled with this methods in +/// [deltas mode](https://substreams.streamingfast.io/documentation/develop/manifest-modules/types#deltas-mode). pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: impl StoreAdd) { let mut previous_ordinal = HashMap::::new(); deltas @@ -49,6 +63,22 @@ pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: impl StoreAdd for tycho::ContractChange { } } +/// Extracts relevant contract changes from the block. +/// +/// Contract changes include changes in storage, code and native balance. +/// +/// ## Arguments +/// +/// * `block` - The block to extract changes from. Must be the extended block model. +/// * `inclusion_predicate` - A predicate function that determines if a contract address is relevant. +/// * `transaction_contract_changes` - A mutable map to store the contract changes in. +/// +/// ## Panics +/// Will panic in case the detail level of the block is not extended. pub fn extract_contract_changes bool>( block: ð::v2::Block, inclusion_predicate: F, transaction_contract_changes: &mut HashMap, ) { + if block.detail_level != Into::::into(DetailLevel::DetaillevelExtended) { + panic!("Only extended blocks are supported"); + } let mut changed_contracts: HashMap, InterimContractChange> = HashMap::new(); // Collect all accounts created in this block diff --git a/crates/tycho-substreams/src/pb/mod.rs b/crates/tycho-substreams/src/pb/mod.rs index ceced59..5e6abdd 100644 --- a/crates/tycho-substreams/src/pb/mod.rs +++ b/crates/tycho-substreams/src/pb/mod.rs @@ -8,6 +8,7 @@ pub mod tycho { // @@protoc_insertion_point(tycho.evm.v1) impl TransactionContractChanges { + /// Creates a new empty `TransactionContractChanges` instance. pub fn new(tx: &Transaction) -> Self { Self { tx: Some(tx.clone()), @@ -46,6 +47,9 @@ pub mod tycho { } impl ProtocolComponent { + /// Creates a new empty `ProtocolComponent` instance. + /// + /// You can use the `with_*` methods to set the fields in a convience way. pub fn new(id: &str, tx: &Transaction) -> Self { Self { id: id.to_string(), @@ -58,6 +62,10 @@ pub mod tycho { } } + /// Shorthand to create a component with a 1-1 relationship to a contract. + /// + /// Will set the component id to a hex encoded address with a 0x prefix + /// and add the contract to contracts attributes. pub fn at_contract(id: &[u8], tx: &Transaction) -> Self { Self { id: format!("0x{}", hex::encode(id)), @@ -70,6 +78,7 @@ pub mod tycho { } } + /// Replaces the tokens on this component. pub fn with_tokens>(mut self, tokens: &[B]) -> Self { self.tokens = tokens .iter() @@ -78,6 +87,7 @@ pub mod tycho { self } + /// Replaces the contracts associated with this component. pub fn with_contracts>(mut self, contracts: &[B]) -> Self { self.contracts = contracts .iter() @@ -86,6 +96,9 @@ pub mod tycho { self } + /// Replaces the static attributes on this component. + /// + /// The change type will be set to Creation. pub fn with_attributes, V: AsRef<[u8]>>( mut self, attributes: &[(K, V)], @@ -101,6 +114,10 @@ pub mod tycho { self } + /// Sets the protocol_type on this component. + /// + /// Will set the `financial_type` to FinancialType::Swap and the + /// `attribute_schema` to an empty list. pub fn as_swap_type( mut self, name: &str, From ba2a4ced08a4da23da2be668c3bc5f9c95995444 Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 23:19:45 +0000 Subject: [PATCH 7/8] Use expect instead of unwrap. --- crates/tycho-substreams/src/balances.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/tycho-substreams/src/balances.rs b/crates/tycho-substreams/src/balances.rs index 66ff2b5..8046fe1 100644 --- a/crates/tycho-substreams/src/balances.rs +++ b/crates/tycho-substreams/src/balances.rs @@ -97,7 +97,9 @@ pub fn aggregate_balances_changes( let big_endian_bytes_balance = balance.to_bytes_be().1; ( - balance_delta.tx.unwrap(), + balance_delta + .tx + .expect("Missing transaction on delta"), BalanceChange { token: hex::decode(token_id).expect("Token ID not valid hex"), balance: big_endian_bytes_balance, From 29e5984a3a8be00a77f0efe9eb095d8e6f1e31fb Mon Sep 17 00:00:00 2001 From: kayibal Date: Wed, 13 Mar 2024 23:49:40 +0000 Subject: [PATCH 8/8] Add prelude; Move pb implementations. Else the implementations would get overwritten. --- crates/tycho-substreams/src/lib.rs | 7 +- crates/tycho-substreams/src/models.rs | 123 +++++++++++++++++ crates/tycho-substreams/src/pb/mod.rs | 127 ------------------ substreams/ethereum-balancer/src/modules.rs | 22 ++- .../ethereum-balancer/src/pool_factories.rs | 8 +- 5 files changed, 141 insertions(+), 146 deletions(-) create mode 100644 crates/tycho-substreams/src/models.rs diff --git a/crates/tycho-substreams/src/lib.rs b/crates/tycho-substreams/src/lib.rs index 78c7e2e..03e9e03 100644 --- a/crates/tycho-substreams/src/lib.rs +++ b/crates/tycho-substreams/src/lib.rs @@ -1,4 +1,9 @@ pub mod balances; pub mod contract; mod mock_store; -pub mod pb; +pub mod models; +mod pb; + +pub mod prelude { + pub use super::models::*; +} diff --git a/crates/tycho-substreams/src/models.rs b/crates/tycho-substreams/src/models.rs new file mode 100644 index 0000000..14ab6f5 --- /dev/null +++ b/crates/tycho-substreams/src/models.rs @@ -0,0 +1,123 @@ +use substreams_ethereum::pb::eth::v2::{self as sf}; + +// re-export the protobuf types here. +pub use crate::pb::tycho::evm::v1::*; + +impl TransactionContractChanges { + /// Creates a new empty `TransactionContractChanges` instance. + pub fn new(tx: &Transaction) -> Self { + Self { + tx: Some(tx.clone()), + contract_changes: vec![], + component_changes: vec![], + balance_changes: vec![], + } + } +} + +impl From<&sf::TransactionTrace> for Transaction { + fn from(tx: &sf::TransactionTrace) -> Self { + Self { + hash: tx.hash.clone(), + from: tx.from.clone(), + to: tx.to.clone(), + index: tx.index.into(), + } + } +} + +impl From<&sf::Block> for Block { + fn from(block: &sf::Block) -> Self { + Self { + number: block.number, + hash: block.hash.clone(), + parent_hash: block + .header + .as_ref() + .expect("Block header not present") + .parent_hash + .clone(), + ts: block.timestamp_seconds(), + } + } +} + +impl ProtocolComponent { + /// Creates a new empty `ProtocolComponent` instance. + /// + /// You can use the `with_*` methods to set the fields in a convience way. + pub fn new(id: &str, tx: &Transaction) -> Self { + Self { + id: id.to_string(), + tokens: vec![], + contracts: vec![], + static_att: vec![], + change: ChangeType::Creation.into(), + protocol_type: None, + tx: Some(tx.clone()), + } + } + + /// Shorthand to create a component with a 1-1 relationship to a contract. + /// + /// Will set the component id to a hex encoded address with a 0x prefix + /// and add the contract to contracts attributes. + pub fn at_contract(id: &[u8], tx: &Transaction) -> Self { + Self { + id: format!("0x{}", hex::encode(id)), + tokens: vec![], + contracts: vec![id.to_vec()], + static_att: vec![], + change: ChangeType::Creation.into(), + protocol_type: None, + tx: Some(tx.clone()), + } + } + + /// Replaces the tokens on this component. + pub fn with_tokens>(mut self, tokens: &[B]) -> Self { + self.tokens = tokens + .iter() + .map(|e| e.as_ref().to_vec()) + .collect::>>(); + self + } + + /// Replaces the contracts associated with this component. + pub fn with_contracts>(mut self, contracts: &[B]) -> Self { + self.contracts = contracts + .iter() + .map(|e| e.as_ref().to_vec()) + .collect::>>(); + self + } + + /// Replaces the static attributes on this component. + /// + /// The change type will be set to Creation. + pub fn with_attributes, V: AsRef<[u8]>>(mut self, attributes: &[(K, V)]) -> Self { + self.static_att = attributes + .iter() + .map(|(k, v)| Attribute { + name: k.as_ref().to_string(), + value: v.as_ref().to_vec(), + change: ChangeType::Creation.into(), + }) + .collect::>(); + self + } + + /// Sets the protocol_type on this component. + /// + /// Will set the `financial_type` to FinancialType::Swap and the + /// `attribute_schema` to an empty list. + pub fn as_swap_type(mut self, name: &str, implementation_type: ImplementationType) -> Self { + self.protocol_type = Some(ProtocolType { + name: name.to_string(), + financial_type: FinancialType::Swap.into(), + attribute_schema: vec![], + implementation_type: implementation_type.into(), + }); + self + } +} diff --git a/crates/tycho-substreams/src/pb/mod.rs b/crates/tycho-substreams/src/pb/mod.rs index 5e6abdd..43d8838 100644 --- a/crates/tycho-substreams/src/pb/mod.rs +++ b/crates/tycho-substreams/src/pb/mod.rs @@ -3,135 +3,8 @@ pub mod tycho { pub mod evm { // @@protoc_insertion_point(attribute:tycho.evm.v1) pub mod v1 { - use substreams_ethereum::pb::eth::v2::{self as sf}; include!("tycho.evm.v1.rs"); // @@protoc_insertion_point(tycho.evm.v1) - - impl TransactionContractChanges { - /// Creates a new empty `TransactionContractChanges` instance. - pub fn new(tx: &Transaction) -> Self { - Self { - tx: Some(tx.clone()), - contract_changes: vec![], - component_changes: vec![], - balance_changes: vec![], - } - } - } - - impl From<&sf::TransactionTrace> for Transaction { - fn from(tx: &sf::TransactionTrace) -> Self { - Self { - hash: tx.hash.clone(), - from: tx.from.clone(), - to: tx.to.clone(), - index: tx.index.into(), - } - } - } - - impl From<&sf::Block> for Block { - fn from(block: &sf::Block) -> Self { - Self { - number: block.number, - hash: block.hash.clone(), - parent_hash: block - .header - .as_ref() - .expect("Block header not present") - .parent_hash - .clone(), - ts: block.timestamp_seconds(), - } - } - } - - impl ProtocolComponent { - /// Creates a new empty `ProtocolComponent` instance. - /// - /// You can use the `with_*` methods to set the fields in a convience way. - pub fn new(id: &str, tx: &Transaction) -> Self { - Self { - id: id.to_string(), - tokens: vec![], - contracts: vec![], - static_att: vec![], - change: ChangeType::Creation.into(), - protocol_type: None, - tx: Some(tx.clone()), - } - } - - /// Shorthand to create a component with a 1-1 relationship to a contract. - /// - /// Will set the component id to a hex encoded address with a 0x prefix - /// and add the contract to contracts attributes. - pub fn at_contract(id: &[u8], tx: &Transaction) -> Self { - Self { - id: format!("0x{}", hex::encode(id)), - tokens: vec![], - contracts: vec![id.to_vec()], - static_att: vec![], - change: ChangeType::Creation.into(), - protocol_type: None, - tx: Some(tx.clone()), - } - } - - /// Replaces the tokens on this component. - pub fn with_tokens>(mut self, tokens: &[B]) -> Self { - self.tokens = tokens - .iter() - .map(|e| e.as_ref().to_vec()) - .collect::>>(); - self - } - - /// Replaces the contracts associated with this component. - pub fn with_contracts>(mut self, contracts: &[B]) -> Self { - self.contracts = contracts - .iter() - .map(|e| e.as_ref().to_vec()) - .collect::>>(); - self - } - - /// Replaces the static attributes on this component. - /// - /// The change type will be set to Creation. - pub fn with_attributes, V: AsRef<[u8]>>( - mut self, - attributes: &[(K, V)], - ) -> Self { - self.static_att = attributes - .iter() - .map(|(k, v)| Attribute { - name: k.as_ref().to_string(), - value: v.as_ref().to_vec(), - change: ChangeType::Creation.into(), - }) - .collect::>(); - self - } - - /// Sets the protocol_type on this component. - /// - /// Will set the `financial_type` to FinancialType::Swap and the - /// `attribute_schema` to an empty list. - pub fn as_swap_type( - mut self, - name: &str, - implementation_type: ImplementationType, - ) -> Self { - self.protocol_type = Some(ProtocolType { - name: name.to_string(), - financial_type: FinancialType::Swap.into(), - attribute_schema: vec![], - implementation_type: implementation_type.into(), - }); - self - } - } } } } diff --git a/substreams/ethereum-balancer/src/modules.rs b/substreams/ethereum-balancer/src/modules.rs index 81411f4..728a53e 100644 --- a/substreams/ethereum-balancer/src/modules.rs +++ b/substreams/ethereum-balancer/src/modules.rs @@ -9,12 +9,9 @@ use substreams::store::{ }; use substreams_ethereum::pb::eth; use substreams_ethereum::Event; -use tycho_substreams::balances; +use tycho_substreams::balances::aggregate_balances_changes; use tycho_substreams::contract::extract_contract_changes; -use tycho_substreams::pb::tycho::evm::v1::{ - self as tycho, BalanceDelta, BlockBalanceDeltas, BlockTransactionProtocolComponents, - TransactionProtocolComponents, -}; +use tycho_substreams::prelude::*; const VAULT_ADDRESS: &[u8] = &hex!("BA12222222228d8Ba445958a75a0704d566BF2C8"); @@ -145,7 +142,7 @@ pub fn map_balance_deltas( /// store key to ensure that there's a unique balance being tallied for each. #[substreams::handlers::store] pub fn store_balance_changes(deltas: BlockBalanceDeltas, store: StoreAddBigInt) { - balances::store_balance_changes(deltas, store); + tycho_substreams::balances::store_balance_changes(deltas, store); } /// This is the main map that handles most of the indexing of this substream. @@ -161,11 +158,10 @@ pub fn map_changes( deltas: BlockBalanceDeltas, components_store: StoreGetInt64, balance_store: StoreDeltas, // Note, this map module is using the `deltas` mode for the store. -) -> Result { +) -> Result { // We merge contract changes by transaction (identified by transaction index) making it easy to // sort them at the very end. - let mut transaction_contract_changes: HashMap<_, tycho::TransactionContractChanges> = - HashMap::new(); + let mut transaction_contract_changes: HashMap<_, TransactionContractChanges> = HashMap::new(); // `ProtocolComponents` are gathered from `map_pools_created` which just need a bit of work to // convert into `TransactionContractChanges` @@ -176,7 +172,7 @@ pub fn map_changes( let tx = tx_component.tx.as_ref().unwrap(); transaction_contract_changes .entry(tx.index) - .or_insert_with(|| tycho::TransactionContractChanges::new(&tx)) + .or_insert_with(|| TransactionContractChanges::new(&tx)) .component_changes .extend_from_slice(&tx_component.components); }); @@ -185,12 +181,12 @@ pub fn map_changes( // `BlockBalanceDeltas`. We essentially just process the changes that occurred to the `store` this // block. Then, these balance changes are merged onto the existing map of tx contract changes, // inserting a new one if it doesn't exist. - balances::aggregate_balances_changes(balance_store, deltas) + aggregate_balances_changes(balance_store, deltas) .into_iter() .for_each(|(_, (tx, balances))| { transaction_contract_changes .entry(tx.index) - .or_insert_with(|| tycho::TransactionContractChanges::new(&tx)) + .or_insert_with(|| TransactionContractChanges::new(&tx)) .balance_changes .extend(balances.into_iter().map(|(_, change)| change)); }); @@ -208,7 +204,7 @@ pub fn map_changes( // Process all `transaction_contract_changes` for final output in the `BlockContractChanges`, // sorted by transaction index (the key). - Ok(tycho::BlockContractChanges { + Ok(BlockContractChanges { block: Some((&block).into()), changes: transaction_contract_changes .drain() diff --git a/substreams/ethereum-balancer/src/pool_factories.rs b/substreams/ethereum-balancer/src/pool_factories.rs index 2ef1bc8..16691c3 100644 --- a/substreams/ethereum-balancer/src/pool_factories.rs +++ b/substreams/ethereum-balancer/src/pool_factories.rs @@ -1,11 +1,9 @@ -use substreams_ethereum::pb::eth::v2::{Call, Log}; -use substreams_ethereum::{Event, Function}; - use crate::abi; use substreams::hex; -use tycho_substreams::pb::tycho::evm::v1::{ImplementationType, ProtocolComponent, Transaction}; - use substreams::scalar::BigInt; +use substreams_ethereum::pb::eth::v2::{Call, Log}; +use substreams_ethereum::{Event, Function}; +use tycho_substreams::prelude::*; /// This trait defines some helpers for serializing and deserializing `Vec