From 8a7271bc59e73d6d64e6495315ad575293888b09 Mon Sep 17 00:00:00 2001 From: pistomat Date: Wed, 20 Dec 2023 14:29:26 +0100 Subject: [PATCH 1/6] Clone proto and substreams from tycho-indexer --- .gitignore | 12 + proto/buf.lock | 2 + proto/buf.yaml | 7 + proto/sf/substreams/internal/v2/deltas.proto | 36 + proto/sf/substreams/internal/v2/service.proto | 88 ++ proto/sf/substreams/options.proto | 20 + proto/sf/substreams/rpc/v2/service.proto | 235 ++++ .../substreams/sink/service/v1/service.proto | 142 +++ proto/sf/substreams/v1/clock.proto | 20 + proto/sf/substreams/v1/modules.proto | 98 ++ proto/sf/substreams/v1/package.proto | 42 + proto/tycho/evm/v1/common.proto | 44 + proto/tycho/evm/v1/entity.proto | 22 + proto/tycho/evm/v1/vm.proto | 34 + substreams/.gitkeep | 0 substreams/Readme.md | 95 ++ substreams/ethereum-ambient/Cargo.lock | 1095 +++++++++++++++++ substreams/ethereum-ambient/Cargo.toml | 18 + substreams/ethereum-ambient/src/lib.rs | 389 ++++++ substreams/ethereum-ambient/src/pb/mod.rs | 10 + .../ethereum-ambient/src/pb/tycho.evm.v1.rs | 166 +++ substreams/ethereum-ambient/substreams.yaml | 24 + 22 files changed, 2599 insertions(+) create mode 100644 proto/buf.lock create mode 100644 proto/buf.yaml create mode 100644 proto/sf/substreams/internal/v2/deltas.proto create mode 100644 proto/sf/substreams/internal/v2/service.proto create mode 100644 proto/sf/substreams/options.proto create mode 100644 proto/sf/substreams/rpc/v2/service.proto create mode 100644 proto/sf/substreams/sink/service/v1/service.proto create mode 100644 proto/sf/substreams/v1/clock.proto create mode 100644 proto/sf/substreams/v1/modules.proto create mode 100644 proto/sf/substreams/v1/package.proto create mode 100644 proto/tycho/evm/v1/common.proto create mode 100644 proto/tycho/evm/v1/entity.proto create mode 100644 proto/tycho/evm/v1/vm.proto delete mode 100644 substreams/.gitkeep create mode 100644 substreams/Readme.md create mode 100644 substreams/ethereum-ambient/Cargo.lock create mode 100644 substreams/ethereum-ambient/Cargo.toml create mode 100644 substreams/ethereum-ambient/src/lib.rs create mode 100644 substreams/ethereum-ambient/src/pb/mod.rs create mode 100644 substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs create mode 100644 substreams/ethereum-ambient/substreams.yaml diff --git a/.gitignore b/.gitignore index 485dee6..2c65d74 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,13 @@ +### Rust ### +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Substreams spkg files are build artifacts +*.spkg + +.env +.vscode .idea +*.log diff --git a/proto/buf.lock b/proto/buf.lock new file mode 100644 index 0000000..c91b581 --- /dev/null +++ b/proto/buf.lock @@ -0,0 +1,2 @@ +# Generated by buf. DO NOT EDIT. +version: v1 diff --git a/proto/buf.yaml b/proto/buf.yaml new file mode 100644 index 0000000..d4ff52d --- /dev/null +++ b/proto/buf.yaml @@ -0,0 +1,7 @@ +version: v1 +breaking: + use: + - FILE +lint: + use: + - BASIC diff --git a/proto/sf/substreams/internal/v2/deltas.proto b/proto/sf/substreams/internal/v2/deltas.proto new file mode 100644 index 0000000..21b9e40 --- /dev/null +++ b/proto/sf/substreams/internal/v2/deltas.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package sf.substreams.internal.v2; + +import "google/protobuf/any.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/internal/v2;pbssinternal"; + +message StoreDeltas { + repeated StoreDelta store_deltas = 1; +} + +message StoreDelta { + enum Operation { + UNSET = 0; + CREATE = 1; + UPDATE = 2; + DELETE = 3; + } + Operation operation = 1; + uint64 ordinal = 2; + string key = 3; + bytes old_value = 4; + bytes new_value = 5; +} + +message ModuleOutput { + string module_name = 1; + oneof data { + google.protobuf.Any map_output = 2; + StoreDeltas store_deltas = 3; + } + repeated string logs = 4; + bool debug_logs_truncated = 5; + bool cached = 6; +} diff --git a/proto/sf/substreams/internal/v2/service.proto b/proto/sf/substreams/internal/v2/service.proto new file mode 100644 index 0000000..f695707 --- /dev/null +++ b/proto/sf/substreams/internal/v2/service.proto @@ -0,0 +1,88 @@ +syntax = "proto3"; + +package sf.substreams.internal.v2; + +import "sf/substreams/v1/modules.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/internal/v2;pbssinternal"; + +service Substreams { + rpc ProcessRange(ProcessRangeRequest) returns (stream ProcessRangeResponse); +} + +message ProcessRangeRequest { + uint64 start_block_num = 1; + uint64 stop_block_num = 2; + string output_module = 3; + sf.substreams.v1.Modules modules = 4; + uint32 stage = 5; // 0-based index of stage to execute up to +} + +message ProcessRangeResponse { + reserved 1; // previously string module_name = 1; + + reserved 2; // previously in oneof(type): BlockRange processed_range + reserved 3; // previously in oneof(type): ProcessedBytes processed_bytes + + oneof type { + Failed failed = 4; + Completed completed = 5; + Update update = 6; + } +} + +message Update { + uint64 duration_ms = 1; + uint64 processed_blocks = 2; + uint64 total_bytes_read = 3; + uint64 total_bytes_written = 4; + + repeated ModuleStats modules_stats = 5; +} + +message ModuleStats { + string name = 1; + uint64 processing_time_ms = 2; + uint64 store_operation_time_ms = 3; + uint64 store_read_count = 4; + + repeated ExternalCallMetric external_call_metrics = 5; + + // store-specific (will be 0 on mappers) + uint64 store_write_count = 10; + uint64 store_deleteprefix_count = 11; + uint64 store_size_bytes = 12; +} + +message ExternalCallMetric { + string name = 1; + uint64 count = 2; + uint64 time_ms = 3; +} + +message Completed { + repeated BlockRange all_processed_ranges = 1; + + // TraceId represents the producer's trace id that produced the partial files. + // This is present here so that the consumer can use it to identify the + // right partial files that needs to be squashed together. + // + // The TraceId can be empty in which case it should be assumed by the tier1 + // consuming this message that the tier2 that produced those partial files + // is not yet updated to produce a trace id and a such, the tier1 should + // generate a legacy partial file name. + string trace_id = 2; +} + +message Failed { + string reason = 1; + repeated string logs = 2; + // FailureLogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 3; +} + +message BlockRange { + uint64 start_block = 2; + uint64 end_block = 3; +} diff --git a/proto/sf/substreams/options.proto b/proto/sf/substreams/options.proto new file mode 100644 index 0000000..9ea5537 --- /dev/null +++ b/proto/sf/substreams/options.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +package sf.substreams; + +import "google/protobuf/descriptor.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams;pbsubstreams"; + +message FieldOptions { + // this option informs the `substreams pack` command that it should treat the corresponding manifest value as a path to a file, putting its content as bytes in this field. + // must be applied to a `bytes` or `string` field + bool load_from_file = 1; + + // this option informs the `substreams pack` command that it should treat the corresponding manifest value as a path to a folder, zipping its content and putting the zip content as bytes in this field. + // must be applied to a `bytes` field + bool zip_from_folder = 2; +} + +extend google.protobuf.FieldOptions { + optional FieldOptions options = 2200; +} diff --git a/proto/sf/substreams/rpc/v2/service.proto b/proto/sf/substreams/rpc/v2/service.proto new file mode 100644 index 0000000..05e69c0 --- /dev/null +++ b/proto/sf/substreams/rpc/v2/service.proto @@ -0,0 +1,235 @@ +syntax = "proto3"; + +package sf.substreams.rpc.v2; + +import "google/protobuf/any.proto"; +import "sf/substreams/v1/clock.proto"; +import "sf/substreams/v1/modules.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2;pbsubstreamsrpc"; + +service Stream { + rpc Blocks(Request) returns (stream Response); +} + +message Request { + int64 start_block_num = 1; + string start_cursor = 2; + uint64 stop_block_num = 3; + + // With final_block_only, you only receive blocks that are irreversible: + // 'final_block_height' will be equal to current block and no 'undo_signal' will ever be sent + bool final_blocks_only = 4; + + // Substreams has two mode when executing your module(s) either development mode or production + // mode. Development and production modes impact the execution of Substreams, important aspects + // of execution include: + // * The time required to reach the first byte. + // * The speed that large ranges get executed. + // * The module logs and outputs sent back to the client. + // + // By default, the engine runs in developer mode, with richer and deeper output. Differences + // between production and development modes include: + // * Forward parallel execution is enabled in production mode and disabled in development mode + // * The time required to reach the first byte in development mode is faster than in production mode. + // + // Specific attributes of development mode include: + // * The client will receive all of the executed module's logs. + // * It's possible to request specific store snapshots in the execution tree (via `debug_initial_store_snapshot_for_modules`). + // * Multiple module's output is possible. + // + // With production mode`, however, you trade off functionality for high speed enabling forward + // parallel execution of module ahead of time. + bool production_mode = 5; + + string output_module = 6; + + sf.substreams.v1.Modules modules = 7; + + // Available only in developer mode + repeated string debug_initial_store_snapshot_for_modules = 10; +} + +message Response { + oneof message { + SessionInit session = 1; // Always sent first + ModulesProgress progress = 2; // Progress of data preparation, before sending in the stream of `data` events. + BlockScopedData block_scoped_data = 3; + BlockUndoSignal block_undo_signal = 4; + Error fatal_error = 5; + + // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + InitialSnapshotData debug_snapshot_data = 10; + // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + InitialSnapshotComplete debug_snapshot_complete = 11; + } +} + +// BlockUndoSignal informs you that every bit of data +// with a block number above 'last_valid_block' has been reverted +// on-chain. Delete that data and restart from 'last_valid_cursor' +message BlockUndoSignal { + sf.substreams.v1.BlockRef last_valid_block = 1; + string last_valid_cursor = 2; +} + +message BlockScopedData { + MapModuleOutput output = 1; + sf.substreams.v1.Clock clock = 2; + string cursor = 3; + + // Non-deterministic, allows substreams-sink to let go of their undo data. + uint64 final_block_height = 4; + + repeated MapModuleOutput debug_map_outputs = 10; + repeated StoreModuleOutput debug_store_outputs = 11; +} + +message SessionInit { + string trace_id = 1; + uint64 resolved_start_block = 2; + uint64 linear_handoff_block = 3; + uint64 max_parallel_workers = 4; +} + +message InitialSnapshotComplete { + string cursor = 1; +} + +message InitialSnapshotData { + string module_name = 1; + repeated StoreDelta deltas = 2; + uint64 sent_keys = 4; + uint64 total_keys = 3; +} + +message MapModuleOutput { + string name = 1; + google.protobuf.Any map_output = 2; + // DebugOutputInfo is available in non-production mode only + OutputDebugInfo debug_info = 10; +} + +// StoreModuleOutput are produced for store modules in development mode. +// It is not possible to retrieve store models in production, with parallelization +// enabled. If you need the deltas directly, write a pass through mapper module +// that will get them down to you. +message StoreModuleOutput { + string name = 1; + repeated StoreDelta debug_store_deltas = 2; + OutputDebugInfo debug_info = 10; +} + +message OutputDebugInfo { + repeated string logs = 1; + // LogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 2; + bool cached = 3; +} + +// ModulesProgress is a message that is sent every 500ms +message ModulesProgress { + // previously: repeated ModuleProgress modules = 1; + // these previous `modules` messages were sent in bursts and are not sent anymore. + reserved 1; + // List of jobs running on tier2 servers + repeated Job running_jobs = 2; + // Execution statistics for each module + repeated ModuleStats modules_stats = 3; + // Stages definition and completed block ranges + repeated Stage stages = 4; + + ProcessedBytes processed_bytes = 5; +} + +message ProcessedBytes { + uint64 total_bytes_read = 1; + uint64 total_bytes_written = 2; +} + +message Error { + string module = 1; + string reason = 2; + repeated string logs = 3; + // FailureLogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 4; +} + +message Job { + uint32 stage = 1; + uint64 start_block = 2; + uint64 stop_block = 3; + uint64 processed_blocks = 4; + uint64 duration_ms = 5; +} + +message Stage { + repeated string modules = 1; + repeated BlockRange completed_ranges = 2; +} + +// ModuleStats gathers metrics and statistics from each module, running on tier1 or tier2 +// All the 'count' and 'time_ms' values may include duplicate for each stage going over that module +message ModuleStats { + // name of the module + string name = 1; + + // total_processed_blocks is the sum of blocks sent to that module code + uint64 total_processed_block_count = 2; + // total_processing_time_ms is the sum of all time spent running that module code + uint64 total_processing_time_ms = 3; + + //// external_calls are chain-specific intrinsics, like "Ethereum RPC calls". + repeated ExternalCallMetric external_call_metrics = 4; + + // total_store_operation_time_ms is the sum of all time spent running that module code waiting for a store operation (ex: read, write, delete...) + uint64 total_store_operation_time_ms = 5; + // total_store_read_count is the sum of all the store Read operations called from that module code + uint64 total_store_read_count = 6; + + // total_store_write_count is the sum of all store Write operations called from that module code (store-only) + uint64 total_store_write_count = 10; + + // total_store_deleteprefix_count is the sum of all store DeletePrefix operations called from that module code (store-only) + // note that DeletePrefix can be a costly operation on large stores + uint64 total_store_deleteprefix_count = 11; + + // store_size_bytes is the uncompressed size of the full KV store for that module, from the last 'merge' operation (store-only) + uint64 store_size_bytes = 12; + + // total_store_merging_time_ms is the time spent merging partial stores into a full KV store for that module (store-only) + uint64 total_store_merging_time_ms = 13; + + // store_currently_merging is true if there is a merging operation (partial store to full KV store) on the way. + bool store_currently_merging = 14; + + // highest_contiguous_block is the highest block in the highest merged full KV store of that module (store-only) + uint64 highest_contiguous_block = 15; +} + +message ExternalCallMetric { + string name = 1; + uint64 count = 2; + uint64 time_ms = 3; +} + +message StoreDelta { + enum Operation { + UNSET = 0; + CREATE = 1; + UPDATE = 2; + DELETE = 3; + } + Operation operation = 1; + uint64 ordinal = 2; + string key = 3; + bytes old_value = 4; + bytes new_value = 5; +} + +message BlockRange { + uint64 start_block = 2; + uint64 end_block = 3; +} diff --git a/proto/sf/substreams/sink/service/v1/service.proto b/proto/sf/substreams/sink/service/v1/service.proto new file mode 100644 index 0000000..4c85e9f --- /dev/null +++ b/proto/sf/substreams/sink/service/v1/service.proto @@ -0,0 +1,142 @@ +syntax = "proto3"; + +package sf.substreams.sink.service.v1; + +import "sf/substreams/v1/package.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/sink/service/v1;pbsinksvc"; + +service Provider { + rpc Deploy(DeployRequest) returns (DeployResponse); + rpc Update(UpdateRequest) returns (UpdateResponse); + rpc Info(InfoRequest) returns (InfoResponse); + rpc List(ListRequest) returns (ListResponse); + rpc Pause(PauseRequest) returns (PauseResponse); + rpc Stop(StopRequest) returns (StopResponse); + rpc Resume(ResumeRequest) returns (ResumeResponse); + rpc Remove(RemoveRequest) returns (RemoveResponse); +} + +message DeployRequest { + sf.substreams.v1.Package substreams_package = 1; + + bool development_mode = 2; + repeated Parameter parameters = 3; +} + +message Parameter { + string key = 1; + string value = 2; +} + +message DeployResponse { + DeploymentStatus status = 1; + + // deployment_id is a short name (max 8 characters) that uniquely identifies your deployment + string deployment_id = 2; + + map services = 3; + string reason = 4; + string motd = 5; +} + +message UpdateRequest { + sf.substreams.v1.Package substreams_package = 1; + string deployment_id = 2; + bool reset = 3; +} + +message UpdateResponse { + DeploymentStatus status = 1; + map services = 2; + string reason = 3; + string motd = 4; +} + +message InfoRequest { + string deployment_id = 1; +} + +message InfoResponse { + DeploymentStatus status = 1; + map services = 2; + string reason = 3; + PackageInfo package_info = 4; + SinkProgress progress = 5; + string motd = 6; +} + +message SinkProgress { + uint64 last_processed_block = 1; +} + +message PackageInfo { + string name = 1; + string version = 2; + string output_module_name = 3; + string output_module_hash = 4; +} + +message ListRequest {} + +message ListResponse { + repeated DeploymentWithStatus deployments = 1; +} + +message DeploymentWithStatus { + string id = 1; + DeploymentStatus status = 2; + string reason = 3; + PackageInfo package_info = 4; + SinkProgress progress = 5; + string motd = 6; +} + +enum DeploymentStatus { + UNKNOWN = 0; + RUNNING = 1; + FAILING = 2; + PAUSED = 3; + STOPPED = 4; + + STARTING = 5; + PAUSING = 6; + STOPPING = 7; + REMOVING = 8; + RESUMING = 9; +} + +message RemoveRequest { + string deployment_id = 1; +} + +message RemoveResponse { + DeploymentStatus previous_status = 1; +} + +message PauseRequest { + string deployment_id = 1; +} + +message PauseResponse { + DeploymentStatus previous_status = 1; + DeploymentStatus new_status = 2; +} + +message StopRequest { + string deployment_id = 1; +} + +message StopResponse { + DeploymentStatus previous_status = 1; + DeploymentStatus new_status = 2; +} + +message ResumeRequest { + string deployment_id = 1; +} + +message ResumeResponse { + DeploymentStatus previous_status = 1; + DeploymentStatus new_status = 2; +} diff --git a/proto/sf/substreams/v1/clock.proto b/proto/sf/substreams/v1/clock.proto new file mode 100644 index 0000000..9d3e4bd --- /dev/null +++ b/proto/sf/substreams/v1/clock.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package sf.substreams.v1; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/v1;pbsubstreams"; + +// Clock is a pointer to a block with added timestamp +message Clock { + string id = 1; + uint64 number = 2; + google.protobuf.Timestamp timestamp = 3; +} + +// BlockRef is a pointer to a block to which we don't know the timestamp +message BlockRef { + string id = 1; + uint64 number = 2; +} diff --git a/proto/sf/substreams/v1/modules.proto b/proto/sf/substreams/v1/modules.proto new file mode 100644 index 0000000..bde9c27 --- /dev/null +++ b/proto/sf/substreams/v1/modules.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; + +package sf.substreams.v1; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/v1;pbsubstreams"; + +message Modules { + repeated Module modules = 1; + repeated Binary binaries = 2; +} + +// Binary represents some code compiled to its binary form. +message Binary { + string type = 1; + bytes content = 2; +} + +message Module { + string name = 1; + oneof kind { + KindMap kind_map = 2; + KindStore kind_store = 3; + } + + uint32 binary_index = 4; + string binary_entrypoint = 5; + + repeated Input inputs = 6; + Output output = 7; + + uint64 initial_block = 8; + + message KindMap { + string output_type = 1; + } + + message KindStore { + // The `update_policy` determines the functions available to mutate the store + // (like `set()`, `set_if_not_exists()` or `sum()`, etc..) in + // order to ensure that parallel operations are possible and deterministic + // + // Say a store cumulates keys from block 0 to 1M, and a second store + // cumulates keys from block 1M to 2M. When we want to use this + // store as a dependency for a downstream module, we will merge the + // two stores according to this policy. + UpdatePolicy update_policy = 1; + string value_type = 2; + + enum UpdatePolicy { + UPDATE_POLICY_UNSET = 0; + // Provides a store where you can `set()` keys, and the latest key wins + UPDATE_POLICY_SET = 1; + // Provides a store where you can `set_if_not_exists()` keys, and the first key wins + UPDATE_POLICY_SET_IF_NOT_EXISTS = 2; + // Provides a store where you can `add_*()` keys, where two stores merge by summing its values. + UPDATE_POLICY_ADD = 3; + // Provides a store where you can `min_*()` keys, where two stores merge by leaving the minimum value. + UPDATE_POLICY_MIN = 4; + // Provides a store where you can `max_*()` keys, where two stores merge by leaving the maximum value. + UPDATE_POLICY_MAX = 5; + // Provides a store where you can `append()` keys, where two stores merge by concatenating the bytes in order. + UPDATE_POLICY_APPEND = 6; + } + } + + message Input { + oneof input { + Source source = 1; + Map map = 2; + Store store = 3; + Params params = 4; + } + + message Source { + string type = 1; // ex: "sf.ethereum.type.v1.Block" + } + message Map { + string module_name = 1; // ex: "block_to_pairs" + } + message Store { + string module_name = 1; + Mode mode = 2; + + enum Mode { + UNSET = 0; + GET = 1; + DELTAS = 2; + } + } + message Params { + string value = 1; + } + } + + message Output { + string type = 1; + } +} diff --git a/proto/sf/substreams/v1/package.proto b/proto/sf/substreams/v1/package.proto new file mode 100644 index 0000000..3bf7707 --- /dev/null +++ b/proto/sf/substreams/v1/package.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +package sf.substreams.v1; + +import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; +import "sf/substreams/v1/modules.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/v1;pbsubstreams"; + +message Package { + // Needs to be one so this file can be used _directly_ as a + // buf `Image` andor a ProtoSet for grpcurl and other tools + repeated google.protobuf.FileDescriptorProto proto_files = 1; + reserved 2 to 4; // Reserved for future: in case protosets adds fields + + uint64 version = 5; + sf.substreams.v1.Modules modules = 6; + repeated ModuleMetadata module_meta = 7; + repeated PackageMetadata package_meta = 8; + + // Source network for Substreams to fetch its data from. + string network = 9; + + google.protobuf.Any sink_config = 10; + string sink_module = 11; + // image is the bytes to a JPEG, WebP or PNG file. Max size is 2 MiB + bytes image = 12; +} + +message PackageMetadata { + string version = 1; + string url = 2; + string name = 3; + string doc = 4; +} + +message ModuleMetadata { + // Corresponds to the index in `Package.metadata.package_meta` + uint64 package_index = 1; + string doc = 2; +} diff --git a/proto/tycho/evm/v1/common.proto b/proto/tycho/evm/v1/common.proto new file mode 100644 index 0000000..cbf9eb8 --- /dev/null +++ b/proto/tycho/evm/v1/common.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +package tycho.evm.v1; + +message Block { + bytes hash = 1; + bytes parent_hash = 2; + uint64 number = 3; + uint64 ts = 4; +} + +message Transaction { + bytes hash = 1; + bytes from = 2; + bytes to = 3; + uint64 index = 4; +} + +enum ChangeType { + CHANGE_TYPE_UNSPECIFIED = 0; + CHANGE_TYPE_UPDATE = 1; + CHANGE_TYPE_CREATION = 2; + CHANGE_TYPE_DELETION = 3; +} + +message Attribute { + string name = 1; + bytes value = 2; + ChangeType change = 3; +} + +message ProtocolComponent { + string id = 1; + repeated bytes tokens = 2; + repeated string contracts = 3; + repeated Attribute static_att = 4; + ChangeType change = 5; +} + +message BalanceChange { + bytes token = 1; + bytes balance = 2; + bytes component_id = 3; +} diff --git a/proto/tycho/evm/v1/entity.proto b/proto/tycho/evm/v1/entity.proto new file mode 100644 index 0000000..7a940ea --- /dev/null +++ b/proto/tycho/evm/v1/entity.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package tycho.evm.v1; + +import "tycho/evm/v1/common.proto"; + +message EntityChanges { + string component_id = 1; + repeated Attribute attributes = 2; +} + +message TransactionEntityChanges { + Transaction tx = 1; + repeated EntityChanges entity_changes = 2; + repeated ProtocolComponent component_changes = 3; + repeated BalanceChange balance_changes = 4; +} + +message BlockEntityChanges { + Block block = 1; + repeated TransactionEntityChanges changes = 2; +} diff --git a/proto/tycho/evm/v1/vm.proto b/proto/tycho/evm/v1/vm.proto new file mode 100644 index 0000000..8305f69 --- /dev/null +++ b/proto/tycho/evm/v1/vm.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package tycho.evm.v1; + +import "tycho/evm/v1/common.proto"; + +message ContractSlot { + bytes slot = 2; + bytes value = 3; +} + +message ContractChange { + bytes address = 1; + // empty bytes indicates no change + bytes balance = 2; + // empty bytes indicates no change + bytes code = 3; + // empty sequence indicates no change + repeated ContractSlot slots = 4; + // Whether this is an update, creation or deletion + ChangeType change = 5; +} + +message TransactionContractChanges { + Transaction tx = 1; + repeated ContractChange contract_changes = 2; + repeated ProtocolComponent component_changes = 3; + repeated BalanceChange balance_changes = 4; +} + +message BlockContractChanges { + Block block = 1; + repeated TransactionContractChanges changes = 2; +} diff --git a/substreams/.gitkeep b/substreams/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/substreams/Readme.md b/substreams/Readme.md new file mode 100644 index 0000000..0940442 --- /dev/null +++ b/substreams/Readme.md @@ -0,0 +1,95 @@ +# Subtreams packages + +This directory contains all substream packages that are used by the extractors to access certain data from diffrent +blockchains. + +## Adding a new package + +To add a new package add folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. In this new folder add a manifest +file `substreams.yaml`. You can use the template below to get started: + +```yaml +specVersion: v0.1.0 +package: + name: 'substreams_[CHAIN]_[PROTOCOL_SYSTEM]' + version: v0.1.0 + +protobuf: + files: + - vm.proto + - common.proto + importPaths: + # This is different compared to the substreams example, + # we need to share protobuf definitions with tycho you + # are invited to reuse existing definitions if they are + # useful to you. + - ../../proto/evm/v1 + # any private message types only used in internal modules + # can remain local to the crate. + - ./proto + +binaries: + default: + type: wasm/rust-v1 + # this points to the workspace target directory we use a special + # substreams build profile to optimise wasm binaries + file: ../../target/wasm32-unknown-unknown/substreams/substreams_[CHAIN]_[PROTOCOL_SYSTEM].wasm + +modules: + # sample module provides access to blocks. + - name: map_block + kind: map + inputs: + - source: sf.ethereum.type.v2.Block + output: + type: proto:acme.block_meta.v1.BlockMeta +``` + +Substreams packages are Rust crates so we also need a `cargo.toml`. +The example from the official docs will serve us just well: + +```toml +[package] +name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]" +version = "0.1.0" +edition = "2021" + +[lib] +name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]" +crate-type = ["cdylib"] + +[dependencies] +substreams = "0.5" +substreams-ethereum = "0.9" +prost = "0.11" + +``` + +Now we can generate the Rust protobuf code: + +``` +substreams protogen substreams.yaml --exclude-paths="sf/substreams,google" +``` + +The command above should put the generate rust files under `/src/pb`. You +can start using these now in your module handlers: See +the [official substreams documentation](https://thegraph.com/docs/en/substreams/getting-started/quickstart/#create-substreams-module-handlers) +on +how to implement module handlers. + +You can also look into already existing substreams packages to see how it +is done. E.g. [ethereum-ambient](./ethereum-ambient/) provides a pretty good +example of how to get access to raw contract storage. + +# Tests + +To create a block test asset for ethereum do the following: + +- Follow [this tutorial](https://substreams.streamingfast.io/tutorials/overview/map_block_meta_module). Make sure you + set up the substreams-explorer repo in the same directory as this repo. + - Comment out `image: ./ethereum.png` in `ethereum-explorer/substreams.yaml` + - Add `prost-types = "0.11.0"` to `ethereum-explorer/Cargo.toml` +- Make sure you set up your key env vars. +- Run `sh scripts/download-ethereum-block-to-s3 BLOCK_NUMBER` + +Do not commit the block files (they are quite big). \ No newline at end of file diff --git a/substreams/ethereum-ambient/Cargo.lock b/substreams/ethereum-ambient/Cargo.lock new file mode 100644 index 0000000..31a9d60 --- /dev/null +++ b/substreams/ethereum-ambient/Cargo.lock @@ -0,0 +1,1095 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "ethabi" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +dependencies = [ + "ethereum-types", + "hex", + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "thiserror", + "uint", +] + +[[package]] +name = "ethbloom" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +dependencies = [ + "crunchy", + "fixed-hash", + "impl-rlp", + "impl-serde", + "tiny-keccak", +] + +[[package]] +name = "ethereum-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +dependencies = [ + "ethbloom", + "fixed-hash", + "impl-rlp", + "impl-serde", + "primitive-types", + "uint", +] + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-rlp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +dependencies = [ + "rlp", +] + +[[package]] +name = "impl-serde" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" + +[[package]] +name = "linux-raw-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "pad" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "parity-scale-codec" +version = "3.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dec8a8073036902368c2cdc0387e85ff9a37054d7e7c98e592145e0c92cd4fb" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312270ee71e1cd70289dacf597cab7b207aa107d2f28191c2ae45b2ece18a260" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "rlp" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" +dependencies = [ + "bytes", + "rustc-hex", +] + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustix" +version = "0.38.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "substreams" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af972e374502cdfc9998132f5343848d1c58f27a295dc061a89804371f408a46" +dependencies = [ + "anyhow", + "bigdecimal", + "hex", + "hex-literal 0.3.4", + "num-bigint", + "num-traits", + "pad", + "prost", + "prost-build", + "prost-types", + "substreams-macro", + "thiserror", +] + +[[package]] +name = "substreams-ethereum" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78effc18ed321399fe15ec082806e96a58d213f79741d078c1cd26dd6dd53025" +dependencies = [ + "getrandom", + "num-bigint", + "substreams", + "substreams-ethereum-abigen", + "substreams-ethereum-core", + "substreams-ethereum-derive", +] + +[[package]] +name = "substreams-ethereum-abigen" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a176f39a6e09553c17a287edacd1854e5686fd20ffea3c9655dfc44d94b35e" +dependencies = [ + "anyhow", + "ethabi", + "heck", + "hex", + "prettyplease", + "proc-macro2", + "quote", + "substreams-ethereum-core", + "syn 1.0.109", +] + +[[package]] +name = "substreams-ethereum-ambient" +version = "0.3.0" +dependencies = [ + "hex-literal 0.4.1", + "prost", + "substreams", + "substreams-ethereum", +] + +[[package]] +name = "substreams-ethereum-core" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4700cfe408b75634a3c6b3a0caf7bddba4879601d2085c811485ea54cbde2d" +dependencies = [ + "bigdecimal", + "ethabi", + "getrandom", + "num-bigint", + "prost", + "prost-build", + "prost-types", + "substreams", +] + +[[package]] +name = "substreams-ethereum-derive" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d6d278d926fe3f0775d996ee2b5e1dc822c1b4bf4f7bf07c7fbb5bce6c79a9" +dependencies = [ + "ethabi", + "heck", + "hex", + "num-bigint", + "proc-macro2", + "quote", + "substreams-ethereum-abigen", + "syn 1.0.109", +] + +[[package]] +name = "substreams-macro" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6521ccd011a4c3f52cd3c31fc7400733e4feba2094e0e0e6354adca25b2b3f37" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "winnow" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +dependencies = [ + "memchr", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] diff --git a/substreams/ethereum-ambient/Cargo.toml b/substreams/ethereum-ambient/Cargo.toml new file mode 100644 index 0000000..f7d246b --- /dev/null +++ b/substreams/ethereum-ambient/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substreams-ethereum-ambient" +version = "0.3.0" +edition = "2021" + +[lib] +name = "substreams_ethereum_ambient" +crate-type = ["cdylib"] + +[dependencies] +substreams = "0.5" +substreams-ethereum = "0.9" +prost = "0.11" +hex-literal = "0.4.1" +ethabi = "18.0.0" +hex = "0.4.2" +bytes = "1.5.0" +anyhow = "1.0.75" diff --git a/substreams/ethereum-ambient/src/lib.rs b/substreams/ethereum-ambient/src/lib.rs new file mode 100644 index 0000000..f8f9cb7 --- /dev/null +++ b/substreams/ethereum-ambient/src/lib.rs @@ -0,0 +1,389 @@ +use std::collections::{hash_map::Entry, HashMap}; + +use anyhow::{anyhow, bail}; +use ethabi::{decode, ParamType}; +use hex_literal::hex; +use substreams_ethereum::pb::eth::{self}; + +use pb::tycho::evm::v1::{self as tycho, ChangeType}; + +mod pb; + +const AMBIENT_CONTRACT: [u8; 20] = hex!("aaaaaaaaa24eeeb8d57d431224f73832bc34f688"); +const INIT_POOL_CODE: u8 = 71; +const USER_CMD_FN_SIG: [u8; 4] = [0xA1, 0x51, 0x12, 0xF9]; + +struct SlotValue { + new_value: Vec, + start_value: Vec, +} + +impl SlotValue { + fn has_changed(&self) -> bool { + self.start_value != self.new_value + } +} + +// uses a map for slots, protobuf does not +// allow bytes in hashmap keys +struct InterimContractChange { + address: Vec, + balance: Vec, + code: Vec, + slots: HashMap, SlotValue>, + change: tycho::ChangeType, +} + +impl From for tycho::ContractChange { + fn from(value: InterimContractChange) -> Self { + tycho::ContractChange { + address: value.address, + balance: value.balance, + code: value.code, + slots: value + .slots + .into_iter() + .filter(|(_, value)| value.has_changed()) + .map(|(slot, value)| tycho::ContractSlot { slot, value: value.new_value }) + .collect(), + change: value.change.into(), + } + } +} + +/// Extracts all contract changes relevant to vm simulations +/// +/// This implementation has currently two major limitations: +/// 1. It is hardwired to only care about changes to the ambient main contract, this is ok for this +/// particular use case but for a more general purpose implementation this is not ideal +/// 2. Changes are processed separately, this means that if there are any side effects between each +/// other (e.g. if account is deleted and then created again in ethereum all the storage is set +/// to 0. So there is a side effect between account creation and contract storage.) these might +/// not be properly accounted for. Most of the time this should not be a major issue but may lead +/// to wrong results so consume this implementation with care. See example below for a concrete +/// case where this is problematic. +/// +/// ## A very contrived example: +/// 1. Some existing contract receives a transaction that changes it state, the state is updated +/// 2. Next, this contract has self destruct called on itself +/// 3. The contract is created again using CREATE2 at the same address +/// 4. The contract receives a transaction that changes it state +/// 5. We would emit this as as contract creation with slots set from 1 and from 4, although we +/// should only emit the slots changed from 4. +#[substreams::handlers::map] +fn map_changes( + block: eth::v2::Block, +) -> Result { + let mut block_changes = tycho::BlockContractChanges { block: None, changes: Vec::new() }; + + let mut tx_change = tycho::TransactionContractChanges::default(); + + let mut changed_contracts: HashMap, InterimContractChange> = HashMap::new(); + + let created_accounts: HashMap<_, _> = block + .transactions() + .flat_map(|tx| { + tx.calls.iter().flat_map(|call| { + call.account_creations + .iter() + .map(|ac| (&ac.account, ac.ordinal)) + }) + }) + .collect(); + + for block_tx in block.transactions() { + // extract storage changes + let mut storage_changes = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .flat_map(|call| { + call.storage_changes + .iter() + .filter(|c| c.address == AMBIENT_CONTRACT) + }) + .collect::>(); + storage_changes.sort_unstable_by_key(|change| change.ordinal); + + let ambient_calls = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .filter(|call| call.address == AMBIENT_CONTRACT) + .collect::>(); + + for call in ambient_calls { + if call.input.len() < 4 { + continue; + } + if call.input[0..4] == USER_CMD_FN_SIG { + let user_cmd_external_abi_types = &[ + // index of the proxy sidecar the command is being called on + ParamType::Uint(16), + // call data for internal UserCmd method + ParamType::Bytes, + ]; + let user_cmd_internal_abi_types = &[ + ParamType::Uint(8), // command + ParamType::Address, // base + ParamType::Address, // quote + ParamType::Uint(256), // pool index + ParamType::Uint(128), // price + ]; + + // Decode external call to UserCmd + if let Ok(external_params) = decode(user_cmd_external_abi_types, &call.input[4..]) { + let cmd_bytes = external_params[1] + .to_owned() + .into_bytes() + .ok_or_else(|| { + anyhow!("Failed to convert to bytes: {:?}", &external_params[1]) + })?; + + // Call data is structured differently depending on the cmd code, so only + // decode if this is an init pool code. + if cmd_bytes[31] == INIT_POOL_CODE { + // Decode internal call to UserCmd + if let Ok(internal_params) = decode(user_cmd_internal_abi_types, &cmd_bytes) + { + let base = internal_params[1] + .to_owned() + .into_address() + .ok_or_else(|| { + anyhow!( + "Failed to convert to address: {:?}", + &internal_params[1] + ) + })? + .to_fixed_bytes() + .to_vec(); + + let quote = internal_params[2] + .to_owned() + .into_address() + .ok_or_else(|| { + anyhow!( + "Failed to convert to address: {:?}", + &internal_params[2] + ) + })? + .to_fixed_bytes() + .to_vec(); + + let pool_index = internal_params[3] + .to_owned() + .into_uint() + .ok_or_else(|| anyhow!("Failed to convert to u32".to_string()))? + .as_u32(); + + let static_attribute = tycho::Attribute { + name: String::from("pool_index"), + value: pool_index.to_be_bytes().to_vec(), + change: ChangeType::Creation.into(), + }; + + let mut tokens: Vec> = vec![base.clone(), quote.clone()]; + tokens.sort(); + + let new_component = tycho::ProtocolComponent { + id: format!( + "{}{}{}", + hex::encode(base.clone()), + hex::encode(quote.clone()), + pool_index + ), + tokens, + contracts: vec![hex::encode(AMBIENT_CONTRACT)], + static_att: vec![static_attribute], + change: ChangeType::Creation.into(), + }; + tx_change + .component_changes + .push(new_component); + } else { + bail!("Failed to decode ABI internal call.".to_string()); + } + } + } else { + bail!("Failed to decode ABI external call.".to_string()); + } + } + } + + // Note: some contracts change slot values and change them back to their + // original value before the transactions ends we remember the initial + // value before the first change and in the end filter found deltas + // that ended up not actually changing anything. + for storage_change in storage_changes.iter() { + match changed_contracts.entry(storage_change.address.clone()) { + // We have already an entry recording a change about this contract + // only append the change about this storage slot + Entry::Occupied(mut e) => { + let contract_change = e.get_mut(); + match contract_change + .slots + .entry(storage_change.key.clone()) + { + // The storage slot was already changed before, simply + // update new_value + Entry::Occupied(mut v) => { + let slot_value = v.get_mut(); + slot_value + .new_value + .copy_from_slice(&storage_change.new_value); + } + // The storage slots is being initialised for the first time + Entry::Vacant(v) => { + v.insert(SlotValue { + new_value: storage_change.new_value.clone(), + start_value: storage_change.old_value.clone(), + }); + } + } + } + // Intialise a new contract change after obsering a storage change + Entry::Vacant(e) => { + let mut slots = HashMap::new(); + slots.insert( + storage_change.key.clone(), + SlotValue { + new_value: storage_change.new_value.clone(), + start_value: storage_change.old_value.clone(), + }, + ); + e.insert(InterimContractChange { + address: storage_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots, + change: if created_accounts.contains_key(&storage_change.address) { + ChangeType::Creation + } else { + ChangeType::Update + }, + }); + } + } + } + + // extract balance changes + let mut balance_changes = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .flat_map(|call| { + call.balance_changes + .iter() + .filter(|c| c.address == AMBIENT_CONTRACT) + }) + .collect::>(); + balance_changes.sort_unstable_by_key(|change| change.ordinal); + + for balance_change in balance_changes.iter() { + match changed_contracts.entry(balance_change.address.clone()) { + Entry::Occupied(mut e) => { + let contract_change = e.get_mut(); + if let Some(new_balance) = &balance_change.new_value { + contract_change.balance.clear(); + contract_change + .balance + .extend_from_slice(&new_balance.bytes); + } + } + Entry::Vacant(e) => { + if let Some(new_balance) = &balance_change.new_value { + e.insert(InterimContractChange { + address: balance_change.address.clone(), + balance: new_balance.bytes.clone(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&balance_change.address) { + ChangeType::Creation + } else { + ChangeType::Update + }, + }); + } + } + } + } + + // extract code changes + let mut code_changes = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .flat_map(|call| { + call.code_changes + .iter() + .filter(|c| c.address == AMBIENT_CONTRACT) + }) + .collect::>(); + code_changes.sort_unstable_by_key(|change| change.ordinal); + + for code_change in code_changes.iter() { + match changed_contracts.entry(code_change.address.clone()) { + Entry::Occupied(mut e) => { + let contract_change = e.get_mut(); + contract_change.code.clear(); + contract_change + .code + .extend_from_slice(&code_change.new_code); + } + Entry::Vacant(e) => { + e.insert(InterimContractChange { + address: code_change.address.clone(), + balance: Vec::new(), + code: code_change.new_code.clone(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&code_change.address) { + ChangeType::Creation + } else { + ChangeType::Update + }, + }); + } + } + } + + // if there were any changes, add transaction and push the changes + if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() { + tx_change.tx = Some(tycho::Transaction { + hash: block_tx.hash.clone(), + from: block_tx.from.clone(), + to: block_tx.to.clone(), + index: block_tx.index as u64, + }); + + // reuse changed_contracts hash map by draining it, next iteration + // will start empty. This avoids a costly reallocation + for (_, change) in changed_contracts.drain() { + tx_change + .contract_changes + .push(change.into()) + } + + block_changes + .changes + .push(tx_change.clone()); + + // clear out the interim contract changes after we pushed those. + tx_change.tx = None; + tx_change.contract_changes.clear(); + } + } + + block_changes.block = Some(tycho::Block { + number: block.number, + hash: block.hash.clone(), + parent_hash: block + .header + .as_ref() + .expect("Block header not present") + .parent_hash + .clone(), + ts: block.timestamp_seconds(), + }); + + Ok(block_changes) +} diff --git a/substreams/ethereum-ambient/src/pb/mod.rs b/substreams/ethereum-ambient/src/pb/mod.rs new file mode 100644 index 0000000..43d8838 --- /dev/null +++ b/substreams/ethereum-ambient/src/pb/mod.rs @@ -0,0 +1,10 @@ +// @generated +pub mod tycho { + pub mod evm { + // @@protoc_insertion_point(attribute:tycho.evm.v1) + pub mod v1 { + include!("tycho.evm.v1.rs"); + // @@protoc_insertion_point(tycho.evm.v1) + } + } +} diff --git a/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs b/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs new file mode 100644 index 0000000..f59fcf2 --- /dev/null +++ b/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs @@ -0,0 +1,166 @@ +// @generated +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub parent_hash: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="3")] + pub number: u64, + #[prost(uint64, tag="4")] + pub ts: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub from: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub to: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="4")] + pub index: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Attribute { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub value: ::prost::alloc::vec::Vec, + #[prost(enumeration="ChangeType", tag="3")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtocolComponent { + #[prost(string, tag="1")] + pub id: ::prost::alloc::string::String, + #[prost(bytes="vec", repeated, tag="2")] + pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + #[prost(string, repeated, tag="3")] + pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag="4")] + pub static_att: ::prost::alloc::vec::Vec, + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceChange { + #[prost(bytes="vec", tag="1")] + pub token: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub component_id: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ChangeType { + Unspecified = 0, + Update = 1, + Creation = 2, + Deletion = 3, +} +impl ChangeType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED", + ChangeType::Update => "CHANGE_TYPE_UPDATE", + ChangeType::Creation => "CHANGE_TYPE_CREATION", + ChangeType::Deletion => "CHANGE_TYPE_DELETION", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified), + "CHANGE_TYPE_UPDATE" => Some(Self::Update), + "CHANGE_TYPE_CREATION" => Some(Self::Creation), + "CHANGE_TYPE_DELETION" => Some(Self::Deletion), + _ => None, + } + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EntityChanges { + #[prost(string, tag="1")] + pub component_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag="2")] + pub attributes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionEntityChanges { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub entity_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockEntityChanges { + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractSlot { + #[prost(bytes="vec", tag="2")] + pub slot: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub value: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractChange { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + /// empty bytes indicates no change + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + /// empty bytes indicates no change + #[prost(bytes="vec", tag="3")] + pub code: ::prost::alloc::vec::Vec, + /// empty sequence indicates no change + #[prost(message, repeated, tag="4")] + pub slots: ::prost::alloc::vec::Vec, + /// Whether this is an update, creation or deletion + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionContractChanges { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub contract_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockContractChanges { + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-ambient/substreams.yaml b/substreams/ethereum-ambient/substreams.yaml new file mode 100644 index 0000000..8007c2d --- /dev/null +++ b/substreams/ethereum-ambient/substreams.yaml @@ -0,0 +1,24 @@ +specVersion: v0.1.0 +package: + name: "substreams_ethereum_ambient" + version: v0.3.0 + +protobuf: + files: + - vm.proto + - common.proto + importPaths: + - ../../proto/tycho/evm/v1/ + +binaries: + default: + type: wasm/rust-v1 + file: ../../target/wasm32-unknown-unknown/substreams/substreams_ethereum_ambient.wasm + +modules: + - name: map_changes + kind: map + inputs: + - source: sf.ethereum.type.v2.Block + output: + type: proto:tycho.evm.state.v1.BlockContractChanges From 5f87b1d0af3a200a38ab7a824d70a3502481182c Mon Sep 17 00:00:00 2001 From: pistomat Date: Wed, 20 Dec 2023 14:44:06 +0100 Subject: [PATCH 2/6] Add a simple top level readme --- README.md | 5 +++++ docs/indexing/substreams-integration/tutorial-ambient.md | 2 ++ docs/indexing/substreams-integration/tutorial-uniswapv2.md | 2 -- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 README.md create mode 100644 docs/indexing/substreams-integration/tutorial-ambient.md delete mode 100644 docs/indexing/substreams-integration/tutorial-uniswapv2.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e88d75b --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Propeller Protocol Lib + +Protocol lib is a library used by Propellerheads.xyz solvers to integrate decentralized protocols. Currently, only swap/exchange protocols are supported. + +Please refer to the [README.md](docs/README.md) in the [docs](docs) folder for more information. diff --git a/docs/indexing/substreams-integration/tutorial-ambient.md b/docs/indexing/substreams-integration/tutorial-ambient.md new file mode 100644 index 0000000..5364956 --- /dev/null +++ b/docs/indexing/substreams-integration/tutorial-ambient.md @@ -0,0 +1,2 @@ +# Tutorial: Ambient + diff --git a/docs/indexing/substreams-integration/tutorial-uniswapv2.md b/docs/indexing/substreams-integration/tutorial-uniswapv2.md deleted file mode 100644 index b0af89f..0000000 --- a/docs/indexing/substreams-integration/tutorial-uniswapv2.md +++ /dev/null @@ -1,2 +0,0 @@ -# Tutorial: UniswapV2 - From 0f3a5b819c3d944598d3d3b0904aa14c7ae6d3c4 Mon Sep 17 00:00:00 2001 From: pistomat Date: Wed, 20 Dec 2023 18:12:41 +0100 Subject: [PATCH 3/6] Add substreams integration template, readmes --- README.md | 2 +- docs/README.md | 6 +- docs/SUMMARY.md | 2 +- .../indexing/substreams-integration/README.md | 69 +- docs/logic/vm-integration/README.md | 4 +- substreams/Readme.md | 14 +- substreams/ethereum-ambient/Cargo.lock | 106 +- substreams/ethereum-ambient/src/lib.rs | 38 +- substreams/ethereum-template/Cargo.lock | 1135 +++++++++++++++++ substreams/ethereum-template/Cargo.toml | 13 + substreams/ethereum-template/src/lib.rs | 13 + substreams/ethereum-template/src/pb/mod.rs | 10 + .../ethereum-template/src/pb/tycho.evm.v1.rs | 166 +++ substreams/ethereum-template/substreams.yaml | 24 + 14 files changed, 1549 insertions(+), 53 deletions(-) create mode 100644 substreams/ethereum-template/Cargo.lock create mode 100644 substreams/ethereum-template/Cargo.toml create mode 100644 substreams/ethereum-template/src/lib.rs create mode 100644 substreams/ethereum-template/src/pb/mod.rs create mode 100644 substreams/ethereum-template/src/pb/tycho.evm.v1.rs create mode 100644 substreams/ethereum-template/substreams.yaml diff --git a/README.md b/README.md index e88d75b..ad0fba5 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,4 @@ Protocol lib is a library used by Propellerheads.xyz solvers to integrate decentralized protocols. Currently, only swap/exchange protocols are supported. -Please refer to the [README.md](docs/README.md) in the [docs](docs) folder for more information. +Please refer to the [README.md](docs/README.md) for more information. diff --git a/docs/README.md b/docs/README.md index fa597fa..549e27e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,7 +22,7 @@ While VM integration is certainly the quickest and probably most accessible one ### Indexing -For indexing purposes, it is required that you provide a [substreams](https://thegraph.com/docs/en/substreams/) package that emits a specified set of messages. If your protocol already has a [substreams](https://thegraph.com/docs/en/substreams/) package for indexing implemented, you can adjust it to emit the required messages. - -_Specifications coming soon._ +For indexing purposes, it is required that you provide a [substreams](https://substreams.streamingfast.io/) package that emits a specified set of messages. If your protocol already has a [substreams package](https://github.com/messari/substreams) package for indexing implemented, you can adjust it to emit the required messages. +**VM Integration** Currently the only supported integration is for EVM protocols in order to complement the Solidity protocol logic. **[Read more here.](indexing/vm-integration/README.md)** +**Custom Entity Integration** Coming soon, this integration will complement the upcoming native Rust protocol logic. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 34da702..1e70a1c 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -11,4 +11,4 @@ ## Indexing * [Substreams Integration](indexing/substreams-integration/README.md) - * [Tutorial: UniswapV2](indexing/substreams-integration/tutorial-uniswapv2.md) + * [Tutorial: Ambient](indexing/substreams-integration/tutorial-ambient.md) diff --git a/docs/indexing/substreams-integration/README.md b/docs/indexing/substreams-integration/README.md index 056fff1..273bf21 100644 --- a/docs/indexing/substreams-integration/README.md +++ b/docs/indexing/substreams-integration/README.md @@ -1,3 +1,70 @@ # Substreams Integration -Coming Soon +## Example + +We have integrated the **Ambient** protocol as a reference, see `/substreams/ethereum-ambient` for more information. + +## Step by step + +1. Install [Rust](https://www.rust-lang.org/tools/install), you can do so with the following command: + + ```bash + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + ``` + +1. Install [Substreams CLI](https://substreams.streamingfast.io/getting-started/installing-the-cli), you can either use brew: + + ```bash + brew install streamingfast/tap/substreams + ``` + use precompiled binaries + ```bash + # Use correct binary for your platform + LINK=$(curl -s https://api.github.com/repos/streamingfast/substreams/releases/latest | awk '/download.url.*linux/ {print $2}' | sed 's/"//g') + curl -L $LINK | tar zxf - + ``` + or compile from source: + ```bash + git clone https://github.com/streamingfast/substreams + cd substreams + go install -v ./cmd/substreams + ``` + +1. Start by making a local copy of the Propeller Protocol Lib repository: + ```bash + git clone https://github.com/propeller-heads/propeller-protocol-lib + ``` + +## Understanding the Substreams integration + +Substreams is a new indexing technology, which uses Rust modules to compose raw blockchain data streams into higher level data streams, in out case specific to the protocol. These modules together with the protobuf definitions and manifest are then wrapped into SPKG packages (more info [here](https://substreams.streamingfast.io/quick-access/glossary#spkg-.spkg)) that is then run remotely on the Substreams server. + +For more information, read the [quick explanation of Substreams](https://thegraph.com/docs/en/substreams/) or jump into the [Substreams documentation](https://substreams.streamingfast.io/). It describes the functions that need to be implemented as well as the manifest file. + +### ProtoBuf files + +Generally these describe the raw blockchain data that we get on the input stream and the output data that we want to produce using the Rust module. + +If you are unfamiliar with ProtoBuf at all, you can start with the [official documentation](https://protobuf.dev/overview/). + +First get familiar with the raw ProtoBuf definitions provided by us: +- [common.proto](../../../proto/tycho/evm/v1/common.proto) - Common types used by all integration types +- [vm.proto](../../../proto/tycho/evm/v1/vm.proto) - Types specific to the VM integration + +You can also create your own intermediate ProtoBufs. These files should reside in your own substreams package, e.g. `./substreams/ethereum-template/proto/custom-messages.proto`. You have to link these files in the `substreams.yaml` file, see the [manifest docs](https://substreams.streamingfast.io/developers-guide/creating-your-manifest) for more information or you can look at the official substreams example integration of [UniswapV2](https://github.com/messari/substreams/blob/master/uniswap-v2/substreams.yaml#L20-L22). + +*Note: Internally we are referring to the substreams integration as `Tycho`, which is why our protobuf files are under the `proto/tycho` directory.* + +### Rust module + +The goal of the rust module is to implement the logic that will transform the raw blockchain data into the desired output data. + +*This is the actual integration code that you will be writing!* + +The module is a Rust library that is compiled into a SPKG (`.spkg`) file using the Substreams CLI and then loaded by the Substreams server. It is defined by the `lib.rs` file (see the [Ambient reference example](../../../substreams/ethereum-ambient/src/lib.rs)). + +Read our [Substreams README.md](../../../substreams/README.md) for more information on how to write the Rust module. + +### Testing + +Read the [Substreams testing docs](../../../substreams/README.md#testing-your-implementation) for more information on how to test your integration. diff --git a/docs/logic/vm-integration/README.md b/docs/logic/vm-integration/README.md index bcc5e58..90a19d9 100644 --- a/docs/logic/vm-integration/README.md +++ b/docs/logic/vm-integration/README.md @@ -24,12 +24,12 @@ Following exchanges have been integrated using VM approach: foundryup ``` -2. Start by making a local copy of the Propeller Protocol Lib repository: +1. Start by making a local copy of the Propeller Protocol Lib repository: ```bash git clone https://github.com/propeller-heads/propeller-protocol-lib ``` -3. Install forge dependencies: +1. Install forge dependencies: ```bash cd ./propeller-protocol-lib/evm/ forge install diff --git a/substreams/Readme.md b/substreams/Readme.md index 0940442..459546b 100644 --- a/substreams/Readme.md +++ b/substreams/Readme.md @@ -5,8 +5,10 @@ blockchains. ## Adding a new package -To add a new package add folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. In this new folder add a manifest -file `substreams.yaml`. You can use the template below to get started: +To add a new package add folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. + +### Manifest +In this new folder add a manifest file `substreams.yaml`. You can use the template below to get started: ```yaml specVersion: v0.1.0 @@ -36,13 +38,12 @@ binaries: file: ../../target/wasm32-unknown-unknown/substreams/substreams_[CHAIN]_[PROTOCOL_SYSTEM].wasm modules: - # sample module provides access to blocks. - - name: map_block + - name: map_changes kind: map inputs: - source: sf.ethereum.type.v2.Block output: - type: proto:acme.block_meta.v1.BlockMeta + type: proto:tycho.evm.state.v1.BlockContractChanges ``` Substreams packages are Rust crates so we also need a `cargo.toml`. @@ -65,6 +66,9 @@ prost = "0.11" ``` +There are already some generated rust files in the `src/pb` directory. These are generated from the protobuf files in the + + Now we can generate the Rust protobuf code: ``` diff --git a/substreams/ethereum-ambient/Cargo.lock b/substreams/ethereum-ambient/Cargo.lock index 31a9d60..03629cf 100644 --- a/substreams/ethereum-ambient/Cargo.lock +++ b/substreams/ethereum-ambient/Cargo.lock @@ -180,7 +180,24 @@ version = "17.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" dependencies = [ - "ethereum-types", + "ethereum-types 0.13.1", + "hex", + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "thiserror", + "uint", +] + +[[package]] +name = "ethabi" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7413c5f74cc903ea37386a8965a936cbeb334bd270862fdece542c1b2dcbc898" +dependencies = [ + "ethereum-types 0.14.1", "hex", "once_cell", "regex", @@ -198,9 +215,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" dependencies = [ "crunchy", - "fixed-hash", + "fixed-hash 0.7.0", "impl-rlp", - "impl-serde", + "impl-serde 0.3.2", + "tiny-keccak", +] + +[[package]] +name = "ethbloom" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c22d4b5885b6aa2fe5e8b9329fb8d232bf739e434e6b87347c63bdd00c120f60" +dependencies = [ + "crunchy", + "fixed-hash 0.8.0", + "impl-rlp", + "impl-serde 0.4.0", "tiny-keccak", ] @@ -210,11 +240,25 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" dependencies = [ - "ethbloom", - "fixed-hash", + "ethbloom 0.12.1", + "fixed-hash 0.7.0", "impl-rlp", - "impl-serde", - "primitive-types", + "impl-serde 0.3.2", + "primitive-types 0.11.1", + "uint", +] + +[[package]] +name = "ethereum-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d215cbf040552efcbe99a38372fe80ab9d00268e20012b79fcd0f073edd8ee" +dependencies = [ + "ethbloom 0.13.0", + "fixed-hash 0.8.0", + "impl-rlp", + "impl-serde 0.4.0", + "primitive-types 0.12.2", "uint", ] @@ -236,6 +280,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixed-hash" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" +dependencies = [ + "byteorder", + "rand", + "rustc-hex", + "static_assertions", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -335,6 +391,15 @@ dependencies = [ "serde", ] +[[package]] +name = "impl-serde" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd" +dependencies = [ + "serde", +] + [[package]] name = "impl-trait-for-tuples" version = "0.2.2" @@ -519,10 +584,23 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" dependencies = [ - "fixed-hash", + "fixed-hash 0.7.0", "impl-codec", "impl-rlp", - "impl-serde", + "impl-serde 0.3.2", + "uint", +] + +[[package]] +name = "primitive-types" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" +dependencies = [ + "fixed-hash 0.8.0", + "impl-codec", + "impl-rlp", + "impl-serde 0.4.0", "uint", ] @@ -805,7 +883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a176f39a6e09553c17a287edacd1854e5686fd20ffea3c9655dfc44d94b35e" dependencies = [ "anyhow", - "ethabi", + "ethabi 17.2.0", "heck", "hex", "prettyplease", @@ -819,6 +897,10 @@ dependencies = [ name = "substreams-ethereum-ambient" version = "0.3.0" dependencies = [ + "anyhow", + "bytes", + "ethabi 18.0.0", + "hex", "hex-literal 0.4.1", "prost", "substreams", @@ -832,7 +914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db4700cfe408b75634a3c6b3a0caf7bddba4879601d2085c811485ea54cbde2d" dependencies = [ "bigdecimal", - "ethabi", + "ethabi 17.2.0", "getrandom", "num-bigint", "prost", @@ -847,7 +929,7 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40d6d278d926fe3f0775d996ee2b5e1dc822c1b4bf4f7bf07c7fbb5bce6c79a9" dependencies = [ - "ethabi", + "ethabi 17.2.0", "heck", "hex", "num-bigint", diff --git a/substreams/ethereum-ambient/src/lib.rs b/substreams/ethereum-ambient/src/lib.rs index f8f9cb7..ae169db 100644 --- a/substreams/ethereum-ambient/src/lib.rs +++ b/substreams/ethereum-ambient/src/lib.rs @@ -3,9 +3,9 @@ use std::collections::{hash_map::Entry, HashMap}; use anyhow::{anyhow, bail}; use ethabi::{decode, ParamType}; use hex_literal::hex; -use substreams_ethereum::pb::eth::{self}; +use substreams_ethereum::pb::eth; -use pb::tycho::evm::v1::{self as tycho, ChangeType}; +use pb::tycho::evm::v1::{self as tycho}; mod pb; @@ -52,24 +52,6 @@ impl From for tycho::ContractChange { } /// Extracts all contract changes relevant to vm simulations -/// -/// This implementation has currently two major limitations: -/// 1. It is hardwired to only care about changes to the ambient main contract, this is ok for this -/// particular use case but for a more general purpose implementation this is not ideal -/// 2. Changes are processed separately, this means that if there are any side effects between each -/// other (e.g. if account is deleted and then created again in ethereum all the storage is set -/// to 0. So there is a side effect between account creation and contract storage.) these might -/// not be properly accounted for. Most of the time this should not be a major issue but may lead -/// to wrong results so consume this implementation with care. See example below for a concrete -/// case where this is problematic. -/// -/// ## A very contrived example: -/// 1. Some existing contract receives a transaction that changes it state, the state is updated -/// 2. Next, this contract has self destruct called on itself -/// 3. The contract is created again using CREATE2 at the same address -/// 4. The contract receives a transaction that changes it state -/// 5. We would emit this as as contract creation with slots set from 1 and from 4, although we -/// should only emit the slots changed from 4. #[substreams::handlers::map] fn map_changes( block: eth::v2::Block, @@ -179,7 +161,7 @@ fn map_changes( let static_attribute = tycho::Attribute { name: String::from("pool_index"), value: pool_index.to_be_bytes().to_vec(), - change: ChangeType::Creation.into(), + change: tycho::ChangeType::Creation.into(), }; let mut tokens: Vec> = vec![base.clone(), quote.clone()]; @@ -195,7 +177,7 @@ fn map_changes( tokens, contracts: vec![hex::encode(AMBIENT_CONTRACT)], static_att: vec![static_attribute], - change: ChangeType::Creation.into(), + change: tycho::ChangeType::Creation.into(), }; tx_change .component_changes @@ -257,9 +239,9 @@ fn map_changes( code: Vec::new(), slots, change: if created_accounts.contains_key(&storage_change.address) { - ChangeType::Creation + tycho::ChangeType::Creation } else { - ChangeType::Update + tycho::ChangeType::Update }, }); } @@ -298,9 +280,9 @@ fn map_changes( code: Vec::new(), slots: HashMap::new(), change: if created_accounts.contains_key(&balance_change.address) { - ChangeType::Creation + tycho::ChangeType::Creation } else { - ChangeType::Update + tycho::ChangeType::Update }, }); } @@ -337,9 +319,9 @@ fn map_changes( code: code_change.new_code.clone(), slots: HashMap::new(), change: if created_accounts.contains_key(&code_change.address) { - ChangeType::Creation + tycho::ChangeType::Creation } else { - ChangeType::Update + tycho::ChangeType::Update }, }); } diff --git a/substreams/ethereum-template/Cargo.lock b/substreams/ethereum-template/Cargo.lock new file mode 100644 index 0000000..ada8eab --- /dev/null +++ b/substreams/ethereum-template/Cargo.lock @@ -0,0 +1,1135 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "ethabi" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +dependencies = [ + "ethereum-types", + "hex", + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "thiserror", + "uint", +] + +[[package]] +name = "ethbloom" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +dependencies = [ + "crunchy", + "fixed-hash", + "impl-rlp", + "impl-serde", + "tiny-keccak", +] + +[[package]] +name = "ethereum-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +dependencies = [ + "ethbloom", + "fixed-hash", + "impl-rlp", + "impl-serde", + "primitive-types", + "uint", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-rlp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +dependencies = [ + "rlp", +] + +[[package]] +name = "impl-serde" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" + +[[package]] +name = "linux-raw-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pad" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "parity-scale-codec" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881331e34fa842a2fb61cc2db9643a8fedc615e47cfcc52597d1af0db9a7e8fe" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be30eaf4b0a9fba5336683b38de57bb86d179a35862ba6bfcf57625d006bde5b" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97dc5fea232fc28d2f597b37c4876b348a40e33f3b02cc975c8d006d78d94b1a" +dependencies = [ + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "rlp" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" +dependencies = [ + "bytes", + "rustc-hex", +] + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustix" +version = "0.38.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "serde" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.193" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "substreams" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3524a4e2931ff6cd58783e62adbd7e44f461752eca0c423793cfb462351f24" +dependencies = [ + "anyhow", + "bigdecimal", + "hex", + "hex-literal", + "num-bigint", + "num-integer", + "num-traits", + "pad", + "prost", + "prost-build", + "prost-types", + "substreams-macro", + "thiserror", +] + +[[package]] +name = "substreams-ethereum" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f45dc04be50b7ca08d6d5c4560ee3eeba16ccaa1c124d0361bb30b5b84e28b" +dependencies = [ + "getrandom", + "num-bigint", + "substreams", + "substreams-ethereum-abigen", + "substreams-ethereum-core", + "substreams-ethereum-derive", +] + +[[package]] +name = "substreams-ethereum-abigen" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c04307913a355aaf2a1bb7186d4bc7e36875f3d4aff77b47e83f1b63b24da55" +dependencies = [ + "anyhow", + "ethabi", + "heck", + "hex", + "prettyplease", + "proc-macro2", + "quote", + "substreams-ethereum-core", + "syn 1.0.109", +] + +[[package]] +name = "substreams-ethereum-core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9048cc9a66873ab7069ef958c2684994e6ee323da49c186b19156fdb4ca131" +dependencies = [ + "bigdecimal", + "ethabi", + "getrandom", + "num-bigint", + "prost", + "prost-build", + "prost-types", + "substreams", +] + +[[package]] +name = "substreams-ethereum-derive" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e862928bee8653f5c9291ac619c8dc0da14ca61d8cd8d89b3acdbbde4d0bf304" +dependencies = [ + "ethabi", + "heck", + "hex", + "num-bigint", + "proc-macro2", + "quote", + "substreams-ethereum-abigen", + "syn 1.0.109", +] + +[[package]] +name = "substreams-ethereum-template" +version = "0.1.0" +dependencies = [ + "prost", + "substreams", + "substreams-ethereum", +] + +[[package]] +name = "substreams-macro" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63c2b15adf5b4d7a6d1a73c73df951a6b2df6fbb4f0b41304dc28c5550ce0ed0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "thiserror" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + +[[package]] +name = "toml_edit" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + +[[package]] +name = "winnow" +version = "0.5.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" +dependencies = [ + "memchr", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] diff --git a/substreams/ethereum-template/Cargo.toml b/substreams/ethereum-template/Cargo.toml new file mode 100644 index 0000000..82ae8f3 --- /dev/null +++ b/substreams/ethereum-template/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "substreams-ethereum-template" +version = "0.1.0" +edition = "2021" + +[lib] +name = "substreams_ethereum_template" +crate-type = ["cdylib"] + +[dependencies] +substreams = "0.5" +substreams-ethereum = "0.9" +prost = "0.11" diff --git a/substreams/ethereum-template/src/lib.rs b/substreams/ethereum-template/src/lib.rs new file mode 100644 index 0000000..6e10475 --- /dev/null +++ b/substreams/ethereum-template/src/lib.rs @@ -0,0 +1,13 @@ + +use substreams_ethereum::pb::eth; + +use pb::tycho::evm::v1::{self as tycho}; + +mod pb; + +#[substreams::handlers::map] +fn map_changes( + block: eth::v2::Block, +) -> Result { + todo!("Not implemented") +} \ No newline at end of file diff --git a/substreams/ethereum-template/src/pb/mod.rs b/substreams/ethereum-template/src/pb/mod.rs new file mode 100644 index 0000000..43d8838 --- /dev/null +++ b/substreams/ethereum-template/src/pb/mod.rs @@ -0,0 +1,10 @@ +// @generated +pub mod tycho { + pub mod evm { + // @@protoc_insertion_point(attribute:tycho.evm.v1) + pub mod v1 { + include!("tycho.evm.v1.rs"); + // @@protoc_insertion_point(tycho.evm.v1) + } + } +} diff --git a/substreams/ethereum-template/src/pb/tycho.evm.v1.rs b/substreams/ethereum-template/src/pb/tycho.evm.v1.rs new file mode 100644 index 0000000..f59fcf2 --- /dev/null +++ b/substreams/ethereum-template/src/pb/tycho.evm.v1.rs @@ -0,0 +1,166 @@ +// @generated +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub parent_hash: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="3")] + pub number: u64, + #[prost(uint64, tag="4")] + pub ts: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub from: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub to: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="4")] + pub index: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Attribute { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub value: ::prost::alloc::vec::Vec, + #[prost(enumeration="ChangeType", tag="3")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtocolComponent { + #[prost(string, tag="1")] + pub id: ::prost::alloc::string::String, + #[prost(bytes="vec", repeated, tag="2")] + pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + #[prost(string, repeated, tag="3")] + pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag="4")] + pub static_att: ::prost::alloc::vec::Vec, + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceChange { + #[prost(bytes="vec", tag="1")] + pub token: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub component_id: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ChangeType { + Unspecified = 0, + Update = 1, + Creation = 2, + Deletion = 3, +} +impl ChangeType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED", + ChangeType::Update => "CHANGE_TYPE_UPDATE", + ChangeType::Creation => "CHANGE_TYPE_CREATION", + ChangeType::Deletion => "CHANGE_TYPE_DELETION", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified), + "CHANGE_TYPE_UPDATE" => Some(Self::Update), + "CHANGE_TYPE_CREATION" => Some(Self::Creation), + "CHANGE_TYPE_DELETION" => Some(Self::Deletion), + _ => None, + } + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EntityChanges { + #[prost(string, tag="1")] + pub component_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag="2")] + pub attributes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionEntityChanges { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub entity_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockEntityChanges { + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractSlot { + #[prost(bytes="vec", tag="2")] + pub slot: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub value: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractChange { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + /// empty bytes indicates no change + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + /// empty bytes indicates no change + #[prost(bytes="vec", tag="3")] + pub code: ::prost::alloc::vec::Vec, + /// empty sequence indicates no change + #[prost(message, repeated, tag="4")] + pub slots: ::prost::alloc::vec::Vec, + /// Whether this is an update, creation or deletion + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionContractChanges { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub contract_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockContractChanges { + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-template/substreams.yaml b/substreams/ethereum-template/substreams.yaml new file mode 100644 index 0000000..b3fa3fb --- /dev/null +++ b/substreams/ethereum-template/substreams.yaml @@ -0,0 +1,24 @@ +specVersion: v0.1.0 +package: + name: "substreams_ethereum_template" + version: v0.1.0 + +protobuf: + files: + - vm.proto + - common.proto + importPaths: + - ../../proto/tycho/evm/v1/ + +binaries: + default: + type: wasm/rust-v1 + file: ../../target/wasm32-unknown-unknown/substreams/substreams_ethereum_template.wasm + +modules: + - name: map_changes + kind: map + inputs: + - source: sf.ethereum.type.v2.Block + output: + type: proto:tycho.evm.state.v1.BlockContractChanges From 761698769a6eb81b15018664ea551bfc116f2737 Mon Sep 17 00:00:00 2001 From: pistomat Date: Thu, 21 Dec 2023 15:47:49 +0100 Subject: [PATCH 4/6] Upstream changes for proto --- proto/tycho/evm/v1/common.proto | 32 +++++++++++++++++++++++++++++++- proto/tycho/evm/v1/entity.proto | 10 ++++++++++ proto/tycho/evm/v1/vm.proto | 23 +++++++++++++++++++---- 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/proto/tycho/evm/v1/common.proto b/proto/tycho/evm/v1/common.proto index cbf9eb8..f9f53aa 100644 --- a/proto/tycho/evm/v1/common.proto +++ b/proto/tycho/evm/v1/common.proto @@ -2,20 +2,33 @@ syntax = "proto3"; package tycho.evm.v1; +// This file contains the proto definitions for Substreams common to all integrations. + +// A struct describing a block. message Block { + // The blocks hash. bytes hash = 1; + // The parent blocks hash. bytes parent_hash = 2; + // The block number. uint64 number = 3; + // The block timestamp. uint64 ts = 4; } +// A struct describing a transaction. message Transaction { + // The transaction hash. bytes hash = 1; + // The sender of the transaction. bytes from = 2; + // The receiver of the transaction. bytes to = 3; + // The transactions index within the block. uint64 index = 4; } +// Enum to specify the type of a change. enum ChangeType { CHANGE_TYPE_UNSPECIFIED = 0; CHANGE_TYPE_UPDATE = 1; @@ -23,22 +36,39 @@ enum ChangeType { CHANGE_TYPE_DELETION = 3; } +// A custom struct representing an arbitrary attribute of a protocol component. message Attribute { + // The name of the attribute. string name = 1; + // The value of the attribute. bytes value = 2; + // The type of change the attribute underwent. ChangeType change = 3; } +// A struct describing a part of the protocol. message ProtocolComponent { + // A unique identifier for the component within the protocol. + // Can be a stringified address or a string describing the trading pair. string id = 1; + // Addresses of the ERC20 tokens used by the component. repeated bytes tokens = 2; - repeated string contracts = 3; + // Addresses of the contracts used by the component. + repeated bytes contracts = 3; + // Attributes of the component. + // The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent. repeated Attribute static_att = 4; + // Type of change the component underwent. ChangeType change = 5; } +// A struct for following the changes of Total Value Locked (TVL) of a protocol component. +// Note that if the ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. message BalanceChange { + // The address of the ERC20 token whose balance changed. bytes token = 1; + // The new balance of the token. bytes balance = 2; + // The id of the component whose TVL is tracked. bytes component_id = 3; } diff --git a/proto/tycho/evm/v1/entity.proto b/proto/tycho/evm/v1/entity.proto index 7a940ea..14539e4 100644 --- a/proto/tycho/evm/v1/entity.proto +++ b/proto/tycho/evm/v1/entity.proto @@ -4,19 +4,29 @@ package tycho.evm.v1; import "tycho/evm/v1/common.proto"; +// This file contains the definition for the native integration of Substreams. + +// A component is a set of attributes that are associated with a custom entity. message EntityChanges { + // A unique identifier of the entity within the protocol. string component_id = 1; + // The set of attributes that are associated with the entity. repeated Attribute attributes = 2; } message TransactionEntityChanges { Transaction tx = 1; repeated EntityChanges entity_changes = 2; + // An array of newly added components. repeated ProtocolComponent component_changes = 3; + // An array of balance changes to components. repeated BalanceChange balance_changes = 4; } +// A set of transaction changes within a single block. message BlockEntityChanges { + // The block for which these changes are collectively computed. Block block = 1; + // The set of transaction changes observed in the specified block. repeated TransactionEntityChanges changes = 2; } diff --git a/proto/tycho/evm/v1/vm.proto b/proto/tycho/evm/v1/vm.proto index 8305f69..467bc77 100644 --- a/proto/tycho/evm/v1/vm.proto +++ b/proto/tycho/evm/v1/vm.proto @@ -4,31 +4,46 @@ package tycho.evm.v1; import "tycho/evm/v1/common.proto"; +// This file contains proto definitions specific to the VM integration. + +// A key value entry into contract storage. message ContractSlot { + // A contract's storage slot. bytes slot = 2; + // The new value for this storage slot. bytes value = 3; } +// Changes made to a single contract's state. message ContractChange { + // The contract's address bytes address = 1; - // empty bytes indicates no change + // The new balance of the contract, empty bytes indicates no change. bytes balance = 2; - // empty bytes indicates no change + // The new code of the contract, empty bytes indicates no change. bytes code = 3; - // empty sequence indicates no change + // The changes to this contract's slots, empty sequence indicates no change. repeated ContractSlot slots = 4; - // Whether this is an update, creation or deletion + // Whether this is an update, a creation or a deletion. ChangeType change = 5; } +// A set of changes aggregated by transaction. message TransactionContractChanges { + // The transaction instance that results in the changes. Transaction tx = 1; + // Contains the changes induced by the above transaction, aggregated on a per-contract basis. repeated ContractChange contract_changes = 2; + // An array of newly added components. repeated ProtocolComponent component_changes = 3; + // An array of balance changes to components. repeated BalanceChange balance_changes = 4; } +// A set of transaction changes within a single block. message BlockContractChanges { + // The block for which these changes are collectively computed. Block block = 1; + // The set of transaction changes observed in the specified block. repeated TransactionContractChanges changes = 2; } From 071f02f85686acf8015eca59de5b6d00364565ed Mon Sep 17 00:00:00 2001 From: pistomat Date: Thu, 21 Dec 2023 16:37:57 +0100 Subject: [PATCH 5/6] Update readmes and add upstream changes --- .../indexing/substreams-integration/README.md | 27 ++++++ .../tutorial-ambient.md | 2 - proto/tycho/evm/v1/common.proto | 11 ++- proto/tycho/evm/v1/vm.proto | 3 +- substreams/ethereum-ambient/buf.gen.yaml | 12 +++ substreams/ethereum-ambient/src/lib.rs | 39 ++++++--- .../ethereum-ambient/src/pb/tycho.evm.v1.rs | 85 +++++++++++-------- substreams/ethereum-template/buf.gen.yaml | 12 +++ .../ethereum-template/src/pb/tycho.evm.v1.rs | 85 +++++++++++-------- 9 files changed, 191 insertions(+), 85 deletions(-) delete mode 100644 docs/indexing/substreams-integration/tutorial-ambient.md create mode 100644 substreams/ethereum-ambient/buf.gen.yaml create mode 100644 substreams/ethereum-template/buf.gen.yaml diff --git a/docs/indexing/substreams-integration/README.md b/docs/indexing/substreams-integration/README.md index 273bf21..e5c6fbe 100644 --- a/docs/indexing/substreams-integration/README.md +++ b/docs/indexing/substreams-integration/README.md @@ -65,6 +65,33 @@ The module is a Rust library that is compiled into a SPKG (`.spkg`) file using t Read our [Substreams README.md](../../../substreams/README.md) for more information on how to write the Rust module. +### How to implement the integration + +1. Create a new directory for your integration by cloning the template, rename all the references to `ethereum-template` to `[CHAIN]-[PROTOCOL_SYSTEM]`: + + ```bash + cp -r ./substreams/ethereum-template ./substreams/[CHAIN]-[PROTOCOL_SYSTEM] + ``` +1. Implement the logic in the Rust module `lib.rs`. The main function to implement is the `map_changes` function, which is called for every block. + + ```rust + #[substreams::handlers::map] + fn map_changes( + block: eth::v2::Block, + ) -> Result {} + ``` + The `map_changes` function takes a raw block as input and returns a `BlockContractChanges` struct, which is derived from the `BlockContractChanges` protobuf message in [vm.proto](../../../proto/tycho/evm/v1/vm.proto). + + +1. The `BlockContractChanges` is a list of `TransactionContractChanges`, which includes these main fields: + - list of `ContractChange` - All storage slots that have changed in the transaction for every contract tracked by any ProtocolComponent + - list of `ProtocolComponent` - All the protocol component changes in the transaction + - list of `BalanceChange` - All the contract component changes in the transaction + + See the [Ambient reference example](../../../substreams/ethereum-ambient/src/lib.rs) for more information. + + + ### Testing Read the [Substreams testing docs](../../../substreams/README.md#testing-your-implementation) for more information on how to test your integration. diff --git a/docs/indexing/substreams-integration/tutorial-ambient.md b/docs/indexing/substreams-integration/tutorial-ambient.md deleted file mode 100644 index 5364956..0000000 --- a/docs/indexing/substreams-integration/tutorial-ambient.md +++ /dev/null @@ -1,2 +0,0 @@ -# Tutorial: Ambient - diff --git a/proto/tycho/evm/v1/common.proto b/proto/tycho/evm/v1/common.proto index f9f53aa..2acbd0a 100644 --- a/proto/tycho/evm/v1/common.proto +++ b/proto/tycho/evm/v1/common.proto @@ -37,6 +37,7 @@ enum ChangeType { } // A custom struct representing an arbitrary attribute of a protocol component. +// This is mainly used by the native integration to track the necessary information about the protocol. message Attribute { // The name of the attribute. string name = 1; @@ -47,6 +48,11 @@ message Attribute { } // A struct describing a part of the protocol. +// Note: For example this can be a UniswapV2 pair, that would track the two ERC20 tokens used by the pair, +// the contract would be only the single sontract. The attributes would be empty for the VM integration, +// because we track all the relevant info via storage slots and balance changes. +// It can also be a wrapping contract, like WETH, that has a constant price, but it allows swapping tokens. +// This is why the name ProtocolComponent is used instead of "Pool" or "Pair". message ProtocolComponent { // A unique identifier for the component within the protocol. // Can be a stringified address or a string describing the trading pair. @@ -54,8 +60,9 @@ message ProtocolComponent { // Addresses of the ERC20 tokens used by the component. repeated bytes tokens = 2; // Addresses of the contracts used by the component. + // Usually it is a single contract, but some protocols use multiple contracts. repeated bytes contracts = 3; - // Attributes of the component. + // Attributes of the component. Used mainly be the native integration. // The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent. repeated Attribute static_att = 4; // Type of change the component underwent. @@ -64,11 +71,13 @@ message ProtocolComponent { // A struct for following the changes of Total Value Locked (TVL) of a protocol component. // Note that if the ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. +// E.g. for UniswapV2 pair WETH/USDC, this tracks the USDC and WETH balance of the pair contract. message BalanceChange { // The address of the ERC20 token whose balance changed. bytes token = 1; // The new balance of the token. bytes balance = 2; // The id of the component whose TVL is tracked. + // If the protocol component includes multiple contracts, the balance change must be aggregated to reflect how much tokens can be traded. bytes component_id = 3; } diff --git a/proto/tycho/evm/v1/vm.proto b/proto/tycho/evm/v1/vm.proto index 467bc77..95f9c1e 100644 --- a/proto/tycho/evm/v1/vm.proto +++ b/proto/tycho/evm/v1/vm.proto @@ -33,8 +33,9 @@ message TransactionContractChanges { // The transaction instance that results in the changes. Transaction tx = 1; // Contains the changes induced by the above transaction, aggregated on a per-contract basis. + // Must include changes to every contract that is tracked by all ProtocolComponents. repeated ContractChange contract_changes = 2; - // An array of newly added components. + // An array of any component changes. repeated ProtocolComponent component_changes = 3; // An array of balance changes to components. repeated BalanceChange balance_changes = 4; diff --git a/substreams/ethereum-ambient/buf.gen.yaml b/substreams/ethereum-ambient/buf.gen.yaml new file mode 100644 index 0000000..d2e6544 --- /dev/null +++ b/substreams/ethereum-ambient/buf.gen.yaml @@ -0,0 +1,12 @@ + +version: v1 +plugins: +- plugin: buf.build/community/neoeinstein-prost:v0.2.2 + out: src/pb + opt: + - file_descriptor_set=false + +- plugin: buf.build/community/neoeinstein-prost-crate:v0.3.1 + out: src/pb + opt: + - no_features diff --git a/substreams/ethereum-ambient/src/lib.rs b/substreams/ethereum-ambient/src/lib.rs index ae169db..b9a241b 100644 --- a/substreams/ethereum-ambient/src/lib.rs +++ b/substreams/ethereum-ambient/src/lib.rs @@ -3,7 +3,7 @@ use std::collections::{hash_map::Entry, HashMap}; use anyhow::{anyhow, bail}; use ethabi::{decode, ParamType}; use hex_literal::hex; -use substreams_ethereum::pb::eth; +use substreams_ethereum::pb::eth::{self}; use pb::tycho::evm::v1::{self as tycho}; @@ -24,8 +24,7 @@ impl SlotValue { } } -// uses a map for slots, protobuf does not -// allow bytes in hashmap keys +// Uses a map for slots, protobuf does not allow bytes in hashmap keys struct InterimContractChange { address: Vec, balance: Vec, @@ -52,16 +51,25 @@ impl From for tycho::ContractChange { } /// Extracts all contract changes relevant to vm simulations +/// +/// This is the main logic of the substreams integration. It takes a raw ethereum block on input and extracts the BlockContractChanges stream. It includes tracking: +/// - new pool initializations +/// - all storage slot changes for the Ambient contract +/// - all ERC20 balance changes for the Ambient pools +/// - all code changes and balance updates of the Ambient contract +/// +/// Generally we detect all changes in transactions sequentially and detect if it is a CREATE or UPDATE change based on already present data. #[substreams::handlers::map] fn map_changes( block: eth::v2::Block, ) -> Result { - let mut block_changes = tycho::BlockContractChanges { block: None, changes: Vec::new() }; + let mut block_changes = tycho::BlockContractChanges::default(); let mut tx_change = tycho::TransactionContractChanges::default(); let mut changed_contracts: HashMap, InterimContractChange> = HashMap::new(); + // Collect all accounts created in this block let created_accounts: HashMap<_, _> = block .transactions() .flat_map(|tx| { @@ -74,7 +82,7 @@ fn map_changes( .collect(); for block_tx in block.transactions() { - // extract storage changes + // Extract storage changes for all contracts relevant to this ProtocolComponent (i.e. Ambient) let mut storage_changes = block_tx .calls .iter() @@ -87,6 +95,7 @@ fn map_changes( .collect::>(); storage_changes.sort_unstable_by_key(|change| change.ordinal); + // Detect all call to the Ambient contracts, even inner calls let ambient_calls = block_tx .calls .iter() @@ -94,6 +103,8 @@ fn map_changes( .filter(|call| call.address == AMBIENT_CONTRACT) .collect::>(); + // Detect all pool initializations + // Official documentation: https://docs.ambient.finance/developers/dex-contract-interface/pool-initialization for call in ambient_calls { if call.input.len() < 4 { continue; @@ -175,7 +186,7 @@ fn map_changes( pool_index ), tokens, - contracts: vec![hex::encode(AMBIENT_CONTRACT)], + contracts: vec![AMBIENT_CONTRACT.to_vec()], static_att: vec![static_attribute], change: tycho::ChangeType::Creation.into(), }; @@ -192,10 +203,12 @@ fn map_changes( } } + // Extract all contract changes. + // We cache the data in a general interim contract > slot > value data structure. // Note: some contracts change slot values and change them back to their - // original value before the transactions ends we remember the initial - // value before the first change and in the end filter found deltas - // that ended up not actually changing anything. + // original value before the transactions ends we remember the initial + // value before the first change and in the end filter found deltas + // that ended up not actually changing anything. for storage_change in storage_changes.iter() { match changed_contracts.entry(storage_change.address.clone()) { // We have already an entry recording a change about this contract @@ -223,7 +236,7 @@ fn map_changes( } } } - // Intialise a new contract change after obsering a storage change + // Intialise a new contract change after observing a storage change Entry::Vacant(e) => { let mut slots = HashMap::new(); slots.insert( @@ -248,7 +261,7 @@ fn map_changes( } } - // extract balance changes + // Extract balance changes let mut balance_changes = block_tx .calls .iter() @@ -290,7 +303,7 @@ fn map_changes( } } - // extract code changes + // Extract code changes let mut code_changes = block_tx .calls .iter() @@ -328,7 +341,7 @@ fn map_changes( } } - // if there were any changes, add transaction and push the changes + // If there were any changes, add transaction and push the changes if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() { tx_change.tx = Some(tycho::Transaction { hash: block_tx.hash.clone(), diff --git a/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs b/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs index f59fcf2..b59b5f7 100644 --- a/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs @@ -1,62 +1,92 @@ // @generated +// This file contains the proto definitions for Substreams common to all integrations. + +/// A struct describing a block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { + /// The blocks hash. #[prost(bytes="vec", tag="1")] pub hash: ::prost::alloc::vec::Vec, + /// The parent blocks hash. #[prost(bytes="vec", tag="2")] pub parent_hash: ::prost::alloc::vec::Vec, + /// The block number. #[prost(uint64, tag="3")] pub number: u64, + /// The block timestamp. #[prost(uint64, tag="4")] pub ts: u64, } +/// A struct describing a transaction. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Transaction { + /// The transaction hash. #[prost(bytes="vec", tag="1")] pub hash: ::prost::alloc::vec::Vec, + /// The sender of the transaction. #[prost(bytes="vec", tag="2")] pub from: ::prost::alloc::vec::Vec, + /// The receiver of the transaction. #[prost(bytes="vec", tag="3")] pub to: ::prost::alloc::vec::Vec, + /// The transactions index within the block. #[prost(uint64, tag="4")] pub index: u64, } +/// A custom struct representing an arbitrary attribute of a protocol component. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Attribute { + /// The name of the attribute. #[prost(string, tag="1")] pub name: ::prost::alloc::string::String, + /// The value of the attribute. #[prost(bytes="vec", tag="2")] pub value: ::prost::alloc::vec::Vec, + /// The type of change the attribute underwent. #[prost(enumeration="ChangeType", tag="3")] pub change: i32, } +/// A struct describing a part of the protocol. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtocolComponent { + /// A unique identifier for the component within the protocol. + /// Can be a stringified address or a string describing the trading pair. #[prost(string, tag="1")] pub id: ::prost::alloc::string::String, + /// Addresses of the ERC20 tokens used by the component. #[prost(bytes="vec", repeated, tag="2")] pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, - #[prost(string, repeated, tag="3")] - pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Addresses of the contracts used by the component. + #[prost(bytes="vec", repeated, tag="3")] + pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// Attributes of the component. + /// The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent. #[prost(message, repeated, tag="4")] pub static_att: ::prost::alloc::vec::Vec, + /// Type of change the component underwent. #[prost(enumeration="ChangeType", tag="5")] pub change: i32, } +/// A struct for following the changes of Total Value Locked (TVL) of a protocol component. +/// Note that if the ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BalanceChange { + /// The address of the ERC20 token whose balance changed. #[prost(bytes="vec", tag="1")] pub token: ::prost::alloc::vec::Vec, + /// The new balance of the token. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. #[prost(bytes="vec", tag="3")] pub component_id: ::prost::alloc::vec::Vec, } +/// Enum to specify the type of a change. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ChangeType { @@ -89,77 +119,64 @@ impl ChangeType { } } } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityChanges { - #[prost(string, tag="1")] - pub component_id: ::prost::alloc::string::String, - #[prost(message, repeated, tag="2")] - pub attributes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionEntityChanges { - #[prost(message, optional, tag="1")] - pub tx: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub entity_changes: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="3")] - pub component_changes: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub balance_changes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockEntityChanges { - #[prost(message, optional, tag="1")] - pub block: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub changes: ::prost::alloc::vec::Vec, -} +// This file contains proto definitions specific to the VM integration. + +/// A key value entry into contract storage. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ContractSlot { + /// A contract's storage slot. #[prost(bytes="vec", tag="2")] pub slot: ::prost::alloc::vec::Vec, + /// The new value for this storage slot. #[prost(bytes="vec", tag="3")] pub value: ::prost::alloc::vec::Vec, } +/// Changes made to a single contract's state. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ContractChange { + /// The contract's address #[prost(bytes="vec", tag="1")] pub address: ::prost::alloc::vec::Vec, - /// empty bytes indicates no change + /// The new balance of the contract, empty bytes indicates no change. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, - /// empty bytes indicates no change + /// The new code of the contract, empty bytes indicates no change. #[prost(bytes="vec", tag="3")] pub code: ::prost::alloc::vec::Vec, - /// empty sequence indicates no change + /// The changes to this contract's slots, empty sequence indicates no change. #[prost(message, repeated, tag="4")] pub slots: ::prost::alloc::vec::Vec, - /// Whether this is an update, creation or deletion + /// Whether this is an update, a creation or a deletion. #[prost(enumeration="ChangeType", tag="5")] pub change: i32, } +/// A set of changes aggregated by transaction. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionContractChanges { + /// The transaction instance that results in the changes. #[prost(message, optional, tag="1")] pub tx: ::core::option::Option, + /// Contains the changes induced by the above transaction, aggregated on a per-contract basis. #[prost(message, repeated, tag="2")] pub contract_changes: ::prost::alloc::vec::Vec, + /// An array of newly added components. #[prost(message, repeated, tag="3")] pub component_changes: ::prost::alloc::vec::Vec, + /// An array of balance changes to components. #[prost(message, repeated, tag="4")] pub balance_changes: ::prost::alloc::vec::Vec, } +/// A set of transaction changes within a single block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockContractChanges { + /// The block for which these changes are collectively computed. #[prost(message, optional, tag="1")] pub block: ::core::option::Option, + /// The set of transaction changes observed in the specified block. #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } diff --git a/substreams/ethereum-template/buf.gen.yaml b/substreams/ethereum-template/buf.gen.yaml new file mode 100644 index 0000000..d2e6544 --- /dev/null +++ b/substreams/ethereum-template/buf.gen.yaml @@ -0,0 +1,12 @@ + +version: v1 +plugins: +- plugin: buf.build/community/neoeinstein-prost:v0.2.2 + out: src/pb + opt: + - file_descriptor_set=false + +- plugin: buf.build/community/neoeinstein-prost-crate:v0.3.1 + out: src/pb + opt: + - no_features diff --git a/substreams/ethereum-template/src/pb/tycho.evm.v1.rs b/substreams/ethereum-template/src/pb/tycho.evm.v1.rs index f59fcf2..b59b5f7 100644 --- a/substreams/ethereum-template/src/pb/tycho.evm.v1.rs +++ b/substreams/ethereum-template/src/pb/tycho.evm.v1.rs @@ -1,62 +1,92 @@ // @generated +// This file contains the proto definitions for Substreams common to all integrations. + +/// A struct describing a block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { + /// The blocks hash. #[prost(bytes="vec", tag="1")] pub hash: ::prost::alloc::vec::Vec, + /// The parent blocks hash. #[prost(bytes="vec", tag="2")] pub parent_hash: ::prost::alloc::vec::Vec, + /// The block number. #[prost(uint64, tag="3")] pub number: u64, + /// The block timestamp. #[prost(uint64, tag="4")] pub ts: u64, } +/// A struct describing a transaction. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Transaction { + /// The transaction hash. #[prost(bytes="vec", tag="1")] pub hash: ::prost::alloc::vec::Vec, + /// The sender of the transaction. #[prost(bytes="vec", tag="2")] pub from: ::prost::alloc::vec::Vec, + /// The receiver of the transaction. #[prost(bytes="vec", tag="3")] pub to: ::prost::alloc::vec::Vec, + /// The transactions index within the block. #[prost(uint64, tag="4")] pub index: u64, } +/// A custom struct representing an arbitrary attribute of a protocol component. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Attribute { + /// The name of the attribute. #[prost(string, tag="1")] pub name: ::prost::alloc::string::String, + /// The value of the attribute. #[prost(bytes="vec", tag="2")] pub value: ::prost::alloc::vec::Vec, + /// The type of change the attribute underwent. #[prost(enumeration="ChangeType", tag="3")] pub change: i32, } +/// A struct describing a part of the protocol. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ProtocolComponent { + /// A unique identifier for the component within the protocol. + /// Can be a stringified address or a string describing the trading pair. #[prost(string, tag="1")] pub id: ::prost::alloc::string::String, + /// Addresses of the ERC20 tokens used by the component. #[prost(bytes="vec", repeated, tag="2")] pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, - #[prost(string, repeated, tag="3")] - pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Addresses of the contracts used by the component. + #[prost(bytes="vec", repeated, tag="3")] + pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// Attributes of the component. + /// The inner ChangeType of the attribute has to match the ChangeType of the ProtocolComponent. #[prost(message, repeated, tag="4")] pub static_att: ::prost::alloc::vec::Vec, + /// Type of change the component underwent. #[prost(enumeration="ChangeType", tag="5")] pub change: i32, } +/// A struct for following the changes of Total Value Locked (TVL) of a protocol component. +/// Note that if the ProtocolComponent contains multiple contracts, the TVL is tracked for the component as a whole. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BalanceChange { + /// The address of the ERC20 token whose balance changed. #[prost(bytes="vec", tag="1")] pub token: ::prost::alloc::vec::Vec, + /// The new balance of the token. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, + /// The id of the component whose TVL is tracked. #[prost(bytes="vec", tag="3")] pub component_id: ::prost::alloc::vec::Vec, } +/// Enum to specify the type of a change. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ChangeType { @@ -89,77 +119,64 @@ impl ChangeType { } } } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityChanges { - #[prost(string, tag="1")] - pub component_id: ::prost::alloc::string::String, - #[prost(message, repeated, tag="2")] - pub attributes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionEntityChanges { - #[prost(message, optional, tag="1")] - pub tx: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub entity_changes: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="3")] - pub component_changes: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub balance_changes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockEntityChanges { - #[prost(message, optional, tag="1")] - pub block: ::core::option::Option, - #[prost(message, repeated, tag="2")] - pub changes: ::prost::alloc::vec::Vec, -} +// This file contains proto definitions specific to the VM integration. + +/// A key value entry into contract storage. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ContractSlot { + /// A contract's storage slot. #[prost(bytes="vec", tag="2")] pub slot: ::prost::alloc::vec::Vec, + /// The new value for this storage slot. #[prost(bytes="vec", tag="3")] pub value: ::prost::alloc::vec::Vec, } +/// Changes made to a single contract's state. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ContractChange { + /// The contract's address #[prost(bytes="vec", tag="1")] pub address: ::prost::alloc::vec::Vec, - /// empty bytes indicates no change + /// The new balance of the contract, empty bytes indicates no change. #[prost(bytes="vec", tag="2")] pub balance: ::prost::alloc::vec::Vec, - /// empty bytes indicates no change + /// The new code of the contract, empty bytes indicates no change. #[prost(bytes="vec", tag="3")] pub code: ::prost::alloc::vec::Vec, - /// empty sequence indicates no change + /// The changes to this contract's slots, empty sequence indicates no change. #[prost(message, repeated, tag="4")] pub slots: ::prost::alloc::vec::Vec, - /// Whether this is an update, creation or deletion + /// Whether this is an update, a creation or a deletion. #[prost(enumeration="ChangeType", tag="5")] pub change: i32, } +/// A set of changes aggregated by transaction. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionContractChanges { + /// The transaction instance that results in the changes. #[prost(message, optional, tag="1")] pub tx: ::core::option::Option, + /// Contains the changes induced by the above transaction, aggregated on a per-contract basis. #[prost(message, repeated, tag="2")] pub contract_changes: ::prost::alloc::vec::Vec, + /// An array of newly added components. #[prost(message, repeated, tag="3")] pub component_changes: ::prost::alloc::vec::Vec, + /// An array of balance changes to components. #[prost(message, repeated, tag="4")] pub balance_changes: ::prost::alloc::vec::Vec, } +/// A set of transaction changes within a single block. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockContractChanges { + /// The block for which these changes are collectively computed. #[prost(message, optional, tag="1")] pub block: ::core::option::Option, + /// The set of transaction changes observed in the specified block. #[prost(message, repeated, tag="2")] pub changes: ::prost::alloc::vec::Vec, } From e9ebf6f0c6dac64bc0710547bca7f108f9e04046 Mon Sep 17 00:00:00 2001 From: pistomat Date: Thu, 21 Dec 2023 16:43:23 +0100 Subject: [PATCH 6/6] Add modules reference to docs --- docs/indexing/substreams-integration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/indexing/substreams-integration/README.md b/docs/indexing/substreams-integration/README.md index e5c6fbe..4e7fb2b 100644 --- a/docs/indexing/substreams-integration/README.md +++ b/docs/indexing/substreams-integration/README.md @@ -90,7 +90,7 @@ Read our [Substreams README.md](../../../substreams/README.md) for more informat See the [Ambient reference example](../../../substreams/ethereum-ambient/src/lib.rs) for more information. - +1. If you are more advanced with Substreams, you can define more steps than a single "map" step, including defining your own protobuf files. Add these protobuf files in your `pb` folder and update the manifest accordingly. This allows for better parallelization of the indexing process. See the official documentation of [modules](https://substreams.streamingfast.io/concepts-and-fundamentals/modules#modules-basics-overview). ### Testing