diff --git a/.gitignore b/.gitignore index 485dee6..2c65d74 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,13 @@ +### Rust ### +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Substreams spkg files are build artifacts +*.spkg + +.env +.vscode .idea +*.log diff --git a/proto/buf.lock b/proto/buf.lock new file mode 100644 index 0000000..c91b581 --- /dev/null +++ b/proto/buf.lock @@ -0,0 +1,2 @@ +# Generated by buf. DO NOT EDIT. +version: v1 diff --git a/proto/buf.yaml b/proto/buf.yaml new file mode 100644 index 0000000..d4ff52d --- /dev/null +++ b/proto/buf.yaml @@ -0,0 +1,7 @@ +version: v1 +breaking: + use: + - FILE +lint: + use: + - BASIC diff --git a/proto/sf/substreams/internal/v2/deltas.proto b/proto/sf/substreams/internal/v2/deltas.proto new file mode 100644 index 0000000..21b9e40 --- /dev/null +++ b/proto/sf/substreams/internal/v2/deltas.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package sf.substreams.internal.v2; + +import "google/protobuf/any.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/internal/v2;pbssinternal"; + +message StoreDeltas { + repeated StoreDelta store_deltas = 1; +} + +message StoreDelta { + enum Operation { + UNSET = 0; + CREATE = 1; + UPDATE = 2; + DELETE = 3; + } + Operation operation = 1; + uint64 ordinal = 2; + string key = 3; + bytes old_value = 4; + bytes new_value = 5; +} + +message ModuleOutput { + string module_name = 1; + oneof data { + google.protobuf.Any map_output = 2; + StoreDeltas store_deltas = 3; + } + repeated string logs = 4; + bool debug_logs_truncated = 5; + bool cached = 6; +} diff --git a/proto/sf/substreams/internal/v2/service.proto b/proto/sf/substreams/internal/v2/service.proto new file mode 100644 index 0000000..f695707 --- /dev/null +++ b/proto/sf/substreams/internal/v2/service.proto @@ -0,0 +1,88 @@ +syntax = "proto3"; + +package sf.substreams.internal.v2; + +import "sf/substreams/v1/modules.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/internal/v2;pbssinternal"; + +service Substreams { + rpc ProcessRange(ProcessRangeRequest) returns (stream ProcessRangeResponse); +} + +message ProcessRangeRequest { + uint64 start_block_num = 1; + uint64 stop_block_num = 2; + string output_module = 3; + sf.substreams.v1.Modules modules = 4; + uint32 stage = 5; // 0-based index of stage to execute up to +} + +message ProcessRangeResponse { + reserved 1; // previously string module_name = 1; + + reserved 2; // previously in oneof(type): BlockRange processed_range + reserved 3; // previously in oneof(type): ProcessedBytes processed_bytes + + oneof type { + Failed failed = 4; + Completed completed = 5; + Update update = 6; + } +} + +message Update { + uint64 duration_ms = 1; + uint64 processed_blocks = 2; + uint64 total_bytes_read = 3; + uint64 total_bytes_written = 4; + + repeated ModuleStats modules_stats = 5; +} + +message ModuleStats { + string name = 1; + uint64 processing_time_ms = 2; + uint64 store_operation_time_ms = 3; + uint64 store_read_count = 4; + + repeated ExternalCallMetric external_call_metrics = 5; + + // store-specific (will be 0 on mappers) + uint64 store_write_count = 10; + uint64 store_deleteprefix_count = 11; + uint64 store_size_bytes = 12; +} + +message ExternalCallMetric { + string name = 1; + uint64 count = 2; + uint64 time_ms = 3; +} + +message Completed { + repeated BlockRange all_processed_ranges = 1; + + // TraceId represents the producer's trace id that produced the partial files. + // This is present here so that the consumer can use it to identify the + // right partial files that needs to be squashed together. + // + // The TraceId can be empty in which case it should be assumed by the tier1 + // consuming this message that the tier2 that produced those partial files + // is not yet updated to produce a trace id and a such, the tier1 should + // generate a legacy partial file name. + string trace_id = 2; +} + +message Failed { + string reason = 1; + repeated string logs = 2; + // FailureLogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 3; +} + +message BlockRange { + uint64 start_block = 2; + uint64 end_block = 3; +} diff --git a/proto/sf/substreams/options.proto b/proto/sf/substreams/options.proto new file mode 100644 index 0000000..9ea5537 --- /dev/null +++ b/proto/sf/substreams/options.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +package sf.substreams; + +import "google/protobuf/descriptor.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams;pbsubstreams"; + +message FieldOptions { + // this option informs the `substreams pack` command that it should treat the corresponding manifest value as a path to a file, putting its content as bytes in this field. + // must be applied to a `bytes` or `string` field + bool load_from_file = 1; + + // this option informs the `substreams pack` command that it should treat the corresponding manifest value as a path to a folder, zipping its content and putting the zip content as bytes in this field. + // must be applied to a `bytes` field + bool zip_from_folder = 2; +} + +extend google.protobuf.FieldOptions { + optional FieldOptions options = 2200; +} diff --git a/proto/sf/substreams/rpc/v2/service.proto b/proto/sf/substreams/rpc/v2/service.proto new file mode 100644 index 0000000..05e69c0 --- /dev/null +++ b/proto/sf/substreams/rpc/v2/service.proto @@ -0,0 +1,235 @@ +syntax = "proto3"; + +package sf.substreams.rpc.v2; + +import "google/protobuf/any.proto"; +import "sf/substreams/v1/clock.proto"; +import "sf/substreams/v1/modules.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/rpc/v2;pbsubstreamsrpc"; + +service Stream { + rpc Blocks(Request) returns (stream Response); +} + +message Request { + int64 start_block_num = 1; + string start_cursor = 2; + uint64 stop_block_num = 3; + + // With final_block_only, you only receive blocks that are irreversible: + // 'final_block_height' will be equal to current block and no 'undo_signal' will ever be sent + bool final_blocks_only = 4; + + // Substreams has two mode when executing your module(s) either development mode or production + // mode. Development and production modes impact the execution of Substreams, important aspects + // of execution include: + // * The time required to reach the first byte. + // * The speed that large ranges get executed. + // * The module logs and outputs sent back to the client. + // + // By default, the engine runs in developer mode, with richer and deeper output. Differences + // between production and development modes include: + // * Forward parallel execution is enabled in production mode and disabled in development mode + // * The time required to reach the first byte in development mode is faster than in production mode. + // + // Specific attributes of development mode include: + // * The client will receive all of the executed module's logs. + // * It's possible to request specific store snapshots in the execution tree (via `debug_initial_store_snapshot_for_modules`). + // * Multiple module's output is possible. + // + // With production mode`, however, you trade off functionality for high speed enabling forward + // parallel execution of module ahead of time. + bool production_mode = 5; + + string output_module = 6; + + sf.substreams.v1.Modules modules = 7; + + // Available only in developer mode + repeated string debug_initial_store_snapshot_for_modules = 10; +} + +message Response { + oneof message { + SessionInit session = 1; // Always sent first + ModulesProgress progress = 2; // Progress of data preparation, before sending in the stream of `data` events. + BlockScopedData block_scoped_data = 3; + BlockUndoSignal block_undo_signal = 4; + Error fatal_error = 5; + + // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + InitialSnapshotData debug_snapshot_data = 10; + // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + InitialSnapshotComplete debug_snapshot_complete = 11; + } +} + +// BlockUndoSignal informs you that every bit of data +// with a block number above 'last_valid_block' has been reverted +// on-chain. Delete that data and restart from 'last_valid_cursor' +message BlockUndoSignal { + sf.substreams.v1.BlockRef last_valid_block = 1; + string last_valid_cursor = 2; +} + +message BlockScopedData { + MapModuleOutput output = 1; + sf.substreams.v1.Clock clock = 2; + string cursor = 3; + + // Non-deterministic, allows substreams-sink to let go of their undo data. + uint64 final_block_height = 4; + + repeated MapModuleOutput debug_map_outputs = 10; + repeated StoreModuleOutput debug_store_outputs = 11; +} + +message SessionInit { + string trace_id = 1; + uint64 resolved_start_block = 2; + uint64 linear_handoff_block = 3; + uint64 max_parallel_workers = 4; +} + +message InitialSnapshotComplete { + string cursor = 1; +} + +message InitialSnapshotData { + string module_name = 1; + repeated StoreDelta deltas = 2; + uint64 sent_keys = 4; + uint64 total_keys = 3; +} + +message MapModuleOutput { + string name = 1; + google.protobuf.Any map_output = 2; + // DebugOutputInfo is available in non-production mode only + OutputDebugInfo debug_info = 10; +} + +// StoreModuleOutput are produced for store modules in development mode. +// It is not possible to retrieve store models in production, with parallelization +// enabled. If you need the deltas directly, write a pass through mapper module +// that will get them down to you. +message StoreModuleOutput { + string name = 1; + repeated StoreDelta debug_store_deltas = 2; + OutputDebugInfo debug_info = 10; +} + +message OutputDebugInfo { + repeated string logs = 1; + // LogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 2; + bool cached = 3; +} + +// ModulesProgress is a message that is sent every 500ms +message ModulesProgress { + // previously: repeated ModuleProgress modules = 1; + // these previous `modules` messages were sent in bursts and are not sent anymore. + reserved 1; + // List of jobs running on tier2 servers + repeated Job running_jobs = 2; + // Execution statistics for each module + repeated ModuleStats modules_stats = 3; + // Stages definition and completed block ranges + repeated Stage stages = 4; + + ProcessedBytes processed_bytes = 5; +} + +message ProcessedBytes { + uint64 total_bytes_read = 1; + uint64 total_bytes_written = 2; +} + +message Error { + string module = 1; + string reason = 2; + repeated string logs = 3; + // FailureLogsTruncated is a flag that tells you if you received all the logs or if they + // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + bool logs_truncated = 4; +} + +message Job { + uint32 stage = 1; + uint64 start_block = 2; + uint64 stop_block = 3; + uint64 processed_blocks = 4; + uint64 duration_ms = 5; +} + +message Stage { + repeated string modules = 1; + repeated BlockRange completed_ranges = 2; +} + +// ModuleStats gathers metrics and statistics from each module, running on tier1 or tier2 +// All the 'count' and 'time_ms' values may include duplicate for each stage going over that module +message ModuleStats { + // name of the module + string name = 1; + + // total_processed_blocks is the sum of blocks sent to that module code + uint64 total_processed_block_count = 2; + // total_processing_time_ms is the sum of all time spent running that module code + uint64 total_processing_time_ms = 3; + + //// external_calls are chain-specific intrinsics, like "Ethereum RPC calls". + repeated ExternalCallMetric external_call_metrics = 4; + + // total_store_operation_time_ms is the sum of all time spent running that module code waiting for a store operation (ex: read, write, delete...) + uint64 total_store_operation_time_ms = 5; + // total_store_read_count is the sum of all the store Read operations called from that module code + uint64 total_store_read_count = 6; + + // total_store_write_count is the sum of all store Write operations called from that module code (store-only) + uint64 total_store_write_count = 10; + + // total_store_deleteprefix_count is the sum of all store DeletePrefix operations called from that module code (store-only) + // note that DeletePrefix can be a costly operation on large stores + uint64 total_store_deleteprefix_count = 11; + + // store_size_bytes is the uncompressed size of the full KV store for that module, from the last 'merge' operation (store-only) + uint64 store_size_bytes = 12; + + // total_store_merging_time_ms is the time spent merging partial stores into a full KV store for that module (store-only) + uint64 total_store_merging_time_ms = 13; + + // store_currently_merging is true if there is a merging operation (partial store to full KV store) on the way. + bool store_currently_merging = 14; + + // highest_contiguous_block is the highest block in the highest merged full KV store of that module (store-only) + uint64 highest_contiguous_block = 15; +} + +message ExternalCallMetric { + string name = 1; + uint64 count = 2; + uint64 time_ms = 3; +} + +message StoreDelta { + enum Operation { + UNSET = 0; + CREATE = 1; + UPDATE = 2; + DELETE = 3; + } + Operation operation = 1; + uint64 ordinal = 2; + string key = 3; + bytes old_value = 4; + bytes new_value = 5; +} + +message BlockRange { + uint64 start_block = 2; + uint64 end_block = 3; +} diff --git a/proto/sf/substreams/sink/service/v1/service.proto b/proto/sf/substreams/sink/service/v1/service.proto new file mode 100644 index 0000000..4c85e9f --- /dev/null +++ b/proto/sf/substreams/sink/service/v1/service.proto @@ -0,0 +1,142 @@ +syntax = "proto3"; + +package sf.substreams.sink.service.v1; + +import "sf/substreams/v1/package.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/sink/service/v1;pbsinksvc"; + +service Provider { + rpc Deploy(DeployRequest) returns (DeployResponse); + rpc Update(UpdateRequest) returns (UpdateResponse); + rpc Info(InfoRequest) returns (InfoResponse); + rpc List(ListRequest) returns (ListResponse); + rpc Pause(PauseRequest) returns (PauseResponse); + rpc Stop(StopRequest) returns (StopResponse); + rpc Resume(ResumeRequest) returns (ResumeResponse); + rpc Remove(RemoveRequest) returns (RemoveResponse); +} + +message DeployRequest { + sf.substreams.v1.Package substreams_package = 1; + + bool development_mode = 2; + repeated Parameter parameters = 3; +} + +message Parameter { + string key = 1; + string value = 2; +} + +message DeployResponse { + DeploymentStatus status = 1; + + // deployment_id is a short name (max 8 characters) that uniquely identifies your deployment + string deployment_id = 2; + + map services = 3; + string reason = 4; + string motd = 5; +} + +message UpdateRequest { + sf.substreams.v1.Package substreams_package = 1; + string deployment_id = 2; + bool reset = 3; +} + +message UpdateResponse { + DeploymentStatus status = 1; + map services = 2; + string reason = 3; + string motd = 4; +} + +message InfoRequest { + string deployment_id = 1; +} + +message InfoResponse { + DeploymentStatus status = 1; + map services = 2; + string reason = 3; + PackageInfo package_info = 4; + SinkProgress progress = 5; + string motd = 6; +} + +message SinkProgress { + uint64 last_processed_block = 1; +} + +message PackageInfo { + string name = 1; + string version = 2; + string output_module_name = 3; + string output_module_hash = 4; +} + +message ListRequest {} + +message ListResponse { + repeated DeploymentWithStatus deployments = 1; +} + +message DeploymentWithStatus { + string id = 1; + DeploymentStatus status = 2; + string reason = 3; + PackageInfo package_info = 4; + SinkProgress progress = 5; + string motd = 6; +} + +enum DeploymentStatus { + UNKNOWN = 0; + RUNNING = 1; + FAILING = 2; + PAUSED = 3; + STOPPED = 4; + + STARTING = 5; + PAUSING = 6; + STOPPING = 7; + REMOVING = 8; + RESUMING = 9; +} + +message RemoveRequest { + string deployment_id = 1; +} + +message RemoveResponse { + DeploymentStatus previous_status = 1; +} + +message PauseRequest { + string deployment_id = 1; +} + +message PauseResponse { + DeploymentStatus previous_status = 1; + DeploymentStatus new_status = 2; +} + +message StopRequest { + string deployment_id = 1; +} + +message StopResponse { + DeploymentStatus previous_status = 1; + DeploymentStatus new_status = 2; +} + +message ResumeRequest { + string deployment_id = 1; +} + +message ResumeResponse { + DeploymentStatus previous_status = 1; + DeploymentStatus new_status = 2; +} diff --git a/proto/sf/substreams/v1/clock.proto b/proto/sf/substreams/v1/clock.proto new file mode 100644 index 0000000..9d3e4bd --- /dev/null +++ b/proto/sf/substreams/v1/clock.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package sf.substreams.v1; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/v1;pbsubstreams"; + +// Clock is a pointer to a block with added timestamp +message Clock { + string id = 1; + uint64 number = 2; + google.protobuf.Timestamp timestamp = 3; +} + +// BlockRef is a pointer to a block to which we don't know the timestamp +message BlockRef { + string id = 1; + uint64 number = 2; +} diff --git a/proto/sf/substreams/v1/modules.proto b/proto/sf/substreams/v1/modules.proto new file mode 100644 index 0000000..bde9c27 --- /dev/null +++ b/proto/sf/substreams/v1/modules.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; + +package sf.substreams.v1; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/v1;pbsubstreams"; + +message Modules { + repeated Module modules = 1; + repeated Binary binaries = 2; +} + +// Binary represents some code compiled to its binary form. +message Binary { + string type = 1; + bytes content = 2; +} + +message Module { + string name = 1; + oneof kind { + KindMap kind_map = 2; + KindStore kind_store = 3; + } + + uint32 binary_index = 4; + string binary_entrypoint = 5; + + repeated Input inputs = 6; + Output output = 7; + + uint64 initial_block = 8; + + message KindMap { + string output_type = 1; + } + + message KindStore { + // The `update_policy` determines the functions available to mutate the store + // (like `set()`, `set_if_not_exists()` or `sum()`, etc..) in + // order to ensure that parallel operations are possible and deterministic + // + // Say a store cumulates keys from block 0 to 1M, and a second store + // cumulates keys from block 1M to 2M. When we want to use this + // store as a dependency for a downstream module, we will merge the + // two stores according to this policy. + UpdatePolicy update_policy = 1; + string value_type = 2; + + enum UpdatePolicy { + UPDATE_POLICY_UNSET = 0; + // Provides a store where you can `set()` keys, and the latest key wins + UPDATE_POLICY_SET = 1; + // Provides a store where you can `set_if_not_exists()` keys, and the first key wins + UPDATE_POLICY_SET_IF_NOT_EXISTS = 2; + // Provides a store where you can `add_*()` keys, where two stores merge by summing its values. + UPDATE_POLICY_ADD = 3; + // Provides a store where you can `min_*()` keys, where two stores merge by leaving the minimum value. + UPDATE_POLICY_MIN = 4; + // Provides a store where you can `max_*()` keys, where two stores merge by leaving the maximum value. + UPDATE_POLICY_MAX = 5; + // Provides a store where you can `append()` keys, where two stores merge by concatenating the bytes in order. + UPDATE_POLICY_APPEND = 6; + } + } + + message Input { + oneof input { + Source source = 1; + Map map = 2; + Store store = 3; + Params params = 4; + } + + message Source { + string type = 1; // ex: "sf.ethereum.type.v1.Block" + } + message Map { + string module_name = 1; // ex: "block_to_pairs" + } + message Store { + string module_name = 1; + Mode mode = 2; + + enum Mode { + UNSET = 0; + GET = 1; + DELTAS = 2; + } + } + message Params { + string value = 1; + } + } + + message Output { + string type = 1; + } +} diff --git a/proto/sf/substreams/v1/package.proto b/proto/sf/substreams/v1/package.proto new file mode 100644 index 0000000..3bf7707 --- /dev/null +++ b/proto/sf/substreams/v1/package.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +package sf.substreams.v1; + +import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; +import "sf/substreams/v1/modules.proto"; + +option go_package = "github.com/streamingfast/substreams/pb/sf/substreams/v1;pbsubstreams"; + +message Package { + // Needs to be one so this file can be used _directly_ as a + // buf `Image` andor a ProtoSet for grpcurl and other tools + repeated google.protobuf.FileDescriptorProto proto_files = 1; + reserved 2 to 4; // Reserved for future: in case protosets adds fields + + uint64 version = 5; + sf.substreams.v1.Modules modules = 6; + repeated ModuleMetadata module_meta = 7; + repeated PackageMetadata package_meta = 8; + + // Source network for Substreams to fetch its data from. + string network = 9; + + google.protobuf.Any sink_config = 10; + string sink_module = 11; + // image is the bytes to a JPEG, WebP or PNG file. Max size is 2 MiB + bytes image = 12; +} + +message PackageMetadata { + string version = 1; + string url = 2; + string name = 3; + string doc = 4; +} + +message ModuleMetadata { + // Corresponds to the index in `Package.metadata.package_meta` + uint64 package_index = 1; + string doc = 2; +} diff --git a/proto/tycho/evm/v1/common.proto b/proto/tycho/evm/v1/common.proto new file mode 100644 index 0000000..cbf9eb8 --- /dev/null +++ b/proto/tycho/evm/v1/common.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +package tycho.evm.v1; + +message Block { + bytes hash = 1; + bytes parent_hash = 2; + uint64 number = 3; + uint64 ts = 4; +} + +message Transaction { + bytes hash = 1; + bytes from = 2; + bytes to = 3; + uint64 index = 4; +} + +enum ChangeType { + CHANGE_TYPE_UNSPECIFIED = 0; + CHANGE_TYPE_UPDATE = 1; + CHANGE_TYPE_CREATION = 2; + CHANGE_TYPE_DELETION = 3; +} + +message Attribute { + string name = 1; + bytes value = 2; + ChangeType change = 3; +} + +message ProtocolComponent { + string id = 1; + repeated bytes tokens = 2; + repeated string contracts = 3; + repeated Attribute static_att = 4; + ChangeType change = 5; +} + +message BalanceChange { + bytes token = 1; + bytes balance = 2; + bytes component_id = 3; +} diff --git a/proto/tycho/evm/v1/entity.proto b/proto/tycho/evm/v1/entity.proto new file mode 100644 index 0000000..7a940ea --- /dev/null +++ b/proto/tycho/evm/v1/entity.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package tycho.evm.v1; + +import "tycho/evm/v1/common.proto"; + +message EntityChanges { + string component_id = 1; + repeated Attribute attributes = 2; +} + +message TransactionEntityChanges { + Transaction tx = 1; + repeated EntityChanges entity_changes = 2; + repeated ProtocolComponent component_changes = 3; + repeated BalanceChange balance_changes = 4; +} + +message BlockEntityChanges { + Block block = 1; + repeated TransactionEntityChanges changes = 2; +} diff --git a/proto/tycho/evm/v1/vm.proto b/proto/tycho/evm/v1/vm.proto new file mode 100644 index 0000000..8305f69 --- /dev/null +++ b/proto/tycho/evm/v1/vm.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package tycho.evm.v1; + +import "tycho/evm/v1/common.proto"; + +message ContractSlot { + bytes slot = 2; + bytes value = 3; +} + +message ContractChange { + bytes address = 1; + // empty bytes indicates no change + bytes balance = 2; + // empty bytes indicates no change + bytes code = 3; + // empty sequence indicates no change + repeated ContractSlot slots = 4; + // Whether this is an update, creation or deletion + ChangeType change = 5; +} + +message TransactionContractChanges { + Transaction tx = 1; + repeated ContractChange contract_changes = 2; + repeated ProtocolComponent component_changes = 3; + repeated BalanceChange balance_changes = 4; +} + +message BlockContractChanges { + Block block = 1; + repeated TransactionContractChanges changes = 2; +} diff --git a/substreams/.gitkeep b/substreams/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/substreams/Readme.md b/substreams/Readme.md new file mode 100644 index 0000000..0940442 --- /dev/null +++ b/substreams/Readme.md @@ -0,0 +1,95 @@ +# Subtreams packages + +This directory contains all substream packages that are used by the extractors to access certain data from diffrent +blockchains. + +## Adding a new package + +To add a new package add folder. The naming convention is `[CHAIN]-[PROTOCOL_SYSTEM]`. In this new folder add a manifest +file `substreams.yaml`. You can use the template below to get started: + +```yaml +specVersion: v0.1.0 +package: + name: 'substreams_[CHAIN]_[PROTOCOL_SYSTEM]' + version: v0.1.0 + +protobuf: + files: + - vm.proto + - common.proto + importPaths: + # This is different compared to the substreams example, + # we need to share protobuf definitions with tycho you + # are invited to reuse existing definitions if they are + # useful to you. + - ../../proto/evm/v1 + # any private message types only used in internal modules + # can remain local to the crate. + - ./proto + +binaries: + default: + type: wasm/rust-v1 + # this points to the workspace target directory we use a special + # substreams build profile to optimise wasm binaries + file: ../../target/wasm32-unknown-unknown/substreams/substreams_[CHAIN]_[PROTOCOL_SYSTEM].wasm + +modules: + # sample module provides access to blocks. + - name: map_block + kind: map + inputs: + - source: sf.ethereum.type.v2.Block + output: + type: proto:acme.block_meta.v1.BlockMeta +``` + +Substreams packages are Rust crates so we also need a `cargo.toml`. +The example from the official docs will serve us just well: + +```toml +[package] +name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]" +version = "0.1.0" +edition = "2021" + +[lib] +name = "substreams_[CHAIN]_[PROTOCOL_SYSTEM]" +crate-type = ["cdylib"] + +[dependencies] +substreams = "0.5" +substreams-ethereum = "0.9" +prost = "0.11" + +``` + +Now we can generate the Rust protobuf code: + +``` +substreams protogen substreams.yaml --exclude-paths="sf/substreams,google" +``` + +The command above should put the generate rust files under `/src/pb`. You +can start using these now in your module handlers: See +the [official substreams documentation](https://thegraph.com/docs/en/substreams/getting-started/quickstart/#create-substreams-module-handlers) +on +how to implement module handlers. + +You can also look into already existing substreams packages to see how it +is done. E.g. [ethereum-ambient](./ethereum-ambient/) provides a pretty good +example of how to get access to raw contract storage. + +# Tests + +To create a block test asset for ethereum do the following: + +- Follow [this tutorial](https://substreams.streamingfast.io/tutorials/overview/map_block_meta_module). Make sure you + set up the substreams-explorer repo in the same directory as this repo. + - Comment out `image: ./ethereum.png` in `ethereum-explorer/substreams.yaml` + - Add `prost-types = "0.11.0"` to `ethereum-explorer/Cargo.toml` +- Make sure you set up your key env vars. +- Run `sh scripts/download-ethereum-block-to-s3 BLOCK_NUMBER` + +Do not commit the block files (they are quite big). \ No newline at end of file diff --git a/substreams/ethereum-ambient/Cargo.lock b/substreams/ethereum-ambient/Cargo.lock new file mode 100644 index 0000000..31a9d60 --- /dev/null +++ b/substreams/ethereum-ambient/Cargo.lock @@ -0,0 +1,1095 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "ethabi" +version = "17.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +dependencies = [ + "ethereum-types", + "hex", + "once_cell", + "regex", + "serde", + "serde_json", + "sha3", + "thiserror", + "uint", +] + +[[package]] +name = "ethbloom" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +dependencies = [ + "crunchy", + "fixed-hash", + "impl-rlp", + "impl-serde", + "tiny-keccak", +] + +[[package]] +name = "ethereum-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" +dependencies = [ + "ethbloom", + "fixed-hash", + "impl-rlp", + "impl-serde", + "primitive-types", + "uint", +] + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" + +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-rlp" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +dependencies = [ + "rlp", +] + +[[package]] +name = "impl-serde" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.148" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" + +[[package]] +name = "linux-raw-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memchr" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "pad" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "parity-scale-codec" +version = "3.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dec8a8073036902368c2cdc0387e85ff9a37054d7e7c98e592145e0c92cd4fb" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312270ee71e1cd70289dacf597cab7b207aa107d2f28191c2ae45b2ece18a260" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "primitive-types" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-rlp", + "impl-serde", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + +[[package]] +name = "rlp" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" +dependencies = [ + "bytes", + "rustc-hex", +] + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustix" +version = "0.38.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "substreams" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af972e374502cdfc9998132f5343848d1c58f27a295dc061a89804371f408a46" +dependencies = [ + "anyhow", + "bigdecimal", + "hex", + "hex-literal 0.3.4", + "num-bigint", + "num-traits", + "pad", + "prost", + "prost-build", + "prost-types", + "substreams-macro", + "thiserror", +] + +[[package]] +name = "substreams-ethereum" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78effc18ed321399fe15ec082806e96a58d213f79741d078c1cd26dd6dd53025" +dependencies = [ + "getrandom", + "num-bigint", + "substreams", + "substreams-ethereum-abigen", + "substreams-ethereum-core", + "substreams-ethereum-derive", +] + +[[package]] +name = "substreams-ethereum-abigen" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a176f39a6e09553c17a287edacd1854e5686fd20ffea3c9655dfc44d94b35e" +dependencies = [ + "anyhow", + "ethabi", + "heck", + "hex", + "prettyplease", + "proc-macro2", + "quote", + "substreams-ethereum-core", + "syn 1.0.109", +] + +[[package]] +name = "substreams-ethereum-ambient" +version = "0.3.0" +dependencies = [ + "hex-literal 0.4.1", + "prost", + "substreams", + "substreams-ethereum", +] + +[[package]] +name = "substreams-ethereum-core" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4700cfe408b75634a3c6b3a0caf7bddba4879601d2085c811485ea54cbde2d" +dependencies = [ + "bigdecimal", + "ethabi", + "getrandom", + "num-bigint", + "prost", + "prost-build", + "prost-types", + "substreams", +] + +[[package]] +name = "substreams-ethereum-derive" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d6d278d926fe3f0775d996ee2b5e1dc822c1b4bf4f7bf07c7fbb5bce6c79a9" +dependencies = [ + "ethabi", + "heck", + "hex", + "num-bigint", + "proc-macro2", + "quote", + "substreams-ethereum-abigen", + "syn 1.0.109", +] + +[[package]] +name = "substreams-macro" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6521ccd011a4c3f52cd3c31fc7400733e4feba2094e0e0e6354adca25b2b3f37" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "thiserror", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "uint" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "winnow" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +dependencies = [ + "memchr", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] diff --git a/substreams/ethereum-ambient/Cargo.toml b/substreams/ethereum-ambient/Cargo.toml new file mode 100644 index 0000000..f7d246b --- /dev/null +++ b/substreams/ethereum-ambient/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "substreams-ethereum-ambient" +version = "0.3.0" +edition = "2021" + +[lib] +name = "substreams_ethereum_ambient" +crate-type = ["cdylib"] + +[dependencies] +substreams = "0.5" +substreams-ethereum = "0.9" +prost = "0.11" +hex-literal = "0.4.1" +ethabi = "18.0.0" +hex = "0.4.2" +bytes = "1.5.0" +anyhow = "1.0.75" diff --git a/substreams/ethereum-ambient/src/lib.rs b/substreams/ethereum-ambient/src/lib.rs new file mode 100644 index 0000000..f8f9cb7 --- /dev/null +++ b/substreams/ethereum-ambient/src/lib.rs @@ -0,0 +1,389 @@ +use std::collections::{hash_map::Entry, HashMap}; + +use anyhow::{anyhow, bail}; +use ethabi::{decode, ParamType}; +use hex_literal::hex; +use substreams_ethereum::pb::eth::{self}; + +use pb::tycho::evm::v1::{self as tycho, ChangeType}; + +mod pb; + +const AMBIENT_CONTRACT: [u8; 20] = hex!("aaaaaaaaa24eeeb8d57d431224f73832bc34f688"); +const INIT_POOL_CODE: u8 = 71; +const USER_CMD_FN_SIG: [u8; 4] = [0xA1, 0x51, 0x12, 0xF9]; + +struct SlotValue { + new_value: Vec, + start_value: Vec, +} + +impl SlotValue { + fn has_changed(&self) -> bool { + self.start_value != self.new_value + } +} + +// uses a map for slots, protobuf does not +// allow bytes in hashmap keys +struct InterimContractChange { + address: Vec, + balance: Vec, + code: Vec, + slots: HashMap, SlotValue>, + change: tycho::ChangeType, +} + +impl From for tycho::ContractChange { + fn from(value: InterimContractChange) -> Self { + tycho::ContractChange { + address: value.address, + balance: value.balance, + code: value.code, + slots: value + .slots + .into_iter() + .filter(|(_, value)| value.has_changed()) + .map(|(slot, value)| tycho::ContractSlot { slot, value: value.new_value }) + .collect(), + change: value.change.into(), + } + } +} + +/// Extracts all contract changes relevant to vm simulations +/// +/// This implementation has currently two major limitations: +/// 1. It is hardwired to only care about changes to the ambient main contract, this is ok for this +/// particular use case but for a more general purpose implementation this is not ideal +/// 2. Changes are processed separately, this means that if there are any side effects between each +/// other (e.g. if account is deleted and then created again in ethereum all the storage is set +/// to 0. So there is a side effect between account creation and contract storage.) these might +/// not be properly accounted for. Most of the time this should not be a major issue but may lead +/// to wrong results so consume this implementation with care. See example below for a concrete +/// case where this is problematic. +/// +/// ## A very contrived example: +/// 1. Some existing contract receives a transaction that changes it state, the state is updated +/// 2. Next, this contract has self destruct called on itself +/// 3. The contract is created again using CREATE2 at the same address +/// 4. The contract receives a transaction that changes it state +/// 5. We would emit this as as contract creation with slots set from 1 and from 4, although we +/// should only emit the slots changed from 4. +#[substreams::handlers::map] +fn map_changes( + block: eth::v2::Block, +) -> Result { + let mut block_changes = tycho::BlockContractChanges { block: None, changes: Vec::new() }; + + let mut tx_change = tycho::TransactionContractChanges::default(); + + let mut changed_contracts: HashMap, InterimContractChange> = HashMap::new(); + + let created_accounts: HashMap<_, _> = block + .transactions() + .flat_map(|tx| { + tx.calls.iter().flat_map(|call| { + call.account_creations + .iter() + .map(|ac| (&ac.account, ac.ordinal)) + }) + }) + .collect(); + + for block_tx in block.transactions() { + // extract storage changes + let mut storage_changes = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .flat_map(|call| { + call.storage_changes + .iter() + .filter(|c| c.address == AMBIENT_CONTRACT) + }) + .collect::>(); + storage_changes.sort_unstable_by_key(|change| change.ordinal); + + let ambient_calls = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .filter(|call| call.address == AMBIENT_CONTRACT) + .collect::>(); + + for call in ambient_calls { + if call.input.len() < 4 { + continue; + } + if call.input[0..4] == USER_CMD_FN_SIG { + let user_cmd_external_abi_types = &[ + // index of the proxy sidecar the command is being called on + ParamType::Uint(16), + // call data for internal UserCmd method + ParamType::Bytes, + ]; + let user_cmd_internal_abi_types = &[ + ParamType::Uint(8), // command + ParamType::Address, // base + ParamType::Address, // quote + ParamType::Uint(256), // pool index + ParamType::Uint(128), // price + ]; + + // Decode external call to UserCmd + if let Ok(external_params) = decode(user_cmd_external_abi_types, &call.input[4..]) { + let cmd_bytes = external_params[1] + .to_owned() + .into_bytes() + .ok_or_else(|| { + anyhow!("Failed to convert to bytes: {:?}", &external_params[1]) + })?; + + // Call data is structured differently depending on the cmd code, so only + // decode if this is an init pool code. + if cmd_bytes[31] == INIT_POOL_CODE { + // Decode internal call to UserCmd + if let Ok(internal_params) = decode(user_cmd_internal_abi_types, &cmd_bytes) + { + let base = internal_params[1] + .to_owned() + .into_address() + .ok_or_else(|| { + anyhow!( + "Failed to convert to address: {:?}", + &internal_params[1] + ) + })? + .to_fixed_bytes() + .to_vec(); + + let quote = internal_params[2] + .to_owned() + .into_address() + .ok_or_else(|| { + anyhow!( + "Failed to convert to address: {:?}", + &internal_params[2] + ) + })? + .to_fixed_bytes() + .to_vec(); + + let pool_index = internal_params[3] + .to_owned() + .into_uint() + .ok_or_else(|| anyhow!("Failed to convert to u32".to_string()))? + .as_u32(); + + let static_attribute = tycho::Attribute { + name: String::from("pool_index"), + value: pool_index.to_be_bytes().to_vec(), + change: ChangeType::Creation.into(), + }; + + let mut tokens: Vec> = vec![base.clone(), quote.clone()]; + tokens.sort(); + + let new_component = tycho::ProtocolComponent { + id: format!( + "{}{}{}", + hex::encode(base.clone()), + hex::encode(quote.clone()), + pool_index + ), + tokens, + contracts: vec![hex::encode(AMBIENT_CONTRACT)], + static_att: vec![static_attribute], + change: ChangeType::Creation.into(), + }; + tx_change + .component_changes + .push(new_component); + } else { + bail!("Failed to decode ABI internal call.".to_string()); + } + } + } else { + bail!("Failed to decode ABI external call.".to_string()); + } + } + } + + // Note: some contracts change slot values and change them back to their + // original value before the transactions ends we remember the initial + // value before the first change and in the end filter found deltas + // that ended up not actually changing anything. + for storage_change in storage_changes.iter() { + match changed_contracts.entry(storage_change.address.clone()) { + // We have already an entry recording a change about this contract + // only append the change about this storage slot + Entry::Occupied(mut e) => { + let contract_change = e.get_mut(); + match contract_change + .slots + .entry(storage_change.key.clone()) + { + // The storage slot was already changed before, simply + // update new_value + Entry::Occupied(mut v) => { + let slot_value = v.get_mut(); + slot_value + .new_value + .copy_from_slice(&storage_change.new_value); + } + // The storage slots is being initialised for the first time + Entry::Vacant(v) => { + v.insert(SlotValue { + new_value: storage_change.new_value.clone(), + start_value: storage_change.old_value.clone(), + }); + } + } + } + // Intialise a new contract change after obsering a storage change + Entry::Vacant(e) => { + let mut slots = HashMap::new(); + slots.insert( + storage_change.key.clone(), + SlotValue { + new_value: storage_change.new_value.clone(), + start_value: storage_change.old_value.clone(), + }, + ); + e.insert(InterimContractChange { + address: storage_change.address.clone(), + balance: Vec::new(), + code: Vec::new(), + slots, + change: if created_accounts.contains_key(&storage_change.address) { + ChangeType::Creation + } else { + ChangeType::Update + }, + }); + } + } + } + + // extract balance changes + let mut balance_changes = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .flat_map(|call| { + call.balance_changes + .iter() + .filter(|c| c.address == AMBIENT_CONTRACT) + }) + .collect::>(); + balance_changes.sort_unstable_by_key(|change| change.ordinal); + + for balance_change in balance_changes.iter() { + match changed_contracts.entry(balance_change.address.clone()) { + Entry::Occupied(mut e) => { + let contract_change = e.get_mut(); + if let Some(new_balance) = &balance_change.new_value { + contract_change.balance.clear(); + contract_change + .balance + .extend_from_slice(&new_balance.bytes); + } + } + Entry::Vacant(e) => { + if let Some(new_balance) = &balance_change.new_value { + e.insert(InterimContractChange { + address: balance_change.address.clone(), + balance: new_balance.bytes.clone(), + code: Vec::new(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&balance_change.address) { + ChangeType::Creation + } else { + ChangeType::Update + }, + }); + } + } + } + } + + // extract code changes + let mut code_changes = block_tx + .calls + .iter() + .filter(|call| !call.state_reverted) + .flat_map(|call| { + call.code_changes + .iter() + .filter(|c| c.address == AMBIENT_CONTRACT) + }) + .collect::>(); + code_changes.sort_unstable_by_key(|change| change.ordinal); + + for code_change in code_changes.iter() { + match changed_contracts.entry(code_change.address.clone()) { + Entry::Occupied(mut e) => { + let contract_change = e.get_mut(); + contract_change.code.clear(); + contract_change + .code + .extend_from_slice(&code_change.new_code); + } + Entry::Vacant(e) => { + e.insert(InterimContractChange { + address: code_change.address.clone(), + balance: Vec::new(), + code: code_change.new_code.clone(), + slots: HashMap::new(), + change: if created_accounts.contains_key(&code_change.address) { + ChangeType::Creation + } else { + ChangeType::Update + }, + }); + } + } + } + + // if there were any changes, add transaction and push the changes + if !storage_changes.is_empty() || !balance_changes.is_empty() || !code_changes.is_empty() { + tx_change.tx = Some(tycho::Transaction { + hash: block_tx.hash.clone(), + from: block_tx.from.clone(), + to: block_tx.to.clone(), + index: block_tx.index as u64, + }); + + // reuse changed_contracts hash map by draining it, next iteration + // will start empty. This avoids a costly reallocation + for (_, change) in changed_contracts.drain() { + tx_change + .contract_changes + .push(change.into()) + } + + block_changes + .changes + .push(tx_change.clone()); + + // clear out the interim contract changes after we pushed those. + tx_change.tx = None; + tx_change.contract_changes.clear(); + } + } + + block_changes.block = Some(tycho::Block { + number: block.number, + hash: block.hash.clone(), + parent_hash: block + .header + .as_ref() + .expect("Block header not present") + .parent_hash + .clone(), + ts: block.timestamp_seconds(), + }); + + Ok(block_changes) +} diff --git a/substreams/ethereum-ambient/src/pb/mod.rs b/substreams/ethereum-ambient/src/pb/mod.rs new file mode 100644 index 0000000..43d8838 --- /dev/null +++ b/substreams/ethereum-ambient/src/pb/mod.rs @@ -0,0 +1,10 @@ +// @generated +pub mod tycho { + pub mod evm { + // @@protoc_insertion_point(attribute:tycho.evm.v1) + pub mod v1 { + include!("tycho.evm.v1.rs"); + // @@protoc_insertion_point(tycho.evm.v1) + } + } +} diff --git a/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs b/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs new file mode 100644 index 0000000..f59fcf2 --- /dev/null +++ b/substreams/ethereum-ambient/src/pb/tycho.evm.v1.rs @@ -0,0 +1,166 @@ +// @generated +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Block { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub parent_hash: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="3")] + pub number: u64, + #[prost(uint64, tag="4")] + pub ts: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(bytes="vec", tag="1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub from: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub to: ::prost::alloc::vec::Vec, + #[prost(uint64, tag="4")] + pub index: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Attribute { + #[prost(string, tag="1")] + pub name: ::prost::alloc::string::String, + #[prost(bytes="vec", tag="2")] + pub value: ::prost::alloc::vec::Vec, + #[prost(enumeration="ChangeType", tag="3")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProtocolComponent { + #[prost(string, tag="1")] + pub id: ::prost::alloc::string::String, + #[prost(bytes="vec", repeated, tag="2")] + pub tokens: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + #[prost(string, repeated, tag="3")] + pub contracts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag="4")] + pub static_att: ::prost::alloc::vec::Vec, + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BalanceChange { + #[prost(bytes="vec", tag="1")] + pub token: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub component_id: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ChangeType { + Unspecified = 0, + Update = 1, + Creation = 2, + Deletion = 3, +} +impl ChangeType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ChangeType::Unspecified => "CHANGE_TYPE_UNSPECIFIED", + ChangeType::Update => "CHANGE_TYPE_UPDATE", + ChangeType::Creation => "CHANGE_TYPE_CREATION", + ChangeType::Deletion => "CHANGE_TYPE_DELETION", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CHANGE_TYPE_UNSPECIFIED" => Some(Self::Unspecified), + "CHANGE_TYPE_UPDATE" => Some(Self::Update), + "CHANGE_TYPE_CREATION" => Some(Self::Creation), + "CHANGE_TYPE_DELETION" => Some(Self::Deletion), + _ => None, + } + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EntityChanges { + #[prost(string, tag="1")] + pub component_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag="2")] + pub attributes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionEntityChanges { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub entity_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockEntityChanges { + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractSlot { + #[prost(bytes="vec", tag="2")] + pub slot: ::prost::alloc::vec::Vec, + #[prost(bytes="vec", tag="3")] + pub value: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContractChange { + #[prost(bytes="vec", tag="1")] + pub address: ::prost::alloc::vec::Vec, + /// empty bytes indicates no change + #[prost(bytes="vec", tag="2")] + pub balance: ::prost::alloc::vec::Vec, + /// empty bytes indicates no change + #[prost(bytes="vec", tag="3")] + pub code: ::prost::alloc::vec::Vec, + /// empty sequence indicates no change + #[prost(message, repeated, tag="4")] + pub slots: ::prost::alloc::vec::Vec, + /// Whether this is an update, creation or deletion + #[prost(enumeration="ChangeType", tag="5")] + pub change: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionContractChanges { + #[prost(message, optional, tag="1")] + pub tx: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub contract_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="3")] + pub component_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="4")] + pub balance_changes: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockContractChanges { + #[prost(message, optional, tag="1")] + pub block: ::core::option::Option, + #[prost(message, repeated, tag="2")] + pub changes: ::prost::alloc::vec::Vec, +} +// @@protoc_insertion_point(module) diff --git a/substreams/ethereum-ambient/substreams.yaml b/substreams/ethereum-ambient/substreams.yaml new file mode 100644 index 0000000..8007c2d --- /dev/null +++ b/substreams/ethereum-ambient/substreams.yaml @@ -0,0 +1,24 @@ +specVersion: v0.1.0 +package: + name: "substreams_ethereum_ambient" + version: v0.3.0 + +protobuf: + files: + - vm.proto + - common.proto + importPaths: + - ../../proto/tycho/evm/v1/ + +binaries: + default: + type: wasm/rust-v1 + file: ../../target/wasm32-unknown-unknown/substreams/substreams_ethereum_ambient.wasm + +modules: + - name: map_changes + kind: map + inputs: + - source: sf.ethereum.type.v2.Block + output: + type: proto:tycho.evm.state.v1.BlockContractChanges