From 2b104ff1d5fc2474e34d644194b5ff1e34159021 Mon Sep 17 00:00:00 2001 From: colindickson Date: Wed, 1 Nov 2023 10:18:13 -0400 Subject: [PATCH] solana-explorer: add map_filter_transactions module --- solana-explorer/README.md | 85 +++++++++++++++++-- .../proto/{block_meta.proto => block.proto} | 2 +- .../proto/program_instructions.proto | 16 ---- solana-explorer/proto/transactions.proto | 24 ++++++ solana-explorer/src/lib.rs | 1 + solana-explorer/src/map_block_meta.rs | 2 +- .../src/map_filter_instructions.rs | 13 +-- .../src/map_filter_transactions.rs | 69 +++++++++++++++ solana-explorer/src/pb/mod.rs | 16 ++-- .../{sol.block_meta.v1.rs => sol.block.v1.rs} | 0 ....programs.v1.rs => sol.transactions.v1.rs} | 18 +++- solana-explorer/substreams.yaml | 17 +++- 12 files changed, 214 insertions(+), 49 deletions(-) rename solana-explorer/proto/{block_meta.proto => block.proto} (79%) delete mode 100644 solana-explorer/proto/program_instructions.proto create mode 100644 solana-explorer/proto/transactions.proto create mode 100644 solana-explorer/src/map_filter_transactions.rs rename solana-explorer/src/pb/{sol.block_meta.v1.rs => sol.block.v1.rs} (100%) rename solana-explorer/src/pb/{sol.programs.v1.rs => sol.transactions.v1.rs} (54%) diff --git a/solana-explorer/README.md b/solana-explorer/README.md index 1a52945..bb92355 100644 --- a/solana-explorer/README.md +++ b/solana-explorer/README.md @@ -87,8 +87,6 @@ params: "SysvarRent111111111111111111111111111111111" ], "data": "1111jgvBfvfPuc2GLifVk9vJpXFTL5urtkjvyJHP6p7TqNR9G47jkYe3RGab9rya96yeGycaY8SVbcpaBkRLZonTJL7y4GyDWwyEmYmvNz74HWew5cPsky18ppTT1WyqmjxvmznzUWXfRogakvUscqJ8C54tP", - "slotNumber": "153000012", - "blockHash": "EtMvzAbrkdWpawfP7ozcL86yocFNYxvWvtS4f8pMrajD" }, { "programId": "Stake11111111111111111111111111111111111111", @@ -101,8 +99,6 @@ params: "GJYanGsBPyRUgAYkXBTyBYdZiH5wppXoJys9HaJwitzk" ], "data": "3xyZh", - "slotNumber": "153000012", - "blockHash": "EtMvzAbrkdWpawfP7ozcL86yocFNYxvWvtS4f8pMrajD" } ] } @@ -138,8 +134,6 @@ params: "4sUsH6ZzqWU2g1xzM8udFErB4qN3GSL1QbNjiSHc3QxN" ], "data": "8QwQj", - "slotNumber": "153000028", - "blockHash": "5xj9iYo6nuAiXvPFDUieUKBcRLbg9rz5qiBQg2NYjyBv" } ] } @@ -167,4 +161,81 @@ params: ----------- BLOCK #153,000,048 (2E8ncPZMAqa5RKPvEUAMt6abLNcDZM7J6nawyaiSboGD) --------------- ----------- BLOCK #153,000,049 (GyG9Rw4Mo8sYb9WLzz8oqHgV3k4sYa3UWeveGG2uLGWX) --------------- all done -``` \ No newline at end of file +``` + +### Running the "map_filter_transactions" Module + +To run this module, you must provide a transaction signature value in the parameters in your substreams.yaml manifest. + +This example will use the default `21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t` program. + +```yaml +params: + map_filter_transactions: "signature=21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t" +``` + +```bash +substreams run -e mainnet.sol.streamingfast.io:443 ./solana-explorer-v0.1.0.spkg map_filter_transactions -s 153000000 -t +30 +Connected (trace ID 27d4c57714280d824e0cd37f7cd8dc17) +Progress messages received: 0 (0/sec) +Backprocessing history up to requested target block 153000028: +(hit 'm' to switch mode) +----------- BLOCK #153,000,000 (ENZNjBKCxXQqgFxX63T6kn7DJwc7fQiwy3W3vyehtEGA) --------------- +----------- BLOCK #153,000,001 (8q2n7y3ozKWUWrAGVbNc3wrkxdmT6iSdgf8CHgqrvSv1) --------------- +----------- BLOCK #153,000,002 (AfE4vBBsvcCXzdexP4sX9byHYrdZsUpd2GTTTTvkShxu) --------------- +----------- BLOCK #153,000,003 (H5RZGAShGm7sMW6NvaPY3TsX36WEvugZ64Y1gSBRPyLV) --------------- +----------- BLOCK #153,000,004 (Egf4PFv5636rycdntsBfi26Bou2Mt3Z5UAgH4hBbkZat) --------------- +----------- BLOCK #153,000,005 (DmkkhPG2RDWkJnLbSiG1XTUotJVpLbjqBeRa9FoQB3Tg) --------------- +----------- BLOCK #153,000,006 (Dy9YQQrTed69j7fsy7z3F3sa1x34T4CGmCVQVFoWobRE) --------------- +----------- BLOCK #153,000,007 (3bHscyCgsvzLo6u4CpvCzJe7ELkuf9k8MSwHyLUtif9L) --------------- +----------- BLOCK #153,000,008 (CkDjN7c4hjYqtuYXH9Vw8MTmk29XjgMnbEtAzrjbyjnU) --------------- +----------- BLOCK #153,000,009 (8SVnGrwbwpAWCPdRXKgyNCvJRedEqmYnLazB3mB7Rehg) --------------- +----------- BLOCK #153,000,010 (BTVDWyLK26P2CBjGjCvsHwNqaUcn15xUauwCJmkCSAx1) --------------- +----------- BLOCK #153,000,011 (DVzw63xEAHmWTdo9iA9ca7yTiaondmGGvr4VczQnFhbn) --------------- +----------- BLOCK #153,000,012 (EtMvzAbrkdWpawfP7ozcL86yocFNYxvWvtS4f8pMrajD) --------------- +----------- BLOCK #153,000,013 (7NVSAZZefTrNkK7JSg227yoponB6Tig27PXnEm7kQFJf) --------------- +----------- BLOCK #153,000,014 (3nfE81F85ggdWxjRZUENBPhvXStAJsVxnwrQ99VcTMME) --------------- +----------- BLOCK #153,000,015 (4N3sqbbJEZobfu1pL6rZEZf6Kpx2WYB8fHm1hbqASa6S) --------------- +----------- BLOCK #153,000,016 (812uX2UUSvvep5APemSmTDk4bDnMkZnJtMBSa6enBmFN) --------------- +----------- BLOCK #153,000,017 (CXsKxq7LmJVW6P3YjBVbrfgDiyMyzYDX5jNhgdsryMfv) --------------- +----------- BLOCK #153,000,018 (FnVqabMm8VEKCr9pchMFxipxSssNeioxx7bCaqyGbzgX) --------------- +----------- BLOCK #153,000,019 (J49iZE979hGtK4hnxUigpxQiNsU3oNq9UMMJqknGLJVT) --------------- +----------- BLOCK #153,000,020 (GJ1BNqsBtyxucvqkipkZDaHW2KGdsU3sR41ebmCxth24) --------------- +----------- BLOCK #153,000,021 (Bxc99wYm1heyzfY1ViXNnJFmf81BFHzLKrwizW5TPemn) --------------- +----------- BLOCK #153,000,022 (2iLweM1VHWnkmXpF7G2fHr52zGuwsEjFXd8ZtpwwUcDw) --------------- +----------- BLOCK #153,000,023 (6JpfNdC2qTErHpXd8e74E5aAkm11USA2rJJbCvQdtHyg) --------------- +----------- BLOCK #153,000,024 (AjN8rk6c8ya5TNYRBUWKUgEYEJceqtJsPMMDdweg4MPR) --------------- +----------- BLOCK #153,000,025 (F7dvT2mwGusrLmRwdJrGZ7cAXsxq43kHsrKpzeseywwg) --------------- +----------- BLOCK #153,000,026 (aGdXha9hhhaETuNywV2XY39Y6B719NBZ6xbUzAu3wah) --------------- +----------- BLOCK #153,000,027 (GfoPvdzekjLV1QHvvHzyVG7xbjajDKA4Ev7gXum2Ry1S) --------------- +----------- BLOCK #153,000,028 (5xj9iYo6nuAiXvPFDUieUKBcRLbg9rz5qiBQg2NYjyBv) --------------- +{ + "@module": "map_filter_transactions", + "@block": 153000028, + "@type": "sol.transactions.v1.Transactions", + "@data": { + "transactions": [ + { + "signatures": [ + "21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t" + ], + "instructions": [ + { + "programId": "Vote111111111111111111111111111111111111111", + "accounts": [ + "HRfK8kbqCaKYsHk3R8HCtLNDp4aTneq1eSQ9ZrA2Kb2q", + "SysvarS1otHashes111111111111111111111111111", + "SysvarC1ock11111111111111111111111111111111", + "HEJzPiLSg9ty5CGAyJm5y7ef1NPzr1u1aHLYaAMXsgH9" + ], + "data": "29z5mr1JoRmJYQ6yybpMZZDfJB6dWDGQsB8jimJo3eTNYA7qEJvjo3dwvmP25dQtpxkKPQdEy37P5hDAMoAtGnaE1vGpXq" + } + ] + } + ] + } +} + +----------- BLOCK #153,000,029 (3LBYvzhWfdtcAwuCvoBzTCkUbpVbckyBFAcJezdScGhP) --------------- +all done +``` diff --git a/solana-explorer/proto/block_meta.proto b/solana-explorer/proto/block.proto similarity index 79% rename from solana-explorer/proto/block_meta.proto rename to solana-explorer/proto/block.proto index 8e4a7ce..81cb0bc 100644 --- a/solana-explorer/proto/block_meta.proto +++ b/solana-explorer/proto/block.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package sol.block_meta.v1; +package sol.block.v1; message BlockMeta { uint64 slot = 1; diff --git a/solana-explorer/proto/program_instructions.proto b/solana-explorer/proto/program_instructions.proto deleted file mode 100644 index 6141c2b..0000000 --- a/solana-explorer/proto/program_instructions.proto +++ /dev/null @@ -1,16 +0,0 @@ -syntax = "proto3"; - -package sol.programs.v1; - -message Instructions { - repeated Instruction instructions = 1; -} - -message Instruction { - string program_id = 1; - repeated string accounts = 2; - string data = 3; - - uint64 slot_number = 4; - string block_hash = 5; -} diff --git a/solana-explorer/proto/transactions.proto b/solana-explorer/proto/transactions.proto new file mode 100644 index 0000000..ff3ce04 --- /dev/null +++ b/solana-explorer/proto/transactions.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package sol.transactions.v1; + + +message Instructions { + repeated Instruction instructions = 1; +} + +message Instruction { + string program_id = 1; + repeated string accounts = 2; + string data = 3; +} + +message Transactions { + repeated Transaction transactions = 1; +} + +message Transaction { + repeated string signatures = 1; + + repeated Instruction instructions = 2; +} \ No newline at end of file diff --git a/solana-explorer/src/lib.rs b/solana-explorer/src/lib.rs index 76a8f1c..9b6f90e 100644 --- a/solana-explorer/src/lib.rs +++ b/solana-explorer/src/lib.rs @@ -2,3 +2,4 @@ mod pb; mod map_block_full; mod map_block_meta; mod map_filter_instructions; +mod map_filter_transactions; diff --git a/solana-explorer/src/map_block_meta.rs b/solana-explorer/src/map_block_meta.rs index 53ab91a..9bc6d4f 100644 --- a/solana-explorer/src/map_block_meta.rs +++ b/solana-explorer/src/map_block_meta.rs @@ -1,5 +1,5 @@ use substreams_solana::pb::sf::solana::r#type::v1::Block; -use crate::pb::sol::block_meta::v1::BlockMeta; +use crate::pb::sol::block::v1::BlockMeta; #[substreams::handlers::map] fn map_block_meta(blk: Block) -> Result { diff --git a/solana-explorer/src/map_filter_instructions.rs b/solana-explorer/src/map_filter_instructions.rs index 69e86c8..bc516ae 100644 --- a/solana-explorer/src/map_filter_instructions.rs +++ b/solana-explorer/src/map_filter_instructions.rs @@ -1,14 +1,13 @@ use anyhow::anyhow; use serde::Deserialize; use substreams_solana::pb::sf::solana::r#type::v1::{Block, CompiledInstruction}; -use crate::pb::sol::programs::v1::{Instruction, Instructions}; +use crate::pb::sol::transactions::v1::{Instruction, Instructions}; #[derive(Deserialize, Debug)] -struct TransactionFilterParams { +struct InstructionFilterParams { program_id: Option, } - #[substreams::handlers::map] fn map_filter_instructions(params: String, blk: Block) -> Result> { let filters = parse_filters_from_params(params)?; @@ -17,14 +16,11 @@ fn map_filter_instructions(params: String, blk: Block) -> Result = msg.instructions.iter() .filter(|inst| apply_filter(inst, &filters, acct_keys.to_vec())) .map(|inst| { Instruction { - slot_number: blk.slot, - block_hash: blk.blockhash.clone(), program_id: bs58::encode(acct_keys[inst.program_id_index as usize].to_vec()).into_string(), accounts: inst.accounts.iter().map(|acct| bs58::encode(acct_keys[*acct as usize].to_vec()).into_string()).collect(), data: bs58::encode(inst.data.clone()).into_string(), @@ -36,7 +32,7 @@ fn map_filter_instructions(params: String, blk: Block) -> Result Result> { +fn parse_filters_from_params(params: String) -> Result> { let parsed_result = serde_qs::from_str(¶ms); if parsed_result.is_err() { return Err(Vec::from([anyhow!("Unexpected error while parsing parameters")])); @@ -48,7 +44,7 @@ fn parse_filters_from_params(params: String) -> Result>) -> bool { +fn apply_filter(instruction: &CompiledInstruction, filters: &InstructionFilterParams, account_keys: Vec>) -> bool { if filters.program_id.is_none() { return true; } @@ -60,7 +56,6 @@ fn apply_filter(instruction: &CompiledInstruction, filters: &TransactionFilterPa } let program_account_key_val = bs58::encode(program_account_key.unwrap()).into_string(); - //check if account key value is equal to the program id if program_account_key_val != program_id_filter { return false; } diff --git a/solana-explorer/src/map_filter_transactions.rs b/solana-explorer/src/map_filter_transactions.rs new file mode 100644 index 0000000..672c142 --- /dev/null +++ b/solana-explorer/src/map_filter_transactions.rs @@ -0,0 +1,69 @@ +use anyhow::anyhow; +use serde::Deserialize; +use substreams_solana::pb::sf::solana::r#type::v1::{Block, ConfirmedTransaction}; +use crate::pb::sol::transactions::v1::{Instruction, Transaction, Transactions}; + +#[derive(Deserialize, Debug)] +struct TransactionFilterParams { + signature: Option, +} + +#[substreams::handlers::map] +fn map_filter_transactions(params: String, blk: Block) -> Result> { + let filters = parse_filters_from_params(params)?; + + let mut transactions : Vec = Vec::new(); + + blk.transactions.iter() + .filter(|tx| apply_filter(tx, &filters)) + .for_each(|tx| { + let msg = tx.transaction.clone().unwrap().message.unwrap(); + let acct_keys = msg.account_keys.as_slice(); + + let insts : Vec = msg.instructions.iter() + .map(|inst| { + Instruction { + program_id: bs58::encode(acct_keys[inst.program_id_index as usize].to_vec()).into_string(), + accounts: inst.accounts.iter().map(|acct| bs58::encode(acct_keys[*acct as usize].to_vec()).into_string()).collect(), + data: bs58::encode(inst.data.clone()).into_string(), + } + }).collect(); + + let t = Transaction { + signatures: tx.transaction.clone().unwrap().signatures.iter().map(|sig| bs58::encode(sig).into_string()).collect(), + instructions: insts + }; + transactions.push(t); + }); + + Ok(Transactions { transactions }) +} + +fn parse_filters_from_params(params: String) -> Result> { + let parsed_result = serde_qs::from_str(¶ms); + if parsed_result.is_err() { + return Err(Vec::from([anyhow!("Unexpected error while parsing parameters")])); + } + + let filters = parsed_result.unwrap(); + //todo: verify_filters(&filters)?; + + Ok(filters) +} + +fn apply_filter(transaction: &&ConfirmedTransaction, filters: &TransactionFilterParams) -> bool { + if filters.signature.is_none() { + return true; + } + + let mut found = false; + + transaction.transaction.as_ref().unwrap().signatures.iter().for_each(|sig| { + let xsig = bs58::encode(&sig).into_string(); + if xsig == filters.signature.clone().unwrap() { + found = true; + } + }); + + found +} \ No newline at end of file diff --git a/solana-explorer/src/pb/mod.rs b/solana-explorer/src/pb/mod.rs index aec8beb..7d6bd36 100644 --- a/solana-explorer/src/pb/mod.rs +++ b/solana-explorer/src/pb/mod.rs @@ -11,18 +11,18 @@ pub mod sf { } } pub mod sol { - pub mod block_meta { - // @@protoc_insertion_point(attribute:sol.block_meta.v1) + pub mod block { + // @@protoc_insertion_point(attribute:sol.block.v1) pub mod v1 { - include!("sol.block_meta.v1.rs"); - // @@protoc_insertion_point(sol.block_meta.v1) + include!("sol.block.v1.rs"); + // @@protoc_insertion_point(sol.block.v1) } } - pub mod programs { - // @@protoc_insertion_point(attribute:sol.programs.v1) + pub mod transactions { + // @@protoc_insertion_point(attribute:sol.transactions.v1) pub mod v1 { - include!("sol.programs.v1.rs"); - // @@protoc_insertion_point(sol.programs.v1) + include!("sol.transactions.v1.rs"); + // @@protoc_insertion_point(sol.transactions.v1) } } } diff --git a/solana-explorer/src/pb/sol.block_meta.v1.rs b/solana-explorer/src/pb/sol.block.v1.rs similarity index 100% rename from solana-explorer/src/pb/sol.block_meta.v1.rs rename to solana-explorer/src/pb/sol.block.v1.rs diff --git a/solana-explorer/src/pb/sol.programs.v1.rs b/solana-explorer/src/pb/sol.transactions.v1.rs similarity index 54% rename from solana-explorer/src/pb/sol.programs.v1.rs rename to solana-explorer/src/pb/sol.transactions.v1.rs index 8e8784f..f4c1817 100644 --- a/solana-explorer/src/pb/sol.programs.v1.rs +++ b/solana-explorer/src/pb/sol.transactions.v1.rs @@ -14,9 +14,19 @@ pub struct Instruction { pub accounts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, #[prost(string, tag="3")] pub data: ::prost::alloc::string::String, - #[prost(uint64, tag="4")] - pub slot_number: u64, - #[prost(string, tag="5")] - pub block_hash: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transactions { + #[prost(message, repeated, tag="1")] + pub transactions: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Transaction { + #[prost(string, repeated, tag="1")] + pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag="2")] + pub instructions: ::prost::alloc::vec::Vec, } // @@protoc_insertion_point(module) diff --git a/solana-explorer/substreams.yaml b/solana-explorer/substreams.yaml index 2d82e22..3aa6431 100644 --- a/solana-explorer/substreams.yaml +++ b/solana-explorer/substreams.yaml @@ -10,8 +10,8 @@ imports: protobuf: files: - - block_meta.proto - - program_instructions.proto + - block.proto + - transactions.proto importPaths: - ./proto @@ -45,9 +45,20 @@ modules: - params: string - source: sf.solana.type.v1.Block output: - type: proto:sol.programs.v1.Instructions + type: proto:sol.transactions.v1.Instructions doc: | `map_filter_instructions` allows you to get instructions executed by a certain `program_id`, and returns the raw Instruction data model. + - name: map_filter_transactions + kind: map + inputs: + - params: string + - source: sf.solana.type.v1.Block + output: + type: proto:sol.transactions.v1.Transactions + doc: | + `map_filter_transaction` allows you to find a transaction by hash. You need to scope your search to the blocks you know you're going to find that transaction. Check with your preferred block explorer first. + params: map_filter_instructions: "program_id=Stake11111111111111111111111111111111111111" + map_filter_transactions: "signature=21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t"