solana-explorer: add map_filter_transactions module

solana-explorer/README.md

@@ -87,8 +87,6 @@ params:
           "SysvarRent111111111111111111111111111111111"
         ],
         "data": "1111jgvBfvfPuc2GLifVk9vJpXFTL5urtkjvyJHP6p7TqNR9G47jkYe3RGab9rya96yeGycaY8SVbcpaBkRLZonTJL7y4GyDWwyEmYmvNz74HWew5cPsky18ppTT1WyqmjxvmznzUWXfRogakvUscqJ8C54tP",
-        "slotNumber": "153000012",
-        "blockHash": "EtMvzAbrkdWpawfP7ozcL86yocFNYxvWvtS4f8pMrajD"
       },
       {
         "programId": "Stake11111111111111111111111111111111111111",
@@ -101,8 +99,6 @@ params:
           "GJYanGsBPyRUgAYkXBTyBYdZiH5wppXoJys9HaJwitzk"
         ],
         "data": "3xyZh",
-        "slotNumber": "153000012",
-        "blockHash": "EtMvzAbrkdWpawfP7ozcL86yocFNYxvWvtS4f8pMrajD"
       }
     ]
   }
@@ -138,8 +134,6 @@ params:
           "4sUsH6ZzqWU2g1xzM8udFErB4qN3GSL1QbNjiSHc3QxN"
         ],
         "data": "8QwQj",
-        "slotNumber": "153000028",
-        "blockHash": "5xj9iYo6nuAiXvPFDUieUKBcRLbg9rz5qiBQg2NYjyBv"
       }
     ]
   }
@@ -167,4 +161,81 @@ params:
 ----------- BLOCK #153,000,048 (2E8ncPZMAqa5RKPvEUAMt6abLNcDZM7J6nawyaiSboGD) ---------------
 ----------- BLOCK #153,000,049 (GyG9Rw4Mo8sYb9WLzz8oqHgV3k4sYa3UWeveGG2uLGWX) ---------------
 all done
-```
+```
+
+### Running the "map_filter_transactions" Module
+
+To run this module, you must provide a transaction signature value in the parameters in your substreams.yaml manifest.
+
+This example will use the default `21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t` program.
+
+```yaml
+params:
+  map_filter_transactions: "signature=21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t"
+```
+
+```bash
+substreams run -e mainnet.sol.streamingfast.io:443 ./solana-explorer-v0.1.0.spkg map_filter_transactions -s 153000000 -t +30
+Connected (trace ID 27d4c57714280d824e0cd37f7cd8dc17)
+Progress messages received: 0 (0/sec)
+Backprocessing history up to requested target block 153000028:
+(hit 'm' to switch mode)
+----------- BLOCK #153,000,000 (ENZNjBKCxXQqgFxX63T6kn7DJwc7fQiwy3W3vyehtEGA) ---------------
+----------- BLOCK #153,000,001 (8q2n7y3ozKWUWrAGVbNc3wrkxdmT6iSdgf8CHgqrvSv1) ---------------
+----------- BLOCK #153,000,002 (AfE4vBBsvcCXzdexP4sX9byHYrdZsUpd2GTTTTvkShxu) ---------------
+----------- BLOCK #153,000,003 (H5RZGAShGm7sMW6NvaPY3TsX36WEvugZ64Y1gSBRPyLV) ---------------
+----------- BLOCK #153,000,004 (Egf4PFv5636rycdntsBfi26Bou2Mt3Z5UAgH4hBbkZat) ---------------
+----------- BLOCK #153,000,005 (DmkkhPG2RDWkJnLbSiG1XTUotJVpLbjqBeRa9FoQB3Tg) ---------------
+----------- BLOCK #153,000,006 (Dy9YQQrTed69j7fsy7z3F3sa1x34T4CGmCVQVFoWobRE) ---------------
+----------- BLOCK #153,000,007 (3bHscyCgsvzLo6u4CpvCzJe7ELkuf9k8MSwHyLUtif9L) ---------------
+----------- BLOCK #153,000,008 (CkDjN7c4hjYqtuYXH9Vw8MTmk29XjgMnbEtAzrjbyjnU) ---------------
+----------- BLOCK #153,000,009 (8SVnGrwbwpAWCPdRXKgyNCvJRedEqmYnLazB3mB7Rehg) ---------------
+----------- BLOCK #153,000,010 (BTVDWyLK26P2CBjGjCvsHwNqaUcn15xUauwCJmkCSAx1) ---------------
+----------- BLOCK #153,000,011 (DVzw63xEAHmWTdo9iA9ca7yTiaondmGGvr4VczQnFhbn) ---------------
+----------- BLOCK #153,000,012 (EtMvzAbrkdWpawfP7ozcL86yocFNYxvWvtS4f8pMrajD) ---------------
+----------- BLOCK #153,000,013 (7NVSAZZefTrNkK7JSg227yoponB6Tig27PXnEm7kQFJf) ---------------
+----------- BLOCK #153,000,014 (3nfE81F85ggdWxjRZUENBPhvXStAJsVxnwrQ99VcTMME) ---------------
+----------- BLOCK #153,000,015 (4N3sqbbJEZobfu1pL6rZEZf6Kpx2WYB8fHm1hbqASa6S) ---------------
+----------- BLOCK #153,000,016 (812uX2UUSvvep5APemSmTDk4bDnMkZnJtMBSa6enBmFN) ---------------
+----------- BLOCK #153,000,017 (CXsKxq7LmJVW6P3YjBVbrfgDiyMyzYDX5jNhgdsryMfv) ---------------
+----------- BLOCK #153,000,018 (FnVqabMm8VEKCr9pchMFxipxSssNeioxx7bCaqyGbzgX) ---------------
+----------- BLOCK #153,000,019 (J49iZE979hGtK4hnxUigpxQiNsU3oNq9UMMJqknGLJVT) ---------------
+----------- BLOCK #153,000,020 (GJ1BNqsBtyxucvqkipkZDaHW2KGdsU3sR41ebmCxth24) ---------------
+----------- BLOCK #153,000,021 (Bxc99wYm1heyzfY1ViXNnJFmf81BFHzLKrwizW5TPemn) ---------------
+----------- BLOCK #153,000,022 (2iLweM1VHWnkmXpF7G2fHr52zGuwsEjFXd8ZtpwwUcDw) ---------------
+----------- BLOCK #153,000,023 (6JpfNdC2qTErHpXd8e74E5aAkm11USA2rJJbCvQdtHyg) ---------------
+----------- BLOCK #153,000,024 (AjN8rk6c8ya5TNYRBUWKUgEYEJceqtJsPMMDdweg4MPR) ---------------
+----------- BLOCK #153,000,025 (F7dvT2mwGusrLmRwdJrGZ7cAXsxq43kHsrKpzeseywwg) ---------------
+----------- BLOCK #153,000,026 (aGdXha9hhhaETuNywV2XY39Y6B719NBZ6xbUzAu3wah) ---------------
+----------- BLOCK #153,000,027 (GfoPvdzekjLV1QHvvHzyVG7xbjajDKA4Ev7gXum2Ry1S) ---------------
+----------- BLOCK #153,000,028 (5xj9iYo6nuAiXvPFDUieUKBcRLbg9rz5qiBQg2NYjyBv) ---------------
+{
+  "@module": "map_filter_transactions",
+  "@block": 153000028,
+  "@type": "sol.transactions.v1.Transactions",
+  "@data": {
+    "transactions": [
+      {
+        "signatures": [
+          "21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t"
+        ],
+        "instructions": [
+          {
+            "programId": "Vote111111111111111111111111111111111111111",
+            "accounts": [
+              "HRfK8kbqCaKYsHk3R8HCtLNDp4aTneq1eSQ9ZrA2Kb2q",
+              "SysvarS1otHashes111111111111111111111111111",
+              "SysvarC1ock11111111111111111111111111111111",
+              "HEJzPiLSg9ty5CGAyJm5y7ef1NPzr1u1aHLYaAMXsgH9"
+            ],
+            "data": "29z5mr1JoRmJYQ6yybpMZZDfJB6dWDGQsB8jimJo3eTNYA7qEJvjo3dwvmP25dQtpxkKPQdEy37P5hDAMoAtGnaE1vGpXq"
+          }
+        ]
+      }
+    ]
+  }
+}
+
+----------- BLOCK #153,000,029 (3LBYvzhWfdtcAwuCvoBzTCkUbpVbckyBFAcJezdScGhP) ---------------
+all done
+```

solana-explorer/proto/block.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package sol.block_meta.v1;
+package sol.block.v1;
 
 message BlockMeta {
   uint64 slot = 1;

solana-explorer/proto/program_instructions.proto

@@ -1,16 +0,0 @@
-syntax = "proto3";
-
-package sol.programs.v1;
-
-message Instructions {
-  repeated Instruction instructions = 1;
-}
-
-message Instruction {
-  string program_id = 1;
-  repeated string accounts = 2;
-  string data = 3;
-
-  uint64 slot_number = 4;
-  string block_hash = 5;
-}

solana-explorer/proto/transactions.proto

@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package sol.transactions.v1;
+
+
+message Instructions {
+  repeated Instruction instructions = 1;
+}
+
+message Instruction {
+  string program_id = 1;
+  repeated string accounts = 2;
+  string data = 3;
+}
+
+message Transactions {
+  repeated Transaction transactions = 1;
+}
+
+message Transaction {
+  repeated string signatures = 1;
+
+  repeated Instruction instructions = 2;
+}

solana-explorer/src/lib.rs

@@ -2,3 +2,4 @@ mod pb;
 mod map_block_full;
 mod map_block_meta;
 mod map_filter_instructions;
+mod map_filter_transactions;

solana-explorer/src/map_block_meta.rs

@@ -1,5 +1,5 @@
 use substreams_solana::pb::sf::solana::r#type::v1::Block;
-use crate::pb::sol::block_meta::v1::BlockMeta;
+use crate::pb::sol::block::v1::BlockMeta;
 
 #[substreams::handlers::map]
 fn map_block_meta(blk: Block) -> Result<BlockMeta, substreams::errors::Error> {

solana-explorer/src/map_filter_instructions.rs

@@ -1,14 +1,13 @@
 use anyhow::anyhow;
 use serde::Deserialize;
 use substreams_solana::pb::sf::solana::r#type::v1::{Block, CompiledInstruction};
-use crate::pb::sol::programs::v1::{Instruction, Instructions};
+use crate::pb::sol::transactions::v1::{Instruction, Instructions};
 
 #[derive(Deserialize, Debug)]
-struct TransactionFilterParams {
+struct InstructionFilterParams {
     program_id: Option<String>,
 }
 
-
 #[substreams::handlers::map]
 fn map_filter_instructions(params: String, blk: Block) -> Result<Instructions, Vec<substreams::errors::Error>> {
     let filters = parse_filters_from_params(params)?;
@@ -17,14 +16,11 @@ fn map_filter_instructions(params: String, blk: Block) -> Result<Instructions, V
 
     blk.transactions.iter().for_each(|tx| {
         let msg = tx.transaction.clone().unwrap().message.unwrap();
-        tx.transaction.clone().unwrap().
         let acct_keys = msg.account_keys.as_slice();
         let insts : Vec<Instruction> = msg.instructions.iter()
             .filter(|inst| apply_filter(inst, &filters, acct_keys.to_vec()))
             .map(|inst| {
             Instruction {
-                slot_number: blk.slot,
-                block_hash: blk.blockhash.clone(),
                 program_id: bs58::encode(acct_keys[inst.program_id_index as usize].to_vec()).into_string(),
                 accounts: inst.accounts.iter().map(|acct| bs58::encode(acct_keys[*acct as usize].to_vec()).into_string()).collect(),
                 data: bs58::encode(inst.data.clone()).into_string(),
@@ -36,7 +32,7 @@ fn map_filter_instructions(params: String, blk: Block) -> Result<Instructions, V
     Ok(Instructions { instructions })
 }
 
-fn parse_filters_from_params(params: String) -> Result<TransactionFilterParams, Vec<substreams::errors::Error>> {
+fn parse_filters_from_params(params: String) -> Result<InstructionFilterParams, Vec<substreams::errors::Error>> {
     let parsed_result = serde_qs::from_str(&params);
     if parsed_result.is_err() {
         return Err(Vec::from([anyhow!("Unexpected error while parsing parameters")]));
@@ -48,7 +44,7 @@ fn parse_filters_from_params(params: String) -> Result<TransactionFilterParams,
     Ok(filters)
 }
 
-fn apply_filter(instruction: &CompiledInstruction, filters: &TransactionFilterParams, account_keys: Vec<Vec<u8>>) -> bool {
+fn apply_filter(instruction: &CompiledInstruction, filters: &InstructionFilterParams, account_keys: Vec<Vec<u8>>) -> bool {
     if filters.program_id.is_none() {
         return true;
     }
@@ -60,7 +56,6 @@ fn apply_filter(instruction: &CompiledInstruction, filters: &TransactionFilterPa
     }
     let program_account_key_val = bs58::encode(program_account_key.unwrap()).into_string();
 
-    //check if account key value is equal to the program id
     if program_account_key_val != program_id_filter {
         return false;
     }

solana-explorer/src/map_filter_transactions.rs

@@ -0,0 +1,69 @@
+use anyhow::anyhow;
+use serde::Deserialize;
+use substreams_solana::pb::sf::solana::r#type::v1::{Block, ConfirmedTransaction};
+use crate::pb::sol::transactions::v1::{Instruction, Transaction, Transactions};
+
+#[derive(Deserialize, Debug)]
+struct TransactionFilterParams {
+    signature: Option<String>,
+}
+
+#[substreams::handlers::map]
+fn map_filter_transactions(params: String, blk: Block) -> Result<Transactions, Vec<substreams::errors::Error>> {
+    let filters = parse_filters_from_params(params)?;
+
+    let mut transactions : Vec<Transaction> = Vec::new();
+
+    blk.transactions.iter()
+        .filter(|tx| apply_filter(tx, &filters))
+        .for_each(|tx| {
+            let msg = tx.transaction.clone().unwrap().message.unwrap();
+            let acct_keys = msg.account_keys.as_slice();
+
+            let insts : Vec<Instruction> = msg.instructions.iter()
+                .map(|inst| {
+                    Instruction {
+                        program_id: bs58::encode(acct_keys[inst.program_id_index as usize].to_vec()).into_string(),
+                        accounts: inst.accounts.iter().map(|acct| bs58::encode(acct_keys[*acct as usize].to_vec()).into_string()).collect(),
+                        data: bs58::encode(inst.data.clone()).into_string(),
+                    }
+                }).collect();
+
+            let t = Transaction {
+                signatures: tx.transaction.clone().unwrap().signatures.iter().map(|sig| bs58::encode(sig).into_string()).collect(),
+                instructions: insts
+            };
+            transactions.push(t);
+        });
+
+    Ok(Transactions { transactions })
+}
+
+fn parse_filters_from_params(params: String) -> Result<TransactionFilterParams, Vec<substreams::errors::Error>> {
+    let parsed_result = serde_qs::from_str(&params);
+    if parsed_result.is_err() {
+        return Err(Vec::from([anyhow!("Unexpected error while parsing parameters")]));
+    }
+
+    let filters = parsed_result.unwrap();
+    //todo: verify_filters(&filters)?;
+
+    Ok(filters)
+}
+
+fn apply_filter(transaction: &&ConfirmedTransaction, filters: &TransactionFilterParams) -> bool {
+    if filters.signature.is_none() {
+        return true;
+    }
+
+    let mut found = false;
+
+    transaction.transaction.as_ref().unwrap().signatures.iter().for_each(|sig| {
+        let xsig = bs58::encode(&sig).into_string();
+        if xsig == filters.signature.clone().unwrap() {
+            found = true;
+        }
+    });
+
+    found
+}

solana-explorer/src/pb/mod.rs

@@ -11,18 +11,18 @@ pub mod sf {
     }
 }
 pub mod sol {
-    pub mod block_meta {
+    pub mod block {
-        // @@protoc_insertion_point(attribute:sol.block_meta.v1)
+        // @@protoc_insertion_point(attribute:sol.block.v1)
         pub mod v1 {
-            include!("sol.block_meta.v1.rs");
+            include!("sol.block.v1.rs");
-            // @@protoc_insertion_point(sol.block_meta.v1)
+            // @@protoc_insertion_point(sol.block.v1)
         }
     }
-    pub mod programs {
+    pub mod transactions {
-        // @@protoc_insertion_point(attribute:sol.programs.v1)
+        // @@protoc_insertion_point(attribute:sol.transactions.v1)
         pub mod v1 {
-            include!("sol.programs.v1.rs");
+            include!("sol.transactions.v1.rs");
-            // @@protoc_insertion_point(sol.programs.v1)
+            // @@protoc_insertion_point(sol.transactions.v1)
         }
     }
 }

solana-explorer/src/pb/sol.transactions.v1.rs

@@ -14,9 +14,19 @@ pub struct Instruction {
     pub accounts: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
     #[prost(string, tag="3")]
     pub data: ::prost::alloc::string::String,
-    #[prost(uint64, tag="4")]
+}
-    pub slot_number: u64,
+#[allow(clippy::derive_partial_eq_without_eq)]
-    #[prost(string, tag="5")]
+#[derive(Clone, PartialEq, ::prost::Message)]
-    pub block_hash: ::prost::alloc::string::String,
+pub struct Transactions {
+    #[prost(message, repeated, tag="1")]
+    pub transactions: ::prost::alloc::vec::Vec<Transaction>,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Transaction {
+    #[prost(string, repeated, tag="1")]
+    pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    #[prost(message, repeated, tag="2")]
+    pub instructions: ::prost::alloc::vec::Vec<Instruction>,
 }
 // @@protoc_insertion_point(module)

solana-explorer/substreams.yaml

@@ -10,8 +10,8 @@ imports:
 
 protobuf:
   files:
-    - block_meta.proto
+    - block.proto
-    - program_instructions.proto
+    - transactions.proto
   importPaths:
     - ./proto
 
@@ -45,9 +45,20 @@ modules:
       - params: string
       - source: sf.solana.type.v1.Block
     output:
-      type: proto:sol.programs.v1.Instructions
+      type: proto:sol.transactions.v1.Instructions
     doc: |
       `map_filter_instructions` allows you to get instructions executed by a certain `program_id`, and returns the raw Instruction data model.
 
+  - name: map_filter_transactions
+    kind: map
+    inputs:
+      - params: string
+      - source: sf.solana.type.v1.Block
+    output:
+      type: proto:sol.transactions.v1.Transactions
+    doc: |
+      `map_filter_transaction` allows you to find a transaction by hash. You need to scope your search to the blocks you know you're going to find that transaction. Check with your preferred block explorer first.
+
 params:
   map_filter_instructions: "program_id=Stake11111111111111111111111111111111111111"
+  map_filter_transactions: "signature=21ED2HBGuLUwgbaBb77cGwFR8MkVQfjR9KszzCb7jZkeSysJkHAVew6RaaBh3r1zTefpdq9Kf5geFp19P3nUXB3t"