data pipeline refactor and fix

protobuf/ingestor.proto

@@ -24,6 +24,9 @@ message DataRequest {
   // Flink uses this to determine notification topic
   optional string client_id = 6;
 
+  // Job ID assigned by the IngestorBroker for work tracking and heartbeating
+  optional string job_id = 7;
+
   enum RequestType {
     HISTORICAL_OHLC = 0;
     REALTIME_TICKS = 1;
@@ -326,4 +329,41 @@ message FieldValue {
     bytes bytes_val = 5;
     uint64 timestamp_val = 6;
   }
+}
+
+// ─── Ingestor Broker Protocol (Flink ROUTER ↔ Ingestor DEALER, port 5567) ───
+// Message type IDs 0x20–0x25
+
+// Ingestor → Flink: register as available (type 0x20)
+// Sent on DEALER connect and after every COMPLETE.
+message WorkerReady {
+  // Exchanges this ingestor supports (e.g. ["BINANCE", "COINBASE"])
+  repeated string exchanges = 1;
+}
+
+// Ingestor → Flink: historical job finished (type 0x21)
+message WorkComplete {
+  string job_id = 1;
+  bool success = 2;
+  optional string error_message = 3;
+}
+
+// Ingestor → Flink: realtime job still alive — sent every 5s (type 0x22)
+message WorkHeartbeat {
+  string job_id = 1;
+}
+
+// Ingestor → Flink: unable to handle this job (type 0x23)
+message WorkReject {
+  string job_id = 1;
+  string reason = 2;
+}
+
+// Flink → Ingestor: dispatch a job — wraps DataRequest (type 0x24)
+// DataRequest.job_id is populated by IngestorBroker
+// (DataRequest itself is type 0x01; this is the framing type for broker dispatch)
+
+// Flink → Ingestor: stop a realtime job (type 0x25)
+message WorkStop {
+  string job_id = 1;
 }

protobuf/ohlc.proto

@@ -58,4 +58,8 @@ message OHLCBatchMetadata {
 
   // Error message if status is ERROR
   optional string error_message = 8;
+
+  // True on the final page of a historical query (including error/not-found markers).
+  // Flink publishes HistoryReadyNotification only when this is true.
+  bool is_last_page = 9;
 }