dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data pipeline refactor and fix

Browse Source
This commit is contained in:
Tim Olson
2026-04-13 18:30:04 -04:00
parent 6418729b16
commit 326bf80846
96 changed files with 7107 additions and 1763 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
relay/src/config.rs
View File

@@ -16,14 +16,15 @@ pub struct Config {
#[serde(default = "default_market_data_pub_port")]
pub market_data_pub_port: u16,
/// Ingestor work queue port (PUB - publish work with exchange prefix)
#[serde(default = "default_ingestor_work_port")]
pub ingestor_work_port: u16,
/// Flink market data endpoint (XSUB - relay subscribes to Flink)
/// Flink market data endpoint (XSUB - relay subscribes to Flink XPUB)
#[serde(default = "default_flink_market_data_endpoint")]
pub flink_market_data_endpoint: String,
/// Flink request endpoint (PUSH - relay forwards client requests to Flink PULL)
/// Flink's IngestorBroker binds a PULL socket on port 5566
#[serde(default = "default_flink_request_endpoint")]
pub flink_request_endpoint: String,
/// Request timeout in seconds
#[serde(default = "default_request_timeout_secs")]
pub request_timeout_secs: u64,
@@ -45,14 +46,14 @@ fn default_market_data_pub_port() -> u16 {
5558
}
fn default_ingestor_work_port() -> u16 {
5555
}
fn default_flink_market_data_endpoint() -> String {
"tcp://flink-jobmanager:5558".to_string()
}
fn default_flink_request_endpoint() -> String {
"tcp://flink-jobmanager:5566".to_string()
}
fn default_request_timeout_secs() -> u64 {
30
}
@@ -67,8 +68,8 @@ impl Default for Config {
bind_address: default_bind_address(),
client_request_port: default_client_request_port(),
market_data_pub_port: default_market_data_pub_port(),
ingestor_work_port: default_ingestor_work_port(),
flink_market_data_endpoint: default_flink_market_data_endpoint(),
flink_request_endpoint: default_flink_request_endpoint(),
request_timeout_secs: default_request_timeout_secs(),
high_water_mark: default_hwm(),
}

118
relay/src/relay.rs
View File

@@ -7,8 +7,6 @@ use tracing::{debug, error, info, warn};
const PROTOCOL_VERSION: u8 = 0x01;
const MSG_TYPE_SUBMIT_REQUEST: u8 = 0x10;
const MSG_TYPE_SUBMIT_RESPONSE: u8 = 0x11;
const MSG_TYPE_DATA_REQUEST: u8 = 0x01;
const MSG_TYPE_HISTORY_READY: u8 = 0x12;
pub struct Relay {
config: Config,
@@ -26,24 +24,21 @@ impl Relay {
}
pub async fn run(self) -> Result<()> {
info!("Initializing Stateless ZMQ Relay");
info!("Initializing ZMQ Relay");
// Bind sockets
let client_request_socket = self.create_client_request_socket()?;
let market_data_frontend = self.create_market_data_frontend()?;
let market_data_backend = self.create_market_data_backend()?;
let ingestor_work_socket = self.create_ingestor_work_socket()?;
let flink_request_socket = self.create_flink_request_socket()?;
info!("All sockets initialized successfully - relay is STATELESS");
info!("No pending requests tracked - all async via pub/sub");
info!("All sockets initialized — relay forwards requests to Flink");
// Run main loop
tokio::task::spawn_blocking(move || {
Self::proxy_loop(
client_request_socket,
market_data_frontend,
market_data_backend,
ingestor_work_socket,
flink_request_socket,
)
})
.await?
@@ -58,7 +53,6 @@ impl Relay {
let endpoint = format!("{}:{}", self.config.bind_address, self.config.client_request_port);
socket.bind(&endpoint)?;
info!("Client request socket (ROUTER) bound to {}", endpoint);
info!(" → Accepts SubmitHistoricalRequest, returns SubmitResponse immediately");
Ok(socket)
}
@@ -71,7 +65,7 @@ impl Relay {
let endpoint = format!("{}:{}", self.config.bind_address, self.config.market_data_pub_port);
socket.bind(&endpoint)?;
info!("Market data frontend (XPUB) bound to {}", endpoint);
info!(" → Clients subscribe here for HistoryReadyNotification and market data");
info!(" → Clients subscribe here; subscription events forwarded to Flink for realtime activation");
Ok(socket)
}
@@ -82,20 +76,19 @@ impl Relay {
socket.connect(&self.config.flink_market_data_endpoint)?;
info!("Market data backend (XSUB) connected to {}", self.config.flink_market_data_endpoint);
info!(" → Receives HistoryReadyNotification and market data from Flink");
info!(" → Receives market data and notifications from Flink");
Ok(socket)
}
fn create_ingestor_work_socket(&self) -> Result<zmq::Socket> {
let socket = self.context.socket(zmq::PUB)?;
fn create_flink_request_socket(&self) -> Result<zmq::Socket> {
let socket = self.context.socket(zmq::PUSH)?;
socket.set_sndhwm(self.config.high_water_mark)?;
socket.set_linger(1000)?;
let endpoint = format!("{}:{}", self.config.bind_address, self.config.ingestor_work_port);
socket.bind(&endpoint)?;
info!("Ingestor work queue (PUB) bound to {}", endpoint);
info!(" → Publishes DataRequest with exchange prefix");
socket.connect(&self.config.flink_request_endpoint)?;
info!("Flink request socket (PUSH) connected to {}", self.config.flink_request_endpoint);
info!(" → Forwards SubmitHistoricalRequest to Flink for dispatch to ingestors");
Ok(socket)
}
@@ -104,7 +97,7 @@ impl Relay {
client_request_socket: zmq::Socket,
market_data_frontend: zmq::Socket,
market_data_backend: zmq::Socket,
ingestor_work_socket: zmq::Socket,
flink_request_socket: zmq::Socket,
) -> Result<()> {
let mut items = [
client_request_socket.as_poll_item(zmq::POLLIN),
@@ -112,10 +105,9 @@ impl Relay {
market_data_backend.as_poll_item(zmq::POLLIN),
];
info!("Entering stateless proxy loop");
info!("Entering relay proxy loop");
loop {
// Poll with 100ms timeout
zmq::poll(&mut items, 100)
.context("Failed to poll sockets")?;
@@ -123,21 +115,20 @@ impl Relay {
if items[0].is_readable() {
if let Err(e) = Self::handle_client_submission(
&client_request_socket,
&ingestor_work_socket,
&flink_request_socket,
) {
error!("Error handling client submission: {}", e);
}
}
// Handle market data subscriptions from clients (XPUB → XSUB)
// Proxy client subscription events → Flink (XPUB → XSUB)
if items[1].is_readable() {
if let Err(e) = Self::proxy_subscription(&market_data_frontend, &market_data_backend) {
error!("Error proxying subscription: {}", e);
}
}
// Handle market data from Flink (XSUB → XPUB)
// This includes HistoryReadyNotification and regular market data
// Proxy market data from Flink → clients (XSUB → XPUB)
if items[2].is_readable() {
if let Err(e) = Self::proxy_market_data(&market_data_backend, &market_data_frontend) {
error!("Error proxying market data: {}", e);
@@ -148,7 +139,7 @@ impl Relay {
fn handle_client_submission(
client_socket: &zmq::Socket,
ingestor_socket: &zmq::Socket,
flink_socket: &zmq::Socket,
) -> Result<()> {
// Receive from client: [identity][empty][version][message]
let identity = client_socket.recv_bytes(0)?;
@@ -177,7 +168,7 @@ impl Relay {
identity,
payload,
client_socket,
ingestor_socket,
flink_socket,
)?;
}
_ => {
@@ -192,61 +183,27 @@ impl Relay {
client_identity: Vec<u8>,
payload: &[u8],
client_socket: &zmq::Socket,
ingestor_socket: &zmq::Socket,
flink_socket: &zmq::Socket,
) -> Result<()> {
// Parse protobuf request
// Parse just enough to build the SubmitResponse — relay stays thin
let request = proto::SubmitHistoricalRequest::decode(payload)
.context("Failed to parse SubmitHistoricalRequest")?;
let request_id = request.request_id.clone();
let ticker = request.ticker.clone();
let client_id = request.client_id.clone();
info!("Handling request submission: request_id={}, ticker={}, client_id={:?}",
request_id, ticker, client_id);
info!("Forwarding request to Flink: request_id={}, ticker={}", request_id, request.ticker);
// Extract exchange suffix from ticker (Nautilus format: "BTC/USDT.BINANCE")
let exchange_prefix = ticker.rsplitn(2, '.').next()
.map(|s| format!("{}.", s))
.unwrap_or_else(|| String::from(""));
if exchange_prefix.is_empty() {
warn!("Ticker '{}' missing exchange suffix", ticker);
}
// Build DataRequest protobuf for ingestors
let data_request = proto::DataRequest {
request_id: request_id.clone(),
r#type: proto::data_request::RequestType::HistoricalOhlc as i32,
ticker: ticker.clone(),
historical: Some(proto::HistoricalParams {
start_time: request.start_time,
end_time: request.end_time,
period_seconds: request.period_seconds,
limit: request.limit,
}),
realtime: None,
client_id: client_id.clone(),
};
let mut data_request_bytes = Vec::new();
data_request.encode(&mut data_request_bytes)?;
// Publish to ingestors with exchange prefix
// Forward the raw request to Flink via PUSH
// Flink builds DataRequest and dispatches to ingestors via IngestorBroker
let version_frame = vec![PROTOCOL_VERSION];
let mut message_frame = vec![MSG_TYPE_DATA_REQUEST];
message_frame.extend_from_slice(&data_request_bytes);
let mut message_frame = vec![MSG_TYPE_SUBMIT_REQUEST];
message_frame.extend_from_slice(payload);
ingestor_socket.send(&exchange_prefix, zmq::SNDMORE)?;
ingestor_socket.send(&version_frame, zmq::SNDMORE)?;
ingestor_socket.send(&message_frame, 0)?;
flink_socket.send(&version_frame, zmq::SNDMORE)?;
flink_socket.send(&message_frame, 0)?;
info!("Published to ingestors: prefix={}, request_id={}", exchange_prefix, request_id);
// Build SubmitResponse protobuf
// NOTE: This topic is DETERMINISTIC based on client-generated values.
// Client should have already subscribed to this topic BEFORE sending the request
// to prevent race condition where notification arrives before client subscribes.
// Build SubmitResponse — relay still acks the client immediately
let notification_topic = if let Some(cid) = &client_id {
format!("RESPONSE:{}", cid)
} else {
@@ -263,20 +220,16 @@ impl Relay {
let mut response_bytes = Vec::new();
response.encode(&mut response_bytes)?;
// Send immediate response to client
let version_frame = vec![PROTOCOL_VERSION];
let mut message_frame = vec![MSG_TYPE_SUBMIT_RESPONSE];
message_frame.extend_from_slice(&response_bytes);
let mut resp_message_frame = vec![MSG_TYPE_SUBMIT_RESPONSE];
resp_message_frame.extend_from_slice(&response_bytes);
client_socket.send(&client_identity, zmq::SNDMORE)?;
client_socket.send(&[] as &[u8], zmq::SNDMORE)?;
client_socket.send(&version_frame, zmq::SNDMORE)?;
client_socket.send(&message_frame, 0)?;
client_socket.send(&resp_message_frame, 0)?;
info!("Sent SubmitResponse to client: request_id={}, topic={}", request_id, notification_topic);
// Relay is now DONE with this request - completely stateless!
// Client will receive notification via pub/sub when Flink publishes HistoryReadyNotification
info!("Acked client and forwarded to Flink: request_id={}, notification_topic={}", request_id, notification_topic);
Ok(())
}
@@ -285,7 +238,7 @@ impl Relay {
frontend: &zmq::Socket,
backend: &zmq::Socket,
) -> Result<()> {
// Forward subscription message from XPUB to XSUB
// Forward subscription event from XPUB to XSUB so Flink can detect realtime interest
let msg = frontend.recv_bytes(0)?;
backend.send(&msg, 0)?;
@@ -302,10 +255,7 @@ impl Relay {
backend: &zmq::Socket,
frontend: &zmq::Socket,
) -> Result<()> {
// Forward all messages from XSUB to XPUB (zero-copy proxy)
// This includes:
// - Regular market data (ticks, OHLC)
// - HistoryReadyNotification from Flink
// Zero-copy proxy: XSUB (Flink) → XPUB (clients)
loop {
let msg = backend.recv_bytes(0)?;
let more = backend.get_rcvmore()?;