backend redesign

deploy/k8s/prod/configs/flink-config.yaml

@@ -0,0 +1,30 @@
+# Flink Job Configuration
+
+# ZeroMQ bind address and ports
+zmq_bind_address: "tcp://*"
+zmq_ingestor_work_queue_port: 5555
+zmq_ingestor_response_port: 5556
+zmq_ingestor_control_port: 5557
+zmq_market_data_pub_port: 5558
+zmq_client_request_port: 5559
+zmq_cep_webhook_port: 5560
+
+# Notification publisher endpoint (Flink → Relay)
+# Relay connects XSUB to this endpoint and proxies to clients
+notification_publish_endpoint: "tcp://*:5557"
+
+# Kafka configuration
+kafka_bootstrap_servers: "kafka:9092"
+kafka_tick_topic: "market-tick"
+kafka_ohlc_topic: "market-ohlc"
+
+# Iceberg catalog
+iceberg_catalog_uri: "http://iceberg-catalog:8181"
+iceberg_warehouse: "s3://trading-warehouse/"
+iceberg_namespace: "trading"
+iceberg_table_prefix: "market"
+hadoop_conf_dir: "/etc/hadoop/conf"
+
+# Flink configuration
+flink_parallelism: 2
+flink_checkpoint_interval_ms: 60000

deploy/k8s/prod/configs/ingestor-config.yaml

@@ -0,0 +1,24 @@
+# CCXT Ingestor Configuration
+
+# Relay ZMQ endpoints (relay is the well-known gateway)
+flink_hostname: relay
+ingestor_work_port: 5555    # SUB - receives DataRequest with exchange prefix
+# Note: No response port needed - async architecture via Kafka!
+
+# Supported exchanges (subscribe to these prefixes)
+supported_exchanges:
+  - BINANCE
+  - COINBASE
+  - KRAKEN
+
+# Kafka configuration
+kafka_brokers:
+  - kafka:9092
+kafka_topic: market-0
+
+# Worker configuration
+max_concurrent: 10
+poll_interval_ms: 10000
+
+# Logging
+log_level: info

deploy/k8s/prod/configs/relay-config.yaml

@@ -0,0 +1,19 @@
+# ZMQ Relay Configuration
+
+# Bind address for all relay sockets
+bind_address: "tcp://*"
+
+# Client-facing ports
+client_request_port: 5559          # ROUTER - Client historical data requests
+market_data_pub_port: 5558         # XPUB - Market data fanout to clients
+
+# Ingestor-facing ports
+ingestor_work_port: 5555           # PUB - Distribute work with exchange prefix
+ingestor_response_port: 5556       # ROUTER - Receive responses from ingestors
+
+# Flink connection
+flink_market_data_endpoint: "tcp://flink-jobmanager:5557"  # XSUB - Subscribe to Flink market data
+
+# Timeouts and limits
+request_timeout_secs: 30           # Timeout for pending client requests
+high_water_mark: 10000             # ZMQ high water mark for all sockets