backend redesign

deploy/k8s/dev/configs/flink-config.yaml

@@ -0,0 +1,40 @@
+# Flink Job Configuration
+
+# ZeroMQ bind address and ports
+zmq_bind_address: "tcp://*"
+zmq_ingestor_work_queue_port: 5555
+zmq_ingestor_response_port: 5556
+zmq_ingestor_control_port: 5557
+zmq_market_data_pub_port: 5558
+zmq_client_request_port: 5559
+zmq_cep_webhook_port: 5560
+
+# Notification endpoints
+# Task managers PUSH to job manager PULL socket at this address
+notification_publish_endpoint: "tcp://flink-jobmanager:5561"
+# Job manager binds PULL socket on this port to receive from task managers
+notification_pull_port: 5561
+
+# Kafka configuration
+kafka_bootstrap_servers: "kafka:9092"
+kafka_tick_topic: "market-tick"
+kafka_ohlc_topic: "market-ohlc"
+kafka_topics_file: "/topics-dev.yaml"  # Use topics-dev.yaml for single broker dev environment
+
+# Iceberg catalog
+iceberg_catalog_uri: "http://iceberg-catalog:8181"
+iceberg_warehouse: "s3://trading-warehouse/"
+iceberg_namespace: "trading"
+iceberg_table_prefix: "market"
+hadoop_conf_dir: "/etc/hadoop/conf"
+
+# Flink configuration
+flink_parallelism: 1
+flink_checkpoint_interval_ms: 60000
+
+# Flink memory configuration (required)
+jobmanager.memory.process.size: 1600m
+taskmanager.memory.process.size: 1728m
+taskmanager.numberOfTaskSlots: 2
+jobmanager.rpc.address: flink-jobmanager
+jobmanager.rpc.port: 6123

deploy/k8s/dev/configs/ingestor-config.yaml

@@ -0,0 +1,24 @@
+# CCXT Ingestor Configuration
+
+# Relay ZMQ endpoints (relay is the well-known gateway)
+flink_hostname: relay
+ingestor_work_port: 5555    # SUB - receives DataRequest with exchange prefix
+# Note: No response port needed - async architecture via Kafka!
+
+# Supported exchanges (subscribe to these prefixes)
+supported_exchanges:
+  - BINANCE
+  - COINBASE
+  - KRAKEN
+
+# Kafka configuration
+kafka_brokers:
+  - kafka:9092
+kafka_topic: market-ohlc
+
+# Worker configuration
+max_concurrent: 10
+poll_interval_ms: 10000
+
+# Logging
+log_level: info

deploy/k8s/dev/configs/relay-config.yaml

@@ -0,0 +1,19 @@
+# ZMQ Relay Configuration
+
+# Bind address for all relay sockets
+bind_address: "tcp://*"
+
+# Client-facing ports
+client_request_port: 5559          # ROUTER - Client historical data requests
+market_data_pub_port: 5558         # XPUB - Market data fanout to clients
+
+# Ingestor-facing ports
+ingestor_work_port: 5555           # PUB - Distribute work with exchange prefix
+ingestor_response_port: 5556       # ROUTER - Receive responses from ingestors
+
+# Flink connection
+flink_market_data_endpoint: "tcp://flink-jobmanager:5558"  # XSUB - Subscribe to Flink market data (MARKET_DATA_PUB)
+
+# Timeouts and limits
+request_timeout_secs: 30           # Timeout for pending client requests
+high_water_mark: 10000             # ZMQ high water mark for all sockets