data pipeline refactor and fix

deploy/k8s/prod/configs/ingestor-config.yaml

@@ -1,9 +1,8 @@
 # CCXT Ingestor Configuration
 
-# Relay ZMQ endpoints (relay is the well-known gateway)
-flink_hostname: relay
-ingestor_work_port: 5555    # SUB - receives DataRequest with exchange prefix
-# Note: No response port needed - async architecture via Kafka!
+# Flink IngestorBroker (ROUTER) endpoint
+flink_hostname: flink-jobmanager
+ingestor_broker_port: 5567
 
 # Supported exchanges (subscribe to these prefixes)
 supported_exchanges:

deploy/k8s/prod/infrastructure.yaml

@@ -451,6 +451,14 @@ spec:
       protocol: TCP
       port: 5561
       targetPort: 5561
+    - name: zmq-client-pull
+      protocol: TCP
+      port: 5566
+      targetPort: 5566
+    - name: zmq-ingestor-broker
+      protocol: TCP
+      port: 5567
+      targetPort: 5567
   type: ClusterIP
 ---
 apiVersion: apps/v1
@@ -579,14 +587,6 @@ spec:
   selector:
     app: relay
   ports:
-    - name: work-queue
-      protocol: TCP
-      port: 5555
-      targetPort: 5555
-    - name: responses
-      protocol: TCP
-      port: 5556
-      targetPort: 5556
     - name: market-data
       protocol: TCP
       port: 5558
@@ -616,10 +616,6 @@ spec:
         image: dexorder/ai-relay
         imagePullPolicy: Always
         ports:
-        - containerPort: 5555
-          name: work-queue
-        - containerPort: 5556
-          name: responses
         - containerPort: 5558
           name: market-data
         - containerPort: 5559
@@ -653,9 +649,9 @@ spec:
         app: ingestor
     spec:
       initContainers:
-      - name: wait-for-relay
+      - name: wait-for-flink
         image: busybox:1.36
-        command: ['sh', '-c', 'until nc -z relay 5555; do echo waiting for relay; sleep 2; done;']
+        command: ['sh', '-c', 'until nc -z flink-jobmanager 5567; do echo waiting for flink broker; sleep 2; done;']
       - name: wait-for-kafka
         image: busybox:1.36
         command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']

deploy/k8s/prod/sandbox-config.yaml

@@ -22,6 +22,7 @@ data:
         catalog_uri: "http://iceberg-catalog.ai.svc.cluster.local:8181"
         namespace: "trading"
         s3_endpoint: "http://minio.ai.svc.cluster.local:9000"
+        s3_region: "us-east-1"
 
       relay:
         endpoint: "tcp://relay.ai.svc.cluster.local:5559"