data pipeline refactor and fix

deploy/k8s/dev/configs/gateway-config.yaml.tpl

@@ -56,8 +56,8 @@ data:
       namespace: sandbox
       service_namespace: default
       in_cluster: true
-      sandbox_image: dexorder/ai-sandbox:dev20260409143116
-      sidecar_image: dexorder/ai-lifecycle-sidecar:dev20260408103634
+      sandbox_image: SANDBOX_IMAGE_TAG
+      sidecar_image: SIDECAR_IMAGE_TAG
       storage_class: standard
       image_pull_policy: Never  # For minikube dev - use local images
 

deploy/k8s/dev/configs/ingestor-config.yaml

@@ -1,9 +1,8 @@
 # CCXT Ingestor Configuration
 
-# Relay ZMQ endpoints (relay is the well-known gateway)
-flink_hostname: relay
-ingestor_work_port: 5555    # SUB - receives DataRequest with exchange prefix
-# Note: No response port needed - async architecture via Kafka!
+# Flink IngestorBroker (ROUTER) endpoint
+flink_hostname: flink-jobmanager
+ingestor_broker_port: 5567
 
 # Supported exchanges (subscribe to these prefixes)
 supported_exchanges:

deploy/k8s/dev/infrastructure.yaml

@@ -455,6 +455,14 @@ spec:
       protocol: TCP
       port: 5561
       targetPort: 5561
+    - name: zmq-client-pull
+      protocol: TCP
+      port: 5566
+      targetPort: 5566
+    - name: zmq-ingestor-broker
+      protocol: TCP
+      port: 5567
+      targetPort: 5567
   type: ClusterIP
 ---
 apiVersion: apps/v1
@@ -583,14 +591,6 @@ spec:
   selector:
     app: relay
   ports:
-    - name: work-queue
-      protocol: TCP
-      port: 5555
-      targetPort: 5555
-    - name: responses
-      protocol: TCP
-      port: 5556
-      targetPort: 5556
     - name: market-data
       protocol: TCP
       port: 5558
@@ -620,10 +620,6 @@ spec:
         image: dexorder/ai-relay
         imagePullPolicy: Never
         ports:
-        - containerPort: 5555
-          name: work-queue
-        - containerPort: 5556
-          name: responses
         - containerPort: 5558
           name: market-data
         - containerPort: 5559
@@ -657,9 +653,9 @@ spec:
         app: ingestor
     spec:
       initContainers:
-      - name: wait-for-relay
+      - name: wait-for-flink
         image: busybox:1.36
-        command: ['sh', '-c', 'until nc -z relay 5555; do echo waiting for relay; sleep 2; done;']
+        command: ['sh', '-c', 'until nc -z flink-jobmanager 5567; do echo waiting for flink broker; sleep 2; done;']
       - name: wait-for-kafka
         image: busybox:1.36
         command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']

deploy/k8s/dev/sandbox-config.yaml

@@ -30,6 +30,7 @@ data:
         namespace: "trading"
         # S3 endpoint for MinIO in default namespace
         s3_endpoint: "http://minio.default.svc.cluster.local:9000"
+        s3_region: "us-east-1"
 
       relay:
         endpoint: "tcp://relay.default.svc.cluster.local:5559"