data pipeline refactor and fix

deploy/k8s/dev/infrastructure.yaml

@@ -455,6 +455,14 @@ spec:
       protocol: TCP
       port: 5561
       targetPort: 5561
+    - name: zmq-client-pull
+      protocol: TCP
+      port: 5566
+      targetPort: 5566
+    - name: zmq-ingestor-broker
+      protocol: TCP
+      port: 5567
+      targetPort: 5567
   type: ClusterIP
 ---
 apiVersion: apps/v1
@@ -583,14 +591,6 @@ spec:
   selector:
     app: relay
   ports:
-    - name: work-queue
-      protocol: TCP
-      port: 5555
-      targetPort: 5555
-    - name: responses
-      protocol: TCP
-      port: 5556
-      targetPort: 5556
     - name: market-data
       protocol: TCP
       port: 5558
@@ -620,10 +620,6 @@ spec:
         image: dexorder/ai-relay
         imagePullPolicy: Never
         ports:
-        - containerPort: 5555
-          name: work-queue
-        - containerPort: 5556
-          name: responses
         - containerPort: 5558
           name: market-data
         - containerPort: 5559
@@ -657,9 +653,9 @@ spec:
         app: ingestor
     spec:
       initContainers:
-      - name: wait-for-relay
+      - name: wait-for-flink
         image: busybox:1.36
-        command: ['sh', '-c', 'until nc -z relay 5555; do echo waiting for relay; sleep 2; done;']
+        command: ['sh', '-c', 'until nc -z flink-jobmanager 5567; do echo waiting for flink broker; sleep 2; done;']
       - name: wait-for-kafka
         image: busybox:1.36
         command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']