diff --git a/deploy/k8s/base/flink.yaml b/deploy/k8s/base/flink.yaml new file mode 100644 index 00000000..5ab679b7 --- /dev/null +++ b/deploy/k8s/base/flink.yaml @@ -0,0 +1,152 @@ +# Flink JobManager + TaskManager +--- +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager +spec: + selector: + app: flink-jobmanager + ports: + - name: rpc + protocol: TCP + port: 6123 + targetPort: 6123 + - name: ui + protocol: TCP + port: 8081 + targetPort: 8081 + - name: zmq-market-data + protocol: TCP + port: 5558 + targetPort: 5558 + - name: zmq-notif-pull + protocol: TCP + port: 5561 + targetPort: 5561 + - name: zmq-client-pull + protocol: TCP + port: 5566 + targetPort: 5566 + - name: zmq-ingestor-broker + protocol: TCP + port: 5567 + targetPort: 5567 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink-jobmanager + template: + metadata: + labels: + app: flink-jobmanager + spec: + initContainers: + - name: wait-for-kafka + image: busybox:1.36 + command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;'] + - name: wait-for-iceberg-catalog + image: busybox:1.36 + command: ['sh', '-c', 'until nc -z iceberg-catalog 8181; do echo waiting for iceberg-catalog; sleep 2; done;'] + containers: + - name: flink-jobmanager + image: dexorder/ai-flink + imagePullPolicy: Always + args: ["standalone-job", "--job-classname", "com.dexorder.flink.TradingFlinkApp"] + ports: + - containerPort: 6123 + name: rpc + - containerPort: 8081 + name: ui + - containerPort: 5558 + name: zmq-market-data + - containerPort: 5561 + name: zmq-notif-pull + env: + - name: JOB_MANAGER_RPC_ADDRESS + value: flink-jobmanager + - name: AWS_REGION + value: us-east-1 + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: flink-secrets + key: minio-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: flink-secrets + key: minio-secret-key + volumeMounts: + - name: flink-config + mountPath: /etc/config/config.yaml + subPath: config.yaml + - name: flink-secrets + mountPath: /etc/secrets + volumes: + - name: flink-config + configMap: + name: flink-config + - name: flink-secrets + secret: + secretName: flink-secrets +--- +# Flink TaskManager +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink-taskmanager + template: + metadata: + labels: + app: flink-taskmanager + spec: + initContainers: + - name: wait-for-jobmanager + image: busybox:1.36 + command: ['sh', '-c', 'until nc -z flink-jobmanager 6123; do echo waiting for jobmanager; sleep 2; done;'] + containers: + - name: flink-taskmanager + image: dexorder/ai-flink + imagePullPolicy: Always + args: ["taskmanager"] + env: + - name: JOB_MANAGER_RPC_ADDRESS + value: flink-jobmanager + - name: AWS_REGION + value: us-east-1 + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: flink-secrets + key: minio-access-key + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: flink-secrets + key: minio-secret-key + volumeMounts: + - name: flink-config + mountPath: /etc/config/config.yaml + subPath: config.yaml + - name: flink-secrets + mountPath: /etc/secrets + volumes: + - name: flink-config + configMap: + name: flink-config + - name: flink-secrets + secret: + secretName: flink-secrets diff --git a/deploy/k8s/base/relay.yaml b/deploy/k8s/base/relay.yaml new file mode 100644 index 00000000..8d8e6157 --- /dev/null +++ b/deploy/k8s/base/relay.yaml @@ -0,0 +1,55 @@ +# Relay (ZMQ router) +--- +apiVersion: v1 +kind: Service +metadata: + name: relay +spec: + selector: + app: relay + ports: + - name: market-data + protocol: TCP + port: 5558 + targetPort: 5558 + - name: client-requests + protocol: TCP + port: 5559 + targetPort: 5559 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: relay +spec: + replicas: 1 + selector: + matchLabels: + app: relay + template: + metadata: + labels: + app: relay + spec: + containers: + - name: relay + image: dexorder/ai-relay + imagePullPolicy: Always + ports: + - containerPort: 5558 + name: market-data + - containerPort: 5559 + name: client-requests + env: + - name: RUST_LOG + value: relay=info + - name: CONFIG_PATH + value: /config/config.yaml + volumeMounts: + - name: relay-config + mountPath: /config + volumes: + - name: relay-config + configMap: + name: relay-config