153 lines
3.8 KiB
YAML
153 lines
3.8 KiB
YAML
# Flink JobManager + TaskManager
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: flink-jobmanager
|
|
spec:
|
|
selector:
|
|
app: flink-jobmanager
|
|
ports:
|
|
- name: rpc
|
|
protocol: TCP
|
|
port: 6123
|
|
targetPort: 6123
|
|
- name: ui
|
|
protocol: TCP
|
|
port: 8081
|
|
targetPort: 8081
|
|
- name: zmq-market-data
|
|
protocol: TCP
|
|
port: 5558
|
|
targetPort: 5558
|
|
- name: zmq-notif-pull
|
|
protocol: TCP
|
|
port: 5561
|
|
targetPort: 5561
|
|
- name: zmq-client-pull
|
|
protocol: TCP
|
|
port: 5566
|
|
targetPort: 5566
|
|
- name: zmq-ingestor-broker
|
|
protocol: TCP
|
|
port: 5567
|
|
targetPort: 5567
|
|
type: ClusterIP
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: flink-jobmanager
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: flink-jobmanager
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: flink-jobmanager
|
|
spec:
|
|
initContainers:
|
|
- name: wait-for-kafka
|
|
image: busybox:1.36
|
|
command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']
|
|
- name: wait-for-iceberg-catalog
|
|
image: busybox:1.36
|
|
command: ['sh', '-c', 'until nc -z iceberg-catalog 8181; do echo waiting for iceberg-catalog; sleep 2; done;']
|
|
containers:
|
|
- name: flink-jobmanager
|
|
image: dexorder/ai-flink
|
|
imagePullPolicy: Always
|
|
args: ["standalone-job", "--job-classname", "com.dexorder.flink.TradingFlinkApp"]
|
|
ports:
|
|
- containerPort: 6123
|
|
name: rpc
|
|
- containerPort: 8081
|
|
name: ui
|
|
- containerPort: 5558
|
|
name: zmq-market-data
|
|
- containerPort: 5561
|
|
name: zmq-notif-pull
|
|
env:
|
|
- name: JOB_MANAGER_RPC_ADDRESS
|
|
value: flink-jobmanager
|
|
- name: AWS_REGION
|
|
value: us-east-1
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: flink-secrets
|
|
key: minio-access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: flink-secrets
|
|
key: minio-secret-key
|
|
volumeMounts:
|
|
- name: flink-config
|
|
mountPath: /etc/config/config.yaml
|
|
subPath: config.yaml
|
|
- name: flink-secrets
|
|
mountPath: /etc/secrets
|
|
volumes:
|
|
- name: flink-config
|
|
configMap:
|
|
name: flink-config
|
|
- name: flink-secrets
|
|
secret:
|
|
secretName: flink-secrets
|
|
---
|
|
# Flink TaskManager
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: flink-taskmanager
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: flink-taskmanager
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: flink-taskmanager
|
|
spec:
|
|
initContainers:
|
|
- name: wait-for-jobmanager
|
|
image: busybox:1.36
|
|
command: ['sh', '-c', 'until nc -z flink-jobmanager 6123; do echo waiting for jobmanager; sleep 2; done;']
|
|
containers:
|
|
- name: flink-taskmanager
|
|
image: dexorder/ai-flink
|
|
imagePullPolicy: Always
|
|
args: ["taskmanager"]
|
|
env:
|
|
- name: JOB_MANAGER_RPC_ADDRESS
|
|
value: flink-jobmanager
|
|
- name: AWS_REGION
|
|
value: us-east-1
|
|
- name: AWS_ACCESS_KEY_ID
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: flink-secrets
|
|
key: minio-access-key
|
|
- name: AWS_SECRET_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: flink-secrets
|
|
key: minio-secret-key
|
|
volumeMounts:
|
|
- name: flink-config
|
|
mountPath: /etc/config/config.yaml
|
|
subPath: config.yaml
|
|
- name: flink-secrets
|
|
mountPath: /etc/secrets
|
|
volumes:
|
|
- name: flink-config
|
|
configMap:
|
|
name: flink-config
|
|
- name: flink-secrets
|
|
secret:
|
|
secretName: flink-secrets
|