bugfixes

deploy/k8s/base/ingestor.yaml

@@ -0,0 +1,63 @@
+# Ingestor deployment
+# CCXT market data ingestor worker — connects to Flink via ZMQ, writes to Kafka
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ingestor
+  labels:
+    app: ingestor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ingestor
+  template:
+    metadata:
+      labels:
+        app: ingestor
+    spec:
+      initContainers:
+        - name: wait-for-kafka
+          image: busybox:1.36
+          command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']
+        - name: wait-for-flink
+          image: busybox:1.36
+          command: ['sh', '-c', 'until nc -z flink-jobmanager 5567; do echo waiting for flink-jobmanager; sleep 2; done;']
+
+      volumes:
+        - name: config
+          configMap:
+            name: ingestor-config
+        - name: secrets
+          secret:
+            secretName: ingestor-secrets
+
+      containers:
+        - name: ingestor
+          image: dexorder/ai-ingestor
+          imagePullPolicy: Always
+
+          volumeMounts:
+            - name: config
+              mountPath: /config/config.yaml
+              subPath: config.yaml
+              readOnly: true
+            - name: secrets
+              mountPath: /config/secrets.yaml
+              subPath: secrets.yaml
+              readOnly: true
+
+          env:
+            - name: CONFIG_PATH
+              value: "/config/config.yaml"
+            - name: SECRETS_PATH
+              value: "/config/secrets.yaml"
+
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "1Gi"
+              cpu: "500m"

deploy/k8s/dev/infrastructure.yaml

@@ -668,11 +668,17 @@ spec:
           value: info
         - name: CONFIG_PATH
           value: /config/config.yaml
+        - name: SECRETS_PATH
+          value: /config/secrets.yaml
         volumeMounts:
         - name: ingestor-config
-          mountPath: /config
+          mountPath: /config/config.yaml
+          subPath: config.yaml
+          readOnly: true
         - name: ingestor-secrets
-          mountPath: /secrets
+          mountPath: /config/secrets.yaml
+          subPath: secrets.yaml
+          readOnly: true
       volumes:
       - name: ingestor-config
         configMap:

deploy/k8s/prod/infrastructure.yaml

@@ -664,11 +664,17 @@ spec:
           value: info
         - name: CONFIG_PATH
           value: /config/config.yaml
+        - name: SECRETS_PATH
+          value: /config/secrets.yaml
         volumeMounts:
         - name: ingestor-config
-          mountPath: /config
+          mountPath: /config/config.yaml
+          subPath: config.yaml
+          readOnly: true
         - name: ingestor-secrets
-          mountPath: /secrets
+          mountPath: /config/secrets.yaml
+          subPath: secrets.yaml
+          readOnly: true
       volumes:
       - name: ingestor-config
         configMap:

deploy/k8s/prod/secrets/ingestor-secrets.tpl.yaml

@@ -5,9 +5,10 @@ metadata:
   namespace: ai
 type: Opaque
 stringData:
-  binance-api-key: "{{ op://AI Prod/Ingestor/binance_api_key }}"
+  secrets.yaml: |
-  binance-api-secret: "{{ op://AI Prod/Ingestor/binance_api_secret }}"
+    binance_api_key: "{{ op://AI Prod/Ingestor/binance_api_key }}"
-  coinbase-api-key: "{{ op://AI Prod/Ingestor/coinbase_api_key }}"
+    binance_api_secret: "{{ op://AI Prod/Ingestor/binance_api_secret }}"
-  coinbase-api-secret: "{{ op://AI Prod/Ingestor/coinbase_api_secret }}"
+    coinbase_api_key: "{{ op://AI Prod/Ingestor/coinbase_api_key }}"
-  kraken-api-key: "{{ op://AI Prod/Ingestor/kraken_api_key }}"
+    coinbase_api_secret: "{{ op://AI Prod/Ingestor/coinbase_api_secret }}"
-  kraken-api-secret: "{{ op://AI Prod/Ingestor/kraken_api_secret }}"
+    kraken_api_key: "{{ op://AI Prod/Ingestor/kraken_api_key }}"
+    kraken_api_secret: "{{ op://AI Prod/Ingestor/kraken_api_secret }}"