backend redesign

deploy/k8s/prod/configs/flink-config.yaml

@@ -0,0 +1,30 @@
+# Flink Job Configuration
+
+# ZeroMQ bind address and ports
+zmq_bind_address: "tcp://*"
+zmq_ingestor_work_queue_port: 5555
+zmq_ingestor_response_port: 5556
+zmq_ingestor_control_port: 5557
+zmq_market_data_pub_port: 5558
+zmq_client_request_port: 5559
+zmq_cep_webhook_port: 5560
+
+# Notification publisher endpoint (Flink → Relay)
+# Relay connects XSUB to this endpoint and proxies to clients
+notification_publish_endpoint: "tcp://*:5557"
+
+# Kafka configuration
+kafka_bootstrap_servers: "kafka:9092"
+kafka_tick_topic: "market-tick"
+kafka_ohlc_topic: "market-ohlc"
+
+# Iceberg catalog
+iceberg_catalog_uri: "http://iceberg-catalog:8181"
+iceberg_warehouse: "s3://trading-warehouse/"
+iceberg_namespace: "trading"
+iceberg_table_prefix: "market"
+hadoop_conf_dir: "/etc/hadoop/conf"
+
+# Flink configuration
+flink_parallelism: 2
+flink_checkpoint_interval_ms: 60000

deploy/k8s/prod/configs/ingestor-config.yaml

@@ -0,0 +1,24 @@
+# CCXT Ingestor Configuration
+
+# Relay ZMQ endpoints (relay is the well-known gateway)
+flink_hostname: relay
+ingestor_work_port: 5555    # SUB - receives DataRequest with exchange prefix
+# Note: No response port needed - async architecture via Kafka!
+
+# Supported exchanges (subscribe to these prefixes)
+supported_exchanges:
+  - BINANCE
+  - COINBASE
+  - KRAKEN
+
+# Kafka configuration
+kafka_brokers:
+  - kafka:9092
+kafka_topic: market-0
+
+# Worker configuration
+max_concurrent: 10
+poll_interval_ms: 10000
+
+# Logging
+log_level: info

deploy/k8s/prod/configs/relay-config.yaml

@@ -0,0 +1,19 @@
+# ZMQ Relay Configuration
+
+# Bind address for all relay sockets
+bind_address: "tcp://*"
+
+# Client-facing ports
+client_request_port: 5559          # ROUTER - Client historical data requests
+market_data_pub_port: 5558         # XPUB - Market data fanout to clients
+
+# Ingestor-facing ports
+ingestor_work_port: 5555           # PUB - Distribute work with exchange prefix
+ingestor_response_port: 5556       # ROUTER - Receive responses from ingestors
+
+# Flink connection
+flink_market_data_endpoint: "tcp://flink-jobmanager:5557"  # XSUB - Subscribe to Flink market data
+
+# Timeouts and limits
+request_timeout_secs: 30           # Timeout for pending client requests
+high_water_mark: 10000             # ZMQ high water mark for all sockets

deploy/k8s/prod/kustomization.yaml

@@ -0,0 +1,40 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: default
+
+# Base resources (backend, web, ingress, init/gVisor)
+resources:
+  - ../base
+
+# Production patches
+patches:
+  - path: patches.yaml
+
+# ConfigMaps for service configs
+# In production, these might come from external sources
+# or be managed separately, but we'll include them here for consistency
+configMapGenerator:
+  - name: relay-config
+    files:
+      - config.yaml=../../configmaps/relay-config.yaml
+  - name: ingestor-config
+    files:
+      - config.yaml=../../configmaps/ingestor-config.yaml
+  - name: flink-config
+    files:
+      - config.yaml=../../configmaps/flink-config.yaml
+
+# Secrets (managed via kubectl, not committed)
+# These are created by bin/secret-update prod
+secretGenerator: []
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+# Images
+images:
+  - name: dexorder/ai-backend
+    newTag: latest
+  - name: dexorder/ai-web
+    newTag: latest

deploy/k8s/prod/patches.yaml

@@ -0,0 +1,52 @@
+---
+# Production backend patches
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ai-backend
+spec:
+  replicas: 2
+  template:
+    spec:
+      runtimeClassName: gvisor
+      containers:
+      - name: ai-backend
+        image: dexorder/ai-backend:latest
+        imagePullPolicy: Always
+        env:
+        - name: CONFIG
+          value: "prod"
+        resources:
+          requests:
+            memory: "2Gi"
+            cpu: "1000m"
+          limits:
+            memory: "4Gi"
+            cpu: "2000m"
+---
+# Production web patches
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ai-web
+spec:
+  replicas: 2
+  template:
+    spec:
+      runtimeClassName: gvisor
+      containers:
+      - name: ai-web
+        image: dexorder/ai-web:latest
+        imagePullPolicy: Always
+        env:
+        - name: VITE_BASE_PATH
+          value: "/cryptochimp/"
+        - name: VITE_WS_URL
+          value: "wss://dexorder.ai/ws"
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "250m"
+          limits:
+            memory: "1Gi"
+            cpu: "500m"

deploy/k8s/prod/secrets/ai-secrets.yaml.example

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ai-secrets
+type: Opaque
+stringData:
+  anthropic-api-key: "sk-ant-YOUR_KEY_HERE"

deploy/k8s/prod/secrets/ingestor-secrets.yaml.example

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ingestor-secrets
+type: Opaque
+stringData:
+  # Exchange API keys (if needed for authenticated endpoints)
+  binance-api-key: ""
+  binance-api-secret: ""
+  coinbase-api-key: ""
+  coinbase-api-secret: ""
+  kraken-api-key: ""
+  kraken-api-secret: ""

deploy/k8s/prod/secrets/minio-secret.yaml.example

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secret
+type: Opaque
+stringData:
+  root-user: "CHANGE_THIS_IN_PRODUCTION"
+  root-password: "CHANGE_THIS_IN_PRODUCTION"

deploy/k8s/prod/secrets/postgres-secret.yaml.example

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-secret
+type: Opaque
+stringData:
+  password: "CHANGE_THIS_IN_PRODUCTION"