backend redesign

deploy/k8s/dev/configs/flink-config.yaml

@@ -0,0 +1,40 @@
+# Flink Job Configuration
+
+# ZeroMQ bind address and ports
+zmq_bind_address: "tcp://*"
+zmq_ingestor_work_queue_port: 5555
+zmq_ingestor_response_port: 5556
+zmq_ingestor_control_port: 5557
+zmq_market_data_pub_port: 5558
+zmq_client_request_port: 5559
+zmq_cep_webhook_port: 5560
+
+# Notification endpoints
+# Task managers PUSH to job manager PULL socket at this address
+notification_publish_endpoint: "tcp://flink-jobmanager:5561"
+# Job manager binds PULL socket on this port to receive from task managers
+notification_pull_port: 5561
+
+# Kafka configuration
+kafka_bootstrap_servers: "kafka:9092"
+kafka_tick_topic: "market-tick"
+kafka_ohlc_topic: "market-ohlc"
+kafka_topics_file: "/topics-dev.yaml"  # Use topics-dev.yaml for single broker dev environment
+
+# Iceberg catalog
+iceberg_catalog_uri: "http://iceberg-catalog:8181"
+iceberg_warehouse: "s3://trading-warehouse/"
+iceberg_namespace: "trading"
+iceberg_table_prefix: "market"
+hadoop_conf_dir: "/etc/hadoop/conf"
+
+# Flink configuration
+flink_parallelism: 1
+flink_checkpoint_interval_ms: 60000
+
+# Flink memory configuration (required)
+jobmanager.memory.process.size: 1600m
+taskmanager.memory.process.size: 1728m
+taskmanager.numberOfTaskSlots: 2
+jobmanager.rpc.address: flink-jobmanager
+jobmanager.rpc.port: 6123

deploy/k8s/dev/configs/ingestor-config.yaml

@@ -0,0 +1,24 @@
+# CCXT Ingestor Configuration
+
+# Relay ZMQ endpoints (relay is the well-known gateway)
+flink_hostname: relay
+ingestor_work_port: 5555    # SUB - receives DataRequest with exchange prefix
+# Note: No response port needed - async architecture via Kafka!
+
+# Supported exchanges (subscribe to these prefixes)
+supported_exchanges:
+  - BINANCE
+  - COINBASE
+  - KRAKEN
+
+# Kafka configuration
+kafka_brokers:
+  - kafka:9092
+kafka_topic: market-ohlc
+
+# Worker configuration
+max_concurrent: 10
+poll_interval_ms: 10000
+
+# Logging
+log_level: info

deploy/k8s/dev/configs/relay-config.yaml

@@ -0,0 +1,19 @@
+# ZMQ Relay Configuration
+
+# Bind address for all relay sockets
+bind_address: "tcp://*"
+
+# Client-facing ports
+client_request_port: 5559          # ROUTER - Client historical data requests
+market_data_pub_port: 5558         # XPUB - Market data fanout to clients
+
+# Ingestor-facing ports
+ingestor_work_port: 5555           # PUB - Distribute work with exchange prefix
+ingestor_response_port: 5556       # ROUTER - Receive responses from ingestors
+
+# Flink connection
+flink_market_data_endpoint: "tcp://flink-jobmanager:5558"  # XSUB - Subscribe to Flink market data (MARKET_DATA_PUB)
+
+# Timeouts and limits
+request_timeout_secs: 30           # Timeout for pending client requests
+high_water_mark: 10000             # ZMQ high water mark for all sockets

deploy/k8s/dev/infrastructure.yaml

@@ -0,0 +1,519 @@
+---
+# Kafka (KRaft mode - no Zookeeper needed)
+# Using apache/kafka:3.9.0 instead of confluentinc/cp-kafka because:
+# - cp-kafka's entrypoint script has issues with KRaft configuration
+# - apache/kafka allows explicit command configuration
+# - For production, use Strimzi operator (see kafka/ directory)
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka
+spec:
+  selector:
+    app: kafka
+  ports:
+    - name: broker
+      protocol: TCP
+      port: 9092
+      targetPort: 9092
+    - name: controller
+      protocol: TCP
+      port: 9093
+      targetPort: 9093
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kafka
+spec:
+  serviceName: kafka
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka
+  template:
+    metadata:
+      labels:
+        app: kafka
+    spec:
+      containers:
+      - name: kafka
+        image: apache/kafka:3.9.0
+        ports:
+        - containerPort: 9092
+          name: broker
+        - containerPort: 9093
+          name: controller
+        command:
+        - sh
+        - -c
+        - |
+          CLUSTER_ID="dexorder-dev-cluster"
+          if [ ! -f /var/lib/kafka/data/meta.properties ]; then
+            /opt/kafka/bin/kafka-storage.sh format -t $CLUSTER_ID -c /opt/kafka/config/kraft/server.properties
+          fi
+          /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties \
+            --override node.id=1 \
+            --override process.roles=broker,controller \
+            --override listeners=PLAINTEXT://:9092,CONTROLLER://:9093 \
+            --override advertised.listeners=PLAINTEXT://kafka:9092 \
+            --override controller.quorum.voters=1@kafka:9093 \
+            --override controller.listener.names=CONTROLLER \
+            --override listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \
+            --override log.dirs=/var/lib/kafka/data \
+            --override offsets.topic.replication.factor=1 \
+            --override transaction.state.log.replication.factor=1 \
+            --override transaction.state.log.min.isr=1
+        env: []
+        volumeMounts:
+        - name: kafka-data
+          mountPath: /var/lib/kafka/data
+  volumeClaimTemplates:
+  - metadata:
+      name: kafka-data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 5Gi
+---
+# PostgreSQL (for Iceberg catalog metadata)
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  selector:
+    app: postgres
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+spec:
+  serviceName: postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+      - name: postgres
+        image: postgres:15
+        ports:
+        - containerPort: 5432
+        env:
+        - name: POSTGRES_USER
+          value: postgres
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: postgres-secret
+              key: password
+        - name: POSTGRES_DB
+          value: iceberg
+        volumeMounts:
+        - name: postgres-data
+          mountPath: /var/lib/postgresql/data
+  volumeClaimTemplates:
+  - metadata:
+      name: postgres-data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 2Gi
+---
+# MinIO (S3-compatible object storage)
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+spec:
+  selector:
+    app: minio
+  ports:
+    - name: api
+      protocol: TCP
+      port: 9000
+      targetPort: 9000
+    - name: console
+      protocol: TCP
+      port: 9001
+      targetPort: 9001
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: minio
+spec:
+  serviceName: minio
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      containers:
+      - name: minio
+        image: minio/minio:latest
+        args:
+        - server
+        - /data
+        - --console-address
+        - ":9001"
+        ports:
+        - containerPort: 9000
+          name: api
+        - containerPort: 9001
+          name: console
+        env:
+        - name: MINIO_ROOT_USER
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: root-user
+        - name: MINIO_ROOT_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: root-password
+        volumeMounts:
+        - name: minio-data
+          mountPath: /data
+  volumeClaimTemplates:
+  - metadata:
+      name: minio-data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 10Gi
+---
+# Iceberg REST Catalog
+apiVersion: v1
+kind: Service
+metadata:
+  name: iceberg-catalog
+spec:
+  selector:
+    app: iceberg-catalog
+  ports:
+    - protocol: TCP
+      port: 8181
+      targetPort: 8181
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: iceberg-catalog
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: iceberg-catalog
+  template:
+    metadata:
+      labels:
+        app: iceberg-catalog
+    spec:
+      initContainers:
+      - name: wait-for-postgres
+        image: busybox:1.36
+        command: ['sh', '-c', 'until nc -z postgres 5432; do echo waiting for postgres; sleep 2; done;']
+      - name: wait-for-minio
+        image: busybox:1.36
+        command: ['sh', '-c', 'until nc -z minio 9000; do echo waiting for minio; sleep 2; done;']
+      containers:
+      - name: iceberg-catalog
+        image: tabulario/iceberg-rest:latest
+        ports:
+        - containerPort: 8181
+        env:
+        - name: CATALOG_WAREHOUSE
+          value: "s3://warehouse/"
+        - name: CATALOG_IO__IMPL
+          value: "org.apache.iceberg.aws.s3.S3FileIO"
+        - name: CATALOG_S3_ENDPOINT
+          value: "http://minio:9000"
+        - name: CATALOG_S3_ACCESS__KEY__ID
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: root-user
+        - name: CATALOG_S3_SECRET__ACCESS__KEY
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: root-password
+        - name: CATALOG_S3_PATH__STYLE__ACCESS
+          value: "true"
+        - name: AWS_REGION
+          value: "us-east-1"
+---
+# Flink JobManager
+apiVersion: v1
+kind: Service
+metadata:
+  name: flink-jobmanager
+spec:
+  selector:
+    app: flink-jobmanager
+  ports:
+    - name: rpc
+      protocol: TCP
+      port: 6123
+      targetPort: 6123
+    - name: ui
+      protocol: TCP
+      port: 8081
+      targetPort: 8081
+    - name: zmq-market-data
+      protocol: TCP
+      port: 5558
+      targetPort: 5558
+    - name: zmq-notif-pull
+      protocol: TCP
+      port: 5561
+      targetPort: 5561
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flink-jobmanager
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flink-jobmanager
+  template:
+    metadata:
+      labels:
+        app: flink-jobmanager
+    spec:
+      initContainers:
+      - name: wait-for-kafka
+        image: busybox:1.36
+        command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']
+      containers:
+      - name: flink-jobmanager
+        image: dexorder/flink:latest
+        imagePullPolicy: Never
+        args: ["standalone-job", "--job-classname", "com.dexorder.flink.TradingFlinkApp"]
+        ports:
+        - containerPort: 6123
+          name: rpc
+        - containerPort: 8081
+          name: ui
+        - containerPort: 5558
+          name: zmq-market-data
+        - containerPort: 5561
+          name: zmq-notif-pull
+        env:
+        - name: JOB_MANAGER_RPC_ADDRESS
+          value: flink-jobmanager
+        - name: AWS_REGION
+          value: us-east-1
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: flink-secrets
+              key: minio-access-key
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: flink-secrets
+              key: minio-secret-key
+        volumeMounts:
+        - name: flink-config
+          mountPath: /etc/config/config.yaml
+          subPath: config.yaml
+        - name: flink-secrets
+          mountPath: /etc/secrets
+      volumes:
+      - name: flink-config
+        configMap:
+          name: flink-config
+      - name: flink-secrets
+        secret:
+          secretName: flink-secrets
+---
+# Flink TaskManager
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flink-taskmanager
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flink-taskmanager
+  template:
+    metadata:
+      labels:
+        app: flink-taskmanager
+    spec:
+      initContainers:
+      - name: wait-for-jobmanager
+        image: busybox:1.36
+        command: ['sh', '-c', 'until nc -z flink-jobmanager 6123; do echo waiting for jobmanager; sleep 2; done;']
+      containers:
+      - name: flink-taskmanager
+        image: dexorder/flink:latest
+        imagePullPolicy: Never
+        args: ["taskmanager"]
+        env:
+        - name: JOB_MANAGER_RPC_ADDRESS
+          value: flink-jobmanager
+        - name: AWS_REGION
+          value: us-east-1
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: flink-secrets
+              key: minio-access-key
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: flink-secrets
+              key: minio-secret-key
+        volumeMounts:
+        - name: flink-config
+          mountPath: /etc/config/config.yaml
+          subPath: config.yaml
+        - name: flink-secrets
+          mountPath: /etc/secrets
+      volumes:
+      - name: flink-config
+        configMap:
+          name: flink-config
+      - name: flink-secrets
+        secret:
+          secretName: flink-secrets
+---
+# Relay (ZMQ router)
+apiVersion: v1
+kind: Service
+metadata:
+  name: relay
+spec:
+  selector:
+    app: relay
+  ports:
+    - name: work-queue
+      protocol: TCP
+      port: 5555
+      targetPort: 5555
+    - name: responses
+      protocol: TCP
+      port: 5556
+      targetPort: 5556
+    - name: market-data
+      protocol: TCP
+      port: 5558
+      targetPort: 5558
+    - name: client-requests
+      protocol: TCP
+      port: 5559
+      targetPort: 5559
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: relay
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: relay
+  template:
+    metadata:
+      labels:
+        app: relay
+    spec:
+      containers:
+      - name: relay
+        image: dexorder/relay:latest
+        imagePullPolicy: Never
+        ports:
+        - containerPort: 5555
+          name: work-queue
+        - containerPort: 5556
+          name: responses
+        - containerPort: 5558
+          name: market-data
+        - containerPort: 5559
+          name: client-requests
+        env:
+        - name: RUST_LOG
+          value: relay=info
+        - name: CONFIG_PATH
+          value: /config/config.yaml
+        volumeMounts:
+        - name: relay-config
+          mountPath: /config
+      volumes:
+      - name: relay-config
+        configMap:
+          name: relay-config
+---
+# Ingestor (CCXT data fetcher)
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ingestor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ingestor
+  template:
+    metadata:
+      labels:
+        app: ingestor
+    spec:
+      initContainers:
+      - name: wait-for-relay
+        image: busybox:1.36
+        command: ['sh', '-c', 'until nc -z relay 5555; do echo waiting for relay; sleep 2; done;']
+      - name: wait-for-kafka
+        image: busybox:1.36
+        command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']
+      containers:
+      - name: ingestor
+        image: dexorder/ingestor:latest
+        imagePullPolicy: Never
+        env:
+        - name: LOG_LEVEL
+          value: info
+        - name: CONFIG_PATH
+          value: /config/config.yaml
+        volumeMounts:
+        - name: ingestor-config
+          mountPath: /config
+        - name: ingestor-secrets
+          mountPath: /secrets
+      volumes:
+      - name: ingestor-config
+        configMap:
+          name: ingestor-config
+      - name: ingestor-secrets
+        secret:
+          secretName: ingestor-secrets

deploy/k8s/dev/ingress-dev.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ai-ingress
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: dexorder.local
+    http:
+      paths: []

deploy/k8s/dev/kustomization.yaml

@@ -0,0 +1,32 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: default
+
+# Base resources
+resources:
+  - ../base
+  - infrastructure.yaml
+
+# No patches needed currently
+patches: []
+  # ingress-dev.yaml - removed until we have services to expose
+
+# ConfigMaps for service configs
+configMapGenerator:
+  - name: relay-config
+    files:
+      - config.yaml=configs/relay-config.yaml
+  - name: ingestor-config
+    files:
+      - config.yaml=configs/ingestor-config.yaml
+  - name: flink-config
+    files:
+      - config.yaml=configs/flink-config.yaml
+
+# Secrets (managed via kubectl, not committed)
+# These are created by bin/secret-update
+secretGenerator: []
+
+generatorOptions:
+  disableNameSuffixHash: true

deploy/k8s/dev/secrets/ai-secrets.yaml.example

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ai-secrets
+type: Opaque
+stringData:
+  anthropic-api-key: "sk-ant-YOUR_KEY_HERE"

deploy/k8s/dev/secrets/flink-secrets.yaml.example

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: flink-secrets
+type: Opaque
+stringData:
+  # MinIO/S3 credentials for Iceberg S3FileIO
+  minio-access-key: "minio"
+  minio-secret-key: "minio123"

deploy/k8s/dev/secrets/ingestor-secrets.yaml.example

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ingestor-secrets
+type: Opaque
+stringData:
+  # Exchange API keys (if needed for authenticated endpoints)
+  binance-api-key: ""
+  binance-api-secret: ""
+  coinbase-api-key: ""
+  coinbase-api-secret: ""
+  kraken-api-key: ""
+  kraken-api-secret: ""

deploy/k8s/dev/secrets/minio-secret.yaml.example

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secret
+type: Opaque
+stringData:
+  root-user: "minio"
+  root-password: "minio123"

deploy/k8s/dev/secrets/postgres-secret.yaml.example

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-secret
+type: Opaque
+stringData:
+  password: "password"