backend redesign

kafka/README.md

@@ -0,0 +1,74 @@
+# Kafka Deployment for K8s Cluster
+
+## Install Strimzi Operator
+
+```bash
+# Add the Strimzi Helm repository (if using Helm 3.6+)
+helm install strimzi-kafka-operator oci://quay.io/strimzi-helm/strimzi-kafka-operator \
+  -f values.yaml \
+  --namespace kafka --create-namespace
+
+# Wait for operator to be ready
+kubectl wait --for=condition=ready pod -l name=strimzi-cluster-operator -n kafka --timeout=300s
+```
+
+## Deploy Kafka Cluster
+
+```bash
+# Apply the metrics ConfigMap first
+kubectl apply -f kafka-metrics-config.yaml
+
+# Apply the Kafka cluster manifest
+kubectl apply -f kafka-cluster.yaml
+
+# Wait for Kafka to be ready (this may take a few minutes)
+kubectl wait kafka/trading-cluster --for=condition=Ready --timeout=300s -n default
+```
+
+## Verify Installation
+
+```bash
+# Check Kafka cluster status
+kubectl get kafka -n default
+
+# Check all pods
+kubectl get pods -n default | grep trading-cluster
+
+# Check Kafka cluster details
+kubectl describe kafka trading-cluster -n default
+```
+
+## Connect to Kafka
+
+Internal connection string (from within cluster):
+- **Plaintext**: `trading-cluster-kafka-bootstrap.default.svc:9092`
+- **TLS**: `trading-cluster-kafka-bootstrap.default.svc:9093`
+
+## Prometheus Metrics
+
+Kafka and ZooKeeper expose metrics on port 9404. Prometheus will automatically discover and scrape these metrics via pod annotations:
+- `prometheus.io/scrape: "true"`
+- `prometheus.io/port: "9404"`
+- `prometheus.io/path: "/metrics"`
+
+To verify metrics are being exported:
+```bash
+kubectl exec -it trading-cluster-kafka-0 -n default -- curl localhost:9404/metrics
+```
+
+## Create a Test Topic
+
+```bash
+kubectl run kafka-producer -ti --image=quay.io/strimzi/kafka:0.43.0-kafka-3.7.0 --rm=true --restart=Never -- \
+  bin/kafka-topics.sh --create --topic test-topic --bootstrap-server trading-cluster-kafka-bootstrap:9092 --partitions 3 --replication-factor 3
+```
+
+## Uninstall
+
+```bash
+# Delete Kafka cluster
+kubectl delete kafka trading-cluster -n default
+
+# Delete operator
+helm uninstall strimzi-kafka-operator -n kafka
+```

kafka/kafka-cluster.yaml

@@ -0,0 +1,85 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: trading-cluster
+  namespace: default
+  labels:
+    app: kafka
+spec:
+  kafka:
+    version: 3.7.0
+    replicas: 1
+    metricsConfig:
+      type: jmxPrometheusExporter
+      valueFrom:
+        configMapKeyRef:
+          name: kafka-metrics
+          key: kafka-metrics-config.yml
+    template:
+      pod:
+        metadata:
+          annotations:
+            prometheus.io/scrape: "true"
+            prometheus.io/port: "9404"
+            prometheus.io/path: "/metrics"
+    listeners:
+      - name: plain
+        port: 9092
+        type: internal
+        tls: false
+      - name: tls
+        port: 9093
+        type: internal
+        tls: true
+    config:
+      offsets.topic.replication.factor: 1
+      transaction.state.log.replication.factor: 1
+      transaction.state.log.min.isr: 1
+      default.replication.factor: 1
+      min.insync.replicas: 1
+      log.retention.hours: 168
+      log.segment.bytes: 1073741824
+      log.retention.check.interval.ms: 300000
+    storage:
+      type: jbod
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: 5Gi
+        deleteClaim: false
+    resources:
+      requests:
+        memory: 512Mi
+        cpu: 250m
+      limits:
+        memory: 1Gi
+        cpu: 500m
+  zookeeper:
+    replicas: 1
+    metricsConfig:
+      type: jmxPrometheusExporter
+      valueFrom:
+        configMapKeyRef:
+          name: kafka-metrics
+          key: zookeeper-metrics-config.yml
+    template:
+      pod:
+        metadata:
+          annotations:
+            prometheus.io/scrape: "true"
+            prometheus.io/port: "9404"
+            prometheus.io/path: "/metrics"
+    storage:
+      type: persistent-claim
+      size: 2Gi
+      deleteClaim: false
+    resources:
+      requests:
+        memory: 256Mi
+        cpu: 100m
+      limits:
+        memory: 512Mi
+        cpu: 250m
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}

kafka/kafka-metrics-config.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kafka-metrics
+  namespace: default
+data:
+  kafka-metrics-config.yml: |
+    lowercaseOutputName: true
+    rules:
+    # Kafka broker metrics
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
+      labels:
+        clientId: "$3"
+        topic: "$4"
+        partition: "$5"
+    - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), brokerHost=(.+), brokerPort=(.+)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
+      labels:
+        clientId: "$3"
+        broker: "$4:$5"
+    - pattern: kafka.server<type=(.+), name=(.+)><>Value
+      name: kafka_server_$1_$2
+      type: GAUGE
+    # Kafka network metrics
+    - pattern: kafka.network<type=(.+), name=(.+)><>Value
+      name: kafka_network_$1_$2
+      type: GAUGE
+    # Kafka log metrics
+    - pattern: kafka.log<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
+      name: kafka_log_$1_$2
+      type: GAUGE
+      labels:
+        topic: "$3"
+        partition: "$4"
+
+  zookeeper-metrics-config.yml: |
+    lowercaseOutputName: true
+    rules:
+    - pattern: "org.apache.ZooKeeperService<name0=(.+)><>(\\w+)"
+      name: zookeeper_$2
+      type: GAUGE

kafka/values.yaml

@@ -0,0 +1,9 @@
+# Strimzi Kafka Operator Helm Values
+# Install with: helm install strimzi-kafka-operator oci://quay.io/strimzi-helm/strimzi-kafka-operator
+
+# This values file is for the operator installation
+# The operator itself is lightweight
+#watchNamespaces: []  # Empty = watch all namespaces
+
+watchNamespaces:
+  - default