chart data loading

deploy/k8s/dev/infrastructure.yaml

@@ -158,9 +158,24 @@ spec:
         - -c
         - |
           CLUSTER_ID="dexorder-dev-cluster"
-          if [ ! -f /var/lib/kafka/data/meta.properties ]; then
-            /opt/kafka/bin/kafka-storage.sh format -t $CLUSTER_ID -c /opt/kafka/config/kraft/server.properties
+          LOG_DIR="/var/lib/kafka/data"
+
+          # Ensure log directory exists
+          mkdir -p $LOG_DIR
+
+          # Create temporary config with custom log.dirs for formatting
+          cp /opt/kafka/config/kraft/server.properties /tmp/server.properties
+          echo "log.dirs=$LOG_DIR" >> /tmp/server.properties
+
+          # Format storage if not already formatted
+          if [ ! -f $LOG_DIR/meta.properties ]; then
+            echo "Formatting Kafka storage with cluster ID: $CLUSTER_ID"
+            /opt/kafka/bin/kafka-storage.sh format -t $CLUSTER_ID -c /tmp/server.properties
+          else
+            echo "Kafka storage already formatted, skipping format step"
           fi
+
+          # Start Kafka server
           /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties \
             --override node.id=1 \
             --override process.roles=broker,controller \
@@ -169,7 +184,7 @@ spec:
             --override controller.quorum.voters=1@kafka:9093 \
             --override controller.listener.names=CONTROLLER \
             --override listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \
-            --override log.dirs=/var/lib/kafka/data \
+            --override log.dirs=$LOG_DIR \
             --override offsets.topic.replication.factor=1 \
             --override transaction.state.log.replication.factor=1 \
             --override transaction.state.log.min.isr=1
@@ -315,6 +330,44 @@ spec:
         requests:
           storage: 10Gi
 ---
+# MinIO bucket initialization job
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: minio-init-buckets
+spec:
+  ttlSecondsAfterFinished: 100
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: create-buckets
+        image: minio/mc:latest
+        command:
+        - sh
+        - -c
+        - |
+          echo "Waiting for MinIO to be ready..."
+          until mc alias set minio http://minio:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD; do
+            sleep 2
+          done
+
+          echo "Creating warehouse bucket..."
+          mc mb minio/warehouse --ignore-existing
+
+          echo "Buckets initialized successfully"
+        env:
+        - name: MINIO_ROOT_USER
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: root-user
+        - name: MINIO_ROOT_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: minio-secret
+              key: root-password
+---
 # Iceberg REST Catalog
 apiVersion: v1
 kind: Service
@@ -422,6 +475,9 @@ spec:
       - name: wait-for-kafka
         image: busybox:1.36
         command: ['sh', '-c', 'until nc -z kafka 9092; do echo waiting for kafka; sleep 2; done;']
+      - name: wait-for-iceberg-catalog
+        image: busybox:1.36
+        command: ['sh', '-c', 'until nc -z iceberg-catalog 8181; do echo waiting for iceberg-catalog; sleep 2; done;']
       containers:
       - name: flink-jobmanager
         image: dexorder/flink:latest