data pipeline refactor and fix

This commit is contained in:
2026-04-13 18:30:04 -04:00
parent 6418729b16
commit 326bf80846
96 changed files with 7107 additions and 1763 deletions

View File

@@ -45,6 +45,29 @@ else
MCP_URL="http://localhost:8080/mcp"
fi
# ---------- MinIO Bucket Initialization ----------
echo ""
echo -e "${BLUE}=== MinIO Storage Setup ===${NC}"
echo ""
echo -e "${BLUE}Waiting for MinIO pod...${NC}"
$KUBECTL wait --for=condition=ready --timeout=120s pod -l app=minio 2>/dev/null || {
echo -e "${YELLOW}⚠️ MinIO not ready after 120s, skipping bucket setup${NC}"
}
MINIO_POD=$($KUBECTL get pods -l app=minio -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -n "$MINIO_POD" ]; then
echo -e "${GREEN}→${NC} Ensuring warehouse bucket exists..."
MINIO_USER=$($KUBECTL exec "$MINIO_POD" -- sh -c 'echo $MINIO_ROOT_USER' 2>/dev/null | tr -d '\r')
MINIO_PASS=$($KUBECTL exec "$MINIO_POD" -- sh -c 'echo $MINIO_ROOT_PASSWORD' 2>/dev/null | tr -d '\r')
$KUBECTL exec "$MINIO_POD" -- mc alias set local http://localhost:9000 "$MINIO_USER" "$MINIO_PASS" > /dev/null 2>&1
$KUBECTL exec "$MINIO_POD" -- mc mb --ignore-existing local/warehouse > /dev/null 2>&1
echo -e "${GREEN}✓ Warehouse bucket ready${NC}"
else
echo -e "${YELLOW}⚠️ MinIO pod not found, skipping bucket setup${NC}"
fi
# ---------- Schema Initialization ----------
echo ""