dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

data pipeline refactor and fix

Browse Source
This commit is contained in:
Tim Olson
2026-04-13 18:30:04 -04:00
parent 6418729b16
commit 326bf80846
96 changed files with 7107 additions and 1763 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
bin/create-user
View File

@@ -21,6 +21,10 @@ usage() {
}
ENV="${1:-dev}"
ARG_EMAIL="${2:-}"
ARG_PASSWORD="${3:-}"
ARG_NAME="${4:-}"
ARG_LICENSE="${5:-}"
if [[ "$ENV" != "dev" && "$ENV" != "prod" ]]; then
echo -e "${RED}Error: Environment must be 'dev' or 'prod'${NC}"
@@ -44,16 +48,36 @@ if [ -z "$PG_POD" ]; then
exit 1
fi
# Prompt for credentials
read -p "Email: " USER_EMAIL
read -rs -p "Password (min 8 chars): " USER_PASSWORD
echo ""
# Get credentials — from args or interactively
if [[ -n "$ARG_EMAIL" ]]; then
USER_EMAIL="$ARG_EMAIL"
else
read -p "Email: " USER_EMAIL
fi
if [[ -n "$ARG_PASSWORD" ]]; then
USER_PASSWORD="$ARG_PASSWORD"
else
read -rs -p "Password (min 8 chars): " USER_PASSWORD
echo ""
fi
if [[ ${#USER_PASSWORD} -lt 8 ]]; then
echo -e "${RED}✗ Password must be at least 8 characters${NC}"
exit 1
fi
read -p "Display name: " USER_NAME
read -p "License type [free|pro|enterprise] (default: pro): " LICENSE_TYPE
if [[ -n "$ARG_NAME" ]]; then
USER_NAME="$ARG_NAME"
else
read -p "Display name: " USER_NAME
fi
if [[ -n "$ARG_LICENSE" ]]; then
LICENSE_TYPE="$ARG_LICENSE"
else
read -p "License type [free|pro|enterprise] (default: pro): " LICENSE_TYPE
fi
LICENSE_TYPE="${LICENSE_TYPE:-pro}"
# Check if user already exists

2
bin/deploy
View File

@@ -43,7 +43,7 @@ if [ "$PROJECT" == "dev" ]; then
fi
if [ "$DEV" == "1" ]; then
TAG="dev`date +%Y%m%d%H%M%S`"
TAG="dev`date -u +%Y%m%d%H%M%S`"
if [ "$1" != "" ]; then
CONFIG=$1
shift

158
bin/deploy-all Executable file
View File

@@ -0,0 +1,158 @@
#!/usr/bin/env bash
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
KUBECTL="kubectl --context=prod"
CLEAR_SANDBOXES=0
usage() {
echo "Usage: $0 [--sandboxes]"
echo ""
echo "Deploy all services to production. Does NOT update secrets (use bin/secret-update)."
echo ""
echo "Steps performed:"
echo " 1. Apply base kustomize manifests (namespaces, RBAC, policies)"
echo " 2. Apply infrastructure.yaml (statefulsets, deployments)"
echo " 3. Run bin/config-update prod"
echo " 4. Build and deploy all application images"
echo " 5. Wait for rollouts"
echo ""
echo "Options:"
echo " --sandboxes Delete sandbox Deployments and Services (PVCs are retained)."
echo " The gateway will recreate sandboxes on next user login."
echo ""
exit 1
}
for arg in "$@"; do
case "$arg" in
--sandboxes)
CLEAR_SANDBOXES=1
;;
--help|-h)
usage
;;
*)
echo -e "${RED}Unknown argument: $arg${NC}"
usage
;;
esac
done
echo -e "${YELLOW}╔══════════════════════════════════════════╗${NC}"
echo -e "${YELLOW}║ PRODUCTION FULL DEPLOY ║${NC}"
echo -e "${YELLOW}╚══════════════════════════════════════════╝${NC}"
echo ""
echo -e "${YELLOW}⚠️ This will update ALL production services.${NC}"
echo -e "${YELLOW} Secrets are NOT updated (run bin/secret-update prod separately).${NC}"
if [ "$CLEAR_SANDBOXES" == "1" ]; then
echo -e "${YELLOW} Sandbox deployments will be DELETED (PVCs retained).${NC}"
fi
echo ""
read -p "Are you sure you want to continue? (yes/no): " confirm
if [[ "$confirm" != "yes" ]]; then
echo "Aborted."
exit 0
fi
step() {
echo ""
echo -e "${BLUE}━━━ $1 ━━━${NC}"
}
ok() {
echo -e "${GREEN}✓${NC} $1"
}
fail() {
echo -e "${RED}✗ $1${NC}"
exit 1
}
# ── Step 1: Base kustomize manifests ─────────────────────────────────────────
step "Step 1/5: Applying base kustomize manifests"
cd "$ROOT_DIR"
$KUBECTL apply -k deploy/k8s/prod/
ok "Base manifests applied (namespaces, RBAC, policies, quotas)"
# ── Step 2: Infrastructure ────────────────────────────────────────────────────
step "Step 2/5: Applying infrastructure.yaml"
$KUBECTL -n ai apply -f deploy/k8s/prod/infrastructure.yaml
ok "Infrastructure applied"
# ── Step 3: Configs ───────────────────────────────────────────────────────────
step "Step 3/5: Updating configs"
# config-update prod will prompt for confirmation; we already confirmed above,
# so feed "yes" automatically via stdin.
echo "yes" | "$SCRIPT_DIR/config-update" prod
ok "Configs updated"
# ── Step 4: Build and deploy all application images ───────────────────────────
step "Step 4/5: Building and deploying application images"
echo ""
SERVICES=(gateway web sandbox lifecycle-sidecar flink relay ingestor)
for service in "${SERVICES[@]}"; do
echo -e "${GREEN}→${NC} Deploying $service..."
"$SCRIPT_DIR/deploy" "$service" prod
ok "$service deployed"
echo ""
done
# ── Step 4b: Optionally clear sandbox deployments ─────────────────────────────
if [ "$CLEAR_SANDBOXES" == "1" ]; then
step "Step 4b: Clearing sandbox deployments"
SANDBOX_DEPLOYS=$($KUBECTL -n sandbox get deployments -o name 2>/dev/null || true)
SANDBOX_SVCS=$($KUBECTL -n sandbox get services -o name 2>/dev/null || true)
if [ -z "$SANDBOX_DEPLOYS" ]; then
echo " No sandbox deployments found."
else
echo " Deleting sandbox deployments..."
echo "$SANDBOX_DEPLOYS" | xargs $KUBECTL -n sandbox delete
ok "Sandbox deployments deleted"
fi
if [ -n "$SANDBOX_SVCS" ]; then
echo " Deleting sandbox services..."
echo "$SANDBOX_SVCS" | xargs $KUBECTL -n sandbox delete
ok "Sandbox services deleted"
fi
echo -e "${YELLOW} PVCs retained — gateway will recreate sandboxes on next login.${NC}"
fi
# ── Step 5: Wait for rollouts ─────────────────────────────────────────────────
step "Step 5/5: Waiting for rollouts"
ROLLOUTS=(
"deployment/gateway"
"deployment/ai-web"
"deployment/relay"
"deployment/ingestor"
"deployment/flink-jobmanager"
"deployment/flink-taskmanager"
)
for r in "${ROLLOUTS[@]}"; do
echo -e "${GREEN}→${NC} Waiting for $r..."
$KUBECTL -n ai rollout status "$r" --timeout=180s || echo -e "${YELLOW} ⚠ $r did not become ready within 3 minutes${NC}"
done
echo ""
echo -e "${GREEN}╔══════════════════════════════════════════╗${NC}"
echo -e "${GREEN}║ Deploy complete! ║${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════╝${NC}"
echo ""
echo " Verify: curl -I https://dexorder.ai/api/health"
echo ""

46
bin/dev
View File

@@ -99,6 +99,12 @@ start_minikube() {
fi
}
generate_gateway_config_dev() {
sed "s|SANDBOX_IMAGE_TAG|dexorder/ai-sandbox:$SANDBOX_TAG|g; s|SIDECAR_IMAGE_TAG|dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" \
"$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml.tpl" \
> "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
}
rebuild_images() {
local service="${1:-all}"
echo -e "${BLUE}Building custom images...${NC}"
@@ -221,12 +227,7 @@ deploy_services() {
# Update configs
echo -e "${GREEN}→${NC} Updating configs..."
# Template gateway-config.yaml with actual image tags (backup first for safe restore)
local _gw_bak
_gw_bak=$(mktemp)
cp "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml" "$_gw_bak"
sed -i "s|sandbox_image: dexorder/ai-sandbox:.*|sandbox_image: dexorder/ai-sandbox:$SANDBOX_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
sed -i "s|sidecar_image: dexorder/ai-lifecycle-sidecar:.*|sidecar_image: dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
generate_gateway_config_dev
"$SCRIPT_DIR/config-update" dev
@@ -264,10 +265,6 @@ EOF
# Clean up the appended image tags from kustomization.yaml
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
# Restore gateway-config.yaml from backup
cp "$_gw_bak" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
rm "$_gw_bak"
echo -e "${GREEN}✓ Services deployed${NC}"
echo ""
@@ -525,6 +522,9 @@ deep_restart() {
;;
esac
echo -e "${GREEN}→${NC} Rebuilding application images..."
rebuild_images
echo -e "${GREEN}→${NC} Redeploying services..."
deploy_services
@@ -589,11 +589,7 @@ deploy_service() {
gateway)
image_name="dexorder/ai-gateway"
image_tag="$GATEWAY_TAG"
# Also need to template gateway-config.yaml (backup for safe restore)
_gw_bak_single=$(mktemp)
cp "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml" "$_gw_bak_single"
sed -i "s|sandbox_image: dexorder/ai-sandbox:.*|sandbox_image: dexorder/ai-sandbox:$SANDBOX_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
sed -i "s|sidecar_image: dexorder/ai-lifecycle-sidecar:.*|sidecar_image: dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
generate_gateway_config_dev
"$SCRIPT_DIR/config-update" dev
;;
web)
@@ -623,12 +619,6 @@ EOF
# Clean up the appended image tags from kustomization.yaml
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
# Restore gateway-config.yaml from backup if we modified it
if [ "$service" == "gateway" ]; then
cp "$_gw_bak_single" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
rm "$_gw_bak_single"
fi
echo -e "${GREEN}✓ $service deployed${NC}"
}
@@ -713,15 +703,10 @@ case "$COMMAND" in
cd "$ROOT_DIR/deploy/k8s/dev"
# Template gateway-config if gateway is in the list (backup for safe restore)
_ms_gw_bak=""
# Regenerate gateway-config if gateway is in the list
for svc in "${deploy_services_list[@]}"; do
if [ "$svc" == "gateway" ]; then
_ms_gw_bak=$(mktemp)
cp "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml" "$_ms_gw_bak"
sed -i "s|sandbox_image: dexorder/ai-sandbox:.*|sandbox_image: dexorder/ai-sandbox:$SANDBOX_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
sed -i "s|sidecar_image: dexorder/ai-lifecycle-sidecar:.*|sidecar_image: dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
"$SCRIPT_DIR/config-update" dev
generate_gateway_config_dev
break
fi
done
@@ -744,11 +729,6 @@ case "$COMMAND" in
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
# Restore gateway-config from backup if we modified it
if [ -n "$_ms_gw_bak" ]; then
cp "$_ms_gw_bak" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
rm "$_ms_gw_bak"
fi
fi
# Handle sandbox separately

23
bin/init
View File

@@ -45,6 +45,29 @@ else
MCP_URL="http://localhost:8080/mcp"
fi
# ---------- MinIO Bucket Initialization ----------
echo ""
echo -e "${BLUE}=== MinIO Storage Setup ===${NC}"
echo ""
echo -e "${BLUE}Waiting for MinIO pod...${NC}"
$KUBECTL wait --for=condition=ready --timeout=120s pod -l app=minio 2>/dev/null || {
echo -e "${YELLOW}⚠️ MinIO not ready after 120s, skipping bucket setup${NC}"
}
MINIO_POD=$($KUBECTL get pods -l app=minio -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -n "$MINIO_POD" ]; then
echo -e "${GREEN}→${NC} Ensuring warehouse bucket exists..."
MINIO_USER=$($KUBECTL exec "$MINIO_POD" -- sh -c 'echo $MINIO_ROOT_USER' 2>/dev/null | tr -d '\r')
MINIO_PASS=$($KUBECTL exec "$MINIO_POD" -- sh -c 'echo $MINIO_ROOT_PASSWORD' 2>/dev/null | tr -d '\r')
$KUBECTL exec "$MINIO_POD" -- mc alias set local http://localhost:9000 "$MINIO_USER" "$MINIO_PASS" > /dev/null 2>&1
$KUBECTL exec "$MINIO_POD" -- mc mb --ignore-existing local/warehouse > /dev/null 2>&1
echo -e "${GREEN}✓ Warehouse bucket ready${NC}"
else
echo -e "${YELLOW}⚠️ MinIO pod not found, skipping bucket setup${NC}"
fi
# ---------- Schema Initialization ----------
echo ""