data pipeline refactor and fix

This commit is contained in:
2026-04-13 18:30:04 -04:00
parent 6418729b16
commit 326bf80846
96 changed files with 7107 additions and 1763 deletions

46
bin/dev
View File

@@ -99,6 +99,12 @@ start_minikube() {
fi
}
generate_gateway_config_dev() {
sed "s|SANDBOX_IMAGE_TAG|dexorder/ai-sandbox:$SANDBOX_TAG|g; s|SIDECAR_IMAGE_TAG|dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" \
"$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml.tpl" \
> "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
}
rebuild_images() {
local service="${1:-all}"
echo -e "${BLUE}Building custom images...${NC}"
@@ -221,12 +227,7 @@ deploy_services() {
# Update configs
echo -e "${GREEN}→${NC} Updating configs..."
# Template gateway-config.yaml with actual image tags (backup first for safe restore)
local _gw_bak
_gw_bak=$(mktemp)
cp "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml" "$_gw_bak"
sed -i "s|sandbox_image: dexorder/ai-sandbox:.*|sandbox_image: dexorder/ai-sandbox:$SANDBOX_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
sed -i "s|sidecar_image: dexorder/ai-lifecycle-sidecar:.*|sidecar_image: dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
generate_gateway_config_dev
"$SCRIPT_DIR/config-update" dev
@@ -264,10 +265,6 @@ EOF
# Clean up the appended image tags from kustomization.yaml
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
# Restore gateway-config.yaml from backup
cp "$_gw_bak" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
rm "$_gw_bak"
echo -e "${GREEN}✓ Services deployed${NC}"
echo ""
@@ -525,6 +522,9 @@ deep_restart() {
;;
esac
echo -e "${GREEN}→${NC} Rebuilding application images..."
rebuild_images
echo -e "${GREEN}→${NC} Redeploying services..."
deploy_services
@@ -589,11 +589,7 @@ deploy_service() {
gateway)
image_name="dexorder/ai-gateway"
image_tag="$GATEWAY_TAG"
# Also need to template gateway-config.yaml (backup for safe restore)
_gw_bak_single=$(mktemp)
cp "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml" "$_gw_bak_single"
sed -i "s|sandbox_image: dexorder/ai-sandbox:.*|sandbox_image: dexorder/ai-sandbox:$SANDBOX_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
sed -i "s|sidecar_image: dexorder/ai-lifecycle-sidecar:.*|sidecar_image: dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
generate_gateway_config_dev
"$SCRIPT_DIR/config-update" dev
;;
web)
@@ -623,12 +619,6 @@ EOF
# Clean up the appended image tags from kustomization.yaml
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
# Restore gateway-config.yaml from backup if we modified it
if [ "$service" == "gateway" ]; then
cp "$_gw_bak_single" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
rm "$_gw_bak_single"
fi
echo -e "${GREEN}✓ $service deployed${NC}"
}
@@ -713,15 +703,10 @@ case "$COMMAND" in
cd "$ROOT_DIR/deploy/k8s/dev"
# Template gateway-config if gateway is in the list (backup for safe restore)
_ms_gw_bak=""
# Regenerate gateway-config if gateway is in the list
for svc in "${deploy_services_list[@]}"; do
if [ "$svc" == "gateway" ]; then
_ms_gw_bak=$(mktemp)
cp "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml" "$_ms_gw_bak"
sed -i "s|sandbox_image: dexorder/ai-sandbox:.*|sandbox_image: dexorder/ai-sandbox:$SANDBOX_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
sed -i "s|sidecar_image: dexorder/ai-lifecycle-sidecar:.*|sidecar_image: dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG|g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
"$SCRIPT_DIR/config-update" dev
generate_gateway_config_dev
break
fi
done
@@ -744,11 +729,6 @@ case "$COMMAND" in
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
# Restore gateway-config from backup if we modified it
if [ -n "$_ms_gw_bak" ]; then
cp "$_ms_gw_bak" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
rm "$_ms_gw_bak"
fi
fi
# Handle sandbox separately