backend redesign
This commit is contained in:
364
bin/dev
Executable file
364
bin/dev
Executable file
@@ -0,0 +1,364 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [COMMAND]"
|
||||
echo ""
|
||||
echo "Manage the minikube development environment"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " start Start minikube and deploy all services"
|
||||
echo " stop Stop minikube"
|
||||
echo " restart [svc] Rebuild and redeploy all services, or just one (relay|ingestor|flink)"
|
||||
echo " rebuild [svc] Rebuild all custom images, or just one"
|
||||
echo " deploy [svc] Deploy/update all services, or just one"
|
||||
echo " status Show status of all services"
|
||||
echo " logs Tail logs for a service"
|
||||
echo " shell Open a shell in a service pod"
|
||||
echo " clean Delete all resources and volumes"
|
||||
echo " tunnel Start minikube tunnel (for LoadBalancer access)"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 start # Start minikube and deploy everything"
|
||||
echo " $0 rebuild # Rebuild all custom images"
|
||||
echo " $0 logs relay # Tail logs for relay service"
|
||||
echo " $0 shell ingestor # Open shell in ingestor pod"
|
||||
exit 1
|
||||
}
|
||||
|
||||
COMMAND="${1:-start}"
|
||||
|
||||
check_minikube() {
|
||||
if ! command -v minikube &> /dev/null; then
|
||||
echo -e "${RED}Error: minikube not found. Please install minikube first.${NC}"
|
||||
echo "https://minikube.sigs.k8s.io/docs/start/"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_kubectl() {
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}Error: kubectl not found. Please install kubectl first.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
start_minikube() {
|
||||
echo -e "${BLUE}Starting minikube...${NC}"
|
||||
|
||||
if minikube status &> /dev/null; then
|
||||
echo -e "${GREEN}✓ Minikube already running${NC}"
|
||||
else
|
||||
minikube start --cpus=6 --memory=12g --driver=docker
|
||||
echo -e "${GREEN}✓ Minikube started${NC}"
|
||||
fi
|
||||
|
||||
# Enable ingress addon
|
||||
echo -e "${BLUE}Enabling ingress addon...${NC}"
|
||||
minikube addons enable ingress
|
||||
echo -e "${GREEN}✓ Ingress enabled${NC}"
|
||||
|
||||
# Set docker environment
|
||||
echo -e "${YELLOW}Setting docker environment to minikube...${NC}"
|
||||
eval $(minikube docker-env)
|
||||
echo -e "${GREEN}✓ Docker environment set${NC}"
|
||||
|
||||
# Add /etc/hosts entry
|
||||
MINIKUBE_IP=$(minikube ip)
|
||||
if ! grep -q "dexorder.local" /etc/hosts; then
|
||||
echo -e "${YELLOW}Adding dexorder.local to /etc/hosts (requires sudo)...${NC}"
|
||||
echo "$MINIKUBE_IP dexorder.local" | sudo tee -a /etc/hosts
|
||||
else
|
||||
echo -e "${GREEN}✓ /etc/hosts entry exists${NC}"
|
||||
fi
|
||||
}
|
||||
|
||||
rebuild_images() {
|
||||
local service="${1:-all}"
|
||||
echo -e "${BLUE}Building custom images...${NC}"
|
||||
|
||||
# Use minikube's docker daemon
|
||||
eval $(minikube docker-env)
|
||||
|
||||
# Build images using the standard bin/build script with dev flag
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# Load existing tags so we preserve whichever services we're not rebuilding
|
||||
if [ -f "$ROOT_DIR/.dev-image-tag" ]; then
|
||||
source "$ROOT_DIR/.dev-image-tag"
|
||||
fi
|
||||
|
||||
# Helper: run build, show output, and return just the dev tag via stdout
|
||||
# Build output goes to stderr so the caller can capture only the tag via $()
|
||||
build_and_get_tag() {
|
||||
local svc="$1"
|
||||
local output
|
||||
output=$("$SCRIPT_DIR/build" "$svc" dev 2>&1) || { echo "$output" >&2; return 1; }
|
||||
echo "$output" >&2
|
||||
# Extract tag from "built <remote>/ai-<svc>:<tag>" line
|
||||
echo "$output" | grep -oE "ai-${svc}:dev[0-9]+" | tail -1 | cut -d: -f2
|
||||
}
|
||||
|
||||
if [ "$service" == "all" ] || [ "$service" == "relay" ]; then
|
||||
echo -e "${GREEN}→${NC} Building relay..."
|
||||
RELAY_TAG=$(build_and_get_tag relay) || exit 1
|
||||
docker tag "dexorder/ai-relay:$RELAY_TAG" "dexorder/relay:$RELAY_TAG"
|
||||
fi
|
||||
|
||||
if [ "$service" == "all" ] || [ "$service" == "ingestor" ]; then
|
||||
echo -e "${GREEN}→${NC} Building ingestor..."
|
||||
INGEST_TAG=$(build_and_get_tag ingestor) || exit 1
|
||||
docker tag "dexorder/ai-ingestor:$INGEST_TAG" "dexorder/ingestor:$INGEST_TAG"
|
||||
fi
|
||||
|
||||
if [ "$service" == "all" ] || [ "$service" == "flink" ]; then
|
||||
echo -e "${GREEN}→${NC} Building flink..."
|
||||
FLINK_TAG=$(build_and_get_tag flink) || exit 1
|
||||
docker tag "dexorder/ai-flink:$FLINK_TAG" "dexorder/flink:$FLINK_TAG"
|
||||
fi
|
||||
|
||||
# Save the tags for deployment (all three, preserving any we didn't rebuild)
|
||||
echo "RELAY_TAG=$RELAY_TAG" > "$ROOT_DIR/.dev-image-tag"
|
||||
echo "INGEST_TAG=$INGEST_TAG" >> "$ROOT_DIR/.dev-image-tag"
|
||||
echo "FLINK_TAG=$FLINK_TAG" >> "$ROOT_DIR/.dev-image-tag"
|
||||
|
||||
echo -e "${GREEN}✓ Images built: relay=$RELAY_TAG, ingestor=$INGEST_TAG, flink=$FLINK_TAG${NC}"
|
||||
}
|
||||
|
||||
deploy_services() {
|
||||
echo -e "${BLUE}Deploying services to minikube...${NC}"
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# Get the dev image tags
|
||||
if [ -f "$ROOT_DIR/.dev-image-tag" ]; then
|
||||
source "$ROOT_DIR/.dev-image-tag"
|
||||
echo -e "${BLUE}Using image tags:${NC}"
|
||||
echo -e " Relay: $RELAY_TAG"
|
||||
echo -e " Ingestor: $INGEST_TAG"
|
||||
echo -e " Flink: $FLINK_TAG"
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ No dev tags found. Using 'latest'. Run rebuild first.${NC}"
|
||||
RELAY_TAG="latest"
|
||||
INGEST_TAG="latest"
|
||||
FLINK_TAG="latest"
|
||||
fi
|
||||
|
||||
# Create secrets first (if they exist)
|
||||
echo -e "${GREEN}→${NC} Checking secrets..."
|
||||
if ls deploy/k8s/dev/secrets/*.yaml &> /dev/null; then
|
||||
"$SCRIPT_DIR/secret-update" dev || echo -e "${YELLOW} (Some secrets missing - copy from .example files)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠️ No secrets found. Copy from .example files:${NC}"
|
||||
echo -e "${YELLOW} cd deploy/k8s/dev/secrets${NC}"
|
||||
echo -e "${YELLOW} cp ai-secrets.yaml.example ai-secrets.yaml${NC}"
|
||||
echo -e "${YELLOW} # Edit with actual values, then run: bin/secret-update dev${NC}"
|
||||
fi
|
||||
|
||||
# Update configs
|
||||
echo -e "${GREEN}→${NC} Updating configs..."
|
||||
"$SCRIPT_DIR/config-update" dev
|
||||
|
||||
# Apply kustomize with image tag substitution
|
||||
echo -e "${GREEN}→${NC} Applying Kubernetes manifests..."
|
||||
kubectl kustomize deploy/k8s/dev/ | \
|
||||
sed "s|image: dexorder/flink:latest|image: dexorder/flink:$FLINK_TAG|g" | \
|
||||
sed "s|image: dexorder/relay:latest|image: dexorder/relay:$RELAY_TAG|g" | \
|
||||
sed "s|image: dexorder/ingestor:latest|image: dexorder/ingestor:$INGEST_TAG|g" | \
|
||||
kubectl apply -f -
|
||||
|
||||
echo -e "${GREEN}✓ Services deployed${NC}"
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}Waiting for deployments to be ready...${NC}"
|
||||
kubectl wait --for=condition=available --timeout=300s \
|
||||
deployment/relay \
|
||||
deployment/ingestor \
|
||||
deployment/iceberg-catalog \
|
||||
deployment/flink-jobmanager \
|
||||
deployment/flink-taskmanager \
|
||||
2>/dev/null || echo -e "${YELLOW}(Some deployments not ready yet)${NC}"
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}✓ Dev environment ready!${NC}"
|
||||
echo ""
|
||||
echo -e "${BLUE}Access the application:${NC}"
|
||||
echo -e " Web UI: http://dexorder.local/cryptochimp/"
|
||||
echo -e " Backend WS: ws://dexorder.local/ws"
|
||||
echo ""
|
||||
echo -e "${BLUE}Admin UIs (use port-forward):${NC}"
|
||||
echo -e " Flink UI: kubectl port-forward svc/flink-jobmanager 8081:8081"
|
||||
echo -e " Then open http://localhost:8081"
|
||||
echo -e " MinIO Console: kubectl port-forward svc/minio 9001:9001"
|
||||
echo -e " Then open http://localhost:9001"
|
||||
echo ""
|
||||
echo -e "${YELLOW}Note: Run 'minikube tunnel' in another terminal for dexorder.local ingress to work${NC}"
|
||||
}
|
||||
|
||||
show_status() {
|
||||
echo -e "${BLUE}Kubernetes Resources:${NC}"
|
||||
echo ""
|
||||
kubectl get pods,svc,ingress
|
||||
}
|
||||
|
||||
show_logs() {
|
||||
local service="$1"
|
||||
if [ -z "$service" ]; then
|
||||
echo -e "${RED}Error: Please specify a service name${NC}"
|
||||
echo "Available services: relay, ingestor, flink-jobmanager, flink-taskmanager, kafka, postgres, minio, iceberg-catalog"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Try to find pod by label or name
|
||||
local pod=$(kubectl get pods -l app="$service" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
||||
if [ -z "$pod" ]; then
|
||||
pod=$(kubectl get pods | grep "$service" | head -n1 | awk '{print $1}')
|
||||
fi
|
||||
|
||||
if [ -z "$pod" ]; then
|
||||
echo -e "${RED}Error: No pod found for service '$service'${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}Tailing logs for $pod...${NC}"
|
||||
kubectl logs -f "$pod"
|
||||
}
|
||||
|
||||
open_shell() {
|
||||
local service="$1"
|
||||
if [ -z "$service" ]; then
|
||||
echo -e "${RED}Error: Please specify a service name${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local pod=$(kubectl get pods -l app="$service" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
||||
if [ -z "$pod" ]; then
|
||||
pod=$(kubectl get pods | grep "$service" | head -n1 | awk '{print $1}')
|
||||
fi
|
||||
|
||||
if [ -z "$pod" ]; then
|
||||
echo -e "${RED}Error: No pod found for service '$service'${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}Opening shell in $pod...${NC}"
|
||||
kubectl exec -it "$pod" -- /bin/sh || kubectl exec -it "$pod" -- /bin/bash
|
||||
}
|
||||
|
||||
clean_all() {
|
||||
echo -e "${RED}⚠️ WARNING: This will delete all resources and volumes!${NC}"
|
||||
read -p "Are you sure? (yes/no): " confirm
|
||||
if [[ "$confirm" != "yes" ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}Deleting all resources...${NC}"
|
||||
kubectl delete -k deploy/k8s/dev/ || true
|
||||
kubectl delete pvc --all || true
|
||||
echo -e "${GREEN}✓ Resources deleted${NC}"
|
||||
}
|
||||
|
||||
start_tunnel() {
|
||||
echo -e "${BLUE}Starting minikube tunnel...${NC}"
|
||||
echo -e "${YELLOW}This requires sudo and will run in the foreground.${NC}"
|
||||
echo -e "${YELLOW}Press Ctrl+C to stop.${NC}"
|
||||
echo ""
|
||||
minikube tunnel
|
||||
}
|
||||
|
||||
# Deploy a single service using kubectl set image with the dev tag (never uses 'latest')
|
||||
deploy_service() {
|
||||
local service="$1"
|
||||
|
||||
if [ -f "$ROOT_DIR/.dev-image-tag" ]; then
|
||||
source "$ROOT_DIR/.dev-image-tag"
|
||||
fi
|
||||
|
||||
local image
|
||||
case "$service" in
|
||||
relay) image="dexorder/relay:$RELAY_TAG" ;;
|
||||
ingestor) image="dexorder/ingestor:$INGEST_TAG" ;;
|
||||
flink) image="dexorder/flink:$FLINK_TAG" ;;
|
||||
*)
|
||||
echo -e "${RED}Unknown service: $service. Use relay, ingestor, or flink.${NC}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo -e "${GREEN}→${NC} Deploying $service with image $image..."
|
||||
case "$service" in
|
||||
flink)
|
||||
kubectl set image deployment/flink-jobmanager flink-jobmanager=$image
|
||||
kubectl set image deployment/flink-taskmanager flink-taskmanager=$image
|
||||
;;
|
||||
*)
|
||||
kubectl set image deployment/$service $service=$image
|
||||
;;
|
||||
esac
|
||||
echo -e "${GREEN}✓ $service updated to $image${NC}"
|
||||
}
|
||||
|
||||
# Main command routing
|
||||
check_minikube
|
||||
check_kubectl
|
||||
|
||||
case "$COMMAND" in
|
||||
start)
|
||||
start_minikube
|
||||
rebuild_images
|
||||
deploy_services
|
||||
;;
|
||||
stop)
|
||||
echo -e "${BLUE}Stopping minikube...${NC}"
|
||||
minikube stop
|
||||
echo -e "${GREEN}✓ Minikube stopped${NC}"
|
||||
;;
|
||||
restart)
|
||||
if [ -n "$2" ]; then
|
||||
rebuild_images "$2"
|
||||
deploy_service "$2"
|
||||
else
|
||||
rebuild_images
|
||||
deploy_services
|
||||
fi
|
||||
;;
|
||||
rebuild)
|
||||
rebuild_images "${2:-}"
|
||||
;;
|
||||
deploy)
|
||||
if [ -n "$2" ]; then
|
||||
deploy_service "$2"
|
||||
else
|
||||
deploy_services
|
||||
fi
|
||||
;;
|
||||
status)
|
||||
show_status
|
||||
;;
|
||||
logs)
|
||||
show_logs "$2"
|
||||
;;
|
||||
shell)
|
||||
open_shell "$2"
|
||||
;;
|
||||
clean)
|
||||
clean_all
|
||||
;;
|
||||
tunnel)
|
||||
start_tunnel
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user