chart data loading

This commit is contained in:
2026-03-24 21:37:49 -04:00
parent f6bd22a8ef
commit c76887ab92
65 changed files with 6350 additions and 713 deletions

View File

@@ -3,9 +3,9 @@
#REMOTE=northamerica-northeast2-docker.pkg.dev/dexorder-430504/dexorder
REMOTE=${REMOTE:-git.dxod.org/dexorder/dexorder}
if [ "$1" != "flink" ] && [ "$1" != "relay" ] && [ "$1" != "ingestor" ] && [ "$1" != "web" ]; then
if [ "$1" != "flink" ] && [ "$1" != "relay" ] && [ "$1" != "ingestor" ] && [ "$1" != "web" ] && [ "$1" != "gateway" ] && [ "$1" != "lifecycle-sidecar" ] && [ "$1" != "client-py" ]; then
echo
echo usage: "$0 "'{flink|relay|ingestor|web} [''dev''] [config] [deployment] [kubernetes] [image_tag]'
echo usage: "$0 "'{flink|relay|ingestor|web|gateway|lifecycle-sidecar|client-py} [''dev''] [config] [deployment] [kubernetes] [image_tag]'
echo
echo ' [''dev''] if the literal string ''dev'' is not the second argument, then the build refuses to run if source code is not checked in. Otherwise, the git revision numbers are used in the image tag.'
echo
@@ -94,9 +94,11 @@ fi
echo $ACTION $PROJECT config=$CONFIG deployment=$DEPLOYMENT '=>' $TAG
# Copy protobuf definitions into project directory for Docker build
# Copy protobuf definitions into project directory for Docker build (if not gateway or lifecycle-sidecar)
# Using rsync --checksum so unchanged files keep their timestamps (preserves docker layer cache)
rsync -a --checksum --delete protobuf/ $PROJECT/protobuf/
if [ "$PROJECT" != "lifecycle-sidecar" ]; then
rsync -a --checksum --delete protobuf/ $PROJECT/protobuf/
fi
docker build $NO_CACHE -f $PROJECT/Dockerfile --build-arg="CONFIG=$CONFIG" --build-arg="DEPLOYMENT=$DEPLOYMENT" -t dexorder/ai-$PROJECT:latest $PROJECT || exit 1
@@ -110,6 +112,9 @@ if [ "$IMG_TAG" != "" ]; then
fi
echo "$(date)" built $REMOTE/ai-$PROJECT:$TAG
# Output just the tag for scripting purposes (to stderr so scripts can capture it)
echo "$TAG" >&2
if [ "$DEPLOY" == "1" ]; then
docker push $REMOTE/ai-$PROJECT:$TAG
YAML=$(sed "s#image: dexorder/ai-$PROJECT*#image: $REMOTE/ai-$PROJECT:$TAG#" deploy/k8s/$KUBERNETES.yaml)

260
bin/dev
View File

@@ -19,7 +19,7 @@ usage() {
echo "Commands:"
echo " start Start minikube and deploy all services"
echo " stop [--keep-data] Stop minikube (deletes PVCs by default)"
echo " restart [svc] Rebuild and redeploy all services, or just one (relay|ingestor|flink|gateway|sidecar|web)"
echo " restart [svc] Rebuild and redeploy all services, or just one (relay|ingestor|flink|gateway|sidecar|web|client-py)"
echo " deep-restart [svc] Restart StatefulSet(s) and delete their PVCs (kafka|postgres|minio|qdrant|all)"
echo " rebuild [svc] Rebuild all custom images, or just one"
echo " deploy [svc] Deploy/update all services, or just one"
@@ -115,14 +115,18 @@ rebuild_images() {
fi
# Helper: run build, show output, and return just the dev tag via stdout
# Build output goes to stderr so the caller can capture only the tag via $()
# bin/build now outputs the tag on its last line to stderr
build_and_get_tag() {
local svc="$1"
local output
local tag
# Capture stderr (which contains both output and the tag)
output=$("$SCRIPT_DIR/build" "$svc" dev 2>&1) || { echo "$output" >&2; return 1; }
echo "$output" >&2
# Extract tag from "built <remote>/ai-<svc>:<tag>" line
echo "$output" | grep -oE "ai-${svc}:dev[0-9]+" | tail -1 | cut -d: -f2
# Show the build output (excluding the final tag line)
echo "$output" | head -n -1 >&2
# Return just the tag (last line)
tag=$(echo "$output" | tail -n 1)
echo "$tag"
}
if [ "$service" == "all" ] || [ "$service" == "relay" ]; then
@@ -146,31 +150,27 @@ rebuild_images() {
# Build gateway (Node.js application)
if [ "$service" == "all" ] || [ "$service" == "gateway" ]; then
echo -e "${GREEN}→${NC} Building gateway..."
cd "$ROOT_DIR/gateway"
GATEWAY_TAG="dev$(date +%Y%m%d%H%M%S)"
docker build -t dexorder/gateway:latest -t dexorder/gateway:$GATEWAY_TAG . || exit 1
echo -e "${GREEN}✓ Built dexorder/gateway:$GATEWAY_TAG${NC}"
cd "$ROOT_DIR"
GATEWAY_TAG=$(build_and_get_tag gateway) || exit 1
docker tag "dexorder/ai-gateway:$GATEWAY_TAG" "dexorder/gateway:$GATEWAY_TAG"
fi
# Build lifecycle-sidecar (Go binary)
if [ "$service" == "all" ] || [ "$service" == "lifecycle-sidecar" ] || [ "$service" == "sidecar" ]; then
echo -e "${GREEN}→${NC} Building lifecycle-sidecar..."
cd "$ROOT_DIR/lifecycle-sidecar"
SIDECAR_TAG="dev$(date +%Y%m%d%H%M%S)"
docker build -t lifecycle-sidecar:latest -t lifecycle-sidecar:$SIDECAR_TAG . || exit 1
echo -e "${GREEN}✓ Built lifecycle-sidecar:$SIDECAR_TAG${NC}"
cd "$ROOT_DIR"
SIDECAR_TAG=$(build_and_get_tag lifecycle-sidecar) || exit 1
docker tag "dexorder/ai-lifecycle-sidecar:$SIDECAR_TAG" "lifecycle-sidecar:$SIDECAR_TAG"
fi
# Build web (Vue.js application)
if [ "$service" == "all" ] || [ "$service" == "web" ]; then
echo -e "${GREEN}→${NC} Building web..."
cd "$ROOT_DIR/web"
WEB_TAG="dev$(date +%Y%m%d%H%M%S)"
docker build -t dexorder/ai-web:latest -t dexorder/ai-web:$WEB_TAG . || exit 1
echo -e "${GREEN}✓ Built dexorder/ai-web:$WEB_TAG${NC}"
cd "$ROOT_DIR"
WEB_TAG=$(build_and_get_tag web) || exit 1
fi
# Build client-py (Python client library)
if [ "$service" == "all" ] || [ "$service" == "client-py" ]; then
echo -e "${GREEN}→${NC} Building client-py..."
CLIENT_PY_TAG=$(build_and_get_tag client-py) || exit 1
fi
# Save the tags for deployment (all services, preserving any we didn't rebuild)
@@ -180,8 +180,9 @@ rebuild_images() {
echo "GATEWAY_TAG=$GATEWAY_TAG" >> "$ROOT_DIR/.dev-image-tag"
echo "SIDECAR_TAG=$SIDECAR_TAG" >> "$ROOT_DIR/.dev-image-tag"
echo "WEB_TAG=$WEB_TAG" >> "$ROOT_DIR/.dev-image-tag"
echo "CLIENT_PY_TAG=$CLIENT_PY_TAG" >> "$ROOT_DIR/.dev-image-tag"
echo -e "${GREEN}✓ Images built: relay=$RELAY_TAG, ingestor=$INGEST_TAG, flink=$FLINK_TAG, gateway=$GATEWAY_TAG, sidecar=$SIDECAR_TAG, web=$WEB_TAG${NC}"
echo -e "${GREEN}✓ Images built: relay=$RELAY_TAG, ingestor=$INGEST_TAG, flink=$FLINK_TAG, gateway=$GATEWAY_TAG, sidecar=$SIDECAR_TAG, web=$WEB_TAG, client-py=$CLIENT_PY_TAG${NC}"
}
deploy_services() {
@@ -268,89 +269,43 @@ EOF
pg_pod=$(kubectl get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -n "$pg_pod" ]; then
# Wait for postgres to actually be ready to accept connections
echo -e "${GREEN}→${NC} Verifying postgres is ready to accept connections..."
for i in {1..30}; do
if kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -c "SELECT 1;" > /dev/null 2>&1; then
echo -e "${GREEN}✓ Postgres ready${NC}"
break
fi
if [ $i -eq 30 ]; then
echo -e "${RED}✗ Postgres not ready after 30 seconds${NC}"
exit 1
fi
sleep 1
done
table_count=$(kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'user';" 2>/dev/null | tr -d ' ')
if [ "$table_count" = "1" ]; then
echo -e "${GREEN}✓ Gateway schema already exists${NC}"
else
echo -e "${GREEN}→${NC} Applying gateway schema..."
kubectl exec -i "$pg_pod" -- psql -U postgres -d iceberg < "$ROOT_DIR/gateway/schema.sql" > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo -e "${GREEN}✓ Gateway schema initialized${NC}"
if kubectl exec -i "$pg_pod" -- psql -U postgres -d iceberg < "$ROOT_DIR/gateway/schema.sql" > /dev/null 2>&1; then
# Verify schema was actually created
sleep 1
table_count=$(kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'user';" 2>/dev/null | tr -d ' ')
if [ "$table_count" = "1" ]; then
echo -e "${GREEN}✓ Gateway schema initialized${NC}"
else
echo -e "${RED}✗ Failed to verify schema creation${NC}"
exit 1
fi
else
echo -e "${YELLOW}⚠️ Failed to initialize gateway schema${NC}"
echo -e "${RED}✗ Failed to initialize gateway schema${NC}"
exit 1
fi
fi
# Create dev user via Better Auth API (skip if already exists)
echo -e "${GREEN}→${NC} Checking for dev user..."
user_id=$(kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -t -c "SELECT id FROM \"user\" WHERE email = 'cryptochimp@dexorder.ai';" 2>/dev/null | tr -d ' ')
if [ -n "$user_id" ]; then
echo -e "${GREEN}✓ Dev user already exists (cryptochimp@dexorder.ai)${NC}"
else
echo -e "${GREEN}→${NC} Creating dev user via Better Auth API..."
echo -e "${BLUE}Waiting for gateway to be ready...${NC}"
kubectl wait --for=condition=available --timeout=120s deployment/gateway 2>/dev/null || {
echo -e "${YELLOW}⚠️ Gateway not ready after 120s${NC}"
}
# Give gateway a few seconds to start accepting requests
sleep 5
# Create user via custom auth endpoint
response=$(curl -s -w "\n%{http_code}" -X POST "http://dexorder.local/api/auth/register" \
-H "Content-Type: application/json" \
-d '{
"email": "cryptochimp@dexorder.ai",
"password": "moon2the",
"name": "Crypto Chimp"
}' 2>&1)
http_code=$(echo "$response" | tail -n1)
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
echo -e "${GREEN}✓ User created via auth API${NC}"
elif [ "$http_code" = "400" ]; then
echo -e "${YELLOW}⚠️ User may already exist (status 400)${NC}"
else
echo -e "${YELLOW}⚠️ API call returned status $http_code${NC}"
fi
# Wait a moment for database to be updated
sleep 2
# Check again if user exists now
user_id=$(kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -t -c "SELECT id FROM \"user\" WHERE email = 'cryptochimp@dexorder.ai';" 2>/dev/null | tr -d ' ')
if [ -n "$user_id" ]; then
echo -e "${GREEN}✓ Dev user confirmed in database${NC}"
fi
fi
if [ -n "$user_id" ]; then
# Create/update license for the user
echo -e "${GREEN}→${NC} Creating pro license for dev user..."
kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -c "
INSERT INTO user_licenses (user_id, email, license_type, mcp_server_url, features, resource_limits, preferred_model)
VALUES (
'$user_id',
'cryptochimp@dexorder.ai',
'pro',
'http://localhost:8080/mcp',
'{\"maxIndicators\":50,\"maxStrategies\":20,\"maxBacktestDays\":365,\"realtimeData\":true,\"customExecutors\":true,\"apiAccess\":true}',
'{\"maxConcurrentSessions\":5,\"maxMessagesPerDay\":1000,\"maxTokensPerMessage\":8192,\"rateLimitPerMinute\":60}',
'{\"provider\":\"anthropic\",\"model\":\"claude-3-5-sonnet-20241022\",\"temperature\":0.7}'
)
ON CONFLICT (user_id) DO UPDATE SET
license_type = EXCLUDED.license_type,
features = EXCLUDED.features,
resource_limits = EXCLUDED.resource_limits,
preferred_model = EXCLUDED.preferred_model,
updated_at = NOW();
" > /dev/null 2>&1
echo -e "${GREEN}✓ Dev user ready (cryptochimp@dexorder.ai / moon2the)${NC}"
else
echo -e "${YELLOW}⚠️ Could not create dev user (gateway may not be ready)${NC}"
fi
# Create dev user (refactored into reusable function)
create_dev_user
fi
echo ""
@@ -369,6 +324,97 @@ EOF
echo -e "${YELLOW}Note: Run 'minikube tunnel' in another terminal for dexorder.local ingress to work${NC}"
}
create_dev_user() {
# Dev user configuration (single source of truth)
local DEV_EMAIL="tim@dexorder.ai"
local DEV_PASSWORD="test1234"
local DEV_NAME="Tim"
local LICENSE_TYPE="pro"
echo -e "${BLUE}Initializing dev user...${NC}"
# Find postgres pod
local pg_pod=$(kubectl get pods -l app=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
if [ -z "$pg_pod" ]; then
echo -e "${YELLOW}⚠️ Postgres pod not found${NC}"
return 1
fi
# Check if user already exists
echo -e "${GREEN}→${NC} Checking for dev user..."
local user_id=$(kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -t -c "SELECT id FROM \"user\" WHERE email = '$DEV_EMAIL';" 2>/dev/null | tr -d ' ')
if [ -n "$user_id" ]; then
echo -e "${GREEN}✓ Dev user already exists ($DEV_EMAIL)${NC}"
else
echo -e "${GREEN}→${NC} Creating dev user via Better Auth API..."
echo -e "${BLUE}Waiting for gateway to be ready...${NC}"
kubectl wait --for=condition=available --timeout=120s deployment/gateway 2>/dev/null || {
echo -e "${YELLOW}⚠️ Gateway not ready after 120s${NC}"
}
# Give gateway a few seconds to start accepting requests
sleep 5
# Create user via custom auth endpoint
local response=$(curl -s -w "\n%{http_code}" -X POST "http://dexorder.local/api/auth/register" \
-H "Content-Type: application/json" \
-d '{
"email": "'"$DEV_EMAIL"'",
"password": "'"$DEV_PASSWORD"'",
"name": "'"$DEV_NAME"'"
}' 2>&1)
local http_code=$(echo "$response" | tail -n1)
if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then
echo -e "${GREEN}✓ User created via auth API${NC}"
elif [ "$http_code" = "400" ]; then
echo -e "${YELLOW}⚠️ User may already exist (status 400)${NC}"
else
echo -e "${YELLOW}⚠️ API call returned status $http_code${NC}"
local body=$(echo "$response" | head -n -1)
echo -e "${YELLOW}Response: $body${NC}"
fi
# Wait a moment for database to be updated
sleep 2
# Check again if user exists now
user_id=$(kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -t -c "SELECT id FROM \"user\" WHERE email = '$DEV_EMAIL';" 2>/dev/null | tr -d ' ')
if [ -n "$user_id" ]; then
echo -e "${GREEN}✓ Dev user confirmed in database${NC}"
fi
fi
if [ -n "$user_id" ]; then
# Create/update license for the user
echo -e "${GREEN}→${NC} Creating $LICENSE_TYPE license for dev user..."
kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -c "
INSERT INTO user_licenses (user_id, email, license_type, mcp_server_url, features, resource_limits, preferred_model)
VALUES (
'$user_id',
'$DEV_EMAIL',
'$LICENSE_TYPE',
'http://localhost:8080/mcp',
'{\"maxIndicators\":50,\"maxStrategies\":20,\"maxBacktestDays\":365,\"realtimeData\":true,\"customExecutors\":true,\"apiAccess\":true}',
'{\"maxConcurrentSessions\":5,\"maxMessagesPerDay\":1000,\"maxTokensPerMessage\":8192,\"rateLimitPerMinute\":60}',
'{\"provider\":\"anthropic\",\"model\":\"claude-3-5-sonnet-20241022\",\"temperature\":0.7}'
)
ON CONFLICT (user_id) DO UPDATE SET
license_type = EXCLUDED.license_type,
features = EXCLUDED.features,
resource_limits = EXCLUDED.resource_limits,
preferred_model = EXCLUDED.preferred_model,
updated_at = NOW();
" > /dev/null 2>&1
echo -e "${GREEN}✓ Dev user ready ($DEV_EMAIL / $DEV_PASSWORD)${NC}"
else
echo -e "${YELLOW}⚠️ Could not create dev user (gateway may not be ready)${NC}"
return 1
fi
}
show_status() {
echo -e "${BLUE}Kubernetes Resources:${NC}"
echo ""
@@ -472,12 +518,18 @@ deep_restart() {
kubectl delete statefulset postgres || true
sleep 2
delete_pvcs postgres
# Force restart iceberg-catalog since it depends on postgres
echo -e "${GREEN}→${NC} Force restarting iceberg-catalog (depends on postgres)..."
kubectl delete pod -l app=iceberg-catalog 2>/dev/null || true
;;
minio)
echo -e "${GREEN}→${NC} Deleting minio StatefulSet..."
kubectl delete statefulset minio || true
sleep 2
delete_pvcs minio
# Force restart iceberg-catalog since it depends on minio
echo -e "${GREEN}→${NC} Force restarting iceberg-catalog (depends on minio)..."
kubectl delete pod -l app=iceberg-catalog 2>/dev/null || true
;;
qdrant)
echo -e "${GREEN}→${NC} Deleting qdrant StatefulSet..."
@@ -490,6 +542,9 @@ deep_restart() {
kubectl delete statefulset kafka postgres minio qdrant || true
sleep 2
delete_pvcs all
# Force restart iceberg-catalog since it depends on postgres and minio
echo -e "${GREEN}→${NC} Force restarting iceberg-catalog (depends on postgres/minio)..."
kubectl delete pod -l app=iceberg-catalog 2>/dev/null || true
;;
*)
echo -e "${RED}Error: Unknown service '$service'${NC}"
@@ -501,6 +556,8 @@ deep_restart() {
echo -e "${GREEN}→${NC} Redeploying services..."
deploy_services
# Note: deploy_services already calls create_dev_user, so no need to call it again here
echo -e "${GREEN}✓ Deep restart complete${NC}"
}
@@ -600,12 +657,17 @@ case "$COMMAND" in
fi
;;
restart)
if [ -n "$2" ]; then
rebuild_images "$2"
deploy_service "$2"
else
shift # Remove 'restart' from args
if [ $# -eq 0 ]; then
# No services specified, restart all
rebuild_images
deploy_services
else
# Multiple services specified
for service in "$@"; do
rebuild_images "$service"
deploy_service "$service"
done
fi
;;
rebuild)