sandbox connected and streaming
This commit is contained in:
70
bin/deploy
70
bin/deploy
@@ -3,9 +3,9 @@
|
||||
#REMOTE=northamerica-northeast2-docker.pkg.dev/dexorder-430504/dexorder
|
||||
REMOTE=${REMOTE:-git.dxod.org/dexorder/dexorder}
|
||||
|
||||
if [ "$1" != "flink" ] && [ "$1" != "relay" ] && [ "$1" != "ingestor" ] && [ "$1" != "web" ] && [ "$1" != "gateway" ] && [ "$1" != "lifecycle-sidecar" ] && [ "$1" != "client-py" ]; then
|
||||
if [ "$1" != "flink" ] && [ "$1" != "relay" ] && [ "$1" != "ingestor" ] && [ "$1" != "web" ] && [ "$1" != "gateway" ] && [ "$1" != "lifecycle-sidecar" ] && [ "$1" != "sandbox" ]; then
|
||||
echo
|
||||
echo usage: "$0 "'{flink|relay|ingestor|web|gateway|lifecycle-sidecar|client-py} [''dev''] [config] [deployment] [kubernetes] [image_tag]'
|
||||
echo usage: "$0 "'{flink|relay|ingestor|web|gateway|lifecycle-sidecar|sandbox} [''dev''] [config] [deployment] [kubernetes] [image_tag]'
|
||||
echo
|
||||
echo ' [''dev''] if the literal string ''dev'' is not the second argument, then the build refuses to run if source code is not checked in. Otherwise, the git revision numbers are used in the image tag.'
|
||||
echo
|
||||
@@ -100,6 +100,72 @@ if [ "$PROJECT" != "lifecycle-sidecar" ]; then
|
||||
rsync -a --checksum --delete protobuf/ $PROJECT/protobuf/
|
||||
fi
|
||||
|
||||
# For gateway: copy Python API files for research subagent
|
||||
if [ "$PROJECT" == "gateway" ]; then
|
||||
echo "Copying Python API files for research subagent..."
|
||||
|
||||
# Create api-source directory
|
||||
mkdir -p gateway/src/harness/subagents/research/api-source
|
||||
|
||||
# Copy all Python API files (for easy future expansion)
|
||||
cp sandbox/dexorder/api/*.py gateway/src/harness/subagents/research/api-source/
|
||||
|
||||
# Generate api-reference.md with verbatim Python source code
|
||||
API_REF="gateway/src/harness/subagents/research/memory/api-reference.md"
|
||||
|
||||
cat > "$API_REF" << 'HEADER'
|
||||
# Dexorder Research API Reference
|
||||
|
||||
This file contains the complete Python API source code with full docstrings.
|
||||
These files are copied verbatim from `sandbox/dexorder/api/`.
|
||||
|
||||
The API provides access to market data and charting capabilities for research scripts.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Research scripts access the API via:
|
||||
```python
|
||||
from dexorder.api import get_api
|
||||
api = get_api()
|
||||
```
|
||||
|
||||
The API instance provides:
|
||||
- `api.data` - DataAPI for fetching OHLC market data
|
||||
- `api.charting` - ChartingAPI for creating financial charts
|
||||
|
||||
---
|
||||
|
||||
## Complete API Source Code
|
||||
|
||||
The following sections contain the verbatim Python source files with complete
|
||||
type hints, docstrings, and examples.
|
||||
|
||||
HEADER
|
||||
|
||||
# Append each Python file
|
||||
for py_file in api.py data_api.py charting_api.py __init__.py; do
|
||||
if [ -f "sandbox/dexorder/api/$py_file" ]; then
|
||||
echo "" >> "$API_REF"
|
||||
echo "### $py_file" >> "$API_REF"
|
||||
echo '```python' >> "$API_REF"
|
||||
cat "sandbox/dexorder/api/$py_file" >> "$API_REF"
|
||||
echo '```' >> "$API_REF"
|
||||
echo "" >> "$API_REF"
|
||||
fi
|
||||
done
|
||||
|
||||
cat >> "$API_REF" << 'FOOTER'
|
||||
|
||||
---
|
||||
|
||||
For practical usage patterns and complete working examples, see `usage-examples.md`.
|
||||
FOOTER
|
||||
|
||||
echo "Generated api-reference.md with Python API source code"
|
||||
fi
|
||||
|
||||
docker build $NO_CACHE -f $PROJECT/Dockerfile --build-arg="CONFIG=$CONFIG" --build-arg="DEPLOYMENT=$DEPLOYMENT" -t dexorder/ai-$PROJECT:latest $PROJECT || exit 1
|
||||
|
||||
# Cleanup is handled by trap
|
||||
|
||||
133
bin/dev
133
bin/dev
@@ -19,7 +19,7 @@ usage() {
|
||||
echo "Commands:"
|
||||
echo " start Start minikube and deploy all services"
|
||||
echo " stop [--keep-data] Stop minikube (deletes PVCs by default)"
|
||||
echo " restart [svc] Rebuild and redeploy all services, or just one (relay|ingestor|flink|gateway|sidecar|web|client-py)"
|
||||
echo " restart [svc] Rebuild and redeploy all services, or just one (relay|ingestor|flink|gateway|sidecar|web|sandbox)"
|
||||
echo " deep-restart [svc] Restart StatefulSet(s) and delete their PVCs (kafka|postgres|minio|qdrant|all)"
|
||||
echo " rebuild [svc] Rebuild all custom images, or just one"
|
||||
echo " deploy [svc] Deploy/update all services, or just one"
|
||||
@@ -132,19 +132,16 @@ rebuild_images() {
|
||||
if [ "$service" == "all" ] || [ "$service" == "relay" ]; then
|
||||
echo -e "${GREEN}→${NC} Building relay..."
|
||||
RELAY_TAG=$(build_and_get_tag relay) || exit 1
|
||||
docker tag "dexorder/ai-relay:$RELAY_TAG" "dexorder/relay:$RELAY_TAG"
|
||||
fi
|
||||
|
||||
if [ "$service" == "all" ] || [ "$service" == "ingestor" ]; then
|
||||
echo -e "${GREEN}→${NC} Building ingestor..."
|
||||
INGEST_TAG=$(build_and_get_tag ingestor) || exit 1
|
||||
docker tag "dexorder/ai-ingestor:$INGEST_TAG" "dexorder/ingestor:$INGEST_TAG"
|
||||
fi
|
||||
|
||||
if [ "$service" == "all" ] || [ "$service" == "flink" ]; then
|
||||
echo -e "${GREEN}→${NC} Building flink..."
|
||||
FLINK_TAG=$(build_and_get_tag flink) || exit 1
|
||||
docker tag "dexorder/ai-flink:$FLINK_TAG" "dexorder/flink:$FLINK_TAG"
|
||||
fi
|
||||
|
||||
# Build gateway (Node.js application)
|
||||
@@ -165,10 +162,10 @@ rebuild_images() {
|
||||
WEB_TAG=$(build_and_get_tag web) || exit 1
|
||||
fi
|
||||
|
||||
# Build client-py (Python client library)
|
||||
if [ "$service" == "all" ] || [ "$service" == "client-py" ]; then
|
||||
echo -e "${GREEN}→${NC} Building client-py..."
|
||||
CLIENT_PY_TAG=$(build_and_get_tag client-py) || exit 1
|
||||
# Build sandbox (Python client library)
|
||||
if [ "$service" == "all" ] || [ "$service" == "sandbox" ]; then
|
||||
echo -e "${GREEN}→${NC} Building sandbox..."
|
||||
SANDBOX_TAG=$(build_and_get_tag sandbox) || exit 1
|
||||
fi
|
||||
|
||||
# Save the tags for deployment (all services, preserving any we didn't rebuild)
|
||||
@@ -178,9 +175,9 @@ rebuild_images() {
|
||||
echo "GATEWAY_TAG=$GATEWAY_TAG" >> "$ROOT_DIR/.dev-image-tag"
|
||||
echo "SIDECAR_TAG=$SIDECAR_TAG" >> "$ROOT_DIR/.dev-image-tag"
|
||||
echo "WEB_TAG=$WEB_TAG" >> "$ROOT_DIR/.dev-image-tag"
|
||||
echo "CLIENT_PY_TAG=$CLIENT_PY_TAG" >> "$ROOT_DIR/.dev-image-tag"
|
||||
echo "SANDBOX_TAG=$SANDBOX_TAG" >> "$ROOT_DIR/.dev-image-tag"
|
||||
|
||||
echo -e "${GREEN}✓ Images built: relay=$RELAY_TAG, ingestor=$INGEST_TAG, flink=$FLINK_TAG, gateway=$GATEWAY_TAG, sidecar=$SIDECAR_TAG, web=$WEB_TAG, client-py=$CLIENT_PY_TAG${NC}"
|
||||
echo -e "${GREEN}✓ Images built: relay=$RELAY_TAG, ingestor=$INGEST_TAG, flink=$FLINK_TAG, gateway=$GATEWAY_TAG, sidecar=$SIDECAR_TAG, web=$WEB_TAG, sandbox=$SANDBOX_TAG${NC}"
|
||||
}
|
||||
|
||||
deploy_services() {
|
||||
@@ -219,6 +216,11 @@ deploy_services() {
|
||||
|
||||
# Update configs
|
||||
echo -e "${GREEN}→${NC} Updating configs..."
|
||||
|
||||
# Template the gateway-config.yaml with actual image tags
|
||||
sed -i "s/SANDBOX_TAG_PLACEHOLDER/$SANDBOX_TAG/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
sed -i "s/SIDECAR_TAG_PLACEHOLDER/$SIDECAR_TAG/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
|
||||
"$SCRIPT_DIR/config-update" dev
|
||||
|
||||
# Create a temporary kustomization overlay with image tags
|
||||
@@ -227,25 +229,39 @@ deploy_services() {
|
||||
|
||||
# Image tags (added by bin/dev)
|
||||
images:
|
||||
- name: dexorder/relay
|
||||
- name: dexorder/ai-relay
|
||||
newTag: $RELAY_TAG
|
||||
- name: dexorder/ingestor
|
||||
- name: dexorder/ai-ingestor
|
||||
newTag: $INGEST_TAG
|
||||
- name: dexorder/flink
|
||||
- name: dexorder/ai-flink
|
||||
newTag: $FLINK_TAG
|
||||
- name: dexorder/gateway
|
||||
- name: dexorder/ai-gateway
|
||||
newTag: $GATEWAY_TAG
|
||||
- name: dexorder/ai-web
|
||||
newTag: $WEB_TAG
|
||||
- name: dexorder/ai-sandbox
|
||||
newTag: $SANDBOX_TAG
|
||||
- name: dexorder/ai-lifecycle-sidecar
|
||||
newTag: $SIDECAR_TAG
|
||||
EOF
|
||||
|
||||
# Apply kustomize
|
||||
echo -e "${GREEN}→${NC} Applying Kubernetes manifests..."
|
||||
kubectl apply -k .
|
||||
|
||||
# Apply sandbox-namespace secrets (must be after kustomize creates the dexorder-sandboxes namespace)
|
||||
echo -e "${GREEN}→${NC} Applying sandbox secrets..."
|
||||
if [ -f "$ROOT_DIR/deploy/k8s/dev/secrets/sandbox-secrets.yaml" ]; then
|
||||
kubectl apply -f "$ROOT_DIR/deploy/k8s/dev/secrets/sandbox-secrets.yaml"
|
||||
fi
|
||||
|
||||
# Clean up the appended image tags from kustomization.yaml
|
||||
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
|
||||
|
||||
# Restore gateway-config.yaml placeholders
|
||||
sed -i "s/$SANDBOX_TAG/SANDBOX_TAG_PLACEHOLDER/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
sed -i "s/$SIDECAR_TAG/SIDECAR_TAG_PLACEHOLDER/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
|
||||
echo -e "${GREEN}✓ Services deployed${NC}"
|
||||
|
||||
echo ""
|
||||
@@ -389,21 +405,15 @@ create_dev_user() {
|
||||
# Create/update license for the user
|
||||
echo -e "${GREEN}→${NC} Creating $LICENSE_TYPE license for dev user..."
|
||||
kubectl exec "$pg_pod" -- psql -U postgres -d iceberg -c "
|
||||
INSERT INTO user_licenses (user_id, email, license_type, mcp_server_url, features, resource_limits, preferred_model)
|
||||
INSERT INTO user_licenses (user_id, email, license, mcp_server_url)
|
||||
VALUES (
|
||||
'$user_id',
|
||||
'$DEV_EMAIL',
|
||||
'$LICENSE_TYPE',
|
||||
'http://localhost:8080/mcp',
|
||||
'{\"maxIndicators\":50,\"maxStrategies\":20,\"maxBacktestDays\":365,\"realtimeData\":true,\"customExecutors\":true,\"apiAccess\":true}',
|
||||
'{\"maxConcurrentSessions\":5,\"maxMessagesPerDay\":1000,\"maxTokensPerMessage\":8192,\"rateLimitPerMinute\":60}',
|
||||
'{\"provider\":\"anthropic\",\"model\":\"claude-sonnet-4-6\",\"temperature\":0.7}'
|
||||
'{\"licenseType\":\"$LICENSE_TYPE\",\"features\":{\"maxIndicators\":50,\"maxStrategies\":20,\"maxBacktestDays\":365,\"realtimeData\":true,\"customExecutors\":true,\"apiAccess\":true},\"resourceLimits\":{\"maxConcurrentSessions\":5,\"maxMessagesPerDay\":1000,\"maxTokensPerMessage\":8192,\"rateLimitPerMinute\":60},\"k8sResources\":{\"memoryRequest\":\"512Mi\",\"memoryLimit\":\"2Gi\",\"cpuRequest\":\"250m\",\"cpuLimit\":\"2000m\",\"storage\":\"10Gi\",\"tmpSizeLimit\":\"256Mi\",\"enableIdleShutdown\":true,\"idleTimeoutMinutes\":60},\"preferredModel\":{\"provider\":\"anthropic\",\"model\":\"claude-sonnet-4-6\",\"temperature\":0.7}}',
|
||||
'http://localhost:8080/mcp'
|
||||
)
|
||||
ON CONFLICT (user_id) DO UPDATE SET
|
||||
license_type = EXCLUDED.license_type,
|
||||
features = EXCLUDED.features,
|
||||
resource_limits = EXCLUDED.resource_limits,
|
||||
preferred_model = EXCLUDED.preferred_model,
|
||||
license = EXCLUDED.license,
|
||||
updated_at = NOW();
|
||||
" > /dev/null 2>&1
|
||||
echo -e "${GREEN}✓ Dev user ready ($DEV_EMAIL / $DEV_PASSWORD)${NC}"
|
||||
@@ -595,21 +605,52 @@ deploy_service() {
|
||||
# This ensures all patches (including imagePullPolicy) are properly applied
|
||||
cd "$ROOT_DIR/deploy/k8s/dev"
|
||||
|
||||
# Create a temporary kustomization overlay with image tags
|
||||
# Map service names to image names and tags
|
||||
local image_name=""
|
||||
local image_tag=""
|
||||
|
||||
case "$service" in
|
||||
relay)
|
||||
image_name="dexorder/ai-relay"
|
||||
image_tag="$RELAY_TAG"
|
||||
;;
|
||||
ingestor)
|
||||
image_name="dexorder/ai-ingestor"
|
||||
image_tag="$INGEST_TAG"
|
||||
;;
|
||||
flink)
|
||||
image_name="dexorder/ai-flink"
|
||||
image_tag="$FLINK_TAG"
|
||||
;;
|
||||
gateway)
|
||||
image_name="dexorder/ai-gateway"
|
||||
image_tag="$GATEWAY_TAG"
|
||||
# Also need to template gateway-config.yaml
|
||||
sed -i "s/SANDBOX_TAG_PLACEHOLDER/$SANDBOX_TAG/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
sed -i "s/SIDECAR_TAG_PLACEHOLDER/$SIDECAR_TAG/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
"$SCRIPT_DIR/config-update" dev
|
||||
;;
|
||||
web)
|
||||
image_name="dexorder/ai-web"
|
||||
image_tag="$WEB_TAG"
|
||||
;;
|
||||
lifecycle-sidecar|sidecar)
|
||||
image_name="dexorder/ai-lifecycle-sidecar"
|
||||
image_tag="$SIDECAR_TAG"
|
||||
;;
|
||||
*)
|
||||
echo -e "${RED}Error: Unknown service '$service'${NC}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Create a temporary kustomization overlay with ONLY this service's image tag
|
||||
cat >> kustomization.yaml <<EOF
|
||||
|
||||
# Image tags (added by bin/dev)
|
||||
images:
|
||||
- name: dexorder/relay
|
||||
newTag: $RELAY_TAG
|
||||
- name: dexorder/ingestor
|
||||
newTag: $INGEST_TAG
|
||||
- name: dexorder/flink
|
||||
newTag: $FLINK_TAG
|
||||
- name: dexorder/gateway
|
||||
newTag: $GATEWAY_TAG
|
||||
- name: dexorder/ai-web
|
||||
newTag: $WEB_TAG
|
||||
- name: $image_name
|
||||
newTag: $image_tag
|
||||
EOF
|
||||
|
||||
kubectl apply -k .
|
||||
@@ -617,6 +658,12 @@ EOF
|
||||
# Clean up the appended image tags from kustomization.yaml
|
||||
sed -i '/# Image tags (added by bin\/dev)/,$d' kustomization.yaml
|
||||
|
||||
# Restore gateway-config.yaml placeholders if we modified it
|
||||
if [ "$service" == "gateway" ]; then
|
||||
sed -i "s/$SANDBOX_TAG/SANDBOX_TAG_PLACEHOLDER/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
sed -i "s/$SIDECAR_TAG/SIDECAR_TAG_PLACEHOLDER/g" "$ROOT_DIR/deploy/k8s/dev/configs/gateway-config.yaml"
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✓ $service deployed${NC}"
|
||||
}
|
||||
|
||||
@@ -649,9 +696,9 @@ case "$COMMAND" in
|
||||
kubectl wait --for=delete pod -l app=qdrant --timeout=60s 2>/dev/null || true
|
||||
# Now delete PVCs
|
||||
delete_pvcs all
|
||||
# Delete dexorder-agents namespace
|
||||
echo -e "${GREEN}→${NC} Deleting dexorder-agents namespace..."
|
||||
kubectl delete namespace dexorder-agents 2>/dev/null || true
|
||||
# Delete dexorder-sandboxes namespace
|
||||
echo -e "${GREEN}→${NC} Deleting dexorder-sandboxes namespace..."
|
||||
kubectl delete namespace dexorder-sandboxes 2>/dev/null || true
|
||||
minikube stop
|
||||
echo -e "${GREEN}✓ Minikube stopped and PVCs deleted${NC}"
|
||||
echo -e "${YELLOW}Tip: Use 'bin/dev stop --keep-data' to preserve PVCs${NC}"
|
||||
@@ -667,7 +714,15 @@ case "$COMMAND" in
|
||||
# Multiple services specified
|
||||
for service in "$@"; do
|
||||
rebuild_images "$service"
|
||||
deploy_service "$service"
|
||||
|
||||
# Special handling for sandbox: delete sandbox deployments instead of applying kustomization
|
||||
if [ "$service" == "sandbox" ]; then
|
||||
echo -e "${GREEN}→${NC} Deleting user container deployments in dexorder-sandboxes namespace..."
|
||||
kubectl delete deployments --all -n dexorder-sandboxes 2>/dev/null || true
|
||||
echo -e "${GREEN}✓ User containers will be recreated by gateway on next login${NC}"
|
||||
else
|
||||
deploy_service "$service"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
;;
|
||||
|
||||
@@ -14,7 +14,7 @@ NC='\033[0m' # No Color
|
||||
usage() {
|
||||
echo "Usage: $0 [COMMAND]"
|
||||
echo ""
|
||||
echo "Test client-py against the development environment"
|
||||
echo "Test sandbox against the development environment"
|
||||
echo ""
|
||||
echo "Commands:"
|
||||
echo " ohlc Test OHLCClient API (default)"
|
||||
@@ -107,10 +107,10 @@ run_ohlc_test() {
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# Install client-py in development mode
|
||||
pip install -e client-py >/dev/null 2>&1 || {
|
||||
echo -e "${YELLOW}Installing client-py dependencies...${NC}"
|
||||
pip install -e client-py
|
||||
# Install sandbox in development mode
|
||||
pip install -e sandbox >/dev/null 2>&1 || {
|
||||
echo -e "${YELLOW}Installing sandbox dependencies...${NC}"
|
||||
pip install -e sandbox
|
||||
}
|
||||
|
||||
# Run the test
|
||||
@@ -123,10 +123,10 @@ run_history_test() {
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# Install client-py in development mode
|
||||
pip install -e client-py >/dev/null 2>&1 || {
|
||||
echo -e "${YELLOW}Installing client-py dependencies...${NC}"
|
||||
pip install -e client-py
|
||||
# Install sandbox in development mode
|
||||
pip install -e sandbox >/dev/null 2>&1 || {
|
||||
echo -e "${YELLOW}Installing sandbox dependencies...${NC}"
|
||||
pip install -e sandbox
|
||||
}
|
||||
|
||||
# Run the low-level test
|
||||
@@ -139,10 +139,10 @@ open_shell() {
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# Install client-py in development mode
|
||||
pip install -e client-py >/dev/null 2>&1 || {
|
||||
echo -e "${YELLOW}Installing client-py dependencies...${NC}"
|
||||
pip install -e client-py
|
||||
# Install sandbox in development mode
|
||||
pip install -e sandbox >/dev/null 2>&1 || {
|
||||
echo -e "${YELLOW}Installing sandbox dependencies...${NC}"
|
||||
pip install -e sandbox
|
||||
}
|
||||
|
||||
echo -e "${BLUE}Example usage:${NC}"
|
||||
@@ -156,7 +156,7 @@ open_shell() {
|
||||
python3 -i -c "
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.join(os.getcwd(), 'client-py'))
|
||||
sys.path.insert(0, os.path.join(os.getcwd(), 'sandbox'))
|
||||
from dexorder import OHLCClient, HistoryClient, IcebergClient
|
||||
import asyncio
|
||||
print('✓ dexorder package imported')
|
||||
@@ -94,6 +94,7 @@ else
|
||||
"ingestor-secrets"
|
||||
"flink-secrets"
|
||||
"gateway-secrets"
|
||||
"sandbox-secrets"
|
||||
)
|
||||
|
||||
FAILED=0
|
||||
|
||||
Reference in New Issue
Block a user