sandbox connected and streaming

sandbox/.dockerignore

@@ -0,0 +1,57 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Config and secrets (should come from k8s mounts)
+config.yaml
+secrets.yaml
+*.local.yaml
+
+# Data directories
+data/
+
+# Git
+.git/
+.gitignore
+
+# Documentation
+*.md
+docs/
+
+# Example files
+*.example.*

sandbox/Dockerfile

@@ -0,0 +1,96 @@
+# Multi-stage build for DexOrder user container
+FROM continuumio/miniconda3:latest AS builder
+
+WORKDIR /build
+
+# Install build dependencies including protobuf compiler
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    g++ \
+    protobuf-compiler \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy dependency specifications
+COPY setup.py .
+COPY environment.yml .
+COPY dexorder/ dexorder/
+
+# Copy protobuf definitions (copied by bin/build from canonical /protobuf/)
+COPY protobuf/ protobuf/
+
+# Compile protobufs to Python
+RUN mkdir -p dexorder/generated && \
+    protoc --python_out=dexorder/generated --proto_path=protobuf protobuf/*.proto && \
+    touch dexorder/generated/__init__.py
+
+# Create conda environment and install dependencies
+RUN conda env create -f environment.yml -p /build/env && \
+    conda clean -afy
+
+# Install the local package into the conda environment
+RUN /build/env/bin/pip install --no-cache-dir .
+
+# =============================================================================
+# Runtime stage
+# =============================================================================
+FROM continuumio/miniconda3:latest
+
+WORKDIR /app
+
+# Install runtime dependencies only
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libzmq5 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user
+RUN groupadd -r dexorder && useradd -r -g dexorder -u 1000 dexorder
+
+# Copy conda environment from builder
+COPY --from=builder /build/env /opt/conda/envs/dexorder
+
+# Copy application code
+COPY dexorder/ /app/dexorder/
+COPY main.py /app/
+
+# Copy generated protobuf code from builder
+COPY --from=builder /build/dexorder/generated/ /app/dexorder/generated/
+
+# Create directories for config, secrets, and data
+# Note: /app will be read-only at runtime except for /app/data (mounted volume)
+RUN mkdir -p /app/config /app/secrets /app/data && \
+    chown -R root:root /app && \
+    chmod -R 755 /app && \
+    chown dexorder:dexorder /app/data && \
+    chmod 700 /app/data
+
+# Create writable tmp directory (read-only rootfs requirement)
+RUN mkdir -p /tmp && chmod 1777 /tmp
+
+# Copy entrypoint script
+COPY entrypoint.sh /app/
+RUN chmod 755 /app/entrypoint.sh && chown root:root /app/entrypoint.sh
+
+# Switch to non-root user
+USER dexorder
+
+# Environment variables (can be overridden in k8s)
+ENV PYTHONUNBUFFERED=1 \
+    MPLCONFIGDIR=/tmp \
+    LOG_LEVEL=INFO \
+    CONFIG_PATH=/app/config/config.yaml \
+    SECRETS_PATH=/app/config/secrets.yaml \
+    ZMQ_XPUB_PORT=5570 \
+    ZMQ_GATEWAY_ENDPOINT=tcp://gateway:5571 \
+    MCP_SERVER_NAME=dexorder-user \
+    MCP_TRANSPORT=sse \
+    MCP_HTTP_PORT=3000 \
+    MCP_HTTP_HOST=0.0.0.0 \
+    IDLE_TIMEOUT_MINUTES=15 \
+    ENABLE_IDLE_SHUTDOWN=true
+
+# Health check endpoint (simple check if process is running)
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+    CMD /opt/conda/envs/dexorder/bin/python -c "import sys; sys.exit(0)"
+
+# Run the main application using conda environment via entrypoint
+ENTRYPOINT ["/app/entrypoint.sh"]

sandbox/README.md

@@ -0,0 +1,259 @@
+# DexOrder Python Client Library
+
+High-level Python API for accessing historical OHLC data from the DexOrder trading platform.
+
+## Features
+
+- **Smart Caching**: Automatically checks Iceberg warehouse before requesting new data
+- **Async Request/Response**: Non-blocking historical data requests via relay
+- **Gap Detection**: Identifies and requests only missing data ranges
+- **Transparent Access**: Single API for both cached and on-demand data
+
+## Installation
+
+```bash
+cd redesign/sandbox
+pip install -e .
+```
+
+## Quick Start
+
+```python
+import asyncio
+from dexorder import OHLCClient
+
+async def main():
+    # Initialize client
+    client = OHLCClient(
+        iceberg_catalog_uri="http://iceberg-catalog:8181",
+        relay_endpoint="tcp://relay:5555",
+        notification_endpoint="tcp://flink:5557"
+    )
+
+    # Start background notification listener
+    await client.start()
+
+    try:
+        # Fetch OHLC data (automatically checks cache and requests missing data)
+        df = await client.fetch_ohlc(
+            ticker="BINANCE:BTC/USDT",
+            period_seconds=3600,  # 1-hour candles
+            start_time=1735689600000000,  # microseconds
+            end_time=1736294399000000
+        )
+
+        print(f"Fetched {len(df)} candles")
+        print(df.head())
+
+    finally:
+        await client.stop()
+
+# Run
+asyncio.run(main())
+```
+
+## Using Context Manager
+
+```python
+async def main():
+    async with OHLCClient(...) as client:
+        df = await client.fetch_ohlc(...)
+```
+
+## Architecture
+
+### Components
+
+1. **OHLCClient**: High-level API with smart caching
+2. **IcebergClient**: Direct queries to Iceberg warehouse
+3. **HistoryClient**: Submit requests via relay and wait for notifications
+
+### Data Flow
+
+```
+┌─────────┐
+│  Client │
+└────┬────┘
+     │ 1. fetch_ohlc()
+     ▼
+┌─────────────────┐
+│  OHLCClient     │
+└────┬────────────┘
+     │ 2. Check Iceberg
+     ▼
+┌─────────────────┐      ┌──────────┐
+│ IcebergClient   │─────▶│ Iceberg  │
+└─────────────────┘      └──────────┘
+     │ 3. Missing data?
+     ▼
+┌─────────────────┐      ┌──────────┐
+│ HistoryClient   │─────▶│  Relay   │
+└────┬────────────┘      └──────────┘
+     │                          │
+     │ 4. Wait for notification │
+     │◀─────────────────────────┘
+     │ 5. Query Iceberg again
+     ▼
+┌─────────────────┐
+│  Return data    │
+└─────────────────┘
+```
+
+## API Reference
+
+### OHLCClient
+
+#### `__init__(iceberg_catalog_uri, relay_endpoint, notification_endpoint, namespace="trading")`
+
+Initialize the client with connection parameters.
+
+#### `async fetch_ohlc(ticker, period_seconds, start_time, end_time, request_timeout=30.0)`
+
+Fetch OHLC data with smart caching.
+
+**Parameters:**
+- `ticker` (str): Market identifier (e.g., "BINANCE:BTC/USDT")
+- `period_seconds` (int): OHLC period in seconds (60, 300, 3600, 86400, etc.)
+- `start_time` (int): Start timestamp in microseconds
+- `end_time` (int): End timestamp in microseconds
+- `request_timeout` (float): Timeout for historical requests in seconds
+
+**Returns:** `pd.DataFrame` with columns:
+- `ticker`: Market identifier
+- `period_seconds`: Period in seconds
+- `timestamp`: Candle timestamp (microseconds)
+- `open`, `high`, `low`, `close`: Prices (integer format)
+- `volume`: Trading volume
+- Additional fields: `buy_vol`, `sell_vol`, `open_interest`, etc.
+
+### IcebergClient
+
+Direct access to Iceberg warehouse.
+
+#### `query_ohlc(ticker, period_seconds, start_time, end_time)`
+
+Query OHLC data directly from Iceberg.
+
+#### `find_missing_ranges(ticker, period_seconds, start_time, end_time)`
+
+Identify missing data ranges. Returns list of `(start_time, end_time)` tuples.
+
+#### `has_data(ticker, period_seconds, start_time, end_time)`
+
+Check if any data exists for the given parameters.
+
+### HistoryClient
+
+Low-level client for submitting historical data requests.
+
+**IMPORTANT**: Always call `connect()` before making requests to prevent race condition.
+
+#### `async connect()`
+
+Connect to relay and start notification listener. **MUST be called before making any requests.**
+
+This subscribes to the notification topic `RESPONSE:{client_id}` BEFORE any requests are sent,
+preventing the race condition where notifications arrive before subscription.
+
+#### `async request_historical_ohlc(ticker, period_seconds, start_time, end_time, timeout=30.0, limit=None)`
+
+Submit historical data request and wait for completion notification.
+
+**Returns:** dict with keys:
+- `request_id`: The request ID
+- `status`: 'OK', 'NOT_FOUND', or 'ERROR'
+- `error_message`: Error message if status is 'ERROR'
+- `iceberg_namespace`, `iceberg_table`, `row_count`: Available when status is 'OK'
+
+**Example:**
+```python
+from dexorder import HistoryClient
+
+client = HistoryClient(
+    relay_endpoint="tcp://relay:5559",
+    notification_endpoint="tcp://relay:5558"
+)
+
+# CRITICAL: Connect first to prevent race condition
+await client.connect()
+
+# Now safe to make requests
+result = await client.request_historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,
+    start_time=1735689600000000,
+    end_time=1736294399000000
+)
+
+await client.close()
+```
+
+## Configuration
+
+The client requires the following endpoints:
+
+- **Iceberg Catalog URI**: REST API endpoint for Iceberg metadata (default: `http://iceberg-catalog:8181`)
+- **Relay Endpoint**: ZMQ REQ/REP endpoint for submitting requests (default: `tcp://relay:5555`)
+- **Notification Endpoint**: ZMQ PUB/SUB endpoint for receiving notifications (default: `tcp://flink:5557`)
+
+## Development
+
+### Generate Protobuf Files
+
+```bash
+cd redesign/protobuf
+protoc -I . --python_out=../sandbox/dexorder ingestor.proto ohlc.proto
+```
+
+### Run Tests
+
+```bash
+pytest tests/
+```
+
+## Examples
+
+See `../relay/test/async_client.py` for a complete example.
+
+## Timestamp Format
+
+All timestamps are in **microseconds since epoch**:
+
+```python
+# Convert from datetime
+from datetime import datetime, timezone
+
+dt = datetime(2024, 1, 1, tzinfo=timezone.utc)
+timestamp_micros = int(dt.timestamp() * 1_000_000)
+
+# Convert to datetime
+dt = datetime.fromtimestamp(timestamp_micros / 1_000_000, tz=timezone.utc)
+```
+
+## Period Seconds
+
+Common period values:
+- `60` - 1 minute
+- `300` - 5 minutes
+- `900` - 15 minutes
+- `3600` - 1 hour
+- `14400` - 4 hours
+- `86400` - 1 day
+- `604800` - 1 week
+
+## Error Handling
+
+```python
+try:
+    df = await client.fetch_ohlc(...)
+except TimeoutError:
+    print("Request timed out")
+except ValueError as e:
+    print(f"Request failed: {e}")
+except ConnectionError:
+    print("Unable to connect to relay")
+```
+
+## License
+
+Internal use only.

sandbox/RESEARCH_API_USAGE.md

@@ -0,0 +1,221 @@
+# Research Script API Usage
+
+Research scripts executed via the `execute_research` MCP tool have access to the global API instance, which provides both data fetching and charting capabilities.
+
+## Accessing the API
+
+```python
+from dexorder.api import get_api
+import asyncio
+
+# Get the global API instance
+api = get_api()
+```
+
+## Using the Data API
+
+The data API provides access to historical OHLC (Open, High, Low, Close) market data with smart caching via Iceberg.
+
+### Fetching Historical Data
+
+The API accepts flexible timestamp formats for convenience:
+
+```python
+from dexorder.api import get_api
+import asyncio
+from datetime import datetime
+
+api = get_api()
+
+# Method 1: Using Unix timestamps (seconds)
+df = asyncio.run(api.data.historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,  # 1 hour candles
+    start_time=1640000000,  # Unix timestamp in seconds
+    end_time=1640086400,
+    extra_columns=["volume"]
+))
+
+# Method 2: Using date strings
+df = asyncio.run(api.data.historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,
+    start_time="2021-12-20",  # Simple date string
+    end_time="2021-12-21",
+    extra_columns=["volume"]
+))
+
+# Method 3: Using date strings with time
+df = asyncio.run(api.data.historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,
+    start_time="2021-12-20 00:00:00",
+    end_time="2021-12-20 23:59:59",
+    extra_columns=["volume"]
+))
+
+# Method 4: Using datetime objects
+df = asyncio.run(api.data.historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,
+    start_time=datetime(2021, 12, 20),
+    end_time=datetime(2021, 12, 21),
+    extra_columns=["volume"]
+))
+
+print(f"Loaded {len(df)} candles")
+print(df.head())
+```
+
+### Available Extra Columns
+
+- `"volume"` - Total volume
+- `"buy_vol"` - Buy-side volume
+- `"sell_vol"` - Sell-side volume
+- `"open_time"`, `"high_time"`, `"low_time"`, `"close_time"` - Timestamps for each price point
+- `"open_interest"` - Open interest (for futures)
+- `"ticker"` - Market identifier
+- `"period_seconds"` - Period in seconds
+
+## Using the Charting API
+
+The charting API provides styled financial charts with OHLC candlesticks and technical indicators.
+
+### Creating a Basic Candlestick Chart
+
+```python
+from dexorder.api import get_api
+import asyncio
+from datetime import datetime
+
+api = get_api()
+
+# Fetch data
+df = asyncio.run(api.data.historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,
+    start_time="2021-12-20",
+    end_time="2021-12-21",
+    extra_columns=["volume"]
+))
+
+# Create candlestick chart (synchronous)
+fig, ax = api.charting.plot_ohlc(
+    df,
+    title="BTC/USDT 1H",
+    volume=True,  # Show volume bars
+    style="charles"  # Chart style
+)
+
+# The figure is automatically captured and returned to the MCP client
+```
+
+### Adding Indicator Panels
+
+```python
+from dexorder.api import get_api
+import asyncio
+import pandas as pd
+
+api = get_api()
+
+# Fetch data
+df = asyncio.run(api.data.historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,
+    start_time="2021-12-20",
+    end_time="2021-12-21"
+))
+
+# Calculate a simple moving average
+df['sma_20'] = df['close'].rolling(window=20).mean()
+
+# Create chart
+fig, ax = api.charting.plot_ohlc(df, title="BTC/USDT with SMA")
+
+# Overlay the SMA on the price chart
+ax.plot(df.index, df['sma_20'], label="SMA 20", color="blue", linewidth=2)
+ax.legend()
+
+# Add RSI indicator panel below
+df['rsi'] = calculate_rsi(df['close'], 14)  # Your RSI calculation
+rsi_ax = api.charting.add_indicator_panel(
+    fig, df,
+    columns=["rsi"],
+    ylabel="RSI",
+    ylim=(0, 100)
+)
+rsi_ax.axhline(70, color='red', linestyle='--', alpha=0.5)
+rsi_ax.axhline(30, color='green', linestyle='--', alpha=0.5)
+```
+
+## Complete Example
+
+```python
+from dexorder.api import get_api
+import asyncio
+import pandas as pd
+
+# Get API instance
+api = get_api()
+
+# Fetch historical data using date strings (easiest for research)
+df = asyncio.run(api.data.historical_ohlc(
+    ticker="BINANCE:BTC/USDT",
+    period_seconds=3600,  # 1 hour
+    start_time="2021-12-20",
+    end_time="2021-12-21",
+    extra_columns=["volume"]
+))
+
+# Add some analysis
+df['sma_20'] = df['close'].rolling(window=20).mean()
+df['sma_50'] = df['close'].rolling(window=50).mean()
+
+# Create chart with volume
+fig, ax = api.charting.plot_ohlc(
+    df,
+    title="BTC/USDT Analysis",
+    volume=True,
+    style="charles"
+)
+
+# Overlay moving averages
+ax.plot(df.index, df['sma_20'], label="SMA 20", color="blue", linewidth=1.5)
+ax.plot(df.index, df['sma_50'], label="SMA 50", color="red", linewidth=1.5)
+ax.legend()
+
+# Print summary statistics
+print(f"Period: {len(df)} candles")
+print(f"High: {df['high'].max()}")
+print(f"Low: {df['low'].min()}")
+print(f"Mean Volume: {df['volume'].mean():.2f}")
+```
+
+## Notes
+
+- **Async vs Sync**: Data API methods are async and require `asyncio.run()`. Charting API methods are synchronous.
+- **Figure Capture**: All matplotlib figures created during script execution are automatically captured and returned as PNG images.
+- **Print Statements**: All `print()` output is captured and returned as text content.
+- **Errors**: Exceptions are caught and reported in the execution results.
+- **Timestamps**: The API accepts flexible timestamp formats:
+  - Unix timestamps in **seconds** (int or float) - e.g., `1640000000`
+  - Date strings - e.g., `"2021-12-20"` or `"2021-12-20 12:00:00"`
+  - datetime objects - e.g., `datetime(2021, 12, 20)`
+  - pandas Timestamp objects
+  - Internally, the system uses microseconds since epoch, but you don't need to worry about this conversion.
+- **Price/Volume Values**: All prices and volumes are returned as decimal floats, automatically converted from internal storage format using market metadata. No manual conversion is needed.
+
+## Available Chart Styles
+
+- `"charles"` (default)
+- `"binance"`
+- `"blueskies"`
+- `"brasil"`
+- `"checkers"`
+- `"classic"`
+- `"mike"`
+- `"nightclouds"`
+- `"sas"`
+- `"starsandstripes"`
+- `"yahoo"`

sandbox/__init__.py

@@ -0,0 +1,3 @@
+import logging
+
+log = logging.getLogger(__name__)

sandbox/config.example.yaml

@@ -0,0 +1,35 @@
+# Example configuration file for DexOrder user container
+# Mount this at /app/config/config.yaml in k8s
+
+# Data directory for persistent storage (workspace, strategies, etc.)
+# Defaults to ./data relative to working directory if not set
+# In k8s this is mounted as a PVC at /app/data
+data_dir: "/app/data"
+
+# User-specific settings
+user:
+  timezone: "UTC"
+
+# Data sources
+data:
+  iceberg:
+    catalog_name: "dexorder"
+    # Catalog properties loaded from secrets
+
+  relay:
+    endpoint: "tcp://relay:5560"
+    timeout_ms: 5000
+
+# Strategy settings
+strategies:
+  max_concurrent: 5
+  default_timeout_minutes: 60
+
+# Alert settings
+alerts:
+  max_active: 100
+
+# Logging
+logging:
+  level: "INFO"
+  include_timestamps: true

sandbox/dexorder/__init__.py

@@ -0,0 +1,53 @@
+"""
+DexOrder Trading Platform Python Client
+
+Provides high-level APIs for:
+- Historical OHLC data retrieval with smart caching
+- Async request/response via relay
+- Iceberg data warehouse queries
+- User container event publishing
+- Container lifecycle management
+"""
+
+__version__ = "0.1.0"
+
+# Suppress the protobuf warning - it's handled at import time
+import warnings
+warnings.filterwarnings('ignore', message='Warning: Protobuf files not found')
+
+from .ohlc_client import OHLCClient
+from .iceberg_client import IcebergClient
+from .history_client import HistoryClient
+from .lifecycle_manager import (
+    LifecycleManager,
+    get_lifecycle_manager,
+    start_lifecycle_manager,
+)
+
+# Event system
+from .events import (
+    EventPublisher,
+    EventType,
+    Priority,
+    ChannelType,
+    DeliverySpec,
+    UserEvent,
+)
+
+__all__ = [
+    # Data clients
+    'OHLCClient',
+    'IcebergClient',
+    'HistoryClient',
+    # Lifecycle management
+    'LifecycleManager',
+    'get_lifecycle_manager',
+    'start_lifecycle_manager',
+    # Event system
+    'EventPublisher',
+    'EventType',
+    'Priority',
+    'ChannelType',
+    'DeliverySpec',
+    'UserEvent',
+]

sandbox/dexorder/api/__init__.py

@@ -0,0 +1,67 @@
+"""
+DexOrder API - market data and charting for research and trading.
+
+For research scripts, import and use get_api() to access the API:
+
+    from dexorder.api import get_api
+    import asyncio
+
+    api = get_api()
+    df = asyncio.run(api.data.historical_ohlc(...))
+    fig, ax = api.charting.plot_ohlc(df)
+"""
+
+import logging
+from typing import Optional
+
+from dexorder.api.api import API
+from dexorder.api.charting_api import ChartingAPI
+from dexorder.api.data_api import DataAPI
+
+log = logging.getLogger(__name__)
+
+# Global API instance - managed by main.py
+_global_api: Optional[API] = None
+
+
+def get_api() -> API:
+    """
+    Get the global API instance for accessing market data and charts.
+
+    Use this in research scripts to access the data and charting APIs.
+
+    Returns:
+        API instance with data and charting capabilities
+
+    Raises:
+        RuntimeError: If called before API initialization (should not happen in research scripts)
+
+    Example:
+        from dexorder.api import get_api
+        import asyncio
+
+        api = get_api()
+
+        # Fetch data
+        df = asyncio.run(api.data.historical_ohlc(
+            ticker="BINANCE:BTC/USDT",
+            period_seconds=3600,
+            start_time="2021-12-20",
+            end_time="2021-12-21"
+        ))
+
+        # Create chart
+        fig, ax = api.charting.plot_ohlc(df, title="BTC/USDT")
+    """
+    if _global_api is None:
+        raise RuntimeError("API not initialized")
+    return _global_api
+
+
+def set_api(api: API) -> None:
+    """Set the global API instance. Internal use only."""
+    global _global_api
+    _global_api = api
+
+
+__all__ = ['API', 'ChartingAPI', 'DataAPI', 'get_api', 'set_api']

sandbox/dexorder/api/api.py

@@ -0,0 +1,44 @@
+"""
+Main DexOrder API - provides access to market data and charting.
+"""
+
+import logging
+
+from .charting_api import ChartingAPI
+from .data_api import DataAPI
+
+log = logging.getLogger(__name__)
+
+
+class API:
+    """
+    Main API for accessing market data and creating charts.
+
+    This is the primary interface for research scripts and trading strategies.
+    Access this via get_api() in research scripts.
+
+    Attributes:
+        data: DataAPI for fetching historical and current market data
+        charting: ChartingAPI for creating candlestick charts and visualizations
+
+    Example:
+        from dexorder.api import get_api
+        import asyncio
+
+        api = get_api()
+
+        # Fetch data
+        df = asyncio.run(api.data.historical_ohlc(
+            ticker="BINANCE:BTC/USDT",
+            period_seconds=3600,
+            start_time="2021-12-20",
+            end_time="2021-12-21"
+        ))
+
+        # Create chart
+        fig, ax = api.charting.plot_ohlc(df, title="BTC/USDT 1H")
+    """
+
+    def __init__(self, charting: ChartingAPI, data: DataAPI):
+        self.charting: ChartingAPI = charting
+        self.data: DataAPI = data

sandbox/dexorder/api/charting_api.py

@@ -0,0 +1,155 @@
+import logging
+from abc import abstractmethod, ABC
+from typing import Optional, Tuple, List
+
+import pandas as pd
+from matplotlib import pyplot as plt
+from matplotlib.figure import Figure
+
+
+class ChartingAPI(ABC):
+    """
+    API for creating financial charts and visualizations.
+
+    Provides methods to create candlestick charts, add technical indicator panels,
+    and build custom visualizations. All figures are automatically captured and
+    returned to the client as images.
+
+    Basic workflow:
+    1. Create a chart with plot_ohlc() → returns Figure and Axes
+    2. Optionally overlay indicators on the main axes (e.g., moving averages)
+    3. Optionally add indicator panels below with add_indicator_panel()
+    4. Figures are automatically captured (no need to save manually)
+    """
+
+    @abstractmethod
+    def plot_ohlc(
+        self,
+        df: pd.DataFrame,
+        title: Optional[str] = None,
+        volume: bool = False,
+        style: str = "charles",
+        figsize: Tuple[int, int] = (12, 8),
+        **kwargs
+    ) -> Tuple[Figure, plt.Axes]:
+        """
+        Create a candlestick chart from OHLC data.
+
+        Args:
+            df: DataFrame with OHLC data. Required columns: open, high, low, close.
+               Column names are case-insensitive.
+            title: Chart title (optional)
+            volume: If True, shows volume bars below the candlesticks (requires 'volume' column)
+            style: Visual style for the chart. Available styles:
+                  "charles" (default), "binance", "blueskies", "brasil", "checkers",
+                  "classic", "mike", "nightclouds", "sas", "starsandstripes", "yahoo"
+            figsize: Figure size as (width, height) in inches. Default: (12, 8)
+            **kwargs: Additional styling arguments
+
+        Returns:
+            Tuple of (Figure, Axes):
+            - Figure: matplotlib Figure object
+            - Axes: Main candlestick axes (use for overlaying indicators)
+
+        Examples:
+            # Basic chart
+            fig, ax = api.plot_ohlc(df)
+
+            # With volume and title
+            fig, ax = api.plot_ohlc(
+                df,
+                title="BTC/USDT 1H",
+                volume=True,
+                style="binance"
+            )
+
+            # Overlay moving average
+            fig, ax = api.plot_ohlc(df)
+            ax.plot(df.index, df['sma_20'], label="SMA 20", color="blue")
+            ax.legend()
+        """
+        pass
+
+    @abstractmethod
+    def add_indicator_panel(
+        self,
+        fig: Figure,
+        df: pd.DataFrame,
+        columns: Optional[List[str]] = None,
+        ylabel: Optional[str] = None,
+        height_ratio: float = 0.3,
+        ylim: Optional[Tuple[float, float]] = None,
+        **kwargs
+    ) -> plt.Axes:
+        """
+        Add an indicator panel below the chart with time-aligned x-axis.
+
+        Use this to display indicators that should be shown separately from the
+        price chart (e.g., RSI, MACD, volume).
+
+        Args:
+            fig: Figure object from plot_ohlc()
+            df: DataFrame with indicator data (must have same index as OHLC data)
+            columns: Column names to plot. If None, plots all numeric columns.
+            ylabel: Y-axis label (e.g., "RSI", "MACD")
+            height_ratio: Panel height relative to main chart (default: 0.3 = 30%)
+            ylim: Y-axis limits as (min, max). If None, auto-scales.
+            **kwargs: Line styling options (color, linewidth, linestyle, alpha)
+
+        Returns:
+            Axes object for the new panel (use for further customization)
+
+        Examples:
+            # Add RSI panel with reference lines
+            fig, ax = api.plot_ohlc(df)
+            rsi_ax = api.add_indicator_panel(
+                fig, df,
+                columns=["rsi"],
+                ylabel="RSI",
+                ylim=(0, 100)
+            )
+            rsi_ax.axhline(30, color='green', linestyle='--', alpha=0.5)
+            rsi_ax.axhline(70, color='red', linestyle='--', alpha=0.5)
+
+            # Add MACD panel
+            fig, ax = api.plot_ohlc(df)
+            api.add_indicator_panel(
+                fig, df,
+                columns=["macd", "macd_signal"],
+                ylabel="MACD"
+            )
+        """
+        pass
+
+    @abstractmethod
+    def create_figure(
+        self,
+        figsize: Tuple[int, int] = (12, 8),
+        style: str = "charles"
+    ) -> Tuple[Figure, plt.Axes]:
+        """
+        Create a styled figure for custom visualizations.
+
+        Use this when you want to create charts other than candlesticks
+        (e.g., histograms, scatter plots, heatmaps).
+
+        Args:
+            figsize: Figure size as (width, height) in inches. Default: (12, 8)
+            style: Style name for consistent theming. Default: "charles"
+
+        Returns:
+            Tuple of (Figure, Axes) ready for plotting
+
+        Examples:
+            # Histogram
+            fig, ax = api.create_figure()
+            ax.hist(returns, bins=50)
+            ax.set_title("Return Distribution")
+
+            # Heatmap
+            fig, ax = api.create_figure(figsize=(10, 10))
+            import seaborn as sns
+            sns.heatmap(correlation_matrix, ax=ax)
+            ax.set_title("Correlation Matrix")
+        """
+        pass

sandbox/dexorder/api/data_api.py

@@ -0,0 +1,162 @@
+from abc import ABC, abstractmethod
+from typing import Optional, List
+
+import pandas as pd
+
+from dexorder.utils import TimestampInput
+
+
+class DataAPI(ABC):
+    """
+    API for accessing market data.
+
+    Provides methods to query OHLC (Open, High, Low, Close) candlestick data
+    for cryptocurrency markets.
+    """
+
+    @abstractmethod
+    async def historical_ohlc(
+        self,
+        ticker: str,
+        period_seconds: int,
+        start_time: TimestampInput,
+        end_time: TimestampInput,
+        extra_columns: Optional[List[str]] = None,
+    ) -> pd.DataFrame:
+        """
+        Fetch historical OHLC candlestick data for a market.
+
+        Args:
+            ticker: Market identifier in format "EXCHANGE:SYMBOL"
+                   Examples: "BINANCE:BTC/USDT", "COINBASE:ETH/USD"
+            period_seconds: Candle period in seconds
+                   Common values:
+                   - 60 (1 minute)
+                   - 300 (5 minutes)
+                   - 900 (15 minutes)
+                   - 3600 (1 hour)
+                   - 86400 (1 day)
+                   - 604800 (1 week)
+            start_time: Start of time range. Accepts:
+                       - Unix timestamp in seconds (int/float): 1640000000
+                       - Date string: "2021-12-20" or "2021-12-20 12:00:00"
+                       - datetime object: datetime(2021, 12, 20)
+                       - pandas Timestamp: pd.Timestamp("2021-12-20")
+            end_time: End of time range. Same formats as start_time.
+            extra_columns: Optional additional columns to include beyond the standard
+                          OHLC columns. Available options:
+                          - "volume" - Total volume (decimal float)
+                          - "buy_vol" - Buy-side volume (decimal float)
+                          - "sell_vol" - Sell-side volume (decimal float)
+                          - "open_time", "high_time", "low_time", "close_time" (timestamps)
+                          - "open_interest" (for futures markets)
+                          - "ticker", "period_seconds"
+
+        Returns:
+            DataFrame with candlestick data sorted by timestamp (ascending).
+            Standard columns (always included):
+            - timestamp: Period start time in microseconds
+            - open: Opening price (decimal float)
+            - high: Highest price (decimal float)
+            - low: Lowest price (decimal float)
+            - close: Closing price (decimal float)
+
+            Plus any columns specified in extra_columns.
+
+            All prices and volumes are automatically converted to decimal floats
+            using market metadata. No manual conversion is needed.
+
+            Returns empty DataFrame if no data is available.
+
+        Examples:
+            # Basic OHLC with Unix timestamp
+            df = await api.historical_ohlc(
+                ticker="BINANCE:BTC/USDT",
+                period_seconds=3600,
+                start_time=1640000000,
+                end_time=1640086400
+            )
+
+            # Using date strings with volume
+            df = await api.historical_ohlc(
+                ticker="BINANCE:BTC/USDT",
+                period_seconds=3600,
+                start_time="2021-12-20",
+                end_time="2021-12-21",
+                extra_columns=["volume"]
+            )
+
+            # Using datetime objects
+            from datetime import datetime
+            df = await api.historical_ohlc(
+                ticker="COINBASE:ETH/USD",
+                period_seconds=300,
+                start_time=datetime(2021, 12, 20, 9, 30),
+                end_time=datetime(2021, 12, 20, 16, 30),
+                extra_columns=["volume", "buy_vol", "sell_vol"]
+            )
+        """
+        pass
+
+    @abstractmethod
+    async def latest_ohlc(
+        self,
+        ticker: str,
+        period_seconds: int,
+        length: int = 1,
+        extra_columns: Optional[List[str]] = None,
+    ) -> pd.DataFrame:
+        """
+        Query the most recent OHLC candles for a ticker.
+
+        This method fetches the latest N completed candles without needing to
+        specify exact timestamps. Useful for real-time analysis and indicators.
+
+        Args:
+            ticker: Market identifier in format "EXCHANGE:SYMBOL"
+                   Examples: "BINANCE:BTC/USDT", "COINBASE:ETH/USD"
+            period_seconds: OHLC candle period in seconds
+                   Common values: 60 (1m), 300 (5m), 900 (15m), 3600 (1h),
+                                 86400 (1d), 604800 (1w)
+            length: Number of most recent candles to return (default: 1)
+            extra_columns: Optional list of additional column names to include.
+                          Same column options as historical_ohlc:
+                          - "volume", "buy_vol", "sell_vol"
+                          - "open_time", "high_time", "low_time", "close_time"
+                          - "open_interest", "ticker", "period_seconds"
+
+        Returns:
+            Pandas DataFrame with the same column structure as historical_ohlc,
+            containing the N most recent completed candles sorted by timestamp.
+            Returns empty DataFrame if no data is available.
+
+        Examples:
+            # Get the last candle
+            df = await api.latest_ohlc(
+                ticker="BINANCE:BTC/USDT",
+                period_seconds=3600
+            )
+            # Returns: timestamp, open, high, low, close
+
+            # Get the last 50 5-minute candles with volume
+            df = await api.latest_ohlc(
+                ticker="COINBASE:ETH/USD",
+                period_seconds=300,
+                length=50,
+                extra_columns=["volume", "buy_vol", "sell_vol"]
+            )
+
+            # Get recent candles with all timing data
+            df = await api.latest_ohlc(
+                ticker="BINANCE:BTC/USDT",
+                period_seconds=60,
+                length=100,
+                extra_columns=["open_time", "high_time", "low_time", "close_time"]
+            )
+
+        Note:
+            This method returns only completed candles. The current (incomplete)
+            candle is not included.
+        """
+        pass
+

sandbox/dexorder/conda_manager.py

@@ -0,0 +1,400 @@
+"""
+Conda Package Manager
+
+Manages dynamic installation and cleanup of conda packages for user components.
+Scans metadata files to determine required packages and syncs the conda environment.
+"""
+
+import json
+import logging
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional, Set
+
+log = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Conda Environment Detection
+# =============================================================================
+
+def get_conda_env_path() -> Optional[Path]:
+    """
+    Detect the active conda environment path.
+
+    Returns:
+        Path to conda environment, or None if not in a conda environment
+    """
+    # Check for CONDA_PREFIX environment variable
+    import os
+    conda_prefix = os.getenv("CONDA_PREFIX")
+    if conda_prefix:
+        return Path(conda_prefix)
+
+    # Check if python executable is in a conda environment
+    python_path = Path(sys.executable)
+
+    # Look for conda-meta directory (indicates conda environment)
+    for parent in [python_path.parent, python_path.parent.parent]:
+        if (parent / "conda-meta").exists():
+            return parent
+
+    return None
+
+
+def get_conda_executable() -> Optional[Path]:
+    """
+    Find the conda executable.
+
+    Returns:
+        Path to conda executable, or None if not found
+    """
+    env_path = get_conda_env_path()
+    if not env_path:
+        return None
+
+    # Try common locations
+    for conda_name in ["conda", "mamba"]:
+        # Look in environment bin
+        conda_bin = env_path / "bin" / conda_name
+        if conda_bin.exists():
+            return conda_bin
+
+        # Look in parent conda installation
+        parent_conda = env_path.parent.parent / "bin" / conda_name
+        if parent_conda.exists():
+            return parent_conda
+
+    return None
+
+
+# =============================================================================
+# Package Management
+# =============================================================================
+
+def get_installed_packages() -> Set[str]:
+    """
+    Get set of currently installed conda packages.
+
+    Returns:
+        Set of package names
+    """
+    try:
+        result = subprocess.run(
+            [sys.executable, "-m", "conda", "list", "--json"],
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+
+        if result.returncode == 0:
+            packages = json.loads(result.stdout)
+            return {pkg["name"] for pkg in packages}
+        else:
+            log.error(f"Failed to list conda packages: {result.stderr}")
+            return set()
+    except subprocess.TimeoutExpired:
+        log.error("Timeout while listing conda packages")
+        return set()
+    except Exception as e:
+        log.error(f"Error listing conda packages: {e}")
+        return set()
+
+
+def install_packages(packages: list[str]) -> dict:
+    """
+    Install conda packages if not already installed.
+
+    Args:
+        packages: List of package names to install
+
+    Returns:
+        dict with:
+        - success: bool
+        - installed: list[str] - packages that were installed
+        - skipped: list[str] - packages already installed
+        - failed: list[str] - packages that failed to install
+        - error: str (if any)
+    """
+    if not packages:
+        return {
+            "success": True,
+            "installed": [],
+            "skipped": [],
+            "failed": [],
+        }
+
+    # Get currently installed packages
+    installed = get_installed_packages()
+
+    # Filter out already installed packages
+    to_install = [pkg for pkg in packages if pkg not in installed]
+    skipped = [pkg for pkg in packages if pkg in installed]
+
+    if not to_install:
+        log.info(f"All packages already installed: {skipped}")
+        return {
+            "success": True,
+            "installed": [],
+            "skipped": skipped,
+            "failed": [],
+        }
+
+    # Install missing packages
+    log.info(f"Installing conda packages: {to_install}")
+
+    try:
+        result = subprocess.run(
+            [sys.executable, "-m", "conda", "install", "-y", "-c", "conda-forge"] + to_install,
+            capture_output=True,
+            text=True,
+            timeout=300,  # 5 minute timeout
+        )
+
+        if result.returncode == 0:
+            log.info(f"Successfully installed packages: {to_install}")
+            return {
+                "success": True,
+                "installed": to_install,
+                "skipped": skipped,
+                "failed": [],
+            }
+        else:
+            log.error(f"Failed to install packages: {result.stderr}")
+            return {
+                "success": False,
+                "installed": [],
+                "skipped": skipped,
+                "failed": to_install,
+                "error": result.stderr,
+            }
+    except subprocess.TimeoutExpired:
+        log.error("Timeout while installing conda packages")
+        return {
+            "success": False,
+            "installed": [],
+            "skipped": skipped,
+            "failed": to_install,
+            "error": "Installation timeout",
+        }
+    except Exception as e:
+        log.error(f"Error installing conda packages: {e}")
+        return {
+            "success": False,
+            "installed": [],
+            "skipped": skipped,
+            "failed": to_install,
+            "error": str(e),
+        }
+
+
+def remove_packages(packages: list[str]) -> dict:
+    """
+    Remove conda packages.
+
+    Args:
+        packages: List of package names to remove
+
+    Returns:
+        dict with:
+        - success: bool
+        - removed: list[str] - packages that were removed
+        - error: str (if any)
+    """
+    if not packages:
+        return {
+            "success": True,
+            "removed": [],
+        }
+
+    log.info(f"Removing conda packages: {packages}")
+
+    try:
+        result = subprocess.run(
+            [sys.executable, "-m", "conda", "remove", "-y"] + packages,
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+
+        if result.returncode == 0:
+            log.info(f"Successfully removed packages: {packages}")
+            return {
+                "success": True,
+                "removed": packages,
+            }
+        else:
+            log.error(f"Failed to remove packages: {result.stderr}")
+            return {
+                "success": False,
+                "removed": [],
+                "error": result.stderr,
+            }
+    except subprocess.TimeoutExpired:
+        log.error("Timeout while removing conda packages")
+        return {
+            "success": False,
+            "removed": [],
+            "error": "Removal timeout",
+        }
+    except Exception as e:
+        log.error(f"Error removing conda packages: {e}")
+        return {
+            "success": False,
+            "removed": [],
+            "error": str(e),
+        }
+
+
+# =============================================================================
+# Metadata Scanning
+# =============================================================================
+
+def scan_metadata_packages(data_dir: Path) -> Set[str]:
+    """
+    Scan all metadata files to find required conda packages.
+
+    Args:
+        data_dir: Base data directory containing category subdirectories
+
+    Returns:
+        Set of all required package names
+    """
+    packages = set()
+
+    # Scan all category directories
+    for category_dir in data_dir.iterdir():
+        if not category_dir.is_dir():
+            continue
+
+        # Scan all items in this category
+        for item_dir in category_dir.iterdir():
+            if not item_dir.is_dir():
+                continue
+
+            metadata_path = item_dir / "metadata.json"
+            if not metadata_path.exists():
+                continue
+
+            try:
+                metadata = json.loads(metadata_path.read_text())
+                conda_packages = metadata.get("conda_packages", [])
+                if conda_packages:
+                    packages.update(conda_packages)
+                    log.debug(f"Found packages in {item_dir.name}: {conda_packages}")
+            except Exception as e:
+                log.error(f"Failed to read metadata from {metadata_path}: {e}")
+
+    return packages
+
+
+def get_base_packages(environment_yml: Path) -> Set[str]:
+    """
+    Get base packages from environment.yml.
+
+    Args:
+        environment_yml: Path to environment.yml file
+
+    Returns:
+        Set of base package names
+    """
+    if not environment_yml.exists():
+        log.warning(f"environment.yml not found at {environment_yml}")
+        return set()
+
+    try:
+        import yaml
+        with open(environment_yml) as f:
+            env_spec = yaml.safe_load(f)
+
+        packages = set()
+
+        # Add conda packages
+        for dep in env_spec.get("dependencies", []):
+            if isinstance(dep, str):
+                # Extract package name (before version spec)
+                pkg_name = dep.split(">=")[0].split("=")[0].split("<")[0].split(">")[0].strip()
+                packages.add(pkg_name)
+
+        return packages
+    except Exception as e:
+        log.error(f"Failed to parse environment.yml: {e}")
+        return set()
+
+
+# =============================================================================
+# Sync Operation
+# =============================================================================
+
+def sync_packages(data_dir: Path, environment_yml: Optional[Path] = None) -> dict:
+    """
+    Sync conda packages with metadata requirements.
+
+    Scans all metadata files, computes desired package set, and removes
+    packages that are no longer needed (excluding base environment packages).
+
+    Args:
+        data_dir: Base data directory
+        environment_yml: Path to environment.yml (optional)
+
+    Returns:
+        dict with:
+        - success: bool
+        - required: list[str] - packages required by metadata
+        - base: list[str] - base packages from environment.yml
+        - installed: list[str] - currently installed packages
+        - to_remove: list[str] - packages to be removed
+        - removed: list[str] - packages that were removed
+        - error: str (if any)
+    """
+    log.info("Starting conda package sync")
+
+    # Get required packages from metadata
+    required_packages = scan_metadata_packages(data_dir)
+    log.info(f"Required packages from metadata: {required_packages}")
+
+    # Get base packages from environment.yml
+    base_packages = set()
+    if environment_yml and environment_yml.exists():
+        base_packages = get_base_packages(environment_yml)
+        log.info(f"Base packages from environment.yml: {base_packages}")
+
+    # Get currently installed packages
+    installed_packages = get_installed_packages()
+    log.info(f"Currently installed packages: {len(installed_packages)} total")
+
+    # Compute packages to remove
+    # Remove packages that are:
+    # - Currently installed
+    # - Not in base packages
+    # - Not in required packages
+    protected = base_packages | required_packages
+    to_remove = [pkg for pkg in installed_packages if pkg not in protected]
+
+    # Filter out critical system packages (be conservative)
+    system_prefixes = ["python", "conda", "pip", "setuptools", "wheel", "_"]
+    to_remove = [pkg for pkg in to_remove if not any(pkg.startswith(prefix) for prefix in system_prefixes)]
+
+    log.info(f"Packages to remove: {to_remove}")
+
+    result = {
+        "success": True,
+        "required": sorted(required_packages),
+        "base": sorted(base_packages),
+        "installed": sorted(installed_packages),
+        "to_remove": to_remove,
+        "removed": [],
+    }
+
+    # Remove packages if any
+    if to_remove:
+        remove_result = remove_packages(to_remove)
+        result["success"] = remove_result["success"]
+        result["removed"] = remove_result.get("removed", [])
+        if not remove_result["success"]:
+            result["error"] = remove_result.get("error", "Unknown error")
+
+    log.info(f"Conda package sync complete: {len(result['removed'])} packages removed")
+
+    return result

sandbox/dexorder/events/__init__.py

@@ -0,0 +1,57 @@
+"""
+User Container Event System
+
+Publishes events to the gateway via dual ZMQ patterns:
+- XPUB for informational events (fire-and-forget to active sessions)
+- DEALER for critical events (guaranteed delivery with ack)
+
+See doc/protocol.md and doc/user_container_events.md for details.
+"""
+
+from .types import (
+    # Enums
+    EventType,
+    Priority,
+    ChannelType,
+    AckStatus,
+    # Message types
+    ChannelPreference,
+    DeliverySpec,
+    UserEvent,
+    EventAck,
+    # Serialization
+    MSG_TYPE_USER_EVENT,
+    MSG_TYPE_EVENT_ACK,
+    serialize_user_event,
+    deserialize_user_event,
+    serialize_event_ack,
+    deserialize_event_ack,
+)
+
+from .publisher import EventPublisher
+
+from .pending_store import PendingStore
+
+__all__ = [
+    # Enums
+    "EventType",
+    "Priority",
+    "ChannelType",
+    "AckStatus",
+    # Message types
+    "ChannelPreference",
+    "DeliverySpec",
+    "UserEvent",
+    "EventAck",
+    # Serialization
+    "MSG_TYPE_USER_EVENT",
+    "MSG_TYPE_EVENT_ACK",
+    "serialize_user_event",
+    "deserialize_user_event",
+    "serialize_event_ack",
+    "deserialize_event_ack",
+    # Publisher
+    "EventPublisher",
+    # Storage
+    "PendingStore",
+]

sandbox/dexorder/events/pending_store.py

@@ -0,0 +1,120 @@
+"""
+Pending event store for crash recovery.
+
+Persists critical events that haven't been acknowledged to disk,
+so they can be replayed after a container restart.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import List, Optional
+import asyncio
+import aiofiles
+
+from .types import UserEvent
+
+logger = logging.getLogger(__name__)
+
+
+class PendingStore:
+    """
+    Persists pending critical events to disk for crash recovery.
+
+    Events are written to a JSON file when the publisher shuts down,
+    and loaded back when it starts up.
+    """
+
+    def __init__(self, path: Optional[str] = None):
+        """
+        Initialize pending store.
+
+        Args:
+            path: Path to the pending events file.
+                  If None, persistence is disabled.
+        """
+        self.path = Path(path) if path else None
+        self._lock = asyncio.Lock()
+
+    async def save_pending(self, events: List[UserEvent]) -> None:
+        """
+        Save pending events to disk.
+
+        Args:
+            events: List of events to persist
+        """
+        if not self.path:
+            return
+
+        if not events:
+            # No events to save, remove file if exists
+            await self._remove_file()
+            return
+
+        async with self._lock:
+            try:
+                # Ensure parent directory exists
+                self.path.parent.mkdir(parents=True, exist_ok=True)
+
+                # Serialize events
+                data = [event.to_dict() for event in events]
+                json_str = json.dumps(data, indent=2)
+
+                # Write atomically (write to temp, then rename)
+                temp_path = self.path.with_suffix(".tmp")
+                async with aiofiles.open(temp_path, "w") as f:
+                    await f.write(json_str)
+
+                # Atomic rename
+                temp_path.rename(self.path)
+
+                logger.info("Saved %d pending events to %s", len(events), self.path)
+
+            except Exception as e:
+                logger.error("Failed to save pending events: %s", e, exc_info=True)
+
+    async def load_pending(self) -> List[UserEvent]:
+        """
+        Load pending events from disk.
+
+        Returns:
+            List of pending events, or empty list if none/error
+        """
+        if not self.path or not self.path.exists():
+            return []
+
+        async with self._lock:
+            try:
+                async with aiofiles.open(self.path, "r") as f:
+                    content = await f.read()
+
+                data = json.loads(content)
+                events = [UserEvent.from_dict(d) for d in data]
+
+                # Remove file after successful load
+                await self._remove_file()
+
+                logger.info("Loaded %d pending events from %s", len(events), self.path)
+                return events
+
+            except json.JSONDecodeError as e:
+                logger.error("Failed to parse pending events file: %s", e)
+                await self._remove_file()
+                return []
+
+            except Exception as e:
+                logger.error("Failed to load pending events: %s", e, exc_info=True)
+                return []
+
+    async def _remove_file(self) -> None:
+        """Remove the pending events file."""
+        if self.path and self.path.exists():
+            try:
+                self.path.unlink()
+                logger.debug("Removed pending events file: %s", self.path)
+            except Exception as e:
+                logger.warning("Failed to remove pending events file: %s", e)
+
+    def has_pending(self) -> bool:
+        """Check if there are pending events on disk."""
+        return self.path is not None and self.path.exists()

sandbox/dexorder/events/publisher.py

@@ -0,0 +1,441 @@
+"""
+Event publisher for user containers.
+
+Publishes events via dual ZMQ patterns:
+- XPUB for informational events (fire-and-forget)
+- DEALER for critical events (guaranteed delivery with ack)
+
+The publisher automatically routes events based on:
+1. Event priority (INFORMATIONAL always uses XPUB if subscribed)
+2. Whether an active gateway session is subscribed (tracked via XPUB)
+"""
+
+import asyncio
+import logging
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Dict, Optional, Set
+
+import zmq
+import zmq.asyncio
+
+from .types import (
+    UserEvent,
+    EventAck,
+    Priority,
+    AckStatus,
+    serialize_user_event,
+    deserialize_event_ack,
+    get_event_type_name,
+)
+from .pending_store import PendingStore
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PendingEvent:
+    """Tracks a pending critical event awaiting acknowledgment."""
+
+    event: UserEvent
+    sent_at: float
+    retries: int = 0
+
+
+class EventPublisher:
+    """
+    Publishes user events via dual ZMQ channels.
+
+    - XPUB socket (bind): For informational events to active sessions
+    - DEALER socket (connect): For critical events with guaranteed delivery
+
+    The publisher tracks XPUB subscriptions to know when a gateway has an
+    active session for this user. If subscribed, events go via XPUB (fast).
+    If not subscribed and priority > INFORMATIONAL, events go via DEALER.
+
+    Usage:
+        publisher = EventPublisher(user_id="user-123")
+        await publisher.start()
+
+        # Informational event (dropped if no active session)
+        await publisher.publish(UserEvent(
+            event_type=EventType.INDICATOR_UPDATED,
+            payload={"indicator": "RSI", "value": 65.5},
+            delivery=DeliverySpec.informational(),
+        ))
+
+        # Critical event (guaranteed delivery)
+        await publisher.publish(UserEvent(
+            event_type=EventType.ORDER_FILLED,
+            payload={"order_id": "123", "symbol": "BTC/USDT", ...},
+            delivery=DeliverySpec.critical(),
+        ))
+
+        await publisher.stop()
+    """
+
+    def __init__(
+        self,
+        user_id: str,
+        xpub_port: int = 5570,
+        gateway_router_endpoint: str = "tcp://gateway:5571",
+        ack_timeout: float = 30.0,
+        max_retries: int = 3,
+        pending_store_path: Optional[str] = None,
+    ):
+        """
+        Initialize event publisher.
+
+        Args:
+            user_id: User ID for this container
+            xpub_port: Port to bind XPUB socket on (gateway connects here)
+            gateway_router_endpoint: Gateway ROUTER socket endpoint (we connect)
+            ack_timeout: Seconds to wait for ack before retrying
+            max_retries: Maximum retries for critical events
+            pending_store_path: Path to persist pending events (for crash recovery)
+        """
+        self.user_id = user_id
+        self.xpub_port = xpub_port
+        self.gateway_router_endpoint = gateway_router_endpoint
+        self.ack_timeout = ack_timeout
+        self.max_retries = max_retries
+
+        # ZMQ context and sockets
+        self.ctx: Optional[zmq.asyncio.Context] = None
+        self.xpub_socket: Optional[zmq.asyncio.Socket] = None
+        self.dealer_socket: Optional[zmq.asyncio.Socket] = None
+
+        # Track active subscriptions (set of topic strings)
+        self.active_subscriptions: Set[str] = set()
+
+        # Track pending critical events awaiting ack
+        self.pending_events: Dict[str, PendingEvent] = {}
+
+        # Persistent store for crash recovery
+        self.pending_store = PendingStore(pending_store_path)
+
+        # Background tasks
+        self._subscription_task: Optional[asyncio.Task] = None
+        self._ack_task: Optional[asyncio.Task] = None
+        self._retry_task: Optional[asyncio.Task] = None
+        self._running = False
+
+        # Statistics
+        self.stats = {
+            "events_published": 0,
+            "events_via_xpub": 0,
+            "events_via_dealer": 0,
+            "events_dropped": 0,
+            "events_delivered": 0,
+            "events_failed": 0,
+            "retries": 0,
+        }
+
+    async def start(self) -> None:
+        """Start the event publisher."""
+        if self._running:
+            logger.warning("Event publisher already running")
+            return
+
+        logger.info(
+            "Starting event publisher: user_id=%s, xpub_port=%d, gateway=%s",
+            self.user_id,
+            self.xpub_port,
+            self.gateway_router_endpoint,
+        )
+
+        # Create ZMQ context
+        self.ctx = zmq.asyncio.Context()
+
+        # Create XPUB socket for informational events
+        self.xpub_socket = self.ctx.socket(zmq.XPUB)
+        self.xpub_socket.setsockopt(zmq.XPUB_VERBOSE, 1)  # Receive all sub/unsub
+        self.xpub_socket.bind(f"tcp://*:{self.xpub_port}")
+        logger.info("XPUB socket bound on port %d", self.xpub_port)
+
+        # Create DEALER socket for critical events
+        self.dealer_socket = self.ctx.socket(zmq.DEALER)
+        self.dealer_socket.setsockopt_string(zmq.IDENTITY, f"container-{self.user_id}")
+        self.dealer_socket.connect(self.gateway_router_endpoint)
+        logger.info("DEALER socket connected to %s", self.gateway_router_endpoint)
+
+        # Load any persisted pending events
+        persisted = await self.pending_store.load_pending()
+        for event in persisted:
+            self.pending_events[event.event_id] = PendingEvent(
+                event=event,
+                sent_at=time.time(),
+                retries=0,
+            )
+        if persisted:
+            logger.info("Loaded %d pending events from disk", len(persisted))
+
+        # Start background tasks
+        self._running = True
+        self._subscription_task = asyncio.create_task(self._subscription_loop())
+        self._ack_task = asyncio.create_task(self._ack_loop())
+        self._retry_task = asyncio.create_task(self._retry_loop())
+
+        # Resend any loaded pending events
+        for pending in list(self.pending_events.values()):
+            await self._send_via_dealer(pending.event)
+
+        logger.info("Event publisher started")
+
+    async def stop(self) -> None:
+        """Stop the event publisher and persist pending events."""
+        if not self._running:
+            return
+
+        logger.info("Stopping event publisher")
+        self._running = False
+
+        # Cancel background tasks
+        for task in [self._subscription_task, self._ack_task, self._retry_task]:
+            if task:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+
+        # Persist pending critical events for crash recovery
+        if self.pending_events:
+            events = [pe.event for pe in self.pending_events.values()]
+            await self.pending_store.save_pending(events)
+            logger.info("Persisted %d pending events", len(events))
+
+        # Close sockets
+        if self.xpub_socket:
+            self.xpub_socket.close()
+        if self.dealer_socket:
+            self.dealer_socket.close()
+        if self.ctx:
+            self.ctx.term()
+
+        logger.info(
+            "Event publisher stopped. Stats: %s",
+            self.stats,
+        )
+
+    def has_active_subscriber(self) -> bool:
+        """Check if any gateway is subscribed to this user's events."""
+        topic = f"USER:{self.user_id}"
+        return topic in self.active_subscriptions
+
+    async def publish(self, event: UserEvent) -> None:
+        """
+        Publish an event via the appropriate channel.
+
+        Routing logic:
+        - INFORMATIONAL: XPUB only if subscribed, else drop
+        - NORMAL/CRITICAL + subscribed: XPUB (fast path)
+        - NORMAL/CRITICAL + not subscribed: DEALER (guaranteed)
+
+        Args:
+            event: Event to publish
+        """
+        # Ensure event has required fields
+        if not event.event_id:
+            event.event_id = str(uuid.uuid4())
+        if not event.user_id:
+            event.user_id = self.user_id
+        if not event.timestamp:
+            event.timestamp = int(time.time() * 1000)
+
+        priority = event.delivery.priority
+        has_subscriber = self.has_active_subscriber()
+
+        logger.debug(
+            "Publishing event: id=%s, type=%s, priority=%s, has_subscriber=%s",
+            event.event_id,
+            get_event_type_name(event.event_type),
+            Priority(priority).name,
+            has_subscriber,
+        )
+
+        self.stats["events_published"] += 1
+
+        if priority == Priority.INFORMATIONAL:
+            # Fire and forget - only send if someone's listening
+            if has_subscriber:
+                await self._send_via_xpub(event)
+                self.stats["events_via_xpub"] += 1
+            else:
+                logger.debug(
+                    "Dropping informational event (no subscriber): %s",
+                    event.event_id,
+                )
+                self.stats["events_dropped"] += 1
+
+        elif has_subscriber:
+            # Active session exists - use fast path
+            await self._send_via_xpub(event)
+            self.stats["events_via_xpub"] += 1
+
+        else:
+            # No active session - use guaranteed delivery
+            await self._send_via_dealer(event)
+            self.stats["events_via_dealer"] += 1
+
+            # Track for ack
+            self.pending_events[event.event_id] = PendingEvent(
+                event=event,
+                sent_at=time.time(),
+                retries=0,
+            )
+
+    async def _send_via_xpub(self, event: UserEvent) -> None:
+        """Send event via XPUB socket (fire-and-forget)."""
+        topic = f"USER:{self.user_id}"
+        payload = serialize_user_event(event)
+
+        await self.xpub_socket.send_multipart([topic.encode(), payload])
+
+        logger.debug(
+            "Sent event via XPUB: id=%s, type=%s",
+            event.event_id,
+            get_event_type_name(event.event_type),
+        )
+
+    async def _send_via_dealer(self, event: UserEvent) -> None:
+        """Send event via DEALER socket (with ack tracking)."""
+        payload = serialize_user_event(event)
+        await self.dealer_socket.send(payload)
+
+        logger.debug(
+            "Sent event via DEALER: id=%s, type=%s",
+            event.event_id,
+            get_event_type_name(event.event_type),
+        )
+
+    async def _subscription_loop(self) -> None:
+        """Process XPUB subscription/unsubscription messages."""
+        logger.debug("Starting subscription loop")
+
+        while self._running:
+            try:
+                # Poll with timeout to allow shutdown
+                if await self.xpub_socket.poll(100):
+                    msg = await self.xpub_socket.recv()
+
+                    # First byte: 1 = subscribe, 0 = unsubscribe
+                    # Remaining bytes: topic
+                    is_subscribe = msg[0] == 1
+                    topic = msg[1:].decode()
+
+                    if is_subscribe:
+                        self.active_subscriptions.add(topic)
+                        logger.info("Gateway subscribed to topic: %s", topic)
+                    else:
+                        self.active_subscriptions.discard(topic)
+                        logger.info("Gateway unsubscribed from topic: %s", topic)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error("Error in subscription loop: %s", e, exc_info=True)
+
+        logger.debug("Subscription loop ended")
+
+    async def _ack_loop(self) -> None:
+        """Process EventAck messages from gateway."""
+        logger.debug("Starting ack loop")
+
+        while self._running:
+            try:
+                # Poll with timeout
+                if await self.dealer_socket.poll(100):
+                    payload = await self.dealer_socket.recv()
+                    ack = deserialize_event_ack(payload)
+
+                    logger.debug(
+                        "Received ack: event_id=%s, status=%s",
+                        ack.event_id,
+                        AckStatus(ack.status).name,
+                    )
+
+                    if ack.event_id in self.pending_events:
+                        pending = self.pending_events.pop(ack.event_id)
+
+                        if ack.status == AckStatus.DELIVERED:
+                            logger.info(
+                                "Event delivered: id=%s, type=%s, via=%s",
+                                ack.event_id,
+                                get_event_type_name(pending.event.event_type),
+                                ack.delivered_via.name if ack.delivered_via else "unknown",
+                            )
+                            self.stats["events_delivered"] += 1
+
+                        elif ack.status == AckStatus.QUEUED:
+                            logger.info(
+                                "Event queued for delivery: id=%s",
+                                ack.event_id,
+                            )
+                            self.stats["events_delivered"] += 1
+
+                        elif ack.status == AckStatus.ACK_ERROR:
+                            logger.error(
+                                "Event delivery failed: id=%s, error=%s",
+                                ack.event_id,
+                                ack.error_message,
+                            )
+                            self.stats["events_failed"] += 1
+                    else:
+                        logger.debug("Received ack for unknown event: %s", ack.event_id)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error("Error in ack loop: %s", e, exc_info=True)
+
+        logger.debug("Ack loop ended")
+
+    async def _retry_loop(self) -> None:
+        """Retry pending events that haven't been acked."""
+        logger.debug("Starting retry loop")
+
+        while self._running:
+            try:
+                await asyncio.sleep(5)  # Check every 5 seconds
+
+                now = time.time()
+                for event_id, pending in list(self.pending_events.items()):
+                    if now - pending.sent_at > self.ack_timeout:
+                        if pending.retries >= self.max_retries:
+                            # Give up
+                            logger.error(
+                                "Event exceeded max retries, dropping: id=%s, type=%s",
+                                event_id,
+                                get_event_type_name(pending.event.event_type),
+                            )
+                            del self.pending_events[event_id]
+                            self.stats["events_failed"] += 1
+                        else:
+                            # Retry
+                            pending.retries += 1
+                            pending.sent_at = now
+                            await self._send_via_dealer(pending.event)
+                            logger.info(
+                                "Retrying event: id=%s, attempt=%d/%d",
+                                event_id,
+                                pending.retries,
+                                self.max_retries,
+                            )
+                            self.stats["retries"] += 1
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error("Error in retry loop: %s", e, exc_info=True)
+
+        logger.debug("Retry loop ended")
+
+    def get_stats(self) -> Dict[str, int]:
+        """Get publisher statistics."""
+        return {
+            **self.stats,
+            "pending_events": len(self.pending_events),
+            "active_subscriptions": len(self.active_subscriptions),
+        }

sandbox/dexorder/events/types.py

@@ -0,0 +1,384 @@
+"""
+User Event Types for Container → Gateway communication.
+
+These types mirror the protobuf definitions in protobuf/user_events.proto
+and the TypeScript types in gateway/src/events/types.ts.
+
+Message Type IDs (must match protocol.md):
+- UserEvent: 0x20
+- EventAck: 0x21
+"""
+
+import json
+import uuid
+import time
+from dataclasses import dataclass, field
+from enum import IntEnum
+from typing import List, Optional, Any
+
+# =============================================================================
+# Message Type IDs
+# =============================================================================
+
+MSG_TYPE_USER_EVENT = 0x20
+MSG_TYPE_EVENT_ACK = 0x21
+
+# =============================================================================
+# Enums
+# =============================================================================
+
+
+class EventType(IntEnum):
+    """Types of events that containers can emit."""
+
+    # Trading events
+    ORDER_PLACED = 0
+    ORDER_FILLED = 1
+    ORDER_CANCELLED = 2
+    ORDER_REJECTED = 3
+    ORDER_EXPIRED = 4
+
+    # Alert events
+    ALERT_TRIGGERED = 10
+    ALERT_CREATED = 11
+    ALERT_DELETED = 12
+
+    # Position events
+    POSITION_OPENED = 20
+    POSITION_CLOSED = 21
+    POSITION_UPDATED = 22
+    POSITION_LIQUIDATED = 23
+
+    # Workspace/chart events
+    WORKSPACE_CHANGED = 30
+    CHART_ANNOTATION_ADDED = 31
+    CHART_ANNOTATION_REMOVED = 32
+    INDICATOR_UPDATED = 33
+
+    # Strategy events
+    STRATEGY_STARTED = 40
+    STRATEGY_STOPPED = 41
+    STRATEGY_LOG = 42
+    STRATEGY_ERROR = 43
+    BACKTEST_COMPLETED = 44
+
+    # System events
+    CONTAINER_STARTING = 50
+    CONTAINER_READY = 51
+    CONTAINER_SHUTTING_DOWN = 52
+    EVENT_ERROR = 53
+
+
+class Priority(IntEnum):
+    """
+    Event delivery priority.
+
+    INFORMATIONAL: Drop if no active session (fire-and-forget via XPUB)
+    NORMAL: Best effort - XPUB if subscribed, else DEALER
+    CRITICAL: Must deliver - always uses DEALER with ack
+    """
+
+    INFORMATIONAL = 0
+    NORMAL = 1
+    CRITICAL = 2
+
+
+class ChannelType(IntEnum):
+    """Delivery channel types."""
+
+    ACTIVE_SESSION = 0  # Whatever's currently connected
+    WEB = 1  # WebSocket to web UI
+    TELEGRAM = 2  # Telegram bot message
+    EMAIL = 3  # Email notification
+    PUSH = 4  # Mobile push notification
+    DISCORD = 5  # Discord webhook
+    SLACK = 6  # Slack webhook
+
+
+class AckStatus(IntEnum):
+    """Event acknowledgment status."""
+
+    DELIVERED = 0  # Successfully delivered to at least one channel
+    QUEUED = 1  # Accepted and queued for delivery
+    ACK_ERROR = 2  # Permanent failure - all channels failed
+
+
+# =============================================================================
+# Message Types
+# =============================================================================
+
+
+@dataclass
+class ChannelPreference:
+    """
+    Channel delivery preference.
+
+    Args:
+        channel: Which channel to deliver to
+        only_if_active: If True, skip this channel if user is not connected
+    """
+
+    channel: ChannelType
+    only_if_active: bool = False
+
+    def to_dict(self) -> dict:
+        return {
+            "channel": int(self.channel),
+            "only_if_active": self.only_if_active,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ChannelPreference":
+        return cls(
+            channel=ChannelType(data["channel"]),
+            only_if_active=data.get("only_if_active", False),
+        )
+
+
+@dataclass
+class DeliverySpec:
+    """
+    Delivery specification for an event.
+
+    Args:
+        priority: Delivery priority (INFORMATIONAL, NORMAL, CRITICAL)
+        channels: Ordered list of channel preferences to try
+    """
+
+    priority: Priority = Priority.NORMAL
+    channels: List[ChannelPreference] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return {
+            "priority": int(self.priority),
+            "channels": [c.to_dict() for c in self.channels],
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "DeliverySpec":
+        return cls(
+            priority=Priority(data.get("priority", Priority.NORMAL)),
+            channels=[
+                ChannelPreference.from_dict(c) for c in data.get("channels", [])
+            ],
+        )
+
+    # -------------------------------------------------------------------------
+    # Convenience constructors
+    # -------------------------------------------------------------------------
+
+    @staticmethod
+    def informational() -> "DeliverySpec":
+        """
+        Drop if no active session.
+        Use for: indicator updates, chart syncs, strategy logs when watching.
+        """
+        return DeliverySpec(
+            priority=Priority.INFORMATIONAL,
+            channels=[ChannelPreference(ChannelType.ACTIVE_SESSION, only_if_active=True)],
+        )
+
+    @staticmethod
+    def active_or_telegram() -> "DeliverySpec":
+        """
+        Active session preferred, fallback to Telegram.
+        Use for: alerts, position updates.
+        """
+        return DeliverySpec(
+            priority=Priority.NORMAL,
+            channels=[
+                ChannelPreference(ChannelType.ACTIVE_SESSION, only_if_active=True),
+                ChannelPreference(ChannelType.TELEGRAM, only_if_active=False),
+            ],
+        )
+
+    @staticmethod
+    def active_or_push() -> "DeliverySpec":
+        """
+        Active session preferred, fallback to push notification.
+        Use for: alerts, position updates on mobile.
+        """
+        return DeliverySpec(
+            priority=Priority.NORMAL,
+            channels=[
+                ChannelPreference(ChannelType.ACTIVE_SESSION, only_if_active=True),
+                ChannelPreference(ChannelType.PUSH, only_if_active=False),
+            ],
+        )
+
+    @staticmethod
+    def critical() -> "DeliverySpec":
+        """
+        Must deliver through any available channel.
+        Use for: order fills, liquidations, critical errors.
+        """
+        return DeliverySpec(
+            priority=Priority.CRITICAL,
+            channels=[
+                ChannelPreference(ChannelType.ACTIVE_SESSION, only_if_active=True),
+                ChannelPreference(ChannelType.TELEGRAM, only_if_active=False),
+                ChannelPreference(ChannelType.PUSH, only_if_active=False),
+                ChannelPreference(ChannelType.EMAIL, only_if_active=False),
+            ],
+        )
+
+    @staticmethod
+    def telegram_only() -> "DeliverySpec":
+        """
+        Deliver only to Telegram, regardless of active session.
+        Use for: scheduled reports, digest notifications.
+        """
+        return DeliverySpec(
+            priority=Priority.NORMAL,
+            channels=[
+                ChannelPreference(ChannelType.TELEGRAM, only_if_active=False),
+            ],
+        )
+
+
+@dataclass
+class UserEvent:
+    """
+    Event emitted by user container to gateway.
+
+    Args:
+        user_id: User ID this event belongs to
+        event_id: Unique event ID (UUID) for deduplication and ack
+        timestamp: Unix milliseconds when event was generated
+        event_type: Type of event
+        payload: Event data (will be JSON encoded)
+        delivery: Delivery specification
+    """
+
+    event_type: EventType
+    payload: Any = None
+    delivery: DeliverySpec = field(default_factory=DeliverySpec)
+    user_id: str = ""
+    event_id: str = ""
+    timestamp: int = 0
+
+    def __post_init__(self):
+        if not self.event_id:
+            self.event_id = str(uuid.uuid4())
+        if not self.timestamp:
+            self.timestamp = int(time.time() * 1000)
+
+    def to_dict(self) -> dict:
+        return {
+            "user_id": self.user_id,
+            "event_id": self.event_id,
+            "timestamp": self.timestamp,
+            "event_type": int(self.event_type),
+            "payload": self.payload,
+            "delivery": self.delivery.to_dict(),
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "UserEvent":
+        return cls(
+            user_id=data.get("user_id", ""),
+            event_id=data.get("event_id", ""),
+            timestamp=data.get("timestamp", 0),
+            event_type=EventType(data["event_type"]),
+            payload=data.get("payload"),
+            delivery=DeliverySpec.from_dict(data.get("delivery", {})),
+        )
+
+
+@dataclass
+class EventAck:
+    """
+    Acknowledgment from gateway for a critical event.
+
+    Args:
+        event_id: Event ID being acknowledged
+        status: Delivery status
+        error_message: Error message if status is ERROR
+        delivered_via: Which channel successfully delivered (optional)
+    """
+
+    event_id: str
+    status: AckStatus
+    error_message: str = ""
+    delivered_via: Optional[ChannelType] = None
+
+    def to_dict(self) -> dict:
+        return {
+            "event_id": self.event_id,
+            "status": int(self.status),
+            "error_message": self.error_message,
+            "delivered_via": int(self.delivered_via) if self.delivered_via is not None else None,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "EventAck":
+        delivered_via = data.get("delivered_via")
+        return cls(
+            event_id=data["event_id"],
+            status=AckStatus(data["status"]),
+            error_message=data.get("error_message", ""),
+            delivered_via=ChannelType(delivered_via) if delivered_via is not None else None,
+        )
+
+
+# =============================================================================
+# Serialization
+# =============================================================================
+
+
+def serialize_user_event(event: UserEvent) -> bytes:
+    """
+    Serialize UserEvent to wire format.
+    Format: [1 byte msg type][JSON payload]
+
+    Note: In production, replace with proper protobuf serialization.
+    """
+    json_bytes = json.dumps(event.to_dict()).encode("utf-8")
+    return bytes([MSG_TYPE_USER_EVENT]) + json_bytes
+
+
+def deserialize_user_event(data: bytes) -> UserEvent:
+    """Deserialize UserEvent from wire format."""
+    if len(data) < 2:
+        raise ValueError("Data too short")
+
+    msg_type = data[0]
+    if msg_type != MSG_TYPE_USER_EVENT:
+        raise ValueError(f"Invalid message type: expected {MSG_TYPE_USER_EVENT}, got {msg_type}")
+
+    json_str = data[1:].decode("utf-8")
+    return UserEvent.from_dict(json.loads(json_str))
+
+
+def serialize_event_ack(ack: EventAck) -> bytes:
+    """Serialize EventAck to wire format."""
+    json_bytes = json.dumps(ack.to_dict()).encode("utf-8")
+    return bytes([MSG_TYPE_EVENT_ACK]) + json_bytes
+
+
+def deserialize_event_ack(data: bytes) -> EventAck:
+    """Deserialize EventAck from wire format."""
+    if len(data) < 2:
+        raise ValueError("Data too short")
+
+    msg_type = data[0]
+    if msg_type != MSG_TYPE_EVENT_ACK:
+        raise ValueError(f"Invalid message type: expected {MSG_TYPE_EVENT_ACK}, got {msg_type}")
+
+    json_str = data[1:].decode("utf-8")
+    return EventAck.from_dict(json.loads(json_str))
+
+
+# =============================================================================
+# Helpers
+# =============================================================================
+
+
+def get_event_type_name(event_type: EventType) -> str:
+    """Get human-readable event type name."""
+    return event_type.name
+
+
+def get_channel_type_name(channel_type: ChannelType) -> str:
+    """Get human-readable channel type name."""
+    return channel_type.name

sandbox/dexorder/history_client.py

@@ -0,0 +1,322 @@
+"""
+HistoryClient - Submit historical data requests via relay and wait for notifications
+
+RACE CONDITION PREVENTION:
+The client must subscribe to notification topics BEFORE submitting requests.
+Notification topics are deterministic: RESPONSE:{client_id} or HISTORY_READY:{request_id}
+Since both are client-generated, we can subscribe before sending the request.
+"""
+
+import asyncio
+import uuid
+import zmq
+import zmq.asyncio
+from typing import Optional
+import struct
+import sys
+import os
+
+# Import protobuf messages from generated package
+try:
+    from dexorder.generated.ingestor_pb2 import SubmitHistoricalRequest, SubmitResponse, HistoryReadyNotification
+except ImportError:
+    print("Warning: Protobuf files not found. Run: protoc --python_out=dexorder/generated --proto_path=protobuf protobuf/*.proto")
+    raise
+
+
+class HistoryClient:
+    """
+    Client for submitting historical data requests via relay.
+
+    IMPORTANT: Call connect() before making any requests. This ensures the notification
+    listener is running and subscribed BEFORE any requests are submitted, preventing
+    the race condition where notifications arrive before subscription.
+
+    Provides:
+    - Submit historical OHLC data requests
+    - Wait for completion notifications
+    - Handle request timeouts and errors
+    """
+
+    def __init__(self, relay_endpoint: str, notification_endpoint: str, client_id: Optional[str] = None):
+        """
+        Initialize history client.
+
+        Args:
+            relay_endpoint: ZMQ endpoint for relay client requests (e.g., "tcp://relay:5559")
+            notification_endpoint: ZMQ endpoint for notifications (e.g., "tcp://relay:5558")
+            client_id: Optional client ID for notification routing. If not provided, generates one.
+                      All notifications for this client will be sent to topic RESPONSE:{client_id}
+        """
+        self.relay_endpoint = relay_endpoint
+        self.notification_endpoint = notification_endpoint
+        self.client_id = client_id or f"client-{uuid.uuid4().hex[:8]}"
+        self.context: Optional[zmq.asyncio.Context] = None  # created in connect()
+        self.pending_requests = {}  # request_id -> asyncio.Event
+        self.notification_task = None
+        self.connected = False
+
+    async def connect(self):
+        """
+        Connect to relay and start notification listener.
+
+        CRITICAL: This MUST be called before making any requests to prevent race condition.
+        The notification listener subscribes to the deterministic topic RESPONSE:{client_id}
+        BEFORE any requests are sent, ensuring we never miss notifications.
+
+        Safe to call multiple times — handles reconnection after event loop resets
+        (e.g., between successive asyncio.run() calls in research scripts).
+        """
+        if self.connected:
+            return
+
+        # Clean up stale resources from a previous event loop
+        if self.notification_task is not None:
+            if not self.notification_task.done():
+                self.notification_task.cancel()
+                try:
+                    await self.notification_task
+                except asyncio.CancelledError:
+                    pass
+            self.notification_task = None
+
+        # Create a fresh ZMQ context for the current event loop.
+        # zmq.asyncio sockets are bound to the loop they're created in,
+        # so we must not reuse a context from a previous (now-dead) loop.
+        if self.context is not None:
+            self.context.term()
+        self.context = zmq.asyncio.Context()
+
+        # Start notification listener FIRST
+        self.notification_task = asyncio.create_task(self._notification_listener())
+
+        # Give the listener a moment to connect and subscribe
+        await asyncio.sleep(0.1)
+
+        self.connected = True
+
+    async def request_historical_ohlc(
+        self,
+        ticker: str,
+        period_seconds: int,
+        start_time: int,
+        end_time: int,
+        timeout: float = 30.0,
+        limit: Optional[int] = None
+    ) -> dict:
+        """
+        Request historical OHLC data and wait for completion notification.
+
+        IMPORTANT: Call connect() before using this method.
+
+        Args:
+            ticker: Market identifier (e.g., "BINANCE:BTC/USDT")
+            period_seconds: OHLC period in seconds
+            start_time: Start timestamp in microseconds
+            end_time: End timestamp in microseconds
+            timeout: Request timeout in seconds (default: 30)
+            limit: Optional limit on number of candles
+
+        Returns:
+            dict with keys:
+                - request_id: The request ID
+                - status: 'OK', 'NOT_FOUND', or 'ERROR'
+                - error_message: Error message if status is 'ERROR'
+                - iceberg_namespace: Iceberg namespace (if status is 'OK')
+                - iceberg_table: Iceberg table name (if status is 'OK')
+                - row_count: Number of rows written (if status is 'OK')
+
+        Raises:
+            TimeoutError: If request times out
+            ConnectionError: If unable to connect to relay or not connected
+        """
+        # Auto-reconnect if the notification listener task died (e.g., a prior asyncio.run()
+        # created a new event loop, cancelling the background task from the previous loop).
+        if self.connected and self.notification_task is not None and self.notification_task.done():
+            self.connected = False
+
+        if not self.connected:
+            await self.connect()
+
+        request_id = str(uuid.uuid4())
+
+        # Register the pending request BEFORE sending to eliminate any race condition.
+        # The notification topic is deterministic (RESPONSE:{client_id}) and the listener
+        # is already subscribed, so we just need pending_requests populated before Flink
+        # could possibly publish the notification.
+        event = asyncio.Event()
+        self.pending_requests[request_id] = {
+            'event': event,
+            'result': None
+        }
+
+        try:
+            # Create protobuf request with client_id for notification routing
+            request = SubmitHistoricalRequest(
+                request_id=request_id,
+                ticker=ticker,
+                period_seconds=period_seconds,
+                start_time=start_time,
+                end_time=end_time,
+                client_id=self.client_id  # CRITICAL: Enables deterministic notification topic
+            )
+
+            if limit is not None:
+                request.limit = limit
+
+            # Encode with ZMQ envelope: version (1 byte) + message type (1 byte) + protobuf payload
+            MESSAGE_TYPE_SUBMIT_HISTORICAL = 0x10
+            version_frame = struct.pack('B', 0x01)
+            message_frame = struct.pack('B', MESSAGE_TYPE_SUBMIT_HISTORICAL) + request.SerializeToString()
+
+            # Send request to relay
+            socket = self.context.socket(zmq.REQ)
+            socket.connect(self.relay_endpoint)
+
+            try:
+                # Send two frames: version, then message
+                await socket.send(version_frame, zmq.SNDMORE)
+                await socket.send(message_frame)
+
+                # Wait for immediate response
+                response_frames = []
+                while True:
+                    frame = await asyncio.wait_for(socket.recv(), timeout=5.0)
+                    response_frames.append(frame)
+                    if not socket.get(zmq.RCVMORE):
+                        break
+
+                # Parse response (expect 2 frames: version, message)
+                if len(response_frames) < 2:
+                    raise ConnectionError(f"Expected 2 frames, got {len(response_frames)}")
+
+                msg_type = response_frames[1][0]
+                response_payload = response_frames[1][1:]
+
+                response = SubmitResponse()
+                response.ParseFromString(response_payload)
+
+                if response.status != 0:
+                    raise ConnectionError(f"Request failed: {response.error_message}")
+
+            finally:
+                socket.close()
+
+            # Wait for Flink notification with timeout
+            try:
+                await asyncio.wait_for(event.wait(), timeout=timeout)
+                result = self.pending_requests[request_id]['result']
+                return result
+            except asyncio.TimeoutError:
+                raise TimeoutError(f"Request {request_id} timed out after {timeout}s")
+
+        finally:
+            self.pending_requests.pop(request_id, None)
+
+    async def _notification_listener(self):
+        """
+        Internal notification listener that subscribes to RESPONSE:{client_id} topic.
+
+        CRITICAL: This runs BEFORE any requests are submitted to prevent race condition.
+        The notification topic is deterministic based on our client_id.
+        """
+        socket = self.context.socket(zmq.SUB)
+        socket.connect(self.notification_endpoint)
+
+        # Subscribe to our client-specific topic
+        # CRITICAL: This topic is deterministic (RESPONSE:{client_id}) and we know it
+        # before sending any requests, so we can subscribe first to prevent race condition
+        notification_topic = f"RESPONSE:{self.client_id}"
+        socket.setsockopt_string(zmq.SUBSCRIBE, notification_topic)
+
+        try:
+            while True:
+                # Receive multi-frame message: [topic][version][message]
+                frames = []
+                while True:
+                    frame = await socket.recv()
+                    frames.append(frame)
+                    if not socket.get(zmq.RCVMORE):
+                        break
+
+                # Parse frames
+                if len(frames) < 3:
+                    continue
+
+                topic_frame = frames[0]
+                version_frame = frames[1]
+                message_frame = frames[2]
+
+                # Validate version
+                if len(version_frame) != 1 or version_frame[0] != 0x01:
+                    continue
+
+                # Validate message type
+                if len(message_frame) < 1:
+                    continue
+
+                msg_type = message_frame[0]
+                payload = message_frame[1:]
+
+                MESSAGE_TYPE_HISTORY_READY = 0x12
+                if msg_type != MESSAGE_TYPE_HISTORY_READY:
+                    continue
+
+                # Parse notification (protobuf)
+                try:
+                    notification = HistoryReadyNotification()
+                    notification.ParseFromString(payload)
+                except Exception as e:
+                    print(f"Warning: failed to parse notification payload: {e}")
+                    continue
+
+                request_id = notification.request_id
+
+                # Check if we're waiting for this request
+                if request_id in self.pending_requests:
+                    # Map protobuf enum to string status
+                    # NotificationStatus: OK=0, NOT_FOUND=1, ERROR=2, TIMEOUT=3
+                    status_map = {0: 'OK', 1: 'NOT_FOUND', 2: 'ERROR', 3: 'TIMEOUT'}
+                    status = status_map.get(notification.status, 'ERROR')
+
+                    result = {
+                        'request_id': request_id,
+                        'status': status,
+                        'error_message': notification.error_message if notification.error_message else None
+                    }
+
+                    # Add Iceberg details if available
+                    if status == 'OK':
+                        result.update({
+                            'iceberg_namespace': notification.iceberg_namespace,
+                            'iceberg_table': notification.iceberg_table,
+                            'row_count': notification.row_count,
+                            'ticker': notification.ticker,
+                            'period_seconds': notification.period_seconds,
+                            'start_time': notification.start_time,
+                            'end_time': notification.end_time,
+                        })
+
+                    self.pending_requests[request_id]['result'] = result
+                    self.pending_requests[request_id]['event'].set()
+
+        except asyncio.CancelledError:
+            pass
+        finally:
+            socket.close()
+
+    async def close(self):
+        """
+        Close the client and cleanup resources.
+        """
+        if self.notification_task:
+            self.notification_task.cancel()
+            try:
+                await self.notification_task
+            except asyncio.CancelledError:
+                pass
+
+        if self.context is not None:
+            self.context.term()
+            self.context = None
+        self.connected = False

sandbox/dexorder/iceberg_client.py

@@ -0,0 +1,236 @@
+"""
+IcebergClient - Query OHLC data from Iceberg warehouse (Iceberg 1.10.1)
+"""
+
+from typing import Optional, List, Tuple
+import pandas as pd
+import logging
+from pyiceberg.catalog import load_catalog
+from pyiceberg.expressions import (
+    And,
+    EqualTo,
+    GreaterThanOrEqual,
+    LessThanOrEqual
+)
+
+log = logging.getLogger(__name__)
+
+
+class IcebergClient:
+    """
+    Client for querying OHLC data from Iceberg warehouse (Iceberg 1.10.1).
+
+    Note: Iceberg 1.x does not enforce primary keys at the table level.
+    Deduplication is handled by:
+    - Flink upsert mode with equality delete files
+    - PyIceberg automatically filters deleted rows during queries
+    - Last-write-wins semantics for duplicates
+
+    Provides:
+    - Query OHLC data by ticker, period, and time range
+    - Identify missing data gaps
+    - Efficient partition pruning for large datasets
+    """
+
+    def __init__(
+        self,
+        catalog_uri: str,
+        namespace: str = "trading",
+        s3_endpoint: Optional[str] = None,
+        s3_access_key: Optional[str] = None,
+        s3_secret_key: Optional[str] = None,
+        metadata_client=None,  # SymbolMetadataClient (avoid circular import)
+    ):
+        """
+        Initialize Iceberg client.
+
+        Args:
+            catalog_uri: URI of the Iceberg catalog (e.g., "http://iceberg-catalog:8181")
+            namespace: Iceberg namespace (default: "trading")
+            s3_endpoint: S3/MinIO endpoint URL (e.g., "http://localhost:9000")
+            s3_access_key: S3/MinIO access key
+            s3_secret_key: S3/MinIO secret key
+            metadata_client: SymbolMetadataClient for price/volume conversion
+        """
+        self.catalog_uri = catalog_uri
+        self.namespace = namespace
+        self.metadata_client = metadata_client
+
+        catalog_props = {"uri": catalog_uri}
+        if s3_endpoint:
+            catalog_props["s3.endpoint"] = s3_endpoint
+            catalog_props["s3.path-style-access"] = "true"
+        if s3_access_key:
+            catalog_props["s3.access-key-id"] = s3_access_key
+        if s3_secret_key:
+            catalog_props["s3.secret-access-key"] = s3_secret_key
+
+        self.catalog = load_catalog("trading", **catalog_props)
+        self.table = self.catalog.load_table(f"{namespace}.ohlc")
+
+    def query_ohlc(
+        self,
+        ticker: str,
+        period_seconds: int,
+        start_time: int,
+        end_time: int,
+        columns: Optional[List[str]] = None
+    ) -> pd.DataFrame:
+        """
+        Query OHLC data for a specific ticker, period, and time range.
+
+        Args:
+            ticker: Market identifier (e.g., "BINANCE:BTC/USDT")
+            period_seconds: OHLC period in seconds (60, 300, 3600, etc.)
+            start_time: Start timestamp in microseconds
+            end_time: End timestamp in microseconds
+            columns: Optional list of columns to select. If None, returns all columns.
+                    Example: ["timestamp", "open", "high", "low", "close", "volume"]
+
+        Returns:
+            DataFrame with OHLC data sorted by timestamp
+        """
+        # Reload table metadata to pick up snapshots committed after this client was initialized
+        self.table = self.catalog.load_table(f"{self.namespace}.ohlc")
+
+        scan = self.table.scan(
+            row_filter=And(
+                EqualTo("ticker", ticker),
+                EqualTo("period_seconds", period_seconds),
+                GreaterThanOrEqual("timestamp", start_time),
+                LessThanOrEqual("timestamp", end_time)
+            )
+        )
+
+        # Select specific columns if requested
+        if columns is not None:
+            scan = scan.select(*columns)
+
+        df = scan.to_pandas()
+
+        if not df.empty:
+            df = df.sort_values("timestamp")
+            # Apply price/volume conversion if metadata client available
+            if self.metadata_client is not None:
+                df = self._apply_denominators(df, ticker)
+
+        return df
+
+    def _apply_denominators(self, df: pd.DataFrame, ticker: str) -> pd.DataFrame:
+        """
+        Convert integer prices and volumes to decimal floats using market metadata.
+
+        Args:
+            df: DataFrame with integer OHLC data
+            ticker: Market identifier for metadata lookup
+
+        Returns:
+            DataFrame with decimal prices and volumes
+
+        Raises:
+            ValueError: If metadata not found for ticker
+        """
+        if df.empty:
+            return df
+
+        # Get metadata for this ticker
+        metadata = self.metadata_client.get_metadata(ticker)
+
+        # Convert price columns (divide by tick_denom)
+        price_columns = ["open", "high", "low", "close"]
+        for col in price_columns:
+            if col in df.columns:
+                df[col] = df[col].astype(float) / metadata.tick_denom
+
+        # Convert volume columns (divide by base_denom)
+        volume_columns = ["volume", "buy_vol", "sell_vol"]
+        for col in volume_columns:
+            if col in df.columns and df[col].notna().any():
+                df[col] = df[col].astype(float) / metadata.base_denom
+
+        log.debug(
+            f"Applied denominators to {ticker}: tick_denom={metadata.tick_denom}, "
+            f"base_denom={metadata.base_denom} ({len(df)} rows)"
+        )
+
+        return df
+
+    def find_missing_ranges(
+        self,
+        ticker: str,
+        period_seconds: int,
+        start_time: int,
+        end_time: int
+    ) -> List[Tuple[int, int]]:
+        """
+        Identify missing data ranges in the requested time period.
+
+        Returns list of (start, end) tuples for missing ranges.
+        Expected candles are calculated based on period_seconds.
+
+        Args:
+            ticker: Market identifier
+            period_seconds: OHLC period in seconds
+            start_time: Start timestamp in microseconds
+            end_time: End timestamp in microseconds
+
+        Returns:
+            List of (start_time, end_time) tuples for missing ranges
+        """
+        df = self.query_ohlc(ticker, period_seconds, start_time, end_time)
+
+        if df.empty:
+            # All data is missing
+            return [(start_time, end_time)]
+
+        # Convert period to microseconds
+        period_micros = period_seconds * 1_000_000
+
+        # Generate expected timestamps
+        expected_timestamps = list(range(start_time, end_time + 1, period_micros))
+        actual_timestamps = set(df['timestamp'].values)
+
+        # Find gaps
+        missing = sorted(set(expected_timestamps) - actual_timestamps)
+
+        if not missing:
+            return []
+
+        # Consolidate consecutive missing timestamps into ranges
+        ranges = []
+        range_start = missing[0]
+        prev_ts = missing[0]
+
+        for ts in missing[1:]:
+            if ts > prev_ts + period_micros:
+                # Gap in missing data - close previous range
+                ranges.append((range_start, prev_ts))
+                range_start = ts
+            prev_ts = ts
+
+        # Close final range
+        ranges.append((range_start, prev_ts))
+
+        return ranges
+
+    def has_data(
+        self,
+        ticker: str,
+        period_seconds: int,
+        start_time: int,
+        end_time: int
+    ) -> bool:
+        """
+        Check if any data exists for the given parameters.
+
+        Args:
+            ticker: Market identifier
+            period_seconds: OHLC period in seconds
+            start_time: Start timestamp in microseconds
+            end_time: End timestamp in microseconds
+
+        Returns:
+            True if at least one candle exists, False otherwise
+        """
+        df = self.query_ohlc(ticker, period_seconds, start_time, end_time)
+        return not df.empty

sandbox/dexorder/impl/__init__.py

@@ -0,0 +1,8 @@
+"""
+Implementation modules for dexorder APIs.
+"""
+
+from .data_api_impl import DataAPIImpl
+from .charting_api_impl import ChartingAPIImpl
+
+__all__ = ['DataAPIImpl', 'ChartingAPIImpl']

sandbox/dexorder/impl/charting_api_impl.py

@@ -0,0 +1,239 @@
+"""
+Implementation of ChartingAPI using mplfinance for professional financial charts.
+"""
+
+import logging
+from typing import Optional, Tuple, List
+import pandas as pd
+import numpy as np
+from matplotlib import pyplot as plt
+from matplotlib.figure import Figure
+import mplfinance as mpf
+
+from dexorder.api.charting_api import ChartingAPI
+
+log = logging.getLogger(__name__)
+
+
+class ChartingAPIImpl(ChartingAPI):
+    """
+    Implementation of ChartingAPI using mplfinance.
+
+    This implementation provides professional-looking financial charts with:
+    - Candlestick plots with various styling options
+    - Easy addition of indicator panels with proper alignment
+    - Consistent theming across all chart types
+    """
+
+    def __init__(self):
+        """Initialize the charting API implementation."""
+        pass
+
+    def plot_ohlc(
+        self,
+        df: pd.DataFrame,
+        title: Optional[str] = None,
+        volume: bool = False,
+        style: str = "charles",
+        figsize: Tuple[int, int] = (12, 8),
+        **kwargs
+    ) -> Tuple[Figure, plt.Axes]:
+        """
+        Create a candlestick chart from OHLC data.
+
+        See ChartingAPI.plot_ohlc for full documentation.
+        """
+        # Prepare the dataframe for mplfinance
+        df_plot = self._prepare_ohlc_dataframe(df)
+
+        # Create the plot
+        fig, axes = mpf.plot(
+            df_plot,
+            type='candle',
+            style=style,
+            title=title,
+            volume=volume,
+            figsize=figsize,
+            returnfig=True,
+            **kwargs
+        )
+
+        # Return the main price axes (first axes is price, second is volume if present)
+        main_ax = axes[0]
+
+        return fig, main_ax
+
+    def add_indicator_panel(
+        self,
+        fig: Figure,
+        df: pd.DataFrame,
+        columns: Optional[List[str]] = None,
+        ylabel: Optional[str] = None,
+        height_ratio: float = 0.3,
+        ylim: Optional[Tuple[float, float]] = None,
+        **kwargs
+    ) -> plt.Axes:
+        """
+        Add a new indicator panel below existing plots with aligned x-axis.
+
+        See ChartingAPI.add_indicator_panel for full documentation.
+        """
+        # Determine which columns to plot
+        if columns is None:
+            # Plot all numeric columns
+            numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+            columns = numeric_cols
+        else:
+            # Validate columns exist
+            missing = set(columns) - set(df.columns)
+            if missing:
+                raise ValueError(f"Columns not found in DataFrame: {missing}")
+
+        # Get existing axes
+        existing_axes = fig.get_axes()
+        if not existing_axes:
+            raise ValueError("Figure has no existing axes. Create a plot first with plot_ohlc().")
+
+        # Calculate new grid layout
+        n_existing = len(existing_axes)
+
+        # Calculate height ratios: existing axes maintain their relative sizes,
+        # new axes gets height_ratio relative to the first (main) axes
+        existing_heights = [ax.get_position().height for ax in existing_axes]
+        main_height = existing_heights[0]
+        new_height = main_height * height_ratio
+
+        # Adjust existing axes positions to make room for new panel
+        total_height = sum(existing_heights) + new_height
+        current_top = 0.98  # Leave small margin at top
+        current_bottom = 0.05  # Leave margin at bottom
+        available_height = current_top - current_bottom
+
+        # Reposition existing axes
+        for i, ax in enumerate(existing_axes):
+            old_pos = ax.get_position()
+            normalized_height = (existing_heights[i] / total_height) * available_height
+            new_top = current_top - (sum(existing_heights[:i]) / total_height) * available_height
+            new_bottom = new_top - normalized_height
+            ax.set_position([old_pos.x0, new_bottom, old_pos.width, normalized_height])
+
+        # Create new axes at the bottom
+        normalized_new_height = (new_height / total_height) * available_height
+        new_bottom = current_bottom
+        new_top = new_bottom + normalized_new_height
+
+        first_ax_pos = existing_axes[0].get_position()
+        new_ax = fig.add_axes([
+            first_ax_pos.x0,
+            new_bottom,
+            first_ax_pos.width,
+            normalized_new_height
+        ])
+
+        # Share x-axis with the first axes for time alignment
+        new_ax.sharex(existing_axes[0])
+
+        # Plot the indicator data
+        for col in columns:
+            if col in df.columns:
+                # Handle potential timestamp index (convert from microseconds)
+                if df.index.name == 'timestamp' or 'timestamp' in str(df.index.dtype):
+                    # Assume microseconds, convert to datetime
+                    plot_index = pd.to_datetime(df.index, unit='us')
+                else:
+                    plot_index = df.index
+
+                new_ax.plot(plot_index, df[col], label=col, **kwargs)
+
+        # Styling
+        if ylabel:
+            new_ax.set_ylabel(ylabel)
+
+        if ylim:
+            new_ax.set_ylim(ylim)
+
+        if len(columns) > 1:
+            new_ax.legend(loc='best')
+
+        new_ax.grid(True, alpha=0.3)
+
+        # Only show x-axis labels on the bottom-most panel
+        for ax in existing_axes:
+            ax.set_xlabel('')
+            plt.setp(ax.get_xticklabels(), visible=False)
+
+        return new_ax
+
+    def create_figure(
+        self,
+        figsize: Tuple[int, int] = (12, 8),
+        style: str = "charles"
+    ) -> Tuple[Figure, plt.Axes]:
+        """
+        Create a styled figure without OHLC data for custom visualizations.
+
+        See ChartingAPI.create_figure for full documentation.
+        """
+        # Get the style parameters from mplfinance
+        mpf_style = mpf.make_mpf_style(base_mpf_style=style)
+
+        # Create figure with the style's colors
+        fig, ax = plt.subplots(figsize=figsize)
+
+        # Apply style colors if available
+        if 'facecolor' in mpf_style:
+            fig.patch.set_facecolor(mpf_style['facecolor'])
+        if 'figcolor' in mpf_style:
+            ax.set_facecolor(mpf_style['figcolor'])
+
+        ax.grid(True, alpha=0.3)
+
+        return fig, ax
+
+    def _prepare_ohlc_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Prepare a DataFrame for mplfinance plotting.
+
+        Ensures the DataFrame has the correct format:
+        - DatetimeIndex
+        - Lowercase column names: open, high, low, close, volume
+
+        Args:
+            df: Input DataFrame with OHLC data
+
+        Returns:
+            DataFrame ready for mplfinance
+        """
+        df_copy = df.copy()
+
+        # Handle timestamp column (in microseconds) -> DatetimeIndex
+        if 'timestamp' in df_copy.columns:
+            df_copy.index = pd.to_datetime(df_copy['timestamp'], unit='us')
+            df_copy = df_copy.drop(columns=['timestamp'])
+        elif df_copy.index.name == 'timestamp' or 'int' in str(df_copy.index.dtype):
+            # Index is timestamp in microseconds
+            df_copy.index = pd.to_datetime(df_copy.index, unit='us')
+
+        # Ensure index is DatetimeIndex
+        if not isinstance(df_copy.index, pd.DatetimeIndex):
+            raise ValueError(
+                "DataFrame must have a DatetimeIndex or a 'timestamp' column in microseconds"
+            )
+
+        # Normalize column names to lowercase
+        df_copy.columns = df_copy.columns.str.lower()
+
+        # Validate required columns
+        required = ['open', 'high', 'low', 'close']
+        missing = set(required) - set(df_copy.columns)
+        if missing:
+            raise ValueError(f"DataFrame missing required OHLC columns: {missing}")
+
+        # Keep only OHLC(V) columns for mplfinance
+        keep_cols = ['open', 'high', 'low', 'close']
+        if 'volume' in df_copy.columns:
+            keep_cols.append('volume')
+
+        df_copy = df_copy[keep_cols]
+
+        return df_copy

sandbox/dexorder/impl/data_api_impl.py

@@ -0,0 +1,169 @@
+"""
+Implementation of DataAPI using OHLCClient for smart caching.
+"""
+
+import logging
+from typing import Optional, List
+import pandas as pd
+
+from dexorder.api.data_api import DataAPI
+from dexorder.ohlc_client import OHLCClient
+from dexorder.utils import TimestampInput, to_microseconds
+
+log = logging.getLogger(__name__)
+
+# Standard OHLC columns always returned
+STANDARD_COLUMNS = ["timestamp", "open", "high", "low", "close"]
+
+# All valid extra columns available in the Iceberg schema
+VALID_EXTRA_COLUMNS = {
+    "volume", "buy_vol", "sell_vol",
+    "open_time", "high_time", "low_time", "close_time",
+    "open_interest",
+    "ticker", "period_seconds"
+}
+
+
+class DataAPIImpl(DataAPI):
+    """
+    Implementation of DataAPI using OHLCClient for querying OHLC data.
+
+    This implementation provides:
+    - Smart caching via Iceberg (checks cache first, fetches missing data on-demand)
+    - Selective column queries to minimize data transfer
+    - Integration with the historical data pipeline via relay
+    """
+
+    def __init__(
+        self,
+        iceberg_catalog_uri: str,
+        relay_endpoint: str,
+        notification_endpoint: str,
+        namespace: str = "trading",
+        s3_endpoint: Optional[str] = None,
+        s3_access_key: Optional[str] = None,
+        s3_secret_key: Optional[str] = None,
+        request_timeout: float = 30.0,
+    ):
+        """
+        Initialize DataAPI implementation.
+
+        Args:
+            iceberg_catalog_uri: URI of Iceberg REST catalog (e.g., "http://iceberg-catalog:8181")
+            relay_endpoint: ZMQ endpoint for relay requests (e.g., "tcp://relay:5559")
+            notification_endpoint: ZMQ endpoint for notifications (e.g., "tcp://relay:5558")
+            namespace: Iceberg namespace (default: "trading")
+            s3_endpoint: S3/MinIO endpoint URL (e.g., "http://minio:9000")
+            s3_access_key: S3/MinIO access key
+            s3_secret_key: S3/MinIO secret key
+            request_timeout: Default timeout for historical data requests in seconds (default: 30)
+        """
+        self.ohlc_client = OHLCClient(
+            iceberg_catalog_uri=iceberg_catalog_uri,
+            relay_endpoint=relay_endpoint,
+            notification_endpoint=notification_endpoint,
+            namespace=namespace,
+            s3_endpoint=s3_endpoint,
+            s3_access_key=s3_access_key,
+            s3_secret_key=s3_secret_key,
+        )
+        self.request_timeout = request_timeout
+        self._started = False
+
+    async def start(self):
+        """
+        Start the DataAPI client.
+
+        Must be called before making any queries. Initializes the background
+        notification listener for historical data requests.
+        """
+        if not self._started:
+            await self.ohlc_client.start()
+            self._started = True
+
+    async def stop(self):
+        """
+        Stop the DataAPI client and cleanup resources.
+        """
+        if self._started:
+            await self.ohlc_client.stop()
+            self._started = False
+
+    async def historical_ohlc(
+        self,
+        ticker: str,
+        period_seconds: int,
+        start_time: TimestampInput,
+        end_time: TimestampInput,
+        extra_columns: Optional[List[str]] = None,
+    ) -> pd.DataFrame:
+        """
+        Query historical OHLC data with smart caching.
+
+        See DataAPI.historical_ohlc for full documentation.
+        """
+        if not self._started:
+            await self.start()
+
+        # Convert timestamps to microseconds
+        start_micros = to_microseconds(start_time)
+        end_micros = to_microseconds(end_time)
+
+        log.debug(f"Fetching OHLC: {ticker}, period={period_seconds}s, "
+                  f"start={start_time} ({start_micros}), end={end_time} ({end_micros})")
+
+        # Validate extra_columns
+        if extra_columns:
+            invalid = set(extra_columns) - VALID_EXTRA_COLUMNS
+            if invalid:
+                raise ValueError(f"Invalid extra columns: {invalid}. Valid options: {VALID_EXTRA_COLUMNS}")
+
+        # Determine which columns to fetch
+        columns_to_fetch = STANDARD_COLUMNS.copy()
+        if extra_columns:
+            columns_to_fetch.extend(extra_columns)
+
+        # Use OHLCClient which handles smart caching:
+        # 1. Check Iceberg for existing data
+        # 2. Request missing data via relay if needed
+        # 3. Wait for notification
+        # 4. Return complete dataset
+        df = await self.ohlc_client.fetch_ohlc(
+            ticker=ticker,
+            period_seconds=period_seconds,
+            start_time=start_micros,
+            end_time=end_micros,
+            request_timeout=self.request_timeout
+        )
+
+        # Select only requested columns (filter out metadata and unrequested fields)
+        if not df.empty:
+            available_cols = [col for col in columns_to_fetch if col in df.columns]
+            df = df[available_cols]
+
+        return df
+
+    async def latest_ohlc(
+        self,
+        ticker: str,
+        period_seconds: int,
+        length: int = 1,
+        extra_columns: Optional[List[str]] = None,
+    ) -> pd.DataFrame:
+        """
+        Query the most recent OHLC candles.
+
+        See DataAPI.latest_ohlc for full documentation.
+
+        Note: This method is not yet implemented.
+        """
+        raise NotImplementedError("latest_ohlc will be implemented in the future")
+
+    async def __aenter__(self):
+        """Support async context manager."""
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Support async context manager."""
+        await self.stop()

sandbox/dexorder/lifecycle_manager.py

@@ -0,0 +1,240 @@
+"""
+Container lifecycle manager for agent containers.
+
+Tracks activity and triggers to determine when the container should shut down.
+Exits with code 42 to signal clean idle shutdown to the lifecycle sidecar.
+"""
+
+import asyncio
+import logging
+import os
+import signal
+import sys
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional, Set
+
+logger = logging.getLogger(__name__)
+
+# Exit code to signal clean idle shutdown to sidecar
+EXIT_CODE_IDLE_SHUTDOWN = 42
+
+# File to write exit code for sidecar to read
+EXIT_CODE_FILE = Path("/var/run/agent/exit_code")
+
+
+class LifecycleManager:
+    """
+    Manages container lifecycle based on activity and triggers.
+    
+    The container shuts itself down when:
+    1. No active triggers (data subscriptions, CEP patterns, etc.)
+    2. No recent user activity (MCP calls)
+    3. Idle timeout has elapsed
+    """
+
+    def __init__(
+        self,
+        idle_timeout_minutes: int = 15,
+        check_interval_seconds: int = 60,
+        enable_shutdown: bool = True,
+    ):
+        """
+        Initialize lifecycle manager.
+
+        Args:
+            idle_timeout_minutes: Minutes of inactivity before shutdown
+            check_interval_seconds: Interval between idle checks
+            enable_shutdown: If False, only log idle state without exiting (for testing)
+        """
+        self.idle_timeout = timedelta(minutes=idle_timeout_minutes)
+        self.check_interval = check_interval_seconds
+        self.enable_shutdown = enable_shutdown
+
+        self.last_activity: datetime = datetime.now()
+        self.active_triggers: Set[str] = set()
+        self._running = False
+        self._check_task: Optional[asyncio.Task] = None
+
+        logger.info(
+            "Lifecycle manager initialized: idle_timeout=%dm, check_interval=%ds, shutdown_enabled=%s",
+            idle_timeout_minutes,
+            check_interval_seconds,
+            enable_shutdown,
+        )
+
+    def record_activity(self) -> None:
+        """
+        Record user activity (called on MCP tool/resource/prompt invocations).
+        Resets the idle timer.
+        """
+        self.last_activity = datetime.now()
+        logger.debug("Activity recorded, idle timer reset")
+
+    def update_triggers(self, triggers: Set[str]) -> None:
+        """
+        Update the set of active triggers.
+
+        Args:
+            triggers: Set of active trigger IDs (data subscriptions, CEP patterns, etc.)
+        """
+        if triggers != self.active_triggers:
+            added = triggers - self.active_triggers
+            removed = self.active_triggers - triggers
+
+            if added:
+                logger.info("Triggers added: %s", added)
+            if removed:
+                logger.info("Triggers removed: %s", removed)
+
+            self.active_triggers = triggers
+            logger.info("Active triggers: %d", len(self.active_triggers))
+
+    def add_trigger(self, trigger_id: str) -> None:
+        """Add a single trigger."""
+        if trigger_id not in self.active_triggers:
+            self.active_triggers.add(trigger_id)
+            logger.info("Trigger added: %s (total: %d)", trigger_id, len(self.active_triggers))
+
+    def remove_trigger(self, trigger_id: str) -> None:
+        """Remove a single trigger."""
+        if trigger_id in self.active_triggers:
+            self.active_triggers.remove(trigger_id)
+            logger.info("Trigger removed: %s (total: %d)", trigger_id, len(self.active_triggers))
+
+    def is_idle(self) -> bool:
+        """
+        Check if container is idle and should shut down.
+
+        Returns:
+            True if no triggers and idle timeout exceeded
+        """
+        has_triggers = len(self.active_triggers) > 0
+        idle_time = datetime.now() - self.last_activity
+        is_past_timeout = idle_time > self.idle_timeout
+
+        if has_triggers:
+            logger.debug("Not idle: has %d active triggers", len(self.active_triggers))
+            return False
+
+        if not is_past_timeout:
+            logger.debug(
+                "Not idle: last activity %s ago (timeout: %s)",
+                idle_time,
+                self.idle_timeout,
+            )
+            return False
+
+        logger.info(
+            "Container is idle: no triggers and %s since last activity", idle_time
+        )
+        return True
+
+    async def start(self) -> None:
+        """Start the lifecycle manager background task."""
+        if self._running:
+            logger.warning("Lifecycle manager already running")
+            return
+
+        self._running = True
+        self._check_task = asyncio.create_task(self._check_loop())
+        logger.info("Lifecycle manager started")
+
+    async def stop(self) -> None:
+        """Stop the lifecycle manager."""
+        self._running = False
+        if self._check_task:
+            self._check_task.cancel()
+            try:
+                await self._check_task
+            except asyncio.CancelledError:
+                pass
+        logger.info("Lifecycle manager stopped")
+
+    async def _check_loop(self) -> None:
+        """Background task that periodically checks if container should shut down."""
+        while self._running:
+            try:
+                await asyncio.sleep(self.check_interval)
+
+                if self.is_idle():
+                    if self.enable_shutdown:
+                        logger.info("Initiating idle shutdown (exit code %d)", EXIT_CODE_IDLE_SHUTDOWN)
+                        self._write_exit_code(EXIT_CODE_IDLE_SHUTDOWN)
+                        
+                        # Give sidecar a moment to see the exit code file
+                        await asyncio.sleep(1)
+                        
+                        # Exit with special code
+                        os._exit(EXIT_CODE_IDLE_SHUTDOWN)
+                    else:
+                        logger.info(
+                            "Container is idle but shutdown is disabled (testing mode)"
+                        )
+
+            except asyncio.CancelledError:
+                logger.info("Check loop cancelled")
+                raise
+            except Exception as e:
+                logger.error("Error in lifecycle check loop: %s", e, exc_info=True)
+
+    def _write_exit_code(self, code: int) -> None:
+        """Write exit code to shared file for sidecar to read."""
+        try:
+            EXIT_CODE_FILE.parent.mkdir(parents=True, exist_ok=True)
+            EXIT_CODE_FILE.write_text(str(code))
+            logger.debug("Wrote exit code %d to %s", code, EXIT_CODE_FILE)
+        except Exception as e:
+            logger.warning("Failed to write exit code file: %s", e)
+
+    def setup_signal_handlers(self) -> None:
+        """
+        Setup signal handlers for graceful shutdown.
+        On SIGTERM/SIGINT, exit normally (not with code 42) to allow restart.
+        """
+
+        def signal_handler(signum, frame):
+            logger.info("Received signal %d, exiting normally", signum)
+            sys.exit(0)
+
+        signal.signal(signal.SIGTERM, signal_handler)
+        signal.signal(signal.SIGINT, signal_handler)
+
+
+# Singleton instance for easy access across the application
+_lifecycle_manager: Optional[LifecycleManager] = None
+
+
+def get_lifecycle_manager(
+    idle_timeout_minutes: Optional[int] = None,
+    enable_shutdown: Optional[bool] = None,
+) -> LifecycleManager:
+    """Get or create the global lifecycle manager instance."""
+    global _lifecycle_manager
+    if _lifecycle_manager is None:
+        # Load configuration from environment or use provided values
+        idle_timeout = idle_timeout_minutes if idle_timeout_minutes is not None else int(os.environ.get("IDLE_TIMEOUT_MINUTES", "15"))
+        check_interval = int(os.environ.get("IDLE_CHECK_INTERVAL_SECONDS", "60"))
+        enable = enable_shutdown if enable_shutdown is not None else os.environ.get("ENABLE_IDLE_SHUTDOWN", "true").lower() == "true"
+
+        _lifecycle_manager = LifecycleManager(
+            idle_timeout_minutes=idle_timeout,
+            check_interval_seconds=check_interval,
+            enable_shutdown=enable,
+        )
+    return _lifecycle_manager
+
+
+async def start_lifecycle_manager(
+    user_id: Optional[str] = None,
+    idle_timeout_minutes: Optional[int] = None,
+    enable_idle_shutdown: Optional[bool] = None,
+) -> LifecycleManager:
+    """Initialize and start the lifecycle manager."""
+    manager = get_lifecycle_manager(
+        idle_timeout_minutes=idle_timeout_minutes,
+        enable_shutdown=enable_idle_shutdown,
+    )
+    manager.setup_signal_handlers()
+    await manager.start()
+    return manager

sandbox/dexorder/mcp_auth_middleware.py

@@ -0,0 +1,43 @@
+# openclaw/auth.py
+
+class MCPAuthMiddleware:
+    """Authenticates incoming MCP connections based on configured mode."""
+
+    def __init__(self, config: AuthConfig):
+        self.config = config
+        self._jwks_client = None  # lazy-loaded for platform mode
+
+    async def authenticate(self, request) -> AuthContext:
+        match self.config.mode:
+            case "local":
+                # stdio transport or localhost-only binding
+                # No auth needed — if you can exec into the container,
+                # you're the user
+                return AuthContext(user_id=self.config.local_user_id,
+                                  source="local")
+
+            case "token":
+                # User-generated API key (standalone remote access)
+                token = extract_bearer_token(request)
+                if not verify_token_hash(token, self.config.tokens):
+                    raise AuthError("Invalid API token")
+                return AuthContext(user_id=self.config.local_user_id,
+                                  source="api_key")
+
+            case "platform":
+                # JWT signed by the OpenClaw platform
+                token = extract_bearer_token(request)
+                claims = await self._verify_platform_jwt(token)
+                if claims["sub"] != self.config.expected_user_id:
+                    raise AuthError("User ID mismatch")
+                return AuthContext(user_id=claims["sub"],
+                                  source="platform",
+                                  scopes=claims.get("scopes", []))
+
+    async def _verify_platform_jwt(self, token: str) -> dict:
+        if not self._jwks_client:
+            self._jwks_client = JWKSClient(self.config.platform_jwks_url)
+        signing_key = await self._jwks_client.get_signing_key_from_jwt(token)
+        return jwt.decode(token, signing_key.key,
+                          algorithms=["RS256"],
+                          audience="openclaw-mcp")

sandbox/dexorder/ohlc_client.py

@@ -0,0 +1,159 @@
+"""
+OHLCClient - High-level API for fetching OHLC data with smart caching
+"""
+
+import asyncio
+import pandas as pd
+import logging
+from typing import Optional
+from .iceberg_client import IcebergClient
+from .history_client import HistoryClient
+from .symbol_metadata_client import SymbolMetadataClient
+
+log = logging.getLogger(__name__)
+
+
+class OHLCClient:
+    """
+    High-level client for fetching OHLC data.
+
+    Workflow:
+    1. Check Iceberg for existing data
+    2. Identify missing ranges
+    3. Request missing data via relay
+    4. Wait for notification
+    5. Query Iceberg for complete dataset
+    6. Return combined results
+
+    This provides transparent caching - clients don't need to know
+    whether data came from cache or was fetched on-demand.
+    """
+
+    def __init__(
+        self,
+        iceberg_catalog_uri: str,
+        relay_endpoint: str,
+        notification_endpoint: str,
+        namespace: str = "trading",
+        s3_endpoint: str = None,
+        s3_access_key: str = None,
+        s3_secret_key: str = None,
+    ):
+        """
+        Initialize OHLC client.
+
+        Args:
+            iceberg_catalog_uri: URI of Iceberg catalog
+            relay_endpoint: ZMQ endpoint for relay requests
+            notification_endpoint: ZMQ endpoint for notifications
+            namespace: Iceberg namespace (default: "trading")
+            s3_endpoint: S3/MinIO endpoint URL (e.g., "http://localhost:9000")
+            s3_access_key: S3/MinIO access key
+            s3_secret_key: S3/MinIO secret key
+        """
+        # Initialize symbol metadata client for price/volume conversion
+        self.metadata = SymbolMetadataClient(
+            iceberg_catalog_uri,
+            namespace=namespace,
+            s3_endpoint=s3_endpoint,
+            s3_access_key=s3_access_key,
+            s3_secret_key=s3_secret_key,
+        )
+
+        # Initialize Iceberg client with metadata client for automatic conversion
+        self.iceberg = IcebergClient(
+            iceberg_catalog_uri, namespace,
+            s3_endpoint=s3_endpoint,
+            s3_access_key=s3_access_key,
+            s3_secret_key=s3_secret_key,
+            metadata_client=self.metadata,
+        )
+
+        self.history = HistoryClient(relay_endpoint, notification_endpoint)
+        log.info("OHLCClient initialized with automatic price/volume conversion")
+
+    async def start(self):
+        """
+        Start the client. Must be called before making requests.
+        Starts background notification listener.
+        """
+        await self.history.connect()
+
+    async def stop(self):
+        """
+        Stop the client and cleanup resources.
+        """
+        await self.history.close()
+
+    async def fetch_ohlc(
+        self,
+        ticker: str,
+        period_seconds: int,
+        start_time: int,
+        end_time: int,
+        request_timeout: float = 30.0
+    ) -> pd.DataFrame:
+        """
+        Fetch OHLC data with smart caching.
+
+        Steps:
+        1. Query Iceberg for existing data
+        2. If complete, return immediately
+        3. If missing data, request via relay
+        4. Wait for completion notification
+        5. Query Iceberg again for complete dataset
+        6. Return results
+
+        Args:
+            ticker: Market identifier (e.g., "BINANCE:BTC/USDT")
+            period_seconds: OHLC period in seconds (60, 300, 3600, etc.)
+            start_time: Start timestamp in microseconds
+            end_time: End timestamp in microseconds
+            request_timeout: Timeout for historical data requests (default: 30s)
+
+        Returns:
+            DataFrame with OHLC data sorted by timestamp
+
+        Raises:
+            TimeoutError: If historical data request times out
+            ValueError: If request fails
+        """
+        # Step 1: Check Iceberg for existing data
+        df = self.iceberg.query_ohlc(ticker, period_seconds, start_time, end_time)
+
+        # Step 2: Identify missing ranges
+        missing_ranges = self.iceberg.find_missing_ranges(
+            ticker, period_seconds, start_time, end_time
+        )
+
+        if not missing_ranges:
+            # All data exists in Iceberg
+            return df
+
+        # Step 3: Request missing data for each range
+        # For simplicity, request entire range (relay can merge adjacent requests)
+        result = await self.history.request_historical_ohlc(
+            ticker=ticker,
+            period_seconds=period_seconds,
+            start_time=start_time,
+            end_time=end_time,
+            timeout=request_timeout
+        )
+
+        # Step 4: Check result status
+        if result['status'] == 'ERROR':
+            raise ValueError(f"Historical data request failed: {result['error_message']}")
+
+        # Step 5: Query Iceberg again for complete dataset
+        df = self.iceberg.query_ohlc(ticker, period_seconds, start_time, end_time)
+
+        return df
+
+    async def __aenter__(self):
+        """Support async context manager."""
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Support async context manager."""
+        await self.stop()

sandbox/dexorder/symbol_metadata_client.py

@@ -0,0 +1,188 @@
+"""
+SymbolMetadataClient - Query symbol metadata from Iceberg for price/volume conversion.
+
+Provides lazy-loaded, cached access to symbol metadata including denominators
+used to convert integer OHLC data to decimal prices and volumes.
+"""
+
+import logging
+from typing import Optional, Dict, NamedTuple
+from pyiceberg.catalog import load_catalog
+from pyiceberg.expressions import EqualTo, And
+
+log = logging.getLogger(__name__)
+
+
+class SymbolMetadata(NamedTuple):
+    """Symbol metadata containing denominators for price/volume conversion."""
+    exchange_id: str
+    market_id: str
+    tick_denom: int  # Denominator for price fields (open, high, low, close)
+    base_denom: int  # Denominator for base asset (volume in base terms)
+    quote_denom: int  # Denominator for quote asset
+    market_type: Optional[str] = None
+    description: Optional[str] = None
+
+
+class SymbolMetadataClient:
+    """
+    Client for querying symbol metadata from Iceberg.
+
+    Provides lazy-loaded, cached access to market metadata including
+    denominators needed to convert integer OHLC prices/volumes to decimals.
+    """
+
+    def __init__(
+        self,
+        catalog_uri: str,
+        namespace: str = "trading",
+        s3_endpoint: Optional[str] = None,
+        s3_access_key: Optional[str] = None,
+        s3_secret_key: Optional[str] = None,
+    ):
+        """
+        Initialize symbol metadata client.
+
+        Args:
+            catalog_uri: URI of the Iceberg catalog
+            namespace: Iceberg namespace (default: "trading")
+            s3_endpoint: S3/MinIO endpoint URL
+            s3_access_key: S3/MinIO access key
+            s3_secret_key: S3/MinIO secret key
+        """
+        self.catalog_uri = catalog_uri
+        self.namespace = namespace
+
+        catalog_props = {"uri": catalog_uri}
+        if s3_endpoint:
+            catalog_props["s3.endpoint"] = s3_endpoint
+            catalog_props["s3.path-style-access"] = "true"
+        if s3_access_key:
+            catalog_props["s3.access-key-id"] = s3_access_key
+        if s3_secret_key:
+            catalog_props["s3.secret-access-key"] = s3_secret_key
+
+        self.catalog = load_catalog("trading", **catalog_props)
+
+        # Lazy load the table
+        self._table = None
+
+        # Cache: ticker -> SymbolMetadata
+        self._cache: Dict[str, SymbolMetadata] = {}
+
+    @property
+    def table(self):
+        """Lazy load the symbol_metadata table."""
+        if self._table is None:
+            try:
+                self._table = self.catalog.load_table(f"{self.namespace}.symbol_metadata")
+                log.info(f"Loaded symbol_metadata table from {self.namespace}")
+            except Exception as e:
+                raise RuntimeError(
+                    f"Failed to load symbol_metadata table from {self.namespace}.symbol_metadata. "
+                    f"This table is required for price/volume conversion. Error: {e}"
+                ) from e
+        return self._table
+
+    def get_metadata(self, ticker: str) -> SymbolMetadata:
+        """
+        Get metadata for a ticker (e.g., "BINANCE:BTC/USDT").
+
+        Args:
+            ticker: Market identifier in format "EXCHANGE:SYMBOL"
+
+        Returns:
+            SymbolMetadata with denominators and market info
+
+        Raises:
+            ValueError: If ticker format is invalid or metadata not found
+            RuntimeError: If symbol_metadata table cannot be loaded
+        """
+        # Check cache first
+        if ticker in self._cache:
+            return self._cache[ticker]
+
+        # Parse ticker into exchange_id and market_id
+        if ":" not in ticker:
+            raise ValueError(
+                f"Invalid ticker format '{ticker}'. Expected format: 'EXCHANGE:SYMBOL' "
+                f"(e.g., 'BINANCE:BTC/USDT')"
+            )
+
+        exchange_id, market_id = ticker.split(":", 1)
+
+        # Query Iceberg for this symbol
+        try:
+            df = self.table.scan(
+                row_filter=And(
+                    EqualTo("exchange_id", exchange_id),
+                    EqualTo("market_id", market_id)
+                )
+            ).to_pandas()
+
+            if df.empty:
+                raise ValueError(
+                    f"No metadata found for ticker '{ticker}' (exchange_id='{exchange_id}', "
+                    f"market_id='{market_id}'). The symbol may not be configured in the system. "
+                    f"Available tickers can be queried from the symbol_metadata table."
+                )
+
+            if len(df) > 1:
+                log.warning(f"Multiple metadata entries found for {ticker}, using first entry")
+
+            row = df.iloc[0]
+
+            # Extract denominators (required fields)
+            tick_denom = row.get("tick_denom")
+            base_denom = row.get("base_denom")
+            quote_denom = row.get("quote_denom")
+
+            if tick_denom is None or tick_denom == 0:
+                raise ValueError(
+                    f"Invalid tick_denom for {ticker}: {tick_denom}. "
+                    f"Denominator must be a positive integer."
+                )
+
+            if base_denom is None or base_denom == 0:
+                raise ValueError(
+                    f"Invalid base_denom for {ticker}: {base_denom}. "
+                    f"Denominator must be a positive integer."
+                )
+
+            if quote_denom is None or quote_denom == 0:
+                raise ValueError(
+                    f"Invalid quote_denom for {ticker}: {quote_denom}. "
+                    f"Denominator must be a positive integer."
+                )
+
+            metadata = SymbolMetadata(
+                exchange_id=exchange_id,
+                market_id=market_id,
+                tick_denom=int(tick_denom),
+                base_denom=int(base_denom),
+                quote_denom=int(quote_denom),
+                market_type=row.get("market_type"),
+                description=row.get("description"),
+            )
+
+            # Cache the result
+            self._cache[ticker] = metadata
+            log.debug(
+                f"Loaded metadata for {ticker}: tick_denom={metadata.tick_denom}, "
+                f"base_denom={metadata.base_denom}, quote_denom={metadata.quote_denom}"
+            )
+
+            return metadata
+
+        except ValueError:
+            # Re-raise ValueError as-is (ticker not found, invalid format, etc.)
+            raise
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to query metadata for ticker '{ticker}': {e}"
+            ) from e
+
+    def clear_cache(self):
+        """Clear the metadata cache (useful for testing or forcing reloads)."""
+        self._cache.clear()
+        log.info("Symbol metadata cache cleared")

sandbox/dexorder/tools/__init__.py

@@ -0,0 +1,3 @@
+import logging
+
+log = logging.getLogger(__name__)

sandbox/dexorder/tools/category_tools.py

@@ -0,0 +1,671 @@
+"""
+Category-based File Management Tools for MCP Server
+
+Provides write/edit/read/list tools for categorized Python scripts with automatic
+directory structure, metadata management, and validation harnesses.
+
+Categories:
+- strategy: Trading strategies with specific data feed requirements
+- indicator: Technical indicators with configurable parameters
+- research: Research scripts with pyplot visualization support
+
+File Structure:
+{DATA_DIR}/{category}/{sanitized_name}/
+  - implementation.py  # Python implementation
+  - metadata.json      # Category-specific metadata (includes name and description)
+
+After write/edit operations, a category-specific test harness runs to validate
+the code and capture errors/output for agent feedback.
+"""
+
+import json
+import logging
+import re
+import subprocess
+import sys
+from dataclasses import dataclass, asdict
+from enum import Enum
+from pathlib import Path
+from typing import Any, Optional
+
+log = logging.getLogger(__name__)
+
+# Path to the research harness script (written to disk, not inline)
+_RESEARCH_HARNESS = Path(__file__).parent / "research_harness.py"
+
+# Import conda manager for package installation
+try:
+    from dexorder.conda_manager import install_packages
+except ImportError:
+    log.warning("conda_manager not available - package installation disabled")
+    install_packages = None
+
+
+# =============================================================================
+# Categories and Metadata
+# =============================================================================
+
+class Category(str, Enum):
+    """Supported file categories."""
+    STRATEGY = "strategy"
+    INDICATOR = "indicator"
+    RESEARCH = "research"
+
+
+@dataclass
+class BaseMetadata:
+    """Base metadata for all categories."""
+    name: str  # Display name (can have special chars)
+    description: str  # LLM-generated description
+
+
+@dataclass
+class StrategyMetadata(BaseMetadata):
+    """Metadata for trading strategies."""
+    data_feeds: list[str] = None  # Required data feeds (e.g., ["BTC/USD", "ETH/USD"])
+    conda_packages: list[str] = None  # Additional conda packages required
+
+    def __post_init__(self):
+        if self.data_feeds is None:
+            self.data_feeds = []
+        if self.conda_packages is None:
+            self.conda_packages = []
+
+
+@dataclass
+class IndicatorMetadata(BaseMetadata):
+    """Metadata for technical indicators."""
+    default_length: int = 14  # Default period/length parameter
+    conda_packages: list[str] = None  # Additional conda packages required
+
+    def __post_init__(self):
+        if self.conda_packages is None:
+            self.conda_packages = []
+
+
+@dataclass
+class ResearchMetadata(BaseMetadata):
+    """Metadata for research scripts."""
+    conda_packages: list[str] = None  # Additional conda packages required
+
+    def __post_init__(self):
+        if self.conda_packages is None:
+            self.conda_packages = []
+
+
+# Metadata class registry
+METADATA_CLASSES = {
+    Category.STRATEGY: StrategyMetadata,
+    Category.INDICATOR: IndicatorMetadata,
+    Category.RESEARCH: ResearchMetadata,
+}
+
+
+# =============================================================================
+# Utilities
+# =============================================================================
+
+def sanitize_name(name: str) -> str:
+    """
+    Sanitize a name for use as a directory name.
+
+    Converts special characters to underscores and preserves alphanumerics.
+    Examples:
+        "Tim's 5/13 EMA+" -> "Tims_5_13_EMA"
+        "My Strategy (v2)" -> "My_Strategy_v2"
+    """
+    # Replace non-alphanumeric chars (except spaces/hyphens) with underscores
+    sanitized = re.sub(r'[^\w\s\-]', '_', name)
+    # Replace spaces and hyphens with underscores
+    sanitized = re.sub(r'[\s\-]+', '_', sanitized)
+    # Remove duplicate underscores
+    sanitized = re.sub(r'_+', '_', sanitized)
+    # Strip leading/trailing underscores
+    sanitized = sanitized.strip('_')
+    return sanitized
+
+
+def get_category_path(data_dir: Path, category: Category, name: str) -> Path:
+    """
+    Get the directory path for a category item.
+
+    Args:
+        data_dir: Base data directory
+        category: Category enum
+        name: Display name (will be sanitized)
+
+    Returns:
+        Path to the category item directory
+    """
+    safe_name = sanitize_name(name)
+    return data_dir / category.value / safe_name
+
+
+# =============================================================================
+# Category File Manager
+# =============================================================================
+
+class CategoryFileManager:
+    """
+    Manages category-based file operations with validation.
+    """
+
+    def __init__(self, data_dir: Path):
+        self.data_dir = data_dir
+
+        # Ensure category directories exist
+        for category in Category:
+            (data_dir / category.value).mkdir(parents=True, exist_ok=True)
+
+    def write(
+        self,
+        category: str,
+        name: str,
+        description: str,
+        code: str,
+        metadata: Optional[dict] = None
+    ) -> dict[str, Any]:
+        """
+        Write a new category file with metadata.
+
+        Args:
+            category: Category name (strategy, indicator, research)
+            name: Display name for the item
+            description: LLM-generated description (required)
+            code: Python implementation code
+            metadata: Additional category-specific metadata fields
+
+        Returns:
+            dict with:
+            - success: bool
+            - path: str - path to the implementation file
+            - validation: dict - results from test harness
+            - error: str (if any)
+        """
+        try:
+            cat = Category(category)
+        except ValueError:
+            return {
+                "success": False,
+                "error": f"Invalid category '{category}'. Must be one of: {', '.join(c.value for c in Category)}"
+            }
+
+        # Get item directory
+        item_dir = get_category_path(self.data_dir, cat, name)
+        item_dir.mkdir(parents=True, exist_ok=True)
+
+        # Write implementation
+        impl_path = item_dir / "implementation.py"
+        try:
+            impl_path.write_text(code)
+            log.info(f"Wrote {cat.value} implementation: {impl_path}")
+        except Exception as e:
+            return {"success": False, "error": f"Failed to write implementation: {e}"}
+
+        # Build metadata
+        meta_dict = metadata or {}
+        meta_dict["name"] = name
+        meta_dict["description"] = description
+
+        # Validate and write metadata
+        try:
+            metadata_class = METADATA_CLASSES[cat]
+            meta_obj = metadata_class(**meta_dict)
+            validated_meta = asdict(meta_obj)
+
+            meta_path = item_dir / "metadata.json"
+            meta_path.write_text(json.dumps(validated_meta, indent=2))
+            log.info(f"Wrote metadata: {meta_path}")
+        except Exception as e:
+            return {"success": False, "error": f"Failed to write metadata: {e}"}
+
+        # Run validation harness
+        validation = self._validate(cat, item_dir)
+
+        result = {
+            "success": validation["success"],
+            "path": str(impl_path),
+            "validation": validation,
+        }
+
+        # Auto-execute research scripts after successful write
+        if cat == Category.RESEARCH and validation["success"]:
+            log.info(f"Auto-executing research script: {name}")
+            execution_result = self.execute_research(name)
+            result["execution"] = execution_result
+
+        return result
+
+    def edit(
+        self,
+        category: str,
+        name: str,
+        code: Optional[str] = None,
+        description: Optional[str] = None,
+        metadata: Optional[dict] = None
+    ) -> dict[str, Any]:
+        """
+        Edit an existing category file.
+
+        Args:
+            category: Category name
+            name: Display name for the item
+            code: Python implementation code (optional, omit to keep existing)
+            description: Updated description (optional, omit to keep existing)
+            metadata: Additional metadata updates (optional)
+
+        Returns:
+            dict with:
+            - success: bool
+            - path: str - path to the implementation file
+            - validation: dict - results from test harness (if code updated)
+            - error: str (if any)
+        """
+        try:
+            cat = Category(category)
+        except ValueError:
+            return {"success": False, "error": f"Invalid category '{category}'"}
+
+        item_dir = get_category_path(self.data_dir, cat, name)
+
+        if not item_dir.exists():
+            return {"success": False, "error": f"Item '{name}' does not exist in category '{category}'"}
+
+        impl_path = item_dir / "implementation.py"
+        meta_path = item_dir / "metadata.json"
+
+        # Load existing metadata
+        try:
+            existing_meta = {}
+            if meta_path.exists():
+                existing_meta = json.loads(meta_path.read_text())
+        except Exception as e:
+            return {"success": False, "error": f"Failed to read existing metadata: {e}"}
+
+        # Update code if provided
+        if code is not None:
+            try:
+                impl_path.write_text(code)
+                log.info(f"Updated {cat.value} implementation: {impl_path}")
+            except Exception as e:
+                return {"success": False, "error": f"Failed to write implementation: {e}"}
+
+        # Update metadata
+        updated_meta = existing_meta.copy()
+        if description is not None:
+            updated_meta["description"] = description
+        if metadata:
+            updated_meta.update(metadata)
+
+        # Validate and write metadata
+        try:
+            metadata_class = METADATA_CLASSES[cat]
+            meta_obj = metadata_class(**updated_meta)
+            validated_meta = asdict(meta_obj)
+
+            meta_path.write_text(json.dumps(validated_meta, indent=2))
+            log.info(f"Updated metadata: {meta_path}")
+        except Exception as e:
+            return {"success": False, "error": f"Failed to write metadata: {e}"}
+
+        # Run validation harness if code was updated
+        validation = None
+        if code is not None:
+            validation = self._validate(cat, item_dir)
+
+        result = {
+            "success": True,
+            "path": str(impl_path),
+        }
+        if validation:
+            result["validation"] = validation
+            result["success"] = validation["success"]
+
+        # Auto-execute research scripts after successful edit (if code was updated)
+        if cat == Category.RESEARCH and code is not None and result["success"]:
+            log.info(f"Auto-executing research script after edit: {name}")
+            execution_result = self.execute_research(name)
+            result["execution"] = execution_result
+
+        return result
+
+    def read(
+        self,
+        category: str,
+        name: str
+    ) -> dict[str, Any]:
+        """
+        Read a category file and its metadata.
+
+        Returns:
+            dict with:
+            - exists: bool
+            - code: str - implementation code
+            - metadata: dict - metadata
+            - error: str (if any)
+        """
+        try:
+            cat = Category(category)
+        except ValueError:
+            return {"exists": False, "error": f"Invalid category '{category}'"}
+
+        item_dir = get_category_path(self.data_dir, cat, name)
+
+        if not item_dir.exists():
+            return {"exists": False}
+
+        impl_path = item_dir / "implementation.py"
+        meta_path = item_dir / "metadata.json"
+
+        try:
+            code = impl_path.read_text() if impl_path.exists() else ""
+            metadata = {}
+            if meta_path.exists():
+                metadata = json.loads(meta_path.read_text())
+
+            return {
+                "exists": True,
+                "code": code,
+                "metadata": metadata,
+            }
+        except Exception as e:
+            return {"exists": False, "error": str(e)}
+
+    def list_items(self, category: str) -> dict[str, Any]:
+        """
+        List all items in a category with names and descriptions.
+
+        Returns:
+            dict with:
+            - items: list of dicts with name, description, and full metadata
+            - error: str (if any)
+        """
+        try:
+            cat = Category(category)
+        except ValueError:
+            return {"error": f"Invalid category '{category}'"}
+
+        cat_dir = self.data_dir / cat.value
+        items = []
+
+        for item_dir in cat_dir.iterdir():
+            if not item_dir.is_dir():
+                continue
+
+            meta_path = item_dir / "metadata.json"
+            if meta_path.exists():
+                try:
+                    metadata = json.loads(meta_path.read_text())
+                    items.append({
+                        "name": metadata.get("name", item_dir.name),
+                        "description": metadata.get("description", ""),
+                        "metadata": metadata,
+                    })
+                except Exception as e:
+                    log.error(f"Failed to read metadata for {item_dir}: {e}")
+
+        return {"items": items}
+
+    def _validate(self, category: Category, item_dir: Path) -> dict[str, Any]:
+        """
+        Run category-specific validation harness.
+
+        Returns:
+            dict with:
+            - success: bool
+            - output: str - stdout/stderr from validation
+            - images: list[dict] - base64-encoded images (for research)
+            - packages_installed: list[str] - packages that were installed
+            - error: str (if any)
+        """
+        impl_path = item_dir / "implementation.py"
+        meta_path = item_dir / "metadata.json"
+
+        # Install required packages before validation
+        packages_installed = []
+        if install_packages and meta_path.exists():
+            try:
+                metadata = json.loads(meta_path.read_text())
+                conda_packages = metadata.get("conda_packages", [])
+                if conda_packages:
+                    log.info(f"Installing packages for validation: {conda_packages}")
+                    install_result = install_packages(conda_packages)
+                    if install_result.get("success"):
+                        packages_installed = install_result.get("installed", [])
+                        if packages_installed:
+                            log.info(f"Installed packages: {packages_installed}")
+                    else:
+                        log.warning(f"Failed to install packages: {install_result.get('error')}")
+            except Exception as e:
+                log.error(f"Error installing packages: {e}")
+
+        # Run validation
+        if category == Category.STRATEGY:
+            result = self._validate_strategy(impl_path)
+        elif category == Category.INDICATOR:
+            result = self._validate_indicator(impl_path)
+        elif category == Category.RESEARCH:
+            result = self._validate_research(impl_path, item_dir)
+        else:
+            result = {"success": False, "error": f"No validator for category {category}"}
+
+        # Add package installation info
+        if packages_installed:
+            result["packages_installed"] = packages_installed
+
+        return result
+
+    def _validate_strategy(self, impl_path: Path) -> dict[str, Any]:
+        """
+        Validate a strategy implementation.
+
+        Runs basic syntax check and imports.
+        """
+        try:
+            result = subprocess.run(
+                [sys.executable, "-m", "py_compile", str(impl_path)],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+
+            if result.returncode == 0:
+                return {
+                    "success": True,
+                    "output": "Strategy syntax valid",
+                }
+            else:
+                return {
+                    "success": False,
+                    "output": result.stderr,
+                    "error": "Syntax error in strategy",
+                }
+        except subprocess.TimeoutExpired:
+            return {"success": False, "error": "Validation timeout"}
+        except Exception as e:
+            return {"success": False, "error": f"Validation failed: {e}"}
+
+    def _validate_indicator(self, impl_path: Path) -> dict[str, Any]:
+        """
+        Validate an indicator implementation.
+
+        Runs basic syntax check and imports.
+        """
+        try:
+            result = subprocess.run(
+                [sys.executable, "-m", "py_compile", str(impl_path)],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+
+            if result.returncode == 0:
+                return {
+                    "success": True,
+                    "output": "Indicator syntax valid",
+                }
+            else:
+                return {
+                    "success": False,
+                    "output": result.stderr,
+                    "error": "Syntax error in indicator",
+                }
+        except subprocess.TimeoutExpired:
+            return {"success": False, "error": "Validation timeout"}
+        except Exception as e:
+            return {"success": False, "error": f"Validation failed: {e}"}
+
+    def _run_research_harness(self, impl_path: Path, item_dir: Path, timeout: int = 30) -> dict[str, Any]:
+        """
+        Run a research script via the on-disk harness and return parsed results.
+
+        The harness (research_harness.py) handles API initialization, stdout/stderr
+        capture, matplotlib figure capture, and outputs JSON to stdout.
+
+        Returns:
+            dict with stdout, stderr, images, error fields — or an error dict.
+        """
+        try:
+            result = subprocess.run(
+                [sys.executable, str(_RESEARCH_HARNESS), str(impl_path)],
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+                cwd=str(item_dir),
+            )
+
+            if result.returncode == 0:
+                try:
+                    return json.loads(result.stdout)
+                except json.JSONDecodeError:
+                    return {
+                        "stdout": result.stdout,
+                        "stderr": result.stderr,
+                        "images": [],
+                        "error": True,
+                    }
+            else:
+                # Harness itself failed (import error, bad args, etc.)
+                return {
+                    "stdout": "",
+                    "stderr": result.stderr,
+                    "images": [],
+                    "error": True,
+                }
+        except subprocess.TimeoutExpired:
+            return {"stdout": "", "stderr": "", "images": [], "error": True,
+                    "_timeout": True}
+        except Exception as e:
+            return {"stdout": "", "stderr": str(e), "images": [], "error": True}
+
+    def _validate_research(self, impl_path: Path, item_dir: Path) -> dict[str, Any]:
+        """
+        Validate a research script.
+
+        Runs the script via the harness and captures output + pyplot images.
+        """
+        data = self._run_research_harness(impl_path, item_dir, timeout=30)
+
+        if data.get("_timeout"):
+            return {"success": False, "error": "Research script timeout"}
+
+        if data["error"]:
+            return {
+                "success": False,
+                "output": data["stderr"],
+                "error": "Research script execution failed",
+            }
+
+        return {
+            "success": True,
+            "output": data["stdout"],
+            "images": data["images"],
+        }
+
+    def execute_research(self, name: str) -> dict[str, Any]:
+        """
+        Execute a research script and return structured content with images.
+
+        Args:
+            name: Display name of the research script
+
+        Returns:
+            dict with:
+            - content: list of TextContent and ImageContent objects (MCP format)
+            - error: str (if any)
+        """
+        item_dir = get_category_path(self.data_dir, Category.RESEARCH, name)
+
+        if not item_dir.exists():
+            return {"error": f"Research script '{name}' does not exist"}
+
+        impl_path = item_dir / "implementation.py"
+        if not impl_path.exists():
+            return {"error": f"Implementation file not found for '{name}'"}
+
+        data = self._run_research_harness(impl_path, item_dir, timeout=300)
+
+        if data.get("_timeout"):
+            log.error(f"execute_research '{name}': timeout")
+            return {"error": "Research script timeout (5 minutes exceeded)"}
+
+        log.info(
+            f"execute_research '{name}': script_error={data.get('error')}, "
+            f"stdout_len={len(data.get('stdout', ''))}, "
+            f"stderr_len={len(data.get('stderr', ''))}, "
+            f"image_count={len(data.get('images', []))}"
+        )
+        if data.get("stderr"):
+            log.warning(f"execute_research '{name}' stderr: {data['stderr'][:500]}")
+
+        # Build MCP structured content
+        from mcp.types import TextContent, ImageContent
+
+        content = []
+
+        # Add text output (stdout/stderr combined)
+        text_parts = []
+        if data["stdout"]:
+            text_parts.append(f"stdout:\n{data['stdout']}")
+        if data["stderr"]:
+            text_parts.append(f"stderr:\n{data['stderr']}")
+
+        if text_parts:
+            content.append(
+                TextContent(type="text", text="\n\n".join(text_parts))
+            )
+
+        # Add images
+        for img in data["images"]:
+            content.append(
+                ImageContent(
+                    type="image",
+                    data=img["data"],
+                    mimeType="image/png"
+                )
+            )
+
+        # If there was an error but we still got output, include error flag
+        if data.get("error") and not content:
+            log.error(f"execute_research '{name}': script failed with no output")
+            return {"error": "Research script execution failed"}
+
+        log.info(f"execute_research '{name}': returning {len(content)} content items")
+        return {"content": content}
+
+
+# =============================================================================
+# Global Manager Instance
+# =============================================================================
+
+_category_manager: Optional[CategoryFileManager] = None
+
+
+def get_category_manager(data_dir: Optional[Path] = None) -> CategoryFileManager:
+    """Get or create the global category file manager."""
+    global _category_manager
+    if _category_manager is None:
+        if data_dir is None:
+            raise ValueError("data_dir required for first initialization")
+        _category_manager = CategoryFileManager(data_dir)
+    return _category_manager

sandbox/dexorder/tools/research_harness.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""
+Research script harness - runs implementation.py in a subprocess with API
+initialization, stdout/stderr capture, and matplotlib figure capture.
+
+This file is written to disk and invoked by category_tools.py rather than
+being passed inline via `python -c`, so the harness code is inspectable and
+not regenerated on every call.
+
+Usage:
+    python -m dexorder.tools.research_harness <implementation_path>
+
+Output (JSON to stdout):
+    {
+        "stdout": "captured user stdout",
+        "stderr": "captured user stderr",
+        "images": [{"format": "png", "data": "<base64>"}],
+        "error": false
+    }
+"""
+
+import sys
+import io
+import os
+import base64
+import json
+from pathlib import Path
+
+# Non-interactive matplotlib backend (must be set before importing pyplot)
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+
+# Ensure dexorder package is importable
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
+
+# ---------------------------------------------------------------------------
+# Initialize API from config files so research scripts can call get_api()
+# ---------------------------------------------------------------------------
+try:
+    import yaml
+
+    config_path = os.environ.get("CONFIG_PATH", "/app/config/config.yaml")
+    secrets_path = os.environ.get("SECRETS_PATH", "/app/config/secrets.yaml")
+
+    config_data = {}
+    secrets_data = {}
+    if Path(config_path).exists():
+        with open(config_path) as f:
+            config_data = yaml.safe_load(f) or {}
+    if Path(secrets_path).exists():
+        with open(secrets_path) as f:
+            secrets_data = yaml.safe_load(f) or {}
+
+    data_cfg = config_data.get("data", {})
+    iceberg_cfg = data_cfg.get("iceberg", {})
+    relay_cfg = data_cfg.get("relay", {})
+
+    from dexorder.api import set_api, API
+    from dexorder.impl.charting_api_impl import ChartingAPIImpl
+    from dexorder.impl.data_api_impl import DataAPIImpl
+
+    _data_api = DataAPIImpl(
+        iceberg_catalog_uri=iceberg_cfg.get("catalog_uri", "http://iceberg-catalog:8181"),
+        relay_endpoint=relay_cfg.get("endpoint", "tcp://relay:5559"),
+        notification_endpoint=relay_cfg.get("notification_endpoint", "tcp://relay:5558"),
+        namespace=iceberg_cfg.get("namespace", "trading"),
+        s3_endpoint=iceberg_cfg.get("s3_endpoint") or secrets_data.get("s3_endpoint"),
+        s3_access_key=iceberg_cfg.get("s3_access_key") or secrets_data.get("s3_access_key"),
+        s3_secret_key=iceberg_cfg.get("s3_secret_key") or secrets_data.get("s3_secret_key"),
+    )
+    # NOTE: We intentionally do NOT call asyncio.run(_data_api.start()) here.
+    # DataAPIImpl.historical_ohlc() auto-starts on first use, which ensures the
+    # ZMQ context and notification listener are created inside the user's own
+    # asyncio.run() event loop — avoiding cross-loop lifecycle issues.
+    set_api(API(charting=ChartingAPIImpl(), data=_data_api))
+except Exception as e:
+    print(f"WARNING: API initialization failed: {e}", file=sys.stderr)
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: research_harness.py <implementation_path>", file=sys.stderr)
+        sys.exit(2)
+
+    impl_path = Path(sys.argv[1])
+    if not impl_path.exists():
+        print(json.dumps({
+            "stdout": "",
+            "stderr": f"Implementation file not found: {impl_path}",
+            "images": [],
+            "error": True,
+        }))
+        sys.exit(0)
+
+    # Capture stdout and stderr
+    old_stdout = sys.stdout
+    old_stderr = sys.stderr
+    sys.stdout = io.StringIO()
+    sys.stderr = io.StringIO()
+
+    error_occurred = False
+    try:
+        exec(compile(impl_path.read_text(), str(impl_path), 'exec'), {})
+    except Exception as e:
+        print(f"ERROR: {e}", file=sys.stderr)
+        import traceback
+        traceback.print_exc(file=sys.stderr)
+        error_occurred = True
+
+    # Restore stdout/stderr
+    stdout_output = sys.stdout.getvalue()
+    stderr_output = sys.stderr.getvalue()
+    sys.stdout = old_stdout
+    sys.stderr = old_stderr
+
+    # Capture all matplotlib figures as base64 PNGs
+    images = []
+    for fig_num in plt.get_fignums():
+        fig = plt.figure(fig_num)
+        buf = io.BytesIO()
+        fig.savefig(buf, format='png', dpi=100, bbox_inches='tight')
+        buf.seek(0)
+        img_b64 = base64.b64encode(buf.read()).decode('utf-8')
+        images.append({"format": "png", "data": img_b64})
+        buf.close()
+    plt.close('all')
+
+    # Output results as JSON to real stdout
+    result = {
+        "stdout": stdout_output,
+        "stderr": stderr_output,
+        "images": images,
+        "error": error_occurred,
+    }
+    print(json.dumps(result))
+
+
+if __name__ == "__main__":
+    main()

sandbox/dexorder/tools/workspace_tools.py

@@ -0,0 +1,384 @@
+"""
+Workspace Tools for MCP Server
+
+Provides read/write/patch tools for workspace stores that are persisted
+in the user container. These stores sync with the gateway and web client.
+
+Storage location: {DATA_DIR}/workspace/{store_name}.json
+
+Available tools:
+- workspace_read(store_name) -> dict
+- workspace_write(store_name, data) -> None
+- workspace_patch(store_name, patch) -> dict
+
+Future: Path-based triggers for container-side reactions to state changes.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+import jsonpatch
+
+log = logging.getLogger(__name__)
+
+# Default workspace storage directory (relative to working dir for local dev)
+# In production, this is overridden by passing workspace_dir from Config
+DEFAULT_WORKSPACE_DIR = Path("data/workspace")
+
+
+class WorkspaceStore:
+    """
+    Manages persistent workspace stores on the filesystem.
+
+    Stores are JSON files at: {workspace_dir}/{store_name}.json
+    """
+
+    def __init__(self, workspace_dir: Path = DEFAULT_WORKSPACE_DIR):
+        self.workspace_dir = workspace_dir
+        self.workspace_dir.mkdir(parents=True, exist_ok=True)
+
+        # Path triggers (for future use)
+        # Map of "store_name/json/pointer/path" -> list of callbacks
+        self._triggers: dict[str, list[Callable[[Any, Any], None]]] = {}
+
+    def _store_path(self, store_name: str) -> Path:
+        """Get the filesystem path for a store."""
+        # Sanitize store name to prevent directory traversal
+        safe_name = store_name.replace("/", "_").replace("\\", "_").replace("..", "_")
+        return self.workspace_dir / f"{safe_name}.json"
+
+    def read(self, store_name: str) -> dict[str, Any]:
+        """
+        Read a workspace store from disk.
+
+        Returns:
+            dict with:
+            - exists: bool - whether the store exists
+            - data: Any - the store data (if exists)
+            - error: str - error message (if any)
+        """
+        path = self._store_path(store_name)
+
+        if not path.exists():
+            log.debug(f"Store '{store_name}' does not exist at {path}")
+            return {"exists": False}
+
+        try:
+            with open(path, "r") as f:
+                data = json.load(f)
+            log.debug(f"Read store '{store_name}' from {path}")
+            return {"exists": True, "data": data}
+        except json.JSONDecodeError as e:
+            log.error(f"Failed to parse store '{store_name}': {e}")
+            return {"exists": False, "error": f"Invalid JSON: {e}"}
+        except Exception as e:
+            log.error(f"Failed to read store '{store_name}': {e}")
+            return {"exists": False, "error": str(e)}
+
+    def write(self, store_name: str, data: Any) -> dict[str, Any]:
+        """
+        Write a workspace store to disk.
+
+        Returns:
+            dict with:
+            - success: bool
+            - error: str - error message (if any)
+        """
+        path = self._store_path(store_name)
+
+        try:
+            # Read old state for triggers
+            old_state = None
+            if path.exists():
+                try:
+                    with open(path, "r") as f:
+                        old_state = json.load(f)
+                except Exception:
+                    pass
+
+            # Write new state
+            with open(path, "w") as f:
+                json.dump(data, f, indent=2)
+
+            log.debug(f"Wrote store '{store_name}' to {path}")
+
+            # Fire triggers if state changed
+            if old_state != data:
+                self._fire_triggers(store_name, old_state, data)
+
+            return {"success": True}
+        except Exception as e:
+            log.error(f"Failed to write store '{store_name}': {e}")
+            return {"success": False, "error": str(e)}
+
+    def patch(self, store_name: str, patch: list[dict]) -> dict[str, Any]:
+        """
+        Apply a JSON patch (RFC 6902) to a store.
+
+        Args:
+            store_name: Name of the store
+            patch: List of JSON patch operations
+
+        Returns:
+            dict with:
+            - success: bool
+            - data: Any - the new state (if success)
+            - error: str - error message (if any)
+        """
+        path = self._store_path(store_name)
+
+        try:
+            # Read current state (or empty dict if doesn't exist)
+            old_state = {}
+            if path.exists():
+                with open(path, "r") as f:
+                    old_state = json.load(f)
+
+            # Apply patch
+            new_state = jsonpatch.apply_patch(old_state, patch)
+
+            # Write new state
+            with open(path, "w") as f:
+                json.dump(new_state, f, indent=2)
+
+            log.debug(f"Patched store '{store_name}' with {len(patch)} operations")
+
+            # Fire triggers
+            self._fire_triggers(store_name, old_state, new_state)
+
+            return {"success": True, "data": new_state}
+        except jsonpatch.JsonPatchConflict as e:
+            log.error(f"Patch conflict for store '{store_name}': {e}")
+            return {"success": False, "error": f"Patch conflict: {e}"}
+        except Exception as e:
+            log.error(f"Failed to patch store '{store_name}': {e}")
+            return {"success": False, "error": str(e)}
+
+    def delete(self, store_name: str) -> dict[str, Any]:
+        """
+        Delete a workspace store.
+
+        Returns:
+            dict with:
+            - success: bool
+            - error: str - error message (if any)
+        """
+        path = self._store_path(store_name)
+
+        try:
+            if path.exists():
+                path.unlink()
+                log.debug(f"Deleted store '{store_name}'")
+            return {"success": True}
+        except Exception as e:
+            log.error(f"Failed to delete store '{store_name}': {e}")
+            return {"success": False, "error": str(e)}
+
+    def list_stores(self) -> list[str]:
+        """List all existing store names."""
+        stores = []
+        for path in self.workspace_dir.glob("*.json"):
+            stores.append(path.stem)
+        return stores
+
+    # =========================================================================
+    # Triggers (for future use)
+    # =========================================================================
+
+    def on_change(
+        self,
+        store_name: str,
+        path: str,
+        callback: Callable[[Any, Any], None]
+    ) -> Callable[[], None]:
+        """
+        Register a trigger for when a path changes.
+
+        Args:
+            store_name: Name of the store to watch
+            path: JSON pointer path (e.g., "/drawings" or "/*" for any change)
+            callback: Function called with (old_value, new_value)
+
+        Returns:
+            Unsubscribe function
+        """
+        key = f"{store_name}{path}"
+        if key not in self._triggers:
+            self._triggers[key] = []
+        self._triggers[key].append(callback)
+
+        log.debug(f"Registered trigger for {key}")
+
+        def unsubscribe():
+            if key in self._triggers:
+                try:
+                    self._triggers[key].remove(callback)
+                    if not self._triggers[key]:
+                        del self._triggers[key]
+                except ValueError:
+                    pass
+
+        return unsubscribe
+
+    def _fire_triggers(self, store_name: str, old_state: Any, new_state: Any) -> None:
+        """Fire triggers for changes between old and new state."""
+        if not self._triggers:
+            return
+
+        # Fire store-level wildcard triggers
+        wildcard_key = f"{store_name}/*"
+        if wildcard_key in self._triggers:
+            for callback in self._triggers[wildcard_key]:
+                try:
+                    callback(old_state, new_state)
+                except Exception as e:
+                    log.error(f"Error in trigger callback for {wildcard_key}: {e}")
+
+        # Fire path-specific triggers by computing a patch and checking paths
+        if old_state is not None and new_state is not None:
+            try:
+                patch = jsonpatch.make_patch(old_state, new_state)
+                for op in patch.patch:
+                    op_path = op.get("path", "")
+                    trigger_key = f"{store_name}{op_path}"
+                    if trigger_key in self._triggers:
+                        old_value = self._get_value_at_path(old_state, op_path)
+                        new_value = self._get_value_at_path(new_state, op_path)
+                        for callback in self._triggers[trigger_key]:
+                            try:
+                                callback(old_value, new_value)
+                            except Exception as e:
+                                log.error(f"Error in trigger callback for {trigger_key}: {e}")
+            except Exception as e:
+                log.error(f"Error computing patch for triggers: {e}")
+
+    def _get_value_at_path(self, obj: Any, path: str) -> Any:
+        """Get value at a JSON pointer path."""
+        if not path or path == "/":
+            return obj
+
+        parts = path.split("/")[1:]  # Skip empty first part
+        current = obj
+
+        for part in parts:
+            if current is None:
+                return None
+            if isinstance(current, dict):
+                current = current.get(part)
+            elif isinstance(current, list):
+                try:
+                    current = current[int(part)]
+                except (ValueError, IndexError):
+                    return None
+            else:
+                return None
+
+        return current
+
+
+# Global workspace store instance
+_workspace_store: Optional[WorkspaceStore] = None
+
+
+def get_workspace_store(workspace_dir: Optional[Path] = None) -> WorkspaceStore:
+    """Get or create the global workspace store."""
+    global _workspace_store
+    if _workspace_store is None:
+        _workspace_store = WorkspaceStore(workspace_dir or DEFAULT_WORKSPACE_DIR)
+    return _workspace_store
+
+
+def register_workspace_tools(server):
+    """
+    Register workspace tools on an MCP server.
+
+    Args:
+        server: MCP Server instance
+    """
+    store = get_workspace_store()
+
+    @server.call_tool()
+    async def handle_tool_call(name: str, arguments: dict) -> Any:
+        """Handle workspace tool calls."""
+        if name == "workspace_read":
+            return store.read(arguments.get("store_name", ""))
+        elif name == "workspace_write":
+            return store.write(
+                arguments.get("store_name", ""),
+                arguments.get("data")
+            )
+        elif name == "workspace_patch":
+            return store.patch(
+                arguments.get("store_name", ""),
+                arguments.get("patch", [])
+            )
+        else:
+            raise ValueError(f"Unknown tool: {name}")
+
+    @server.list_tools()
+    async def list_tools():
+        """List available workspace tools."""
+        return [
+            {
+                "name": "workspace_read",
+                "description": "Read a workspace store from persistent storage",
+                "inputSchema": {
+                    "type": "object",
+                    "properties": {
+                        "store_name": {
+                            "type": "string",
+                            "description": "Name of the store (e.g., 'chartStore', 'userPreferences')"
+                        }
+                    },
+                    "required": ["store_name"]
+                }
+            },
+            {
+                "name": "workspace_write",
+                "description": "Write a workspace store to persistent storage",
+                "inputSchema": {
+                    "type": "object",
+                    "properties": {
+                        "store_name": {
+                            "type": "string",
+                            "description": "Name of the store"
+                        },
+                        "data": {
+                            "description": "Data to write"
+                        }
+                    },
+                    "required": ["store_name", "data"]
+                }
+            },
+            {
+                "name": "workspace_patch",
+                "description": "Apply JSON patch operations to a workspace store",
+                "inputSchema": {
+                    "type": "object",
+                    "properties": {
+                        "store_name": {
+                            "type": "string",
+                            "description": "Name of the store"
+                        },
+                        "patch": {
+                            "type": "array",
+                            "description": "JSON Patch operations (RFC 6902)",
+                            "items": {
+                                "type": "object",
+                                "properties": {
+                                    "op": {"type": "string", "enum": ["add", "remove", "replace", "move", "copy", "test"]},
+                                    "path": {"type": "string"},
+                                    "value": {}
+                                },
+                                "required": ["op", "path"]
+                            }
+                        }
+                    },
+                    "required": ["store_name", "patch"]
+                }
+            }
+        ]
+
+    log.info("Registered workspace tools")

sandbox/dexorder/utils.py

@@ -0,0 +1,118 @@
+"""
+Utility functions for dexorder.
+
+Includes timestamp conversions, date parsing, and other common utilities.
+"""
+
+import logging
+from typing import Union
+from datetime import datetime
+import pandas as pd
+from dateutil import parser as dateparser
+
+log = logging.getLogger(__name__)
+
+# Type alias for flexible timestamp input
+TimestampInput = Union[int, float, str, datetime, pd.Timestamp]
+
+
+def to_microseconds(timestamp: TimestampInput) -> int:
+    """
+    Convert various timestamp formats to microseconds since epoch.
+
+    This is the canonical way to convert user-friendly timestamps (unix seconds,
+    date strings, datetime objects) into the internal microsecond format used
+    throughout the dexorder system.
+
+    Args:
+        timestamp: Can be:
+            - Unix timestamp (int/float) - assumed to be in seconds
+            - ISO date string (str) - parsed using dateutil
+            - datetime object
+            - pandas Timestamp
+
+    Returns:
+        Microseconds since epoch as integer
+
+    Examples:
+        >>> to_microseconds(1640000000)  # Unix timestamp in seconds
+        1640000000000000
+        >>> to_microseconds(1640000000.5)  # Unix timestamp with fractional seconds
+        1640000000500000
+        >>> to_microseconds("2021-12-20")  # Date string
+        1640000000000000
+        >>> to_microseconds("2021-12-20 12:00:00")  # Date string with time
+        1640000000000000
+        >>> to_microseconds(datetime(2021, 12, 20, 12, 0, 0))  # datetime object
+        1640000000000000
+        >>> to_microseconds(pd.Timestamp("2021-12-20 12:00:00"))  # pandas Timestamp
+        1640000000000000
+    """
+    if isinstance(timestamp, (int, float)):
+        # Assume Unix timestamp in seconds
+        return int(timestamp * 1_000_000)
+    elif isinstance(timestamp, str):
+        # Parse date string
+        dt = dateparser.parse(timestamp)
+        if dt is None:
+            raise ValueError(f"Could not parse date string: {timestamp}")
+        return int(dt.timestamp() * 1_000_000)
+    elif isinstance(timestamp, datetime):
+        return int(timestamp.timestamp() * 1_000_000)
+    elif isinstance(timestamp, pd.Timestamp):
+        return int(timestamp.timestamp() * 1_000_000)
+    else:
+        raise TypeError(f"Unsupported timestamp type: {type(timestamp)}")
+
+
+def to_seconds(timestamp_micros: int) -> float:
+    """
+    Convert microseconds since epoch to Unix timestamp in seconds.
+
+    Args:
+        timestamp_micros: Timestamp in microseconds since epoch
+
+    Returns:
+        Unix timestamp in seconds (float)
+
+    Examples:
+        >>> to_seconds(1640000000000000)
+        1640000000.0
+        >>> to_seconds(1640000000500000)
+        1640000000.5
+    """
+    return timestamp_micros / 1_000_000
+
+
+def to_datetime(timestamp_micros: int) -> datetime:
+    """
+    Convert microseconds since epoch to datetime object.
+
+    Args:
+        timestamp_micros: Timestamp in microseconds since epoch
+
+    Returns:
+        datetime object in UTC
+
+    Examples:
+        >>> to_datetime(1640000000000000)
+        datetime.datetime(2021, 12, 20, 12, 0, tzinfo=datetime.timezone.utc)
+    """
+    return datetime.fromtimestamp(timestamp_micros / 1_000_000)
+
+
+def to_timestamp(timestamp_micros: int) -> pd.Timestamp:
+    """
+    Convert microseconds since epoch to pandas Timestamp.
+
+    Args:
+        timestamp_micros: Timestamp in microseconds since epoch
+
+    Returns:
+        pandas Timestamp
+
+    Examples:
+        >>> to_timestamp(1640000000000000)
+        Timestamp('2021-12-20 12:00:00')
+    """
+    return pd.Timestamp(timestamp_micros, unit='us')

sandbox/entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+# Ensure /app/data is the only writable location for dexorder user
+# All other directories should be read-only (enforced by k8s readOnlyRootFilesystem)
+
+# Fix permissions on mounted volume (k8s may mount with different ownership)
+if [ -d /app/data ]; then
+    # Check if we can write to /app/data - if not, something is wrong
+    if [ ! -w /app/data ]; then
+        echo "ERROR: /app/data is not writable by dexorder user"
+        exit 1
+    fi
+else
+    echo "ERROR: /app/data does not exist"
+    exit 1
+fi
+
+# Ensure /app/config and /app/secrets are read-only (should already be via k8s mount)
+for dir in /app/config /app/secrets; do
+    if [ -d "$dir" ] && [ -w "$dir" ]; then
+        echo "WARNING: $dir is writable but should be read-only"
+    fi
+done
+
+# Execute the main application
+exec /opt/conda/envs/dexorder/bin/python /app/main.py "$@"

sandbox/environment.yml

@@ -0,0 +1,52 @@
+name: dexorder
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python>=3.9
+  # Core data science stack
+  - numpy>=1.24.0
+  - pandas>=2.0.0
+  - pyarrow>=14.0.0
+  - scipy>=1.10.0
+  - scikit-learn>=1.3.0
+  # Visualization
+  - matplotlib>=3.7.0
+  - seaborn>=0.12.0
+  # Iceberg integration
+  - pyiceberg>=0.6.0
+  # ZMQ for event system
+  - pyzmq>=25.0.0
+  # Protobuf
+  - protobuf>=4.25.0
+  # YAML support
+  - pyyaml>=6.0
+  # Async file I/O
+  - aiofiles>=23.0.0
+  # Technical analysis
+  - ta-lib>=0.4.0
+  - mplfinance>=0.12.0
+  - pandas-ta>=0.3.14
+  # Statistics & ML
+  - statsmodels>=0.14.0
+  - optuna>=3.5.0
+  - xgboost>=2.0.0
+  - lightgbm>=4.1.0
+  # Performance optimization
+  - numba>=0.58.0
+  - bottleneck>=1.3.7
+  # Date/time utilities
+  - python-dateutil>=2.8.0
+  - pytz>=2023.3
+  # Scheduling
+  - apscheduler>=3.10.0
+  # Backtesting
+  - vectorbt>=0.25.0
+  # Pip packages (not available in conda)
+  - pip
+  - pip:
+    - mcp>=1.0.0
+    - jsonpatch>=1.33
+    - starlette>=0.27.0
+    - uvicorn>=0.27.0
+    - sse-starlette>=1.6.0

sandbox/main.py

@@ -0,0 +1,707 @@
+#!/usr/bin/env python3
+"""
+DexOrder User Container Main Entry Point
+
+Brings together:
+- Config and secrets loading from k8s mounted YAML files
+- ZeroMQ event publisher for user events
+- MCP server with minimal "hello world" resource
+- Lifecycle management integration
+"""
+
+import asyncio
+import logging
+import os
+import signal
+import sys
+from pathlib import Path
+from typing import Optional
+
+import uvicorn
+import yaml
+from mcp.server import Server
+from mcp.server.sse import SseServerTransport
+from mcp.server.stdio import stdio_server
+from mcp.types import Tool, TextContent, ImageContent
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Route, Mount
+
+from dexorder import EventPublisher, start_lifecycle_manager, get_lifecycle_manager
+from dexorder.api import set_api, API
+from dexorder.conda_manager import sync_packages, install_packages
+from dexorder.events import EventType, UserEvent, DeliverySpec
+from dexorder.impl.charting_api_impl import ChartingAPIImpl
+from dexorder.impl.data_api_impl import DataAPIImpl
+from dexorder.tools.category_tools import get_category_manager
+from dexorder.tools.workspace_tools import get_workspace_store
+
+# =============================================================================
+# Global Data Directory
+# =============================================================================
+
+# Default data directory (relative to working directory for local dev)
+DEFAULT_DATA_DIR = Path("data")
+
+# Global data directory - set after config is loaded
+DATA_DIR: Path = DEFAULT_DATA_DIR
+
+
+def get_data_dir() -> Path:
+    """Get the global data directory."""
+    return DATA_DIR
+
+
+# =============================================================================
+# Configuration
+# =============================================================================
+
+class Config:
+    """Application configuration loaded from config.yaml and secrets.yaml"""
+
+    def __init__(self):
+        # User ID (required)
+        self.user_id: str = os.getenv("USER_ID", "")
+        if not self.user_id:
+            raise ValueError("USER_ID environment variable required")
+
+        # Config and secrets paths (k8s mounted)
+        self.config_path = Path(os.getenv("CONFIG_PATH", "/app/config/config.yaml"))
+        self.secrets_path = Path(os.getenv("SECRETS_PATH", "/app/config/secrets.yaml"))
+
+        # ZMQ ports for event system
+        self.zmq_xpub_port: int = int(os.getenv("ZMQ_XPUB_PORT", "5570"))
+        self.zmq_gateway_endpoint: str = os.getenv(
+            "ZMQ_GATEWAY_ENDPOINT",
+            "tcp://gateway:5571"
+        )
+
+        # MCP server settings
+        self.mcp_server_name: str = os.getenv("MCP_SERVER_NAME", "dexorder-user")
+        self.mcp_transport: str = os.getenv("MCP_TRANSPORT", "sse")  # "stdio" or "sse"
+        self.mcp_http_port: int = int(os.getenv("MCP_HTTP_PORT", "3000"))
+        self.mcp_http_host: str = os.getenv("MCP_HTTP_HOST", "0.0.0.0")
+
+        # Lifecycle settings
+        self.idle_timeout_minutes: int = int(os.getenv("IDLE_TIMEOUT_MINUTES", "15"))
+        self.enable_idle_shutdown: bool = os.getenv("ENABLE_IDLE_SHUTDOWN", "true").lower() == "true"
+
+        # Loaded from files
+        self.config_data: dict = {}
+        self.secrets_data: dict = {}
+
+        # Data directory (set after config load)
+        self.data_dir: Path = DEFAULT_DATA_DIR
+
+    def load(self) -> None:
+        """Load configuration and secrets from YAML files"""
+        global DATA_DIR
+
+        # Load config.yaml if exists
+        if self.config_path.exists():
+            with open(self.config_path) as f:
+                self.config_data = yaml.safe_load(f) or {}
+            logging.info(f"Loaded config from {self.config_path}")
+        else:
+            logging.warning(f"Config file not found: {self.config_path}")
+
+        # Load secrets.yaml if exists
+        if self.secrets_path.exists():
+            with open(self.secrets_path) as f:
+                self.secrets_data = yaml.safe_load(f) or {}
+            logging.info(f"Loaded secrets from {self.secrets_path}")
+        else:
+            logging.warning(f"Secrets file not found: {self.secrets_path}")
+
+        # Set data directory from config or environment
+        # Priority: env var > config file > default
+        data_dir_str = os.getenv("DATA_DIR") or self.config_data.get("data_dir")
+        if data_dir_str:
+            self.data_dir = Path(data_dir_str)
+        else:
+            self.data_dir = DEFAULT_DATA_DIR
+
+        # Update global DATA_DIR
+        DATA_DIR = self.data_dir
+
+        # Ensure data directory exists
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        logging.info(f"Data directory: {self.data_dir}")
+
+    @property
+    def workspace_dir(self) -> Path:
+        """Workspace directory under DATA_DIR."""
+        return self.data_dir / "workspace"
+
+
+# =============================================================================
+# MCP Server Setup
+# =============================================================================
+
+def create_mcp_server(config: Config, event_publisher: EventPublisher) -> Server:
+    """Create MCP server with resources and workspace tools"""
+
+    server = Server(config.mcp_server_name)
+
+    # Initialize workspace store
+    workspace_store = get_workspace_store(config.workspace_dir)
+    logging.info(f"Workspace store initialized at {config.workspace_dir}")
+
+    # Initialize category file manager
+    category_manager = get_category_manager(config.data_dir)
+    logging.info(f"Category manager initialized at {config.data_dir}")
+
+    @server.list_resources()
+    async def list_resources():
+        """List available resources"""
+        return [
+            {
+                "uri": f"dexorder://user/{config.user_id}/hello",
+                "name": "Hello World",
+                "description": "A simple hello world resource",
+                "mimeType": "text/plain",
+            }
+        ]
+
+    @server.read_resource()
+    async def read_resource(uri: str):
+        """Read a resource by URI"""
+        if uri == f"dexorder://user/{config.user_id}/hello":
+            # Publish an event when resource is accessed
+            await event_publisher.publish(UserEvent(
+                event_type=EventType.STRATEGY_LOG,
+                payload={
+                    "message": "Hello world resource accessed",
+                    "uri": uri,
+                },
+                delivery=DeliverySpec.informational(),
+            ))
+
+            return {
+                "uri": uri,
+                "mimeType": "text/plain",
+                "text": f"Hello from DexOrder user container!\nUser ID: {config.user_id}\n",
+            }
+        else:
+            raise ValueError(f"Unknown resource: {uri}")
+
+    @server.list_tools()
+    async def list_tools():
+        """List available tools including workspace and category tools"""
+        return [
+            Tool(
+                name="workspace_read",
+                description="Read a workspace store from persistent storage",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "store_name": {
+                            "type": "string",
+                            "description": "Name of the store (e.g., 'chartStore', 'userPreferences')"
+                        }
+                    },
+                    "required": ["store_name"]
+                }
+            ),
+            Tool(
+                name="workspace_write",
+                description="Write a workspace store to persistent storage",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "store_name": {
+                            "type": "string",
+                            "description": "Name of the store"
+                        },
+                        "data": {
+                            "description": "Data to write"
+                        }
+                    },
+                    "required": ["store_name", "data"]
+                }
+            ),
+            Tool(
+                name="workspace_patch",
+                description="Apply JSON patch operations to a workspace store",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "store_name": {
+                            "type": "string",
+                            "description": "Name of the store"
+                        },
+                        "patch": {
+                            "type": "array",
+                            "description": "JSON Patch operations (RFC 6902)",
+                            "items": {
+                                "type": "object",
+                                "properties": {
+                                    "op": {"type": "string", "enum": ["add", "remove", "replace", "move", "copy", "test"]},
+                                    "path": {"type": "string"},
+                                    "value": {}
+                                },
+                                "required": ["op", "path"]
+                            }
+                        }
+                    },
+                    "required": ["store_name", "patch"]
+                }
+            ),
+            Tool(
+                name="category_write",
+                description="Write a new strategy, indicator, or research script with validation",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "category": {
+                            "type": "string",
+                            "enum": ["strategy", "indicator", "research"],
+                            "description": "Category of the script"
+                        },
+                        "name": {
+                            "type": "string",
+                            "description": "Display name (can contain special characters)"
+                        },
+                        "description": {
+                            "type": "string",
+                            "description": "LLM-generated description of what this does (required)"
+                        },
+                        "code": {
+                            "type": "string",
+                            "description": "Python implementation code"
+                        },
+                        "metadata": {
+                            "type": "object",
+                            "description": "Optional category-specific metadata (e.g., default_length for indicators, data_feeds for strategies)"
+                        }
+                    },
+                    "required": ["category", "name", "description", "code"]
+                }
+            ),
+            Tool(
+                name="category_edit",
+                description="Edit an existing category script (updates code, description, or metadata)",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "category": {
+                            "type": "string",
+                            "enum": ["strategy", "indicator", "research"],
+                            "description": "Category of the script"
+                        },
+                        "name": {
+                            "type": "string",
+                            "description": "Display name of the existing item"
+                        },
+                        "code": {
+                            "type": "string",
+                            "description": "Updated Python code (optional, omit to keep existing)"
+                        },
+                        "description": {
+                            "type": "string",
+                            "description": "Updated description (optional, omit to keep existing)"
+                        },
+                        "metadata": {
+                            "type": "object",
+                            "description": "Updated metadata fields (optional)"
+                        }
+                    },
+                    "required": ["category", "name"]
+                }
+            ),
+            Tool(
+                name="category_read",
+                description="Read a category script and its metadata",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "category": {
+                            "type": "string",
+                            "enum": ["strategy", "indicator", "research"],
+                            "description": "Category of the script"
+                        },
+                        "name": {
+                            "type": "string",
+                            "description": "Display name of the item"
+                        }
+                    },
+                    "required": ["category", "name"]
+                }
+            ),
+            Tool(
+                name="category_list",
+                description="List all items in a category with names and descriptions",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "category": {
+                            "type": "string",
+                            "enum": ["strategy", "indicator", "research"],
+                            "description": "Category to list"
+                        }
+                    },
+                    "required": ["category"]
+                }
+            ),
+            Tool(
+                name="conda_sync",
+                description="Sync conda packages: scan all metadata, remove unused packages (excluding base environment)",
+                inputSchema={
+                    "type": "object",
+                    "properties": {},
+                    "required": []
+                }
+            ),
+            Tool(
+                name="conda_install",
+                description="Install conda packages on-demand",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "packages": {
+                            "type": "array",
+                            "items": {"type": "string"},
+                            "description": "List of conda package names to install"
+                        }
+                    },
+                    "required": ["packages"]
+                }
+            ),
+            Tool(
+                name="execute_research",
+                description="Execute a research script and return results with matplotlib images",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "name": {
+                            "type": "string",
+                            "description": "Display name of the research script to execute"
+                        }
+                    },
+                    "required": ["name"]
+                }
+            )
+        ]
+
+
+    @server.call_tool()
+    async def handle_tool_call(name: str, arguments: dict):
+        """Handle tool calls including workspace and category tools"""
+        if name == "workspace_read":
+            return workspace_store.read(arguments.get("store_name", ""))
+        elif name == "workspace_write":
+            return workspace_store.write(
+                arguments.get("store_name", ""),
+                arguments.get("data")
+            )
+        elif name == "workspace_patch":
+            return workspace_store.patch(
+                arguments.get("store_name", ""),
+                arguments.get("patch", [])
+            )
+        elif name == "category_write":
+            result = category_manager.write(
+                category=arguments.get("category", ""),
+                name=arguments.get("name", ""),
+                description=arguments.get("description", ""),
+                code=arguments.get("code", ""),
+                metadata=arguments.get("metadata")
+            )
+            content = []
+            meta_parts = [f"success: {result['success']}", f"path: {result['path']}"]
+            if result.get("validation") and not result["validation"].get("success"):
+                meta_parts.append(f"validation errors: {result['validation'].get('errors', [])}")
+            content.append(TextContent(type="text", text="\n".join(meta_parts)))
+            if result.get("execution"):
+                exec_content = result["execution"].get("content", [])
+                content.extend(exec_content)
+                image_count = sum(1 for item in exec_content if item.type == "image")
+                logging.info(f"category_write '{arguments.get('name')}': returning {len(content)} items, {image_count} images")
+            else:
+                logging.info(f"category_write '{arguments.get('name')}': no execution result (category={arguments.get('category')})")
+            return content
+        elif name == "category_edit":
+            result = category_manager.edit(
+                category=arguments.get("category", ""),
+                name=arguments.get("name", ""),
+                code=arguments.get("code"),
+                description=arguments.get("description"),
+                metadata=arguments.get("metadata")
+            )
+            content = []
+            meta_parts = [f"success: {result['success']}", f"path: {result['path']}"]
+            if result.get("validation") and not result["validation"].get("success"):
+                meta_parts.append(f"validation errors: {result['validation'].get('errors', [])}")
+            content.append(TextContent(type="text", text="\n".join(meta_parts)))
+            if result.get("execution"):
+                exec_content = result["execution"].get("content", [])
+                content.extend(exec_content)
+                image_count = sum(1 for item in exec_content if item.type == "image")
+                logging.info(f"category_edit '{arguments.get('name')}': returning {len(content)} items, {image_count} images")
+            else:
+                logging.info(f"category_edit '{arguments.get('name')}': no execution result")
+            return content
+        elif name == "category_read":
+            return category_manager.read(
+                category=arguments.get("category", ""),
+                name=arguments.get("name", "")
+            )
+        elif name == "category_list":
+            return category_manager.list_items(
+                category=arguments.get("category", "")
+            )
+        elif name == "conda_sync":
+            # Get environment.yml path relative to main.py
+            env_yml = Path(__file__).parent / "environment.yml"
+            return sync_packages(
+                data_dir=get_data_dir(),
+                environment_yml=env_yml if env_yml.exists() else None
+            )
+        elif name == "conda_install":
+            return install_packages(arguments.get("packages", []))
+        elif name == "execute_research":
+            result = category_manager.execute_research(name=arguments.get("name", ""))
+            if "error" in result:
+                logging.error(f"execute_research '{arguments.get('name')}': {result['error']}")
+                return [TextContent(type="text", text=f"Error: {result['error']}")]
+            content = result.get("content", [TextContent(type="text", text="No output")])
+            image_count = sum(1 for item in content if item.type == "image")
+            logging.info(f"execute_research '{arguments.get('name')}': returning {len(content)} items, {image_count} images")
+            return content
+        else:
+            raise ValueError(f"Unknown tool: {name}")
+
+    logging.info(f"MCP server '{config.mcp_server_name}' created with workspace and category tools")
+    return server
+
+
+# =============================================================================
+# SSE Transport Setup
+# =============================================================================
+
+def create_sse_app(mcp_server: Server) -> Starlette:
+    """Create Starlette app with SSE endpoint for MCP"""
+
+    # Create SSE transport instance
+    sse = SseServerTransport("/messages/")
+
+    async def handle_sse(request: Request) -> Response:
+        """Handle SSE connections for MCP"""
+        async with sse.connect_sse(
+            request.scope, request.receive, request._send
+        ) as streams:
+            await mcp_server.run(
+                streams[0],
+                streams[1],
+                mcp_server.create_initialization_options()
+            )
+        return Response()
+
+    async def handle_health(request: Request) -> Response:
+        """Health check endpoint for k8s probes and gateway readiness checks"""
+        return Response(
+            content='{"status":"ok"}',
+            media_type="application/json"
+        )
+
+    app = Starlette(
+        routes=[
+            Route("/sse", handle_sse),
+            Mount("/messages/", app=sse.handle_post_message),
+            Route("/health", handle_health),
+        ]
+    )
+
+    return app
+
+
+# =============================================================================
+# Main Application
+# =============================================================================
+
+class UserContainer:
+    """Main user container application"""
+
+    def __init__(self):
+        self.config = Config()
+        self.event_publisher: Optional[EventPublisher] = None
+        self.mcp_server: Optional[Server] = None
+        self.data_api: Optional[DataAPIImpl] = None
+        self.running = False
+
+    async def start(self) -> None:
+        """Start all subsystems"""
+        logging.info("Starting user container")
+
+        # Load configuration
+        self.config.load()
+
+        # Initialize data and charting API
+        data_cfg = self.config.config_data.get("data", {})
+        iceberg_cfg = data_cfg.get("iceberg", {})
+        relay_cfg = data_cfg.get("relay", {})
+        secrets = self.config.secrets_data
+        s3_cfg = iceberg_cfg  # S3 settings co-located with iceberg config
+
+        self.data_api = DataAPIImpl(
+            iceberg_catalog_uri=iceberg_cfg.get("catalog_uri", "http://iceberg-catalog:8181"),
+            relay_endpoint=relay_cfg.get("endpoint", "tcp://relay:5559"),
+            notification_endpoint=relay_cfg.get("notification_endpoint", "tcp://relay:5558"),
+            namespace=iceberg_cfg.get("namespace", "trading"),
+            s3_endpoint=s3_cfg.get("s3_endpoint") or secrets.get("s3_endpoint"),
+            s3_access_key=s3_cfg.get("s3_access_key") or secrets.get("s3_access_key"),
+            s3_secret_key=s3_cfg.get("s3_secret_key") or secrets.get("s3_secret_key"),
+        )
+        await self.data_api.start()
+        set_api(API(charting=ChartingAPIImpl(), data=self.data_api))
+        logging.info("API initialized")
+
+        # Start lifecycle manager
+        await start_lifecycle_manager(
+            user_id=self.config.user_id,
+            idle_timeout_minutes=self.config.idle_timeout_minutes,
+            enable_idle_shutdown=self.config.enable_idle_shutdown,
+        )
+        logging.info("Lifecycle manager started")
+
+        # Start event publisher
+        self.event_publisher = EventPublisher(
+            user_id=self.config.user_id,
+            xpub_port=self.config.zmq_xpub_port,
+            gateway_router_endpoint=self.config.zmq_gateway_endpoint,
+        )
+        await self.event_publisher.start()
+        logging.info("Event publisher started")
+
+        # Publish CONTAINER_STARTING event
+        await self.event_publisher.publish(UserEvent(
+            event_type=EventType.CONTAINER_STARTING,
+            payload={
+                "user_id": self.config.user_id,
+                "timestamp": None,  # Will be auto-filled
+            },
+            delivery=DeliverySpec.active_or_telegram(),
+        ))
+
+        # Create MCP server
+        self.mcp_server = create_mcp_server(self.config, self.event_publisher)
+
+        # Publish CONTAINER_READY event
+        await self.event_publisher.publish(UserEvent(
+            event_type=EventType.CONTAINER_READY,
+            payload={
+                "user_id": self.config.user_id,
+            },
+            delivery=DeliverySpec.active_or_telegram(),
+        ))
+
+        self.running = True
+        logging.info("User container ready")
+
+    async def stop(self) -> None:
+        """Stop all subsystems"""
+        if not self.running:
+            return
+
+        logging.info("Stopping user container")
+        self.running = False
+
+        # Publish CONTAINER_SHUTTING_DOWN event
+        if self.event_publisher:
+            await self.event_publisher.publish(UserEvent(
+                event_type=EventType.CONTAINER_SHUTTING_DOWN,
+                payload={
+                    "user_id": self.config.user_id,
+                },
+                delivery=DeliverySpec.active_or_telegram(),
+            ))
+
+        # Stop subsystems
+        if self.data_api:
+            await self.data_api.stop()
+            logging.info("Data API stopped")
+
+        if self.event_publisher:
+            await self.event_publisher.stop()
+            logging.info("Event publisher stopped")
+
+        lifecycle = get_lifecycle_manager()
+        if lifecycle:
+            await lifecycle.stop()
+            logging.info("Lifecycle manager stopped")
+
+        logging.info("User container stopped")
+
+    async def run(self) -> None:
+        """Run the MCP server with configured transport"""
+        await self.start()
+
+        try:
+            if self.config.mcp_transport == "stdio":
+                # Run MCP server on stdio (for dev/testing)
+                logging.info("Starting MCP server with stdio transport")
+                async with stdio_server() as (read_stream, write_stream):
+                    await self.mcp_server.run(
+                        read_stream,
+                        write_stream,
+                        self.mcp_server.create_initialization_options()
+                    )
+            elif self.config.mcp_transport == "sse":
+                # Run MCP server via HTTP/SSE (for production)
+                logging.info(f"Starting MCP server with SSE transport on {self.config.mcp_http_host}:{self.config.mcp_http_port}")
+                app = create_sse_app(self.mcp_server)
+                config = uvicorn.Config(
+                    app,
+                    host=self.config.mcp_http_host,
+                    port=self.config.mcp_http_port,
+                    log_level=os.getenv("LOG_LEVEL", "info").lower(),
+                    access_log=True,
+                )
+                server = uvicorn.Server(config)
+                await server.serve()
+            else:
+                raise ValueError(f"Unknown MCP transport: {self.config.mcp_transport}")
+        finally:
+            await self.stop()
+
+
+# =============================================================================
+# Entry Point
+# =============================================================================
+
+async def main():
+    """Main entry point"""
+    # Setup logging
+    log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+    logging.basicConfig(
+        level=getattr(logging, log_level),
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        stream=sys.stderr,  # MCP uses stdout for protocol
+    )
+
+    # Create and run container
+    container = UserContainer()
+
+    # Handle shutdown signals
+    loop = asyncio.get_event_loop()
+
+    def handle_signal(sig):
+        logging.info(f"Received signal {sig}, shutting down...")
+        asyncio.create_task(container.stop())
+        loop.stop()
+
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        loop.add_signal_handler(sig, lambda s=sig: handle_signal(s))
+
+    try:
+        await container.run()
+    except KeyboardInterrupt:
+        logging.info("Keyboard interrupt received")
+    except Exception as e:
+        logging.error(f"Fatal error: {e}", exc_info=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

sandbox/protobuf/ingestor.proto

@@ -0,0 +1,329 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "com.dexorder.proto";
+
+// Request for data ingestion (used in Relay → Ingestor work queue)
+message DataRequest {
+  // Unique request ID for tracking
+  string request_id = 1;
+
+  // Type of request
+  RequestType type = 2;
+
+  // Market identifier
+  string ticker = 3;
+
+  // For historical requests
+  optional HistoricalParams historical = 4;
+
+  // For realtime requests
+  optional RealtimeParams realtime = 5;
+
+  // Optional client ID for notification routing (async architecture)
+  // Flink uses this to determine notification topic
+  optional string client_id = 6;
+
+  enum RequestType {
+    HISTORICAL_OHLC = 0;
+    REALTIME_TICKS = 1;
+  }
+}
+
+message HistoricalParams {
+  // Start time (microseconds since epoch)
+  uint64 start_time = 1;
+
+  // End time (microseconds since epoch)
+  uint64 end_time = 2;
+
+  // OHLC period in seconds (e.g., 60 = 1m, 300 = 5m, 3600 = 1h, 86400 = 1d)
+  uint32 period_seconds = 3;
+
+  // Maximum number of candles to return (optional limit)
+  optional uint32 limit = 4;
+}
+
+message RealtimeParams {
+  // Whether to include tick data
+  bool include_ticks = 1;
+
+  // Whether to include aggregated OHLC
+  bool include_ohlc = 2;
+
+  // OHLC periods to generate in seconds (e.g., [60, 300, 900] for 1m, 5m, 15m)
+  repeated uint32 ohlc_period_seconds = 3;
+}
+
+// Control messages for ingestors (Flink → Ingestor control channel)
+message IngestorControl {
+  // Control action type
+  ControlAction action = 1;
+
+  // Request ID to cancel (for CANCEL action)
+  optional string request_id = 2;
+
+  // Configuration updates (for CONFIG_UPDATE action)
+  optional IngestorConfig config = 3;
+
+  enum ControlAction {
+    CANCEL = 0;          // Cancel a specific request
+    SHUTDOWN = 1;        // Graceful shutdown signal
+    CONFIG_UPDATE = 2;   // Update ingestor configuration
+    HEARTBEAT = 3;       // Keep-alive signal
+  }
+}
+
+message IngestorConfig {
+  // Maximum concurrent requests per ingestor
+  optional uint32 max_concurrent = 1;
+
+  // Request timeout in seconds
+  optional uint32 timeout_seconds = 2;
+
+  // Kafka topic for output
+  optional string kafka_topic = 3;
+}
+
+// Historical data response from ingestor to Flink (Ingestor → Flink response channel)
+message DataResponse {
+  // Request ID this is responding to
+  string request_id = 1;
+
+  // Status of the request
+  ResponseStatus status = 2;
+
+  // Error message if status is not OK
+  optional string error_message = 3;
+
+  // Serialized OHLC data (repeated OHLCV protobuf messages)
+  repeated bytes ohlc_data = 4;
+
+  // Total number of candles returned
+  uint32 total_records = 5;
+
+  enum ResponseStatus {
+    OK = 0;
+    NOT_FOUND = 1;
+    ERROR = 2;
+  }
+}
+
+// Client request submission for historical OHLC data (Client → Relay)
+// Relay immediately responds with SubmitResponse containing request_id
+message SubmitHistoricalRequest {
+  // Client-generated request ID for tracking
+  string request_id = 1;
+
+  // Market identifier (e.g., "BINANCE:BTC/USDT")
+  string ticker = 2;
+
+  // Start time (microseconds since epoch)
+  uint64 start_time = 3;
+
+  // End time (microseconds since epoch)
+  uint64 end_time = 4;
+
+  // OHLC period in seconds (e.g., 60 = 1m, 300 = 5m, 3600 = 1h)
+  uint32 period_seconds = 5;
+
+  // Optional limit on number of candles
+  optional uint32 limit = 6;
+
+  // Optional client ID for notification routing (e.g., "client-abc-123")
+  // Notifications will be published to topic: "RESPONSE:{client_id}"
+  optional string client_id = 7;
+}
+
+// Immediate response to SubmitHistoricalRequest (Relay → Client)
+message SubmitResponse {
+  // Request ID (echoed from request)
+  string request_id = 1;
+
+  // Status of submission
+  SubmitStatus status = 2;
+
+  // Error message if status is not QUEUED
+  optional string error_message = 3;
+
+  // Topic to subscribe to for result notification
+  // e.g., "RESPONSE:client-abc-123" or "HISTORY_READY:{request_id}"
+  string notification_topic = 4;
+
+  enum SubmitStatus {
+    QUEUED = 0;       // Request queued successfully
+    DUPLICATE = 1;    // Request ID already exists
+    INVALID = 2;      // Invalid parameters
+    ERROR = 3;        // Internal error
+  }
+}
+
+// Historical data ready notification (Flink → Relay → Client via pub/sub)
+// Published after Flink writes data to Iceberg
+message HistoryReadyNotification {
+  // Request ID
+  string request_id = 1;
+
+  // Market identifier
+  string ticker = 2;
+
+  // OHLC period in seconds
+  uint32 period_seconds = 3;
+
+  // Start time (microseconds since epoch)
+  uint64 start_time = 4;
+
+  // End time (microseconds since epoch)
+  uint64 end_time = 5;
+
+  // Status of the data fetch
+  NotificationStatus status = 6;
+
+  // Error message if status is not OK
+  optional string error_message = 7;
+
+  // Iceberg table information for client queries
+  string iceberg_namespace = 10;
+  string iceberg_table = 11;
+
+  // Number of records written
+  uint32 row_count = 12;
+
+  // Timestamp when data was written (microseconds since epoch)
+  uint64 completed_at = 13;
+
+  enum NotificationStatus {
+    OK = 0;          // Data successfully written to Iceberg
+    NOT_FOUND = 1;   // No data found for the requested period
+    ERROR = 2;       // Error during fetch or processing
+    TIMEOUT = 3;     // Request timed out
+  }
+}
+
+// Legacy message for backward compatibility (Client → Relay)
+message OHLCRequest {
+  // Request ID for tracking
+  string request_id = 1;
+
+  // Market identifier
+  string ticker = 2;
+
+  // Start time (microseconds since epoch)
+  uint64 start_time = 3;
+
+  // End time (microseconds since epoch)
+  uint64 end_time = 4;
+
+  // OHLC period in seconds (e.g., 60 = 1m, 300 = 5m, 3600 = 1h)
+  uint32 period_seconds = 5;
+
+  // Optional limit on number of candles
+  optional uint32 limit = 6;
+}
+
+// Generic response for any request (Flink → Client)
+message Response {
+  // Request ID this is responding to
+  string request_id = 1;
+
+  // Status of the request
+  ResponseStatus status = 2;
+
+  // Error message if status is not OK
+  optional string error_message = 3;
+
+  // Generic payload data (serialized protobuf messages)
+  repeated bytes data = 4;
+
+  // Total number of records
+  optional uint32 total_records = 5;
+
+  // Whether this is the final response (for paginated results)
+  bool is_final = 6;
+
+  enum ResponseStatus {
+    OK = 0;
+    NOT_FOUND = 1;
+    ERROR = 2;
+  }
+}
+
+// CEP trigger registration (Client → Flink)
+message CEPTriggerRequest {
+  // Unique trigger ID
+  string trigger_id = 1;
+
+  // Flink SQL CEP pattern/condition
+  string sql_pattern = 2;
+
+  // Markets to monitor
+  repeated string tickers = 3;
+
+  // Callback endpoint (for DEALER/ROUTER routing)
+  optional string callback_id = 4;
+
+  // Optional parameters for the CEP query
+  map<string, string> parameters = 5;
+}
+
+// CEP trigger acknowledgment (Flink → Client)
+message CEPTriggerAck {
+  // Trigger ID being acknowledged
+  string trigger_id = 1;
+
+  // Status of registration
+  TriggerStatus status = 2;
+
+  // Error message if status is not OK
+  optional string error_message = 3;
+
+  enum TriggerStatus {
+    REGISTERED = 0;
+    ALREADY_REGISTERED = 1;
+    INVALID_SQL = 2;
+    ERROR = 3;
+  }
+}
+
+// CEP trigger event callback (Flink → Client)
+message CEPTriggerEvent {
+  // Trigger ID that fired
+  string trigger_id = 1;
+
+  // Timestamp when trigger fired (microseconds since epoch)
+  uint64 timestamp = 2;
+
+  // Schema information for the result rows
+  ResultSchema schema = 3;
+
+  // Result rows from the Flink SQL query
+  repeated ResultRow rows = 4;
+
+  // Additional context from the CEP pattern
+  map<string, string> context = 5;
+}
+
+message ResultSchema {
+  // Column names in order
+  repeated string column_names = 1;
+
+  // Column types (using Flink SQL type names)
+  repeated string column_types = 2;
+}
+
+message ResultRow {
+  // Encoded row data (one bytes field per column, in schema order)
+  // Each value is encoded as a protobuf-serialized FieldValue
+  repeated bytes values = 1;
+}
+
+message FieldValue {
+  oneof value {
+    string string_val = 1;
+    int64 int_val = 2;
+    double double_val = 3;
+    bool bool_val = 4;
+    bytes bytes_val = 5;
+    uint64 timestamp_val = 6;
+  }
+}

sandbox/protobuf/market.proto

@@ -0,0 +1,22 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "com.dexorder.proto";
+
+message Market {
+  // The prices and volumes must be adjusted by the rational denominator provided
+  // by the market metadata
+  string exchange_id = 2;  // e.g., BINANCE
+  string market_id = 3;  // e.g., BTC/USDT
+  string market_type = 4;  // e.g., Spot
+  string description = 5;  // e.g., Bitcoin/Tether on Binance
+  repeated string column_names = 6;  // e.g., ['open', 'high', 'low', 'close', 'volume', 'taker_vol', 'maker_vol']
+  string base_asset = 9;
+  string quote_asset = 10;
+  uint64 earliest_time = 11;
+  uint64 tick_denom = 12; // denominator applied to all OHLC price data
+  uint64 base_denom = 13; // denominator applied to base asset units
+  uint64 quote_denom = 14; // denominator applied to quote asset units
+  repeated uint32 supported_period_seconds = 15;
+
+}

sandbox/protobuf/ohlc.proto

@@ -0,0 +1,61 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "com.dexorder.proto";
+
+// Single OHLC row
+message OHLC {
+  // Timestamp in microseconds since epoch
+  uint64 timestamp = 1;
+
+  // The prices and volumes must be adjusted by the rational denominator provided
+  // by the market metadata
+  int64 open = 2;
+  int64 high = 3;
+  int64 low = 4;
+  int64 close = 5;
+  optional int64 volume = 6;
+  optional int64 buy_vol = 7;
+  optional int64 sell_vol = 8;
+  optional int64 open_time = 9;
+  optional int64 high_time = 10;
+  optional int64 low_time = 11;
+  optional int64 close_time = 12;
+  optional int64 open_interest = 13;
+  string ticker = 14;
+}
+
+// Batch of OHLC rows with metadata for historical request tracking
+// Used for Kafka messages from ingestor → Flink
+message OHLCBatch {
+  // Metadata for tracking this request through the pipeline
+  OHLCBatchMetadata metadata = 1;
+
+  // OHLC rows in this batch
+  repeated OHLC rows = 2;
+}
+
+// Metadata for tracking historical data requests through the pipeline
+message OHLCBatchMetadata {
+  // Request ID from client
+  string request_id = 1;
+
+  // Optional client ID for notification routing
+  optional string client_id = 2;
+
+  // Market identifier
+  string ticker = 3;
+
+  // OHLC period in seconds
+  uint32 period_seconds = 4;
+
+  // Time range requested (microseconds since epoch)
+  uint64 start_time = 5;
+  uint64 end_time = 6;
+
+  // Status for marker messages (OK, NOT_FOUND, ERROR)
+  string status = 7;
+
+  // Error message if status is ERROR
+  optional string error_message = 8;
+}

sandbox/protobuf/tick.proto

@@ -0,0 +1,51 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "com.dexorder.proto";
+
+message Tick {
+  // Unique identifier for the trade
+  string trade_id = 1;
+
+  // Market identifier (matches Market.market_id)
+  string ticker = 2;
+
+  // Timestamp in microseconds since epoch
+  uint64 timestamp = 3;
+
+  // Price (must be adjusted by tick_denom from Market metadata)
+  int64 price = 4;
+
+  // Base asset amount (must be adjusted by base_denom from Market metadata)
+  int64 amount = 5;
+
+  // Quote asset amount (must be adjusted by quote_denom from Market metadata)
+  int64 quote_amount = 6;
+
+  // Side: true = taker buy (market buy), false = taker sell (market sell)
+  bool taker_buy = 7;
+
+  // Position effect: true = close position, false = open position
+  // Only relevant for derivatives/futures markets
+  optional bool to_close = 8;
+
+  // Sequence number for ordering (if provided by exchange)
+  optional uint64 sequence = 9;
+
+  // Additional flags for special trade types
+  optional TradeFlags flags = 10;
+}
+
+message TradeFlags {
+  // Liquidation trade
+  bool is_liquidation = 1;
+
+  // Block trade (large OTC trade)
+  bool is_block_trade = 2;
+
+  // Maker side was a post-only order
+  bool maker_post_only = 3;
+
+  // Trade occurred during auction
+  bool is_auction = 4;
+}

sandbox/protobuf/user_events.proto

@@ -0,0 +1,258 @@
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "com.dexorder.proto";
+
+// User container event system for delivering notifications to users
+// via active sessions or external channels (Telegram, email, push).
+//
+// Two ZMQ patterns:
+// - XPUB/SUB (port 5570): Fast path for informational events to active sessions
+// - DEALER/ROUTER (port 5571): Guaranteed delivery for critical events with ack
+//
+// See doc/protocol.md and doc/user_container_events.md for details.
+
+// =============================================================================
+// User Event (Container → Gateway)
+// Message Type ID: 0x20
+// =============================================================================
+
+message UserEvent {
+  // User ID this event belongs to
+  string user_id = 1;
+
+  // Unique event ID for deduplication and ack tracking (UUID)
+  string event_id = 2;
+
+  // Timestamp when event was generated (Unix milliseconds)
+  int64 timestamp = 3;
+
+  // Type of event
+  EventType event_type = 4;
+
+  // Event payload (JSON or nested protobuf, depending on event_type)
+  bytes payload = 5;
+
+  // Delivery specification (priority and channel preferences)
+  DeliverySpec delivery = 6;
+}
+
+enum EventType {
+  // Trading events
+  ORDER_PLACED = 0;
+  ORDER_FILLED = 1;
+  ORDER_CANCELLED = 2;
+  ORDER_REJECTED = 3;
+  ORDER_EXPIRED = 4;
+
+  // Alert events
+  ALERT_TRIGGERED = 10;
+  ALERT_CREATED = 11;
+  ALERT_DELETED = 12;
+
+  // Position events
+  POSITION_OPENED = 20;
+  POSITION_CLOSED = 21;
+  POSITION_UPDATED = 22;
+  POSITION_LIQUIDATED = 23;
+
+  // Workspace/chart events
+  WORKSPACE_CHANGED = 30;
+  CHART_ANNOTATION_ADDED = 31;
+  CHART_ANNOTATION_REMOVED = 32;
+  INDICATOR_UPDATED = 33;
+
+  // Strategy events
+  STRATEGY_STARTED = 40;
+  STRATEGY_STOPPED = 41;
+  STRATEGY_LOG = 42;
+  STRATEGY_ERROR = 43;
+  BACKTEST_COMPLETED = 44;
+
+  // System events
+  CONTAINER_STARTING = 50;
+  CONTAINER_READY = 51;
+  CONTAINER_SHUTTING_DOWN = 52;
+  EVENT_ERROR = 53;
+}
+
+// =============================================================================
+// Delivery Specification
+// =============================================================================
+
+message DeliverySpec {
+  // Priority determines routing behavior
+  Priority priority = 1;
+
+  // Ordered list of channel preferences (try first, then second, etc.)
+  repeated ChannelPreference channels = 2;
+}
+
+enum Priority {
+  // Drop if no active session (fire-and-forget via XPUB)
+  // Use for: indicator updates, chart syncs, strategy logs when watching
+  INFORMATIONAL = 0;
+
+  // Best effort delivery - queue briefly, deliver when possible
+  // Uses XPUB if subscribed, otherwise DEALER
+  // Use for: alerts, position updates
+  NORMAL = 1;
+
+  // Must deliver - retry until acked, escalate channels
+  // Always uses DEALER for guaranteed delivery
+  // Use for: order fills, liquidations, critical errors
+  CRITICAL = 2;
+}
+
+message ChannelPreference {
+  // Channel to deliver to
+  ChannelType channel = 1;
+
+  // If true, skip this channel if user is not connected to it
+  // If false, deliver even if user is not actively connected
+  // (e.g., send Telegram message even if user isn't in Telegram chat)
+  bool only_if_active = 2;
+}
+
+enum ChannelType {
+  // Whatever channel the user currently has open (WebSocket, Telegram session)
+  ACTIVE_SESSION = 0;
+
+  // Specific channels
+  WEB = 1;           // WebSocket to web UI
+  TELEGRAM = 2;      // Telegram bot message
+  EMAIL = 3;         // Email notification
+  PUSH = 4;          // Mobile push notification (iOS/Android)
+  DISCORD = 5;       // Discord webhook (future)
+  SLACK = 6;         // Slack webhook (future)
+}
+
+// =============================================================================
+// Event Acknowledgment (Gateway → Container)
+// Message Type ID: 0x21
+// =============================================================================
+
+message EventAck {
+  // Event ID being acknowledged
+  string event_id = 1;
+
+  // Delivery status
+  AckStatus status = 2;
+
+  // Error message if status is ERROR
+  string error_message = 3;
+
+  // Channel that successfully delivered (for logging/debugging)
+  ChannelType delivered_via = 4;
+}
+
+enum AckStatus {
+  // Successfully delivered to at least one channel
+  DELIVERED = 0;
+
+  // Accepted and queued for delivery (e.g., rate limited, will retry)
+  QUEUED = 1;
+
+  // Permanent failure - all channels failed
+  ACK_ERROR = 2;
+}
+
+// =============================================================================
+// Event Payloads
+// These are JSON-encoded in the UserEvent.payload field.
+// Defined here for documentation; actual encoding is JSON for flexibility.
+// =============================================================================
+
+// Payload for ORDER_PLACED, ORDER_FILLED, ORDER_CANCELLED, etc.
+message OrderEventPayload {
+  string order_id = 1;
+  string symbol = 2;
+  string side = 3;              // "buy" or "sell"
+  string order_type = 4;        // "market", "limit", "stop_limit", etc.
+  string quantity = 5;          // Decimal string
+  string price = 6;             // Decimal string (for limit orders)
+  string fill_price = 7;        // Decimal string (for fills)
+  string fill_quantity = 8;     // Decimal string (for partial fills)
+  string status = 9;            // "open", "filled", "cancelled", etc.
+  string exchange = 10;
+  int64 timestamp = 11;         // Unix milliseconds
+  string strategy_id = 12;      // If order was placed by a strategy
+  string error_message = 13;    // If rejected/failed
+}
+
+// Payload for ALERT_TRIGGERED
+message AlertEventPayload {
+  string alert_id = 1;
+  string symbol = 2;
+  string condition = 3;         // Human-readable condition (e.g., "BTC > 50000")
+  string triggered_price = 4;   // Decimal string
+  int64 timestamp = 5;
+}
+
+// Payload for POSITION_OPENED, POSITION_CLOSED, POSITION_UPDATED
+message PositionEventPayload {
+  string position_id = 1;
+  string symbol = 2;
+  string side = 3;              // "long" or "short"
+  string size = 4;              // Decimal string
+  string entry_price = 5;       // Decimal string
+  string current_price = 6;     // Decimal string
+  string unrealized_pnl = 7;    // Decimal string
+  string realized_pnl = 8;      // Decimal string (for closed positions)
+  string leverage = 9;          // Decimal string (for margin)
+  string liquidation_price = 10;
+  string exchange = 11;
+  int64 timestamp = 12;
+}
+
+// Payload for WORKSPACE_CHANGED, CHART_ANNOTATION_*, INDICATOR_UPDATED
+message WorkspaceEventPayload {
+  string workspace_id = 1;
+  string change_type = 2;       // "symbol_changed", "timeframe_changed", "annotation_added", etc.
+  string symbol = 3;
+  string timeframe = 4;
+
+  // For annotations
+  string annotation_id = 5;
+  string annotation_type = 6;   // "trendline", "horizontal", "rectangle", "text", etc.
+  string annotation_data = 7;   // JSON string with coordinates, style, etc.
+
+  // For indicators
+  string indicator_name = 8;
+  string indicator_params = 9;  // JSON string with indicator parameters
+
+  int64 timestamp = 10;
+}
+
+// Payload for STRATEGY_LOG, STRATEGY_ERROR
+message StrategyEventPayload {
+  string strategy_id = 1;
+  string strategy_name = 2;
+  string log_level = 3;         // "debug", "info", "warn", "error"
+  string message = 4;
+  string details = 5;           // JSON string with additional context
+  int64 timestamp = 6;
+}
+
+// Payload for BACKTEST_COMPLETED
+message BacktestEventPayload {
+  string backtest_id = 1;
+  string strategy_id = 2;
+  string strategy_name = 3;
+  string symbol = 4;
+  string timeframe = 5;
+  int64 start_time = 6;
+  int64 end_time = 7;
+
+  // Results summary
+  int32 total_trades = 8;
+  int32 winning_trades = 9;
+  int32 losing_trades = 10;
+  string total_pnl = 11;        // Decimal string
+  string win_rate = 12;         // Decimal string (0-1)
+  string sharpe_ratio = 13;     // Decimal string
+  string max_drawdown = 14;     // Decimal string (0-1)
+
+  string results_path = 15;     // Path to full results file
+  int64 completed_at = 16;
+}

sandbox/secrets.example.yaml

@@ -0,0 +1,25 @@
+# Example secrets file for DexOrder user container
+# Mount this at /app/config/secrets.yaml in k8s
+# This file should be created from k8s secrets
+
+# Iceberg catalog credentials
+iceberg:
+  s3:
+    endpoint: "http://minio:9000"
+    access_key_id: "minioadmin"
+    secret_access_key: "minioadmin"
+    region: "us-east-1"
+
+  catalog:
+    uri: "http://iceberg-rest:8181"
+    warehouse: "s3://warehouse/"
+
+# API keys for external services (if needed)
+api_keys:
+  telegram_bot_token: ""
+
+# Internal auth (for mode A - platform harness)
+auth:
+  mode: "mtls"  # or "platform_token" or "api_key"
+  # API key hash if using api_key mode
+  api_key_hash: ""

sandbox/setup.py

@@ -0,0 +1,30 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="dexorder-sandbox",
+    version="0.1.0",
+    description="DexOrder Trading Platform Sandbox",
+    packages=find_packages(),
+    python_requires=">=3.9",
+    install_requires=[
+        "pyiceberg>=0.6.0",
+        "pyarrow>=14.0.0",
+        "pandas>=2.0.0",
+        "pyzmq>=25.0.0",
+        "protobuf>=4.25.0",
+        "pyyaml>=6.0",
+        "aiofiles>=23.0.0",
+        "mcp>=1.0.0",
+        "jsonpatch>=1.33",
+        "starlette>=0.27.0",
+        "uvicorn>=0.27.0",
+        "sse-starlette>=1.6.0",
+        "matplotlib>=3.7.0",
+    ],
+    extras_require={
+        "dev": [
+            "pytest>=7.0.0",
+            "pytest-asyncio>=0.21.0",
+        ]
+    },
+)