sandbox connected and streaming
This commit is contained in:
@@ -15,6 +15,8 @@ import { KubernetesClient } from './k8s/client.js';
|
||||
import { ContainerManager } from './k8s/container-manager.js';
|
||||
import { ZMQRelayClient } from './clients/zmq-relay-client.js';
|
||||
import { IcebergClient } from './clients/iceberg-client.js';
|
||||
import { ConversationStore } from './harness/memory/conversation-store.js';
|
||||
import { AgentHarness, type HarnessSessionConfig } from './harness/agent-harness.js';
|
||||
import { OHLCService } from './services/ohlc-service.js';
|
||||
import { SymbolIndexService } from './services/symbol-index-service.js';
|
||||
import { SymbolRoutes } from './routes/symbol-routes.js';
|
||||
@@ -38,6 +40,7 @@ import {
|
||||
} from './events/index.js';
|
||||
import { QdrantClient } from './clients/qdrant-client.js';
|
||||
import { EmbeddingService, RAGRetriever, DocumentLoader } from './harness/memory/index.js';
|
||||
import { initializeToolRegistry } from './tools/tool-registry.js';
|
||||
import { join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname } from 'path';
|
||||
@@ -131,6 +134,9 @@ function loadConfig() {
|
||||
// Redis configuration (for harness memory layer)
|
||||
redisUrl: configData.redis?.url || process.env.REDIS_URL || 'redis://localhost:6379',
|
||||
|
||||
// Conversation history limit: number of prior turns loaded as LLM context and flushed to Iceberg
|
||||
conversationHistoryLimit: configData.agent?.conversation_history_limit || parseInt(process.env.CONVERSATION_HISTORY_LIMIT || '20'),
|
||||
|
||||
// Qdrant configuration (for RAG)
|
||||
qdrant: {
|
||||
url: configData.qdrant?.url || process.env.QDRANT_URL || 'http://localhost:6333',
|
||||
@@ -147,6 +153,7 @@ function loadConfig() {
|
||||
s3Endpoint: configData.iceberg?.s3_endpoint || process.env.S3_ENDPOINT,
|
||||
s3AccessKey: secretsData.iceberg?.s3_access_key || process.env.S3_ACCESS_KEY,
|
||||
s3SecretKey: secretsData.iceberg?.s3_secret_key || process.env.S3_SECRET_KEY,
|
||||
conversationsBucket: configData.iceberg?.conversations_bucket || process.env.CONVERSATIONS_S3_BUCKET,
|
||||
},
|
||||
|
||||
// Relay configuration (for historical data)
|
||||
@@ -165,12 +172,12 @@ function loadConfig() {
|
||||
|
||||
// Kubernetes configuration
|
||||
kubernetes: {
|
||||
namespace: configData.kubernetes?.namespace || process.env.KUBERNETES_NAMESPACE || 'dexorder-agents',
|
||||
namespace: configData.kubernetes?.namespace || process.env.KUBERNETES_NAMESPACE || 'dexorder-sandboxes',
|
||||
inCluster: configData.kubernetes?.in_cluster ?? (process.env.KUBERNETES_IN_CLUSTER === 'true'),
|
||||
context: configData.kubernetes?.context || process.env.KUBERNETES_CONTEXT,
|
||||
agentImage: configData.kubernetes?.agent_image || process.env.AGENT_IMAGE || 'ghcr.io/dexorder/agent:latest',
|
||||
sandboxImage: configData.kubernetes?.sandbox_image || process.env.SANDBOX_IMAGE || 'ghcr.io/dexorder/sandbox:latest',
|
||||
sidecarImage: configData.kubernetes?.sidecar_image || process.env.SIDECAR_IMAGE || 'ghcr.io/dexorder/lifecycle-sidecar:latest',
|
||||
storageClass: configData.kubernetes?.storage_class || process.env.AGENT_STORAGE_CLASS || 'standard',
|
||||
storageClass: configData.kubernetes?.storage_class || process.env.SANDBOX_STORAGE_CLASS || 'standard',
|
||||
imagePullPolicy: configData.kubernetes?.image_pull_policy || process.env.IMAGE_PULL_POLICY || 'Always',
|
||||
},
|
||||
};
|
||||
@@ -261,11 +268,25 @@ const qdrantClient = new QdrantClient(config.qdrant, app.log);
|
||||
// Initialize Iceberg client (for durable storage)
|
||||
// const icebergClient = new IcebergClient(config.iceberg, app.log);
|
||||
|
||||
// Create metadata update callback that will be wired up when SymbolIndexService initializes
|
||||
// This ensures we don't miss notifications sent before the service is ready
|
||||
let symbolIndexService: SymbolIndexService | undefined;
|
||||
const onMetadataUpdate = async () => {
|
||||
if (symbolIndexService) {
|
||||
app.log.info('Reloading symbol metadata from Iceberg');
|
||||
await symbolIndexService.initialize();
|
||||
app.log.info({ stats: symbolIndexService.getStats() }, 'Symbol metadata reloaded');
|
||||
} else {
|
||||
app.log.warn('Received METADATA_UPDATE before SymbolIndexService initialized, ignoring');
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize ZMQ Relay client (for historical data)
|
||||
// Note: onMetadataUpdate callback will be set after symbolIndexService is initialized
|
||||
// Pass onMetadataUpdate callback so it's registered before connection
|
||||
const zmqRelayClient = new ZMQRelayClient({
|
||||
relayRequestEndpoint: config.relay.requestEndpoint,
|
||||
relayNotificationEndpoint: config.relay.notificationEndpoint,
|
||||
onMetadataUpdate,
|
||||
}, app.log);
|
||||
|
||||
app.log.info({
|
||||
@@ -286,7 +307,7 @@ const k8sClient = new KubernetesClient({
|
||||
|
||||
const containerManager = new ContainerManager({
|
||||
k8sClient,
|
||||
agentImage: config.kubernetes.agentImage,
|
||||
sandboxImage: config.kubernetes.sandboxImage,
|
||||
sidecarImage: config.kubernetes.sidecarImage,
|
||||
storageClass: config.kubernetes.storageClass,
|
||||
imagePullPolicy: config.kubernetes.imagePullPolicy,
|
||||
@@ -326,10 +347,13 @@ const eventRouter = new EventRouter({
|
||||
});
|
||||
app.log.debug('Event router initialized');
|
||||
|
||||
// Initialize shared Iceberg client (used by both OHLC service and conversation store)
|
||||
const icebergClient = new IcebergClient(config.iceberg, app.log);
|
||||
app.log.debug('Iceberg client initialized');
|
||||
|
||||
// Initialize OHLC service (optional - only if relay is available)
|
||||
let ohlcService: OHLCService | undefined;
|
||||
try {
|
||||
const icebergClient = new IcebergClient(config.iceberg, app.log);
|
||||
ohlcService = new OHLCService({
|
||||
icebergClient,
|
||||
relayClient: zmqRelayClient,
|
||||
@@ -340,16 +364,30 @@ try {
|
||||
app.log.warn({ error }, 'Failed to initialize OHLC service - historical data will not be available');
|
||||
}
|
||||
|
||||
// Initialize Symbol Index Service (deferred to after server starts)
|
||||
let symbolIndexService: SymbolIndexService | undefined;
|
||||
// Initialize conversation store (Redis hot path + Iceberg cold path)
|
||||
const conversationStore = new ConversationStore(redis, app.log, icebergClient);
|
||||
app.log.debug('Conversation store initialized');
|
||||
|
||||
// Harness factory: captures infrastructure deps; channel handlers stay infrastructure-free
|
||||
function createHarness(sessionConfig: HarnessSessionConfig): AgentHarness {
|
||||
return new AgentHarness({
|
||||
...sessionConfig,
|
||||
providerConfig: config.providerConfig,
|
||||
conversationStore,
|
||||
historyLimit: config.conversationHistoryLimit,
|
||||
});
|
||||
}
|
||||
|
||||
// Symbol Index Service will be initialized after server starts
|
||||
// (declared above near ZMQ client initialization)
|
||||
|
||||
// Initialize channel handlers
|
||||
const websocketHandler = new WebSocketHandler({
|
||||
authenticator,
|
||||
containerManager,
|
||||
providerConfig: config.providerConfig,
|
||||
sessionRegistry,
|
||||
eventSubscriber,
|
||||
createHarness,
|
||||
ohlcService, // Optional
|
||||
symbolIndexService, // Optional
|
||||
});
|
||||
@@ -357,8 +395,8 @@ app.log.debug('WebSocket handler initialized');
|
||||
|
||||
const telegramHandler = new TelegramHandler({
|
||||
authenticator,
|
||||
providerConfig: config.providerConfig,
|
||||
telegramBotToken: config.telegramBotToken,
|
||||
createHarness,
|
||||
});
|
||||
app.log.debug('Telegram handler initialized');
|
||||
|
||||
@@ -477,6 +515,10 @@ app.get('/admin/knowledge-stats', async (_request, reply) => {
|
||||
const shutdown = async () => {
|
||||
app.log.info('Shutting down gracefully...');
|
||||
try {
|
||||
// Flush all active sessions to Iceberg before shutdown
|
||||
await websocketHandler.endAllSessions();
|
||||
await telegramHandler.endAllSessions();
|
||||
|
||||
// Stop event system first
|
||||
await eventSubscriber.stop();
|
||||
await eventRouter.stop();
|
||||
@@ -529,6 +571,53 @@ try {
|
||||
app.log.warn({ error }, 'Qdrant initialization failed - RAG will not be available');
|
||||
}
|
||||
|
||||
// Initialize tool registry
|
||||
app.log.debug('Initializing tool registry...');
|
||||
try {
|
||||
const toolRegistry = initializeToolRegistry(app.log, {
|
||||
// Use getter functions to support lazy initialization
|
||||
ohlcService: () => ohlcService,
|
||||
symbolIndexService: () => symbolIndexService,
|
||||
workspaceManager: undefined, // Will be set per-session
|
||||
});
|
||||
|
||||
// Register agent tool configurations
|
||||
// Main agent: platform tools + user's general MCP tools
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'main',
|
||||
platformTools: ['symbol_lookup', 'get_chart_data'],
|
||||
mcpTools: [], // No MCP tools for main agent by default (can be extended later)
|
||||
});
|
||||
|
||||
// Research subagent: only MCP tools for script creation/execution
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'research',
|
||||
platformTools: [], // No platform tools (works at script level)
|
||||
mcpTools: ['category_*', 'execute_research'],
|
||||
});
|
||||
|
||||
// Code reviewer subagent: no tools by default
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'code-reviewer',
|
||||
platformTools: [],
|
||||
mcpTools: [],
|
||||
});
|
||||
|
||||
app.log.info(
|
||||
{
|
||||
agents: toolRegistry.getRegisteredAgents(),
|
||||
configs: toolRegistry.getRegisteredAgents().map(name => ({
|
||||
name,
|
||||
config: toolRegistry.getAgentToolConfig(name),
|
||||
})),
|
||||
},
|
||||
'Tool registry initialized'
|
||||
);
|
||||
} catch (error) {
|
||||
app.log.error({ error }, 'Failed to initialize tool registry');
|
||||
// Non-fatal - continue without tools
|
||||
}
|
||||
|
||||
// Initialize RAG system and load global knowledge
|
||||
app.log.debug('Initializing RAG system...');
|
||||
try {
|
||||
@@ -586,6 +675,7 @@ try {
|
||||
|
||||
// Initialize Symbol Index Service (after server is running)
|
||||
// This is done asynchronously to not block server startup
|
||||
// The onMetadataUpdate callback is already registered with zmqRelayClient
|
||||
(async () => {
|
||||
try {
|
||||
const icebergClient = new IcebergClient(config.iceberg, app.log);
|
||||
@@ -594,18 +684,13 @@ try {
|
||||
logger: app.log,
|
||||
});
|
||||
await indexService.initialize();
|
||||
|
||||
// Assign to module-level variable so onMetadataUpdate callback can use it
|
||||
symbolIndexService = indexService;
|
||||
|
||||
// Update websocket handler's config so it can use the service
|
||||
(websocketHandler as any).config.symbolIndexService = indexService;
|
||||
|
||||
// Configure ZMQ relay to reload symbol metadata on updates
|
||||
(zmqRelayClient as any).config.onMetadataUpdate = async () => {
|
||||
app.log.info('Reloading symbol metadata from Iceberg');
|
||||
await indexService.initialize();
|
||||
app.log.info({ stats: indexService.getStats() }, 'Symbol metadata reloaded');
|
||||
};
|
||||
|
||||
app.log.info({ stats: symbolIndexService.getStats() }, 'Symbol index service initialized');
|
||||
} catch (error) {
|
||||
app.log.warn({ error }, 'Failed to initialize symbol index service - symbol search will not be available');
|
||||
|
||||
Reference in New Issue
Block a user