major agent refactoring: wiki knowledge base, no RAG, no Qdrant, no Ollama
This commit is contained in:
@@ -41,15 +41,7 @@ import {
|
||||
EventRouter,
|
||||
DeliveryService,
|
||||
} from './events/index.js';
|
||||
import { QdrantClient } from './clients/qdrant-client.js';
|
||||
import { EmbeddingService, RAGRetriever, DocumentLoader } from './harness/memory/index.js';
|
||||
import { initializeToolRegistry } from './tools/tool-registry.js';
|
||||
import { join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname } from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Load configuration from YAML files
|
||||
function loadConfig() {
|
||||
@@ -137,13 +129,6 @@ function loadConfig() {
|
||||
// Conversation history limit: number of prior turns loaded as LLM context and flushed to Iceberg
|
||||
conversationHistoryLimit: configData.agent?.conversation_history_limit || parseInt(process.env.CONVERSATION_HISTORY_LIMIT || '20'),
|
||||
|
||||
// Qdrant configuration (for RAG)
|
||||
qdrant: {
|
||||
url: configData.qdrant?.url || process.env.QDRANT_URL || 'http://localhost:6333',
|
||||
apiKey: secretsData.qdrant?.api_key || process.env.QDRANT_API_KEY,
|
||||
collectionName: configData.qdrant?.collection || process.env.QDRANT_COLLECTION || 'gateway_memory',
|
||||
},
|
||||
|
||||
// Iceberg configuration (for durable storage)
|
||||
iceberg: {
|
||||
catalogUri: configData.iceberg?.catalog_uri || process.env.ICEBERG_CATALOG_URI || 'http://iceberg-catalog:8181',
|
||||
@@ -162,14 +147,6 @@ function loadConfig() {
|
||||
notificationEndpoint: configData.relay?.notification_endpoint || process.env.RELAY_NOTIFICATION_ENDPOINT || 'tcp://relay:5558',
|
||||
},
|
||||
|
||||
// Embedding configuration (for RAG)
|
||||
embedding: {
|
||||
provider: (configData.embedding?.provider || process.env.EMBEDDING_PROVIDER || 'ollama') as 'ollama' | 'openai' | 'anthropic' | 'local' | 'voyage' | 'cohere' | 'none',
|
||||
model: configData.embedding?.model || process.env.EMBEDDING_MODEL,
|
||||
apiKey: secretsData.embedding?.api_key || process.env.EMBEDDING_API_KEY || secretsData.llm_providers?.openai_api_key || process.env.OPENAI_API_KEY,
|
||||
ollamaUrl: configData.embedding?.ollama_url || process.env.OLLAMA_URL || 'http://localhost:11434',
|
||||
},
|
||||
|
||||
// Kubernetes configuration
|
||||
kubernetes: {
|
||||
namespace: configData.kubernetes?.namespace || process.env.KUBERNETES_NAMESPACE || 'sandbox',
|
||||
@@ -265,9 +242,6 @@ const redis = new Redis(config.redisUrl, {
|
||||
lazyConnect: true,
|
||||
});
|
||||
|
||||
// Initialize Qdrant client (for RAG)
|
||||
const qdrantClient = new QdrantClient(config.qdrant, app.log);
|
||||
|
||||
// Initialize Iceberg client (for durable storage)
|
||||
// const icebergClient = new IcebergClient(config.iceberg, app.log);
|
||||
|
||||
@@ -294,10 +268,8 @@ const zmqRelayClient = new ZMQRelayClient({
|
||||
|
||||
app.log.info({
|
||||
redis: config.redisUrl,
|
||||
qdrant: config.qdrant.url,
|
||||
iceberg: config.iceberg.catalogUri,
|
||||
relay: config.relay.requestEndpoint,
|
||||
embeddingProvider: config.embedding.provider,
|
||||
}, 'Harness storage clients configured');
|
||||
|
||||
// Initialize Kubernetes client and container manager
|
||||
@@ -456,77 +428,9 @@ app.get('/health', async () => {
|
||||
processedEvents: eventRouter.getProcessedEventCount(),
|
||||
};
|
||||
|
||||
// Add RAG stats if available
|
||||
if (app.hasDecorator('ragRetriever')) {
|
||||
try {
|
||||
const ragStats = await (app as any).ragRetriever.getStats();
|
||||
health.rag = {
|
||||
vectorCount: ragStats.vectorCount,
|
||||
indexedCount: ragStats.indexedCount,
|
||||
};
|
||||
} catch (error) {
|
||||
// Ignore errors in health check
|
||||
}
|
||||
}
|
||||
|
||||
return health;
|
||||
});
|
||||
|
||||
// Admin endpoints
|
||||
app.post('/admin/reload-knowledge', async (_request, reply) => {
|
||||
if (!app.hasDecorator('documentLoader')) {
|
||||
return reply.code(503).send({
|
||||
error: 'Document loader not initialized',
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
app.log.info('Manual knowledge reload requested');
|
||||
const stats = await (app as any).documentLoader.loadAll();
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stats,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
} catch (error: any) {
|
||||
app.log.error({ error }, 'Failed to reload knowledge');
|
||||
return reply.code(500).send({
|
||||
error: 'Failed to reload knowledge',
|
||||
message: error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/admin/knowledge-stats', async (_request, reply) => {
|
||||
if (!app.hasDecorator('documentLoader')) {
|
||||
return reply.code(503).send({
|
||||
error: 'Document loader not initialized',
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const loaderStats = (app as any).documentLoader.getStats();
|
||||
const ragStats = await (app as any).ragRetriever.getStats();
|
||||
|
||||
return {
|
||||
loader: loaderStats,
|
||||
rag: {
|
||||
vectorCount: ragStats.vectorCount,
|
||||
indexedCount: ragStats.indexedCount,
|
||||
collectionSize: ragStats.collectionSize,
|
||||
},
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
} catch (error: any) {
|
||||
app.log.error({ error }, 'Failed to get knowledge stats');
|
||||
return reply.code(500).send({
|
||||
error: 'Failed to get knowledge stats',
|
||||
message: error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
const shutdown = async () => {
|
||||
app.log.info('Shutting down gracefully...');
|
||||
@@ -578,15 +482,6 @@ try {
|
||||
app.log.warn({ error }, 'ZMQ Relay connection failed - historical data will not be available');
|
||||
}
|
||||
|
||||
// Initialize Qdrant collection
|
||||
app.log.debug('Initializing Qdrant...');
|
||||
try {
|
||||
await qdrantClient.initialize();
|
||||
app.log.info('Qdrant collection initialized');
|
||||
} catch (error) {
|
||||
app.log.warn({ error }, 'Qdrant initialization failed - RAG will not be available');
|
||||
}
|
||||
|
||||
// Initialize tool registry
|
||||
app.log.debug('Initializing tool registry...');
|
||||
try {
|
||||
@@ -602,42 +497,8 @@ try {
|
||||
// Main agent: platform tools + user's general MCP tools
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'main',
|
||||
platformTools: ['symbol_lookup', 'get_chart_data'],
|
||||
mcpTools: ['python_list', 'python_delete', 'backtest_strategy', 'list_active_strategies'],
|
||||
});
|
||||
|
||||
// Research subagent: only MCP tools for script creation/execution
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'research',
|
||||
platformTools: [], // No platform tools (works at script level)
|
||||
mcpTools: ['python_*', 'execute_research'],
|
||||
});
|
||||
|
||||
// Indicator subagent: workspace patch + category tools + evaluate_indicator
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'indicator',
|
||||
platformTools: [],
|
||||
mcpTools: ['workspace_read', 'workspace_patch', 'python_*', 'evaluate_indicator'],
|
||||
});
|
||||
|
||||
// Web explore subagent: platform search/fetch tools only (no MCP needed)
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'web-explore',
|
||||
platformTools: ['web_search', 'fetch_page', 'arxiv_search'],
|
||||
mcpTools: [],
|
||||
});
|
||||
|
||||
// Strategy subagent: all strategy-related MCP tools
|
||||
toolRegistry.registerAgentTools({
|
||||
agentName: 'strategy',
|
||||
platformTools: [],
|
||||
mcpTools: [
|
||||
'python_write', 'python_edit', 'python_read', 'python_list',
|
||||
'python_log', 'python_revert', 'python_delete',
|
||||
'backtest_strategy', 'activate_strategy', 'deactivate_strategy',
|
||||
'list_active_strategies', 'get_backtest_results',
|
||||
'get_strategy_trades', 'get_strategy_events',
|
||||
],
|
||||
platformTools: ['SymbolLookup', 'GetChartData'],
|
||||
mcpTools: ['PythonList', 'PythonDelete', 'BacktestStrategy', 'ListActiveStrategies'],
|
||||
});
|
||||
|
||||
app.log.info(
|
||||
@@ -655,37 +516,6 @@ try {
|
||||
// Non-fatal - continue without tools
|
||||
}
|
||||
|
||||
// Initialize RAG system and load global knowledge
|
||||
app.log.debug('Initializing RAG system...');
|
||||
try {
|
||||
// Initialize embedding service
|
||||
const embeddingService = new EmbeddingService(config.embedding, app.log);
|
||||
const vectorDimension = embeddingService.getDimensions();
|
||||
|
||||
// Initialize RAG retriever
|
||||
const ragRetriever = new RAGRetriever(config.qdrant, app.log, vectorDimension);
|
||||
await ragRetriever.initialize();
|
||||
|
||||
// Initialize document loader
|
||||
const knowledgeDir = join(__dirname, '..', 'knowledge');
|
||||
const documentLoader = new DocumentLoader(
|
||||
{ knowledgeDir },
|
||||
embeddingService,
|
||||
ragRetriever,
|
||||
app.log
|
||||
);
|
||||
|
||||
// Load all knowledge documents
|
||||
const loadStats = await documentLoader.loadAll();
|
||||
app.log.info(loadStats, 'Global knowledge loaded into RAG');
|
||||
|
||||
// Store references for admin endpoints
|
||||
app.decorate('documentLoader', documentLoader);
|
||||
app.decorate('ragRetriever', ragRetriever);
|
||||
} catch (error) {
|
||||
app.log.warn({ error }, 'Failed to load global knowledge - RAG will use existing data');
|
||||
}
|
||||
|
||||
// Start event system
|
||||
app.log.debug('Starting event subscriber...');
|
||||
await eventSubscriber.start();
|
||||
@@ -705,7 +535,6 @@ try {
|
||||
host: config.host,
|
||||
eventRouterBind: config.eventRouterBind,
|
||||
redis: config.redisUrl,
|
||||
qdrant: config.qdrant.url,
|
||||
},
|
||||
'Gateway server started'
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user