redesign fully scaffolded and web login works

2026-03-17 20:10:47 -04:00
parent b9cc397e05
commit f6bd22a8ef
143 changed files with 17317 additions and 693 deletions
--- a/gateway/src/main.ts
+++ b/gateway/src/main.ts
@@ -1,16 +1,149 @@
 import Fastify from 'fastify';
 import websocket from '@fastify/websocket';
 import cors from '@fastify/cors';
+import Redis from 'ioredis';
+import { readFileSync } from 'fs';
+import { load as loadYaml } from 'js-yaml';
 import { UserService } from './db/user-service.js';
 import { Authenticator } from './auth/authenticator.js';
+import { createBetterAuth } from './auth/better-auth-config.js';
+import { AuthService } from './auth/auth-service.js';
+import { AuthRoutes } from './routes/auth-routes.js';
 import { WebSocketHandler } from './channels/websocket-handler.js';
 import { TelegramHandler } from './channels/telegram-handler.js';
 import { KubernetesClient } from './k8s/client.js';
 import { ContainerManager } from './k8s/container-manager.js';

+// Catch unhandled promise rejections for better debugging
+process.on('unhandledRejection', (reason: any, promise) => {
+  console.error('=== UNHANDLED PROMISE REJECTION ===');
+  console.error('Reason:', reason);
+  console.error('Message:', reason?.message);
+  console.error('Stack:', reason?.stack);
+  console.error('Cause:', reason?.cause);
+  console.error('Promise:', promise);
+  console.error('===================================');
+  process.exit(1);
+});
+import {
+  SessionRegistry,
+  EventSubscriber,
+  EventRouter,
+  DeliveryService,
+} from './events/index.js';
+import { QdrantClient } from './clients/qdrant-client.js';
+import { EmbeddingService, RAGRetriever, DocumentLoader } from './harness/memory/index.js';
+import { join } from 'path';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Load configuration from YAML files
+function loadConfig() {
+  const configPath = process.env.CONFIG_PATH || '/config/config.yaml';
+  const secretsPath = process.env.SECRETS_PATH || '/config/secrets.yaml';
+
+  let configData: any = {};
+  let secretsData: any = {};
+
+  try {
+    const configFile = readFileSync(configPath, 'utf8');
+    configData = loadYaml(configFile) || {};
+    console.log(`Loaded configuration from ${configPath}`);
+  } catch (error: any) {
+    console.warn(`Could not load config from ${configPath}: ${error.message}, using defaults`);
+  }
+
+  try {
+    const secretsFile = readFileSync(secretsPath, 'utf8');
+    secretsData = loadYaml(secretsFile) || {};
+    console.log(`Loaded secrets from ${secretsPath}`);
+  } catch (error: any) {
+    console.warn(`Could not load secrets from ${secretsPath}: ${error.message}`);
+  }
+
+  return {
+    port: configData.server?.port || parseInt(process.env.PORT || '3000'),
+    host: configData.server?.host || process.env.HOST || '0.0.0.0',
+    logLevel: configData.server?.log_level || process.env.LOG_LEVEL || 'info',
+    corsOrigin: configData.server?.cors_origin || process.env.CORS_ORIGIN || '*',
+    baseUrl: configData.server?.base_url || process.env.BASE_URL || 'http://localhost:3000',
+    trustedOrigins: configData.server?.trusted_origins || [
+      process.env.BASE_URL || 'http://localhost:3000',
+      'http://localhost:5173',
+      'http://localhost:8080',
+    ],
+    databaseUrl: configData.database?.url || process.env.DATABASE_URL || 'postgresql://localhost/dexorder',
+
+    // Authentication configuration
+    authSecret: secretsData.auth?.secret || process.env.AUTH_SECRET || 'change-me-in-production',
+
+    // LLM provider API keys
+    providerConfig: {
+      anthropicApiKey: secretsData.llm_providers?.anthropic_api_key || process.env.ANTHROPIC_API_KEY,
+      openaiApiKey: secretsData.llm_providers?.openai_api_key || process.env.OPENAI_API_KEY,
+      googleApiKey: secretsData.llm_providers?.google_api_key || process.env.GOOGLE_API_KEY,
+      openrouterApiKey: secretsData.llm_providers?.openrouter_api_key || process.env.OPENROUTER_API_KEY,
+    },
+
+    telegramBotToken: secretsData.telegram?.bot_token || process.env.TELEGRAM_BOT_TOKEN || '',
+
+    // Email service configuration
+    emailServiceKey: secretsData.email?.service_key || process.env.EMAIL_SERVICE_KEY,
+    emailFromAddress: configData.email?.from_address || process.env.EMAIL_FROM_ADDRESS || 'noreply@dexorder.com',
+
+    // Push notification service configuration
+    pushServiceKey: secretsData.push?.service_key || process.env.PUSH_SERVICE_KEY,
+
+    // Event router configuration
+    eventRouterBind: configData.events?.router_bind || process.env.EVENT_ROUTER_BIND || 'tcp://*:5571',
+
+    // Redis configuration (for harness memory layer)
+    redisUrl: configData.redis?.url || process.env.REDIS_URL || 'redis://localhost:6379',
+
+    // Qdrant configuration (for RAG)
+    qdrant: {
+      url: configData.qdrant?.url || process.env.QDRANT_URL || 'http://localhost:6333',
+      apiKey: secretsData.qdrant?.api_key || process.env.QDRANT_API_KEY,
+      collectionName: configData.qdrant?.collection || process.env.QDRANT_COLLECTION || 'gateway_memory',
+    },
+
+    // Iceberg configuration (for durable storage)
+    iceberg: {
+      catalogUri: configData.iceberg?.catalog_uri || process.env.ICEBERG_CATALOG_URI || 'http://iceberg-catalog:8181',
+      namespace: configData.iceberg?.namespace || process.env.ICEBERG_NAMESPACE || 'gateway',
+      s3Endpoint: configData.iceberg?.s3_endpoint || process.env.S3_ENDPOINT,
+      s3AccessKey: secretsData.iceberg?.s3_access_key || process.env.S3_ACCESS_KEY,
+      s3SecretKey: secretsData.iceberg?.s3_secret_key || process.env.S3_SECRET_KEY,
+    },
+
+    // Embedding configuration (for RAG)
+    embedding: {
+      provider: (configData.embedding?.provider || process.env.EMBEDDING_PROVIDER || 'ollama') as 'ollama' | 'openai' | 'anthropic' | 'local' | 'voyage' | 'cohere' | 'none',
+      model: configData.embedding?.model || process.env.EMBEDDING_MODEL,
+      apiKey: secretsData.embedding?.api_key || process.env.EMBEDDING_API_KEY || secretsData.llm_providers?.openai_api_key || process.env.OPENAI_API_KEY,
+      ollamaUrl: configData.embedding?.ollama_url || process.env.OLLAMA_URL || 'http://localhost:11434',
+    },
+
+    // Kubernetes configuration
+    kubernetes: {
+      namespace: configData.kubernetes?.namespace || process.env.KUBERNETES_NAMESPACE || 'dexorder-agents',
+      inCluster: configData.kubernetes?.in_cluster ?? (process.env.KUBERNETES_IN_CLUSTER === 'true'),
+      context: configData.kubernetes?.context || process.env.KUBERNETES_CONTEXT,
+      agentImage: configData.kubernetes?.agent_image || process.env.AGENT_IMAGE || 'ghcr.io/dexorder/agent:latest',
+      sidecarImage: configData.kubernetes?.sidecar_image || process.env.SIDECAR_IMAGE || 'ghcr.io/dexorder/lifecycle-sidecar:latest',
+      storageClass: configData.kubernetes?.storage_class || process.env.AGENT_STORAGE_CLASS || 'standard',
+    },
+  };
+}
+
+const config = loadConfig();
+
 const app = Fastify({
  logger: {
-    level: process.env.LOG_LEVEL || 'info',
+    level: config.logLevel,
    transport: {
      target: 'pino-pretty',
      options: {
@@ -22,33 +155,6 @@ const app = Fastify({
  },
 });

-// Configuration from environment
-const config = {
-  port: parseInt(process.env.PORT || '3000'),
-  host: process.env.HOST || '0.0.0.0',
-  databaseUrl: process.env.DATABASE_URL || 'postgresql://localhost/dexorder',
-
-  // LLM provider API keys
-  providerConfig: {
-    anthropicApiKey: process.env.ANTHROPIC_API_KEY,
-    openaiApiKey: process.env.OPENAI_API_KEY,
-    googleApiKey: process.env.GOOGLE_API_KEY,
-    openrouterApiKey: process.env.OPENROUTER_API_KEY,
-  },
-
-  telegramBotToken: process.env.TELEGRAM_BOT_TOKEN || '',
-
-  // Kubernetes configuration
-  kubernetes: {
-    namespace: process.env.KUBERNETES_NAMESPACE || 'dexorder-agents',
-    inCluster: process.env.KUBERNETES_IN_CLUSTER === 'true',
-    context: process.env.KUBERNETES_CONTEXT,
-    agentImage: process.env.AGENT_IMAGE || 'ghcr.io/dexorder/agent:latest',
-    sidecarImage: process.env.SIDECAR_IMAGE || 'ghcr.io/dexorder/lifecycle-sidecar:latest',
-    storageClass: process.env.AGENT_STORAGE_CLASS || 'standard',
-  },
-};
-
 // Validate at least one LLM provider is configured
 const hasAnyProvider = Object.values(config.providerConfig).some(key => !!key);
 if (!hasAnyProvider) {
@@ -58,7 +164,7 @@ if (!hasAnyProvider) {

 // Register plugins
 await app.register(cors, {
-  origin: process.env.CORS_ORIGIN || '*',
+  origin: config.corsOrigin,
 });

 await app.register(websocket, {
@@ -70,6 +176,61 @@ await app.register(websocket, {
 // Initialize services
 const userService = new UserService(config.databaseUrl);

+// Initialize Better Auth
+let betterAuth;
+try {
+  app.log.info({ databaseUrl: config.databaseUrl.replace(/:[^:@]+@/, ':***@') }, 'Initializing Better Auth');
+  betterAuth = await createBetterAuth({
+    databaseUrl: config.databaseUrl,
+    secret: config.authSecret,
+    baseUrl: config.baseUrl,
+    trustedOrigins: config.trustedOrigins,
+    logger: app.log,
+  });
+  app.log.info('Better Auth initialized successfully');
+} catch (error: any) {
+  app.log.error({ error, message: error.message, stack: error.stack }, 'Failed to initialize Better Auth');
+  throw error;
+}
+
+// Initialize Auth Service
+const authService = new AuthService({
+  auth: betterAuth,
+  pool: userService.getPool(),
+  logger: app.log,
+});
+
+// Connect UserService with AuthService for JWT verification
+userService.setAuthService(authService);
+
+// Initialize Redis client (for harness memory layer)
+const redis = new Redis(config.redisUrl, {
+  maxRetriesPerRequest: 3,
+  connectTimeout: 10000, // 10 seconds
+  retryStrategy: (times) => {
+    if (times > 5) {
+      app.log.error('Redis connection failed after 5 retries');
+      return null; // Stop retrying
+    }
+    const delay = Math.min(times * 50, 2000);
+    return delay;
+  },
+  lazyConnect: true,
+});
+
+// Initialize Qdrant client (for RAG)
+const qdrantClient = new QdrantClient(config.qdrant, app.log);
+
+// Initialize Iceberg client (for durable storage)
+// const icebergClient = new IcebergClient(config.iceberg, app.log);
+
+app.log.info({
+  redis: config.redisUrl,
+  qdrant: config.qdrant.url,
+  iceberg: config.iceberg.catalogUri,
+  embeddingProvider: config.embedding.provider,
+}, 'Harness storage clients configured');
+
 // Initialize Kubernetes client and container manager
 const k8sClient = new KubernetesClient({
  namespace: config.kubernetes.namespace,
@@ -86,41 +247,170 @@ const containerManager = new ContainerManager({
  namespace: config.kubernetes.namespace,
  logger: app.log,
 });
+app.log.debug('Container manager initialized');

 const authenticator = new Authenticator({
  userService,
  containerManager,
  logger: app.log,
 });
+app.log.debug('Authenticator initialized');
+
+// Initialize event system
+const sessionRegistry = new SessionRegistry();
+app.log.debug('Session registry initialized');
+
+const deliveryService = new DeliveryService({
+  telegramBotToken: config.telegramBotToken,
+  emailServiceKey: config.emailServiceKey,
+  emailFromAddress: config.emailFromAddress,
+  pushServiceKey: config.pushServiceKey,
+  logger: app.log,
+});
+app.log.debug('Delivery service initialized');
+
+const eventSubscriber = new EventSubscriber(sessionRegistry, app.log);
+app.log.debug('Event subscriber initialized');
+
+const eventRouter = new EventRouter({
+  sessions: sessionRegistry,
+  delivery: deliveryService,
+  logger: app.log,
+  bindEndpoint: config.eventRouterBind,
+});
+app.log.debug('Event router initialized');

 // Initialize channel handlers
 const websocketHandler = new WebSocketHandler({
  authenticator,
  providerConfig: config.providerConfig,
+  sessionRegistry,
+  eventSubscriber,
 });
+app.log.debug('WebSocket handler initialized');

 const telegramHandler = new TelegramHandler({
  authenticator,
  providerConfig: config.providerConfig,
  telegramBotToken: config.telegramBotToken,
 });
+app.log.debug('Telegram handler initialized');
+
+// Initialize auth routes
+app.log.debug('Initializing auth routes...');
+const authRoutes = new AuthRoutes({
+  authService,
+  betterAuth,
+});

 // Register routes
+app.log.debug('Registering auth routes...');
+try {
+  authRoutes.register(app);
+  app.log.debug('Auth routes registered successfully');
+} catch (error: any) {
+  app.log.error({ error, message: error.message, stack: error.stack }, 'Failed to register auth routes');
+  throw error;
+}
+
+app.log.debug('Registering websocket handler...');
 websocketHandler.register(app);
+app.log.debug('Registering telegram handler...');
 telegramHandler.register(app);
+app.log.debug('All routes registered');

 // Health check
 app.get('/health', async () => {
-  return {
+  const health: any = {
    status: 'ok',
    timestamp: new Date().toISOString(),
+    activeSessions: sessionRegistry.size(),
+    eventSubscriptions: eventSubscriber.getSubscriptionCount(),
+    processedEvents: eventRouter.getProcessedEventCount(),
  };
+
+  // Add RAG stats if available
+  if (app.hasDecorator('ragRetriever')) {
+    try {
+      const ragStats = await (app as any).ragRetriever.getStats();
+      health.rag = {
+        vectorCount: ragStats.vectorCount,
+        indexedCount: ragStats.indexedCount,
+      };
+    } catch (error) {
+      // Ignore errors in health check
+    }
+  }
+
+  return health;
+});
+
+// Admin endpoints
+app.post('/admin/reload-knowledge', async (_request, reply) => {
+  if (!app.hasDecorator('documentLoader')) {
+    return reply.code(503).send({
+      error: 'Document loader not initialized',
+    });
+  }
+
+  try {
+    app.log.info('Manual knowledge reload requested');
+    const stats = await (app as any).documentLoader.loadAll();
+
+    return {
+      success: true,
+      stats,
+      timestamp: new Date().toISOString(),
+    };
+  } catch (error: any) {
+    app.log.error({ error }, 'Failed to reload knowledge');
+    return reply.code(500).send({
+      error: 'Failed to reload knowledge',
+      message: error.message,
+    });
+  }
+});
+
+app.get('/admin/knowledge-stats', async (_request, reply) => {
+  if (!app.hasDecorator('documentLoader')) {
+    return reply.code(503).send({
+      error: 'Document loader not initialized',
+    });
+  }
+
+  try {
+    const loaderStats = (app as any).documentLoader.getStats();
+    const ragStats = await (app as any).ragRetriever.getStats();
+
+    return {
+      loader: loaderStats,
+      rag: {
+        vectorCount: ragStats.vectorCount,
+        indexedCount: ragStats.indexedCount,
+        collectionSize: ragStats.collectionSize,
+      },
+      timestamp: new Date().toISOString(),
+    };
+  } catch (error: any) {
+    app.log.error({ error }, 'Failed to get knowledge stats');
+    return reply.code(500).send({
+      error: 'Failed to get knowledge stats',
+      message: error.message,
+    });
+  }
 });

 // Graceful shutdown
 const shutdown = async () => {
  app.log.info('Shutting down gracefully...');
  try {
+    // Stop event system first
+    await eventSubscriber.stop();
+    await eventRouter.stop();
+
+    // Disconnect Redis
+    redis.disconnect();
+
    await userService.close();
    await app.close();
    app.log.info('Shutdown complete');
@@ -136,6 +426,61 @@ process.on('SIGINT', shutdown);

 // Start server
 try {
+  app.log.debug('Starting server initialization...');
+
+  // Connect to Redis
+  app.log.debug('Connecting to Redis...');
+  await redis.connect();
+  app.log.info('Redis connected');
+
+  // Initialize Qdrant collection
+  app.log.debug('Initializing Qdrant...');
+  try {
+    await qdrantClient.initialize();
+    app.log.info('Qdrant collection initialized');
+  } catch (error) {
+    app.log.warn({ error }, 'Qdrant initialization failed - RAG will not be available');
+  }
+
+  // Initialize RAG system and load global knowledge
+  app.log.debug('Initializing RAG system...');
+  try {
+    // Initialize embedding service
+    const embeddingService = new EmbeddingService(config.embedding, app.log);
+    const vectorDimension = embeddingService.getDimensions();
+
+    // Initialize RAG retriever
+    const ragRetriever = new RAGRetriever(config.qdrant, app.log, vectorDimension);
+    await ragRetriever.initialize();
+
+    // Initialize document loader
+    const knowledgeDir = join(__dirname, '..', 'knowledge');
+    const documentLoader = new DocumentLoader(
+      { knowledgeDir },
+      embeddingService,
+      ragRetriever,
+      app.log
+    );
+
+    // Load all knowledge documents
+    const loadStats = await documentLoader.loadAll();
+    app.log.info(loadStats, 'Global knowledge loaded into RAG');
+
+    // Store references for admin endpoints
+    app.decorate('documentLoader', documentLoader);
+    app.decorate('ragRetriever', ragRetriever);
+  } catch (error) {
+    app.log.warn({ error }, 'Failed to load global knowledge - RAG will use existing data');
+  }
+
+  // Start event system
+  app.log.debug('Starting event subscriber...');
+  await eventSubscriber.start();
+  app.log.debug('Starting event router...');
+  await eventRouter.start();
+  app.log.debug('Event system started');
+
+  app.log.debug('Starting Fastify server...');
  await app.listen({
    port: config.port,
    host: config.host,
@@ -145,6 +490,9 @@ try {
    {
      port: config.port,
      host: config.host,
+      eventRouterBind: config.eventRouterBind,
+      redis: config.redisUrl,
+      qdrant: config.qdrant.url,
    },
    'Gateway server started'
  );