chart data loading

ingestor/src/index.js

@@ -8,6 +8,7 @@ import { ZmqClient } from './zmq-client.js';
 import { KafkaProducer } from './kafka-producer.js';
 import { CCXTFetcher } from './ccxt-fetcher.js';
 import { RealtimePoller } from './realtime-poller.js';
+import { SymbolMetadataGenerator } from './symbol-metadata-generator.js';
 
 // Logger setup
 const logger = pino({
@@ -61,6 +62,10 @@ function loadConfig() {
     max_concurrent: config.max_concurrent || 10,
     poll_interval_ms: config.poll_interval_ms || 10000,
 
+    // Symbol metadata configuration
+    supported_exchanges: config.supported_exchanges || ['binance', 'coinbase', 'kraken'],
+    symbol_metadata_interval_ms: config.symbol_metadata_interval_ms || 6 * 60 * 60 * 1000, // 6 hours
+
     ...secrets
   };
 }
@@ -75,9 +80,16 @@ class IngestorWorker {
       config,
       logger.child({ component: 'kafka' })
     );
+    // Create metadata generator first so ccxtFetcher can use it
+    this.metadataGenerator = new SymbolMetadataGenerator(
+      config,
+      this.kafkaProducer,
+      logger.child({ component: 'metadata' })
+    );
     this.ccxtFetcher = new CCXTFetcher(
       config,
-      logger.child({ component: 'ccxt' })
+      logger.child({ component: 'ccxt' }),
+      this.metadataGenerator
     );
     this.realtimePoller = new RealtimePoller(
       this.ccxtFetcher,
@@ -88,6 +100,10 @@ class IngestorWorker {
     // Track active requests
     this.activeRequests = new Map();
     this.isShutdown = false;
+
+    // Metadata generation interval
+    this.metadataIntervalMs = config.symbol_metadata_interval_ms;
+    this.metadataInterval = null;
   }
 
   /**
@@ -106,6 +122,26 @@ class IngestorWorker {
     // Start work loop
     this.workLoop();
 
+    // Generate symbol metadata on startup
+    this.logger.info('Generating initial symbol metadata');
+    try {
+      const results = await this.metadataGenerator.generateAll();
+      this.logger.info({ results }, 'Initial symbol metadata generated');
+    } catch (error) {
+      this.logger.error({ error: error.message }, 'Failed to generate initial symbol metadata');
+    }
+
+    // Schedule periodic metadata generation
+    this.metadataInterval = setInterval(async () => {
+      this.logger.info('Periodic symbol metadata generation');
+      try {
+        const results = await this.metadataGenerator.generateAll();
+        this.logger.info({ results }, 'Periodic symbol metadata generated');
+      } catch (error) {
+        this.logger.error({ error: error.message }, 'Failed to generate periodic symbol metadata');
+      }
+    }, this.metadataIntervalMs);
+
     this.logger.info('Ingestor worker started successfully');
   }
 
@@ -347,7 +383,8 @@ class IngestorWorker {
     return {
       activeRequests: this.activeRequests.size,
       maxConcurrent: this.config.max_concurrent,
-      pollerStats: this.realtimePoller.getStats()
+      pollerStats: this.realtimePoller.getStats(),
+      metadataStatus: this.metadataGenerator.getStatus()
     };
   }
 
@@ -362,11 +399,17 @@ class IngestorWorker {
     this.isShutdown = true;
     this.logger.info('Shutting down ingestor worker');
 
+    // Stop metadata generation interval
+    if (this.metadataInterval) {
+      clearInterval(this.metadataInterval);
+    }
+
     // Stop polling
     this.realtimePoller.shutdown();
 
     // Close connections
     await this.ccxtFetcher.close();
+    await this.metadataGenerator.close();
     await this.kafkaProducer.disconnect();
     await this.zmqClient.shutdown();