backend redesign

docker-compose.yaml

@@ -0,0 +1,139 @@
+version: '3.8'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.7.0
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: confluentinc/cp-kafka:7.7.0
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    ports:
+      - "9092:9092"
+
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: iceberg
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+
+  # MinIO for S3-compatible storage (Iceberg warehouse)
+  minio:
+    image: minio/minio:latest
+    command: server /data --console-address ":9001"
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - minio_data:/data
+
+  # Iceberg REST Catalog
+  iceberg-catalog:
+    image: tabulario/iceberg-rest:latest
+    environment:
+      - CATALOG_WAREHOUSE=s3://warehouse/
+      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
+      - CATALOG_S3_ENDPOINT=http://minio:9000
+      - CATALOG_S3_ACCESS__KEY__ID=minio
+      - CATALOG_S3_SECRET__ACCESS__KEY=minio123
+      - CATALOG_S3_PATH__STYLE__ACCESS=true
+    ports:
+      - "8181:8181"
+    depends_on:
+      - postgres
+      - minio
+
+  flink-jobmanager:
+    image: flink:1.20-scala_2.12
+    command: jobmanager
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager
+    ports:
+      - "6123:6123"
+      - "8081:8081"
+    depends_on:
+      - kafka
+      - postgres
+
+  flink-taskmanager:
+    image: flink:1.20-scala_2.12
+    command: taskmanager
+    environment:
+      - JOB_MANAGER_RPC_ADDRESS=flink-jobmanager
+    depends_on:
+      - flink-jobmanager
+      - kafka
+
+  relay:
+    build:
+      context: relay
+      dockerfile: relay/Dockerfile
+    ports:
+      - "5555:5555"  # Ingestor work queue
+      - "5556:5556"  # Ingestor responses
+      - "5558:5558"  # Market data (clients)
+      - "5559:5559"  # Client requests
+    environment:
+      - RUST_LOG=relay=info
+      - CONFIG_PATH=/config/config.yaml
+    volumes:
+      - ./relay/config.example.yaml:/config/config.yaml:ro
+    depends_on:
+      - flink-jobmanager
+    restart: unless-stopped
+
+  ingestor:
+    build:
+      context: ingestor
+      dockerfile: ingestor/Dockerfile
+    environment:
+      - LOG_LEVEL=info
+      - CONFIG_PATH=/config/config.yaml
+    volumes:
+      - ./ingestor/config.example.yaml:/config/config.yaml:ro
+    depends_on:
+      - kafka
+      - relay
+    restart: unless-stopped
+
+  history-test-client:
+    build:
+      context: test/history_client
+      dockerfile: test/history_client/Dockerfile
+    depends_on:
+      - relay
+      - ingestor
+      - flink-jobmanager
+      - iceberg-catalog
+    environment:
+      - ICEBERG_CATALOG_URI=http://iceberg-catalog:8181
+      - RELAY_ENDPOINT=tcp://relay:5555
+      - NOTIFICATION_ENDPOINT=tcp://flink:5557
+    volumes:
+      - ./client-py:/client-py:ro
+    profiles:
+      - test
+    # Wait for services to start up, then run new OHLCClient-based test
+    command: sh -c "sleep 10 && pip install -e /client-py && python client_ohlc_api.py"
+
+volumes:
+  postgres_data:
+  minio_data: