backend redesign

flink/src/main/resources/iceberg-schemas/ohlc_schema.sql

@@ -0,0 +1,54 @@
+-- Iceberg OHLC Table Schema (Documentation)
+--
+-- NOTE: This file is kept for documentation purposes only.
+-- The actual table is created by SchemaInitializer.java using the Iceberg API.
+--
+-- Single table for all periods with Iceberg v2 primary key enforcement
+-- Primary key: (ticker, period_seconds, timestamp)
+-- Partition by: (ticker, days(timestamp))
+
+CREATE TABLE IF NOT EXISTS trading.ohlc (
+  -- Primary key fields
+  ticker STRING NOT NULL COMMENT 'Market identifier (e.g., BINANCE:BTC/USDT)',
+  period_seconds INT NOT NULL COMMENT 'OHLC period in seconds (60, 300, 900, 3600, 14400, 86400, 604800, etc.)',
+  timestamp BIGINT NOT NULL COMMENT 'Candle timestamp in microseconds since epoch',
+
+  -- OHLC price data (stored as integers, divide by rational denominator from market metadata)
+  open BIGINT NOT NULL COMMENT 'Opening price',
+  high BIGINT NOT NULL COMMENT 'Highest price',
+  low BIGINT NOT NULL COMMENT 'Lowest price',
+  close BIGINT NOT NULL COMMENT 'Closing price',
+
+  -- Volume data
+  volume BIGINT COMMENT 'Total volume',
+  buy_vol BIGINT COMMENT 'Buy volume',
+  sell_vol BIGINT COMMENT 'Sell volume',
+
+  -- Timing data
+  open_time BIGINT COMMENT 'Timestamp when open price occurred',
+  high_time BIGINT COMMENT 'Timestamp when high price occurred',
+  low_time BIGINT COMMENT 'Timestamp when low price occurred',
+  close_time BIGINT COMMENT 'Timestamp when close price occurred',
+
+  -- Additional fields
+  open_interest BIGINT COMMENT 'Open interest for futures',
+
+  -- Metadata fields for tracking
+  request_id STRING COMMENT 'Request ID that generated this data (for historical requests)',
+  ingested_at BIGINT NOT NULL COMMENT 'Timestamp when data was ingested by Flink'
+)
+USING iceberg
+PARTITIONED BY (ticker, days(timestamp))
+TBLPROPERTIES (
+  'write.format.default' = 'parquet',
+  'write.parquet.compression-codec' = 'snappy',
+  'write.metadata.compression-codec' = 'gzip',
+  'format-version' = '2',
+  'write.upsert.enabled' = 'true'
+);
+
+-- Primary key constraint (enforced by Iceberg v2)
+-- Uniqueness enforced on (ticker, period_seconds, timestamp)
+-- Upserts will replace existing rows with same primary key
+
+COMMENT ON TABLE trading.ohlc IS 'Historical OHLC candle data from exchanges. Single table for all periods with primary key enforcement.';

flink/src/main/resources/topics-dev.yaml

@@ -0,0 +1,29 @@
+topics:
+  # Realtime and historical OHLC data (protobuf encoded)
+  # Individual OHLC messages for realtime data
+  # OHLCBatch messages for historical data (with metadata)
+  - name: market-ohlc
+    partitions: 3
+    replication: 1
+    config:
+      retention.ms: 86400000  # 24 hours
+      compression.type: snappy
+      cleanup.policy: delete
+
+  # Realtime tick data (protobuf encoded)
+  - name: market-tick
+    partitions: 3
+    replication: 1
+    config:
+      retention.ms: 3600000  # 1 hour
+      compression.type: snappy
+      cleanup.policy: delete
+
+  # Order execution events
+  - name: order-event
+    partitions: 2
+    replication: 1
+    config:
+      retention.ms: 2592000000  # 30 days
+      compression.type: snappy
+      cleanup.policy: delete

flink/src/main/resources/topics.yaml

@@ -0,0 +1,29 @@
+topics:
+  # Realtime and historical OHLC data (protobuf encoded)
+  # Individual OHLC messages for realtime data
+  # OHLCBatch messages for historical data (with metadata)
+  - name: market-ohlc
+    partitions: 6
+    replication: 2
+    config:
+      retention.ms: 86400000  # 24 hours
+      compression.type: snappy
+      cleanup.policy: delete
+
+  # Realtime tick data (protobuf encoded)
+  - name: market-tick
+    partitions: 6
+    replication: 2
+    config:
+      retention.ms: 3600000  # 1 hour
+      compression.type: snappy
+      cleanup.policy: delete
+
+  # Order execution events
+  - name: order-event
+    partitions: 3
+    replication: 2
+    config:
+      retention.ms: 2592000000  # 30 days
+      compression.type: snappy
+      cleanup.policy: delete