dexorder/ai
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Symbol & data refactoring for Nautilus

Browse Source
This commit is contained in:
Tim Olson
2026-04-01 00:59:13 -04:00
parent cd28e18e52
commit 93bc8a3a4f
55 changed files with 537 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
flink/src/main/java/com/dexorder/flink/iceberg/SchemaInitializer.java
View File

@@ -81,7 +81,8 @@ public class SchemaInitializer {
// Bump this when the schema changes. Tables with a different (or missing) version
// will be dropped and recreated. Increment by 1 for each incompatible change.
// v2: open/high/low/close changed from required to optional to support null gap bars
private static final String OHLC_SCHEMA_VERSION = "2";
// v3: timestamps changed from microseconds to nanoseconds; ticker format changed to BTC/USDT.BINANCE
private static final String OHLC_SCHEMA_VERSION = "3";
private static final String SCHEMA_VERSION_PROP = "app.schema.version";
private void initializeOhlcTable() {
@@ -124,9 +125,9 @@ public class SchemaInitializer {
// so that GenericRowData.setField() accepts a plain Long value.
Schema schema = new Schema(
// Primary key fields
required(1, "ticker", Types.StringType.get(), "Market identifier (e.g., BINANCE:BTC/USDT)"),
required(1, "ticker", Types.StringType.get(), "Market identifier in Nautilus format (e.g., BTC/USDT.BINANCE)"),
required(2, "period_seconds", Types.IntegerType.get(), "OHLC period in seconds"),
required(3, "timestamp", Types.LongType.get(), "Candle timestamp in microseconds since epoch"),
required(3, "timestamp", Types.LongType.get(), "Candle timestamp in nanoseconds since epoch"),
// OHLC price data — optional to support gap bars (null = no trades that period)
optional(4, "open", Types.LongType.get(), "Opening price"),
@@ -150,7 +151,7 @@ public class SchemaInitializer {
// Metadata fields
optional(16, "request_id", Types.StringType.get(), "Request ID that generated this data"),
required(17, "ingested_at", Types.LongType.get(), "Timestamp when data was ingested by Flink")
required(17, "ingested_at", Types.LongType.get(), "Timestamp when data was ingested by Flink (nanoseconds since epoch)")
);
// Create the table with partitioning and properties
@@ -176,7 +177,10 @@ public class SchemaInitializer {
/**
* Initialize the symbol_metadata table if it doesn't exist.
*/
private static final String SYMBOL_METADATA_SCHEMA_VERSION = "1";
// v2: removed tick_denom/base_denom/quote_denom; added Nautilus instrument fields
// (price_precision, size_precision, tick_size, lot_size, min_notional,
// margin_init, margin_maint, maker_fee, taker_fee, contract_multiplier)
private static final String SYMBOL_METADATA_SCHEMA_VERSION = "2";
private void initializeSymbolMetadataTable() {
TableIdentifier tableId = TableIdentifier.of(namespace, "symbol_metadata");
@@ -220,24 +224,31 @@ public class SchemaInitializer {
required(2, "market_id", Types.StringType.get(), "Market symbol (e.g., BTC/USDT)"),
// Market information
optional(3, "market_type", Types.StringType.get(), "Market type (spot, futures, swap)"),
optional(3, "market_type", Types.StringType.get(), "Market type (spot, futures, swap, CryptoPerpetual)"),
optional(4, "description", Types.StringType.get(), "Human-readable description"),
optional(5, "base_asset", Types.StringType.get(), "Base asset (e.g., BTC)"),
optional(6, "quote_asset", Types.StringType.get(), "Quote asset (e.g., USDT)"),
// Precision/denominator information
optional(7, "tick_denom", Types.LongType.get(), "Tick price denominator (10^n for n decimals)"),
optional(8, "base_denom", Types.LongType.get(), "Base asset denominator"),
optional(9, "quote_denom", Types.LongType.get(), "Quote asset denominator"),
// Supported timeframes
optional(10, "supported_period_seconds", Types.ListType.ofRequired(11, Types.IntegerType.get()), "Supported OHLC periods in seconds"),
// Optional timing information
optional(12, "earliest_time", Types.LongType.get(), "Earliest available data timestamp (microseconds)"),
optional(12, "earliest_time", Types.LongType.get(), "Earliest available data timestamp (nanoseconds since epoch)"),
// Metadata
required(13, "updated_at", Types.LongType.get(), "Timestamp when metadata was last updated (microseconds)")
required(13, "updated_at", Types.LongType.get(), "Timestamp when metadata was last updated (nanoseconds since epoch)"),
// Nautilus Instrument fields — required for constructing Instrument objects in the sandbox bridge
optional(14, "price_precision", Types.IntegerType.get(), "Decimal places for prices (e.g., 2 means $0.01 resolution)"),
optional(15, "size_precision", Types.IntegerType.get(), "Decimal places for quantities"),
optional(16, "tick_size", Types.DoubleType.get(), "Minimum price increment (e.g., 0.01)"),
optional(17, "lot_size", Types.DoubleType.get(), "Minimum order size"),
optional(18, "min_notional", Types.DoubleType.get(), "Minimum order value in quote currency"),
optional(19, "margin_init", Types.DoubleType.get(), "Initial margin requirement (futures/perps only)"),
optional(20, "margin_maint", Types.DoubleType.get(), "Maintenance margin (futures/perps only)"),
optional(21, "maker_fee", Types.DoubleType.get(), "Maker fee rate (e.g., 0.001 = 0.1%)"),
optional(22, "taker_fee", Types.DoubleType.get(), "Taker fee rate"),
optional(23, "contract_multiplier", Types.DoubleType.get(), "Contract multiplier for derivatives (default 1.0)")
);
// Create the table with partitioning and properties

6
flink/src/main/java/com/dexorder/flink/ingestor/IngestorWorkQueue.java
View File

@@ -108,13 +108,13 @@ public class IngestorWorkQueue {
/**
* Send a data request to ingestors via PUB socket with exchange prefix.
* The topic prefix is extracted from the ticker (e.g., "BINANCE:BTC/USDT" -> "BINANCE:")
* The topic prefix is extracted from the ticker (e.g., "BTC/USDT.BINANCE" -> "BINANCE:")
*/
private void sendToIngestors(DataRequestMessage request) {
try {
byte[] protobufData = request.toProtobuf();
// Extract exchange prefix from ticker (e.g., "BINANCE:BTC/USDT" -> "BINANCE:")
// Extract exchange prefix from ticker (e.g., "BTC/USDT.BINANCE" -> "BINANCE:")
String ticker = request.getTicker();
String exchangePrefix = extractExchangePrefix(ticker);
@@ -143,7 +143,7 @@ public class IngestorWorkQueue {
/**
* Extract exchange prefix from ticker string.
* E.g., "BINANCE:BTC/USDT" -> "BINANCE:"
* E.g., "BTC/USDT.BINANCE" -> "BINANCE:"
*/
private String extractExchangePrefix(String ticker) {
int colonIndex = ticker.indexOf(':');

2
flink/src/main/java/com/dexorder/flink/publisher/HistoryNotificationFunction.java
View File

@@ -119,7 +119,7 @@ public class HistoryNotificationFunction extends ProcessFunction<OHLCBatchWrappe
}
private String getIcebergTableName(String ticker, int periodSeconds) {
// Extract exchange from ticker (e.g., "BINANCE:BTC/USDT" -> "binance")
// Extract exchange from ticker (e.g., "BTC/USDT.BINANCE" -> "binance")
String exchange = ticker.split(":")[0].toLowerCase();
// Convert period to human-readable format

15
flink/src/main/java/com/dexorder/flink/publisher/MarketDeserializer.java
View File

@@ -62,9 +62,6 @@ public class MarketDeserializer implements DeserializationSchema<MarketWrapper>
wrapper.setDescription(market.getDescription());
wrapper.setBaseAsset(market.getBaseAsset());
wrapper.setQuoteAsset(market.getQuoteAsset());
wrapper.setTickDenom(market.getTickDenom());
wrapper.setBaseDenom(market.getBaseDenom());
wrapper.setQuoteDenom(market.getQuoteDenom());
// Convert repeated field to List
List<Integer> supportedPeriods = new ArrayList<>(market.getSupportedPeriodSecondsList());
@@ -72,6 +69,18 @@ public class MarketDeserializer implements DeserializationSchema<MarketWrapper>
wrapper.setEarliestTime(market.getEarliestTime());
// Nautilus Instrument fields
wrapper.setPricePrecision(market.getPricePrecision());
wrapper.setSizePrecision(market.getSizePrecision());
wrapper.setTickSize(market.getTickSize());
wrapper.setLotSize(market.getLotSize());
wrapper.setMinNotional(market.getMinNotional());
wrapper.setMarginInit(market.getMarginInit());
wrapper.setMarginMaint(market.getMarginMaint());
wrapper.setMakerFee(market.getMakerFee());
wrapper.setTakerFee(market.getTakerFee());
wrapper.setContractMultiplier(market.getContractMultiplier());
return wrapper;
} catch (Exception e) {
LOG.error("Failed to deserialize Market protobuf", e);

138
flink/src/main/java/com/dexorder/flink/publisher/MarketWrapper.java
View File

@@ -8,7 +8,7 @@ import java.util.List;
* Represents symbol metadata for a trading pair.
*/
public class MarketWrapper implements Serializable {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 2L;
private String exchangeId;
private String marketId;
@@ -16,119 +16,79 @@ public class MarketWrapper implements Serializable {
private String description;
private String baseAsset;
private String quoteAsset;
private long tickDenom;
private long baseDenom;
private long quoteDenom;
private List<Integer> supportedPeriodSeconds;
private long earliestTime;
// Nautilus Instrument fields
private int pricePrecision;
private int sizePrecision;
private double tickSize;
private double lotSize;
private double minNotional;
private double marginInit;
private double marginMaint;
private double makerFee;
private double takerFee;
private double contractMultiplier;
public MarketWrapper() {
}
public MarketWrapper(String exchangeId, String marketId, String marketType, String description,
String baseAsset, String quoteAsset, long tickDenom, long baseDenom,
long quoteDenom, List<Integer> supportedPeriodSeconds, long earliestTime) {
this.exchangeId = exchangeId;
this.marketId = marketId;
this.marketType = marketType;
this.description = description;
this.baseAsset = baseAsset;
this.quoteAsset = quoteAsset;
this.tickDenom = tickDenom;
this.baseDenom = baseDenom;
this.quoteDenom = quoteDenom;
this.supportedPeriodSeconds = supportedPeriodSeconds;
this.earliestTime = earliestTime;
}
// Getters and setters
public String getExchangeId() {
return exchangeId;
}
public void setExchangeId(String exchangeId) {
this.exchangeId = exchangeId;
}
public String getExchangeId() { return exchangeId; }
public void setExchangeId(String exchangeId) { this.exchangeId = exchangeId; }
public String getMarketId() {
return marketId;
}
public String getMarketId() { return marketId; }
public void setMarketId(String marketId) { this.marketId = marketId; }
public void setMarketId(String marketId) {
this.marketId = marketId;
}
public String getMarketType() { return marketType; }
public void setMarketType(String marketType) { this.marketType = marketType; }
public String getMarketType() {
return marketType;
}
public String getDescription() { return description; }
public void setDescription(String description) { this.description = description; }
public void setMarketType(String marketType) {
this.marketType = marketType;
}
public String getBaseAsset() { return baseAsset; }
public void setBaseAsset(String baseAsset) { this.baseAsset = baseAsset; }
public String getDescription() {
return description;
}
public String getQuoteAsset() { return quoteAsset; }
public void setQuoteAsset(String quoteAsset) { this.quoteAsset = quoteAsset; }
public void setDescription(String description) {
this.description = description;
}
public List<Integer> getSupportedPeriodSeconds() { return supportedPeriodSeconds; }
public void setSupportedPeriodSeconds(List<Integer> supportedPeriodSeconds) { this.supportedPeriodSeconds = supportedPeriodSeconds; }
public String getBaseAsset() {
return baseAsset;
}
public long getEarliestTime() { return earliestTime; }
public void setEarliestTime(long earliestTime) { this.earliestTime = earliestTime; }
public void setBaseAsset(String baseAsset) {
this.baseAsset = baseAsset;
}
public int getPricePrecision() { return pricePrecision; }
public void setPricePrecision(int pricePrecision) { this.pricePrecision = pricePrecision; }
public String getQuoteAsset() {
return quoteAsset;
}
public int getSizePrecision() { return sizePrecision; }
public void setSizePrecision(int sizePrecision) { this.sizePrecision = sizePrecision; }
public void setQuoteAsset(String quoteAsset) {
this.quoteAsset = quoteAsset;
}
public double getTickSize() { return tickSize; }
public void setTickSize(double tickSize) { this.tickSize = tickSize; }
public long getTickDenom() {
return tickDenom;
}
public double getLotSize() { return lotSize; }
public void setLotSize(double lotSize) { this.lotSize = lotSize; }
public void setTickDenom(long tickDenom) {
this.tickDenom = tickDenom;
}
public double getMinNotional() { return minNotional; }
public void setMinNotional(double minNotional) { this.minNotional = minNotional; }
public long getBaseDenom() {
return baseDenom;
}
public double getMarginInit() { return marginInit; }
public void setMarginInit(double marginInit) { this.marginInit = marginInit; }
public void setBaseDenom(long baseDenom) {
this.baseDenom = baseDenom;
}
public double getMarginMaint() { return marginMaint; }
public void setMarginMaint(double marginMaint) { this.marginMaint = marginMaint; }
public long getQuoteDenom() {
return quoteDenom;
}
public double getMakerFee() { return makerFee; }
public void setMakerFee(double makerFee) { this.makerFee = makerFee; }
public void setQuoteDenom(long quoteDenom) {
this.quoteDenom = quoteDenom;
}
public double getTakerFee() { return takerFee; }
public void setTakerFee(double takerFee) { this.takerFee = takerFee; }
public List<Integer> getSupportedPeriodSeconds() {
return supportedPeriodSeconds;
}
public void setSupportedPeriodSeconds(List<Integer> supportedPeriodSeconds) {
this.supportedPeriodSeconds = supportedPeriodSeconds;
}
public long getEarliestTime() {
return earliestTime;
}
public void setEarliestTime(long earliestTime) {
this.earliestTime = earliestTime;
}
public double getContractMultiplier() { return contractMultiplier; }
public void setContractMultiplier(double contractMultiplier) { this.contractMultiplier = contractMultiplier; }
@Override
public String toString() {

2
flink/src/main/java/com/dexorder/flink/sink/IcebergOHLCSink.java
View File

@@ -75,7 +75,7 @@ public class IcebergOHLCSink {
String requestId = batch.getRequestId();
String ticker = batch.getTicker();
int periodSeconds = batch.getPeriodSeconds();
long ingestedAt = System.currentTimeMillis() * 1000;
long ingestedAt = System.currentTimeMillis() * 1_000_000L; // nanoseconds
// Emit one RowData for each OHLC row in the batch
for (OHLCBatchWrapper.OHLCRow row : batch.getRows()) {

21
flink/src/main/java/com/dexorder/flink/sink/SymbolMetadataWriter.java
View File

@@ -87,8 +87,8 @@ public class SymbolMetadataWriter extends RichFlatMapFunction<MarketWrapper, Mar
@Override
public void flatMap(MarketWrapper market, Collector<MarketWrapper> out) throws Exception {
// Create unique key for deduplication
String symbolKey = market.getExchangeId() + ":" + market.getMarketId();
// Create unique key for deduplication (internal key, not stored)
String symbolKey = market.getMarketId() + "." + market.getExchangeId();
// Skip if we've already seen this symbol
if (seenSymbols.contains(symbolKey)) {
@@ -110,16 +110,25 @@ public class SymbolMetadataWriter extends RichFlatMapFunction<MarketWrapper, Mar
record.setField("description", market.getDescription());
record.setField("base_asset", market.getBaseAsset());
record.setField("quote_asset", market.getQuoteAsset());
record.setField("tick_denom", market.getTickDenom());
record.setField("base_denom", market.getBaseDenom());
record.setField("quote_denom", market.getQuoteDenom());
// Convert supported_period_seconds to List<Integer>
List<Integer> supportedPeriods = new ArrayList<>(market.getSupportedPeriodSeconds());
record.setField("supported_period_seconds", supportedPeriods);
record.setField("earliest_time", market.getEarliestTime() != 0 ? market.getEarliestTime() : null);
record.setField("updated_at", System.currentTimeMillis() * 1000); // Current time in microseconds
record.setField("updated_at", System.currentTimeMillis() * 1_000_000L); // Current time in nanoseconds
// Nautilus Instrument fields (populated from exchange API data)
record.setField("price_precision", market.getPricePrecision() != 0 ? market.getPricePrecision() : null);
record.setField("size_precision", market.getSizePrecision() != 0 ? market.getSizePrecision() : null);
record.setField("tick_size", market.getTickSize() != 0.0 ? market.getTickSize() : null);
record.setField("lot_size", market.getLotSize() != 0.0 ? market.getLotSize() : null);
record.setField("min_notional", market.getMinNotional() != 0.0 ? market.getMinNotional() : null);
record.setField("margin_init", market.getMarginInit() != 0.0 ? market.getMarginInit() : null);
record.setField("margin_maint", market.getMarginMaint() != 0.0 ? market.getMarginMaint() : null);
record.setField("maker_fee", market.getMakerFee() != 0.0 ? market.getMakerFee() : null);
record.setField("taker_fee", market.getTakerFee() != 0.0 ? market.getTakerFee() : null);
record.setField("contract_multiplier", market.getContractMultiplier() != 0.0 ? market.getContractMultiplier() : null);
// Get or create writer for this exchange
DataWriter<Record> writer = writersByExchange.get(exchangeId);

2
flink/src/main/resources/iceberg-schemas/ohlc_schema.sql
View File

@@ -9,7 +9,7 @@
CREATE TABLE IF NOT EXISTS trading.ohlc (
-- Primary key fields
ticker STRING NOT NULL COMMENT 'Market identifier (e.g., BINANCE:BTC/USDT)',
ticker STRING NOT NULL COMMENT 'Market identifier (e.g., BTC/USDT.BINANCE)',
period_seconds INT NOT NULL COMMENT 'OHLC period in seconds (60, 300, 900, 3600, 14400, 86400, 604800, etc.)',
timestamp BIGINT NOT NULL COMMENT 'Candle timestamp in microseconds since epoch',