backfill walker

bin/backfill-deploy.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+#usage: "$0 "'{backend|server|web} [''dev''] [config] [deployment] [kubernetes]'
+bin/deploy.sh backend dev finaldata alpha backfill

bin/build.sh

@@ -68,18 +68,19 @@ else
   DEPLOY=0
 fi
 
-REMOTE=gcr.io/cointrader-211623/dexorder/$KUBERNETES:$TAG
+REMOTE=gcr.io/cointrader-211623/dexorder
 
 # todo nocache as a script option
 #docker build --no-cache -f deploy/Dockerfile-$PROJECT -t dexorder/$PROJECT:latest . || exit 1
 echo Building $PROJECT config=$CONFIG deployment=$DEPLOYMENT '=>' $TAG
 docker build -f deploy/docker/Dockerfile-$PROJECT --build-arg="CONFIG=$CONFIG" --build-arg="DEPLOYMENT=$DEPLOYMENT" -t dexorder/$KUBERNETES:latest . || exit 1
 docker tag dexorder/$KUBERNETES:latest dexorder/$KUBERNETES:$TAG
-docker tag dexorder/$KUBERNETES:$TAG $REMOTE
-echo "$(date)" built $REMOTE
+docker tag dexorder/$KUBERNETES:$TAG $REMOTE/$KUBERNETES:$TAG
+docker tag $REMOTE/$KUBERNETES:latest $REMOTE/$KUBERNETES:$TAG
+echo "$(date)" built $REMOTE/$KUBERNETES:$TAG
 
 if [ "$DEPLOY" != "0" ]; then
-  docker push $REMOTE
+  docker push $REMOTE/$KUBERNETES:$TAG $REMOTE/$KUBERNETES:latest
   sed "s#dexorder/$KUBERNETES*#$REMOTE#" deploy/k8s/$KUBERNETES.yaml | kubectl apply -f - || (echo kubectl apply failed && exit 1)
-  echo deployed $KUBERNETES.yaml $REMOTE
+  echo deployed $KUBERNETES.yaml $REMOTE/$KUBERNETES:$TAG
 fi

deploy/k8s/backfill.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ohlc-backfill
+spec:
+  accessModes: [ "ReadWriteOnce" ]
+  storageClassName: retained-resizable
+  resources:
+    requests:
+      storage: 100Gi
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: backfill
+spec:
+  replicas: 1
+  serviceName: backfill
+  selector:
+    matchLabels:
+      app: backfill
+  template:
+    metadata:
+      labels:
+        app: backfill
+    spec:
+      terminationGracePeriodSeconds: 120
+      containers:
+        - name: backfill
+          image: dexorder/backfill
+          command: [ "python", "-u", "-m", "dexorder.bin.finaldata",
+            "rpc_url=https://arb-mainnet.g.alchemy.com/v2/icB000_hg01vv7A1z3DhU59dhDLHioby",
+            "backfill=1", # In the beginning...
+            'walker_flush_interval=60',
+            'walker_name=backfill',
+            "ohlc_dir=/ohlc",
+            'concurrent_rpc_connections=4',
+            "ws_url=",
+          ]
+          volumeMounts:
+            - name: ohlc-backfill
+              mountPath: /ohlc
+          resources:
+            requests:
+              cpu: '.1'
+              memory: 200M
+      volumes:
+        - name: ohlc-backfill
+          persistentVolumeClaim:
+            claimName: ohlc-backfill