Talos; Ethereum node

.gitignore

@@ -7,3 +7,4 @@ node_modules
 /.vscode
 /ohlc
 /tradingview
+talosconfig

deploy/k8s/arb1.yaml

@@ -22,11 +22,10 @@ spec:
           value: 'true'
       containers:
         - name: arb1
-          image: offchainlabs/nitro-node:v2.3.4-b4cc111
+          image: offchainlabs/nitro-node:v3.2.1-d81324d
           command: [
-            '--init.url=https://snapshot.arbitrum.foundation/arb1/nitro-archive.tar',
-            '--parent-chain.connection.url=https://eth-mainnet.g.alchemy.com/v2/XiM8sruAHL_FKkhVdxqsOw2ardZFMZxX',
-            '--parent-chain.blob-client.beacon-url=<TODO>'
+            '--init.latest=archive',
+            '--parent-chain.connection.url=https://eth-mainnet.g.alchemy.com/v2/Ns3gyYixF9sHKT54_ZOrzXzdG5GOqUSS',
           ]
           volumeMounts:
             - name: arb1

deploy/k8s/certificate.yaml

@@ -0,0 +1,73 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: dexorder-trade
+spec:
+  secretName: dexorder-trade-tls
+  commonName: dexorder.trade
+  dnsNames:
+    - dexorder.trade
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+
+
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: beta-dexorder-trade
+spec:
+  secretName: beta-dexorder-trade-tls
+  commonName: beta.dexorder.trade
+  dnsNames:
+    - beta.dexorder.trade
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ws-beta-dexorder-trade-tls
+spec:
+  secretName: ws-beta-dexorder-trade-tls
+  commonName: ws.beta.dexorder.trade
+  dnsNames:
+    - ws.beta.dexorder.trade
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: rpc-eth-beau-dexorder-trade
+  namespace: crypto-node
+spec:
+  secretName: rpc-eth-beau-dexorder-trade-tls
+  commonName: rpc.eth.beau.dexorder.trade
+  dnsNames:
+    - rpc.eth.beau.dexorder.trade
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ws-eth-beau-dexorder-trade
+  namespace: crypto-node
+spec:
+  secretName: ws-eth-beau-dexorder-trade-tls
+  commonName: ws.eth.beau.dexorder.trade
+  dnsNames:
+    - ws.eth.beau.dexorder.trade
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+

deploy/k8s/ethereum.yaml

@@ -0,0 +1,248 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: jwt
+  namespace: crypto-node
+data:
+  # openssl rand -hex 32 | tr -d "\n" | base64
+  jwt.hex: 'NTlhMTYxNmUyMjhhNjM1MDg0NGZiMzM5ZWU1NTliMzkxYmMzNmIxNWVmY2Q4NjY5NDdlNzc1NDRhZDE0OTk4Zg=='
+
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ethereum
+  namespace: crypto-node
+spec:
+  replicas: 1
+  serviceName: ethereum
+  selector:
+    matchLabels:
+      app: ethereum
+  template:
+    metadata:
+      labels:
+        app: ethereum
+    spec:
+      terminationGracePeriodSeconds: 120
+      containers:
+        - name: lighthouse
+          image: sigp/lighthouse
+          command: [
+            'lighthouse', 'beacon_node',
+            '--datadir', '/lighthouse',
+            '--network', 'mainnet',
+            '--http',
+            '--http-address', '0.0.0.0',
+            '--execution-endpoint', 'http://reth:8551',
+            '--execution-jwt', '/jwt/jwt.hex',
+            '--checkpoint-sync-url', 'https://mainnet.checkpoint.sigp.io',
+            '--checkpoint-sync-url-timeout', '300',
+            '--disable-deposit-contract-sync',  # since we are not staking
+            '--historic-state-cache-size', '4',  # hierarchical snap-diff cache
+          ]
+          volumeMounts:
+            - name: lighthouse
+              mountPath: /lighthouse
+            - name: jwt
+              mountPath: /jwt
+              readOnly: true
+          ports:
+            - containerPort: 5052
+            # peering ports (if exposed)
+            - containerPort: 9000
+              protocol: TCP
+            - containerPort: 9000
+              protocol: UDP
+            - containerPort: 9001
+              protocol: UDP
+          resources:
+            requests:
+              cpu: '1'
+              memory: '2G'
+        - name: reth
+          image: ghcr.io/paradigmxyz/reth
+          command: [
+            'reth', 'node',
+            '--datadir', '/reth',
+            '--authrpc.jwtsecret', '/jwt/jwt.hex',
+            '--authrpc.addr', '0.0.0.0',
+            '--authrpc.port', '8551',
+            '--http',
+            '--http.addr', '0.0.0.0',
+#            '--http.api', 'eth,net,trace',
+            '--http.api', 'eth',
+            '--ws',
+          ]
+          volumeMounts:
+            - name: reth
+              mountPath: /reth
+            - name: jwt
+              mountPath: /jwt
+              readOnly: true
+          ports:
+            # peering ports (if exposed)
+            - containerPort: 30303
+              protocol: TCP
+            - containerPort: 30303
+              protocol: UDP
+            # execution port
+            - containerPort: 8551
+              protocol: TCP
+              name: execution
+            # rpc port
+            - containerPort: 8545
+              protocol: TCP
+              name: rpc
+            # ws port
+            - containerPort: 8546
+              protocol: TCP
+              name: ws
+          resources:
+            requests:
+              cpu: '2'
+              memory: '10G'
+      volumes:
+        - name: jwt
+          secret:
+            secretName: jwt
+        - name: lighthouse
+          persistentVolumeClaim:
+            claimName: lighthouse
+        - name: reth
+          persistentVolumeClaim:
+            claimName: reth
+  volumeClaimTemplates:
+    - metadata:
+        name: lighthouse
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        volumeName: lighthouse
+        storageClassName: bulk-a
+        resources:
+          requests:
+            storage: 2Ti
+    - metadata:
+        name: reth
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        volumeName: reth
+        storageClassName: bulk-a
+        resources:
+          requests:
+            storage: 10Ti
+
+#---
+#apiVersion: v1
+#kind: Service
+#metadata:
+#  name: lighthouse-peers
+#  namespace: crypto-node
+#spec:
+#  selector:
+#    app: ethereum
+#  type: NodePort
+#  ports:
+#    - protocol: TCP
+#      port: 9000
+#      name: peer1
+#    - protocol: UDP
+#      port: 9000
+#      name: peer2
+#    - protocol: UDP
+#      port: 9001
+#      name: peer3
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: reth
+  namespace: crypto-node
+spec:
+  selector:
+    app: ethereum
+  ports:
+#    - protocol: TCP
+#      port: 30303
+#      name: peer1
+#    - protocol: UDP
+#      port: 30303
+#      name: peer2
+    # expose these ports for the lighthouse container
+    - protocol: TCP
+      port: 8551
+      name: execution
+    - protocol: TCP
+      port: 8546
+      name: ws
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: eth-rpc
+  namespace: crypto-node
+spec:
+  selector:
+    app: ethereum
+  ports:
+    - protocol: TCP
+      port: 8545
+      name: rpc
+    - protocol: TCP
+      port: 8546
+      name: ws
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: eth-rpc
+  namespace: crypto-node
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - rpc.eth.beau.dexorder.trade
+      secretName: rpc-eth-beau-dexorder-trade-tls
+  rules:
+    - host: rpc.eth.beau.dexorder.trade
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: eth-rpc
+                port:
+                  number: 8545
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: eth-ws
+  namespace: crypto-node
+  annotations:
+    # websocket connection timeouts
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "86400"
+    nginx.ingress.kubernetes.io/proxy-write-timeout: "86400"
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - ws.eth.beau.dexorder.trade
+      secretName: ws-eth-beau-dexorder-trade-tls
+  rules:
+    - host: ws.eth.beau.dexorder.trade
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: eth-rpc
+                port:
+                  number: 8546

deploy/k8s/ingress.yaml

@@ -104,48 +104,3 @@ spec:
                 name: server
                 port:
                   number: 3001
-
-
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: dexorder-trade
-spec:
-  secretName: dexorder-trade-tls
-  commonName: dexorder.trade
-  dnsNames:
-    - dexorder.trade
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
-
-
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: beta-dexorder-trade
-spec:
-  secretName: beta-dexorder-trade-tls
-  commonName: beta.dexorder.trade
-  dnsNames:
-    - beta.dexorder.trade
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
-
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: ws-beta-dexorder-trade-tls
-spec:
-  secretName: ws-beta-dexorder-trade-tls
-  commonName: ws.beta.dexorder.trade
-  dnsNames:
-    - ws.beta.dexorder.trade
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
-

deploy/k8s/init-cluster-talos.yaml

@@ -0,0 +1,43 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    email: tim@dexorder.trade
+    server: https://acme-v02.api.letsencrypt.org/directory
+    privateKeySecretRef:
+      name: letsencrypt-privkey-prod
+    solvers:
+    - http01:
+        ingress:
+          ingressClassName: nginx
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: lighthouse
+spec:
+  storageClassName: bulk-a
+  capacity:
+    storage: 10Ti
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /var/mnt/bulk/a/lighthouse
+
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: reth
+spec:
+  storageClassName: bulk-a
+  capacity:
+    storage: 10Ti
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /var/mnt/bulk/a/reth

deploy/k8s/redis.yaml

@@ -1,10 +1,11 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: redis
   labels:
     app: redis
 spec:
+  serviceName: "redis"
   replicas: 1
   selector:
     matchLabels:
@@ -15,30 +16,14 @@ spec:
         app: redis
     spec:
       containers:
-      - name: redis
-        image: docker.dragonflydb.io/dragonflydb/dragonfly:latest
-        ports:
-        - containerPort: 6379
-        resources:
-          limits:
-            cpu: '1'
-            memory: 2Gi
-          requests:
-            cpu: 100m
-            memory: 500Mi
-
-
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-  labels:
-    app: redis
-spec:
-  selector:
-    app: redis
-  ports:
-    - protocol: TCP
-      port: 6379
-      name: redis
+        - name: redis
+          image: docker.dragonflydb.io/dragonflydb/dragonfly:latest
+          ports:
+            - containerPort: 6379
+          resources:
+            limits:
+              cpu: '1'
+              memory: 2Gi
+            requests:
+              cpu: 100m
+              memory: 500Mi

deploy/talos/local-static-provisioner.helm.yaml

@@ -0,0 +1,61 @@
+classes:
+  - name: bulk-a # Defines name of storage classes.
+    hostDir: /var/mnt/bulk/a
+    mountDir: /var/mnt/bulk/a
+    volumeMode: Filesystem
+    fsType: ext4
+    namePattern: "*"
+    allowedTopologies:
+    blockCleanerCommand:
+      #  Do a quick reset of the block device during its cleanup.
+      - "/scripts/quick_reset.sh"
+      #  or use dd to zero out block dev in two iterations by uncommenting these lines
+      #  - "/scripts/dd_zero.sh"
+      #  - "2"
+      # or run shred utility for 2 iteration.s
+      # - "/scripts/shred.sh"
+      # - "2"
+      # or blkdiscard utility by uncommenting the line below.
+      #  - "/scripts/blkdiscard.sh"
+    storageClass:
+      reclaimPolicy: Retain
+  - name: bulk-b # Defines name of storage classes.
+    hostDir: /var/mnt/bulk/b
+    mountDir: /var/mnt/bulk/b
+    volumeMode: Filesystem
+    fsType: ext4
+    namePattern: "*"
+    allowedTopologies:
+    blockCleanerCommand:
+      #  Do a quick reset of the block device during its cleanup.
+      - "/scripts/quick_reset.sh"
+      #  or use dd to zero out block dev in two iterations by uncommenting these lines
+      #  - "/scripts/dd_zero.sh"
+      #  - "2"
+      # or run shred utility for 2 iteration.s
+      #  - "/scripts/shred.sh"
+      #  - "2"
+      # or blkdiscard utility by uncommenting the line below.
+      #  - "/scripts/blkdiscard.sh"
+    storageClass:
+      reclaimPolicy: Retain
+  - name: fast-a # Defines name of storage classes.
+    hostDir: /var/mnt/fast/a
+    mountDir: /var/mnt/fast/a
+    volumeMode: Filesystem
+    fsType: ext4
+    namePattern: "*"
+    allowedTopologies:
+    blockCleanerCommand:
+      #  Do a quick reset of the block device during its cleanup.
+      - "/scripts/quick_reset.sh"
+      #  or use dd to zero out block dev in two iterations by uncommenting these lines
+      #  - "/scripts/dd_zero.sh"
+      #  - "2"
+      # or run shred utility for 2 iteration.s
+      #  - "/scripts/shred.sh"
+      #  - "2"
+      # or blkdiscard utility by uncommenting the line below.
+      #  - "/scripts/blkdiscard.sh"
+    storageClass:
+      reclaimPolicy: Retain

doc/new-cluster.md

@@ -1,5 +1,8 @@
-1. Create cluster in GKE
-2. [Install ingress-nginx](https://kubernetes.github.io/ingress-nginx/deploy/#gce-gke)
+1. Create k8s cluster
+2. [Install ingress-nginx](https://kubernetes.github.io/ingress-nginx/deploy/)
+   * for Talos, I deployed nginx as a daemonset using hostPort mode to bind 80/443. This also required changes to the 
+     * `helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace --values ingress-nginx.helm.yaml`
+   * local storage https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner/blob/master/helm/README.md#install-local-volume-provisioner-with-helm
 3. [Install cert-manager](https://cert-manager.io/docs/installation/kubectl/)
 4. `cd deploy/k8s`
 5. `kubectl apply -f init-cluster.yaml -f postgres.yaml -f redis.yaml -f ingress.yaml`