WIP

Signed-off-by: Alejandro Mery <amery@jpi.io>

.env

@@ -0,0 +1,51 @@
+export CLOUDFLARE_DNS_API_TOKEN=wFpklBgp0Z1A4yDs5zNhlTKne3W2Si8GLwkl10Oz
+
+if [ -n "$JPICTL" ]; then
+	:
+elif JPICTL=$(which jpictl); then
+	echo 2 >&2
+elif [ -d ./cmd/jpictl ]; then
+	JPICTL="go run ./cmd/jpictl/"
+	if [ -d "$(go env GOBIN)" ]; then
+		echo 3a >&2
+	elif [ -d "$(go env GOPATH)" ]; then
+		export GOBIN="$(go env GOPATH)/bin"
+		echo 3b >&2
+	elif WS=$(x --root); then
+		export GOBIN="$WS/bin"
+		unset -v WS
+		echo 3c >&2
+	else
+		echo 3d >&2
+	fi
+elif [ -d "${WS:-}" -a -x "${WS:+$WS/bin/jpictl}" ]; then
+	JPICTL="$WS/bin/jpictl"
+	export PATH="$WS/bin:$PATH"
+	echo "4a" >&2
+else
+	for WS in \
+		"$(x --root)" \
+		"$HOME/projects/apptly" \
+		; do
+		if [ -x "$WS/bin/jpictl" ]; then
+			JPICTL="$WS/bin/jpictl"
+			export PATH="$WS/bin:$PATH"
+			echo "4b: '$WS'" >&2
+			break
+		fi
+	done
+	unset -v WS
+
+	[ -n "$JPICTL" ] || echo "4c" >&2
+fi
+
+if [ -n "$JPICTL" ]; then
+	echo "JPICTL='$JPICTL'" >&2
+
+	export JPICTL
+	jpictl() {
+		$JPICTL "$@"
+	}
+else
+	echo "JPICTL="
+fi

.vscode/launch.json

@@ -0,0 +1,38 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "write",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "cmd/jpictl",
+            "args": [ "write" ],
+            "cwd": "${workspaceFolder}"
+        },
+        {
+            "name": "dump",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "cmd/jpictl",
+            "args": [ "dump", "-vvvv" ],
+            "cwd": "${workspaceFolder}"
+        },
+        {
+            "name": "env",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "cmd/jpictl",
+            "args": [ "env", "-vvvv" ],
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}
+
+    ]
+}

cloud.yaml

@@ -0,0 +1,144 @@
+dir: m
+name: m
+domain: jpi.cloud
+ceph_fsid: 28180b9b-6d5d-4be0-bb5b-e4b04bf03804
+regions:
+  - name: de
+  - name: eu
+    id: 1
+    cluster: ""
+    regions:
+      - de
+      - nl
+  - name: nl
+  - name: uk
+    id: 2
+    cluster: ""
+  - name: us
+    id: 3
+    cluster: ""
+  - name: europe
+    regions:
+      - eu
+      - uk
+  - name: global
+    regions:
+      - europe
+      - us
+zones:
+  - id: 1
+    name: ssd-lon
+    regions:
+      - uk
+    machines:
+      - id: 3
+        public:
+          - 63.250.53.5
+          - 2602:ff16:7:0:1:66:0:1
+        rings:
+          - ring: 0
+            enabled: true
+            keys:
+              privatekey: WJY1yrd14c0gY4qqq1O8BPck46TVYebXrT+OoMbEqG8=
+              publickey: aDXNwapVjufcuQKfBN+waJObe1TaeeaSNl1suA0u+00=
+          - ring: 1
+            enabled: true
+            keys:
+              privatekey: 2CDO+x8HVKfw1/iqC2WkwIEQyWZsC9XPW1otqxarR0o=
+              publickey: untmMfW2W4OiMgG/6nAysoivXtDh7cZ9/rxhgJ4OlC4=
+      - id: 7
+        ceph_monitor: true
+        public:
+          - 107.155.122.3
+          - 2602:ff16:7:0:1:6d:0:1
+        rings:
+          - ring: 1
+            enabled: true
+            keys:
+              privatekey: iN4Rc3J36APlkIP/ksSpHIwAuxC/ehYLwGQ5va1AhG8=
+              publickey: XGVXDmxQajHpsgpzgrp+r/ZWxZUaodTpvGBJuLlZ0n4=
+  - id: 2
+    name: ssd-ams
+    regions:
+      - nl
+    machines:
+      - id: 3
+        public:
+          - 89.233.107.128
+          - 2602:ff16:9:0:1:1a7:0:1
+        rings:
+          - ring: 0
+            enabled: true
+            keys:
+              privatekey: oHVQY976pFH1lAhCrz9c40vR/e5PIXw7ZXv6pU6A5Ho=
+              publickey: y/Gkid2Mjxo1WO9Zi7mh+DETNYKT/AAc3K3bvr9xp3c=
+          - ring: 1
+            enabled: true
+            keys:
+              privatekey: 6I/ZMxaWUTbNc+SIGkNKxgovONqv6EAvg7NeeTRrR1c=
+              publickey: 9i0XI358OOYYoxoxV5lX8siqE8uiEwCsuWLo6LoFu0Y=
+      - id: 4
+        ceph_monitor: true
+        public:
+          - 89.233.107.251
+          - 2602:ff16:9:0:1:130:0:1
+        rings:
+          - ring: 1
+            enabled: true
+            keys:
+              privatekey: SPBd5Ka29ZecKYy7BW85rW7FO9dwKwoUHlSibybPG1s=
+              publickey: QpJupYlnJc7wc3+gAlEmbFMonsqpVnYRy7f/gnPqqBw=
+      - id: 5
+        inactive: true
+        public:
+          - 89.233.107.107
+          - 2602:ff16:9:0:1:74:0:1
+        rings:
+          - ring: 1
+            enabled: true
+            keys:
+              privatekey: mJaFvu9TBwE2i0rjhfhvpB3xZmp++3BVwa7QAvWXImY=
+              publickey: HDLVEwLv0uugyZ9C1C9ZWuiaWQ1qIqqe/DAUxYI0whM=
+  - id: 3
+    name: htz-fsn
+    regions:
+      - de
+    machines:
+      - id: 1
+        ceph_monitor: true
+        public:
+          - 157.90.209.125
+          - 2a01:4f8:252:168f::2
+        rings:
+          - ring: 0
+            enabled: true
+            keys:
+              privatekey: AMz/pCgOMVh1/2rT9uCFMox+LQgH5FrE4xUeqSs0x0M=
+              publickey: K9LYog8vtRFN4nW6dJWMQ9trmRrYuLpbu/Ze5kIiBGw=
+          - ring: 1
+            enabled: true
+            keys:
+              privatekey: 2JpA1G0Dp+sEuDhKnzToz0h76o2iLP+S7Qb9itlgXVA=
+              publickey: qWmYfgeRAQ563gWLdeYs65XjGJTSE+W8WTbpLZ0e1CQ=
+  - id: 4
+    name: ssd-nyc
+    regions:
+      - us
+    machines:
+      - id: 3
+        inactive: true
+        ceph_monitor: true
+        public:
+          - 208.87.134.41
+          - 2602:ff16:3:0:1:4fa:0:1
+        rings:
+          - ring: 0
+            enabled: true
+            keys:
+              privatekey: YNDxj+QSO3p5pXw9h7lBx0cLM0kJDv3v7BYLfc8TR3I=
+              publickey: 1UFgleUufTszPM1voIcwVhd0DpYZ1HwV05U6p3IS2DE=
+          - ring: 1
+            enabled: true
+            keys:
+              privatekey: APXGKqOxx1gOr587FSN3O3gRGpkGgSjt6zGlTu49UlU=
+              publickey: av8ni/9ZUyozabzqTjIy1sOnSJuQ9p63Tu9ECVmHB2I=

cmd/jpictl/list.go

@@ -108,7 +108,7 @@ func (*inventory) renderRingOneZone(out *tools.LazyBuffer, r *cluster.Region, z
 	z.ForEachMachine(func(m *cluster.Machine) bool {
 		addr := m.RingOneAddress()
 		cidr := netip.PrefixFrom(addr, 32)
-		_ = out.Printf("%s\t\t%s\n", cidr, m.Name)
+		_ = out.Printf("%s\t\t%s-%v\n", cidr, m.Name, 1)
 		return false
 	})
 	return nil

jpictl.code-workspace

@@ -0,0 +1,14 @@
+{
+	"folders": [
+		{
+			"path": "."
+		},
+		{
+			"path": "../../../bitbucket.org/jpi/m.jpi.cloud"
+		},
+		{
+			"path": "../../../darvaza.org/core"
+		}
+	],
+	"settings": {}
+}

m/ceph.conf

@@ -0,0 +1,8 @@
+[global]
+fsid = 28180b9b-6d5d-4be0-bb5b-e4b04bf03804
+mon_initial_members = ssd-lon-7, ssd-ams-4, htz-fsn-1, ssd-nyc-3
+mon_host = 10.2.16.7, 10.1.32.4, 10.1.48.1, 10.3.64.3
+cluster_network = 10.0.0.0/8
+
+; don't rewrite labels on startup
+osd_class_update_on_start = false

m/eu/regions

@@ -0,0 +1,2 @@
+de
+nl

m/europe/regions

@@ -0,0 +1,2 @@
+eu
+uk

m/global/regions

@@ -0,0 +1,2 @@
+europe
+us

m/htz-fsn/htz-fsn-1/hosts

@@ -0,0 +1,22 @@
+127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+10.2.16.3  ssd-lon-3
+10.2.16.7  ssd-lon-7 ssd-lon-ceph ssd-lon-k3s
+10.1.32.3  ssd-ams-3
+10.1.32.4  ssd-ams-4 ssd-ams-ceph ssd-ams-k3s
+10.1.32.5  ssd-ams-5
+10.1.48.1  htz-fsn-1 htz-fsn-ceph htz-fsn-k3s ceph k3s
+10.3.64.3  ssd-nyc-3 ssd-nyc-ceph ssd-nyc-k3s
+
+10.0.33.3  ssd-lon-3-0
+10.0.18.3  ssd-ams-3-0
+10.0.19.1  htz-fsn-1-0
+10.0.52.3  ssd-nyc-3-0

m/htz-fsn/htz-fsn-1/wg0.conf

@@ -0,0 +1,23 @@
+[Interface]
+# Name: htz-fsn-1-0
+Address = 10.0.19.1
+PrivateKey = AMz/pCgOMVh1/2rT9uCFMox+LQgH5FrE4xUeqSs0x0M=
+ListenPort = 51800
+
+[Peer]
+# Name: ssd-lon-3-0
+PublicKey = aDXNwapVjufcuQKfBN+waJObe1TaeeaSNl1suA0u+00=
+Endpoint = ssd-lon-3.m.jpi.cloud:51800
+AllowedIPs = 10.2.16.0/20, 10.0.33.3/32
+
+[Peer]
+# Name: ssd-ams-3-0
+PublicKey = y/Gkid2Mjxo1WO9Zi7mh+DETNYKT/AAc3K3bvr9xp3c=
+Endpoint = ssd-ams-3.m.jpi.cloud:51800
+AllowedIPs = 10.1.32.0/20, 10.0.18.3/32
+
+[Peer]
+# Name: ssd-nyc-3-0
+PublicKey = 1UFgleUufTszPM1voIcwVhd0DpYZ1HwV05U6p3IS2DE=
+Endpoint = ssd-nyc-3.m.jpi.cloud:51800
+AllowedIPs = 10.3.64.0/20, 10.0.52.3/32

m/htz-fsn/htz-fsn-1/wg0.key

@@ -0,0 +1 @@
+AMz/pCgOMVh1/2rT9uCFMox+LQgH5FrE4xUeqSs0x0M=

m/htz-fsn/htz-fsn-1/wg0.pub

@@ -0,0 +1 @@
+K9LYog8vtRFN4nW6dJWMQ9trmRrYuLpbu/Ze5kIiBGw=

m/htz-fsn/htz-fsn-1/wg1.conf

@@ -0,0 +1,5 @@
+[Interface]
+# Name: htz-fsn-1-1
+Address = 10.1.48.1
+PrivateKey = 2JpA1G0Dp+sEuDhKnzToz0h76o2iLP+S7Qb9itlgXVA=
+ListenPort = 51810

m/htz-fsn/htz-fsn-1/wg1.key

@@ -0,0 +1 @@
+2JpA1G0Dp+sEuDhKnzToz0h76o2iLP+S7Qb9itlgXVA=

m/htz-fsn/htz-fsn-1/wg1.pub

@@ -0,0 +1 @@
+qWmYfgeRAQ563gWLdeYs65XjGJTSE+W8WTbpLZ0e1CQ=

m/htz-fsn/regions

@@ -0,0 +1 @@
+de

m/ssd-ams/regions

@@ -0,0 +1 @@
+nl

m/ssd-ams/ssd-ams-3/hosts

@@ -0,0 +1,22 @@
+127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+10.2.16.3  ssd-lon-3
+10.2.16.7  ssd-lon-7 ssd-lon-ceph ssd-lon-k3s
+10.1.32.3  ssd-ams-3
+10.1.32.4  ssd-ams-4 ssd-ams-ceph ssd-ams-k3s ceph k3s
+10.1.32.5  ssd-ams-5
+10.1.48.1  htz-fsn-1 htz-fsn-ceph htz-fsn-k3s
+10.3.64.3  ssd-nyc-3 ssd-nyc-ceph ssd-nyc-k3s
+
+10.0.33.3  ssd-lon-3-0
+10.0.18.3  ssd-ams-3-0
+10.0.19.1  htz-fsn-1-0
+10.0.52.3  ssd-nyc-3-0

m/ssd-ams/ssd-ams-3/wg0.conf

@@ -0,0 +1,23 @@
+[Interface]
+# Name: ssd-ams-3-0
+Address = 10.0.18.3
+PrivateKey = oHVQY976pFH1lAhCrz9c40vR/e5PIXw7ZXv6pU6A5Ho=
+ListenPort = 51800
+
+[Peer]
+# Name: ssd-lon-3-0
+PublicKey = aDXNwapVjufcuQKfBN+waJObe1TaeeaSNl1suA0u+00=
+Endpoint = ssd-lon-3.m.jpi.cloud:51800
+AllowedIPs = 10.2.16.0/20, 10.0.33.3/32
+
+[Peer]
+# Name: htz-fsn-1-0
+PublicKey = K9LYog8vtRFN4nW6dJWMQ9trmRrYuLpbu/Ze5kIiBGw=
+Endpoint = htz-fsn-1.m.jpi.cloud:51800
+AllowedIPs = 10.1.48.0/20, 10.0.19.1/32
+
+[Peer]
+# Name: ssd-nyc-3-0
+PublicKey = 1UFgleUufTszPM1voIcwVhd0DpYZ1HwV05U6p3IS2DE=
+Endpoint = ssd-nyc-3.m.jpi.cloud:51800
+AllowedIPs = 10.3.64.0/20, 10.0.52.3/32

m/ssd-ams/ssd-ams-3/wg0.key

@@ -0,0 +1 @@
+oHVQY976pFH1lAhCrz9c40vR/e5PIXw7ZXv6pU6A5Ho=

m/ssd-ams/ssd-ams-3/wg0.pub

@@ -0,0 +1 @@
+y/Gkid2Mjxo1WO9Zi7mh+DETNYKT/AAc3K3bvr9xp3c=

m/ssd-ams/ssd-ams-3/wg1.conf

@@ -0,0 +1,17 @@
+[Interface]
+# Name: ssd-ams-3-1
+Address = 10.1.32.3
+PrivateKey = 6I/ZMxaWUTbNc+SIGkNKxgovONqv6EAvg7NeeTRrR1c=
+ListenPort = 51810
+
+[Peer]
+# Name: ssd-ams-4-1
+PublicKey = QpJupYlnJc7wc3+gAlEmbFMonsqpVnYRy7f/gnPqqBw=
+Endpoint = ssd-ams-4.m.jpi.cloud:51810
+AllowedIPs = 10.1.32.4/32
+
+[Peer]
+# Name: ssd-ams-5-1
+PublicKey = HDLVEwLv0uugyZ9C1C9ZWuiaWQ1qIqqe/DAUxYI0whM=
+Endpoint = ssd-ams-5.m.jpi.cloud:51810
+AllowedIPs = 10.1.32.5/32

m/ssd-ams/ssd-ams-3/wg1.key

@@ -0,0 +1 @@
+6I/ZMxaWUTbNc+SIGkNKxgovONqv6EAvg7NeeTRrR1c=

m/ssd-ams/ssd-ams-3/wg1.pub

@@ -0,0 +1 @@
+9i0XI358OOYYoxoxV5lX8siqE8uiEwCsuWLo6LoFu0Y=

m/ssd-ams/ssd-ams-4/hosts

@@ -0,0 +1,22 @@
+127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+10.2.16.3  ssd-lon-3
+10.2.16.7  ssd-lon-7 ssd-lon-ceph ssd-lon-k3s
+10.1.32.3  ssd-ams-3
+10.1.32.4  ssd-ams-4 ssd-ams-ceph ssd-ams-k3s ceph k3s
+10.1.32.5  ssd-ams-5
+10.1.48.1  htz-fsn-1 htz-fsn-ceph htz-fsn-k3s
+10.3.64.3  ssd-nyc-3 ssd-nyc-ceph ssd-nyc-k3s
+
+10.0.33.3  ssd-lon-3-0
+10.0.18.3  ssd-ams-3-0
+10.0.19.1  htz-fsn-1-0
+10.0.52.3  ssd-nyc-3-0

m/ssd-ams/ssd-ams-4/wg1.conf

@@ -0,0 +1,17 @@
+[Interface]
+# Name: ssd-ams-4-1
+Address = 10.1.32.4
+PrivateKey = SPBd5Ka29ZecKYy7BW85rW7FO9dwKwoUHlSibybPG1s=
+ListenPort = 51810
+
+[Peer]
+# Name: ssd-ams-3-1
+PublicKey = 9i0XI358OOYYoxoxV5lX8siqE8uiEwCsuWLo6LoFu0Y=
+Endpoint = ssd-ams-3.m.jpi.cloud:51810
+AllowedIPs = 10.1.32.3/32, 10.2.16.0/20, 10.1.48.0/20, 10.3.64.0/20, 10.0.33.3/32, 10.0.18.3/32, 10.0.19.1/32, 10.0.52.3/32
+
+[Peer]
+# Name: ssd-ams-5-1
+PublicKey = HDLVEwLv0uugyZ9C1C9ZWuiaWQ1qIqqe/DAUxYI0whM=
+Endpoint = ssd-ams-5.m.jpi.cloud:51810
+AllowedIPs = 10.1.32.5/32

m/ssd-ams/ssd-ams-4/wg1.key

@@ -0,0 +1 @@
+SPBd5Ka29ZecKYy7BW85rW7FO9dwKwoUHlSibybPG1s=

m/ssd-ams/ssd-ams-4/wg1.pub

@@ -0,0 +1 @@
+QpJupYlnJc7wc3+gAlEmbFMonsqpVnYRy7f/gnPqqBw=

m/ssd-ams/ssd-ams-5/hosts

@@ -0,0 +1,22 @@
+127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+10.2.16.3  ssd-lon-3
+10.2.16.7  ssd-lon-7 ssd-lon-ceph ssd-lon-k3s
+10.1.32.3  ssd-ams-3
+10.1.32.4  ssd-ams-4 ssd-ams-ceph ssd-ams-k3s ceph k3s
+10.1.32.5  ssd-ams-5
+10.1.48.1  htz-fsn-1 htz-fsn-ceph htz-fsn-k3s
+10.3.64.3  ssd-nyc-3 ssd-nyc-ceph ssd-nyc-k3s
+
+10.0.33.3  ssd-lon-3-0
+10.0.18.3  ssd-ams-3-0
+10.0.19.1  htz-fsn-1-0
+10.0.52.3  ssd-nyc-3-0

m/ssd-ams/ssd-ams-5/region

@@ -0,0 +1 @@
+none

m/ssd-ams/ssd-ams-5/wg1.conf

@@ -0,0 +1,17 @@
+[Interface]
+# Name: ssd-ams-5-1
+Address = 10.1.32.5
+PrivateKey = mJaFvu9TBwE2i0rjhfhvpB3xZmp++3BVwa7QAvWXImY=
+ListenPort = 51810
+
+[Peer]
+# Name: ssd-ams-3-1
+PublicKey = 9i0XI358OOYYoxoxV5lX8siqE8uiEwCsuWLo6LoFu0Y=
+Endpoint = ssd-ams-3.m.jpi.cloud:51810
+AllowedIPs = 10.1.32.3/32, 10.2.16.0/20, 10.1.48.0/20, 10.3.64.0/20, 10.0.33.3/32, 10.0.18.3/32, 10.0.19.1/32, 10.0.52.3/32
+
+[Peer]
+# Name: ssd-ams-4-1
+PublicKey = QpJupYlnJc7wc3+gAlEmbFMonsqpVnYRy7f/gnPqqBw=
+Endpoint = ssd-ams-4.m.jpi.cloud:51810
+AllowedIPs = 10.1.32.4/32

m/ssd-ams/ssd-ams-5/wg1.key

@@ -0,0 +1 @@
+mJaFvu9TBwE2i0rjhfhvpB3xZmp++3BVwa7QAvWXImY=

m/ssd-ams/ssd-ams-5/wg1.pub

@@ -0,0 +1 @@
+HDLVEwLv0uugyZ9C1C9ZWuiaWQ1qIqqe/DAUxYI0whM=

m/ssd-lon/regions

@@ -0,0 +1 @@
+uk

m/ssd-lon/ssd-lon-3/hosts

@@ -0,0 +1,22 @@
+127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+10.2.16.3  ssd-lon-3
+10.2.16.7  ssd-lon-7 ssd-lon-ceph ssd-lon-k3s ceph k3s
+10.1.32.3  ssd-ams-3
+10.1.32.4  ssd-ams-4 ssd-ams-ceph ssd-ams-k3s
+10.1.32.5  ssd-ams-5
+10.1.48.1  htz-fsn-1 htz-fsn-ceph htz-fsn-k3s
+10.3.64.3  ssd-nyc-3 ssd-nyc-ceph ssd-nyc-k3s
+
+10.0.33.3  ssd-lon-3-0
+10.0.18.3  ssd-ams-3-0
+10.0.19.1  htz-fsn-1-0
+10.0.52.3  ssd-nyc-3-0

m/ssd-lon/ssd-lon-3/wg0.conf

@@ -0,0 +1,23 @@
+[Interface]
+# Name: ssd-lon-3-0
+Address = 10.0.33.3
+PrivateKey = WJY1yrd14c0gY4qqq1O8BPck46TVYebXrT+OoMbEqG8=
+ListenPort = 51800
+
+[Peer]
+# Name: ssd-ams-3-0
+PublicKey = y/Gkid2Mjxo1WO9Zi7mh+DETNYKT/AAc3K3bvr9xp3c=
+Endpoint = ssd-ams-3.m.jpi.cloud:51800
+AllowedIPs = 10.1.32.0/20, 10.0.18.3/32
+
+[Peer]
+# Name: htz-fsn-1-0
+PublicKey = K9LYog8vtRFN4nW6dJWMQ9trmRrYuLpbu/Ze5kIiBGw=
+Endpoint = htz-fsn-1.m.jpi.cloud:51800
+AllowedIPs = 10.1.48.0/20, 10.0.19.1/32
+
+[Peer]
+# Name: ssd-nyc-3-0
+PublicKey = 1UFgleUufTszPM1voIcwVhd0DpYZ1HwV05U6p3IS2DE=
+Endpoint = ssd-nyc-3.m.jpi.cloud:51800
+AllowedIPs = 10.3.64.0/20, 10.0.52.3/32

m/ssd-lon/ssd-lon-3/wg0.key

@@ -0,0 +1 @@
+WJY1yrd14c0gY4qqq1O8BPck46TVYebXrT+OoMbEqG8=

m/ssd-lon/ssd-lon-3/wg0.pub

@@ -0,0 +1 @@
+aDXNwapVjufcuQKfBN+waJObe1TaeeaSNl1suA0u+00=

m/ssd-lon/ssd-lon-3/wg1.conf

@@ -0,0 +1,11 @@
+[Interface]
+# Name: ssd-lon-3-1
+Address = 10.2.16.3
+PrivateKey = 2CDO+x8HVKfw1/iqC2WkwIEQyWZsC9XPW1otqxarR0o=
+ListenPort = 51810
+
+[Peer]
+# Name: ssd-lon-7-1
+PublicKey = XGVXDmxQajHpsgpzgrp+r/ZWxZUaodTpvGBJuLlZ0n4=
+Endpoint = ssd-lon-7.m.jpi.cloud:51810
+AllowedIPs = 10.2.16.7/32

m/ssd-lon/ssd-lon-3/wg1.key

@@ -0,0 +1 @@
+2CDO+x8HVKfw1/iqC2WkwIEQyWZsC9XPW1otqxarR0o=

m/ssd-lon/ssd-lon-3/wg1.pub

@@ -0,0 +1 @@
+untmMfW2W4OiMgG/6nAysoivXtDh7cZ9/rxhgJ4OlC4=

m/ssd-lon/ssd-lon-7/hosts

@@ -0,0 +1,22 @@
+127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+10.2.16.3  ssd-lon-3
+10.2.16.7  ssd-lon-7 ssd-lon-ceph ssd-lon-k3s ceph k3s
+10.1.32.3  ssd-ams-3
+10.1.32.4  ssd-ams-4 ssd-ams-ceph ssd-ams-k3s
+10.1.32.5  ssd-ams-5
+10.1.48.1  htz-fsn-1 htz-fsn-ceph htz-fsn-k3s
+10.3.64.3  ssd-nyc-3 ssd-nyc-ceph ssd-nyc-k3s
+
+10.0.33.3  ssd-lon-3-0
+10.0.18.3  ssd-ams-3-0
+10.0.19.1  htz-fsn-1-0
+10.0.52.3  ssd-nyc-3-0

m/ssd-lon/ssd-lon-7/wg1.conf

@@ -0,0 +1,11 @@
+[Interface]
+# Name: ssd-lon-7-1
+Address = 10.2.16.7
+PrivateKey = iN4Rc3J36APlkIP/ksSpHIwAuxC/ehYLwGQ5va1AhG8=
+ListenPort = 51810
+
+[Peer]
+# Name: ssd-lon-3-1
+PublicKey = untmMfW2W4OiMgG/6nAysoivXtDh7cZ9/rxhgJ4OlC4=
+Endpoint = ssd-lon-3.m.jpi.cloud:51810
+AllowedIPs = 10.2.16.3/32, 10.1.32.0/20, 10.1.48.0/20, 10.3.64.0/20, 10.0.33.3/32, 10.0.18.3/32, 10.0.19.1/32, 10.0.52.3/32

m/ssd-lon/ssd-lon-7/wg1.key

@@ -0,0 +1 @@
+iN4Rc3J36APlkIP/ksSpHIwAuxC/ehYLwGQ5va1AhG8=

m/ssd-lon/ssd-lon-7/wg1.pub

@@ -0,0 +1 @@
+XGVXDmxQajHpsgpzgrp+r/ZWxZUaodTpvGBJuLlZ0n4=

m/ssd-nyc/regions

@@ -0,0 +1 @@
+us

m/ssd-nyc/ssd-nyc-3/hosts

@@ -0,0 +1,22 @@
+127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+10.2.16.3  ssd-lon-3
+10.2.16.7  ssd-lon-7 ssd-lon-ceph ssd-lon-k3s
+10.1.32.3  ssd-ams-3
+10.1.32.4  ssd-ams-4 ssd-ams-ceph ssd-ams-k3s
+10.1.32.5  ssd-ams-5
+10.1.48.1  htz-fsn-1 htz-fsn-ceph htz-fsn-k3s
+10.3.64.3  ssd-nyc-3 ssd-nyc-ceph ssd-nyc-k3s ceph k3s
+
+10.0.33.3  ssd-lon-3-0
+10.0.18.3  ssd-ams-3-0
+10.0.19.1  htz-fsn-1-0
+10.0.52.3  ssd-nyc-3-0

m/ssd-nyc/ssd-nyc-3/region

@@ -0,0 +1 @@
+none

m/ssd-nyc/ssd-nyc-3/wg0.conf

@@ -0,0 +1,23 @@
+[Interface]
+# Name: ssd-nyc-3-0
+Address = 10.0.52.3
+PrivateKey = YNDxj+QSO3p5pXw9h7lBx0cLM0kJDv3v7BYLfc8TR3I=
+ListenPort = 51800
+
+[Peer]
+# Name: ssd-lon-3-0
+PublicKey = aDXNwapVjufcuQKfBN+waJObe1TaeeaSNl1suA0u+00=
+Endpoint = ssd-lon-3.m.jpi.cloud:51800
+AllowedIPs = 10.2.16.0/20, 10.0.33.3/32
+
+[Peer]
+# Name: ssd-ams-3-0
+PublicKey = y/Gkid2Mjxo1WO9Zi7mh+DETNYKT/AAc3K3bvr9xp3c=
+Endpoint = ssd-ams-3.m.jpi.cloud:51800
+AllowedIPs = 10.1.32.0/20, 10.0.18.3/32
+
+[Peer]
+# Name: htz-fsn-1-0
+PublicKey = K9LYog8vtRFN4nW6dJWMQ9trmRrYuLpbu/Ze5kIiBGw=
+Endpoint = htz-fsn-1.m.jpi.cloud:51800
+AllowedIPs = 10.1.48.0/20, 10.0.19.1/32

m/ssd-nyc/ssd-nyc-3/wg0.key

@@ -0,0 +1 @@
+YNDxj+QSO3p5pXw9h7lBx0cLM0kJDv3v7BYLfc8TR3I=

m/ssd-nyc/ssd-nyc-3/wg0.pub

@@ -0,0 +1 @@
+1UFgleUufTszPM1voIcwVhd0DpYZ1HwV05U6p3IS2DE=

m/ssd-nyc/ssd-nyc-3/wg1.conf

@@ -0,0 +1,5 @@
+[Interface]
+# Name: ssd-nyc-3-1
+Address = 10.3.64.3
+PrivateKey = APXGKqOxx1gOr587FSN3O3gRGpkGgSjt6zGlTu49UlU=
+ListenPort = 51810

m/ssd-nyc/ssd-nyc-3/wg1.key

@@ -0,0 +1 @@
+APXGKqOxx1gOr587FSN3O3gRGpkGgSjt6zGlTu49UlU=

m/ssd-nyc/ssd-nyc-3/wg1.pub

@@ -0,0 +1 @@
+av8ni/9ZUyozabzqTjIy1sOnSJuQ9p63Tu9ECVmHB2I=

pkg/cluster/errors.go

@@ -25,5 +25,5 @@ var (
 // ErrInvalidRing returns an error indicating the [rings.RingID]
 // can't be used for the intended purpose
 func ErrInvalidRing(ringID rings.RingID) error {
-	return core.QuietWrap(fs.ErrInvalid, "invalid ring %v", ringID-1)
+	return core.QuietWrap(fs.ErrInvalid, "invalid ring %v", ringID)
 }

pkg/cluster/rings.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"fmt"
 	"io/fs"
+	"log"
 	"net/netip"
 	"strconv"
 
@@ -27,6 +28,8 @@ func AsWireguardInterfaceID(ring rings.RingID) (WireguardInterfaceID, error) {
 	switch ring {
 	case rings.RingZeroID:
 		return 0, nil
+	case rings.RingOneID:
+		return 1, nil
 	default:
 		return 0, ErrInvalidRing(ring)
 	}
@@ -146,9 +149,17 @@ var (
 		Decode: rings.DecodeRingZeroAddress,
 		Encode: rings.RingZeroAddress,
 	}
+	// RingOne is a wg1 address encoder/decoder
+	RingOne = RingAddressEncoder{
+		ID:     rings.RingOneID,
+		Port:   RingOnePort,
+		Decode: rings.DecodeRingOneAddress,
+		Encode: rings.RingOneAddress,
+	}
 	// Rings provides indexed access to the ring address encoders
 	Rings = []RingAddressEncoder{
 		RingZero,
+		RingOne,
 	}
 )
 
@@ -191,17 +202,71 @@ func (r *Ring) AddPeer(p *Machine) bool {
 		},
 	}
 
-	r.setRingZeroAllowedIPs(rp)
+	switch {
+	case r.ID == rings.RingZeroID:
+		r.setRingZeroAllowedIPs(rp)
+	case p.IsGateway():
+		r.setRingOneGatewayAllowedIPs(rp)
+	default:
+		r.setRingOneNodeAllowedIPs(rp)
+	}
+
 	r.Peers = append(r.Peers, rp)
 	return true
 }
 
-func (*Ring) setRingZeroAllowedIPs(rp *RingPeer) {
+func (r *Ring) setRingZeroAllowedIPs(rp *RingPeer) {
-	// ring0 peer
+	regionID, zoneID, _, _ := r.Decode(rp.Address)
+
+	// everyone on ring0 is a gateway to ring1
+	subnet, _ := rings.RingOnePrefix(regionID, zoneID)
+	rp.AllowSubnet(subnet)
+
+	// peer
+	rp.AllowCIDR(rp.Address, 32)
+}
+
+func (r *Ring) setRingOneGatewayAllowedIPs(rp *RingPeer) {
+	regionID, zoneID, _, _ := r.Decode(rp.Address)
+
+	// peer
 	rp.AllowCIDR(rp.Address, 32)
 
-	// everyone on ring0 has a leg on ring1
+	log.Println(rp.Node.Name, "0:", rp.Address, regionID, zoneID)
-	rp.AllowCIDR(rp.Node.RingOneAddress(), 32)
+
+	// ring1 gateways connect to all other ring1 networks
+	r.ForEachZone(func(z *Zone) bool {
+		log.Println(rp.Node.Name, "1:", z.Name, z.RegionID(), z.ID)
+
+		if !z.Is(regionID, zoneID) {
+			subnet := z.RingOnePrefix()
+			rp.AllowSubnet(subnet)
+			log.Println(rp.Node.Name, "1.1:", rp.PeerConfig.AllowedIPs)
+		}
+		return false
+	})
+
+	// ring1 gateways also connect to all ring0 addresses
+	r.ForEachZone(func(z *Zone) bool {
+		log.Println(rp.Node.Name, "2:", z.Name, z.RegionID(), z.ID)
+		z.ForEachMachine(func(p *Machine) bool {
+			log.Println(rp.Node.Name, "2.1:", p.Name, p.IsGateway())
+			if p.IsGateway() {
+				addr, _ := p.RingZeroAddress()
+				rp.AllowCIDR(addr, 32)
+				log.Println(rp.Node.Name, "2.2:", rp.PeerConfig.AllowedIPs)
+			}
+			return false
+		})
+		return false
+	})
+
+	log.Println(rp.Node.Name, "3:", rp.PeerConfig.AllowedIPs)
+}
+
+func (*Ring) setRingOneNodeAllowedIPs(rp *RingPeer) {
+	// only to the peer itself
+	rp.AllowCIDR(rp.Address, 32)
 }
 
 // ForEachMachine calls a function for each Machine in the ring

pkg/cluster/wireguard.go

@@ -82,6 +82,13 @@ func (m *Cluster) WriteWireguardConfig(ring rings.RingID) error {
 	switch ring {
 	case rings.RingZeroID:
 		return writeWireguardConfig(m, m, ring)
+	case rings.RingOneID:
+		var err error
+		m.ForEachZone(func(z *Zone) bool {
+			err = writeWireguardConfig(m, z, ring)
+			return err != nil
+		})
+		return err
 	default:
 		return ErrInvalidRing(ring)
 	}
@@ -93,6 +100,8 @@ func (z *Zone) WriteWireguardConfig(ring rings.RingID) error {
 	switch ring {
 	case rings.RingZeroID:
 		return writeWireguardConfig(z.zones, z.zones, ring)
+	case rings.RingOneID:
+		return writeWireguardConfig(z.zones, z, ring)
 	default:
 		return ErrInvalidRing(ring)
 	}
@@ -156,6 +165,13 @@ func (m *Cluster) SyncWireguardConfig(ring rings.RingID) error {
 	switch ring {
 	case rings.RingZeroID:
 		return syncWireguardConfig(m, m, ring)
+	case rings.RingOneID:
+		var err error
+		m.ForEachZone(func(z *Zone) bool {
+			err = syncWireguardConfig(m, z, ring)
+			return err != nil
+		})
+		return err
 	default:
 		return ErrInvalidRing(ring)
 	}
@@ -167,6 +183,8 @@ func (z *Zone) SyncWireguardConfig(ring rings.RingID) error {
 	switch ring {
 	case rings.RingZeroID:
 		return syncWireguardConfig(z.zones, z.zones, ring)
+	case rings.RingOneID:
+		return syncWireguardConfig(z.zones, z, ring)
 	default:
 		return ErrInvalidRing(ring)
 	}