Merge pull request 'cluster: use typed IDs, introduce Region.ID, and pre-compute primary region' (#49) from pr-amery-regions into main

Reviewed-on: #49

Makefile

@@ -15,7 +15,8 @@ TMPDIR ?= .tmp
 REVIVE ?= $(GOBIN)/revive
 REVIVE_CONF ?= $(TOOLSDIR)/revive.toml
 REVIVE_RUN_ARGS ?= -config $(REVIVE_CONF) -formatter friendly
-REVIVE_INSTALL_URL ?= github.com/mgechev/revive@master
+REVIVE_VERSION ?= v1.3.7
+REVIVE_INSTALL_URL ?= github.com/mgechev/revive@$(REVIVE_VERSION)
 
 GO_INSTALL_URLS = \
 	$(REVIVE_INSTALL_URL) \

cmd/jpictl/dns.go

@@ -52,7 +52,7 @@ func populateDNSManager(mgr *dns.Manager, m *cluster.Cluster) error {
 
 	m.ForEachZone(func(z *cluster.Zone) bool {
 		z.ForEachMachine(func(p *cluster.Machine) bool {
-			err = mgr.AddHost(ctx, z.Name, p.ID, p.IsActive(), p.PublicAddresses...)
+			err = mgr.AddHost(ctx, z.Name, int(p.ID), p.IsActive(), p.PublicAddresses...)
 			return err != nil
 		})
 

cmd/jpictl/gateway.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"os"
-	"strconv"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -128,7 +127,7 @@ func gatewayListAll(zi cluster.ZoneIterator) error {
 			return false
 		}
 		for _, i := range ids {
-			sIDs = append(sIDs, strconv.Itoa(i))
+			sIDs = append(sIDs, i.String())
 		}
 		b.WriteString(strings.Join(sIDs, ", "))
 		b.WriteString("\n")

pkg/cluster/cluster_import.go

@@ -6,16 +6,18 @@ import (
 	"os"
 
 	"gopkg.in/yaml.v3"
+
+	"git.jpi.io/amery/jpictl/pkg/rings"
 )
 
 func (m *Cluster) init(opts *ScanOptions) error {
 	for _, fn := range []func(*ScanOptions) error{
 		m.initZones,
+		m.initRegions,
 		m.scanZoneIDs,
 		m.scanSort,
 		m.scanGateways,
 		m.initCephMonitors,
-		m.initRegions,
 	} {
 		if err := fn(opts); err != nil {
 			return err
@@ -45,7 +47,7 @@ func (m *Cluster) initZones(opts *ScanOptions) error {
 
 func (m *Cluster) initZone(z *Zone, _ *ScanOptions) error {
 	var hasMissing bool
-	var lastMachineID int
+	var lastMachineID rings.NodeID
 
 	z.zones = m
 	z.logger = m
@@ -58,7 +60,7 @@ func (m *Cluster) initZone(z *Zone, _ *ScanOptions) error {
 		case p.ID == 0:
 			hasMissing = true
 		case p.ID > lastMachineID:
-			lastMachineID = z.ID
+			lastMachineID = p.ID
 		}
 
 		return false

pkg/cluster/cluster_scan.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 
 	"darvaza.org/core"
+
+	"git.jpi.io/amery/jpictl/pkg/rings"
 )
 
 const (
@@ -23,9 +25,9 @@ func (m *Cluster) scan(opts *ScanOptions) error {
 	for _, fn := range []func(*ScanOptions) error{
 		m.scanDirectory,
 		m.scanMachines,
+		m.initRegions,
 		m.scanZoneIDs,
 		m.scanSort,
-		m.initRegions,
 		m.scanGateways,
 		m.scanCephMonitors,
 	} {
@@ -114,7 +116,7 @@ func (m *Cluster) scanMachines(opts *ScanOptions) error {
 
 func (m *Cluster) scanZoneIDs(_ *ScanOptions) error {
 	var hasMissing bool
-	var lastZoneID int
+	var lastZoneID rings.ZoneID
 
 	m.ForEachZone(func(z *Zone) bool {
 		switch {

pkg/cluster/env.go

@@ -6,6 +6,8 @@ import (
 	"io"
 	"sort"
 	"strings"
+
+	"git.jpi.io/amery/jpictl/pkg/rings"
 )
 
 // Env is a shell environment factory for this cluster
@@ -35,8 +37,8 @@ func (m *Cluster) Env(export bool) (*Env, error) {
 }
 
 // Zones returns the list of Zone IDs
-func (m *Env) Zones() []int {
+func (m *Env) Zones() []rings.ZoneID {
-	var zones []int
+	var zones []rings.ZoneID
 
 	m.ForEachZone(func(z *Zone) bool {
 		zones = append(zones, z.ID)
@@ -51,7 +53,7 @@ func (m *Env) Regions() []string {
 	var regions []string
 
 	m.ForEachRegion(func(r *Region) bool {
-		if r.Cluster != nil {
+		if r.IsPrimary() {
 			regions = append(regions, r.Name)
 		}
 
@@ -70,8 +72,8 @@ func (m *Env) WriteTo(w io.Writer) (int64, error) {
 		m.writeEnvVar(&buf, m.cephFSID, "FSID")
 	}
 
-	m.writeEnvVarStrings(&buf, m.Regions(), "REGIONS")
+	m.writeEnvVar(&buf, genEnvStrings(m.Regions()), "REGIONS")
-	m.writeEnvVarInts(&buf, m.Zones(), "ZONES")
+	m.writeEnvVar(&buf, genEnvInts(m.Zones()), "ZONES")
 
 	m.ForEachZone(func(z *Zone) bool {
 		m.writeEnvZone(&buf, z)
@@ -92,7 +94,7 @@ func (m *Env) writeEnvZone(w io.Writer, z *Zone) {
 
 	// ZONE{zoneID}_GW
 	gateways, _ := z.GatewayIDs()
-	m.writeEnvVarInts(w, gateways, "ZONE%v_%s", zoneID, "GW")
+	m.writeEnvVar(w, genEnvInts(gateways), "ZONE%v_%s", zoneID, "GW")
 
 	// ZONE{zoneID}_REGION
 	m.writeEnvVar(w, genEnvZoneRegion(z), "ZONE%v_%s", zoneID, "REGION")
@@ -107,32 +109,6 @@ func (m *Env) writeEnvZone(w io.Writer, z *Zone) {
 	m.writeEnvVar(w, genEnvZoneCephMonIDs(monitors), "MON%v_%s", zoneID, "ID")
 }
 
-func (m *Env) writeEnvVarInts(w io.Writer, value []int, name string, args ...any) {
-	var buf bytes.Buffer
-
-	for _, v := range value {
-		if buf.Len() > 0 {
-			_, _ = fmt.Fprint(&buf, " ")
-		}
-		_, _ = fmt.Fprintf(&buf, "%v", v)
-	}
-
-	m.writeEnvVar(w, buf.String(), name, args...)
-}
-
-func (m *Env) writeEnvVarStrings(w io.Writer, value []string, name string, args ...any) {
-	var buf bytes.Buffer
-
-	for _, v := range value {
-		if buf.Len() > 0 {
-			_, _ = fmt.Fprint(&buf, " ")
-		}
-		_, _ = fmt.Fprintf(&buf, "%s", v)
-	}
-
-	m.writeEnvVar(w, buf.String(), name, args...)
-}
-
 func (m *Env) writeEnvVar(w io.Writer, value string, name string, args ...any) {
 	var prefix string
 
@@ -155,6 +131,23 @@ func (m *Env) writeEnvVar(w io.Writer, value string, name string, args ...any) {
 	}
 }
 
+func genEnvInts[T ~int | ~uint](values []T) string {
+	var buf bytes.Buffer
+
+	for _, v := range values {
+		if buf.Len() > 0 {
+			_, _ = buf.WriteRune(' ')
+		}
+		_, _ = buf.WriteString(fmt.Sprintf("%v", v))
+	}
+
+	return buf.String()
+}
+
+func genEnvStrings(values []string) string {
+	return strings.Join(values, " ")
+}
+
 func genEnvZoneNodes(z *Zone) string {
 	if n := z.Len(); n > 0 {
 		s := make([]string, 0, n)
@@ -164,24 +157,16 @@ func genEnvZoneNodes(z *Zone) string {
 			return false
 		})
 
-		return strings.Join(s, " ")
+		return genEnvStrings(s)
 	}
 	return ""
 }
 
 func genEnvZoneRegion(z *Zone) string {
-	var region string
+	if z != nil && z.region != nil {
-
+		return z.region.Name
-	z.ForEachRegion(func(r *Region) bool {
-		if r.Cluster != nil {
-			region = r.Name
-			return true
 	}
-
+	return ""
-		return false
-	})
-
-	return region
 }
 
 func genEnvZoneCephMonNames(m Machines) string {

pkg/cluster/machine.go

@@ -3,6 +3,8 @@ package cluster
 import (
 	"net/netip"
 	"strings"
+
+	"git.jpi.io/amery/jpictl/pkg/rings"
 )
 
 // revive:disable:line-length-limit
@@ -12,7 +14,7 @@ type Machine struct {
 	zone   *Zone
 	logger `json:"-" yaml:"-"`
 
-	ID   int
+	ID   rings.NodeID
 	Name string `json:"-" yaml:"-"`
 
 	Inactive        bool         `json:"inactive,omitempty"     yaml:"inactive,omitempty"`
@@ -74,7 +76,7 @@ func (m *Machine) SetGateway(enabled bool) error {
 }
 
 // Zone indicates the [Zone] this machine belongs to
-func (m *Machine) Zone() int {
+func (m *Machine) Zone() rings.ZoneID {
 	return m.zone.ID
 }
 

pkg/cluster/machine_rings.go

@@ -8,6 +8,7 @@ import (
 
 	"darvaza.org/core"
 
+	"git.jpi.io/amery/jpictl/pkg/rings"
 	"git.jpi.io/amery/jpictl/pkg/wireguard"
 )
 
@@ -223,7 +224,7 @@ func (m *Machine) applyWireguardPeerConfig(ring int, pc wireguard.PeerConfig) er
 	}
 }
 
-func (m *Machine) applyZoneNodeID(zoneID, nodeID int) error {
+func (m *Machine) applyZoneNodeID(zoneID rings.ZoneID, nodeID rings.NodeID) error {
 	switch {
 	case zoneID == 0:
 		return fmt.Errorf("invalid %s", "zoneID")

pkg/cluster/machine_scan.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"darvaza.org/core"
+	"git.jpi.io/amery/jpictl/pkg/rings"
 )
 
 // LookupNetIP uses the DNS Resolver to get the public addresses associated
@@ -65,7 +66,7 @@ func (m *Machine) setID() error {
 		return err
 	}
 
-	m.ID = int(id)
+	m.ID = rings.NodeID(id)
 	return nil
 }
 

pkg/cluster/regions.go

@@ -3,6 +3,8 @@ package cluster
 import (
 	"bytes"
 	"path/filepath"
+
+	"git.jpi.io/amery/jpictl/pkg/rings"
 )
 
 var (
@@ -24,10 +26,17 @@ type Region struct {
 	zones []*Zone
 
 	Name    string
+	ID      rings.RegionID `json:",omitempty" yaml:",omitempty"`
 	Cluster *string        `json:",omitempty" yaml:",omitempty"`
 	Regions []string       `json:",omitempty" yaml:",omitempty"`
 }
 
+// IsPrimary indicates the region is primary and corresponds
+// to a kubernetes cluster.
+func (r *Region) IsPrimary() bool {
+	return r != nil && r.Cluster != nil
+}
+
 // ForEachRegion calls a function for each Region of the cluster
 // until instructed to terminate the loop
 func (m *Cluster) ForEachRegion(fn func(r *Region) bool) {
@@ -92,6 +101,8 @@ func (m *Cluster) initRegions(_ *ScanOptions) error {
 	}
 
 	m.sortRegions()
+	m.scanRegionID()
+	m.computeZonesRegion()
 	return nil
 }
 
@@ -210,6 +221,92 @@ func (m *Cluster) finishRegion(r *Region) {
 	r.Regions = sub
 }
 
+// revive:disable:cognitive-complexity
+func (m *Cluster) scanRegionID() {
+	// revive:enable:cognitive-complexity
+	var max rings.RegionID
+	var missing bool
+
+	// check IDs
+	ids := make(map[rings.RegionID]bool)
+	fn := func(r *Region) bool {
+		var term bool
+
+		switch {
+		case !r.IsPrimary():
+			// secondary, no ID.
+			r.ID = 0
+		case !r.ID.Valid():
+			// primary without ID
+			missing = true
+		case ids[r.ID]:
+			// duplicate
+			m.error(nil).WithField("region", r.Name).Print("duplicate ID")
+			missing = true
+			r.ID = 0
+		default:
+			ids[r.ID] = true
+
+			if r.ID > max {
+				max = r.ID
+			}
+		}
+
+		return term
+	}
+	m.ForEachRegion(fn)
+
+	if missing {
+		// assign missing IDs
+		fn := func(r *Region) bool {
+			var term bool
+
+			switch {
+			case !r.IsPrimary():
+				// ignore secondary
+			case r.ID.Valid():
+				// already has an ID
+			default:
+				r.ID = max + 1
+				max = r.ID
+			}
+
+			return term
+		}
+
+		m.ForEachRegion(fn)
+	}
+}
+
+func (m *Cluster) computeZonesRegion() {
+	fn := func(r *Region, z *Zone) {
+		if z.region != nil {
+			m.error(nil).
+				WithField("zone", z.Name).
+				WithField("region", []string{
+					z.region.Name,
+					r.Name,
+				}).
+				Print("zone in two regions")
+		} else {
+			z.region = r
+		}
+	}
+
+	m.ForEachRegion(func(r *Region) bool {
+		var term bool
+
+		if r.IsPrimary() {
+			r.ForEachZone(func(z *Zone) bool {
+				fn(r, z)
+				return term
+			})
+		}
+
+		return term
+	})
+}
+
 func (m *Cluster) getRegion(name string) (*Region, bool) {
 	for i := range m.Regions {
 		r := &m.Regions[i]

pkg/cluster/rings.go

@@ -5,14 +5,11 @@ import (
 	"io/fs"
 	"net/netip"
 
+	"git.jpi.io/amery/jpictl/pkg/rings"
 	"git.jpi.io/amery/jpictl/pkg/wireguard"
 )
 
 const (
-	// MaxZoneID indicates the highest ID allowed for a Zone
-	MaxZoneID = 0xf
-	// MaxNodeID indicates the highest Machine ID allowed within a Zone
-	MaxNodeID = 0xff - 1
 	// RingsCount indicates how many wireguard rings we have
 	RingsCount = 2
 	// RingZeroPort is the port wireguard uses for ring0
@@ -81,8 +78,8 @@ func canMergeKeyPairs(p1, p2 wireguard.KeyPair) bool {
 type RingAddressEncoder struct {
 	ID     int
 	Port   uint16
-	Encode func(zoneID, nodeID int) (netip.Addr, bool)
+	Encode func(zoneID rings.ZoneID, nodeID rings.NodeID) (netip.Addr, bool)
-	Decode func(addr netip.Addr) (zoneID, nodeID int, ok bool)
+	Decode func(addr netip.Addr) (zoneID rings.ZoneID, nodeID rings.NodeID, ok bool)
 }
 
 var (
@@ -110,42 +107,34 @@ var (
 // ValidZoneID checks if the given zoneID is a valid 4 bit zone number.
 //
 // 0 is reserved, and only allowed when composing CIDRs.
-func ValidZoneID(zoneID int) bool {
+func ValidZoneID(zoneID rings.ZoneID) bool {
-	switch {
+	return zoneID == 0 || zoneID.Valid()
-	case zoneID < 0 || zoneID > MaxZoneID:
-		return false
-	default:
-		return true
-	}
 }
 
 // ValidNodeID checks if the given nodeID is a valid 8 bit number.
 // nodeID is unique within a Zone.
 // 0 is reserved, and only allowed when composing CIDRs.
-func ValidNodeID(nodeID int) bool {
+func ValidNodeID(nodeID rings.NodeID) bool {
-	switch {
+	return nodeID == 0 || nodeID.Valid()
-	case nodeID < 0 || nodeID > MaxNodeID:
-		return false
-	default:
-		return true
-	}
 }
 
 // ParseRingZeroAddress extracts zone and node ID from a wg0 [netip.Addr]
 // wg0 addresses are of the form `10.0.{{zoneID}}.{{nodeID}}`
-func ParseRingZeroAddress(addr netip.Addr) (zoneID int, nodeID int, ok bool) {
+func ParseRingZeroAddress(addr netip.Addr) (zoneID rings.ZoneID, nodeID rings.NodeID, ok bool) {
 	if addr.IsValid() {
 		a4 := addr.As4()
 
 		if a4[0] == 10 && a4[1] == 0 {
-			return int(a4[2]), int(a4[3]), true
+			zoneID = rings.ZoneID(a4[2])
+			nodeID = rings.NodeID(a4[3])
+			return zoneID, nodeID, true
 		}
 	}
 	return 0, 0, false
 }
 
 // RingZeroAddress returns a wg0 IP address
-func RingZeroAddress(zoneID, nodeID int) (netip.Addr, bool) {
+func RingZeroAddress(zoneID rings.ZoneID, nodeID rings.NodeID) (netip.Addr, bool) {
 	switch {
 	case !ValidZoneID(zoneID) || !ValidNodeID(nodeID):
 		return netip.Addr{}, false
@@ -157,13 +146,13 @@ func RingZeroAddress(zoneID, nodeID int) (netip.Addr, bool) {
 
 // ParseRingOneAddress extracts zone and node ID from a wg1 [netip.Addr]
 // wg1 addresses are of the form `10.{{zoneID << 4}}.{{nodeID}}`
-func ParseRingOneAddress(addr netip.Addr) (zoneID int, nodeID int, ok bool) {
+func ParseRingOneAddress(addr netip.Addr) (zoneID rings.ZoneID, nodeID rings.NodeID, ok bool) {
 	if addr.IsValid() {
 		a4 := addr.As4()
 
 		if a4[0] == 10 && a4[2] == 0 {
-			zoneID = int(a4[1] >> 4)
+			zoneID = rings.ZoneID(a4[1] >> 4)
-			nodeID = int(a4[3])
+			nodeID = rings.NodeID(a4[3])
 			return zoneID, nodeID, true
 		}
 	}
@@ -171,7 +160,7 @@ func ParseRingOneAddress(addr netip.Addr) (zoneID int, nodeID int, ok bool) {
 }
 
 // RingOneAddress returns a wg1 IP address
-func RingOneAddress(zoneID, nodeID int) (netip.Addr, bool) {
+func RingOneAddress(zoneID rings.ZoneID, nodeID rings.NodeID) (netip.Addr, bool) {
 	switch {
 	case !ValidZoneID(zoneID) || !ValidNodeID(nodeID):
 		return netip.Addr{}, false

pkg/cluster/zones.go

@@ -2,6 +2,8 @@ package cluster
 
 import (
 	"io/fs"
+
+	"git.jpi.io/amery/jpictl/pkg/rings"
 )
 
 var (
@@ -17,9 +19,10 @@ type ZoneIterator interface {
 // affinity.
 type Zone struct {
 	zones  *Cluster
+	region *Region
 	logger `json:"-" yaml:"-"`
 
-	ID      int
+	ID      rings.ZoneID
 	Name    string
 	Regions []string `json:",omitempty" yaml:",omitempty"`
 
@@ -31,7 +34,7 @@ func (z *Zone) String() string {
 }
 
 // SetGateway configures a machine to be the zone's ring0 gateway
-func (z *Zone) SetGateway(gatewayID int, enabled bool) error {
+func (z *Zone) SetGateway(gatewayID rings.NodeID, enabled bool) error {
 	var err error
 	var found bool
 
@@ -56,8 +59,8 @@ func (z *Zone) SetGateway(gatewayID int, enabled bool) error {
 }
 
 // GatewayIDs returns the list of IDs of machines that act as ring0 gateways
-func (z *Zone) GatewayIDs() ([]int, int) {
+func (z *Zone) GatewayIDs() ([]rings.NodeID, int) {
-	var out []int
+	var out []rings.NodeID
 	z.ForEachMachine(func(p *Machine) bool {
 		if p.IsGateway() {
 			out = append(out, p.ID)

pkg/rings/rings.go

@@ -3,6 +3,8 @@
 package rings
 
 import (
+	"fmt"
+	"strconv"
 	"syscall"
 
 	"darvaza.org/core"
@@ -14,10 +16,10 @@ const (
 	// ZoneMax indicates the highest number that can be used for a [ZoneID].
 	ZoneMax = (1 << 4) - 1
 	// NodeMax indicates the highest number that can be used for a [NodeID].
-	NodeMax = (1 << 12) - 1
+	NodeMax = (1 << 12) - 2
 	// NodeZeroMax indicates the highest number that can be used for a [NodeID]
 	// when its a gateway connected to Ring 0 (backbone).
-	NodeZeroMax = (1 << 8) - 1
+	NodeZeroMax = (1 << 8) - 2
 
 	// RingZeroBits indicates the size of the prefix on the ring 0 (backbone) network.
 	RingZeroBits = 16
@@ -37,12 +39,20 @@ type RegionID int
 // Valid tells a [RegionID] is within the valid range.
 func (n RegionID) Valid() bool { return n > 0 && n <= RegionMax }
 
+func (n RegionID) String() string {
+	return idString(n)
+}
+
 // ZoneID is the identifier of a zone within a region, valid between 1 and [ZoneMax].
 type ZoneID int
 
 // Valid tells a [ZoneID] is within the valid range.
 func (n ZoneID) Valid() bool { return n > 0 && n <= ZoneMax }
 
+func (n ZoneID) String() string {
+	return idString(n)
+}
+
 // NodeID is the identifier of a machine within a zone of a region, valid between
 // 1 and [NodeMax], but between 1 and [NodeZeroMax] if it will be a zone gateway.
 type NodeID int
@@ -53,8 +63,28 @@ func (n NodeID) Valid() bool { return n > 0 && n <= NodeMax }
 // ValidZero tells a [NodeID] is within the valid range for a gateway.
 func (n NodeID) ValidZero() bool { return n > 0 && n <= NodeZeroMax }
 
+func (n NodeID) String() string {
+	return idString(n)
+}
+
 // ErrOutOfRange is an error indicating the value of a field
 // is out of range.
 func ErrOutOfRange[T ~int | ~uint32](value T, field string) error {
 	return core.Wrap(syscall.EINVAL, "%s out of range (%v)", field, value)
 }
+
+type intID interface {
+	~int
+	Valid() bool
+}
+
+func idString[T intID](p T) string {
+	switch {
+	case p == 0:
+		return "unspecified"
+	case p.Valid():
+		return strconv.Itoa(int(p))
+	default:
+		return fmt.Sprintf("invalid (%v)", int(p))
+	}
+}