From 948eff76d3ef4bef5e5e38f372db1b2dba171a84 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Mon, 3 Jun 2024 14:28:38 +0000 Subject: [PATCH] cluster: migrate to using pkg/rings for Addresses Signed-off-by: Alejandro Mery --- pkg/cluster/addr.go | 43 ++++++++++++++ pkg/cluster/ceph.go | 2 +- pkg/cluster/ceph_scan.go | 4 +- pkg/cluster/env.go | 2 +- pkg/cluster/hosts.go | 6 +- pkg/cluster/machine.go | 5 ++ pkg/cluster/machine_rings.go | 32 +++++++---- pkg/cluster/rings.go | 105 ++++++++--------------------------- pkg/cluster/zones.go | 35 ++++++++++++ 9 files changed, 133 insertions(+), 101 deletions(-) create mode 100644 pkg/cluster/addr.go diff --git a/pkg/cluster/addr.go b/pkg/cluster/addr.go new file mode 100644 index 0000000..3037d4c --- /dev/null +++ b/pkg/cluster/addr.go @@ -0,0 +1,43 @@ +package cluster + +import ( + "net/netip" + + "git.jpi.io/amery/jpictl/pkg/rings" +) + +// RingOnePrefix returns the ring 1 subnet of this [Zone]. +func (z *Zone) RingOnePrefix() netip.Prefix { + subnet, err := rings.RingOnePrefix(z.RegionID(), z.ID) + if err != nil { + panic(err) + } + return subnet +} + +// RingOnePrefix returns the ring 1 subnet this [Machine] belongs +// to. +func (m *Machine) RingOnePrefix() netip.Prefix { + return m.zone.RingOnePrefix() +} + +// RingZeroAddress returns the ring 0 address of the [Machine] +// if it can act as gateway. +func (m *Machine) RingZeroAddress() (netip.Addr, bool) { + addr, err := rings.RingZeroAddress(m.Region(), m.Zone(), m.ID) + if err != nil { + return netip.Addr{}, false + } + + return addr, true +} + +// RingOneAddress returns the ring 1 address of the [Machine] +func (m *Machine) RingOneAddress() netip.Addr { + addr, err := rings.RingOneAddress(m.Region(), m.Zone(), m.ID) + if err != nil { + panic(err) + } + + return addr +} diff --git a/pkg/cluster/ceph.go b/pkg/cluster/ceph.go index b91b98b..3d55301 100644 --- a/pkg/cluster/ceph.go +++ b/pkg/cluster/ceph.go @@ -66,7 +66,7 @@ func (m *Cluster) GenCephConfig() (*ceph.Config, error) { m.ForEachZone(func(z *Zone) bool { for _, p := range z.GetCephMonitors() { - addr, _ := RingOneAddress(z.ID, p.ID) + addr := p.RingOneAddress() cfg.Global.Monitors = append(cfg.Global.Monitors, p.Name) cfg.Global.MonitorsAddr = append(cfg.Global.MonitorsAddr, addr) diff --git a/pkg/cluster/ceph_scan.go b/pkg/cluster/ceph_scan.go index bd65821..59e5edd 100644 --- a/pkg/cluster/ceph_scan.go +++ b/pkg/cluster/ceph_scan.go @@ -4,6 +4,7 @@ import ( "os" "darvaza.org/slog" + "git.jpi.io/amery/jpictl/pkg/ceph" ) @@ -14,8 +15,7 @@ type cephScanTODO struct { func (todo *cephScanTODO) checkMachine(p *Machine) bool { // on ceph all addresses are ring1 - ring1, _ := RingOneAddress(p.Zone(), p.ID) - addr := ring1.String() + addr := p.RingOneAddress().String() if _, found := todo.names[p.Name]; found { // found on the TODO by name diff --git a/pkg/cluster/env.go b/pkg/cluster/env.go index 5a6674d..42e98a2 100644 --- a/pkg/cluster/env.go +++ b/pkg/cluster/env.go @@ -185,7 +185,7 @@ func genEnvZoneCephMonNames(m Machines) string { func genEnvZoneCephMonIPs(m Machines) string { var buf strings.Builder m.ForEachMachine(func(p *Machine) bool { - addr, _ := RingOneAddress(p.Zone(), p.ID) + addr := p.RingOneAddress() if buf.Len() > 0 { _, _ = buf.WriteRune(' ') diff --git a/pkg/cluster/hosts.go b/pkg/cluster/hosts.go index 359aa75..af7b23a 100644 --- a/pkg/cluster/hosts.go +++ b/pkg/cluster/hosts.go @@ -71,14 +71,14 @@ func (p *Machine) WriteHosts() error { func (z *Zone) genHosts(out *hostsFile, p *Machine) { var names []string - ip, _ := RingOneAddress(p.zone.ID, p.ID) + ip := p.RingOneAddress() names = append(names, p.Name) if p.CephMonitor { names = append(names, fmt.Sprintf("%s-%s", p.zone.Name, "ceph")) names = append(names, fmt.Sprintf("%s-%s", p.zone.Name, "k3s")) - if z.ID == p.zone.ID { + if z.Is(p.Region(), p.Zone()) { names = append(names, "ceph") names = append(names, "k3s") } @@ -94,7 +94,7 @@ func (z *Zone) genHosts(out *hostsFile, p *Machine) { if p.IsGateway() { var s string - ip, _ = RingZeroAddress(p.zone.ID, p.ID) + ip, _ = p.RingZeroAddress() s = fmt.Sprintf("%s-%v", p.Name, 0) entry = hostsEntry{ diff --git a/pkg/cluster/machine.go b/pkg/cluster/machine.go index 2b9fff5..27ffd59 100644 --- a/pkg/cluster/machine.go +++ b/pkg/cluster/machine.go @@ -80,6 +80,11 @@ func (m *Machine) Zone() rings.ZoneID { return m.zone.ID } +// Region indicates the [Region] this machine belongs to +func (m *Machine) Region() rings.RegionID { + return m.zone.RegionID() +} + func (m *Machine) getPeerByName(name string) (*Machine, bool) { return m.zone.zones.GetMachineByName(name) } diff --git a/pkg/cluster/machine_rings.go b/pkg/cluster/machine_rings.go index f2aa397..8e4d744 100644 --- a/pkg/cluster/machine_rings.go +++ b/pkg/cluster/machine_rings.go @@ -139,12 +139,12 @@ func (m *Machine) tryApplyWireguardConfig(ring rings.RingID) error { func (m *Machine) applyWireguardConfigNode(ring rings.RingID, wg *wireguard.Config) error { addr := wg.GetAddress() if !core.IsZero(addr) { - zoneID, nodeID, ok := Rings[ring].Decode(addr) + regionID, zoneID, nodeID, ok := Rings[ring].Decode(addr) if !ok { return fmt.Errorf("%s: invalid address", addr) } - if err := m.applyZoneNodeID(zoneID, nodeID); err != nil { + if err := m.applyZoneNodeID(regionID, zoneID, nodeID); err != nil { return core.Wrap(err, "%s: invalid address", addr) } } @@ -220,7 +220,9 @@ func (m *Machine) applyWireguardInterfaceConfig(ring rings.RingID, return m.applyRingInfo(ring, ri) } -func (m *Machine) applyWireguardPeerConfig(ring rings.RingID, pc wireguard.PeerConfig) error { +func (m *Machine) applyWireguardPeerConfig(ring rings.RingID, + pc wireguard.PeerConfig) error { + // peer, found := m.getPeerByName(pc.Endpoint.Name()) switch { case !found: @@ -243,21 +245,29 @@ func (m *Machine) applyWireguardPeerConfig(ring rings.RingID, pc wireguard.PeerC } } -func (m *Machine) applyZoneNodeID(zoneID rings.ZoneID, nodeID rings.NodeID) error { +func (m *Machine) applyZoneNodeID(regionID rings.RegionID, + zoneID rings.ZoneID, nodeID rings.NodeID) error { + // switch { - case zoneID == 0: + case !regionID.Valid(): + return fmt.Errorf("invalid %s", "regionID") + case !zoneID.Valid(): return fmt.Errorf("invalid %s", "zoneID") - case nodeID == 0: + case !nodeID.Valid(): return fmt.Errorf("invalid %s", "nodeID") case m.ID != nodeID: - return fmt.Errorf("invalid %s: %v ≠ %v", "zoneID", m.ID, nodeID) + return fmt.Errorf("invalid %s: %v ≠ %v", "nodeID", m.ID, nodeID) case m.zone.ID != 0 && m.zone.ID != zoneID: return fmt.Errorf("invalid %s: %v ≠ %v", "zoneID", m.zone.ID, zoneID) - case m.zone.ID == 0: - m.zone.ID = zoneID - } + case m.Region() != regionID: + return fmt.Errorf("invalid %s: %v ≠ %v", "regionID", m.Region(), regionID) + default: + if m.zone.ID == 0 { + m.zone.ID = zoneID + } - return nil + return nil + } } func (m *Machine) setRingDefaults(ri *RingInfo) error { diff --git a/pkg/cluster/rings.go b/pkg/cluster/rings.go index ba56d08..f29c081 100644 --- a/pkg/cluster/rings.go +++ b/pkg/cluster/rings.go @@ -136,8 +136,8 @@ func canMergeKeyPairs(p1, p2 wireguard.KeyPair) bool { type RingAddressEncoder struct { ID rings.RingID Port uint16 - Encode func(zoneID rings.ZoneID, nodeID rings.NodeID) (netip.Addr, bool) - Decode func(addr netip.Addr) (zoneID rings.ZoneID, nodeID rings.NodeID, ok bool) + Encode func(rings.RegionID, rings.ZoneID, rings.NodeID) (netip.Addr, error) + Decode func(addr netip.Addr) (rings.RegionID, rings.ZoneID, rings.NodeID, bool) } var ( @@ -145,15 +145,15 @@ var ( RingZero = RingAddressEncoder{ ID: rings.RingZeroID, Port: RingZeroPort, - Decode: ParseRingZeroAddress, - Encode: RingZeroAddress, + Decode: rings.DecodeRingZeroAddress, + Encode: rings.RingZeroAddress, } // RingOne is a wg1 address encoder/decoder RingOne = RingAddressEncoder{ ID: rings.RingOneID, Port: RingOnePort, - Decode: ParseRingOneAddress, - Encode: RingOneAddress, + Decode: rings.DecodeRingOneAddress, + Encode: rings.RingOneAddress, } // Rings provides indexed access to the ring address encoders Rings = []RingAddressEncoder{ @@ -162,72 +162,6 @@ var ( } ) -// ValidZoneID checks if the given zoneID is a valid 4 bit zone number. -// -// 0 is reserved, and only allowed when composing CIDRs. -func ValidZoneID(zoneID rings.ZoneID) bool { - return zoneID == 0 || zoneID.Valid() -} - -// ValidNodeID checks if the given nodeID is a valid 8 bit number. -// nodeID is unique within a Zone. -// 0 is reserved, and only allowed when composing CIDRs. -func ValidNodeID(nodeID rings.NodeID) bool { - return nodeID == 0 || nodeID.Valid() -} - -// ParseRingZeroAddress extracts zone and node ID from a wg0 [netip.Addr] -// wg0 addresses are of the form `10.0.{{zoneID}}.{{nodeID}}` -func ParseRingZeroAddress(addr netip.Addr) (zoneID rings.ZoneID, nodeID rings.NodeID, ok bool) { - if addr.IsValid() { - a4 := addr.As4() - - if a4[0] == 10 && a4[1] == 0 { - zoneID = rings.ZoneID(a4[2]) - nodeID = rings.NodeID(a4[3]) - return zoneID, nodeID, true - } - } - return 0, 0, false -} - -// RingZeroAddress returns a wg0 IP address -func RingZeroAddress(zoneID rings.ZoneID, nodeID rings.NodeID) (netip.Addr, bool) { - switch { - case !ValidZoneID(zoneID) || !ValidNodeID(nodeID): - return netip.Addr{}, false - default: - a4 := [4]uint8{10, 0, uint8(zoneID), uint8(nodeID)} - return netip.AddrFrom4(a4), true - } -} - -// ParseRingOneAddress extracts zone and node ID from a wg1 [netip.Addr] -// wg1 addresses are of the form `10.{{zoneID << 4}}.{{nodeID}}` -func ParseRingOneAddress(addr netip.Addr) (zoneID rings.ZoneID, nodeID rings.NodeID, ok bool) { - if addr.IsValid() { - a4 := addr.As4() - - if a4[0] == 10 && a4[2] == 0 { - zoneID = rings.ZoneID(a4[1] >> 4) - nodeID = rings.NodeID(a4[3]) - return zoneID, nodeID, true - } - } - return 0, 0, false -} - -// RingOneAddress returns a wg1 IP address -func RingOneAddress(zoneID rings.ZoneID, nodeID rings.NodeID) (netip.Addr, bool) { - switch { - case !ValidZoneID(zoneID) || !ValidNodeID(nodeID): - return netip.Addr{}, false - default: - a4 := [4]uint8{10, uint8(zoneID << 4), 0, uint8(nodeID)} - return netip.AddrFrom4(a4), true - } -} - var ( _ MachineIterator = (*Ring)(nil) _ ZoneIterator = (*Ring)(nil) @@ -250,7 +184,8 @@ func (r *Ring) AddPeer(p *Machine) bool { nodeID := p.ID zoneID := p.Zone() - addr, _ := r.Encode(zoneID, nodeID) + regionID := p.Region() + addr, _ := r.Encode(regionID, zoneID, nodeID) rp := &RingPeer{ Node: p, @@ -280,27 +215,27 @@ func (r *Ring) AddPeer(p *Machine) bool { } func (r *Ring) setRingZeroAllowedIPs(rp *RingPeer) { - zoneID, _, _ := r.Decode(rp.Address) + regionID, zoneID, _, _ := r.Decode(rp.Address) // everyone on ring0 is a gateway to ring1 - addr, _ := RingOneAddress(zoneID, 0) - rp.AllowCIDR(addr, 12) + subnet, _ := rings.RingOnePrefix(regionID, zoneID) + rp.AllowSubnet(subnet) // peer rp.AllowCIDR(rp.Address, 32) } func (r *Ring) setRingOneGatewayAllowedIPs(rp *RingPeer) { - zoneID, _, _ := r.Decode(rp.Address) + regionID, zoneID, _, _ := r.Decode(rp.Address) // peer rp.AllowCIDR(rp.Address, 32) // ring1 gateways connect to all other ring1 networks r.ForEachZone(func(z *Zone) bool { - if z.ID != zoneID { - addr, _ := r.Encode(z.ID, 0) - rp.AllowCIDR(addr, 12) + if !z.Is(regionID, zoneID) { + subnet := z.RingOnePrefix() + rp.AllowSubnet(subnet) } return false }) @@ -309,7 +244,7 @@ func (r *Ring) setRingOneGatewayAllowedIPs(rp *RingPeer) { r.ForEachZone(func(z *Zone) bool { z.ForEachMachine(func(p *Machine) bool { if p.IsGateway() { - addr, _ := RingZeroAddress(z.ID, p.ID) + addr, _ := p.RingZeroAddress() rp.AllowCIDR(addr, 32) } return false @@ -376,8 +311,12 @@ type RingPeer struct { // AllowCIDR allows an IP range via this peer func (rp *RingPeer) AllowCIDR(addr netip.Addr, bits int) { - cidr := netip.PrefixFrom(addr, bits) - rp.PeerConfig.AllowedIPs = append(rp.PeerConfig.AllowedIPs, cidr) + rp.AllowSubnet(netip.PrefixFrom(addr, bits)) +} + +// AllowSubnet allows an IP range via this peer +func (rp *RingPeer) AllowSubnet(subnet netip.Prefix) { + rp.PeerConfig.AllowedIPs = append(rp.PeerConfig.AllowedIPs, subnet) } // NewRing composes a new Ring for Wireguard setup diff --git a/pkg/cluster/zones.go b/pkg/cluster/zones.go index 47d7c91..acfc6c7 100644 --- a/pkg/cluster/zones.go +++ b/pkg/cluster/zones.go @@ -70,3 +70,38 @@ func (z *Zone) GatewayIDs() ([]rings.NodeID, int) { return out, len(out) } + +// RegionID returns the primary [Region] of a [Zone]. +func (z *Zone) RegionID() rings.RegionID { + if z != nil && z.region != nil { + return z.region.ID + } + return 0 +} + +// Is checks if the given [rings.RegionID] and [rings.ZoneID] match +// the [Zone]. +func (z *Zone) Is(regionID rings.RegionID, zoneID rings.ZoneID) bool { + switch { + case z.ID != zoneID: + return false + case z.RegionID() != regionID: + return false + default: + return true + } +} + +// Eq checks if two [Zone]s are the same. +func (z *Zone) Eq(z2 *Zone) bool { + switch { + case z == nil, z2 == nil: + return false + case z.ID != z2.ID: + return false + case z.RegionID() != z2.RegionID(): + return false + default: + return true + } +}