Merge pull request 'cluster: ensure ceph monitors are set when loading a config file' (#23)

Reviewed-on: #23

.vscode/settings.json

@@ -1,9 +1,12 @@
 {
     "cSpell.words": [
+        "asciigoat",
         "ceph",
+        "cyclomatic",
         "darvaza",
         "gofrs",
         "jpictl",
+        "Wrapf",
         "zerolog"
     ]
 }

cmd/jpictl/config.go

@@ -1,11 +1,25 @@
 package main
 
-import "git.jpi.io/amery/jpictl/pkg/cluster"
+import (
+	"os"
+
+	"darvaza.org/core"
+
+	"git.jpi.io/amery/jpictl/pkg/cluster"
+)
+
+const (
+	// DefaultConfigFile is read if -f/--config-file isn't specified.
+	// If it doesn't exist, m/ will be scanned
+	DefaultConfigFile = "cloud.yaml"
+)
 
 // Config describes the repository
 type Config struct {
 	Base   string
 	Domain string
+
+	ConfigFile string
 }
 
 var cfg = &Config{
@@ -14,9 +28,32 @@ var cfg = &Config{
 }
 
 // LoadZones loads all zones and machines in the config directory
+// or file
 func (cfg *Config) LoadZones(resolve bool) (*cluster.Cluster, error) {
+	// try config file first
+	zones, err := cluster.NewFromConfig(cfg.ConfigFile,
+		cluster.ResolvePublicAddresses(resolve),
+		cluster.WithLogger(log),
+	)
+
+	switch {
+	case err == nil:
+		// file was good
+		return zones, nil
+	case !os.IsNotExist(err) || cfg.ConfigFile != DefaultConfigFile:
+		// file was bad
+		return nil, core.Wrapf(err, "NewFromConfig(%q)", cfg.ConfigFile)
+	}
+
+	// default file doesn't exist. scan instead.
 	return cluster.NewFromDirectory(cfg.Base, cfg.Domain,
 		cluster.ResolvePublicAddresses(resolve),
 		cluster.WithLogger(log),
 	)
 }
+
+func init() {
+	rootCmd.PersistentFlags().
+		StringVarP(&cfg.ConfigFile, "config-file", "f",
+			DefaultConfigFile, "config file (JSON or YAML)")
+}

cmd/jpictl/main.go

@@ -37,5 +37,5 @@ func setVerbosity(_ *cobra.Command, _ []string) {
 	if desired > 6 {
 		desired = 6
 	}
-	log = log.WithLevel(slog.LogLevel(desired))
+	log = zerolog.New(nil, slog.LogLevel(desired))
 }

pkg/cluster/ceph_scan.go

@@ -71,7 +71,7 @@ func newCephScanTODO(cfg *ceph.Config) *cephScanTODO {
 	return todo
 }
 
-func (m *Cluster) scanCephMonitors(_ *ScanOptions) error {
+func (m *Cluster) scanCephMonitors(opts *ScanOptions) error {
 	cfg, err := m.GetCephConfig()
 	switch {
 	case os.IsNotExist(err):
@@ -94,6 +94,10 @@ func (m *Cluster) scanCephMonitors(_ *ScanOptions) error {
 		todo.LogMissing(m.log)
 	}
 
+	return m.initCephMonitors(opts)
+}
+
+func (m *Cluster) initCephMonitors(_ *ScanOptions) error {
 	// make sure every zone has one
 	m.ForEachZone(func(z *Zone) bool {
 		_ = z.GetCephMonitors()

pkg/cluster/cluster.go

@@ -27,7 +27,8 @@ type Cluster struct {
 	Domain  string `json:"domain,omitempty" yaml:"domain,omitempty"`
 
 	CephFSID uuid.UUID `json:"ceph_fsid,omitempty" yaml:"ceph_fsid,omitempty"`
-	Zones    []*Zone   `json:"zones,omitempty"     yaml:"zones,omitempty"`
+	Regions  []Region  `json:",omitempty"          yaml:",omitempty"`
+	Zones    []*Zone   `json:",omitempty"          yaml:",omitempty"`
 }
 
 // revive:enable:line-length-limit

pkg/cluster/cluster_fs.go

@@ -0,0 +1,25 @@
+package cluster
+
+import (
+	"io/fs"
+	"path/filepath"
+
+	"github.com/hack-pad/hackpadfs/os"
+)
+
+// DirFS returns a file system (an [fs.FS]) for the tree
+// of files rooted at the directory dir.
+func DirFS(dir string) (fs.FS, error) {
+	dir = filepath.Clean(dir)
+	fullPath, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	sub, err := os.NewFS().Sub(fullPath[1:])
+	if err != nil {
+		return nil, err
+	}
+
+	return sub, nil
+}

pkg/cluster/cluster_import.go

@@ -0,0 +1,139 @@
+package cluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+func (m *Cluster) init(opts *ScanOptions) error {
+	for _, fn := range []func(*ScanOptions) error{
+		m.initZones,
+		m.scanZoneIDs,
+		m.scanSort,
+		m.scanGateways,
+		m.initCephMonitors,
+		m.initRegions,
+	} {
+		if err := fn(opts); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Cluster) initZones(opts *ScanOptions) error {
+	var err error
+
+	sub, err := DirFS(m.BaseDir)
+	if err != nil {
+		return err
+	}
+
+	m.dir = sub
+
+	m.ForEachZone(func(z *Zone) bool {
+		err = m.initZone(z, opts)
+		return err != nil
+	})
+
+	return err
+}
+
+func (m *Cluster) initZone(z *Zone, _ *ScanOptions) error {
+	var hasMissing bool
+	var lastMachineID int
+
+	z.zones = m
+	z.logger = m
+
+	z.ForEachMachine(func(p *Machine) bool {
+		p.zone = z
+		p.logger = z
+
+		switch {
+		case p.ID == 0:
+			hasMissing = true
+		case p.ID > lastMachineID:
+			lastMachineID = z.ID
+		}
+
+		return false
+	})
+
+	if hasMissing {
+		next := lastMachineID + 1
+
+		z.ForEachMachine(func(p *Machine) bool {
+			if p.ID == 0 {
+				p.ID, next = next, next+1
+			}
+
+			return false
+		})
+	}
+
+	z.ForEachMachine(func(p *Machine) bool {
+		p.Name = fmt.Sprintf("%s-%v", z.Name, p.ID)
+		return false
+	})
+
+	return nil
+}
+
+func decodeConfigData(data []byte) (out *Cluster, err error) {
+	// try JSON first
+	out = new(Cluster)
+	err = json.Unmarshal(data, out)
+	if err == nil {
+		// good json
+		return out, nil
+	} else if _, ok := err.(*json.SyntaxError); !ok {
+		// bad json
+		return nil, err
+	}
+
+	out = new(Cluster)
+	err = yaml.Unmarshal(data, out)
+	if err != nil {
+		// bad yaml too
+		return nil, err
+	}
+
+	// good yaml
+	return out, nil
+}
+
+// NewFromConfig loads the cluster data from the given file
+func NewFromConfig(filename string, opts ...ScanOption) (*Cluster, error) {
+	var scanOptions ScanOptions
+
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	m, err := decodeConfigData(data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, opt := range opts {
+		if err = opt(m, &scanOptions); err != nil {
+			return nil, err
+		}
+	}
+
+	if err = m.setScanDefaults(&scanOptions); err != nil {
+		return nil, err
+	}
+
+	if err := m.init(&scanOptions); err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}

pkg/cluster/cluster_scan_options.go

@@ -6,7 +6,6 @@ import (
 
 	"darvaza.org/resolver"
 	"darvaza.org/slog"
-	"github.com/hack-pad/hackpadfs/os"
 )
 
 // A ScanOption pre-configures the Zones before scanning
@@ -101,7 +100,7 @@ func NewFromDirectory(dir, domain string, opts ...ScanOption) (*Cluster, error)
 		return nil, err
 	}
 
-	sub, err := os.NewFS().Sub(fullPath[1:])
+	sub, err := DirFS(dir)
 	if err != nil {
 		return nil, err
 	}

pkg/cluster/hosts.go

@@ -0,0 +1,128 @@
+package cluster
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"text/template"
+)
+
+type hostsFile struct {
+	Ring0 []hostsEntry
+	Ring1 []hostsEntry
+}
+
+type hostsEntry struct {
+	Addr  string
+	Names []string
+}
+
+var hostsTemplate = template.Must(template.New("hosts").Funcs(template.FuncMap{
+	"StringsJoin": strings.Join,
+}).Parse(`127.0.0.1 localhost
+
+# The following lines are desirable for IPv6 capable hosts
+::1     ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+ff02::3 ip6-allhosts
+
+{{range .Ring1 -}}
+{{.Addr}}  {{StringsJoin .Names " "}}
+{{end}}
+{{range .Ring0 -}}
+{{.Addr}}  {{StringsJoin .Names " "}}
+{{end -}}
+`))
+
+// WriteHosts rewrites all hosts files on the tree
+func (m *Cluster) WriteHosts() error {
+	var err error
+
+	m.ForEachZone(func(z *Zone) bool {
+		err = z.WriteHosts()
+		return err != nil
+	})
+
+	return err
+}
+
+// WriteHosts rewrites all hosts files in the zone
+func (z *Zone) WriteHosts() error {
+	var err error
+
+	s := z.Hosts()
+	z.ForEachMachine(func(p *Machine) bool {
+		err = p.WriteStringFile(s, "hosts")
+		return err != nil
+	})
+
+	return err
+}
+
+// WriteHosts rewrites the hosts file
+func (p *Machine) WriteHosts() error {
+	s := p.zone.Hosts()
+	return p.WriteStringFile(s, "hosts")
+}
+
+func (z *Zone) genHosts(out *hostsFile, p *Machine) {
+	var names []string
+
+	ip, _ := RingOneAddress(p.zone.ID, p.ID)
+	names = append(names, p.Name)
+
+	if p.CephMonitor {
+		names = append(names, fmt.Sprintf("%s-%s", p.zone.Name, "ceph"))
+		names = append(names, fmt.Sprintf("%s-%s", p.zone.Name, "k3s"))
+
+		if z.ID == p.zone.ID {
+			names = append(names, "ceph")
+			names = append(names, "k3s")
+		}
+	}
+
+	entry := hostsEntry{
+		Addr:  ip.String(),
+		Names: names,
+	}
+
+	out.Ring1 = append(out.Ring1, entry)
+
+	if p.IsGateway() {
+		var s string
+
+		ip, _ = RingZeroAddress(p.zone.ID, p.ID)
+		s = fmt.Sprintf("%s-%v", p.Name, 0)
+
+		entry = hostsEntry{
+			Addr:  ip.String(),
+			Names: []string{s},
+		}
+
+		out.Ring0 = append(out.Ring0, entry)
+	}
+}
+
+// Hosts renders the /etc/hosts to be used on this zone
+func (z *Zone) Hosts() string {
+	var buf bytes.Buffer
+	var out hostsFile
+
+	z.zones.ForEachZone(func(z2 *Zone) bool {
+		z2.ForEachMachine(func(p *Machine) bool {
+			z.genHosts(&out, p)
+
+			return false
+		})
+		return false
+	})
+
+	if err := hostsTemplate.Execute(&buf, &out); err != nil {
+		panic(err)
+	}
+
+	return buf.String()
+}

pkg/cluster/regions.go

@@ -0,0 +1,88 @@
+package cluster
+
+// Region represents a group of zones geographically related
+type Region struct {
+	m     *Cluster
+	zones []*Zone
+
+	Name    string
+	Regions []string `json:",omitempty" yaml:",omitempty"`
+}
+
+func (m *Cluster) initRegions(_ *ScanOptions) error {
+	regions := make(map[string][]*Zone)
+
+	// first regions defined by zones
+	m.ForEachZone(func(z *Zone) bool {
+		for _, region := range z.Regions {
+			regions[region] = append(regions[region], z)
+		}
+
+		return false
+	})
+
+	// bind first level regions and their zones
+	for name, zones := range regions {
+		m.syncRegions(name, zones...)
+	}
+
+	// and combine zones to produce larger regions
+	for i := range m.Regions {
+		r := &m.Regions[i]
+		m.finishRegion(r)
+	}
+
+	return nil
+}
+
+func (m *Cluster) syncRegions(name string, zones ...*Zone) {
+	for _, r := range m.Regions {
+		if r.Name == name {
+			// found
+			r.m = m
+			r.zones = zones
+			return
+		}
+	}
+
+	// new
+	m.Regions = append(m.Regions, Region{
+		m:     m,
+		zones: zones,
+		Name:  name,
+	})
+}
+
+func (m *Cluster) finishRegion(r *Region) {
+	if r.m != nil {
+		// ready
+		return
+	}
+
+	r.m = m
+	sub := []string{}
+	for _, name := range r.Regions {
+		r2, ok := m.getRegion(name)
+		if !ok {
+			m.warn(nil).WithField("region", name).Print("unknown region")
+			continue
+		}
+
+		sub = append(sub, r2.Name)
+		r.zones = append(r.zones, r2.zones...)
+	}
+	r.Regions = sub
+}
+
+func (m *Cluster) getRegion(name string) (*Region, bool) {
+	for i := range m.Regions {
+		r := &m.Regions[i]
+
+		if name == r.Name {
+			m.finishRegion(r)
+			return r, true
+		}
+	}
+
+	return nil, false
+}

pkg/cluster/sync.go

@@ -5,6 +5,7 @@ func (m *Cluster) SyncAll() error {
 	for _, fn := range []func() error{
 		m.SyncAllWireguard,
 		m.SyncAllCeph,
+		m.WriteHosts,
 	} {
 		if err := fn(); err != nil {
 			return err

pkg/cluster/zones.go

@@ -19,8 +19,9 @@ type Zone struct {
 	zones  *Cluster
 	logger `json:"-" yaml:"-"`
 
-	ID   int
-	Name string
+	ID      int
+	Name    string
+	Regions []string `json:",omitempty" yaml:",omitempty"`
 
 	Machines
 }