From a8849b747c5b46d37721c0f81ed47f5c05d81ed5 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 12:13:26 +0000 Subject: [PATCH 01/13] vscode: add ceph to the dictionary Signed-off-by: Alejandro Mery --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 2e06f2a..7411ff7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "cSpell.words": [ + "ceph", "darvaza", "jpictl", "zerolog" From 0fb8c1d44b8b8322280b4a2e08c32e1fb4874033 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 12:04:22 +0000 Subject: [PATCH 02/13] zones: introduce Zones.CephFSID and Zones.GetCephFSID() the accessor doesn't generate one if needed yet Signed-off-by: Alejandro Mery --- go.mod | 1 + go.sum | 2 ++ pkg/zones/ceph.go | 15 +++++++++++++++ pkg/zones/zones.go | 7 +++++++ 4 files changed, 25 insertions(+) create mode 100644 pkg/zones/ceph.go diff --git a/go.mod b/go.mod index 898b328..6e9c068 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( darvaza.org/sidecar v0.0.2 darvaza.org/slog v0.5.3 github.com/burntSushi/toml v0.3.1 + github.com/gofrs/uuid/v5 v5.0.0 github.com/hack-pad/hackpadfs v0.2.1 github.com/mgechev/revive v1.3.3 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index d04ecf7..8d58d18 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid/v5 v5.0.0 h1:p544++a97kEL+svbcFbCQVM9KFu0Yo25UoISXGNNH9M= +github.com/gofrs/uuid/v5 v5.0.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/hack-pad/hackpadfs v0.2.1 h1:FelFhIhv26gyjujoA/yeFO+6YGlqzmc9la/6iKMIxMw= github.com/hack-pad/hackpadfs v0.2.1/go.mod h1:khQBuCEwGXWakkmq8ZiFUvUZz84ZkJ2KNwKvChs4OrU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/pkg/zones/ceph.go b/pkg/zones/ceph.go new file mode 100644 index 0000000..94ef7d9 --- /dev/null +++ b/pkg/zones/ceph.go @@ -0,0 +1,15 @@ +package zones + +import ( + "darvaza.org/core" + "github.com/gofrs/uuid/v5" +) + +// GetCephFSID returns our Ceph's FSID +func (m *Zones) GetCephFSID() (uuid.UUID, error) { + if core.IsZero(m.CephFSID) { + // TODO: generate one + return uuid.Nil, nil + } + return m.CephFSID, nil +} diff --git a/pkg/zones/zones.go b/pkg/zones/zones.go index f25f264..fad4c30 100644 --- a/pkg/zones/zones.go +++ b/pkg/zones/zones.go @@ -6,6 +6,7 @@ import ( "sort" "darvaza.org/resolver" + "github.com/gofrs/uuid/v5" ) var ( @@ -135,15 +136,21 @@ func (z *Zone) GatewayIDs() ([]int, int) { return out, len(out) } +// revive:disable:line-length-limit + // Zones represents all zones in a cluster type Zones struct { dir fs.FS resolver resolver.Resolver domain string + CephFSID uuid.UUID `toml:"ceph_fsid,omitempty" json:"ceph_fsid,omitempty" yaml:"ceph_fsid,omitempty"` + Zones []*Zone `toml:"zones"` } +// revive:enable:line-length-limit + // ForEachMachine calls a function for each Machine in the cluster // until instructed to terminate the loop func (m *Zones) ForEachMachine(fn func(*Machine) bool) { From 025b9072b4719b03c577b76f9ca11a74a0ce21bb Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 11:25:11 +0000 Subject: [PATCH 03/13] zones: introduce Machine.CephMonitor field Signed-off-by: Alejandro Mery --- pkg/zones/machine.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/zones/machine.go b/pkg/zones/machine.go index 1dd64a2..8500e9e 100644 --- a/pkg/zones/machine.go +++ b/pkg/zones/machine.go @@ -15,6 +15,8 @@ type Machine struct { PublicAddresses []netip.Addr `toml:"public,omitempty" json:"public,omitempty" yaml:"public,omitempty"` Rings []*RingInfo `toml:"rings,omitempty" json:"rings,omitempty" yaml:"rings,omitempty"` + + CephMonitor bool `toml:"ceph_monitor,omitempty" json:"ceph_monitor,omitempty" yaml:"ceph_monitor,omitempty"` } // revive:enable:line-length-limit From 94011a3a030df479d047b6298e31e1d9feb05bf6 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Mon, 4 Sep 2023 19:42:04 +0000 Subject: [PATCH 04/13] ceph: add NewConfigFromReader() and initial ceph.conf parser Signed-off-by: Alejandro Mery --- go.mod | 2 + go.sum | 4 ++ pkg/ceph/ceph.go | 2 + pkg/ceph/config.go | 38 +++++++++++++ pkg/ceph/config_parser.go | 110 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+) create mode 100644 pkg/ceph/ceph.go create mode 100644 pkg/ceph/config.go create mode 100644 pkg/ceph/config_parser.go diff --git a/go.mod b/go.mod index 6e9c068..019ccee 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module git.jpi.io/amery/jpictl go 1.19 require ( + asciigoat.org/ini v0.2.5 darvaza.org/core v0.9.8 darvaza.org/resolver v0.5.4 darvaza.org/sidecar v0.0.2 @@ -18,6 +19,7 @@ require ( ) require ( + asciigoat.org/core v0.3.9 // indirect darvaza.org/slog/handlers/filter v0.4.5 // indirect darvaza.org/slog/handlers/zerolog v0.4.5 // indirect github.com/BurntSushi/toml v1.3.2 // indirect diff --git a/go.sum b/go.sum index 8d58d18..e2f357a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +asciigoat.org/core v0.3.9 h1:hgDDz4ecm3ZvehX++m8A/IzAt+B5oDPiRtxatzfUHPQ= +asciigoat.org/core v0.3.9/go.mod h1:CAaHwyw8MpAq4a1MYtN2dxJrsK+hmIdW50OndaQZYPI= +asciigoat.org/ini v0.2.5 h1:4gRIp9rU+XQt8+HMqZO5R7GavMv9Yl2+N+je6djDIAE= +asciigoat.org/ini v0.2.5/go.mod h1:gmXzJ9XFqf1NLk5nQkj04USQ4tMtdRJHNQX6vp3DzjU= darvaza.org/core v0.9.8 h1:luLxgfUc2pzuusYPo/Z/dC/qr9XZPKpSQw8/kS7zNUM= darvaza.org/core v0.9.8/go.mod h1:Dbme64naxeshQfxcVJX9ZT7AiGyIY8kldfuELVtf8mw= darvaza.org/resolver v0.5.4 h1:dlSBNV14yYsp7Kg7ipwYOMNsLbrpeXa8Z0HBTa0Ryxs= diff --git a/pkg/ceph/ceph.go b/pkg/ceph/ceph.go new file mode 100644 index 0000000..53e1099 --- /dev/null +++ b/pkg/ceph/ceph.go @@ -0,0 +1,2 @@ +// Package ceph deals with ceph config +package ceph diff --git a/pkg/ceph/config.go b/pkg/ceph/config.go new file mode 100644 index 0000000..64c5bff --- /dev/null +++ b/pkg/ceph/config.go @@ -0,0 +1,38 @@ +package ceph + +import ( + "io" + "net/netip" + + "github.com/gofrs/uuid/v5" + + "asciigoat.org/ini/basic" +) + +// Config represents a ceph.conf file +type Config struct { + Global GlobalConfig `ini:"global"` +} + +// GlobalConfig represents the [global] section of a ceph.conf file +type GlobalConfig struct { + FSID uuid.UUID `ini:"fsid"` + Monitors []string `ini:"mon_initial_members,comma"` + MonitorsAddr []netip.Addr `ini:"mon_host,comma"` + ClusterNetwork netip.Prefix `ini:"cluster_network"` +} + +// NewConfigFromReader parses the ceph.conf file +func NewConfigFromReader(r io.Reader) (*Config, error) { + doc, err := basic.Decode(r) + if err != nil { + return nil, err + } + + cfg, err := newConfigFromDocument(doc) + if err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/pkg/ceph/config_parser.go b/pkg/ceph/config_parser.go new file mode 100644 index 0000000..9428e27 --- /dev/null +++ b/pkg/ceph/config_parser.go @@ -0,0 +1,110 @@ +package ceph + +import ( + "io/fs" + "net/netip" + + "asciigoat.org/ini/basic" + "asciigoat.org/ini/parser" + + "darvaza.org/core" +) + +var sectionMap = map[string]func(*Config, *basic.Section) error{ + "global": loadGlobalConfSection, +} + +func loadConfSection(out *Config, src *basic.Section) error { + h, ok := sectionMap[src.Key] + if !ok { + return core.Wrapf(fs.ErrInvalid, "unknown section %q", src.Key) + } + + return h(out, src) +} + +func loadGlobalConfSection(out *Config, src *basic.Section) error { + var cfg GlobalConfig + + for _, field := range src.Fields { + if err := loadGlobalConfField(&cfg, field); err != nil { + return core.Wrap(err, "global") + } + } + + out.Global = cfg + return nil +} + +// revive:disable:cyclomatic +// revive:disable:cognitive-complexity + +func loadGlobalConfField(cfg *GlobalConfig, field basic.Field) error { + // revive:enable:cyclomatic + // revive:enable:cognitive-complexity + + // TODO: refactor when asciigoat's ini parser learns to do reflection + + switch field.Key { + case "fsid": + if !core.IsZero(cfg.FSID) { + return core.Wrapf(fs.ErrInvalid, "duplicate field %q", field.Key) + } + + err := cfg.FSID.UnmarshalText([]byte(field.Value)) + switch { + case err != nil: + return core.Wrap(err, field.Key) + default: + return nil + } + case "mon_host": + entries, _ := parser.SplitCommaArray(field.Value) + for _, s := range entries { + var addr netip.Addr + + if err := addr.UnmarshalText([]byte(s)); err != nil { + return core.Wrap(err, field.Key) + } + + cfg.MonitorsAddr = append(cfg.MonitorsAddr, addr) + } + return nil + case "mon_initial_members": + entries, _ := parser.SplitCommaArray(field.Value) + cfg.Monitors = append(cfg.Monitors, entries...) + return nil + case "cluster_network": + if !core.IsZero(cfg.ClusterNetwork) { + err := core.Wrap(fs.ErrInvalid, "fields before the first section") + return err + } + + err := cfg.ClusterNetwork.UnmarshalText([]byte(field.Value)) + switch { + case err != nil: + return core.Wrap(err, field.Key) + default: + return nil + } + } + return nil +} + +func newConfigFromDocument(doc *basic.Document) (*Config, error) { + var out Config + + if len(doc.Global) > 0 { + err := core.Wrap(fs.ErrInvalid, "fields before the first section") + return nil, err + } + + for i := range doc.Sections { + src := &doc.Sections[i] + if err := loadConfSection(&out, src); err != nil { + return nil, err + } + } + + return &out, nil +} From 05e04c758b25f43c8307bc4b9c1a89b5d9f49d64 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Mon, 4 Sep 2023 19:43:52 +0000 Subject: [PATCH 05/13] zones: introduce Zones.GetCephConfig() accessor for m/ceph.conf Signed-off-by: Alejandro Mery --- pkg/zones/ceph.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/zones/ceph.go b/pkg/zones/ceph.go index 94ef7d9..0968ab6 100644 --- a/pkg/zones/ceph.go +++ b/pkg/zones/ceph.go @@ -1,8 +1,12 @@ package zones import ( + "bytes" + "darvaza.org/core" "github.com/gofrs/uuid/v5" + + "git.jpi.io/amery/jpictl/pkg/ceph" ) // GetCephFSID returns our Ceph's FSID @@ -13,3 +17,14 @@ func (m *Zones) GetCephFSID() (uuid.UUID, error) { } return m.CephFSID, nil } + +// GetCephConfig reads the ceph.conf file +func (m *Zones) GetCephConfig() (*ceph.Config, error) { + data, err := m.ReadFile("ceph.conf") + if err != nil { + return nil, err + } + + r := bytes.NewReader(data) + return ceph.NewConfigFromReader(r) +} From e3ab931eb1627c19c91ebdca6535d9c9b2fdc434 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 11:01:50 +0000 Subject: [PATCH 06/13] zones: introduce Zone.GetCephMonitors() returning the local ceph monitors and setting one if there is none. non-gateway nodes are preferred when setting a monitor automatically Signed-off-by: Alejandro Mery --- pkg/zones/ceph.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pkg/zones/ceph.go b/pkg/zones/ceph.go index 0968ab6..ececc95 100644 --- a/pkg/zones/ceph.go +++ b/pkg/zones/ceph.go @@ -2,6 +2,7 @@ package zones import ( "bytes" + "sort" "darvaza.org/core" "github.com/gofrs/uuid/v5" @@ -28,3 +29,47 @@ func (m *Zones) GetCephConfig() (*ceph.Config, error) { r := bytes.NewReader(data) return ceph.NewConfigFromReader(r) } + +// GetCephMonitors returns the set of Ceph monitors on +// the zone +func (z *Zone) GetCephMonitors() Machines { + var mons Machines + var first, second *Machine + + z.ForEachMachine(func(p *Machine) bool { + switch { + case p.CephMonitor: + // it is a monitor + mons = append(mons, p) + case len(mons) > 0: + // zone has a monitor + case first == nil && !p.IsGateway(): + // first option for monitor + first = p + case second == nil: + // second option for monitor + second = p + } + + return false + }) + + switch { + case len(mons) > 0: + // ready + case first != nil: + // make first option our monitor + first.CephMonitor = true + mons = append(mons, first) + case second != nil: + // make second option our monitor + second.CephMonitor = true + mons = append(mons, second) + default: + // zone without machines?? + panic("unreachable") + } + + sort.Sort(mons) + return mons +} From bf4bfeb3fc1474bc13d1e2949c6c3b35528c7a50 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 11:45:02 +0000 Subject: [PATCH 07/13] zones: introduce GenCephConfig() Signed-off-by: Alejandro Mery --- pkg/zones/ceph.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pkg/zones/ceph.go b/pkg/zones/ceph.go index ececc95..63a030c 100644 --- a/pkg/zones/ceph.go +++ b/pkg/zones/ceph.go @@ -2,6 +2,7 @@ package zones import ( "bytes" + "net/netip" "sort" "darvaza.org/core" @@ -30,6 +31,36 @@ func (m *Zones) GetCephConfig() (*ceph.Config, error) { return ceph.NewConfigFromReader(r) } +// GenCephConfig prepares a ceph.Config using the cluster information +func (m *Zones) GenCephConfig() (*ceph.Config, error) { + fsid, err := m.GetCephFSID() + if err != nil { + return nil, err + } + + cfg := &ceph.Config{ + Global: ceph.GlobalConfig{ + FSID: fsid, + ClusterNetwork: netip.PrefixFrom( + netip.AddrFrom4([4]byte{10, 0, 0, 0}), + 8, + ), + }, + } + + m.ForEachZone(func(z *Zone) bool { + for _, p := range z.GetCephMonitors() { + addr, _ := RingOneAddress(z.ID, p.ID) + + cfg.Global.Monitors = append(cfg.Global.Monitors, p.Name) + cfg.Global.MonitorsAddr = append(cfg.Global.MonitorsAddr, addr) + } + return false + }) + + return cfg, nil +} + // GetCephMonitors returns the set of Ceph monitors on // the zone func (z *Zone) GetCephMonitors() Machines { From 77ad016e99529902cd181275fd664612e4c5ae89 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Mon, 4 Sep 2023 22:01:13 +0000 Subject: [PATCH 08/13] zones: set Machine.CephMonitor if its referenced as monitor on ceph.conf Signed-off-by: Alejandro Mery --- pkg/zones/ceph_scan.go | 173 +++++++++++++++++++++++++++++++++++++++++ pkg/zones/scan.go | 1 + 2 files changed, 174 insertions(+) create mode 100644 pkg/zones/ceph_scan.go diff --git a/pkg/zones/ceph_scan.go b/pkg/zones/ceph_scan.go new file mode 100644 index 0000000..32e3c7f --- /dev/null +++ b/pkg/zones/ceph_scan.go @@ -0,0 +1,173 @@ +package zones + +import ( + "net/netip" + "os" + "strings" + + "darvaza.org/core" + "git.jpi.io/amery/jpictl/pkg/ceph" +) + +// CephMissingMonitorError is an error that contains ceph +// monitors present in ceph.conf but not found on the cluster +type CephMissingMonitorError struct { + Names []string + Addrs []netip.Addr +} + +func (err *CephMissingMonitorError) appendName(name string) { + err.Names = append(err.Names, name) +} + +func (err *CephMissingMonitorError) appendAddr(addr netip.Addr) { + err.Addrs = append(err.Addrs, addr) +} + +// OK tells if this instance actual shouldn't be treated as an error +func (err CephMissingMonitorError) OK() bool { + switch { + case len(err.Names) > 0: + return false + case len(err.Addrs) > 0: + return false + default: + return true + } +} + +func (err CephMissingMonitorError) Error() string { + if !err.OK() { + var buf strings.Builder + + _, _ = buf.WriteString("missing:") + err.writeNames(&buf) + err.writeAddrs(&buf) + + return buf.String() + } + + // no error + return "" +} + +func (err *CephMissingMonitorError) writeNames(w *strings.Builder) { + if len(err.Names) > 0 { + _, _ = w.WriteString(" mon_host:") + for i, name := range err.Names { + if i != 0 { + _, _ = w.WriteRune(',') + } + _, _ = w.WriteString(name) + } + } +} + +func (err *CephMissingMonitorError) writeAddrs(w *strings.Builder) { + _, _ = w.WriteString(" mon_initial_members:") + for i, addr := range err.Addrs { + if i != 0 { + _, _ = w.WriteRune(',') + } + _, _ = w.WriteString(addr.String()) + } +} + +// AsError returns nil if the instance is actually OK +func (err *CephMissingMonitorError) AsError() error { + if err == nil || err.OK() { + return nil + } + + return err +} + +type cephScanTODO struct { + names map[string]bool + addrs map[string]bool +} + +func (todo *cephScanTODO) checkMachine(p *Machine) bool { + // on ceph all addresses are ring1 + ring1, _ := RingOneAddress(p.Zone(), p.ID) + addr := ring1.String() + + if _, found := todo.names[p.Name]; found { + // found on the TODO by name + todo.names[p.Name] = true + todo.addrs[addr] = true + return true + } + + if _, found := todo.addrs[addr]; found { + // found on the TODO by address + todo.names[p.Name] = true + todo.addrs[addr] = true + return true + } + + return false +} + +func (todo *cephScanTODO) Missing() error { + var check CephMissingMonitorError + + for name, found := range todo.names { + if !found { + check.appendName(name) + } + } + + for addr, found := range todo.addrs { + if !found { + var a netip.Addr + // it wouldn't be on the map if it wasn't valid + _ = a.UnmarshalText([]byte(addr)) + + check.appendAddr(a) + } + } + + return check.AsError() +} + +func newCephScanTODO(cfg *ceph.Config) *cephScanTODO { + todo := &cephScanTODO{ + names: make(map[string]bool), + addrs: make(map[string]bool), + } + + for _, name := range cfg.Global.Monitors { + todo.names[name] = false + } + + for _, addr := range cfg.Global.MonitorsAddr { + todo.addrs[addr.String()] = false + } + + return todo +} + +func (m *Zones) scanCephMonitors(_ *ScanOptions) error { + cfg, err := m.GetCephConfig() + switch { + case os.IsNotExist(err): + err = nil + case err != nil: + return err + } + + if cfg != nil { + // flag monitors based on config + todo := newCephScanTODO(cfg) + m.ForEachMachine(func(p *Machine) bool { + p.CephMonitor = todo.checkMachine(p) + return false + }) + if err := todo.Missing(); err != nil { + return core.Wrap(err, "ceph") + } + } + + return nil +} diff --git a/pkg/zones/scan.go b/pkg/zones/scan.go index 08734a3..d13879a 100644 --- a/pkg/zones/scan.go +++ b/pkg/zones/scan.go @@ -12,6 +12,7 @@ func (m *Zones) scan(opts *ScanOptions) error { m.scanZoneIDs, m.scanSort, m.scanGateways, + m.scanCephMonitors, } { if err := fn(opts); err != nil { return err From b772ec0a3d0ec5828da5a1adc9912406c1179592 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 13:40:16 +0000 Subject: [PATCH 09/13] zones: store ceph FSID on scan Signed-off-by: Alejandro Mery --- pkg/zones/ceph_scan.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/zones/ceph_scan.go b/pkg/zones/ceph_scan.go index 32e3c7f..724e24d 100644 --- a/pkg/zones/ceph_scan.go +++ b/pkg/zones/ceph_scan.go @@ -158,6 +158,9 @@ func (m *Zones) scanCephMonitors(_ *ScanOptions) error { } if cfg != nil { + // store FSID + m.CephFSID = cfg.Global.FSID + // flag monitors based on config todo := newCephScanTODO(cfg) m.ForEachMachine(func(p *Machine) bool { From cea8362fe6680187bdc77122739d4d1a7a5aaa98 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 11:03:53 +0000 Subject: [PATCH 10/13] zones: extend scan to ensure every zone has a ceph monitor Signed-off-by: Alejandro Mery --- pkg/zones/ceph_scan.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/zones/ceph_scan.go b/pkg/zones/ceph_scan.go index 724e24d..2180c09 100644 --- a/pkg/zones/ceph_scan.go +++ b/pkg/zones/ceph_scan.go @@ -53,7 +53,7 @@ func (err CephMissingMonitorError) Error() string { func (err *CephMissingMonitorError) writeNames(w *strings.Builder) { if len(err.Names) > 0 { - _, _ = w.WriteString(" mon_host:") + _, _ = w.WriteString(" mon_initial_members:") for i, name := range err.Names { if i != 0 { _, _ = w.WriteRune(',') @@ -64,12 +64,14 @@ func (err *CephMissingMonitorError) writeNames(w *strings.Builder) { } func (err *CephMissingMonitorError) writeAddrs(w *strings.Builder) { - _, _ = w.WriteString(" mon_initial_members:") - for i, addr := range err.Addrs { - if i != 0 { - _, _ = w.WriteRune(',') + if len(err.Addrs) > 0 { + _, _ = w.WriteString(" mon_host:") + for i, addr := range err.Addrs { + if i != 0 { + _, _ = w.WriteRune(',') + } + _, _ = w.WriteString(addr.String()) } - _, _ = w.WriteString(addr.String()) } } @@ -172,5 +174,10 @@ func (m *Zones) scanCephMonitors(_ *ScanOptions) error { } } + // make sure every zone has one + m.ForEachZone(func(z *Zone) bool { + _ = z.GetCephMonitors() + return false + }) return nil } From fe081a42970e9f29c1bd5242c52cfc960ebf7350 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 11:15:38 +0000 Subject: [PATCH 11/13] env: set ceph monitors variables they indicate the ceph monitors on the specified zone * MON{zoneID}_NAME * MON{zoneID}_ID * MON{zoneID}_IP Signed-off-by: Alejandro Mery --- pkg/zones/env.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/pkg/zones/env.go b/pkg/zones/env.go index a1da645..36649f7 100644 --- a/pkg/zones/env.go +++ b/pkg/zones/env.go @@ -59,6 +59,15 @@ func (m *Env) writeEnvZone(w io.Writer, z *Zone) { // ZONE{zoneID}_GW gateways, _ := z.GatewayIDs() m.writeEnvVarInts(w, gateways, "ZONE%v_%s", zoneID, "GW") + + // Ceph + monitors := z.GetCephMonitors() + // MON{zoneID}_NAME + m.writeEnvVar(w, genEnvZoneCephMonNames(monitors), "MON%v_%s", zoneID, "NAME") + // MON{zoneID}_IP + m.writeEnvVar(w, genEnvZoneCephMonIPs(monitors), "MON%v_%s", zoneID, "IP") + // MON{zoneID}_ID + m.writeEnvVar(w, genEnvZoneCephMonIDs(monitors), "MON%v_%s", zoneID, "ID") } func (m *Env) writeEnvVarInts(w io.Writer, value []int, name string, args ...any) { @@ -111,3 +120,44 @@ func genEnvZoneNodes(z *Zone) string { } return "" } + +func genEnvZoneCephMonNames(m Machines) string { + var buf strings.Builder + m.ForEachMachine(func(p *Machine) bool { + if buf.Len() > 0 { + _, _ = buf.WriteRune(' ') + } + _, _ = buf.WriteString(p.Name) + + return false + }) + return buf.String() +} + +func genEnvZoneCephMonIPs(m Machines) string { + var buf strings.Builder + m.ForEachMachine(func(p *Machine) bool { + addr, _ := RingOneAddress(p.Zone(), p.ID) + + if buf.Len() > 0 { + _, _ = buf.WriteRune(' ') + } + _, _ = buf.WriteString(addr.String()) + + return false + }) + return buf.String() +} + +func genEnvZoneCephMonIDs(m Machines) string { + var buf strings.Builder + m.ForEachMachine(func(p *Machine) bool { + if buf.Len() > 0 { + _, _ = buf.WriteRune(' ') + } + _, _ = fmt.Fprintf(&buf, "%v", p.ID) + + return false + }) + return buf.String() +} From ac87757b06b1c016379bc0e65f520680d0759956 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Mon, 4 Sep 2023 19:45:04 +0000 Subject: [PATCH 12/13] ceph: zones.Zones.WriteCephConfig() and ceph.Config.WriteTo() Signed-off-by: Alejandro Mery --- pkg/ceph/config.go | 29 +++++++++++++++++++++++++++++ pkg/zones/ceph.go | 12 ++++++++++++ 2 files changed, 41 insertions(+) diff --git a/pkg/ceph/config.go b/pkg/ceph/config.go index 64c5bff..74e6cf1 100644 --- a/pkg/ceph/config.go +++ b/pkg/ceph/config.go @@ -1,8 +1,11 @@ package ceph import ( + "bytes" + "fmt" "io" "net/netip" + "strings" "github.com/gofrs/uuid/v5" @@ -22,6 +25,32 @@ type GlobalConfig struct { ClusterNetwork netip.Prefix `ini:"cluster_network"` } +// WriteTo writes a Wireguard [Config] onto the provided [io.Writer] +func (cfg *Config) WriteTo(w io.Writer) (int64, error) { + var buf bytes.Buffer + + writeGlobalToBuffer(&buf, &cfg.Global) + return buf.WriteTo(w) +} + +func writeGlobalToBuffer(w *bytes.Buffer, c *GlobalConfig) { + _, _ = w.WriteString("[global]\n") + _, _ = fmt.Fprintf(w, "%s = %s\n", "fsid", c.FSID.String()) + _, _ = fmt.Fprintf(w, "%s = %s\n", "mon_initial_members", strings.Join(c.Monitors, ", ")) + _, _ = fmt.Fprintf(w, "%s = %s\n", "mon_host", joinAddrs(c.MonitorsAddr, ", ")) + _, _ = fmt.Fprintf(w, "%s = %s\n", "cluster_network", c.ClusterNetwork.String()) +} + +func joinAddrs(addrs []netip.Addr, sep string) string { + s := make([]string, len(addrs)) + + for i, addr := range addrs { + s[i] = addr.String() + } + + return strings.Join(s, sep) +} + // NewConfigFromReader parses the ceph.conf file func NewConfigFromReader(r io.Reader) (*Config, error) { doc, err := basic.Decode(r) diff --git a/pkg/zones/ceph.go b/pkg/zones/ceph.go index 63a030c..7b828e3 100644 --- a/pkg/zones/ceph.go +++ b/pkg/zones/ceph.go @@ -31,6 +31,18 @@ func (m *Zones) GetCephConfig() (*ceph.Config, error) { return ceph.NewConfigFromReader(r) } +// WriteCephConfig writes the ceph.conf file +func (m *Zones) WriteCephConfig(cfg *ceph.Config) error { + f, err := m.CreateTruncFile("ceph.conf") + if err != nil { + return err + } + defer f.Close() + + _, err = cfg.WriteTo(f) + return err +} + // GenCephConfig prepares a ceph.Config using the cluster information func (m *Zones) GenCephConfig() (*ceph.Config, error) { fsid, err := m.GetCephFSID() From f10ea1dc22c431cd32e37795ac830d9373846532 Mon Sep 17 00:00:00 2001 From: Alejandro Mery Date: Tue, 5 Sep 2023 12:05:37 +0000 Subject: [PATCH 13/13] jpictl: write m/ceph.conf on sync Signed-off-by: Alejandro Mery --- pkg/zones/sync.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/zones/sync.go b/pkg/zones/sync.go index 71b8951..b7c8b89 100644 --- a/pkg/zones/sync.go +++ b/pkg/zones/sync.go @@ -4,6 +4,7 @@ package zones func (m *Zones) SyncAll() error { for _, fn := range []func() error{ m.SyncAllWireguard, + m.SyncAllCeph, } { if err := fn(); err != nil { return err @@ -31,3 +32,13 @@ func (m *Zones) SyncAllWireguard() error { return nil } + +// SyncAllCeph updates the ceph.conf file +func (m *Zones) SyncAllCeph() error { + cfg, err := m.GenCephConfig() + if err != nil { + return err + } + + return m.WriteCephConfig(cfg) +}