Compare commits

..

2 Commits

Author SHA1 Message Date
lerko ab1194f74d fix(monitor): refresh maintenance cache after loading sites
CI / test (pull_request) Successful in 1m49s
CI / lint (pull_request) Successful in 1m17s
CI / vulncheck (pull_request) Successful in 56s
Move refreshMaintenanceCache() to after sites are loaded into
liveState. Previously it ran before the site loading loop, so on
the first iteration liveState was empty and the cache resolved
to nothing. This caused groups to briefly show DOWN on startup
for children that were in maintenance, until the next poll cycle
repopulated the cache.
2026-06-28 11:15:35 -04:00
lerko 25f4b20b87 feat(monitor): save state changes for group monitors
CI / test (pull_request) Successful in 1m57s
CI / lint (pull_request) Successful in 1m11s
CI / vulncheck (pull_request) Successful in 56s
checkGroup() now detects status transitions and persists them as
state change records, same as individual monitors. Groups also
get StatusChangedAt updates and log entries for down/recovery
events. This enables history and SLA views for group monitors.
2026-06-28 11:00:25 -04:00
3 changed files with 148 additions and 2 deletions
+14
View File
@@ -239,10 +239,24 @@ func (e *Engine) checkGroup(_ context.Context, site models.Site) {
status = models.StatusPending
}
var prev models.Status
e.applyState(site.ID, func(s *models.Site) {
prev = s.Status
s.Status = status
if status != prev && prev != models.StatusPending {
s.StatusChangedAt = time.Now()
}
})
e.recordCheck(site.ID, 0, !status.IsBroken())
if status != prev && prev != models.StatusPending {
e.enqueueWrite(writeStateChange{siteID: site.ID, fromStatus: string(prev), toStatus: string(status)})
if status.IsBroken() {
e.AddLog(fmt.Sprintf("Group '%s' is %s", site.Name, status))
} else if prev.IsBroken() {
e.AddLog(fmt.Sprintf("Group '%s' recovered", site.Name))
}
}
}
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
+4 -2
View File
@@ -283,8 +283,6 @@ func (e *Engine) Start(ctx context.Context) {
default:
}
e.refreshMaintenanceCache(ctx)
configs, err := e.db.GetSites(ctx)
if err != nil {
e.AddLog(fmt.Sprintf("Failed to load sites: %v", err))
@@ -327,6 +325,10 @@ func (e *Engine) Start(ctx context.Context) {
}
}
// Refresh after sites load so the cache covers newly added sites.
// On first iteration liveState was empty before the loop above.
e.refreshMaintenanceCache(ctx)
e.mu.RLock()
var vanished []int
for id := range e.liveState {
+130
View File
@@ -445,6 +445,136 @@ func TestEngineStop_Idempotent(t *testing.T) {
e.Stop() // must not panic or block
}
// --- Group 11: Group State History ---
func TestCheckGroup_SavesStateChange(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 100, Name: "Infra", Type: "group"},
SiteState: models.SiteState{Status: models.StatusUp},
}
child := models.Site{
SiteConfig: models.SiteConfig{ID: 101, Name: "Server", Type: "ping", ParentID: 100},
SiteState: models.SiteState{Status: models.StatusUp},
}
injectSite(e, group)
injectSite(e, child)
// Child goes down → group should transition UP→DOWN.
child.Status = models.StatusDown
injectSite(e, child)
e.checkGroup(context.Background(), group)
s, ok := getSite(e, 100)
if !ok {
t.Fatal("group not found")
}
if s.Status != models.StatusDown {
t.Errorf("expected DOWN, got %s", s.Status)
}
if s.StatusChangedAt.IsZero() {
t.Error("expected StatusChangedAt to be set")
}
logs := e.GetLogs()
found := false
for _, l := range logs {
if containsStr(l.Message, "Group 'Infra' is DOWN") {
found = true
break
}
}
if !found {
t.Error("expected log entry for group state change")
}
// Drain the write channel to verify a state change was enqueued.
select {
case w := <-e.dbWrites:
if w.desc() != "state-change" {
// Could be a check write; try one more.
select {
case w2 := <-e.dbWrites:
if w2.desc() != "state-change" {
t.Errorf("expected state-change write, got %s then %s", w.desc(), w2.desc())
}
default:
t.Errorf("expected state-change write, got only %s", w.desc())
}
}
default:
t.Error("no write enqueued for group state change")
}
}
func TestCheckGroup_NoChangeNoWrite(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 100, Name: "Infra", Type: "group"},
SiteState: models.SiteState{Status: models.StatusUp},
}
child := models.Site{
SiteConfig: models.SiteConfig{ID: 101, Name: "Server", Type: "ping", ParentID: 100},
SiteState: models.SiteState{Status: models.StatusUp},
}
injectSite(e, group)
injectSite(e, child)
e.checkGroup(context.Background(), group)
// Only a check write should be enqueued, no state-change.
drainCount := 0
for {
select {
case w := <-e.dbWrites:
if w.desc() == "state-change" {
t.Error("state-change write enqueued when status didn't change")
}
drainCount++
default:
goto done
}
}
done:
if drainCount == 0 {
t.Error("expected at least a check write")
}
}
func TestCheckGroup_RecoveryLog(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 100, Name: "Infra", Type: "group"},
SiteState: models.SiteState{Status: models.StatusDown},
}
child := models.Site{
SiteConfig: models.SiteConfig{ID: 101, Name: "Server", Type: "ping", ParentID: 100},
SiteState: models.SiteState{Status: models.StatusUp},
}
injectSite(e, group)
injectSite(e, child)
e.checkGroup(context.Background(), group)
logs := e.GetLogs()
found := false
for _, l := range logs {
if containsStr(l.Message, "Group 'Infra' recovered") {
found = true
break
}
}
if !found {
t.Error("expected recovery log entry for group")
}
}
// --- Utilities ---
func containsStr(s, substr string) bool {