feat(monitor): save state changes for group monitors #158
@@ -239,10 +239,24 @@ func (e *Engine) checkGroup(_ context.Context, site models.Site) {
|
|||||||
status = models.StatusPending
|
status = models.StatusPending
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var prev models.Status
|
||||||
e.applyState(site.ID, func(s *models.Site) {
|
e.applyState(site.ID, func(s *models.Site) {
|
||||||
|
prev = s.Status
|
||||||
s.Status = status
|
s.Status = status
|
||||||
|
if status != prev && prev != models.StatusPending {
|
||||||
|
s.StatusChangedAt = time.Now()
|
||||||
|
}
|
||||||
})
|
})
|
||||||
e.recordCheck(site.ID, 0, !status.IsBroken())
|
e.recordCheck(site.ID, 0, !status.IsBroken())
|
||||||
|
|
||||||
|
if status != prev && prev != models.StatusPending {
|
||||||
|
e.enqueueWrite(writeStateChange{siteID: site.ID, fromStatus: string(prev), toStatus: string(status)})
|
||||||
|
if status.IsBroken() {
|
||||||
|
e.AddLog(fmt.Sprintf("Group '%s' is %s", site.Name, status))
|
||||||
|
} else if prev.IsBroken() {
|
||||||
|
e.AddLog(fmt.Sprintf("Group '%s' recovered", site.Name))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
|
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
|
||||||
|
|||||||
@@ -283,8 +283,6 @@ func (e *Engine) Start(ctx context.Context) {
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
e.refreshMaintenanceCache(ctx)
|
|
||||||
|
|
||||||
configs, err := e.db.GetSites(ctx)
|
configs, err := e.db.GetSites(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
e.AddLog(fmt.Sprintf("Failed to load sites: %v", err))
|
e.AddLog(fmt.Sprintf("Failed to load sites: %v", err))
|
||||||
@@ -327,6 +325,10 @@ func (e *Engine) Start(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Refresh after sites load so the cache covers newly added sites.
|
||||||
|
// On first iteration liveState was empty before the loop above.
|
||||||
|
e.refreshMaintenanceCache(ctx)
|
||||||
|
|
||||||
e.mu.RLock()
|
e.mu.RLock()
|
||||||
var vanished []int
|
var vanished []int
|
||||||
for id := range e.liveState {
|
for id := range e.liveState {
|
||||||
|
|||||||
@@ -445,6 +445,136 @@ func TestEngineStop_Idempotent(t *testing.T) {
|
|||||||
e.Stop() // must not panic or block
|
e.Stop() // must not panic or block
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- Group 11: Group State History ---
|
||||||
|
|
||||||
|
func TestCheckGroup_SavesStateChange(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 100, Name: "Infra", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: models.StatusUp},
|
||||||
|
}
|
||||||
|
child := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 101, Name: "Server", Type: "ping", ParentID: 100},
|
||||||
|
SiteState: models.SiteState{Status: models.StatusUp},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child)
|
||||||
|
|
||||||
|
// Child goes down → group should transition UP→DOWN.
|
||||||
|
child.Status = models.StatusDown
|
||||||
|
injectSite(e, child)
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
s, ok := getSite(e, 100)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("group not found")
|
||||||
|
}
|
||||||
|
if s.Status != models.StatusDown {
|
||||||
|
t.Errorf("expected DOWN, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.StatusChangedAt.IsZero() {
|
||||||
|
t.Error("expected StatusChangedAt to be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
logs := e.GetLogs()
|
||||||
|
found := false
|
||||||
|
for _, l := range logs {
|
||||||
|
if containsStr(l.Message, "Group 'Infra' is DOWN") {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("expected log entry for group state change")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drain the write channel to verify a state change was enqueued.
|
||||||
|
select {
|
||||||
|
case w := <-e.dbWrites:
|
||||||
|
if w.desc() != "state-change" {
|
||||||
|
// Could be a check write; try one more.
|
||||||
|
select {
|
||||||
|
case w2 := <-e.dbWrites:
|
||||||
|
if w2.desc() != "state-change" {
|
||||||
|
t.Errorf("expected state-change write, got %s then %s", w.desc(), w2.desc())
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Errorf("expected state-change write, got only %s", w.desc())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Error("no write enqueued for group state change")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGroup_NoChangeNoWrite(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 100, Name: "Infra", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: models.StatusUp},
|
||||||
|
}
|
||||||
|
child := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 101, Name: "Server", Type: "ping", ParentID: 100},
|
||||||
|
SiteState: models.SiteState{Status: models.StatusUp},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child)
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
// Only a check write should be enqueued, no state-change.
|
||||||
|
drainCount := 0
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case w := <-e.dbWrites:
|
||||||
|
if w.desc() == "state-change" {
|
||||||
|
t.Error("state-change write enqueued when status didn't change")
|
||||||
|
}
|
||||||
|
drainCount++
|
||||||
|
default:
|
||||||
|
goto done
|
||||||
|
}
|
||||||
|
}
|
||||||
|
done:
|
||||||
|
if drainCount == 0 {
|
||||||
|
t.Error("expected at least a check write")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGroup_RecoveryLog(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 100, Name: "Infra", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: models.StatusDown},
|
||||||
|
}
|
||||||
|
child := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 101, Name: "Server", Type: "ping", ParentID: 100},
|
||||||
|
SiteState: models.SiteState{Status: models.StatusUp},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child)
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
logs := e.GetLogs()
|
||||||
|
found := false
|
||||||
|
for _, l := range logs {
|
||||||
|
if containsStr(l.Message, "Group 'Infra' recovered") {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("expected recovery log entry for group")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// --- Utilities ---
|
// --- Utilities ---
|
||||||
|
|
||||||
func containsStr(s, substr string) bool {
|
func containsStr(s, substr string) bool {
|
||||||
|
|||||||
Reference in New Issue
Block a user