diff --git a/internal/monitor/alerts.go b/internal/monitor/alerts.go new file mode 100644 index 0000000..e17eaa1 --- /dev/null +++ b/internal/monitor/alerts.go @@ -0,0 +1,272 @@ +package monitor + +import ( + "context" + "fmt" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/alert" + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +type AlertHealth struct { + LastSendAt time.Time + LastSendOK bool + LastError string + SendCount int + FailCount int +} + +// InitAlertHealth restores persisted alert send health so the dashboard shows real +// "last sent" / health state on startup instead of resetting every channel to "never". +func (e *Engine) InitAlertHealth() { + records, err := e.db.LoadAlertHealth(context.Background()) + if err != nil { + return + } + e.alertHealthMu.Lock() + defer e.alertHealthMu.Unlock() + for id, r := range records { + e.alertHealth[id] = AlertHealth{ + LastSendAt: r.LastSendAt, + LastSendOK: r.LastSendOK, + LastError: r.LastError, + SendCount: r.SendCount, + FailCount: r.FailCount, + } + } +} + +// handleStatusChange folds a check result into the live state. snap is the +// stale snapshot the check ran against; the actual mutation is applied onto the +// CURRENT live entry via applyState, so a concurrent pause / config edit / +// heartbeat is never reverted by this write. Logs and alerts are emitted after +// the lock is released, off the critical section. +func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) { + if !e.IsActive() { + return + } + + inMaint := e.isInMaintenance(snap.ID) + status := models.Status(rawStatus) + + var ( + prev, next models.Status + name, typ string + alertID int + failCount, maxRetries int + confirmedDown bool + failedCheck bool + downSince time.Time + sslWarnFire bool + sslDays int + skipped bool + changed bool + ) + + _, exists := e.applyState(snap.ID, func(s *models.Site) { + // A non-UP result computed from a stale snapshot must not override a + // heartbeat (or newer check) that landed while we were evaluating. + if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) { + skipped = true + return + } + + prev = s.Status + name = s.Name + typ = s.Type + alertID = s.AlertID + maxRetries = s.MaxRetries + downSince = s.StatusChangedAt + + // Fresh check results (measured by the run against snap). + s.StatusCode = code + s.Latency = snap.Latency + s.LastCheck = snap.LastCheck + s.HasSSL = snap.HasSSL + s.CertExpiry = snap.CertExpiry + s.LastError = errorReason + if status == models.StatusUp { + s.LastSuccessAt = time.Now() + s.LastError = "" + } + + // Status + failure-count transition, based on the CURRENT live status. + if status == models.StatusUp { + s.FailureCount = 0 + s.Status = models.StatusUp + } else { + if s.FailureCount <= s.MaxRetries { + s.FailureCount++ + } + if s.FailureCount > s.MaxRetries { + if s.Status != status { + confirmedDown = true + } + s.Status = status + s.FailureCount = s.MaxRetries + 1 + } else { + failedCheck = true + } + } + failCount = s.FailureCount + + if s.Status != prev && prev != models.StatusPending { + s.StatusChangedAt = time.Now() + } else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending { + s.StatusChangedAt = time.Now() + } + + // SSL expiry warning (fresh HasSSL/CertExpiry + config threshold). + if typ == "http" && s.CheckSSL && s.HasSSL { + days := int(time.Until(s.CertExpiry).Hours() / 24) + if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp { + sslWarnFire = true + sslDays = days + s.SentSSLWarning = true + } else if days > s.ExpiryThreshold { + s.SentSSLWarning = false + } + } + + next = s.Status + changed = next != prev + }) + + if !exists || skipped { + return + } + + e.recordCheck(snap.ID, latency, status == models.StatusUp) + + if confirmedDown { + if errorReason != "" { + e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason)) + } else { + e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name)) + } + } else if failedCheck { + e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries)) + } + + if changed && prev != models.StatusPending { + e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason}) + } + + if sslWarnFire { + if !inMaint { + e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays)) + } else { + e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name)) + } + } + + if prev == models.StatusUp && next == models.StatusLate { + e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name)) + } + + if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending { + if inMaint { + e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name)) + } else { + msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus) + if errorReason != "" { + msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason) + } + if typ == "push" { + msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name) + } + e.triggerAlert(alertID, "🚨 ALERT", msg) + } + } + if prev.IsBroken() && next == models.StatusUp { + downDur := "" + if !downSince.IsZero() { + downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince))) + } + e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur)) + if !inMaint { + e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur)) + } + } + if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() { + e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name)) + } +} + +func (e *Engine) triggerAlert(alertID int, title, message string) { + if alertID <= 0 { + return + } + cfg, err := e.db.GetAlert(context.Background(), alertID) + if err != nil { + e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err)) + return + } + provider := alert.GetProvider(cfg) + if provider != nil { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout) + defer cancel() + if err := provider.Send(ctx, title, message); err != nil { + e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err)) + e.recordAlertResult(alertID, false, err.Error()) + } else { + e.recordAlertResult(alertID, true, "") + } + }() + } +} + +func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) { + e.alertHealthMu.Lock() + defer e.alertHealthMu.Unlock() + h := e.alertHealth[alertID] + h.LastSendAt = time.Now() + h.LastSendOK = ok + h.SendCount++ + if ok { + h.LastError = "" + } else { + h.LastError = errMsg + h.FailCount++ + } + e.alertHealth[alertID] = h + + // Persist so health survives restarts; DB IO off the alert path. + e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{ + AlertID: alertID, + LastSendAt: h.LastSendAt, + LastSendOK: h.LastSendOK, + LastError: h.LastError, + SendCount: h.SendCount, + FailCount: h.FailCount, + }}) +} + +func (e *Engine) GetAlertHealth(alertID int) AlertHealth { + e.alertHealthMu.RLock() + defer e.alertHealthMu.RUnlock() + return e.alertHealth[alertID] +} + +func (e *Engine) TestAlert(alertID int) error { + cfg, err := e.db.GetAlert(context.Background(), alertID) + if err != nil { + return fmt.Errorf("failed to load alert: %w", err) + } + provider := alert.GetProvider(cfg) + if provider == nil { + return fmt.Errorf("no provider for type %q", cfg.Type) + } + ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout) + defer cancel() + err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name)) + if err != nil { + e.recordAlertResult(alertID, false, err.Error()) + return err + } + e.recordAlertResult(alertID, true, "") + e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name)) + return nil +} diff --git a/internal/monitor/alerts_test.go b/internal/monitor/alerts_test.go new file mode 100644 index 0000000..c5d6518 --- /dev/null +++ b/internal/monitor/alerts_test.go @@ -0,0 +1,487 @@ +package monitor + +import ( + "context" + "testing" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +// --- Group 1: State Machine --- + +func TestHandleStatusChange_PendingToUp(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3, AlertID: 1}, + SiteState: models.SiteState{Status: "PENDING"}, + } + injectSite(e, site) + + e.handleStatusChange(site, "UP", 200, 10*time.Millisecond, "") + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP, got %s", s.Status) + } + if s.FailureCount != 0 { + t.Errorf("expected FailureCount 0, got %d", s.FailureCount) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) != 0 { + t.Error("expected no alert for PENDING→UP") + } +} + +func TestHandleStatusChange_UpIncrementFailure(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3}, + SiteState: models.SiteState{Status: "UP", FailureCount: 0}, + } + injectSite(e, site) + + e.handleStatusChange(site, "DOWN", 500, 0, "test error") + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP (under retry threshold), got %s", s.Status) + } + if s.FailureCount != 1 { + t.Errorf("expected FailureCount 1, got %d", s.FailureCount) + } +} + +func TestHandleStatusChange_UpToDown_ExceedsRetries(t *testing.T) { + ms := newMockStore() + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "discord", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2, AlertID: 1}, + SiteState: models.SiteState{Status: "UP", FailureCount: 2}, + } + injectSite(e, site) + + e.handleStatusChange(site, "DOWN", 500, 0, "test error") + + s, _ := getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected DOWN, got %s", s.Status) + } + if s.FailureCount != 3 { + t.Errorf("expected FailureCount 3, got %d", s.FailureCount) + } + waitAsync() + calls := ms.getAlertCallsSnapshot() + if len(calls) == 0 || calls[0] != 1 { + t.Errorf("expected alert call for alertID 1, got %v", calls) + } +} + +func TestHandleStatusChange_UpToDown_ZeroRetries(t *testing.T) { + ms := newMockStore() + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1}, + SiteState: models.SiteState{Status: "UP", FailureCount: 0}, + } + injectSite(e, site) + + e.handleStatusChange(site, "DOWN", 0, 0, "test error") + + s, _ := getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected DOWN, got %s", s.Status) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) == 0 { + t.Error("expected alert on immediate DOWN") + } +} + +func TestHandleStatusChange_DownToUp_Recovery(t *testing.T) { + ms := newMockStore() + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1}, + SiteState: models.SiteState{Status: "DOWN", FailureCount: 4}, + } + injectSite(e, site) + + e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "") + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP, got %s", s.Status) + } + if s.FailureCount != 0 { + t.Errorf("expected FailureCount 0, got %d", s.FailureCount) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) == 0 { + t.Error("expected recovery alert") + } +} + +func TestHandleStatusChange_DownStaysDown(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2}, + SiteState: models.SiteState{Status: "DOWN", FailureCount: 3}, + } + injectSite(e, site) + + e.handleStatusChange(site, "DOWN", 0, 0, "test error") + + s, _ := getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected DOWN, got %s", s.Status) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) != 0 { + t.Error("expected no re-alert for already DOWN") + } +} + +func TestHandleStatusChange_SSLExpired(t *testing.T) { + ms := newMockStore() + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + e.handleStatusChange(site, "SSL EXP", 0, 0, "SSL certificate expired") + + s, _ := getSite(e, 1) + if s.Status != "SSL EXP" { + t.Errorf("expected SSL EXP, got %s", s.Status) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) == 0 { + t.Error("expected alert on SSL EXP") + } +} + +func TestHandleStatusChange_AlertSuppressedMaintenance(t *testing.T) { + ms := newMockStore() + ms.maintenance[1] = true + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + e.refreshMaintenanceCache(context.Background()) + + e.handleStatusChange(site, "DOWN", 0, 0, "test error") + + s, _ := getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected DOWN, got %s", s.Status) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) != 0 { + t.Error("expected no alert during maintenance") + } + logs := e.GetLogs() + found := false + for _, l := range logs { + if containsStr(l.Message, "suppressed") { + found = true + break + } + } + if !found { + t.Error("expected log mentioning suppressed") + } +} + +func TestHandleStatusChange_RecoverySuppressedMaintenance(t *testing.T) { + ms := newMockStore() + ms.maintenance[1] = true + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1}, + SiteState: models.SiteState{Status: "DOWN"}, + } + injectSite(e, site) + e.refreshMaintenanceCache(context.Background()) + + e.handleStatusChange(site, "UP", 200, 0, "") + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP, got %s", s.Status) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) != 0 { + t.Error("expected no alert during maintenance recovery") + } +} + +func TestHandleStatusChange_SSLWarning(t *testing.T) { + ms := newMockStore() + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1}, + SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)}, + } + injectSite(e, site) + + e.handleStatusChange(site, "UP", 200, 0, "") + + s, _ := getSite(e, 1) + if !s.SentSSLWarning { + t.Error("expected SentSSLWarning=true") + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) == 0 { + t.Error("expected SSL warning alert") + } +} + +func TestHandleStatusChange_SSLWarningNotRepeated(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1}, + SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(15 * 24 * time.Hour)}, + } + injectSite(e, site) + + e.handleStatusChange(site, "UP", 200, 0, "") + + waitAsync() + if len(ms.getAlertCallsSnapshot()) != 0 { + t.Error("expected no repeat SSL warning") + } +} + +func TestHandleStatusChange_SSLWarningReset(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30}, + SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(60 * 24 * time.Hour)}, + } + injectSite(e, site) + + e.handleStatusChange(site, "UP", 200, 0, "") + + s, _ := getSite(e, 1) + if s.SentSSLWarning { + t.Error("expected SentSSLWarning reset to false") + } +} + +func TestHandleStatusChange_SSLWarningSuppressedMaint(t *testing.T) { + ms := newMockStore() + ms.maintenance[1] = true + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1}, + SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)}, + } + injectSite(e, site) + e.refreshMaintenanceCache(context.Background()) + + e.handleStatusChange(site, "UP", 200, 0, "") + + s, _ := getSite(e, 1) + if !s.SentSSLWarning { + t.Error("expected SentSSLWarning=true even in maintenance") + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) != 0 { + t.Error("expected no alert during maintenance") + } +} + +func TestHandleStatusChange_InactiveEngine(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + e.SetActive(false) + + e.handleStatusChange(site, "DOWN", 0, 0, "test error") + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Error("expected no state change when inactive") + } +} + +// --- Group 10: liveState merge (lost-update race) --- + +// A pause that lands while a check is in flight must survive the check's +// write-back. The old code snapshotted the site, ran the check, then wrote the +// whole stale struct back — reverting the pause. +func TestHandleStatusChange_PauseDuringCheckSurvives(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + // `site` is the stale snapshot the check ran against (Paused=false). + // Meanwhile the user pauses the monitor. + e.ToggleSitePause(1) + + // Check completes and folds its result in using the stale snapshot. + e.handleStatusChange(site, "DOWN", 500, 0, "boom") + + s, _ := getSite(e, 1) + if !s.Paused { + t.Error("pause was reverted by a stale check write-back") + } + if s.Status != "DOWN" { + t.Errorf("expected check result still applied (DOWN), got %s", s.Status) + } +} + +// A config edit that lands while a check is in flight must survive; the check +// must not resurrect the old config from its snapshot. +func TestHandleStatusChange_ConfigEditDuringCheckSurvives(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com", Type: "http", MaxRetries: 0, Interval: 30}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + // Config changes mid-check. + e.UpdateSiteConfig(models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Type: "http", Interval: 60}) + + // Stale check (ran against http://old.com) folds its result in. + e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "") + + s, _ := getSite(e, 1) + if s.URL != "http://new.com" { + t.Errorf("config edit reverted: URL=%s", s.URL) + } + if s.Interval != 60 { + t.Errorf("config edit reverted: Interval=%d", s.Interval) + } +} + +// The classic push false-DOWN: a heartbeat marks the monitor UP while a +// staleness evaluation (computed from the older LastCheck) is mid-flight. +// The stale DOWN must not overwrite the fresh heartbeat. +func TestHandleStatusChange_HeartbeatNotOverwrittenByStaleDown(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + // Snapshot the engine would have taken before evaluating staleness: + // LastCheck is old, so checkPush decided "DOWN". + snap := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Token: "tok", Interval: 10}, + SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)}, + } + injectSite(e, snap) + + // A heartbeat lands first, advancing LastCheck and confirming UP. + if !e.RecordHeartbeat("tok") { + t.Fatal("heartbeat rejected") + } + + // Now the in-flight stale evaluation tries to write DOWN. + e.handleStatusChange(snap, "DOWN", 0, 0, "heartbeat missed") + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("stale DOWN overwrote a fresh heartbeat: status=%s", s.Status) + } +} + +// A check result for a site removed mid-check must be dropped, not recreate it. +func TestHandleStatusChange_RemovedSiteDropped(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + e.RemoveSite(1) + e.handleStatusChange(site, "DOWN", 500, 0, "boom") + + if _, ok := getSite(e, 1); ok { + t.Error("removed site was recreated by a late check write-back") + } +} + +// --- Group 12: Phase 3 engine correctness --- + +// PENDING→DOWN must honor MaxRetries instead of alerting on first failure. +func TestHandleStatusChange_PendingRetriesBeforeDown(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "new-monitor", MaxRetries: 2}, + SiteState: models.SiteState{Status: "PENDING"}, + } + injectSite(e, site) + + e.handleStatusChange(site, "DOWN", 0, 0, "timeout") + s, _ := getSite(e, 1) + if s.Status != "PENDING" { + t.Errorf("expected PENDING during retry, got %s", s.Status) + } + if s.FailureCount != 1 { + t.Errorf("expected FailureCount 1, got %d", s.FailureCount) + } + + e.handleStatusChange(s, "DOWN", 0, 0, "timeout") + s, _ = getSite(e, 1) + if s.Status != "PENDING" { + t.Errorf("expected PENDING during retry 2, got %s", s.Status) + } + + e.handleStatusChange(s, "DOWN", 0, 0, "timeout") + s, _ = getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected DOWN after retries exhausted, got %s", s.Status) + } +} + +// LATE→DOWN must also honor MaxRetries. +func TestHandleStatusChange_LateRetriesBeforeDown(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push-mon", MaxRetries: 1}, + SiteState: models.SiteState{Status: "LATE"}, + } + injectSite(e, site) + + e.handleStatusChange(site, "DOWN", 0, 0, "missed heartbeat") + s, _ := getSite(e, 1) + if s.Status != "LATE" { + t.Errorf("expected LATE during retry, got %s", s.Status) + } + + e.handleStatusChange(s, "DOWN", 0, 0, "missed heartbeat") + s, _ = getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected DOWN after retries exhausted, got %s", s.Status) + } +} diff --git a/internal/monitor/checks.go b/internal/monitor/checks.go new file mode 100644 index 0000000..daf7341 --- /dev/null +++ b/internal/monitor/checks.go @@ -0,0 +1,315 @@ +package monitor + +import ( + "context" + "fmt" + "math/rand/v2" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func (e *Engine) RecordHeartbeat(token string) bool { + if !e.IsActive() { + return false + } + + e.mu.RLock() + targetID, ok := e.tokenIndex[token] + e.mu.RUnlock() + if !ok { + return false + } + + var ( + prevStatus models.Status + name string + alertID int + downSince time.Time + ) + _, exists := e.applyState(targetID, func(s *models.Site) { + prevStatus = s.Status + name = s.Name + alertID = s.AlertID + downSince = s.StatusChangedAt // captured before mutation = when it went down + + s.LastCheck = time.Now() + s.Status = models.StatusUp + s.FailureCount = 0 + s.Latency = 0 + s.LastError = "" + s.LastSuccessAt = time.Now() + if prevStatus != models.StatusUp { + s.StatusChangedAt = time.Now() + } + }) + if !exists { + return false + } + + switch prevStatus { + case models.StatusPending: + e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name)) + case models.StatusLate: + e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name)) + case models.StatusStale: + e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name)) + case models.StatusDown: + downDur := "" + if !downSince.IsZero() { + downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince))) + } + e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur)) + go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur)) + } + + e.recordCheck(targetID, 0, true) + + if prevStatus != models.StatusUp && prevStatus != models.StatusPending { + e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)}) + } + + return true +} + +func (e *Engine) getRecheckChan(id int) chan struct{} { + e.recheckMu.Lock() + defer e.recheckMu.Unlock() + ch, ok := e.recheck[id] + if !ok { + ch = make(chan struct{}, 1) + e.recheck[id] = ch + } + return ch +} + +func (e *Engine) signalRecheck(id int) { + ch := e.getRecheckChan(id) + select { + case ch <- struct{}{}: + default: + } +} + +func (e *Engine) monitorRoutine(ctx context.Context, id int) { + recheckCh := e.getRecheckChan(id) + + // Stagger initial check to avoid thundering herd on startup + stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter + select { + case <-time.After(stagger): + case <-ctx.Done(): + return + } + + e.checkByID(ctx, id) + for { + select { + case <-ctx.Done(): + return + default: + } + + if !e.IsActive() { + select { + case <-time.After(pollInterval): + case <-ctx.Done(): + return + case <-recheckCh: + } + continue + } + + e.mu.RLock() + site, exists := e.liveState[id] + e.mu.RUnlock() + if !exists { + return + } + + if site.Paused { + select { + case <-time.After(pollInterval): + case <-ctx.Done(): + return + case <-recheckCh: + } + continue + } + + interval := site.Interval + if interval < minCheckInterval { + interval = minCheckInterval + } + jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter + select { + case <-time.After(time.Duration(interval)*time.Second + jitter): + case <-ctx.Done(): + return + case <-recheckCh: + } + e.checkByID(ctx, id) + } +} + +func (e *Engine) checkByID(ctx context.Context, id int) { + if !e.IsActive() { + return + } + + e.mu.RLock() + site, exists := e.liveState[id] + e.mu.RUnlock() + if !exists || site.Paused { + return + } + + switch site.Type { + case "push": + e.checkPush(ctx, site) + case "group": + e.checkGroup(ctx, site) + default: + result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets) + updatedSite := site + updatedSite.HasSSL = result.HasSSL + updatedSite.CertExpiry = result.CertExpiry + updatedSite.Latency = time.Duration(result.LatencyNs) + updatedSite.LastCheck = time.Now() + e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason) + } +} + +func (e *Engine) checkPush(_ context.Context, site models.Site) { + if site.Status == models.StatusPending { + return + } + + interval := time.Duration(site.Interval) * time.Second + grace := interval / 2 + if grace < minPushGrace { + grace = minPushGrace + } + + overdue := site.LastCheck.Add(interval) + staleMark := overdue.Add(grace / 2) + graceEnd := overdue.Add(grace) + now := time.Now() + + if now.After(graceEnd) { + if site.Status != models.StatusDown { + e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed") + } + } else if now.After(staleMark) { + if site.Status != models.StatusStale { + e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale") + } + } else if now.After(overdue) { + if site.Status != models.StatusLate { + e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue") + } + } +} + +func (e *Engine) checkGroup(_ context.Context, site models.Site) { + e.mu.RLock() + status := models.StatusUp + hasChildren := false + for _, child := range e.liveState { + if child.ParentID != site.ID || child.Type == "group" { + continue + } + hasChildren = true + if child.Paused || e.isInMaintenance(child.ID) { + continue + } + if child.Status == models.StatusDown || child.Status == models.StatusSSLExp { + status = models.StatusDown + } else if child.Status == models.StatusStale && status != models.StatusDown { + status = models.StatusStale + } else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale { + status = models.StatusLate + } else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate { + status = models.StatusPending + } + } + e.mu.RUnlock() + + if !hasChildren { + status = models.StatusPending + } + + e.applyState(site.ID, func(s *models.Site) { + s.Status = status + }) + e.recordCheck(site.ID, 0, !status.IsBroken()) +} + +func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) { + e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp}) +} + +// SetAggStrategy must be called before Start: the field is read by the probe +// aggregation path without synchronization. +func (e *Engine) SetAggStrategy(strategy AggregationStrategy) { + e.aggStrategy = strategy +} + +func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) { + e.mu.RLock() + site, exists := e.liveState[siteID] + e.mu.RUnlock() + if !exists { + return + } + + staleAfter := time.Duration(site.Interval) * time.Second * 3 + if staleAfter < time.Minute { + staleAfter = time.Minute + } + + now := time.Now() + e.probeResultsMu.Lock() + if e.probeResults[siteID] == nil { + e.probeResults[siteID] = make(map[string]NodeResult) + } + e.probeResults[siteID][nodeID] = NodeResult{ + NodeID: nodeID, + IsUp: isUp, + LatencyNs: latencyNs, + CheckedAt: now, + ErrorReason: errorReason, + } + results := make([]NodeResult, 0, len(e.probeResults[siteID])) + for id, r := range e.probeResults[siteID] { + if now.Sub(r.CheckedAt) > staleAfter { + delete(e.probeResults[siteID], id) + continue + } + results = append(results, r) + } + e.probeResultsMu.Unlock() + + aggUp, avgLatency := AggregateStatus(results, e.aggStrategy) + + probeStatus := models.StatusUp + if !aggUp { + probeStatus = models.StatusDown + } + + updatedSite := site + updatedSite.Latency = time.Duration(avgLatency) + updatedSite.LastCheck = time.Now() + e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason) +} + +func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult { + e.probeResultsMu.RLock() + defer e.probeResultsMu.RUnlock() + src := e.probeResults[siteID] + cp := make(map[string]NodeResult, len(src)) + for k, v := range src { + cp[k] = v + } + return cp +} diff --git a/internal/monitor/checks_test.go b/internal/monitor/checks_test.go new file mode 100644 index 0000000..4b0b7a1 --- /dev/null +++ b/internal/monitor/checks_test.go @@ -0,0 +1,389 @@ +package monitor + +import ( + "context" + "testing" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +// --- Group 2: Heartbeat --- + +func TestRecordHeartbeat_ValidToken(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123"}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + if !e.RecordHeartbeat("abc123") { + t.Error("expected true for valid token") + } + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP, got %s", s.Status) + } + if time.Since(s.LastCheck) > time.Second { + t.Error("expected LastCheck to be recent") + } +} + +func TestRecordHeartbeat_RecoveryFromDown(t *testing.T) { + ms := newMockStore() + ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123", AlertID: 1}, + SiteState: models.SiteState{Status: "DOWN", FailureCount: 3}, + } + injectSite(e, site) + + if !e.RecordHeartbeat("abc123") { + t.Error("expected true") + } + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP, got %s", s.Status) + } + if s.FailureCount != 0 { + t.Errorf("expected FailureCount 0, got %d", s.FailureCount) + } + waitAsync() + if len(ms.getAlertCallsSnapshot()) == 0 { + t.Error("expected recovery alert") + } +} + +func TestRecordHeartbeat_UnknownToken(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + + if e.RecordHeartbeat("unknown") { + t.Error("expected false for unknown token") + } +} + +func TestRecordHeartbeat_InactiveEngine(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Type: "push", Token: "abc123"}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + e.SetActive(false) + + if e.RecordHeartbeat("abc123") { + t.Error("expected false when inactive") + } +} + +// --- Group 3: Push Deadline --- + +func TestCheckPush_DeadlineMissed(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 10, MaxRetries: 0}, + SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)}, + } + injectSite(e, site) + + e.checkPush(context.Background(), site) + + s, _ := getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected DOWN after missed deadline, got %s", s.Status) + } +} + +func TestCheckPush_OverdueBecomesLate(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300}, + SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-310 * time.Second)}, + } + injectSite(e, site) + + e.checkPush(context.Background(), site) + + s, _ := getSite(e, 1) + if s.Status != "LATE" { + t.Errorf("expected LATE when overdue but within grace, got %s", s.Status) + } +} + +func TestCheckPush_OverdueBecomesStale(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + // interval=300, grace=150 (300/2), staleMark=overdue+75 + // at 380s: past staleMark(375) but before graceEnd(450) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300}, + SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-380 * time.Second)}, + } + injectSite(e, site) + + e.checkPush(context.Background(), site) + + s, _ := getSite(e, 1) + if s.Status != "STALE" { + t.Errorf("expected STALE when past midpoint of grace, got %s", s.Status) + } +} + +func TestCheckPush_WithinDeadline(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60}, + SiteState: models.SiteState{Status: "UP", LastCheck: time.Now()}, + } + injectSite(e, site) + + e.checkPush(context.Background(), site) + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP, got %s", s.Status) + } +} + +func TestCheckPush_PendingStaysPending(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60}, + SiteState: models.SiteState{Status: "PENDING"}, + } + injectSite(e, site) + + e.checkPush(context.Background(), site) + + s, _ := getSite(e, 1) + if s.Status != "PENDING" { + t.Errorf("expected PENDING to stay until first heartbeat, got %s", s.Status) + } +} + +// --- Group 4: Group Checks --- + +func TestCheckGroup_AllChildrenUp(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + group := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, + SiteState: models.SiteState{Status: "PENDING"}, + } + child1 := models.Site{ + SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, + SiteState: models.SiteState{Status: "UP"}, + } + child2 := models.Site{ + SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, group) + injectSite(e, child1) + injectSite(e, child2) + + e.checkGroup(context.Background(), group) + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected group UP, got %s", s.Status) + } +} + +func TestCheckGroup_OneChildDown(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + group := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, + SiteState: models.SiteState{Status: "UP"}, + } + child1 := models.Site{ + SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, + SiteState: models.SiteState{Status: "UP"}, + } + child2 := models.Site{ + SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1}, + SiteState: models.SiteState{Status: "DOWN"}, + } + injectSite(e, group) + injectSite(e, child1) + injectSite(e, child2) + + e.checkGroup(context.Background(), group) + + s, _ := getSite(e, 1) + if s.Status != "DOWN" { + t.Errorf("expected group DOWN, got %s", s.Status) + } +} + +func TestCheckGroup_PausedChildIgnored(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + group := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, + } + child1 := models.Site{ + SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, + SiteState: models.SiteState{Status: "UP"}, + } + child2 := models.Site{ + SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true}, + SiteState: models.SiteState{Status: "DOWN"}, + } + injectSite(e, group) + injectSite(e, child1) + injectSite(e, child2) + + e.checkGroup(context.Background(), group) + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP (paused child ignored), got %s", s.Status) + } +} + +func TestCheckGroup_MaintenanceChildIgnored(t *testing.T) { + ms := newMockStore() + ms.maintenance[3] = true + e := newTestEngine(ms) + group := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, + } + child1 := models.Site{ + SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, + SiteState: models.SiteState{Status: "UP"}, + } + child2 := models.Site{ + SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1}, + SiteState: models.SiteState{Status: "DOWN"}, + } + injectSite(e, group) + injectSite(e, child1) + injectSite(e, child2) + e.refreshMaintenanceCache(context.Background()) + + e.checkGroup(context.Background(), group) + + s, _ := getSite(e, 1) + if s.Status != "UP" { + t.Errorf("expected UP (maint child ignored), got %s", s.Status) + } +} + +func TestCheckGroup_NoChildren(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + group := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, group) + + e.checkGroup(context.Background(), group) + + s, _ := getSite(e, 1) + if s.Status != "PENDING" { + t.Errorf("expected PENDING for no children, got %s", s.Status) + } +} + +// Groups must not auto-pause when all children are paused — that creates a +// one-way trap because monitorRoutine skips paused sites. +func TestCheckGroup_AllPausedNoAutoFreeze(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + group := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, + SiteState: models.SiteState{Status: "UP"}, + } + child1 := models.Site{ + SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1, Paused: true}, + SiteState: models.SiteState{Status: "UP"}, + } + child2 := models.Site{ + SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, group) + injectSite(e, child1) + injectSite(e, child2) + + e.checkGroup(context.Background(), group) + + s, _ := getSite(e, 1) + if s.Paused { + t.Error("group must not auto-pause when all children are paused") + } +} + +// Dead probe results must be expired so they don't poison aggregation. +func TestIngestProbeResult_ExpiresStaleProbes(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", Interval: 30}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + e.probeResultsMu.Lock() + e.probeResults[1] = map[string]NodeResult{ + "dead-probe": { + NodeID: "dead-probe", + IsUp: false, + CheckedAt: time.Now().Add(-10 * time.Minute), + }, + } + e.probeResultsMu.Unlock() + + e.IngestProbeResult("live-probe", 1, 5000, true, "") + + e.probeResultsMu.RLock() + _, deadExists := e.probeResults[1]["dead-probe"] + _, liveExists := e.probeResults[1]["live-probe"] + e.probeResultsMu.RUnlock() + + if deadExists { + t.Error("stale probe result should have been expired") + } + if !liveExists { + t.Error("live probe result should still exist") + } +} + +// RemoveSite must clean up probeResults. +func TestRemoveSite_CleansProbeResults(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + e.probeResultsMu.Lock() + e.probeResults[1] = map[string]NodeResult{ + "node-a": {NodeID: "node-a", IsUp: true, CheckedAt: time.Now()}, + } + e.probeResultsMu.Unlock() + + e.RemoveSite(1) + + e.probeResultsMu.RLock() + defer e.probeResultsMu.RUnlock() + if _, exists := e.probeResults[1]; exists { + t.Error("probe results should be cleaned up after RemoveSite") + } +} diff --git a/internal/monitor/maintenance.go b/internal/monitor/maintenance.go new file mode 100644 index 0000000..ae479f4 --- /dev/null +++ b/internal/monitor/maintenance.go @@ -0,0 +1,70 @@ +package monitor + +import ( + "context" + "fmt" + "time" +) + +func (e *Engine) maintenancePruner(ctx context.Context) { + ticker := time.NewTicker(maintPruneInterval) + defer ticker.Stop() + + e.pruneMaintenanceWindows(ctx) + + for { + select { + case <-ticker.C: + e.pruneMaintenanceWindows(ctx) + case <-ctx.Done(): + return + } + } +} + +func (e *Engine) pruneMaintenanceWindows(ctx context.Context) { + pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention) + if err != nil { + e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err)) + return + } + if pruned > 0 { + e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned)) + } +} + +func (e *Engine) isInMaintenance(monitorID int) bool { + e.maintCacheMu.RLock() + defer e.maintCacheMu.RUnlock() + return e.maintCache[monitorID] +} + +func (e *Engine) refreshMaintenanceCache(ctx context.Context) { + windows, err := e.db.GetActiveMaintenanceWindows(ctx) + if err != nil { + return + } + + directMaint := make(map[int]bool) + var globalMaint bool + for _, w := range windows { + if w.MonitorID == 0 { + globalMaint = true + } else { + directMaint[w.MonitorID] = true + } + } + + resolved := make(map[int]bool) + e.mu.RLock() + for id, site := range e.liveState { + if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) { + resolved[id] = true + } + } + e.mu.RUnlock() + + e.maintCacheMu.Lock() + e.maintCache = resolved + e.maintCacheMu.Unlock() +} diff --git a/internal/monitor/maintenance_test.go b/internal/monitor/maintenance_test.go new file mode 100644 index 0000000..bcf533e --- /dev/null +++ b/internal/monitor/maintenance_test.go @@ -0,0 +1,53 @@ +package monitor + +import ( + "context" + "testing" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +// Maintenance cache resolves parent relationships correctly. +func TestIsInMaintenance_UsesCache(t *testing.T) { + ms := newMockStore() + ms.maintenance[10] = true // direct maintenance on group + e := newTestEngine(ms) + group := models.Site{ + SiteConfig: models.SiteConfig{ID: 10, Name: "group", Type: "group"}, + SiteState: models.SiteState{Status: "UP"}, + } + child := models.Site{ + SiteConfig: models.SiteConfig{ID: 20, Name: "child", Type: "http", ParentID: 10}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, group) + injectSite(e, child) + e.refreshMaintenanceCache(context.Background()) + + if !e.isInMaintenance(10) { + t.Error("group should be in maintenance (direct)") + } + if !e.isInMaintenance(20) { + t.Error("child should be in maintenance (parent)") + } + if e.isInMaintenance(99) { + t.Error("unknown monitor should not be in maintenance") + } +} + +// Global maintenance (monitor_id=0) applies to all monitors. +func TestIsInMaintenance_GlobalMaintenance(t *testing.T) { + ms := newMockStore() + ms.maintenance[0] = true + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + e.refreshMaintenanceCache(context.Background()) + + if !e.isInMaintenance(1) { + t.Error("all monitors should be in maintenance during global window") + } +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 04c4caf..8b43cc3 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -4,14 +4,12 @@ import ( "context" "crypto/tls" "fmt" - "math/rand/v2" "net/http" "regexp" "strings" "sync" "time" - "gitea.lerkolabs.com/lerkolabs/uptop/internal/alert" "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" "gitea.lerkolabs.com/lerkolabs/uptop/internal/store" ) @@ -28,14 +26,6 @@ const ( dbPruneInterval = 10 * time.Minute ) -type AlertHealth struct { - LastSendAt time.Time - LastSendOK bool - LastError string - SendCount int - FailCount int -} - type Engine struct { mu sync.RWMutex liveState map[int]models.Site @@ -261,26 +251,6 @@ func (e *Engine) InitLogs() { e.logStore = entries } -// InitAlertHealth restores persisted alert send health so the dashboard shows real -// "last sent" / health state on startup instead of resetting every channel to "never". -func (e *Engine) InitAlertHealth() { - records, err := e.db.LoadAlertHealth(context.Background()) - if err != nil { - return - } - e.alertHealthMu.Lock() - defer e.alertHealthMu.Unlock() - for id, r := range records { - e.alertHealth[id] = AlertHealth{ - LastSendAt: r.LastSendAt, - LastSendOK: r.LastSendOK, - LastError: r.LastError, - SendCount: r.SendCount, - FailCount: r.FailCount, - } - } -} - func (e *Engine) GetLogs() []models.LogEntry { e.logMu.RLock() defer e.logMu.RUnlock() @@ -289,123 +259,6 @@ func (e *Engine) GetLogs() []models.LogEntry { return logs } -func (e *Engine) SetActive(active bool) { - e.activeMu.Lock() - defer e.activeMu.Unlock() - if e.isActive != active { - e.isActive = active - status := "RESUMED (Active)" - if !active { - status = "PAUSED (Passive)" - } - e.AddLog(fmt.Sprintf("Engine %s", status)) - } -} - -func (e *Engine) IsActive() bool { - e.activeMu.RLock() - defer e.activeMu.RUnlock() - return e.isActive -} - -func (e *Engine) GetAllSites() []models.Site { - e.mu.RLock() - defer e.mu.RUnlock() - sites := make([]models.Site, 0, len(e.liveState)) - for _, s := range e.liveState { - sites = append(sites, s) - } - return sites -} - -func (e *Engine) GetLiveState() map[int]models.Site { - e.mu.RLock() - defer e.mu.RUnlock() - cp := make(map[int]models.Site, len(e.liveState)) - for k, v := range e.liveState { - cp[k] = v - } - return cp -} - -func (e *Engine) RecordHeartbeat(token string) bool { - if !e.IsActive() { - return false - } - - e.mu.RLock() - targetID, ok := e.tokenIndex[token] - e.mu.RUnlock() - if !ok { - return false - } - - var ( - prevStatus models.Status - name string - alertID int - downSince time.Time - ) - _, exists := e.applyState(targetID, func(s *models.Site) { - prevStatus = s.Status - name = s.Name - alertID = s.AlertID - downSince = s.StatusChangedAt // captured before mutation = when it went down - - s.LastCheck = time.Now() - s.Status = models.StatusUp - s.FailureCount = 0 - s.Latency = 0 - s.LastError = "" - s.LastSuccessAt = time.Now() - if prevStatus != models.StatusUp { - s.StatusChangedAt = time.Now() - } - }) - if !exists { - return false - } - - switch prevStatus { - case models.StatusPending: - e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name)) - case models.StatusLate: - e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name)) - case models.StatusStale: - e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name)) - case models.StatusDown: - downDur := "" - if !downSince.IsZero() { - downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince))) - } - e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur)) - go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur)) - } - - e.recordCheck(targetID, 0, true) - - if prevStatus != models.StatusUp && prevStatus != models.StatusPending { - e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)}) - } - - return true -} - -func (e *Engine) addToTokenIndex(site models.Site) { - if site.Type == "push" && site.Token != "" { - e.tokenIndex[site.Token] = site.ID - } -} - -func (e *Engine) removeFromTokenIndex(id int) { - for token, sid := range e.tokenIndex { - if sid == id { - delete(e.tokenIndex, token) - return - } - } -} - func (e *Engine) Start(ctx context.Context) { // e.cancel is invoked by Stop() to drain and halt the writer; gosec can't // trace the cross-method call, and cancelling the parent reaps this child @@ -496,637 +349,3 @@ func (e *Engine) Start(ctx context.Context) { e.maintenancePruner(ctx) }() } - -func (e *Engine) maintenancePruner(ctx context.Context) { - ticker := time.NewTicker(maintPruneInterval) - defer ticker.Stop() - - e.pruneMaintenanceWindows(ctx) - - for { - select { - case <-ticker.C: - e.pruneMaintenanceWindows(ctx) - case <-ctx.Done(): - return - } - } -} - -func (e *Engine) pruneMaintenanceWindows(ctx context.Context) { - pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention) - if err != nil { - e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err)) - return - } - if pruned > 0 { - e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned)) - } -} - -func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) { - e.mu.Lock() - if existing, ok := e.liveState[cfg.ID]; ok { - e.removeFromTokenIndex(cfg.ID) - existing.SiteConfig = cfg - e.liveState[cfg.ID] = existing - e.addToTokenIndex(existing) - } - e.mu.Unlock() - - e.signalRecheck(cfg.ID) -} - -func (e *Engine) getRecheckChan(id int) chan struct{} { - e.recheckMu.Lock() - defer e.recheckMu.Unlock() - ch, ok := e.recheck[id] - if !ok { - ch = make(chan struct{}, 1) - e.recheck[id] = ch - } - return ch -} - -func (e *Engine) signalRecheck(id int) { - ch := e.getRecheckChan(id) - select { - case ch <- struct{}{}: - default: - } -} - -func (e *Engine) RemoveSite(id int) { - e.mu.Lock() - e.removeFromTokenIndex(id) - delete(e.liveState, id) - e.mu.Unlock() - e.removeHistory(id) - - e.probeResultsMu.Lock() - delete(e.probeResults, id) - e.probeResultsMu.Unlock() - - e.recheckMu.Lock() - delete(e.recheck, id) - e.recheckMu.Unlock() -} - -func (e *Engine) ToggleSitePause(id int) bool { - var ( - paused bool - name string - ) - _, ok := e.applyState(id, func(s *models.Site) { - s.Paused = !s.Paused - paused = s.Paused - name = s.Name - }) - if !ok { - return false - } - if paused { - e.AddLog(fmt.Sprintf("Monitor '%s' paused", name)) - } else { - e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name)) - } - return paused -} - -func (e *Engine) monitorRoutine(ctx context.Context, id int) { - recheckCh := e.getRecheckChan(id) - - // Stagger initial check to avoid thundering herd on startup - stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter - select { - case <-time.After(stagger): - case <-ctx.Done(): - return - } - - e.checkByID(ctx, id) - for { - select { - case <-ctx.Done(): - return - default: - } - - if !e.IsActive() { - select { - case <-time.After(pollInterval): - case <-ctx.Done(): - return - case <-recheckCh: - } - continue - } - - e.mu.RLock() - site, exists := e.liveState[id] - e.mu.RUnlock() - if !exists { - return - } - - if site.Paused { - select { - case <-time.After(pollInterval): - case <-ctx.Done(): - return - case <-recheckCh: - } - continue - } - - interval := site.Interval - if interval < minCheckInterval { - interval = minCheckInterval - } - jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter - select { - case <-time.After(time.Duration(interval)*time.Second + jitter): - case <-ctx.Done(): - return - case <-recheckCh: - } - e.checkByID(ctx, id) - } -} - -// applyState atomically reads, mutates, and writes back the live entry for id. -// The mutator runs under the engine write lock and receives a pointer to the -// CURRENT live state, so concurrent config edits, pauses, and heartbeats are -// never clobbered by a stale snapshot. The mutator must only touch runtime / -// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…) -// are owned by UpdateSiteConfig and must not be written here. Returns the -// post-mutation copy and whether the site still exists. -func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) { - e.mu.Lock() - defer e.mu.Unlock() - cur, ok := e.liveState[id] - if !ok { - return models.Site{}, false - } - mutate(&cur) - e.liveState[id] = cur - return cur, true -} - -func (e *Engine) checkByID(ctx context.Context, id int) { - if !e.IsActive() { - return - } - - e.mu.RLock() - site, exists := e.liveState[id] - e.mu.RUnlock() - if !exists || site.Paused { - return - } - - switch site.Type { - case "push": - e.checkPush(ctx, site) - case "group": - e.checkGroup(ctx, site) - default: - result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets) - updatedSite := site - updatedSite.HasSSL = result.HasSSL - updatedSite.CertExpiry = result.CertExpiry - updatedSite.Latency = time.Duration(result.LatencyNs) - updatedSite.LastCheck = time.Now() - e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason) - } -} - -func (e *Engine) checkPush(_ context.Context, site models.Site) { - if site.Status == models.StatusPending { - return - } - - interval := time.Duration(site.Interval) * time.Second - grace := interval / 2 - if grace < minPushGrace { - grace = minPushGrace - } - - overdue := site.LastCheck.Add(interval) - staleMark := overdue.Add(grace / 2) - graceEnd := overdue.Add(grace) - now := time.Now() - - if now.After(graceEnd) { - if site.Status != models.StatusDown { - e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed") - } - } else if now.After(staleMark) { - if site.Status != models.StatusStale { - e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale") - } - } else if now.After(overdue) { - if site.Status != models.StatusLate { - e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue") - } - } -} - -// handleStatusChange folds a check result into the live state. snap is the -// stale snapshot the check ran against; the actual mutation is applied onto the -// CURRENT live entry via applyState, so a concurrent pause / config edit / -// heartbeat is never reverted by this write. Logs and alerts are emitted after -// the lock is released, off the critical section. -func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) { - if !e.IsActive() { - return - } - - inMaint := e.isInMaintenance(snap.ID) - status := models.Status(rawStatus) - - var ( - prev, next models.Status - name, typ string - alertID int - failCount, maxRetries int - confirmedDown bool - failedCheck bool - downSince time.Time - sslWarnFire bool - sslDays int - skipped bool - changed bool - ) - - _, exists := e.applyState(snap.ID, func(s *models.Site) { - // A non-UP result computed from a stale snapshot must not override a - // heartbeat (or newer check) that landed while we were evaluating. - if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) { - skipped = true - return - } - - prev = s.Status - name = s.Name - typ = s.Type - alertID = s.AlertID - maxRetries = s.MaxRetries - downSince = s.StatusChangedAt - - // Fresh check results (measured by the run against snap). - s.StatusCode = code - s.Latency = snap.Latency - s.LastCheck = snap.LastCheck - s.HasSSL = snap.HasSSL - s.CertExpiry = snap.CertExpiry - s.LastError = errorReason - if status == models.StatusUp { - s.LastSuccessAt = time.Now() - s.LastError = "" - } - - // Status + failure-count transition, based on the CURRENT live status. - if status == models.StatusUp { - s.FailureCount = 0 - s.Status = models.StatusUp - } else { - if s.FailureCount <= s.MaxRetries { - s.FailureCount++ - } - if s.FailureCount > s.MaxRetries { - if s.Status != status { - confirmedDown = true - } - s.Status = status - s.FailureCount = s.MaxRetries + 1 - } else { - failedCheck = true - } - } - failCount = s.FailureCount - - if s.Status != prev && prev != models.StatusPending { - s.StatusChangedAt = time.Now() - } else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending { - s.StatusChangedAt = time.Now() - } - - // SSL expiry warning (fresh HasSSL/CertExpiry + config threshold). - if typ == "http" && s.CheckSSL && s.HasSSL { - days := int(time.Until(s.CertExpiry).Hours() / 24) - if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp { - sslWarnFire = true - sslDays = days - s.SentSSLWarning = true - } else if days > s.ExpiryThreshold { - s.SentSSLWarning = false - } - } - - next = s.Status - changed = next != prev - }) - - if !exists || skipped { - return - } - - e.recordCheck(snap.ID, latency, status == models.StatusUp) - - if confirmedDown { - if errorReason != "" { - e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason)) - } else { - e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name)) - } - } else if failedCheck { - e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries)) - } - - if changed && prev != models.StatusPending { - e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason}) - } - - if sslWarnFire { - if !inMaint { - e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays)) - } else { - e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name)) - } - } - - if prev == models.StatusUp && next == models.StatusLate { - e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name)) - } - - if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending { - if inMaint { - e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name)) - } else { - msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus) - if errorReason != "" { - msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason) - } - if typ == "push" { - msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name) - } - e.triggerAlert(alertID, "🚨 ALERT", msg) - } - } - if prev.IsBroken() && next == models.StatusUp { - downDur := "" - if !downSince.IsZero() { - downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince))) - } - e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur)) - if !inMaint { - e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur)) - } - } - if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() { - e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name)) - } -} - -func (e *Engine) triggerAlert(alertID int, title, message string) { - if alertID <= 0 { - return - } - cfg, err := e.db.GetAlert(context.Background(), alertID) - if err != nil { - e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err)) - return - } - provider := alert.GetProvider(cfg) - if provider != nil { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout) - defer cancel() - if err := provider.Send(ctx, title, message); err != nil { - e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err)) - e.recordAlertResult(alertID, false, err.Error()) - } else { - e.recordAlertResult(alertID, true, "") - } - }() - } -} - -func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) { - e.alertHealthMu.Lock() - defer e.alertHealthMu.Unlock() - h := e.alertHealth[alertID] - h.LastSendAt = time.Now() - h.LastSendOK = ok - h.SendCount++ - if ok { - h.LastError = "" - } else { - h.LastError = errMsg - h.FailCount++ - } - e.alertHealth[alertID] = h - - // Persist so health survives restarts; DB IO off the alert path. - e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{ - AlertID: alertID, - LastSendAt: h.LastSendAt, - LastSendOK: h.LastSendOK, - LastError: h.LastError, - SendCount: h.SendCount, - FailCount: h.FailCount, - }}) -} - -func (e *Engine) GetAlertHealth(alertID int) AlertHealth { - e.alertHealthMu.RLock() - defer e.alertHealthMu.RUnlock() - return e.alertHealth[alertID] -} - -func (e *Engine) TestAlert(alertID int) error { - cfg, err := e.db.GetAlert(context.Background(), alertID) - if err != nil { - return fmt.Errorf("failed to load alert: %w", err) - } - provider := alert.GetProvider(cfg) - if provider == nil { - return fmt.Errorf("no provider for type %q", cfg.Type) - } - ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout) - defer cancel() - err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name)) - if err != nil { - e.recordAlertResult(alertID, false, err.Error()) - return err - } - e.recordAlertResult(alertID, true, "") - e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name)) - return nil -} - -func (e *Engine) isInMaintenance(monitorID int) bool { - e.maintCacheMu.RLock() - defer e.maintCacheMu.RUnlock() - return e.maintCache[monitorID] -} - -func (e *Engine) refreshMaintenanceCache(ctx context.Context) { - windows, err := e.db.GetActiveMaintenanceWindows(ctx) - if err != nil { - return - } - - directMaint := make(map[int]bool) - var globalMaint bool - for _, w := range windows { - if w.MonitorID == 0 { - globalMaint = true - } else { - directMaint[w.MonitorID] = true - } - } - - resolved := make(map[int]bool) - e.mu.RLock() - for id, site := range e.liveState { - if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) { - resolved[id] = true - } - } - e.mu.RUnlock() - - e.maintCacheMu.Lock() - e.maintCache = resolved - e.maintCacheMu.Unlock() -} - -func (e *Engine) GetDisplayStatus(site models.Site) string { - if site.Paused { - return "PAUSED" - } - if e.isInMaintenance(site.ID) { - return "MAINT" - } - return string(site.Status) -} - -func (e *Engine) checkGroup(_ context.Context, site models.Site) { - e.mu.RLock() - status := models.StatusUp - hasChildren := false - for _, child := range e.liveState { - if child.ParentID != site.ID || child.Type == "group" { - continue - } - hasChildren = true - if child.Paused || e.isInMaintenance(child.ID) { - continue - } - if child.Status == models.StatusDown || child.Status == models.StatusSSLExp { - status = models.StatusDown - } else if child.Status == models.StatusStale && status != models.StatusDown { - status = models.StatusStale - } else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale { - status = models.StatusLate - } else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate { - status = models.StatusPending - } - } - e.mu.RUnlock() - - if !hasChildren { - status = models.StatusPending - } - - e.applyState(site.ID, func(s *models.Site) { - s.Status = status - }) - e.recordCheck(site.ID, 0, !status.IsBroken()) -} - -func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) { - e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp}) -} - -// SetAggStrategy must be called before Start: the field is read by the probe -// aggregation path without synchronization. -func (e *Engine) SetAggStrategy(strategy AggregationStrategy) { - e.aggStrategy = strategy -} - -func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) { - e.mu.RLock() - site, exists := e.liveState[siteID] - e.mu.RUnlock() - if !exists { - return - } - - staleAfter := time.Duration(site.Interval) * time.Second * 3 - if staleAfter < time.Minute { - staleAfter = time.Minute - } - - now := time.Now() - e.probeResultsMu.Lock() - if e.probeResults[siteID] == nil { - e.probeResults[siteID] = make(map[string]NodeResult) - } - e.probeResults[siteID][nodeID] = NodeResult{ - NodeID: nodeID, - IsUp: isUp, - LatencyNs: latencyNs, - CheckedAt: now, - ErrorReason: errorReason, - } - results := make([]NodeResult, 0, len(e.probeResults[siteID])) - for id, r := range e.probeResults[siteID] { - if now.Sub(r.CheckedAt) > staleAfter { - delete(e.probeResults[siteID], id) - continue - } - results = append(results, r) - } - e.probeResultsMu.Unlock() - - aggUp, avgLatency := AggregateStatus(results, e.aggStrategy) - - probeStatus := models.StatusUp - if !aggUp { - probeStatus = models.StatusDown - } - - updatedSite := site - updatedSite.Latency = time.Duration(avgLatency) - updatedSite.LastCheck = time.Now() - e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason) -} - -func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult { - e.probeResultsMu.RLock() - defer e.probeResultsMu.RUnlock() - src := e.probeResults[siteID] - cp := make(map[string]NodeResult, len(src)) - for k, v := range src { - cp[k] = v - } - return cp -} - -func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange { - changes, err := e.db.GetStateChanges(context.Background(), siteID, limit) - if err != nil { - return nil - } - return changes -} - -func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange { - changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since) - if err != nil { - return nil - } - return changes -} diff --git a/internal/monitor/monitor_test.go b/internal/monitor/monitor_test.go index 8ebd0c6..8d18af0 100644 --- a/internal/monitor/monitor_test.go +++ b/internal/monitor/monitor_test.go @@ -143,616 +143,6 @@ func (m *mockStore) getAlertCallsSnapshot() []int { return cp } -// --- Group 1: State Machine --- - -func TestHandleStatusChange_PendingToUp(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3, AlertID: 1}, - SiteState: models.SiteState{Status: "PENDING"}, - } - injectSite(e, site) - - e.handleStatusChange(site, "UP", 200, 10*time.Millisecond, "") - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP, got %s", s.Status) - } - if s.FailureCount != 0 { - t.Errorf("expected FailureCount 0, got %d", s.FailureCount) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) != 0 { - t.Error("expected no alert for PENDING→UP") - } -} - -func TestHandleStatusChange_UpIncrementFailure(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3}, - SiteState: models.SiteState{Status: "UP", FailureCount: 0}, - } - injectSite(e, site) - - e.handleStatusChange(site, "DOWN", 500, 0, "test error") - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP (under retry threshold), got %s", s.Status) - } - if s.FailureCount != 1 { - t.Errorf("expected FailureCount 1, got %d", s.FailureCount) - } -} - -func TestHandleStatusChange_UpToDown_ExceedsRetries(t *testing.T) { - ms := newMockStore() - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "discord", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2, AlertID: 1}, - SiteState: models.SiteState{Status: "UP", FailureCount: 2}, - } - injectSite(e, site) - - e.handleStatusChange(site, "DOWN", 500, 0, "test error") - - s, _ := getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected DOWN, got %s", s.Status) - } - if s.FailureCount != 3 { - t.Errorf("expected FailureCount 3, got %d", s.FailureCount) - } - waitAsync() - calls := ms.getAlertCallsSnapshot() - if len(calls) == 0 || calls[0] != 1 { - t.Errorf("expected alert call for alertID 1, got %v", calls) - } -} - -func TestHandleStatusChange_UpToDown_ZeroRetries(t *testing.T) { - ms := newMockStore() - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1}, - SiteState: models.SiteState{Status: "UP", FailureCount: 0}, - } - injectSite(e, site) - - e.handleStatusChange(site, "DOWN", 0, 0, "test error") - - s, _ := getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected DOWN, got %s", s.Status) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) == 0 { - t.Error("expected alert on immediate DOWN") - } -} - -func TestHandleStatusChange_DownToUp_Recovery(t *testing.T) { - ms := newMockStore() - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1}, - SiteState: models.SiteState{Status: "DOWN", FailureCount: 4}, - } - injectSite(e, site) - - e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "") - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP, got %s", s.Status) - } - if s.FailureCount != 0 { - t.Errorf("expected FailureCount 0, got %d", s.FailureCount) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) == 0 { - t.Error("expected recovery alert") - } -} - -func TestHandleStatusChange_DownStaysDown(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2}, - SiteState: models.SiteState{Status: "DOWN", FailureCount: 3}, - } - injectSite(e, site) - - e.handleStatusChange(site, "DOWN", 0, 0, "test error") - - s, _ := getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected DOWN, got %s", s.Status) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) != 0 { - t.Error("expected no re-alert for already DOWN") - } -} - -func TestHandleStatusChange_SSLExpired(t *testing.T) { - ms := newMockStore() - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - e.handleStatusChange(site, "SSL EXP", 0, 0, "SSL certificate expired") - - s, _ := getSite(e, 1) - if s.Status != "SSL EXP" { - t.Errorf("expected SSL EXP, got %s", s.Status) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) == 0 { - t.Error("expected alert on SSL EXP") - } -} - -func TestHandleStatusChange_AlertSuppressedMaintenance(t *testing.T) { - ms := newMockStore() - ms.maintenance[1] = true - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - e.refreshMaintenanceCache(context.Background()) - - e.handleStatusChange(site, "DOWN", 0, 0, "test error") - - s, _ := getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected DOWN, got %s", s.Status) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) != 0 { - t.Error("expected no alert during maintenance") - } - logs := e.GetLogs() - found := false - for _, l := range logs { - if containsStr(l.Message, "suppressed") { - found = true - break - } - } - if !found { - t.Error("expected log mentioning suppressed") - } -} - -func TestHandleStatusChange_RecoverySuppressedMaintenance(t *testing.T) { - ms := newMockStore() - ms.maintenance[1] = true - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1}, - SiteState: models.SiteState{Status: "DOWN"}, - } - injectSite(e, site) - e.refreshMaintenanceCache(context.Background()) - - e.handleStatusChange(site, "UP", 200, 0, "") - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP, got %s", s.Status) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) != 0 { - t.Error("expected no alert during maintenance recovery") - } -} - -func TestHandleStatusChange_SSLWarning(t *testing.T) { - ms := newMockStore() - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1}, - SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)}, - } - injectSite(e, site) - - e.handleStatusChange(site, "UP", 200, 0, "") - - s, _ := getSite(e, 1) - if !s.SentSSLWarning { - t.Error("expected SentSSLWarning=true") - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) == 0 { - t.Error("expected SSL warning alert") - } -} - -func TestHandleStatusChange_SSLWarningNotRepeated(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1}, - SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(15 * 24 * time.Hour)}, - } - injectSite(e, site) - - e.handleStatusChange(site, "UP", 200, 0, "") - - waitAsync() - if len(ms.getAlertCallsSnapshot()) != 0 { - t.Error("expected no repeat SSL warning") - } -} - -func TestHandleStatusChange_SSLWarningReset(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30}, - SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(60 * 24 * time.Hour)}, - } - injectSite(e, site) - - e.handleStatusChange(site, "UP", 200, 0, "") - - s, _ := getSite(e, 1) - if s.SentSSLWarning { - t.Error("expected SentSSLWarning reset to false") - } -} - -func TestHandleStatusChange_SSLWarningSuppressedMaint(t *testing.T) { - ms := newMockStore() - ms.maintenance[1] = true - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1}, - SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)}, - } - injectSite(e, site) - e.refreshMaintenanceCache(context.Background()) - - e.handleStatusChange(site, "UP", 200, 0, "") - - s, _ := getSite(e, 1) - if !s.SentSSLWarning { - t.Error("expected SentSSLWarning=true even in maintenance") - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) != 0 { - t.Error("expected no alert during maintenance") - } -} - -func TestHandleStatusChange_InactiveEngine(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - e.SetActive(false) - - e.handleStatusChange(site, "DOWN", 0, 0, "test error") - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Error("expected no state change when inactive") - } -} - -// --- Group 2: Heartbeat --- - -func TestRecordHeartbeat_ValidToken(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123"}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - if !e.RecordHeartbeat("abc123") { - t.Error("expected true for valid token") - } - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP, got %s", s.Status) - } - if time.Since(s.LastCheck) > time.Second { - t.Error("expected LastCheck to be recent") - } -} - -func TestRecordHeartbeat_RecoveryFromDown(t *testing.T) { - ms := newMockStore() - ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}} - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123", AlertID: 1}, - SiteState: models.SiteState{Status: "DOWN", FailureCount: 3}, - } - injectSite(e, site) - - if !e.RecordHeartbeat("abc123") { - t.Error("expected true") - } - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP, got %s", s.Status) - } - if s.FailureCount != 0 { - t.Errorf("expected FailureCount 0, got %d", s.FailureCount) - } - waitAsync() - if len(ms.getAlertCallsSnapshot()) == 0 { - t.Error("expected recovery alert") - } -} - -func TestRecordHeartbeat_UnknownToken(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - - if e.RecordHeartbeat("unknown") { - t.Error("expected false for unknown token") - } -} - -func TestRecordHeartbeat_InactiveEngine(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Type: "push", Token: "abc123"}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - e.SetActive(false) - - if e.RecordHeartbeat("abc123") { - t.Error("expected false when inactive") - } -} - -// --- Group 3: Push Deadline --- - -func TestCheckPush_DeadlineMissed(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 10, MaxRetries: 0}, - SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)}, - } - injectSite(e, site) - - e.checkPush(context.Background(), site) - - s, _ := getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected DOWN after missed deadline, got %s", s.Status) - } -} - -func TestCheckPush_OverdueBecomesLate(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300}, - SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-310 * time.Second)}, - } - injectSite(e, site) - - e.checkPush(context.Background(), site) - - s, _ := getSite(e, 1) - if s.Status != "LATE" { - t.Errorf("expected LATE when overdue but within grace, got %s", s.Status) - } -} - -func TestCheckPush_OverdueBecomesStale(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - // interval=300, grace=150 (300/2), staleMark=overdue+75 - // at 380s: past staleMark(375) but before graceEnd(450) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300}, - SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-380 * time.Second)}, - } - injectSite(e, site) - - e.checkPush(context.Background(), site) - - s, _ := getSite(e, 1) - if s.Status != "STALE" { - t.Errorf("expected STALE when past midpoint of grace, got %s", s.Status) - } -} - -func TestCheckPush_WithinDeadline(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60}, - SiteState: models.SiteState{Status: "UP", LastCheck: time.Now()}, - } - injectSite(e, site) - - e.checkPush(context.Background(), site) - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP, got %s", s.Status) - } -} - -func TestCheckPush_PendingStaysPending(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60}, - SiteState: models.SiteState{Status: "PENDING"}, - } - injectSite(e, site) - - e.checkPush(context.Background(), site) - - s, _ := getSite(e, 1) - if s.Status != "PENDING" { - t.Errorf("expected PENDING to stay until first heartbeat, got %s", s.Status) - } -} - -// --- Group 4: Group Checks --- - -func TestCheckGroup_AllChildrenUp(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - group := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, - SiteState: models.SiteState{Status: "PENDING"}, - } - child1 := models.Site{ - SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, - SiteState: models.SiteState{Status: "UP"}, - } - child2 := models.Site{ - SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, group) - injectSite(e, child1) - injectSite(e, child2) - - e.checkGroup(context.Background(), group) - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected group UP, got %s", s.Status) - } -} - -func TestCheckGroup_OneChildDown(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - group := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, - SiteState: models.SiteState{Status: "UP"}, - } - child1 := models.Site{ - SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, - SiteState: models.SiteState{Status: "UP"}, - } - child2 := models.Site{ - SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1}, - SiteState: models.SiteState{Status: "DOWN"}, - } - injectSite(e, group) - injectSite(e, child1) - injectSite(e, child2) - - e.checkGroup(context.Background(), group) - - s, _ := getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected group DOWN, got %s", s.Status) - } -} - -func TestCheckGroup_PausedChildIgnored(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - group := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, - } - child1 := models.Site{ - SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, - SiteState: models.SiteState{Status: "UP"}, - } - child2 := models.Site{ - SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true}, - SiteState: models.SiteState{Status: "DOWN"}, - } - injectSite(e, group) - injectSite(e, child1) - injectSite(e, child2) - - e.checkGroup(context.Background(), group) - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP (paused child ignored), got %s", s.Status) - } -} - -func TestCheckGroup_MaintenanceChildIgnored(t *testing.T) { - ms := newMockStore() - ms.maintenance[3] = true - e := newTestEngine(ms) - group := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, - } - child1 := models.Site{ - SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1}, - SiteState: models.SiteState{Status: "UP"}, - } - child2 := models.Site{ - SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1}, - SiteState: models.SiteState{Status: "DOWN"}, - } - injectSite(e, group) - injectSite(e, child1) - injectSite(e, child2) - e.refreshMaintenanceCache(context.Background()) - - e.checkGroup(context.Background(), group) - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("expected UP (maint child ignored), got %s", s.Status) - } -} - -func TestCheckGroup_NoChildren(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - group := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, group) - - e.checkGroup(context.Background(), group) - - s, _ := getSite(e, 1) - if s.Status != "PENDING" { - t.Errorf("expected PENDING for no children, got %s", s.Status) - } -} - // --- Group 5: History --- func TestRecordCheck_Appends(t *testing.T) { @@ -832,133 +222,6 @@ func TestInitHistory_LoadsFromDB(t *testing.T) { } } -// --- Group 6: State Management --- - -func TestUpdateSiteConfig_PreservesRuntime(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com"}, - SiteState: models.SiteState{Status: "DOWN", FailureCount: 3, Latency: 100 * time.Millisecond}, - } - injectSite(e, site) - - updated := models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Interval: 60} - e.UpdateSiteConfig(updated) - - s, _ := getSite(e, 1) - if s.URL != "http://new.com" { - t.Errorf("expected URL updated, got %s", s.URL) - } - if s.Status != "DOWN" { - t.Errorf("expected Status preserved, got %s", s.Status) - } - if s.FailureCount != 3 { - t.Errorf("expected FailureCount preserved, got %d", s.FailureCount) - } - if s.Latency != 100*time.Millisecond { - t.Errorf("expected Latency preserved, got %v", s.Latency) - } -} - -func TestRemoveSite_CleansUp(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "push", Token: "tok1"}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - e.recordCheck(1, 5*time.Millisecond, true) - - e.RemoveSite(1) - - if _, ok := getSite(e, 1); ok { - t.Error("expected site removed from liveState") - } - if e.RecordHeartbeat("tok1") { - t.Error("expected token removed from index") - } - if _, ok := e.GetHistory(1); ok { - t.Error("expected history removed") - } -} - -func TestToggleSitePause(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test"}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - paused := e.ToggleSitePause(1) - if !paused { - t.Error("expected paused=true after first toggle") - } - s, _ := getSite(e, 1) - if !s.Paused { - t.Error("expected Paused=true in state") - } - - paused = e.ToggleSitePause(1) - if paused { - t.Error("expected paused=false after second toggle") - } -} - -func TestToggleSitePause_NonexistentSite(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - if e.ToggleSitePause(999) { - t.Error("expected false for nonexistent site") - } -} - -func TestGetAllSites_ReturnsCopy(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - injectSite(e, models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "s1"}, - SiteState: models.SiteState{Status: "UP"}, - }) - injectSite(e, models.Site{ - SiteConfig: models.SiteConfig{ID: 2, Name: "s2"}, - SiteState: models.SiteState{Status: "DOWN"}, - }) - - sites := e.GetAllSites() - if len(sites) != 2 { - t.Fatalf("expected 2 sites, got %d", len(sites)) - } - sites[0].Name = "mutated" - - fresh := e.GetAllSites() - for _, s := range fresh { - if s.Name == "mutated" { - t.Error("GetAllSites returned reference, not copy") - } - } -} - -func TestGetLiveState_ReturnsCopy(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - injectSite(e, models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "s1"}, - SiteState: models.SiteState{Status: "UP"}, - }) - - state := e.GetLiveState() - state[1] = models.Site{SiteConfig: models.SiteConfig{Name: "mutated"}} - - fresh := e.GetLiveState() - if fresh[1].Name == "mutated" { - t.Error("GetLiveState returned reference, not copy") - } -} - // --- Group 7: Logs --- func TestAddLog_PrependAndCap(t *testing.T) { @@ -1137,108 +400,6 @@ func TestConcurrent_RecordCheckAndGetHistory(t *testing.T) { } } -// --- Group 10: liveState merge (lost-update race) --- - -// A pause that lands while a check is in flight must survive the check's -// write-back. The old code snapshotted the site, ran the check, then wrote the -// whole stale struct back — reverting the pause. -func TestHandleStatusChange_PauseDuringCheckSurvives(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - // `site` is the stale snapshot the check ran against (Paused=false). - // Meanwhile the user pauses the monitor. - e.ToggleSitePause(1) - - // Check completes and folds its result in using the stale snapshot. - e.handleStatusChange(site, "DOWN", 500, 0, "boom") - - s, _ := getSite(e, 1) - if !s.Paused { - t.Error("pause was reverted by a stale check write-back") - } - if s.Status != "DOWN" { - t.Errorf("expected check result still applied (DOWN), got %s", s.Status) - } -} - -// A config edit that lands while a check is in flight must survive; the check -// must not resurrect the old config from its snapshot. -func TestHandleStatusChange_ConfigEditDuringCheckSurvives(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com", Type: "http", MaxRetries: 0, Interval: 30}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - // Config changes mid-check. - e.UpdateSiteConfig(models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Type: "http", Interval: 60}) - - // Stale check (ran against http://old.com) folds its result in. - e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "") - - s, _ := getSite(e, 1) - if s.URL != "http://new.com" { - t.Errorf("config edit reverted: URL=%s", s.URL) - } - if s.Interval != 60 { - t.Errorf("config edit reverted: Interval=%d", s.Interval) - } -} - -// The classic push false-DOWN: a heartbeat marks the monitor UP while a -// staleness evaluation (computed from the older LastCheck) is mid-flight. -// The stale DOWN must not overwrite the fresh heartbeat. -func TestHandleStatusChange_HeartbeatNotOverwrittenByStaleDown(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - // Snapshot the engine would have taken before evaluating staleness: - // LastCheck is old, so checkPush decided "DOWN". - snap := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Token: "tok", Interval: 10}, - SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)}, - } - injectSite(e, snap) - - // A heartbeat lands first, advancing LastCheck and confirming UP. - if !e.RecordHeartbeat("tok") { - t.Fatal("heartbeat rejected") - } - - // Now the in-flight stale evaluation tries to write DOWN. - e.handleStatusChange(snap, "DOWN", 0, 0, "heartbeat missed") - - s, _ := getSite(e, 1) - if s.Status != "UP" { - t.Errorf("stale DOWN overwrote a fresh heartbeat: status=%s", s.Status) - } -} - -// A check result for a site removed mid-check must be dropped, not recreate it. -func TestHandleStatusChange_RemovedSiteDropped(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - e.RemoveSite(1) - e.handleStatusChange(site, "DOWN", 500, 0, "boom") - - if _, ok := getSite(e, 1); ok { - t.Error("removed site was recreated by a late check write-back") - } -} - // --- Group 11: single DB writer --- // Writes enqueued through the engine are persisted by the writer goroutine and @@ -1284,197 +445,6 @@ func TestEngineStop_Idempotent(t *testing.T) { e.Stop() // must not panic or block } -// --- Group 12: Phase 3 engine correctness --- - -// Groups must not auto-pause when all children are paused — that creates a -// one-way trap because monitorRoutine skips paused sites. -func TestCheckGroup_AllPausedNoAutoFreeze(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - group := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"}, - SiteState: models.SiteState{Status: "UP"}, - } - child1 := models.Site{ - SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1, Paused: true}, - SiteState: models.SiteState{Status: "UP"}, - } - child2 := models.Site{ - SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, group) - injectSite(e, child1) - injectSite(e, child2) - - e.checkGroup(context.Background(), group) - - s, _ := getSite(e, 1) - if s.Paused { - t.Error("group must not auto-pause when all children are paused") - } -} - -// PENDING→DOWN must honor MaxRetries instead of alerting on first failure. -func TestHandleStatusChange_PendingRetriesBeforeDown(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "new-monitor", MaxRetries: 2}, - SiteState: models.SiteState{Status: "PENDING"}, - } - injectSite(e, site) - - e.handleStatusChange(site, "DOWN", 0, 0, "timeout") - s, _ := getSite(e, 1) - if s.Status != "PENDING" { - t.Errorf("expected PENDING during retry, got %s", s.Status) - } - if s.FailureCount != 1 { - t.Errorf("expected FailureCount 1, got %d", s.FailureCount) - } - - e.handleStatusChange(s, "DOWN", 0, 0, "timeout") - s, _ = getSite(e, 1) - if s.Status != "PENDING" { - t.Errorf("expected PENDING during retry 2, got %s", s.Status) - } - - e.handleStatusChange(s, "DOWN", 0, 0, "timeout") - s, _ = getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected DOWN after retries exhausted, got %s", s.Status) - } -} - -// LATE→DOWN must also honor MaxRetries. -func TestHandleStatusChange_LateRetriesBeforeDown(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "push-mon", MaxRetries: 1}, - SiteState: models.SiteState{Status: "LATE"}, - } - injectSite(e, site) - - e.handleStatusChange(site, "DOWN", 0, 0, "missed heartbeat") - s, _ := getSite(e, 1) - if s.Status != "LATE" { - t.Errorf("expected LATE during retry, got %s", s.Status) - } - - e.handleStatusChange(s, "DOWN", 0, 0, "missed heartbeat") - s, _ = getSite(e, 1) - if s.Status != "DOWN" { - t.Errorf("expected DOWN after retries exhausted, got %s", s.Status) - } -} - -// Dead probe results must be expired so they don't poison aggregation. -func TestIngestProbeResult_ExpiresStaleProbes(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", Interval: 30}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - e.probeResultsMu.Lock() - e.probeResults[1] = map[string]NodeResult{ - "dead-probe": { - NodeID: "dead-probe", - IsUp: false, - CheckedAt: time.Now().Add(-10 * time.Minute), - }, - } - e.probeResultsMu.Unlock() - - e.IngestProbeResult("live-probe", 1, 5000, true, "") - - e.probeResultsMu.RLock() - _, deadExists := e.probeResults[1]["dead-probe"] - _, liveExists := e.probeResults[1]["live-probe"] - e.probeResultsMu.RUnlock() - - if deadExists { - t.Error("stale probe result should have been expired") - } - if !liveExists { - t.Error("live probe result should still exist") - } -} - -// RemoveSite must clean up probeResults. -func TestRemoveSite_CleansProbeResults(t *testing.T) { - ms := newMockStore() - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - - e.probeResultsMu.Lock() - e.probeResults[1] = map[string]NodeResult{ - "node-a": {NodeID: "node-a", IsUp: true, CheckedAt: time.Now()}, - } - e.probeResultsMu.Unlock() - - e.RemoveSite(1) - - e.probeResultsMu.RLock() - defer e.probeResultsMu.RUnlock() - if _, exists := e.probeResults[1]; exists { - t.Error("probe results should be cleaned up after RemoveSite") - } -} - -// Maintenance cache resolves parent relationships correctly. -func TestIsInMaintenance_UsesCache(t *testing.T) { - ms := newMockStore() - ms.maintenance[10] = true // direct maintenance on group - e := newTestEngine(ms) - group := models.Site{ - SiteConfig: models.SiteConfig{ID: 10, Name: "group", Type: "group"}, - SiteState: models.SiteState{Status: "UP"}, - } - child := models.Site{ - SiteConfig: models.SiteConfig{ID: 20, Name: "child", Type: "http", ParentID: 10}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, group) - injectSite(e, child) - e.refreshMaintenanceCache(context.Background()) - - if !e.isInMaintenance(10) { - t.Error("group should be in maintenance (direct)") - } - if !e.isInMaintenance(20) { - t.Error("child should be in maintenance (parent)") - } - if e.isInMaintenance(99) { - t.Error("unknown monitor should not be in maintenance") - } -} - -// Global maintenance (monitor_id=0) applies to all monitors. -func TestIsInMaintenance_GlobalMaintenance(t *testing.T) { - ms := newMockStore() - ms.maintenance[0] = true - e := newTestEngine(ms) - site := models.Site{ - SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"}, - SiteState: models.SiteState{Status: "UP"}, - } - injectSite(e, site) - e.refreshMaintenanceCache(context.Background()) - - if !e.isInMaintenance(1) { - t.Error("all monitors should be in maintenance during global window") - } -} - // --- Utilities --- func containsStr(s, substr string) bool { diff --git a/internal/monitor/sites.go b/internal/monitor/sites.go new file mode 100644 index 0000000..555aa90 --- /dev/null +++ b/internal/monitor/sites.go @@ -0,0 +1,158 @@ +package monitor + +import ( + "context" + "fmt" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func (e *Engine) SetActive(active bool) { + e.activeMu.Lock() + defer e.activeMu.Unlock() + if e.isActive != active { + e.isActive = active + status := "RESUMED (Active)" + if !active { + status = "PAUSED (Passive)" + } + e.AddLog(fmt.Sprintf("Engine %s", status)) + } +} + +func (e *Engine) IsActive() bool { + e.activeMu.RLock() + defer e.activeMu.RUnlock() + return e.isActive +} + +func (e *Engine) GetAllSites() []models.Site { + e.mu.RLock() + defer e.mu.RUnlock() + sites := make([]models.Site, 0, len(e.liveState)) + for _, s := range e.liveState { + sites = append(sites, s) + } + return sites +} + +func (e *Engine) GetLiveState() map[int]models.Site { + e.mu.RLock() + defer e.mu.RUnlock() + cp := make(map[int]models.Site, len(e.liveState)) + for k, v := range e.liveState { + cp[k] = v + } + return cp +} + +func (e *Engine) addToTokenIndex(site models.Site) { + if site.Type == "push" && site.Token != "" { + e.tokenIndex[site.Token] = site.ID + } +} + +func (e *Engine) removeFromTokenIndex(id int) { + for token, sid := range e.tokenIndex { + if sid == id { + delete(e.tokenIndex, token) + return + } + } +} + +func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) { + e.mu.Lock() + if existing, ok := e.liveState[cfg.ID]; ok { + e.removeFromTokenIndex(cfg.ID) + existing.SiteConfig = cfg + e.liveState[cfg.ID] = existing + e.addToTokenIndex(existing) + } + e.mu.Unlock() + + e.signalRecheck(cfg.ID) +} + +func (e *Engine) RemoveSite(id int) { + e.mu.Lock() + e.removeFromTokenIndex(id) + delete(e.liveState, id) + e.mu.Unlock() + e.removeHistory(id) + + e.probeResultsMu.Lock() + delete(e.probeResults, id) + e.probeResultsMu.Unlock() + + e.recheckMu.Lock() + delete(e.recheck, id) + e.recheckMu.Unlock() +} + +func (e *Engine) ToggleSitePause(id int) bool { + var ( + paused bool + name string + ) + _, ok := e.applyState(id, func(s *models.Site) { + s.Paused = !s.Paused + paused = s.Paused + name = s.Name + }) + if !ok { + return false + } + if paused { + e.AddLog(fmt.Sprintf("Monitor '%s' paused", name)) + } else { + e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name)) + } + return paused +} + +// applyState atomically reads, mutates, and writes back the live entry for id. +// The mutator runs under the engine write lock and receives a pointer to the +// CURRENT live state, so concurrent config edits, pauses, and heartbeats are +// never clobbered by a stale snapshot. The mutator must only touch runtime / +// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…) +// are owned by UpdateSiteConfig and must not be written here. Returns the +// post-mutation copy and whether the site still exists. +func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) { + e.mu.Lock() + defer e.mu.Unlock() + cur, ok := e.liveState[id] + if !ok { + return models.Site{}, false + } + mutate(&cur) + e.liveState[id] = cur + return cur, true +} + +func (e *Engine) GetDisplayStatus(site models.Site) string { + if site.Paused { + return "PAUSED" + } + if e.isInMaintenance(site.ID) { + return "MAINT" + } + return string(site.Status) +} + +func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange { + changes, err := e.db.GetStateChanges(context.Background(), siteID, limit) + if err != nil { + return nil + } + return changes +} + +func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange { + changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since) + if err != nil { + return nil + } + return changes +} diff --git a/internal/monitor/sites_test.go b/internal/monitor/sites_test.go new file mode 100644 index 0000000..ed24051 --- /dev/null +++ b/internal/monitor/sites_test.go @@ -0,0 +1,133 @@ +package monitor + +import ( + "testing" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func TestUpdateSiteConfig_PreservesRuntime(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com"}, + SiteState: models.SiteState{Status: "DOWN", FailureCount: 3, Latency: 100 * time.Millisecond}, + } + injectSite(e, site) + + updated := models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Interval: 60} + e.UpdateSiteConfig(updated) + + s, _ := getSite(e, 1) + if s.URL != "http://new.com" { + t.Errorf("expected URL updated, got %s", s.URL) + } + if s.Status != "DOWN" { + t.Errorf("expected Status preserved, got %s", s.Status) + } + if s.FailureCount != 3 { + t.Errorf("expected FailureCount preserved, got %d", s.FailureCount) + } + if s.Latency != 100*time.Millisecond { + t.Errorf("expected Latency preserved, got %v", s.Latency) + } +} + +func TestRemoveSite_CleansUp(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "push", Token: "tok1"}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + e.recordCheck(1, 5*time.Millisecond, true) + + e.RemoveSite(1) + + if _, ok := getSite(e, 1); ok { + t.Error("expected site removed from liveState") + } + if e.RecordHeartbeat("tok1") { + t.Error("expected token removed from index") + } + if _, ok := e.GetHistory(1); ok { + t.Error("expected history removed") + } +} + +func TestToggleSitePause(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + site := models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "test"}, + SiteState: models.SiteState{Status: "UP"}, + } + injectSite(e, site) + + paused := e.ToggleSitePause(1) + if !paused { + t.Error("expected paused=true after first toggle") + } + s, _ := getSite(e, 1) + if !s.Paused { + t.Error("expected Paused=true in state") + } + + paused = e.ToggleSitePause(1) + if paused { + t.Error("expected paused=false after second toggle") + } +} + +func TestToggleSitePause_NonexistentSite(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + if e.ToggleSitePause(999) { + t.Error("expected false for nonexistent site") + } +} + +func TestGetAllSites_ReturnsCopy(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + injectSite(e, models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "s1"}, + SiteState: models.SiteState{Status: "UP"}, + }) + injectSite(e, models.Site{ + SiteConfig: models.SiteConfig{ID: 2, Name: "s2"}, + SiteState: models.SiteState{Status: "DOWN"}, + }) + + sites := e.GetAllSites() + if len(sites) != 2 { + t.Fatalf("expected 2 sites, got %d", len(sites)) + } + sites[0].Name = "mutated" + + fresh := e.GetAllSites() + for _, s := range fresh { + if s.Name == "mutated" { + t.Error("GetAllSites returned reference, not copy") + } + } +} + +func TestGetLiveState_ReturnsCopy(t *testing.T) { + ms := newMockStore() + e := newTestEngine(ms) + injectSite(e, models.Site{ + SiteConfig: models.SiteConfig{ID: 1, Name: "s1"}, + SiteState: models.SiteState{Status: "UP"}, + }) + + state := e.GetLiveState() + state[1] = models.Site{SiteConfig: models.SiteConfig{Name: "mutated"}} + + fresh := e.GetLiveState() + if fresh[1].Name == "mutated" { + t.Error("GetLiveState returned reference, not copy") + } +} diff --git a/internal/store/sqlstore.go b/internal/store/sqlstore.go index c803f2c..6d511a1 100644 --- a/internal/store/sqlstore.go +++ b/internal/store/sqlstore.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "database/sql" "encoding/hex" - "encoding/json" "fmt" "strings" "time" @@ -218,40 +217,6 @@ func (s *SQLStore) GetSiteByName(ctx context.Context, name string) (models.SiteC return st, err } -func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) { - decrypted, err := s.decryptSettings(raw) - if err != nil { - return nil, fmt.Errorf("decrypt settings: %w", err) - } - var m map[string]string - if err := json.Unmarshal([]byte(decrypted), &m); err != nil { - return nil, fmt.Errorf("unmarshal settings: %w", err) - } - return m, nil -} - -func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) { - jsonBytes, err := json.Marshal(settings) - if err != nil { - return "", err - } - return s.encryptSettings(string(jsonBytes)) -} - -func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) { - var a models.AlertConfig - var settingsRaw string - err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw) - if err != nil { - return a, err - } - a.Settings, err = s.unmarshalSettings(settingsRaw) - if err != nil { - return a, fmt.Errorf("alert %q: %w", name, err) - } - return a, nil -} - func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfig) (int, error) { token := "" if site.Type == "push" { @@ -278,89 +243,6 @@ func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfi return int(id), err } -func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) { - stored, err := s.marshalSettings(settings) - if err != nil { - return 0, err - } - if s.dollar { - var id int - err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id) - return id, err - } - result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored) - if err != nil { - return 0, err - } - id, err := result.LastInsertId() - return int(id), err -} - -func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) { - rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts") - if err != nil { - return nil, err - } - defer rows.Close() - var alerts []models.AlertConfig - for rows.Next() { - var a models.AlertConfig - var settingsRaw string - if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil { - return alerts, err - } - a.Settings, err = s.unmarshalSettings(settingsRaw) - if err != nil { - return alerts, fmt.Errorf("alert %q: %w", a.Name, err) - } - alerts = append(alerts, a) - } - return alerts, rows.Err() -} - -func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) { - var a models.AlertConfig - var settingsRaw string - err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw) - if err != nil { - return a, err - } - a.Settings, err = s.unmarshalSettings(settingsRaw) - if err != nil { - return a, fmt.Errorf("alert %d: %w", id, err) - } - return a, nil -} - -func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error { - stored, err := s.marshalSettings(settings) - if err != nil { - return err - } - _, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored) - return err -} - -func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error { - stored, err := s.marshalSettings(settings) - if err != nil { - return err - } - _, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id) - return err -} - -func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error { - if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil { - return err - } - if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil { - return err - } - s.dialect.ResetSequenceOnEmpty(s.db, "alerts") - return nil -} - func (s *SQLStore) GetAllUsers(ctx context.Context) ([]models.User, error) { rows, err := s.db.QueryContext(ctx, "SELECT id, username, public_key, role FROM users") if err != nil { @@ -393,85 +275,6 @@ func (s *SQLStore) DeleteUser(ctx context.Context, id int) error { return err } -func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error { - _, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"), - siteID, fromStatus, toStatus, errorReason) - return err -} - -func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) { - rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit) - if err != nil { - return nil, err - } - defer rows.Close() - var changes []models.StateChange - for rows.Next() { - var sc models.StateChange - if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil { - return changes, err - } - changes = append(changes, sc) - } - return changes, rows.Err() -} - -func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) { - rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since) - if err != nil { - return nil, err - } - defer rows.Close() - var changes []models.StateChange - for rows.Next() { - var sc models.StateChange - if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil { - return changes, err - } - changes = append(changes, sc) - } - return changes, rows.Err() -} - -func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error { - return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp) -} - -// SaveCheckFromNode inserts a single check row. Retention is handled out of -// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot -// path a plain INSERT. -func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error { - _, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp) - return err -} - -// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per -// site, across all sites, in one pass. Intended to run periodically. -func (s *SQLStore) PruneCheckHistory(ctx context.Context) error { - q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN ( - SELECT id FROM ( - SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn - FROM check_history - ) ranked WHERE rn > %d - )`, maxCheckHistory) - _, err := s.db.ExecContext(ctx, s.q(q)) - return err -} - -// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite -// rows per site. Generous so realistic SLA windows are unaffected; bounds the -// otherwise unbounded growth of a flapping monitor's history. -func (s *SQLStore) PruneStateChanges(ctx context.Context) error { - q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN ( - SELECT id FROM ( - SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn - FROM state_changes - ) ranked WHERE rn > %d - )`, maxStateChangesPerSite) - _, err := s.db.ExecContext(ctx, s.q(q)) - return err -} - func (s *SQLStore) RegisterNode(ctx context.Context, node models.ProbeNode) error { _, err := s.db.ExecContext(ctx, s.dialect.UpsertNodeSQL(), node.ID, node.Name, node.Region, node.Version) return err @@ -582,172 +385,6 @@ func (s *SQLStore) LoadLogs(ctx context.Context, limit int) ([]models.LogEntry, return entries, rows.Err() } -func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) { - result := make(map[int][]models.CheckRecord) - rows, err := s.db.QueryContext(ctx, s.q(` - SELECT site_id, latency_ns, is_up FROM ( - SELECT site_id, latency_ns, is_up, - ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn - FROM check_history - ) sub WHERE rn <= ?`), limit) - if err != nil { - return result, err - } - defer rows.Close() - for rows.Next() { - var r models.CheckRecord - if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil { - return result, err - } - result[r.SiteID] = append(result[r.SiteID], r) - } - for id, records := range result { - for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 { - records[i], records[j] = records[j], records[i] - } - result[id] = records - } - return result, rows.Err() -} - -func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) { - var mw models.MaintenanceWindow - var endTime sql.NullTime - if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil { - return mw, err - } - if endTime.Valid { - mw.EndTime = endTime.Time - } - return mw, nil -} - -func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) { - rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC")) - if err != nil { - return nil, err - } - defer rows.Close() - var windows []models.MaintenanceWindow - for rows.Next() { - mw, err := s.scanMaintenanceWindow(rows) - if err != nil { - return windows, err - } - windows = append(windows, mw) - } - return windows, rows.Err() -} - -func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) { - rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit) - if err != nil { - return nil, err - } - defer rows.Close() - var windows []models.MaintenanceWindow - for rows.Next() { - mw, err := s.scanMaintenanceWindow(rows) - if err != nil { - return windows, err - } - windows = append(windows, mw) - } - return windows, rows.Err() -} - -func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) { - var timeClause string - var args []interface{} - - if endTime.IsZero() { - timeClause = "(end_time IS NULL OR end_time > ?)" - args = append(args, startTime) - } else { - timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?" - args = append(args, startTime, endTime) - } - - var scopeClause string - if monitorID == 0 { - scopeClause = "1=1" - } else { - scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))" - args = append(args, monitorID, monitorID) - } - - query := fmt.Sprintf( - "SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time", - timeClause, scopeClause, - ) - - rows, err := s.db.QueryContext(ctx, s.q(query), args...) - if err != nil { - return nil, err - } - defer rows.Close() - - var windows []models.MaintenanceWindow - for rows.Next() { - mw, err := s.scanMaintenanceWindow(rows) - if err != nil { - return nil, err - } - windows = append(windows, mw) - } - return windows, rows.Err() -} - -func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error { - if mw.StartTime.IsZero() { - mw.StartTime = time.Now() - } - _, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"), - mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy) - return err -} - -func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error { - _, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id) - return err -} - -func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error { - _, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id) - if err != nil { - return err - } - s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows") - return nil -} - -func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) { - cutoff := time.Now().Add(-retention) - result, err := s.db.ExecContext(ctx, - s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"), - cutoff, - ) - if err != nil { - return 0, err - } - return result.RowsAffected() -} - -func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) { - var count int - err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows - WHERE type = 'maintenance' - AND start_time <= CURRENT_TIMESTAMP - AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) - AND (monitor_id = 0 OR monitor_id = ? - OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`), - monitorID, monitorID).Scan(&count) - if err != nil { - return false, err - } - return count > 0, nil -} - func (s *SQLStore) GetPreference(ctx context.Context, key string) (string, error) { var value string err := s.db.QueryRowContext(ctx, s.q("SELECT value FROM preferences WHERE key = ?"), key).Scan(&value) diff --git a/internal/store/sqlstore_alerts.go b/internal/store/sqlstore_alerts.go new file mode 100644 index 0000000..741b606 --- /dev/null +++ b/internal/store/sqlstore_alerts.go @@ -0,0 +1,126 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) { + decrypted, err := s.decryptSettings(raw) + if err != nil { + return nil, fmt.Errorf("decrypt settings: %w", err) + } + var m map[string]string + if err := json.Unmarshal([]byte(decrypted), &m); err != nil { + return nil, fmt.Errorf("unmarshal settings: %w", err) + } + return m, nil +} + +func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) { + jsonBytes, err := json.Marshal(settings) + if err != nil { + return "", err + } + return s.encryptSettings(string(jsonBytes)) +} + +func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) { + var a models.AlertConfig + var settingsRaw string + err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw) + if err != nil { + return a, err + } + a.Settings, err = s.unmarshalSettings(settingsRaw) + if err != nil { + return a, fmt.Errorf("alert %q: %w", name, err) + } + return a, nil +} + +func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) { + stored, err := s.marshalSettings(settings) + if err != nil { + return 0, err + } + if s.dollar { + var id int + err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id) + return id, err + } + result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored) + if err != nil { + return 0, err + } + id, err := result.LastInsertId() + return int(id), err +} + +func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) { + rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts") + if err != nil { + return nil, err + } + defer rows.Close() + var alerts []models.AlertConfig + for rows.Next() { + var a models.AlertConfig + var settingsRaw string + if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil { + return alerts, err + } + a.Settings, err = s.unmarshalSettings(settingsRaw) + if err != nil { + return alerts, fmt.Errorf("alert %q: %w", a.Name, err) + } + alerts = append(alerts, a) + } + return alerts, rows.Err() +} + +func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) { + var a models.AlertConfig + var settingsRaw string + err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw) + if err != nil { + return a, err + } + a.Settings, err = s.unmarshalSettings(settingsRaw) + if err != nil { + return a, fmt.Errorf("alert %d: %w", id, err) + } + return a, nil +} + +func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error { + stored, err := s.marshalSettings(settings) + if err != nil { + return err + } + _, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored) + return err +} + +func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error { + stored, err := s.marshalSettings(settings) + if err != nil { + return err + } + _, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id) + return err +} + +func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error { + if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil { + return err + } + if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil { + return err + } + s.dialect.ResetSequenceOnEmpty(s.db, "alerts") + return nil +} diff --git a/internal/store/sqlstore_alerts_test.go b/internal/store/sqlstore_alerts_test.go new file mode 100644 index 0000000..c98d609 --- /dev/null +++ b/internal/store/sqlstore_alerts_test.go @@ -0,0 +1,102 @@ +package store + +import ( + "context" + "strings" + "testing" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func TestAlertCRUD(t *testing.T) { + s := newTestStore(t) + + if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil { + t.Fatalf("AddAlert: %v", err) + } + + alerts, err := s.GetAllAlerts(context.Background()) + if err != nil { + t.Fatalf("GetAllAlerts: %v", err) + } + if len(alerts) != 1 { + t.Fatalf("expected 1 alert, got %d", len(alerts)) + } + if alerts[0].Type != "discord" { + t.Errorf("expected type 'discord', got '%s'", alerts[0].Type) + } + if alerts[0].Settings["url"] != "https://example.com/hook" { + t.Errorf("settings url mismatch") + } + + a, err := s.GetAlert(context.Background(), alerts[0].ID) + if err != nil { + t.Fatalf("GetAlert: %v", err) + } + if a.Name != "Discord" { + t.Errorf("expected name 'Discord', got '%s'", a.Name) + } + + if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil { + t.Fatalf("UpdateAlert: %v", err) + } + + a, err = s.GetAlert(context.Background(), a.ID) + if err != nil { + t.Fatalf("GetAlert: %v", err) + } + if a.Type != "slack" { + t.Errorf("expected type 'slack', got '%s'", a.Type) + } + + if err := s.DeleteAlert(context.Background(), a.ID); err != nil { + t.Fatalf("DeleteAlert: %v", err) + } + + alerts, err = s.GetAllAlerts(context.Background()) + if err != nil { + t.Fatalf("GetAllAlerts: %v", err) + } + if len(alerts) != 0 { + t.Fatalf("expected 0 alerts after delete, got %d", len(alerts)) + } +} + +// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a +// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext. +func TestImportData_EncryptsAlertSettings(t *testing.T) { + s := newTestStore(t) + enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes + if err != nil { + t.Fatalf("NewEncryptor: %v", err) + } + s.SetEncryptor(enc) + + backup := models.Backup{ + Alerts: []models.AlertConfig{ + {ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}}, + }, + } + if err := s.ImportData(context.Background(), backup); err != nil { + t.Fatalf("ImportData: %v", err) + } + + var raw string + if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil { + t.Fatalf("query settings: %v", err) + } + if !strings.HasPrefix(raw, encryptedPrefix) { + t.Errorf("imported settings not encrypted: %q", raw) + } + if strings.Contains(raw, "SECRET") { + t.Errorf("plaintext secret found in stored column: %q", raw) + } + + alerts, err := s.GetAllAlerts(context.Background()) + if err != nil { + t.Fatalf("GetAllAlerts: %v", err) + } + if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" { + t.Errorf("decrypt round-trip failed: %+v", alerts) + } +} diff --git a/internal/store/sqlstore_history.go b/internal/store/sqlstore_history.go new file mode 100644 index 0000000..37d865d --- /dev/null +++ b/internal/store/sqlstore_history.go @@ -0,0 +1,116 @@ +package store + +import ( + "context" + "fmt" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error { + _, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"), + siteID, fromStatus, toStatus, errorReason) + return err +} + +func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) { + rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var changes []models.StateChange + for rows.Next() { + var sc models.StateChange + if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil { + return changes, err + } + changes = append(changes, sc) + } + return changes, rows.Err() +} + +func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) { + rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since) + if err != nil { + return nil, err + } + defer rows.Close() + var changes []models.StateChange + for rows.Next() { + var sc models.StateChange + if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil { + return changes, err + } + changes = append(changes, sc) + } + return changes, rows.Err() +} + +func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error { + return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp) +} + +// SaveCheckFromNode inserts a single check row. Retention is handled out of +// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot +// path a plain INSERT. +func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error { + _, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp) + return err +} + +// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per +// site, across all sites, in one pass. Intended to run periodically. +func (s *SQLStore) PruneCheckHistory(ctx context.Context) error { + q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN ( + SELECT id FROM ( + SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn + FROM check_history + ) ranked WHERE rn > %d + )`, maxCheckHistory) + _, err := s.db.ExecContext(ctx, s.q(q)) + return err +} + +// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite +// rows per site. Generous so realistic SLA windows are unaffected; bounds the +// otherwise unbounded growth of a flapping monitor's history. +func (s *SQLStore) PruneStateChanges(ctx context.Context) error { + q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN ( + SELECT id FROM ( + SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn + FROM state_changes + ) ranked WHERE rn > %d + )`, maxStateChangesPerSite) + _, err := s.db.ExecContext(ctx, s.q(q)) + return err +} + +func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) { + result := make(map[int][]models.CheckRecord) + rows, err := s.db.QueryContext(ctx, s.q(` + SELECT site_id, latency_ns, is_up FROM ( + SELECT site_id, latency_ns, is_up, + ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn + FROM check_history + ) sub WHERE rn <= ?`), limit) + if err != nil { + return result, err + } + defer rows.Close() + for rows.Next() { + var r models.CheckRecord + if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil { + return result, err + } + result[r.SiteID] = append(result[r.SiteID], r) + } + for id, records := range result { + for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 { + records[i], records[j] = records[j], records[i] + } + result[id] = records + } + return result, rows.Err() +} diff --git a/internal/store/sqlstore_history_test.go b/internal/store/sqlstore_history_test.go new file mode 100644 index 0000000..2f40a94 --- /dev/null +++ b/internal/store/sqlstore_history_test.go @@ -0,0 +1,134 @@ +package store + +import ( + "context" + "testing" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func TestCheckHistory(t *testing.T) { + s := newTestStore(t) + + if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil { + t.Fatalf("SaveCheck: %v", err) + } + if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil { + t.Fatalf("SaveCheck: %v", err) + } + if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil { + t.Fatalf("SaveCheck site 2: %v", err) + } + + history, err := s.LoadAllHistory(context.Background(), 10) + if err != nil { + t.Fatalf("LoadAllHistory: %v", err) + } + if len(history[1]) != 2 { + t.Fatalf("expected 2 records for site 1, got %d", len(history[1])) + } + if len(history[2]) != 1 { + t.Fatalf("expected 1 record for site 2, got %d", len(history[2])) + } + + upCount := 0 + for _, r := range history[1] { + if r.IsUp { + upCount++ + } + } + if upCount != 1 { + t.Errorf("expected 1 up record for site 1, got %d", upCount) + } +} + +func TestPruneCheckHistory(t *testing.T) { + s := newTestStore(t) + + for i := 0; i < maxCheckHistory+5; i++ { + if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil { + t.Fatalf("SaveCheck site 1: %v", err) + } + } + for i := 0; i < 3; i++ { + if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil { + t.Fatalf("SaveCheck site 2: %v", err) + } + } + + if err := s.PruneCheckHistory(context.Background()); err != nil { + t.Fatalf("PruneCheckHistory: %v", err) + } + + history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2) + if err != nil { + t.Fatalf("LoadAllHistory: %v", err) + } + if len(history[1]) != maxCheckHistory { + t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1])) + } + if len(history[2]) != 3 { + t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2])) + } +} + +func TestDeleteSiteCascade(t *testing.T) { + s := newTestStore(t) + + site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30} + if err := s.AddSite(context.Background(), site); err != nil { + t.Fatalf("AddSite: %v", err) + } + sites, err := s.GetSites(context.Background()) + if err != nil { + t.Fatalf("GetSites: %v", err) + } + siteID := sites[0].ID + + if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil { + t.Fatalf("SaveCheck: %v", err) + } + if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil { + t.Fatalf("SaveStateChange: %v", err) + } + mw := models.MaintenanceWindow{ + MonitorID: siteID, + Title: "Test MW", + Type: "maintenance", + StartTime: time.Now(), + } + if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil { + t.Fatalf("AddMaintenanceWindow: %v", err) + } + + if err := s.DeleteSite(context.Background(), siteID); err != nil { + t.Fatalf("DeleteSite: %v", err) + } + + history, err := s.LoadAllHistory(context.Background(), 100) + if err != nil { + t.Fatalf("LoadAllHistory: %v", err) + } + if len(history[siteID]) != 0 { + t.Errorf("expected 0 check_history rows, got %d", len(history[siteID])) + } + + changes, err := s.GetStateChanges(context.Background(), siteID, 100) + if err != nil { + t.Fatalf("GetStateChanges: %v", err) + } + if len(changes) != 0 { + t.Errorf("expected 0 state_changes rows, got %d", len(changes)) + } + + windows, err := s.GetActiveMaintenanceWindows(context.Background()) + if err != nil { + t.Fatalf("GetActiveMaintenanceWindows: %v", err) + } + for _, w := range windows { + if w.MonitorID == siteID { + t.Errorf("orphaned maintenance window found: id=%d", w.ID) + } + } +} diff --git a/internal/store/sqlstore_maintenance.go b/internal/store/sqlstore_maintenance.go new file mode 100644 index 0000000..c3a4bd4 --- /dev/null +++ b/internal/store/sqlstore_maintenance.go @@ -0,0 +1,148 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) { + var mw models.MaintenanceWindow + var endTime sql.NullTime + if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil { + return mw, err + } + if endTime.Valid { + mw.EndTime = endTime.Time + } + return mw, nil +} + +func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) { + rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC")) + if err != nil { + return nil, err + } + defer rows.Close() + var windows []models.MaintenanceWindow + for rows.Next() { + mw, err := s.scanMaintenanceWindow(rows) + if err != nil { + return windows, err + } + windows = append(windows, mw) + } + return windows, rows.Err() +} + +func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) { + rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit) + if err != nil { + return nil, err + } + defer rows.Close() + var windows []models.MaintenanceWindow + for rows.Next() { + mw, err := s.scanMaintenanceWindow(rows) + if err != nil { + return windows, err + } + windows = append(windows, mw) + } + return windows, rows.Err() +} + +func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) { + var timeClause string + var args []interface{} + + if endTime.IsZero() { + timeClause = "(end_time IS NULL OR end_time > ?)" + args = append(args, startTime) + } else { + timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?" + args = append(args, startTime, endTime) + } + + var scopeClause string + if monitorID == 0 { + scopeClause = "1=1" + } else { + scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))" + args = append(args, monitorID, monitorID) + } + + query := fmt.Sprintf( + "SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time", + timeClause, scopeClause, + ) + + rows, err := s.db.QueryContext(ctx, s.q(query), args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var windows []models.MaintenanceWindow + for rows.Next() { + mw, err := s.scanMaintenanceWindow(rows) + if err != nil { + return nil, err + } + windows = append(windows, mw) + } + return windows, rows.Err() +} + +func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error { + if mw.StartTime.IsZero() { + mw.StartTime = time.Now() + } + _, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"), + mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy) + return err +} + +func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error { + _, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id) + return err +} + +func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error { + _, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id) + if err != nil { + return err + } + s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows") + return nil +} + +func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) { + cutoff := time.Now().Add(-retention) + result, err := s.db.ExecContext(ctx, + s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"), + cutoff, + ) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) { + var count int + err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows + WHERE type = 'maintenance' + AND start_time <= CURRENT_TIMESTAMP + AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) + AND (monitor_id = 0 OR monitor_id = ? + OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`), + monitorID, monitorID).Scan(&count) + if err != nil { + return false, err + } + return count > 0, nil +} diff --git a/internal/store/sqlstore_maintenance_test.go b/internal/store/sqlstore_maintenance_test.go new file mode 100644 index 0000000..bae167f --- /dev/null +++ b/internal/store/sqlstore_maintenance_test.go @@ -0,0 +1,168 @@ +package store + +import ( + "context" + "testing" + "time" + + "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" +) + +func TestPruneExpiredMaintenanceWindows(t *testing.T) { + s := newTestStore(t) + + now := time.Now() + + // Expired 10 days ago — should be pruned with 7d retention. + old := models.MaintenanceWindow{ + MonitorID: 0, + Title: "Old Window", + Type: "maintenance", + StartTime: now.Add(-11 * 24 * time.Hour), + EndTime: now.Add(-10 * 24 * time.Hour), + } + if err := s.AddMaintenanceWindow(context.Background(), old); err != nil { + t.Fatalf("AddMaintenanceWindow (old): %v", err) + } + + // Expired 1 day ago — within 7d retention, should survive. + recent := models.MaintenanceWindow{ + MonitorID: 0, + Title: "Recent Window", + Type: "maintenance", + StartTime: now.Add(-2 * 24 * time.Hour), + EndTime: now.Add(-1 * 24 * time.Hour), + } + if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil { + t.Fatalf("AddMaintenanceWindow (recent): %v", err) + } + + // Ongoing — no end time, should survive. + ongoing := models.MaintenanceWindow{ + MonitorID: 0, + Title: "Ongoing Window", + Type: "maintenance", + StartTime: now.Add(-1 * time.Hour), + } + if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil { + t.Fatalf("AddMaintenanceWindow (ongoing): %v", err) + } + + pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour) + if err != nil { + t.Fatalf("PruneExpiredMaintenanceWindows: %v", err) + } + if pruned != 1 { + t.Errorf("expected 1 pruned, got %d", pruned) + } + + all, err := s.GetAllMaintenanceWindows(context.Background(), 100) + if err != nil { + t.Fatalf("GetAllMaintenanceWindows: %v", err) + } + if len(all) != 2 { + t.Fatalf("expected 2 remaining windows, got %d", len(all)) + } + for _, w := range all { + if w.Title == "Old Window" { + t.Error("old window should have been pruned") + } + } +} + +func TestGetOverlappingMaintenanceWindows(t *testing.T) { + s := newTestStore(t) + ctx := context.Background() + + site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30} + if err := s.AddSite(ctx, site); err != nil { + t.Fatalf("AddSite: %v", err) + } + + now := time.Now() + + active := models.MaintenanceWindow{ + MonitorID: 1, + Title: "Deploy v2", + Type: "maintenance", + StartTime: now.Add(-30 * time.Minute), + EndTime: now.Add(30 * time.Minute), + } + if err := s.AddMaintenanceWindow(ctx, active); err != nil { + t.Fatalf("AddMaintenanceWindow: %v", err) + } + + ended := models.MaintenanceWindow{ + MonitorID: 1, + Title: "Old deploy", + Type: "maintenance", + StartTime: now.Add(-3 * time.Hour), + EndTime: now.Add(-2 * time.Hour), + } + if err := s.AddMaintenanceWindow(ctx, ended); err != nil { + t.Fatalf("AddMaintenanceWindow: %v", err) + } + + t.Run("same monitor overlaps", func(t *testing.T) { + overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(overlaps) != 1 { + t.Fatalf("expected 1 overlap, got %d", len(overlaps)) + } + if overlaps[0].Title != "Deploy v2" { + t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title) + } + }) + + t.Run("different monitor no overlap", func(t *testing.T) { + overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(overlaps) != 0 { + t.Errorf("expected 0 overlaps, got %d", len(overlaps)) + } + }) + + t.Run("global window overlaps all", func(t *testing.T) { + global := models.MaintenanceWindow{ + MonitorID: 0, + Title: "Global freeze", + Type: "maintenance", + StartTime: now.Add(-10 * time.Minute), + EndTime: now.Add(2 * time.Hour), + } + if err := s.AddMaintenanceWindow(ctx, global); err != nil { + t.Fatalf("AddMaintenanceWindow: %v", err) + } + overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(overlaps) != 2 { + t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps)) + } + }) + + t.Run("indefinite window overlaps", func(t *testing.T) { + overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(overlaps) < 1 { + t.Error("expected at least 1 overlap for indefinite window") + } + }) + + t.Run("ended window excluded", func(t *testing.T) { + overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(overlaps) != 0 { + t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps)) + } + }) +} diff --git a/internal/store/sqlstore_test.go b/internal/store/sqlstore_test.go index f720e74..ab74aaa 100644 --- a/internal/store/sqlstore_test.go +++ b/internal/store/sqlstore_test.go @@ -3,9 +3,7 @@ package store import ( "context" "fmt" - "strings" "testing" - "time" "gitea.lerkolabs.com/lerkolabs/uptop/internal/models" ) @@ -74,60 +72,6 @@ func TestSiteCRUD(t *testing.T) { } } -func TestAlertCRUD(t *testing.T) { - s := newTestStore(t) - - if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil { - t.Fatalf("AddAlert: %v", err) - } - - alerts, err := s.GetAllAlerts(context.Background()) - if err != nil { - t.Fatalf("GetAllAlerts: %v", err) - } - if len(alerts) != 1 { - t.Fatalf("expected 1 alert, got %d", len(alerts)) - } - if alerts[0].Type != "discord" { - t.Errorf("expected type 'discord', got '%s'", alerts[0].Type) - } - if alerts[0].Settings["url"] != "https://example.com/hook" { - t.Errorf("settings url mismatch") - } - - a, err := s.GetAlert(context.Background(), alerts[0].ID) - if err != nil { - t.Fatalf("GetAlert: %v", err) - } - if a.Name != "Discord" { - t.Errorf("expected name 'Discord', got '%s'", a.Name) - } - - if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil { - t.Fatalf("UpdateAlert: %v", err) - } - - a, err = s.GetAlert(context.Background(), a.ID) - if err != nil { - t.Fatalf("GetAlert: %v", err) - } - if a.Type != "slack" { - t.Errorf("expected type 'slack', got '%s'", a.Type) - } - - if err := s.DeleteAlert(context.Background(), a.ID); err != nil { - t.Fatalf("DeleteAlert: %v", err) - } - - alerts, err = s.GetAllAlerts(context.Background()) - if err != nil { - t.Fatalf("GetAllAlerts: %v", err) - } - if len(alerts) != 0 { - t.Fatalf("expected 0 alerts after delete, got %d", len(alerts)) - } -} - func TestUserCRUD(t *testing.T) { s := newTestStore(t) @@ -301,101 +245,6 @@ func TestImportData_NilUsersPreservesExisting(t *testing.T) { } } -func TestCheckHistory(t *testing.T) { - s := newTestStore(t) - - if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil { - t.Fatalf("SaveCheck: %v", err) - } - if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil { - t.Fatalf("SaveCheck: %v", err) - } - if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil { - t.Fatalf("SaveCheck site 2: %v", err) - } - - history, err := s.LoadAllHistory(context.Background(), 10) - if err != nil { - t.Fatalf("LoadAllHistory: %v", err) - } - if len(history[1]) != 2 { - t.Fatalf("expected 2 records for site 1, got %d", len(history[1])) - } - if len(history[2]) != 1 { - t.Fatalf("expected 1 record for site 2, got %d", len(history[2])) - } - - upCount := 0 - for _, r := range history[1] { - if r.IsUp { - upCount++ - } - } - if upCount != 1 { - t.Errorf("expected 1 up record for site 1, got %d", upCount) - } -} - -func TestDeleteSiteCascade(t *testing.T) { - s := newTestStore(t) - - site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30} - if err := s.AddSite(context.Background(), site); err != nil { - t.Fatalf("AddSite: %v", err) - } - sites, err := s.GetSites(context.Background()) - if err != nil { - t.Fatalf("GetSites: %v", err) - } - siteID := sites[0].ID - - if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil { - t.Fatalf("SaveCheck: %v", err) - } - if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil { - t.Fatalf("SaveStateChange: %v", err) - } - mw := models.MaintenanceWindow{ - MonitorID: siteID, - Title: "Test MW", - Type: "maintenance", - StartTime: time.Now(), - } - if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil { - t.Fatalf("AddMaintenanceWindow: %v", err) - } - - if err := s.DeleteSite(context.Background(), siteID); err != nil { - t.Fatalf("DeleteSite: %v", err) - } - - history, err := s.LoadAllHistory(context.Background(), 100) - if err != nil { - t.Fatalf("LoadAllHistory: %v", err) - } - if len(history[siteID]) != 0 { - t.Errorf("expected 0 check_history rows, got %d", len(history[siteID])) - } - - changes, err := s.GetStateChanges(context.Background(), siteID, 100) - if err != nil { - t.Fatalf("GetStateChanges: %v", err) - } - if len(changes) != 0 { - t.Errorf("expected 0 state_changes rows, got %d", len(changes)) - } - - windows, err := s.GetActiveMaintenanceWindows(context.Background()) - if err != nil { - t.Fatalf("GetActiveMaintenanceWindows: %v", err) - } - for _, w := range windows { - if w.MonitorID == siteID { - t.Errorf("orphaned maintenance window found: id=%d", w.ID) - } - } -} - func TestPruneLogs(t *testing.T) { s := newTestStore(t) @@ -428,231 +277,3 @@ func TestPruneLogs(t *testing.T) { t.Error("oldest log survived prune") } } - -func TestPruneCheckHistory(t *testing.T) { - s := newTestStore(t) - - for i := 0; i < maxCheckHistory+5; i++ { - if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil { - t.Fatalf("SaveCheck site 1: %v", err) - } - } - for i := 0; i < 3; i++ { - if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil { - t.Fatalf("SaveCheck site 2: %v", err) - } - } - - if err := s.PruneCheckHistory(context.Background()); err != nil { - t.Fatalf("PruneCheckHistory: %v", err) - } - - history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2) - if err != nil { - t.Fatalf("LoadAllHistory: %v", err) - } - if len(history[1]) != maxCheckHistory { - t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1])) - } - if len(history[2]) != 3 { - t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2])) - } -} - -func TestPruneExpiredMaintenanceWindows(t *testing.T) { - s := newTestStore(t) - - now := time.Now() - - // Expired 10 days ago — should be pruned with 7d retention. - old := models.MaintenanceWindow{ - MonitorID: 0, - Title: "Old Window", - Type: "maintenance", - StartTime: now.Add(-11 * 24 * time.Hour), - EndTime: now.Add(-10 * 24 * time.Hour), - } - if err := s.AddMaintenanceWindow(context.Background(), old); err != nil { - t.Fatalf("AddMaintenanceWindow (old): %v", err) - } - - // Expired 1 day ago — within 7d retention, should survive. - recent := models.MaintenanceWindow{ - MonitorID: 0, - Title: "Recent Window", - Type: "maintenance", - StartTime: now.Add(-2 * 24 * time.Hour), - EndTime: now.Add(-1 * 24 * time.Hour), - } - if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil { - t.Fatalf("AddMaintenanceWindow (recent): %v", err) - } - - // Ongoing — no end time, should survive. - ongoing := models.MaintenanceWindow{ - MonitorID: 0, - Title: "Ongoing Window", - Type: "maintenance", - StartTime: now.Add(-1 * time.Hour), - } - if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil { - t.Fatalf("AddMaintenanceWindow (ongoing): %v", err) - } - - pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour) - if err != nil { - t.Fatalf("PruneExpiredMaintenanceWindows: %v", err) - } - if pruned != 1 { - t.Errorf("expected 1 pruned, got %d", pruned) - } - - all, err := s.GetAllMaintenanceWindows(context.Background(), 100) - if err != nil { - t.Fatalf("GetAllMaintenanceWindows: %v", err) - } - if len(all) != 2 { - t.Fatalf("expected 2 remaining windows, got %d", len(all)) - } - for _, w := range all { - if w.Title == "Old Window" { - t.Error("old window should have been pruned") - } - } -} - -func TestGetOverlappingMaintenanceWindows(t *testing.T) { - s := newTestStore(t) - ctx := context.Background() - - site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30} - if err := s.AddSite(ctx, site); err != nil { - t.Fatalf("AddSite: %v", err) - } - - now := time.Now() - - active := models.MaintenanceWindow{ - MonitorID: 1, - Title: "Deploy v2", - Type: "maintenance", - StartTime: now.Add(-30 * time.Minute), - EndTime: now.Add(30 * time.Minute), - } - if err := s.AddMaintenanceWindow(ctx, active); err != nil { - t.Fatalf("AddMaintenanceWindow: %v", err) - } - - ended := models.MaintenanceWindow{ - MonitorID: 1, - Title: "Old deploy", - Type: "maintenance", - StartTime: now.Add(-3 * time.Hour), - EndTime: now.Add(-2 * time.Hour), - } - if err := s.AddMaintenanceWindow(ctx, ended); err != nil { - t.Fatalf("AddMaintenanceWindow: %v", err) - } - - t.Run("same monitor overlaps", func(t *testing.T) { - overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(overlaps) != 1 { - t.Fatalf("expected 1 overlap, got %d", len(overlaps)) - } - if overlaps[0].Title != "Deploy v2" { - t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title) - } - }) - - t.Run("different monitor no overlap", func(t *testing.T) { - overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(overlaps) != 0 { - t.Errorf("expected 0 overlaps, got %d", len(overlaps)) - } - }) - - t.Run("global window overlaps all", func(t *testing.T) { - global := models.MaintenanceWindow{ - MonitorID: 0, - Title: "Global freeze", - Type: "maintenance", - StartTime: now.Add(-10 * time.Minute), - EndTime: now.Add(2 * time.Hour), - } - if err := s.AddMaintenanceWindow(ctx, global); err != nil { - t.Fatalf("AddMaintenanceWindow: %v", err) - } - overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(overlaps) != 2 { - t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps)) - } - }) - - t.Run("indefinite window overlaps", func(t *testing.T) { - overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(overlaps) < 1 { - t.Error("expected at least 1 overlap for indefinite window") - } - }) - - t.Run("ended window excluded", func(t *testing.T) { - overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(overlaps) != 0 { - t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps)) - } - }) -} - -// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a -// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext. -func TestImportData_EncryptsAlertSettings(t *testing.T) { - s := newTestStore(t) - enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes - if err != nil { - t.Fatalf("NewEncryptor: %v", err) - } - s.SetEncryptor(enc) - - backup := models.Backup{ - Alerts: []models.AlertConfig{ - {ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}}, - }, - } - if err := s.ImportData(context.Background(), backup); err != nil { - t.Fatalf("ImportData: %v", err) - } - - var raw string - if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil { - t.Fatalf("query settings: %v", err) - } - if !strings.HasPrefix(raw, encryptedPrefix) { - t.Errorf("imported settings not encrypted: %q", raw) - } - if strings.Contains(raw, "SECRET") { - t.Errorf("plaintext secret found in stored column: %q", raw) - } - - alerts, err := s.GetAllAlerts(context.Background()) - if err != nil { - t.Fatalf("GetAllAlerts: %v", err) - } - if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" { - t.Errorf("decrypt round-trip failed: %+v", alerts) - } -}