lerkolabs/uptop
1
1
Fork 0
Code Issues 5 Pull Requests 1 Actions Packages Projects 1 Releases 1 Wiki Activity

refactor: decompose monitor.go and sqlstore.go into focused files #154

Merged
lerko merged 1 commits from refactor/decompose-monitor-sqlstore into main 2026-06-27 23:29:11 +00:00
Conversation 0 Commits 1 Files Changed 18
+2671 -2553
18 changed files with 2671 additions and 2553 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/monitor/alerts.go
+272
View File
@@ -0,0 +1,272 @@
package monitor
import (
"context"
"fmt"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
type AlertHealth struct {
LastSendAt time.Time
LastSendOK bool
LastError string
SendCount int
FailCount int
}
// InitAlertHealth restores persisted alert send health so the dashboard shows real
// "last sent" / health state on startup instead of resetting every channel to "never".
func (e *Engine) InitAlertHealth() {
records, err := e.db.LoadAlertHealth(context.Background())
if err != nil {
return
}
e.alertHealthMu.Lock()
defer e.alertHealthMu.Unlock()
for id, r := range records {
e.alertHealth[id] = AlertHealth{
LastSendAt: r.LastSendAt,
LastSendOK: r.LastSendOK,
LastError: r.LastError,
SendCount: r.SendCount,
FailCount: r.FailCount,
}
}
}
// handleStatusChange folds a check result into the live state. snap is the
// stale snapshot the check ran against; the actual mutation is applied onto the
// CURRENT live entry via applyState, so a concurrent pause / config edit /
// heartbeat is never reverted by this write. Logs and alerts are emitted after
// the lock is released, off the critical section.
func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
if !e.IsActive() {
return
}
inMaint := e.isInMaintenance(snap.ID)
status := models.Status(rawStatus)
var (
prev, next models.Status
name, typ string
alertID int
failCount, maxRetries int
confirmedDown bool
failedCheck bool
downSince time.Time
sslWarnFire bool
sslDays int
skipped bool
changed bool
)
_, exists := e.applyState(snap.ID, func(s *models.Site) {
// A non-UP result computed from a stale snapshot must not override a
// heartbeat (or newer check) that landed while we were evaluating.
if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
skipped = true
return
}
prev = s.Status
name = s.Name
typ = s.Type
alertID = s.AlertID
maxRetries = s.MaxRetries
downSince = s.StatusChangedAt
// Fresh check results (measured by the run against snap).
s.StatusCode = code
s.Latency = snap.Latency
s.LastCheck = snap.LastCheck
s.HasSSL = snap.HasSSL
s.CertExpiry = snap.CertExpiry
s.LastError = errorReason
if status == models.StatusUp {
s.LastSuccessAt = time.Now()
s.LastError = ""
}
// Status + failure-count transition, based on the CURRENT live status.
if status == models.StatusUp {
s.FailureCount = 0
s.Status = models.StatusUp
} else {
if s.FailureCount <= s.MaxRetries {
s.FailureCount++
}
if s.FailureCount > s.MaxRetries {
if s.Status != status {
confirmedDown = true
}
s.Status = status
s.FailureCount = s.MaxRetries + 1
} else {
failedCheck = true
}
}
failCount = s.FailureCount
if s.Status != prev && prev != models.StatusPending {
s.StatusChangedAt = time.Now()
} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
s.StatusChangedAt = time.Now()
}
// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
if typ == "http" && s.CheckSSL && s.HasSSL {
days := int(time.Until(s.CertExpiry).Hours() / 24)
if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
sslWarnFire = true
sslDays = days
s.SentSSLWarning = true
} else if days > s.ExpiryThreshold {
s.SentSSLWarning = false
}
}
next = s.Status
changed = next != prev
})
if !exists || skipped {
return
}
e.recordCheck(snap.ID, latency, status == models.StatusUp)
if confirmedDown {
if errorReason != "" {
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
} else {
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
}
} else if failedCheck {
e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
}
if changed && prev != models.StatusPending {
e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
}
if sslWarnFire {
if !inMaint {
e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
} else {
e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
}
}
if prev == models.StatusUp && next == models.StatusLate {
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
}
if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
if inMaint {
e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
} else {
msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
if errorReason != "" {
msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
}
if typ == "push" {
msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
}
e.triggerAlert(alertID, "🚨 ALERT", msg)
}
}
if prev.IsBroken() && next == models.StatusUp {
downDur := ""
if !downSince.IsZero() {
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
}
e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
if !inMaint {
e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
}
}
if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
}
}
func (e *Engine) triggerAlert(alertID int, title, message string) {
if alertID <= 0 {
return
}
cfg, err := e.db.GetAlert(context.Background(), alertID)
if err != nil {
e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
return
}
provider := alert.GetProvider(cfg)
if provider != nil {
go func() {
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
defer cancel()
if err := provider.Send(ctx, title, message); err != nil {
e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
e.recordAlertResult(alertID, false, err.Error())
} else {
e.recordAlertResult(alertID, true, "")
}
}()
}
}
func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
e.alertHealthMu.Lock()
defer e.alertHealthMu.Unlock()
h := e.alertHealth[alertID]
h.LastSendAt = time.Now()
h.LastSendOK = ok
h.SendCount++
if ok {
h.LastError = ""
} else {
h.LastError = errMsg
h.FailCount++
}
e.alertHealth[alertID] = h
// Persist so health survives restarts; DB IO off the alert path.
e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
AlertID: alertID,
LastSendAt: h.LastSendAt,
LastSendOK: h.LastSendOK,
LastError: h.LastError,
SendCount: h.SendCount,
FailCount: h.FailCount,
}})
}
func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
e.alertHealthMu.RLock()
defer e.alertHealthMu.RUnlock()
return e.alertHealth[alertID]
}
func (e *Engine) TestAlert(alertID int) error {
cfg, err := e.db.GetAlert(context.Background(), alertID)
if err != nil {
return fmt.Errorf("failed to load alert: %w", err)
}
provider := alert.GetProvider(cfg)
if provider == nil {
return fmt.Errorf("no provider for type %q", cfg.Type)
}
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
defer cancel()
err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
if err != nil {
e.recordAlertResult(alertID, false, err.Error())
return err
}
e.recordAlertResult(alertID, true, "")
e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
return nil
}
internal/monitor/alerts_test.go
+487
View File
@@ -0,0 +1,487 @@
package monitor
import (
"context"
"testing"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
// --- Group 1: State Machine ---
func TestHandleStatusChange_PendingToUp(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3, AlertID: 1},
SiteState: models.SiteState{Status: "PENDING"},
}
injectSite(e, site)
e.handleStatusChange(site, "UP", 200, 10*time.Millisecond, "")
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP, got %s", s.Status)
}
if s.FailureCount != 0 {
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) != 0 {
t.Error("expected no alert for PENDING→UP")
}
}
func TestHandleStatusChange_UpIncrementFailure(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3},
SiteState: models.SiteState{Status: "UP", FailureCount: 0},
}
injectSite(e, site)
e.handleStatusChange(site, "DOWN", 500, 0, "test error")
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP (under retry threshold), got %s", s.Status)
}
if s.FailureCount != 1 {
t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
}
}
func TestHandleStatusChange_UpToDown_ExceedsRetries(t *testing.T) {
ms := newMockStore()
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "discord", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2, AlertID: 1},
SiteState: models.SiteState{Status: "UP", FailureCount: 2},
}
injectSite(e, site)
e.handleStatusChange(site, "DOWN", 500, 0, "test error")
s, _ := getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected DOWN, got %s", s.Status)
}
if s.FailureCount != 3 {
t.Errorf("expected FailureCount 3, got %d", s.FailureCount)
}
waitAsync()
calls := ms.getAlertCallsSnapshot()
if len(calls) == 0 || calls[0] != 1 {
t.Errorf("expected alert call for alertID 1, got %v", calls)
}
}
func TestHandleStatusChange_UpToDown_ZeroRetries(t *testing.T) {
ms := newMockStore()
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
SiteState: models.SiteState{Status: "UP", FailureCount: 0},
}
injectSite(e, site)
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
s, _ := getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected DOWN, got %s", s.Status)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) == 0 {
t.Error("expected alert on immediate DOWN")
}
}
func TestHandleStatusChange_DownToUp_Recovery(t *testing.T) {
ms := newMockStore()
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
SiteState: models.SiteState{Status: "DOWN", FailureCount: 4},
}
injectSite(e, site)
e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP, got %s", s.Status)
}
if s.FailureCount != 0 {
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) == 0 {
t.Error("expected recovery alert")
}
}
func TestHandleStatusChange_DownStaysDown(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2},
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3},
}
injectSite(e, site)
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
s, _ := getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected DOWN, got %s", s.Status)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) != 0 {
t.Error("expected no re-alert for already DOWN")
}
}
func TestHandleStatusChange_SSLExpired(t *testing.T) {
ms := newMockStore()
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.handleStatusChange(site, "SSL EXP", 0, 0, "SSL certificate expired")
s, _ := getSite(e, 1)
if s.Status != "SSL EXP" {
t.Errorf("expected SSL EXP, got %s", s.Status)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) == 0 {
t.Error("expected alert on SSL EXP")
}
}
func TestHandleStatusChange_AlertSuppressedMaintenance(t *testing.T) {
ms := newMockStore()
ms.maintenance[1] = true
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.refreshMaintenanceCache(context.Background())
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
s, _ := getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected DOWN, got %s", s.Status)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) != 0 {
t.Error("expected no alert during maintenance")
}
logs := e.GetLogs()
found := false
for _, l := range logs {
if containsStr(l.Message, "suppressed") {
found = true
break
}
}
if !found {
t.Error("expected log mentioning suppressed")
}
}
func TestHandleStatusChange_RecoverySuppressedMaintenance(t *testing.T) {
ms := newMockStore()
ms.maintenance[1] = true
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
SiteState: models.SiteState{Status: "DOWN"},
}
injectSite(e, site)
e.refreshMaintenanceCache(context.Background())
e.handleStatusChange(site, "UP", 200, 0, "")
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP, got %s", s.Status)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) != 0 {
t.Error("expected no alert during maintenance recovery")
}
}
func TestHandleStatusChange_SSLWarning(t *testing.T) {
ms := newMockStore()
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
}
injectSite(e, site)
e.handleStatusChange(site, "UP", 200, 0, "")
s, _ := getSite(e, 1)
if !s.SentSSLWarning {
t.Error("expected SentSSLWarning=true")
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) == 0 {
t.Error("expected SSL warning alert")
}
}
func TestHandleStatusChange_SSLWarningNotRepeated(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
}
injectSite(e, site)
e.handleStatusChange(site, "UP", 200, 0, "")
waitAsync()
if len(ms.getAlertCallsSnapshot()) != 0 {
t.Error("expected no repeat SSL warning")
}
}
func TestHandleStatusChange_SSLWarningReset(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30},
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(60 * 24 * time.Hour)},
}
injectSite(e, site)
e.handleStatusChange(site, "UP", 200, 0, "")
s, _ := getSite(e, 1)
if s.SentSSLWarning {
t.Error("expected SentSSLWarning reset to false")
}
}
func TestHandleStatusChange_SSLWarningSuppressedMaint(t *testing.T) {
ms := newMockStore()
ms.maintenance[1] = true
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
}
injectSite(e, site)
e.refreshMaintenanceCache(context.Background())
e.handleStatusChange(site, "UP", 200, 0, "")
s, _ := getSite(e, 1)
if !s.SentSSLWarning {
t.Error("expected SentSSLWarning=true even in maintenance")
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) != 0 {
t.Error("expected no alert during maintenance")
}
}
func TestHandleStatusChange_InactiveEngine(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.SetActive(false)
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Error("expected no state change when inactive")
}
}
// --- Group 10: liveState merge (lost-update race) ---
// A pause that lands while a check is in flight must survive the check's
// write-back. The old code snapshotted the site, ran the check, then wrote the
// whole stale struct back — reverting the pause.
func TestHandleStatusChange_PauseDuringCheckSurvives(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
// `site` is the stale snapshot the check ran against (Paused=false).
// Meanwhile the user pauses the monitor.
e.ToggleSitePause(1)
// Check completes and folds its result in using the stale snapshot.
e.handleStatusChange(site, "DOWN", 500, 0, "boom")
s, _ := getSite(e, 1)
if !s.Paused {
t.Error("pause was reverted by a stale check write-back")
}
if s.Status != "DOWN" {
t.Errorf("expected check result still applied (DOWN), got %s", s.Status)
}
}
// A config edit that lands while a check is in flight must survive; the check
// must not resurrect the old config from its snapshot.
func TestHandleStatusChange_ConfigEditDuringCheckSurvives(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com", Type: "http", MaxRetries: 0, Interval: 30},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
// Config changes mid-check.
e.UpdateSiteConfig(models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Type: "http", Interval: 60})
// Stale check (ran against http://old.com) folds its result in.
e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
s, _ := getSite(e, 1)
if s.URL != "http://new.com" {
t.Errorf("config edit reverted: URL=%s", s.URL)
}
if s.Interval != 60 {
t.Errorf("config edit reverted: Interval=%d", s.Interval)
}
}
// The classic push false-DOWN: a heartbeat marks the monitor UP while a
// staleness evaluation (computed from the older LastCheck) is mid-flight.
// The stale DOWN must not overwrite the fresh heartbeat.
func TestHandleStatusChange_HeartbeatNotOverwrittenByStaleDown(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
// Snapshot the engine would have taken before evaluating staleness:
// LastCheck is old, so checkPush decided "DOWN".
snap := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Token: "tok", Interval: 10},
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
}
injectSite(e, snap)
// A heartbeat lands first, advancing LastCheck and confirming UP.
if !e.RecordHeartbeat("tok") {
t.Fatal("heartbeat rejected")
}
// Now the in-flight stale evaluation tries to write DOWN.
e.handleStatusChange(snap, "DOWN", 0, 0, "heartbeat missed")
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("stale DOWN overwrote a fresh heartbeat: status=%s", s.Status)
}
}
// A check result for a site removed mid-check must be dropped, not recreate it.
func TestHandleStatusChange_RemovedSiteDropped(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.RemoveSite(1)
e.handleStatusChange(site, "DOWN", 500, 0, "boom")
if _, ok := getSite(e, 1); ok {
t.Error("removed site was recreated by a late check write-back")
}
}
// --- Group 12: Phase 3 engine correctness ---
// PENDING→DOWN must honor MaxRetries instead of alerting on first failure.
func TestHandleStatusChange_PendingRetriesBeforeDown(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "new-monitor", MaxRetries: 2},
SiteState: models.SiteState{Status: "PENDING"},
}
injectSite(e, site)
e.handleStatusChange(site, "DOWN", 0, 0, "timeout")
s, _ := getSite(e, 1)
if s.Status != "PENDING" {
t.Errorf("expected PENDING during retry, got %s", s.Status)
}
if s.FailureCount != 1 {
t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
}
e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
s, _ = getSite(e, 1)
if s.Status != "PENDING" {
t.Errorf("expected PENDING during retry 2, got %s", s.Status)
}
e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
s, _ = getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
}
}
// LATE→DOWN must also honor MaxRetries.
func TestHandleStatusChange_LateRetriesBeforeDown(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push-mon", MaxRetries: 1},
SiteState: models.SiteState{Status: "LATE"},
}
injectSite(e, site)
e.handleStatusChange(site, "DOWN", 0, 0, "missed heartbeat")
s, _ := getSite(e, 1)
if s.Status != "LATE" {
t.Errorf("expected LATE during retry, got %s", s.Status)
}
e.handleStatusChange(s, "DOWN", 0, 0, "missed heartbeat")
s, _ = getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
}
}
internal/monitor/checks.go
+315
View File
@@ -0,0 +1,315 @@
package monitor
import (
"context"
"fmt"
"math/rand/v2"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func (e *Engine) RecordHeartbeat(token string) bool {
if !e.IsActive() {
return false
}
e.mu.RLock()
targetID, ok := e.tokenIndex[token]
e.mu.RUnlock()
if !ok {
return false
}
var (
prevStatus models.Status
name string
alertID int
downSince time.Time
)
_, exists := e.applyState(targetID, func(s *models.Site) {
prevStatus = s.Status
name = s.Name
alertID = s.AlertID
downSince = s.StatusChangedAt // captured before mutation = when it went down
s.LastCheck = time.Now()
s.Status = models.StatusUp
s.FailureCount = 0
s.Latency = 0
s.LastError = ""
s.LastSuccessAt = time.Now()
if prevStatus != models.StatusUp {
s.StatusChangedAt = time.Now()
}
})
if !exists {
return false
}
switch prevStatus {
case models.StatusPending:
e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
case models.StatusLate:
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
case models.StatusStale:
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
case models.StatusDown:
downDur := ""
if !downSince.IsZero() {
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
}
e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
}
e.recordCheck(targetID, 0, true)
if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
}
return true
}
func (e *Engine) getRecheckChan(id int) chan struct{} {
e.recheckMu.Lock()
defer e.recheckMu.Unlock()
ch, ok := e.recheck[id]
if !ok {
ch = make(chan struct{}, 1)
e.recheck[id] = ch
}
return ch
}
func (e *Engine) signalRecheck(id int) {
ch := e.getRecheckChan(id)
select {
case ch <- struct{}{}:
default:
}
}
func (e *Engine) monitorRoutine(ctx context.Context, id int) {
recheckCh := e.getRecheckChan(id)
// Stagger initial check to avoid thundering herd on startup
stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
select {
case <-time.After(stagger):
case <-ctx.Done():
return
}
e.checkByID(ctx, id)
for {
select {
case <-ctx.Done():
return
default:
}
if !e.IsActive() {
select {
case <-time.After(pollInterval):
case <-ctx.Done():
return
case <-recheckCh:
}
continue
}
e.mu.RLock()
site, exists := e.liveState[id]
e.mu.RUnlock()
if !exists {
return
}
if site.Paused {
select {
case <-time.After(pollInterval):
case <-ctx.Done():
return
case <-recheckCh:
}
continue
}
interval := site.Interval
if interval < minCheckInterval {
interval = minCheckInterval
}
jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
select {
case <-time.After(time.Duration(interval)*time.Second + jitter):
case <-ctx.Done():
return
case <-recheckCh:
}
e.checkByID(ctx, id)
}
}
func (e *Engine) checkByID(ctx context.Context, id int) {
if !e.IsActive() {
return
}
e.mu.RLock()
site, exists := e.liveState[id]
e.mu.RUnlock()
if !exists || site.Paused {
return
}
switch site.Type {
case "push":
e.checkPush(ctx, site)
case "group":
e.checkGroup(ctx, site)
default:
result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
updatedSite := site
updatedSite.HasSSL = result.HasSSL
updatedSite.CertExpiry = result.CertExpiry
updatedSite.Latency = time.Duration(result.LatencyNs)
updatedSite.LastCheck = time.Now()
e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
}
}
func (e *Engine) checkPush(_ context.Context, site models.Site) {
if site.Status == models.StatusPending {
return
}
interval := time.Duration(site.Interval) * time.Second
grace := interval / 2
if grace < minPushGrace {
grace = minPushGrace
}
overdue := site.LastCheck.Add(interval)
staleMark := overdue.Add(grace / 2)
graceEnd := overdue.Add(grace)
now := time.Now()
if now.After(graceEnd) {
if site.Status != models.StatusDown {
e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
}
} else if now.After(staleMark) {
if site.Status != models.StatusStale {
e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
}
} else if now.After(overdue) {
if site.Status != models.StatusLate {
e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
}
}
}
func (e *Engine) checkGroup(_ context.Context, site models.Site) {
e.mu.RLock()
status := models.StatusUp
hasChildren := false
for _, child := range e.liveState {
if child.ParentID != site.ID || child.Type == "group" {
continue
}
hasChildren = true
if child.Paused || e.isInMaintenance(child.ID) {
continue
}
if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
status = models.StatusDown
} else if child.Status == models.StatusStale && status != models.StatusDown {
status = models.StatusStale
} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
status = models.StatusLate
} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
status = models.StatusPending
}
}
e.mu.RUnlock()
if !hasChildren {
status = models.StatusPending
}
e.applyState(site.ID, func(s *models.Site) {
s.Status = status
})
e.recordCheck(site.ID, 0, !status.IsBroken())
}
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
}
// SetAggStrategy must be called before Start: the field is read by the probe
// aggregation path without synchronization.
func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
e.aggStrategy = strategy
}
func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
e.mu.RLock()
site, exists := e.liveState[siteID]
e.mu.RUnlock()
if !exists {
return
}
staleAfter := time.Duration(site.Interval) * time.Second * 3
if staleAfter < time.Minute {
staleAfter = time.Minute
}
now := time.Now()
e.probeResultsMu.Lock()
if e.probeResults[siteID] == nil {
e.probeResults[siteID] = make(map[string]NodeResult)
}
e.probeResults[siteID][nodeID] = NodeResult{
NodeID: nodeID,
IsUp: isUp,
LatencyNs: latencyNs,
CheckedAt: now,
ErrorReason: errorReason,
}
results := make([]NodeResult, 0, len(e.probeResults[siteID]))
for id, r := range e.probeResults[siteID] {
if now.Sub(r.CheckedAt) > staleAfter {
delete(e.probeResults[siteID], id)
continue
}
results = append(results, r)
}
e.probeResultsMu.Unlock()
aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
probeStatus := models.StatusUp
if !aggUp {
probeStatus = models.StatusDown
}
updatedSite := site
updatedSite.Latency = time.Duration(avgLatency)
updatedSite.LastCheck = time.Now()
e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
}
func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
e.probeResultsMu.RLock()
defer e.probeResultsMu.RUnlock()
src := e.probeResults[siteID]
cp := make(map[string]NodeResult, len(src))
for k, v := range src {
cp[k] = v
}
return cp
}
internal/monitor/checks_test.go
+389
View File
@@ -0,0 +1,389 @@
package monitor
import (
"context"
"testing"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
// --- Group 2: Heartbeat ---
func TestRecordHeartbeat_ValidToken(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123"},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
if !e.RecordHeartbeat("abc123") {
t.Error("expected true for valid token")
}
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP, got %s", s.Status)
}
if time.Since(s.LastCheck) > time.Second {
t.Error("expected LastCheck to be recent")
}
}
func TestRecordHeartbeat_RecoveryFromDown(t *testing.T) {
ms := newMockStore()
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123", AlertID: 1},
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3},
}
injectSite(e, site)
if !e.RecordHeartbeat("abc123") {
t.Error("expected true")
}
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP, got %s", s.Status)
}
if s.FailureCount != 0 {
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
}
waitAsync()
if len(ms.getAlertCallsSnapshot()) == 0 {
t.Error("expected recovery alert")
}
}
func TestRecordHeartbeat_UnknownToken(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
if e.RecordHeartbeat("unknown") {
t.Error("expected false for unknown token")
}
}
func TestRecordHeartbeat_InactiveEngine(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Type: "push", Token: "abc123"},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.SetActive(false)
if e.RecordHeartbeat("abc123") {
t.Error("expected false when inactive")
}
}
// --- Group 3: Push Deadline ---
func TestCheckPush_DeadlineMissed(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 10, MaxRetries: 0},
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
}
injectSite(e, site)
e.checkPush(context.Background(), site)
s, _ := getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected DOWN after missed deadline, got %s", s.Status)
}
}
func TestCheckPush_OverdueBecomesLate(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-310 * time.Second)},
}
injectSite(e, site)
e.checkPush(context.Background(), site)
s, _ := getSite(e, 1)
if s.Status != "LATE" {
t.Errorf("expected LATE when overdue but within grace, got %s", s.Status)
}
}
func TestCheckPush_OverdueBecomesStale(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
// interval=300, grace=150 (300/2), staleMark=overdue+75
// at 380s: past staleMark(375) but before graceEnd(450)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-380 * time.Second)},
}
injectSite(e, site)
e.checkPush(context.Background(), site)
s, _ := getSite(e, 1)
if s.Status != "STALE" {
t.Errorf("expected STALE when past midpoint of grace, got %s", s.Status)
}
}
func TestCheckPush_WithinDeadline(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now()},
}
injectSite(e, site)
e.checkPush(context.Background(), site)
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP, got %s", s.Status)
}
}
func TestCheckPush_PendingStaysPending(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
SiteState: models.SiteState{Status: "PENDING"},
}
injectSite(e, site)
e.checkPush(context.Background(), site)
s, _ := getSite(e, 1)
if s.Status != "PENDING" {
t.Errorf("expected PENDING to stay until first heartbeat, got %s", s.Status)
}
}
// --- Group 4: Group Checks ---
func TestCheckGroup_AllChildrenUp(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
SiteState: models.SiteState{Status: "PENDING"},
}
child1 := models.Site{
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
SiteState: models.SiteState{Status: "UP"},
}
child2 := models.Site{
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, group)
injectSite(e, child1)
injectSite(e, child2)
e.checkGroup(context.Background(), group)
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected group UP, got %s", s.Status)
}
}
func TestCheckGroup_OneChildDown(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
SiteState: models.SiteState{Status: "UP"},
}
child1 := models.Site{
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
SiteState: models.SiteState{Status: "UP"},
}
child2 := models.Site{
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
SiteState: models.SiteState{Status: "DOWN"},
}
injectSite(e, group)
injectSite(e, child1)
injectSite(e, child2)
e.checkGroup(context.Background(), group)
s, _ := getSite(e, 1)
if s.Status != "DOWN" {
t.Errorf("expected group DOWN, got %s", s.Status)
}
}
func TestCheckGroup_PausedChildIgnored(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
}
child1 := models.Site{
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
SiteState: models.SiteState{Status: "UP"},
}
child2 := models.Site{
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
SiteState: models.SiteState{Status: "DOWN"},
}
injectSite(e, group)
injectSite(e, child1)
injectSite(e, child2)
e.checkGroup(context.Background(), group)
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP (paused child ignored), got %s", s.Status)
}
}
func TestCheckGroup_MaintenanceChildIgnored(t *testing.T) {
ms := newMockStore()
ms.maintenance[3] = true
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
}
child1 := models.Site{
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
SiteState: models.SiteState{Status: "UP"},
}
child2 := models.Site{
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
SiteState: models.SiteState{Status: "DOWN"},
}
injectSite(e, group)
injectSite(e, child1)
injectSite(e, child2)
e.refreshMaintenanceCache(context.Background())
e.checkGroup(context.Background(), group)
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP (maint child ignored), got %s", s.Status)
}
}
func TestCheckGroup_NoChildren(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, group)
e.checkGroup(context.Background(), group)
s, _ := getSite(e, 1)
if s.Status != "PENDING" {
t.Errorf("expected PENDING for no children, got %s", s.Status)
}
}
// Groups must not auto-pause when all children are paused — that creates a
// one-way trap because monitorRoutine skips paused sites.
func TestCheckGroup_AllPausedNoAutoFreeze(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
SiteState: models.SiteState{Status: "UP"},
}
child1 := models.Site{
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1, Paused: true},
SiteState: models.SiteState{Status: "UP"},
}
child2 := models.Site{
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, group)
injectSite(e, child1)
injectSite(e, child2)
e.checkGroup(context.Background(), group)
s, _ := getSite(e, 1)
if s.Paused {
t.Error("group must not auto-pause when all children are paused")
}
}
// Dead probe results must be expired so they don't poison aggregation.
func TestIngestProbeResult_ExpiresStaleProbes(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", Interval: 30},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.probeResultsMu.Lock()
e.probeResults[1] = map[string]NodeResult{
"dead-probe": {
NodeID: "dead-probe",
IsUp: false,
CheckedAt: time.Now().Add(-10 * time.Minute),
},
}
e.probeResultsMu.Unlock()
e.IngestProbeResult("live-probe", 1, 5000, true, "")
e.probeResultsMu.RLock()
_, deadExists := e.probeResults[1]["dead-probe"]
_, liveExists := e.probeResults[1]["live-probe"]
e.probeResultsMu.RUnlock()
if deadExists {
t.Error("stale probe result should have been expired")
}
if !liveExists {
t.Error("live probe result should still exist")
}
}
// RemoveSite must clean up probeResults.
func TestRemoveSite_CleansProbeResults(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.probeResultsMu.Lock()
e.probeResults[1] = map[string]NodeResult{
"node-a": {NodeID: "node-a", IsUp: true, CheckedAt: time.Now()},
}
e.probeResultsMu.Unlock()
e.RemoveSite(1)
e.probeResultsMu.RLock()
defer e.probeResultsMu.RUnlock()
if _, exists := e.probeResults[1]; exists {
t.Error("probe results should be cleaned up after RemoveSite")
}
}
internal/monitor/maintenance.go
+70
View File
@@ -0,0 +1,70 @@
package monitor
import (
"context"
"fmt"
"time"
)
func (e *Engine) maintenancePruner(ctx context.Context) {
ticker := time.NewTicker(maintPruneInterval)
defer ticker.Stop()
e.pruneMaintenanceWindows(ctx)
for {
select {
case <-ticker.C:
e.pruneMaintenanceWindows(ctx)
case <-ctx.Done():
return
}
}
}
func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
if err != nil {
e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
return
}
if pruned > 0 {
e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
}
}
func (e *Engine) isInMaintenance(monitorID int) bool {
e.maintCacheMu.RLock()
defer e.maintCacheMu.RUnlock()
return e.maintCache[monitorID]
}
func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
windows, err := e.db.GetActiveMaintenanceWindows(ctx)
if err != nil {
return
}
directMaint := make(map[int]bool)
var globalMaint bool
for _, w := range windows {
if w.MonitorID == 0 {
globalMaint = true
} else {
directMaint[w.MonitorID] = true
}
}
resolved := make(map[int]bool)
e.mu.RLock()
for id, site := range e.liveState {
if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
resolved[id] = true
}
}
e.mu.RUnlock()
e.maintCacheMu.Lock()
e.maintCache = resolved
e.maintCacheMu.Unlock()
}
internal/monitor/maintenance_test.go
+53
View File
@@ -0,0 +1,53 @@
package monitor
import (
"context"
"testing"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
// Maintenance cache resolves parent relationships correctly.
func TestIsInMaintenance_UsesCache(t *testing.T) {
ms := newMockStore()
ms.maintenance[10] = true // direct maintenance on group
e := newTestEngine(ms)
group := models.Site{
SiteConfig: models.SiteConfig{ID: 10, Name: "group", Type: "group"},
SiteState: models.SiteState{Status: "UP"},
}
child := models.Site{
SiteConfig: models.SiteConfig{ID: 20, Name: "child", Type: "http", ParentID: 10},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, group)
injectSite(e, child)
e.refreshMaintenanceCache(context.Background())
if !e.isInMaintenance(10) {
t.Error("group should be in maintenance (direct)")
}
if !e.isInMaintenance(20) {
t.Error("child should be in maintenance (parent)")
}
if e.isInMaintenance(99) {
t.Error("unknown monitor should not be in maintenance")
}
}
// Global maintenance (monitor_id=0) applies to all monitors.
func TestIsInMaintenance_GlobalMaintenance(t *testing.T) {
ms := newMockStore()
ms.maintenance[0] = true
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.refreshMaintenanceCache(context.Background())
if !e.isInMaintenance(1) {
t.Error("all monitors should be in maintenance during global window")
}
}
internal/monitor/monitor.go
-781
View File
@@ -4,14 +4,12 @@ import (
"context"
"crypto/tls"
"fmt"
"math/rand/v2"
"net/http"
"regexp"
"strings"
"sync"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/store"
)
@@ -28,14 +26,6 @@ const (
dbPruneInterval = 10 * time.Minute
)
type AlertHealth struct {
LastSendAt time.Time
LastSendOK bool
LastError string
SendCount int
FailCount int
}
type Engine struct {
mu sync.RWMutex
liveState map[int]models.Site
@@ -261,26 +251,6 @@ func (e *Engine) InitLogs() {
e.logStore = entries
}
// InitAlertHealth restores persisted alert send health so the dashboard shows real
// "last sent" / health state on startup instead of resetting every channel to "never".
func (e *Engine) InitAlertHealth() {
records, err := e.db.LoadAlertHealth(context.Background())
if err != nil {
return
}
e.alertHealthMu.Lock()
defer e.alertHealthMu.Unlock()
for id, r := range records {
e.alertHealth[id] = AlertHealth{
LastSendAt: r.LastSendAt,
LastSendOK: r.LastSendOK,
LastError: r.LastError,
SendCount: r.SendCount,
FailCount: r.FailCount,
}
}
}
func (e *Engine) GetLogs() []models.LogEntry {
e.logMu.RLock()
defer e.logMu.RUnlock()
@@ -289,123 +259,6 @@ func (e *Engine) GetLogs() []models.LogEntry {
return logs
}
func (e *Engine) SetActive(active bool) {
e.activeMu.Lock()
defer e.activeMu.Unlock()
if e.isActive != active {
e.isActive = active
status := "RESUMED (Active)"
if !active {
status = "PAUSED (Passive)"
}
e.AddLog(fmt.Sprintf("Engine %s", status))
}
}
func (e *Engine) IsActive() bool {
e.activeMu.RLock()
defer e.activeMu.RUnlock()
return e.isActive
}
func (e *Engine) GetAllSites() []models.Site {
e.mu.RLock()
defer e.mu.RUnlock()
sites := make([]models.Site, 0, len(e.liveState))
for _, s := range e.liveState {
sites = append(sites, s)
}
return sites
}
func (e *Engine) GetLiveState() map[int]models.Site {
e.mu.RLock()
defer e.mu.RUnlock()
cp := make(map[int]models.Site, len(e.liveState))
for k, v := range e.liveState {
cp[k] = v
}
return cp
}
func (e *Engine) RecordHeartbeat(token string) bool {
if !e.IsActive() {
return false
}
e.mu.RLock()
targetID, ok := e.tokenIndex[token]
e.mu.RUnlock()
if !ok {
return false
}
var (
prevStatus models.Status
name string
alertID int
downSince time.Time
)
_, exists := e.applyState(targetID, func(s *models.Site) {
prevStatus = s.Status
name = s.Name
alertID = s.AlertID
downSince = s.StatusChangedAt // captured before mutation = when it went down
s.LastCheck = time.Now()
s.Status = models.StatusUp
s.FailureCount = 0
s.Latency = 0
s.LastError = ""
s.LastSuccessAt = time.Now()
if prevStatus != models.StatusUp {
s.StatusChangedAt = time.Now()
}
})
if !exists {
return false
}
switch prevStatus {
case models.StatusPending:
e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
case models.StatusLate:
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
case models.StatusStale:
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
case models.StatusDown:
downDur := ""
if !downSince.IsZero() {
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
}
e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
}
e.recordCheck(targetID, 0, true)
if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
}
return true
}
func (e *Engine) addToTokenIndex(site models.Site) {
if site.Type == "push" && site.Token != "" {
e.tokenIndex[site.Token] = site.ID
}
}
func (e *Engine) removeFromTokenIndex(id int) {
for token, sid := range e.tokenIndex {
if sid == id {
delete(e.tokenIndex, token)
return
}
}
}
func (e *Engine) Start(ctx context.Context) {
// e.cancel is invoked by Stop() to drain and halt the writer; gosec can't
// trace the cross-method call, and cancelling the parent reaps this child
@@ -496,637 +349,3 @@ func (e *Engine) Start(ctx context.Context) {
e.maintenancePruner(ctx)
}()
}
func (e *Engine) maintenancePruner(ctx context.Context) {
ticker := time.NewTicker(maintPruneInterval)
defer ticker.Stop()
e.pruneMaintenanceWindows(ctx)
for {
select {
case <-ticker.C:
e.pruneMaintenanceWindows(ctx)
case <-ctx.Done():
return
}
}
}
func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
if err != nil {
e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
return
}
if pruned > 0 {
e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
}
}
func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
e.mu.Lock()
if existing, ok := e.liveState[cfg.ID]; ok {
e.removeFromTokenIndex(cfg.ID)
existing.SiteConfig = cfg
e.liveState[cfg.ID] = existing
e.addToTokenIndex(existing)
}
e.mu.Unlock()
e.signalRecheck(cfg.ID)
}
func (e *Engine) getRecheckChan(id int) chan struct{} {
e.recheckMu.Lock()
defer e.recheckMu.Unlock()
ch, ok := e.recheck[id]
if !ok {
ch = make(chan struct{}, 1)
e.recheck[id] = ch
}
return ch
}
func (e *Engine) signalRecheck(id int) {
ch := e.getRecheckChan(id)
select {
case ch <- struct{}{}:
default:
}
}
func (e *Engine) RemoveSite(id int) {
e.mu.Lock()
e.removeFromTokenIndex(id)
delete(e.liveState, id)
e.mu.Unlock()
e.removeHistory(id)
e.probeResultsMu.Lock()
delete(e.probeResults, id)
e.probeResultsMu.Unlock()
e.recheckMu.Lock()
delete(e.recheck, id)
e.recheckMu.Unlock()
}
func (e *Engine) ToggleSitePause(id int) bool {
var (
paused bool
name string
)
_, ok := e.applyState(id, func(s *models.Site) {
s.Paused = !s.Paused
paused = s.Paused
name = s.Name
})
if !ok {
return false
}
if paused {
e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
} else {
e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
}
return paused
}
func (e *Engine) monitorRoutine(ctx context.Context, id int) {
recheckCh := e.getRecheckChan(id)
// Stagger initial check to avoid thundering herd on startup
stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
select {
case <-time.After(stagger):
case <-ctx.Done():
return
}
e.checkByID(ctx, id)
for {
select {
case <-ctx.Done():
return
default:
}
if !e.IsActive() {
select {
case <-time.After(pollInterval):
case <-ctx.Done():
return
case <-recheckCh:
}
continue
}
e.mu.RLock()
site, exists := e.liveState[id]
e.mu.RUnlock()
if !exists {
return
}
if site.Paused {
select {
case <-time.After(pollInterval):
case <-ctx.Done():
return
case <-recheckCh:
}
continue
}
interval := site.Interval
if interval < minCheckInterval {
interval = minCheckInterval
}
jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
select {
case <-time.After(time.Duration(interval)*time.Second + jitter):
case <-ctx.Done():
return
case <-recheckCh:
}
e.checkByID(ctx, id)
}
}
// applyState atomically reads, mutates, and writes back the live entry for id.
// The mutator runs under the engine write lock and receives a pointer to the
// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
// never clobbered by a stale snapshot. The mutator must only touch runtime /
// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
// are owned by UpdateSiteConfig and must not be written here. Returns the
// post-mutation copy and whether the site still exists.
func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
e.mu.Lock()
defer e.mu.Unlock()
cur, ok := e.liveState[id]
if !ok {
return models.Site{}, false
}
mutate(&cur)
e.liveState[id] = cur
return cur, true
}
func (e *Engine) checkByID(ctx context.Context, id int) {
if !e.IsActive() {
return
}
e.mu.RLock()
site, exists := e.liveState[id]
e.mu.RUnlock()
if !exists || site.Paused {
return
}
switch site.Type {
case "push":
e.checkPush(ctx, site)
case "group":
e.checkGroup(ctx, site)
default:
result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
updatedSite := site
updatedSite.HasSSL = result.HasSSL
updatedSite.CertExpiry = result.CertExpiry
updatedSite.Latency = time.Duration(result.LatencyNs)
updatedSite.LastCheck = time.Now()
e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
}
}
func (e *Engine) checkPush(_ context.Context, site models.Site) {
if site.Status == models.StatusPending {
return
}
interval := time.Duration(site.Interval) * time.Second
grace := interval / 2
if grace < minPushGrace {
grace = minPushGrace
}
overdue := site.LastCheck.Add(interval)
staleMark := overdue.Add(grace / 2)
graceEnd := overdue.Add(grace)
now := time.Now()
if now.After(graceEnd) {
if site.Status != models.StatusDown {
e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
}
} else if now.After(staleMark) {
if site.Status != models.StatusStale {
e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
}
} else if now.After(overdue) {
if site.Status != models.StatusLate {
e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
}
}
}
// handleStatusChange folds a check result into the live state. snap is the
// stale snapshot the check ran against; the actual mutation is applied onto the
// CURRENT live entry via applyState, so a concurrent pause / config edit /
// heartbeat is never reverted by this write. Logs and alerts are emitted after
// the lock is released, off the critical section.
func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
if !e.IsActive() {
return
}
inMaint := e.isInMaintenance(snap.ID)
status := models.Status(rawStatus)
var (
prev, next models.Status
name, typ string
alertID int
failCount, maxRetries int
confirmedDown bool
failedCheck bool
downSince time.Time
sslWarnFire bool
sslDays int
skipped bool
changed bool
)
_, exists := e.applyState(snap.ID, func(s *models.Site) {
// A non-UP result computed from a stale snapshot must not override a
// heartbeat (or newer check) that landed while we were evaluating.
if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
skipped = true
return
}
prev = s.Status
name = s.Name
typ = s.Type
alertID = s.AlertID
maxRetries = s.MaxRetries
downSince = s.StatusChangedAt
// Fresh check results (measured by the run against snap).
s.StatusCode = code
s.Latency = snap.Latency
s.LastCheck = snap.LastCheck
s.HasSSL = snap.HasSSL
s.CertExpiry = snap.CertExpiry
s.LastError = errorReason
if status == models.StatusUp {
s.LastSuccessAt = time.Now()
s.LastError = ""
}
// Status + failure-count transition, based on the CURRENT live status.
if status == models.StatusUp {
s.FailureCount = 0
s.Status = models.StatusUp
} else {
if s.FailureCount <= s.MaxRetries {
s.FailureCount++
}
if s.FailureCount > s.MaxRetries {
if s.Status != status {
confirmedDown = true
}
s.Status = status
s.FailureCount = s.MaxRetries + 1
} else {
failedCheck = true
}
}
failCount = s.FailureCount
if s.Status != prev && prev != models.StatusPending {
s.StatusChangedAt = time.Now()
} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
s.StatusChangedAt = time.Now()
}
// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
if typ == "http" && s.CheckSSL && s.HasSSL {
days := int(time.Until(s.CertExpiry).Hours() / 24)
if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
sslWarnFire = true
sslDays = days
s.SentSSLWarning = true
} else if days > s.ExpiryThreshold {
s.SentSSLWarning = false
}
}
next = s.Status
changed = next != prev
})
if !exists || skipped {
return
}
e.recordCheck(snap.ID, latency, status == models.StatusUp)
if confirmedDown {
if errorReason != "" {
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
} else {
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
}
} else if failedCheck {
e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
}
if changed && prev != models.StatusPending {
e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
}
if sslWarnFire {
if !inMaint {
e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
} else {
e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
}
}
if prev == models.StatusUp && next == models.StatusLate {
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
}
if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
if inMaint {
e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
} else {
msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
if errorReason != "" {
msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
}
if typ == "push" {
msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
}
e.triggerAlert(alertID, "🚨 ALERT", msg)
}
}
if prev.IsBroken() && next == models.StatusUp {
downDur := ""
if !downSince.IsZero() {
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
}
e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
if !inMaint {
e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
}
}
if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
}
}
func (e *Engine) triggerAlert(alertID int, title, message string) {
if alertID <= 0 {
return
}
cfg, err := e.db.GetAlert(context.Background(), alertID)
if err != nil {
e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
return
}
provider := alert.GetProvider(cfg)
if provider != nil {
go func() {
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
defer cancel()
if err := provider.Send(ctx, title, message); err != nil {
e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
e.recordAlertResult(alertID, false, err.Error())
} else {
e.recordAlertResult(alertID, true, "")
}
}()
}
}
func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
e.alertHealthMu.Lock()
defer e.alertHealthMu.Unlock()
h := e.alertHealth[alertID]
h.LastSendAt = time.Now()
h.LastSendOK = ok
h.SendCount++
if ok {
h.LastError = ""
} else {
h.LastError = errMsg
h.FailCount++
}
e.alertHealth[alertID] = h
// Persist so health survives restarts; DB IO off the alert path.
e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
AlertID: alertID,
LastSendAt: h.LastSendAt,
LastSendOK: h.LastSendOK,
LastError: h.LastError,
SendCount: h.SendCount,
FailCount: h.FailCount,
}})
}
func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
e.alertHealthMu.RLock()
defer e.alertHealthMu.RUnlock()
return e.alertHealth[alertID]
}
func (e *Engine) TestAlert(alertID int) error {
cfg, err := e.db.GetAlert(context.Background(), alertID)
if err != nil {
return fmt.Errorf("failed to load alert: %w", err)
}
provider := alert.GetProvider(cfg)
if provider == nil {
return fmt.Errorf("no provider for type %q", cfg.Type)
}
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
defer cancel()
err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
if err != nil {
e.recordAlertResult(alertID, false, err.Error())
return err
}
e.recordAlertResult(alertID, true, "")
e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
return nil
}
func (e *Engine) isInMaintenance(monitorID int) bool {
e.maintCacheMu.RLock()
defer e.maintCacheMu.RUnlock()
return e.maintCache[monitorID]
}
func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
windows, err := e.db.GetActiveMaintenanceWindows(ctx)
if err != nil {
return
}
directMaint := make(map[int]bool)
var globalMaint bool
for _, w := range windows {
if w.MonitorID == 0 {
globalMaint = true
} else {
directMaint[w.MonitorID] = true
}
}
resolved := make(map[int]bool)
e.mu.RLock()
for id, site := range e.liveState {
if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
resolved[id] = true
}
}
e.mu.RUnlock()
e.maintCacheMu.Lock()
e.maintCache = resolved
e.maintCacheMu.Unlock()
}
func (e *Engine) GetDisplayStatus(site models.Site) string {
if site.Paused {
return "PAUSED"
}
if e.isInMaintenance(site.ID) {
return "MAINT"
}
return string(site.Status)
}
func (e *Engine) checkGroup(_ context.Context, site models.Site) {
e.mu.RLock()
status := models.StatusUp
hasChildren := false
for _, child := range e.liveState {
if child.ParentID != site.ID || child.Type == "group" {
continue
}
hasChildren = true
if child.Paused || e.isInMaintenance(child.ID) {
continue
}
if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
status = models.StatusDown
} else if child.Status == models.StatusStale && status != models.StatusDown {
status = models.StatusStale
} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
status = models.StatusLate
} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
status = models.StatusPending
}
}
e.mu.RUnlock()
if !hasChildren {
status = models.StatusPending
}
e.applyState(site.ID, func(s *models.Site) {
s.Status = status
})
e.recordCheck(site.ID, 0, !status.IsBroken())
}
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
}
// SetAggStrategy must be called before Start: the field is read by the probe
// aggregation path without synchronization.
func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
e.aggStrategy = strategy
}
func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
e.mu.RLock()
site, exists := e.liveState[siteID]
e.mu.RUnlock()
if !exists {
return
}
staleAfter := time.Duration(site.Interval) * time.Second * 3
if staleAfter < time.Minute {
staleAfter = time.Minute
}
now := time.Now()
e.probeResultsMu.Lock()
if e.probeResults[siteID] == nil {
e.probeResults[siteID] = make(map[string]NodeResult)
}
e.probeResults[siteID][nodeID] = NodeResult{
NodeID: nodeID,
IsUp: isUp,
LatencyNs: latencyNs,
CheckedAt: now,
ErrorReason: errorReason,
}
results := make([]NodeResult, 0, len(e.probeResults[siteID]))
for id, r := range e.probeResults[siteID] {
if now.Sub(r.CheckedAt) > staleAfter {
delete(e.probeResults[siteID], id)
continue
}
results = append(results, r)
}
e.probeResultsMu.Unlock()
aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
probeStatus := models.StatusUp
if !aggUp {
probeStatus = models.StatusDown
}
updatedSite := site
updatedSite.Latency = time.Duration(avgLatency)
updatedSite.LastCheck = time.Now()
e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
}
func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
e.probeResultsMu.RLock()
defer e.probeResultsMu.RUnlock()
src := e.probeResults[siteID]
cp := make(map[string]NodeResult, len(src))
for k, v := range src {
cp[k] = v
}
return cp
}
func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
if err != nil {
return nil
}
return changes
}
func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
if err != nil {
return nil
}
return changes
}
internal/monitor/monitor_test.go
-1030
View File
File diff suppressed because it is too large Load Diff
internal/monitor/sites.go
+158
View File
@@ -0,0 +1,158 @@
package monitor
import (
"context"
"fmt"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func (e *Engine) SetActive(active bool) {
e.activeMu.Lock()
defer e.activeMu.Unlock()
if e.isActive != active {
e.isActive = active
status := "RESUMED (Active)"
if !active {
status = "PAUSED (Passive)"
}
e.AddLog(fmt.Sprintf("Engine %s", status))
}
}
func (e *Engine) IsActive() bool {
e.activeMu.RLock()
defer e.activeMu.RUnlock()
return e.isActive
}
func (e *Engine) GetAllSites() []models.Site {
e.mu.RLock()
defer e.mu.RUnlock()
sites := make([]models.Site, 0, len(e.liveState))
for _, s := range e.liveState {
sites = append(sites, s)
}
return sites
}
func (e *Engine) GetLiveState() map[int]models.Site {
e.mu.RLock()
defer e.mu.RUnlock()
cp := make(map[int]models.Site, len(e.liveState))
for k, v := range e.liveState {
cp[k] = v
}
return cp
}
func (e *Engine) addToTokenIndex(site models.Site) {
if site.Type == "push" && site.Token != "" {
e.tokenIndex[site.Token] = site.ID
}
}
func (e *Engine) removeFromTokenIndex(id int) {
for token, sid := range e.tokenIndex {
if sid == id {
delete(e.tokenIndex, token)
return
}
}
}
func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
e.mu.Lock()
if existing, ok := e.liveState[cfg.ID]; ok {
e.removeFromTokenIndex(cfg.ID)
existing.SiteConfig = cfg
e.liveState[cfg.ID] = existing
e.addToTokenIndex(existing)
}
e.mu.Unlock()
e.signalRecheck(cfg.ID)
}
func (e *Engine) RemoveSite(id int) {
e.mu.Lock()
e.removeFromTokenIndex(id)
delete(e.liveState, id)
e.mu.Unlock()
e.removeHistory(id)
e.probeResultsMu.Lock()
delete(e.probeResults, id)
e.probeResultsMu.Unlock()
e.recheckMu.Lock()
delete(e.recheck, id)
e.recheckMu.Unlock()
}
func (e *Engine) ToggleSitePause(id int) bool {
var (
paused bool
name string
)
_, ok := e.applyState(id, func(s *models.Site) {
s.Paused = !s.Paused
paused = s.Paused
name = s.Name
})
if !ok {
return false
}
if paused {
e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
} else {
e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
}
return paused
}
// applyState atomically reads, mutates, and writes back the live entry for id.
// The mutator runs under the engine write lock and receives a pointer to the
// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
// never clobbered by a stale snapshot. The mutator must only touch runtime /
// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
// are owned by UpdateSiteConfig and must not be written here. Returns the
// post-mutation copy and whether the site still exists.
func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
e.mu.Lock()
defer e.mu.Unlock()
cur, ok := e.liveState[id]
if !ok {
return models.Site{}, false
}
mutate(&cur)
e.liveState[id] = cur
return cur, true
}
func (e *Engine) GetDisplayStatus(site models.Site) string {
if site.Paused {
return "PAUSED"
}
if e.isInMaintenance(site.ID) {
return "MAINT"
}
return string(site.Status)
}
func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
if err != nil {
return nil
}
return changes
}
func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
if err != nil {
return nil
}
return changes
}
internal/monitor/sites_test.go
+133
View File
@@ -0,0 +1,133 @@
package monitor
import (
"testing"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func TestUpdateSiteConfig_PreservesRuntime(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com"},
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3, Latency: 100 * time.Millisecond},
}
injectSite(e, site)
updated := models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Interval: 60}
e.UpdateSiteConfig(updated)
s, _ := getSite(e, 1)
if s.URL != "http://new.com" {
t.Errorf("expected URL updated, got %s", s.URL)
}
if s.Status != "DOWN" {
t.Errorf("expected Status preserved, got %s", s.Status)
}
if s.FailureCount != 3 {
t.Errorf("expected FailureCount preserved, got %d", s.FailureCount)
}
if s.Latency != 100*time.Millisecond {
t.Errorf("expected Latency preserved, got %v", s.Latency)
}
}
func TestRemoveSite_CleansUp(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "push", Token: "tok1"},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
e.recordCheck(1, 5*time.Millisecond, true)
e.RemoveSite(1)
if _, ok := getSite(e, 1); ok {
t.Error("expected site removed from liveState")
}
if e.RecordHeartbeat("tok1") {
t.Error("expected token removed from index")
}
if _, ok := e.GetHistory(1); ok {
t.Error("expected history removed")
}
}
func TestToggleSitePause(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "test"},
SiteState: models.SiteState{Status: "UP"},
}
injectSite(e, site)
paused := e.ToggleSitePause(1)
if !paused {
t.Error("expected paused=true after first toggle")
}
s, _ := getSite(e, 1)
if !s.Paused {
t.Error("expected Paused=true in state")
}
paused = e.ToggleSitePause(1)
if paused {
t.Error("expected paused=false after second toggle")
}
}
func TestToggleSitePause_NonexistentSite(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
if e.ToggleSitePause(999) {
t.Error("expected false for nonexistent site")
}
}
func TestGetAllSites_ReturnsCopy(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
injectSite(e, models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
SiteState: models.SiteState{Status: "UP"},
})
injectSite(e, models.Site{
SiteConfig: models.SiteConfig{ID: 2, Name: "s2"},
SiteState: models.SiteState{Status: "DOWN"},
})
sites := e.GetAllSites()
if len(sites) != 2 {
t.Fatalf("expected 2 sites, got %d", len(sites))
}
sites[0].Name = "mutated"
fresh := e.GetAllSites()
for _, s := range fresh {
if s.Name == "mutated" {
t.Error("GetAllSites returned reference, not copy")
}
}
}
func TestGetLiveState_ReturnsCopy(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
injectSite(e, models.Site{
SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
SiteState: models.SiteState{Status: "UP"},
})
state := e.GetLiveState()
state[1] = models.Site{SiteConfig: models.SiteConfig{Name: "mutated"}}
fresh := e.GetLiveState()
if fresh[1].Name == "mutated" {
t.Error("GetLiveState returned reference, not copy")
}
}
internal/store/sqlstore.go
-363
View File
@@ -5,7 +5,6 @@ import (
"crypto/rand"
"database/sql"
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
@@ -218,40 +217,6 @@ func (s *SQLStore) GetSiteByName(ctx context.Context, name string) (models.SiteC
return st, err
}
func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) {
decrypted, err := s.decryptSettings(raw)
if err != nil {
return nil, fmt.Errorf("decrypt settings: %w", err)
}
var m map[string]string
if err := json.Unmarshal([]byte(decrypted), &m); err != nil {
return nil, fmt.Errorf("unmarshal settings: %w", err)
}
return m, nil
}
func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) {
jsonBytes, err := json.Marshal(settings)
if err != nil {
return "", err
}
return s.encryptSettings(string(jsonBytes))
}
func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) {
var a models.AlertConfig
var settingsRaw string
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
if err != nil {
return a, err
}
a.Settings, err = s.unmarshalSettings(settingsRaw)
if err != nil {
return a, fmt.Errorf("alert %q: %w", name, err)
}
return a, nil
}
func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfig) (int, error) {
token := ""
if site.Type == "push" {
@@ -278,89 +243,6 @@ func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfi
return int(id), err
}
func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) {
stored, err := s.marshalSettings(settings)
if err != nil {
return 0, err
}
if s.dollar {
var id int
err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id)
return id, err
}
result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
if err != nil {
return 0, err
}
id, err := result.LastInsertId()
return int(id), err
}
func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) {
rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts")
if err != nil {
return nil, err
}
defer rows.Close()
var alerts []models.AlertConfig
for rows.Next() {
var a models.AlertConfig
var settingsRaw string
if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil {
return alerts, err
}
a.Settings, err = s.unmarshalSettings(settingsRaw)
if err != nil {
return alerts, fmt.Errorf("alert %q: %w", a.Name, err)
}
alerts = append(alerts, a)
}
return alerts, rows.Err()
}
func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) {
var a models.AlertConfig
var settingsRaw string
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
if err != nil {
return a, err
}
a.Settings, err = s.unmarshalSettings(settingsRaw)
if err != nil {
return a, fmt.Errorf("alert %d: %w", id, err)
}
return a, nil
}
func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error {
stored, err := s.marshalSettings(settings)
if err != nil {
return err
}
_, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
return err
}
func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error {
stored, err := s.marshalSettings(settings)
if err != nil {
return err
}
_, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id)
return err
}
func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error {
if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil {
return err
}
if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil {
return err
}
s.dialect.ResetSequenceOnEmpty(s.db, "alerts")
return nil
}
func (s *SQLStore) GetAllUsers(ctx context.Context) ([]models.User, error) {
rows, err := s.db.QueryContext(ctx, "SELECT id, username, public_key, role FROM users")
if err != nil {
@@ -393,85 +275,6 @@ func (s *SQLStore) DeleteUser(ctx context.Context, id int) error {
return err
}
func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error {
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"),
siteID, fromStatus, toStatus, errorReason)
return err
}
func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var changes []models.StateChange
for rows.Next() {
var sc models.StateChange
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
return changes, err
}
changes = append(changes, sc)
}
return changes, rows.Err()
}
func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since)
if err != nil {
return nil, err
}
defer rows.Close()
var changes []models.StateChange
for rows.Next() {
var sc models.StateChange
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
return changes, err
}
changes = append(changes, sc)
}
return changes, rows.Err()
}
func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error {
return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp)
}
// SaveCheckFromNode inserts a single check row. Retention is handled out of
// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot
// path a plain INSERT.
func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error {
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp)
return err
}
// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per
// site, across all sites, in one pass. Intended to run periodically.
func (s *SQLStore) PruneCheckHistory(ctx context.Context) error {
q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN (
SELECT id FROM (
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn
FROM check_history
) ranked WHERE rn > %d
)`, maxCheckHistory)
_, err := s.db.ExecContext(ctx, s.q(q))
return err
}
// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite
// rows per site. Generous so realistic SLA windows are unaffected; bounds the
// otherwise unbounded growth of a flapping monitor's history.
func (s *SQLStore) PruneStateChanges(ctx context.Context) error {
q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN (
SELECT id FROM (
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn
FROM state_changes
) ranked WHERE rn > %d
)`, maxStateChangesPerSite)
_, err := s.db.ExecContext(ctx, s.q(q))
return err
}
func (s *SQLStore) RegisterNode(ctx context.Context, node models.ProbeNode) error {
_, err := s.db.ExecContext(ctx, s.dialect.UpsertNodeSQL(), node.ID, node.Name, node.Region, node.Version)
return err
@@ -582,172 +385,6 @@ func (s *SQLStore) LoadLogs(ctx context.Context, limit int) ([]models.LogEntry,
return entries, rows.Err()
}
func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) {
result := make(map[int][]models.CheckRecord)
rows, err := s.db.QueryContext(ctx, s.q(`
SELECT site_id, latency_ns, is_up FROM (
SELECT site_id, latency_ns, is_up,
ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn
FROM check_history
) sub WHERE rn <= ?`), limit)
if err != nil {
return result, err
}
defer rows.Close()
for rows.Next() {
var r models.CheckRecord
if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil {
return result, err
}
result[r.SiteID] = append(result[r.SiteID], r)
}
for id, records := range result {
for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 {
records[i], records[j] = records[j], records[i]
}
result[id] = records
}
return result, rows.Err()
}
func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) {
var mw models.MaintenanceWindow
var endTime sql.NullTime
if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil {
return mw, err
}
if endTime.Valid {
mw.EndTime = endTime.Time
}
return mw, nil
}
func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC"))
if err != nil {
return nil, err
}
defer rows.Close()
var windows []models.MaintenanceWindow
for rows.Next() {
mw, err := s.scanMaintenanceWindow(rows)
if err != nil {
return windows, err
}
windows = append(windows, mw)
}
return windows, rows.Err()
}
func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit)
if err != nil {
return nil, err
}
defer rows.Close()
var windows []models.MaintenanceWindow
for rows.Next() {
mw, err := s.scanMaintenanceWindow(rows)
if err != nil {
return windows, err
}
windows = append(windows, mw)
}
return windows, rows.Err()
}
func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) {
var timeClause string
var args []interface{}
if endTime.IsZero() {
timeClause = "(end_time IS NULL OR end_time > ?)"
args = append(args, startTime)
} else {
timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?"
args = append(args, startTime, endTime)
}
var scopeClause string
if monitorID == 0 {
scopeClause = "1=1"
} else {
scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))"
args = append(args, monitorID, monitorID)
}
query := fmt.Sprintf(
"SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time",
timeClause, scopeClause,
)
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, err
}
defer rows.Close()
var windows []models.MaintenanceWindow
for rows.Next() {
mw, err := s.scanMaintenanceWindow(rows)
if err != nil {
return nil, err
}
windows = append(windows, mw)
}
return windows, rows.Err()
}
func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error {
if mw.StartTime.IsZero() {
mw.StartTime = time.Now()
}
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"),
mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy)
return err
}
func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error {
_, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id)
return err
}
func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error {
_, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id)
if err != nil {
return err
}
s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows")
return nil
}
func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) {
cutoff := time.Now().Add(-retention)
result, err := s.db.ExecContext(ctx,
s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"),
cutoff,
)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) {
var count int
err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows
WHERE type = 'maintenance'
AND start_time <= CURRENT_TIMESTAMP
AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP)
AND (monitor_id = 0 OR monitor_id = ?
OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`),
monitorID, monitorID).Scan(&count)
if err != nil {
return false, err
}
return count > 0, nil
}
func (s *SQLStore) GetPreference(ctx context.Context, key string) (string, error) {
var value string
err := s.db.QueryRowContext(ctx, s.q("SELECT value FROM preferences WHERE key = ?"), key).Scan(&value)
internal/store/sqlstore_alerts.go
+126
View File
@@ -0,0 +1,126 @@
package store
import (
"context"
"encoding/json"
"fmt"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) {
decrypted, err := s.decryptSettings(raw)
if err != nil {
return nil, fmt.Errorf("decrypt settings: %w", err)
}
var m map[string]string
if err := json.Unmarshal([]byte(decrypted), &m); err != nil {
return nil, fmt.Errorf("unmarshal settings: %w", err)
}
return m, nil
}
func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) {
jsonBytes, err := json.Marshal(settings)
if err != nil {
return "", err
}
return s.encryptSettings(string(jsonBytes))
}
func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) {
var a models.AlertConfig
var settingsRaw string
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
if err != nil {
return a, err
}
a.Settings, err = s.unmarshalSettings(settingsRaw)
if err != nil {
return a, fmt.Errorf("alert %q: %w", name, err)
}
return a, nil
}
func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) {
stored, err := s.marshalSettings(settings)
if err != nil {
return 0, err
}
if s.dollar {
var id int
err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id)
return id, err
}
result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
if err != nil {
return 0, err
}
id, err := result.LastInsertId()
return int(id), err
}
func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) {
rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts")
if err != nil {
return nil, err
}
defer rows.Close()
var alerts []models.AlertConfig
for rows.Next() {
var a models.AlertConfig
var settingsRaw string
if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil {
return alerts, err
}
a.Settings, err = s.unmarshalSettings(settingsRaw)
if err != nil {
return alerts, fmt.Errorf("alert %q: %w", a.Name, err)
}
alerts = append(alerts, a)
}
return alerts, rows.Err()
}
func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) {
var a models.AlertConfig
var settingsRaw string
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
if err != nil {
return a, err
}
a.Settings, err = s.unmarshalSettings(settingsRaw)
if err != nil {
return a, fmt.Errorf("alert %d: %w", id, err)
}
return a, nil
}
func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error {
stored, err := s.marshalSettings(settings)
if err != nil {
return err
}
_, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
return err
}
func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error {
stored, err := s.marshalSettings(settings)
if err != nil {
return err
}
_, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id)
return err
}
func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error {
if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil {
return err
}
if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil {
return err
}
s.dialect.ResetSequenceOnEmpty(s.db, "alerts")
return nil
}
internal/store/sqlstore_alerts_test.go
+102
View File
@@ -0,0 +1,102 @@
package store
import (
"context"
"strings"
"testing"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func TestAlertCRUD(t *testing.T) {
s := newTestStore(t)
if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil {
t.Fatalf("AddAlert: %v", err)
}
alerts, err := s.GetAllAlerts(context.Background())
if err != nil {
t.Fatalf("GetAllAlerts: %v", err)
}
if len(alerts) != 1 {
t.Fatalf("expected 1 alert, got %d", len(alerts))
}
if alerts[0].Type != "discord" {
t.Errorf("expected type 'discord', got '%s'", alerts[0].Type)
}
if alerts[0].Settings["url"] != "https://example.com/hook" {
t.Errorf("settings url mismatch")
}
a, err := s.GetAlert(context.Background(), alerts[0].ID)
if err != nil {
t.Fatalf("GetAlert: %v", err)
}
if a.Name != "Discord" {
t.Errorf("expected name 'Discord', got '%s'", a.Name)
}
if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil {
t.Fatalf("UpdateAlert: %v", err)
}
a, err = s.GetAlert(context.Background(), a.ID)
if err != nil {
t.Fatalf("GetAlert: %v", err)
}
if a.Type != "slack" {
t.Errorf("expected type 'slack', got '%s'", a.Type)
}
if err := s.DeleteAlert(context.Background(), a.ID); err != nil {
t.Fatalf("DeleteAlert: %v", err)
}
alerts, err = s.GetAllAlerts(context.Background())
if err != nil {
t.Fatalf("GetAllAlerts: %v", err)
}
if len(alerts) != 0 {
t.Fatalf("expected 0 alerts after delete, got %d", len(alerts))
}
}
// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a
// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext.
func TestImportData_EncryptsAlertSettings(t *testing.T) {
s := newTestStore(t)
enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes
if err != nil {
t.Fatalf("NewEncryptor: %v", err)
}
s.SetEncryptor(enc)
backup := models.Backup{
Alerts: []models.AlertConfig{
{ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}},
},
}
if err := s.ImportData(context.Background(), backup); err != nil {
t.Fatalf("ImportData: %v", err)
}
var raw string
if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil {
t.Fatalf("query settings: %v", err)
}
if !strings.HasPrefix(raw, encryptedPrefix) {
t.Errorf("imported settings not encrypted: %q", raw)
}
if strings.Contains(raw, "SECRET") {
t.Errorf("plaintext secret found in stored column: %q", raw)
}
alerts, err := s.GetAllAlerts(context.Background())
if err != nil {
t.Fatalf("GetAllAlerts: %v", err)
}
if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" {
t.Errorf("decrypt round-trip failed: %+v", alerts)
}
}
internal/store/sqlstore_history.go
+116
View File
@@ -0,0 +1,116 @@
package store
import (
"context"
"fmt"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error {
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"),
siteID, fromStatus, toStatus, errorReason)
return err
}
func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var changes []models.StateChange
for rows.Next() {
var sc models.StateChange
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
return changes, err
}
changes = append(changes, sc)
}
return changes, rows.Err()
}
func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since)
if err != nil {
return nil, err
}
defer rows.Close()
var changes []models.StateChange
for rows.Next() {
var sc models.StateChange
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
return changes, err
}
changes = append(changes, sc)
}
return changes, rows.Err()
}
func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error {
return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp)
}
// SaveCheckFromNode inserts a single check row. Retention is handled out of
// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot
// path a plain INSERT.
func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error {
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp)
return err
}
// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per
// site, across all sites, in one pass. Intended to run periodically.
func (s *SQLStore) PruneCheckHistory(ctx context.Context) error {
q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN (
SELECT id FROM (
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn
FROM check_history
) ranked WHERE rn > %d
)`, maxCheckHistory)
_, err := s.db.ExecContext(ctx, s.q(q))
return err
}
// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite
// rows per site. Generous so realistic SLA windows are unaffected; bounds the
// otherwise unbounded growth of a flapping monitor's history.
func (s *SQLStore) PruneStateChanges(ctx context.Context) error {
q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN (
SELECT id FROM (
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn
FROM state_changes
) ranked WHERE rn > %d
)`, maxStateChangesPerSite)
_, err := s.db.ExecContext(ctx, s.q(q))
return err
}
func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) {
result := make(map[int][]models.CheckRecord)
rows, err := s.db.QueryContext(ctx, s.q(`
SELECT site_id, latency_ns, is_up FROM (
SELECT site_id, latency_ns, is_up,
ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn
FROM check_history
) sub WHERE rn <= ?`), limit)
if err != nil {
return result, err
}
defer rows.Close()
for rows.Next() {
var r models.CheckRecord
if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil {
return result, err
}
result[r.SiteID] = append(result[r.SiteID], r)
}
for id, records := range result {
for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 {
records[i], records[j] = records[j], records[i]
}
result[id] = records
}
return result, rows.Err()
}
internal/store/sqlstore_history_test.go
+134
View File
@@ -0,0 +1,134 @@
package store
import (
"context"
"testing"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func TestCheckHistory(t *testing.T) {
s := newTestStore(t)
if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil {
t.Fatalf("SaveCheck: %v", err)
}
if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil {
t.Fatalf("SaveCheck: %v", err)
}
if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil {
t.Fatalf("SaveCheck site 2: %v", err)
}
history, err := s.LoadAllHistory(context.Background(), 10)
if err != nil {
t.Fatalf("LoadAllHistory: %v", err)
}
if len(history[1]) != 2 {
t.Fatalf("expected 2 records for site 1, got %d", len(history[1]))
}
if len(history[2]) != 1 {
t.Fatalf("expected 1 record for site 2, got %d", len(history[2]))
}
upCount := 0
for _, r := range history[1] {
if r.IsUp {
upCount++
}
}
if upCount != 1 {
t.Errorf("expected 1 up record for site 1, got %d", upCount)
}
}
func TestPruneCheckHistory(t *testing.T) {
s := newTestStore(t)
for i := 0; i < maxCheckHistory+5; i++ {
if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil {
t.Fatalf("SaveCheck site 1: %v", err)
}
}
for i := 0; i < 3; i++ {
if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil {
t.Fatalf("SaveCheck site 2: %v", err)
}
}
if err := s.PruneCheckHistory(context.Background()); err != nil {
t.Fatalf("PruneCheckHistory: %v", err)
}
history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2)
if err != nil {
t.Fatalf("LoadAllHistory: %v", err)
}
if len(history[1]) != maxCheckHistory {
t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1]))
}
if len(history[2]) != 3 {
t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2]))
}
}
func TestDeleteSiteCascade(t *testing.T) {
s := newTestStore(t)
site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30}
if err := s.AddSite(context.Background(), site); err != nil {
t.Fatalf("AddSite: %v", err)
}
sites, err := s.GetSites(context.Background())
if err != nil {
t.Fatalf("GetSites: %v", err)
}
siteID := sites[0].ID
if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil {
t.Fatalf("SaveCheck: %v", err)
}
if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil {
t.Fatalf("SaveStateChange: %v", err)
}
mw := models.MaintenanceWindow{
MonitorID: siteID,
Title: "Test MW",
Type: "maintenance",
StartTime: time.Now(),
}
if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
if err := s.DeleteSite(context.Background(), siteID); err != nil {
t.Fatalf("DeleteSite: %v", err)
}
history, err := s.LoadAllHistory(context.Background(), 100)
if err != nil {
t.Fatalf("LoadAllHistory: %v", err)
}
if len(history[siteID]) != 0 {
t.Errorf("expected 0 check_history rows, got %d", len(history[siteID]))
}
changes, err := s.GetStateChanges(context.Background(), siteID, 100)
if err != nil {
t.Fatalf("GetStateChanges: %v", err)
}
if len(changes) != 0 {
t.Errorf("expected 0 state_changes rows, got %d", len(changes))
}
windows, err := s.GetActiveMaintenanceWindows(context.Background())
if err != nil {
t.Fatalf("GetActiveMaintenanceWindows: %v", err)
}
for _, w := range windows {
if w.MonitorID == siteID {
t.Errorf("orphaned maintenance window found: id=%d", w.ID)
}
}
}
internal/store/sqlstore_maintenance.go
+148
View File
@@ -0,0 +1,148 @@
package store
import (
"context"
"database/sql"
"fmt"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) {
var mw models.MaintenanceWindow
var endTime sql.NullTime
if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil {
return mw, err
}
if endTime.Valid {
mw.EndTime = endTime.Time
}
return mw, nil
}
func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC"))
if err != nil {
return nil, err
}
defer rows.Close()
var windows []models.MaintenanceWindow
for rows.Next() {
mw, err := s.scanMaintenanceWindow(rows)
if err != nil {
return windows, err
}
windows = append(windows, mw)
}
return windows, rows.Err()
}
func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) {
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit)
if err != nil {
return nil, err
}
defer rows.Close()
var windows []models.MaintenanceWindow
for rows.Next() {
mw, err := s.scanMaintenanceWindow(rows)
if err != nil {
return windows, err
}
windows = append(windows, mw)
}
return windows, rows.Err()
}
func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) {
var timeClause string
var args []interface{}
if endTime.IsZero() {
timeClause = "(end_time IS NULL OR end_time > ?)"
args = append(args, startTime)
} else {
timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?"
args = append(args, startTime, endTime)
}
var scopeClause string
if monitorID == 0 {
scopeClause = "1=1"
} else {
scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))"
args = append(args, monitorID, monitorID)
}
query := fmt.Sprintf(
"SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time",
timeClause, scopeClause,
)
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
if err != nil {
return nil, err
}
defer rows.Close()
var windows []models.MaintenanceWindow
for rows.Next() {
mw, err := s.scanMaintenanceWindow(rows)
if err != nil {
return nil, err
}
windows = append(windows, mw)
}
return windows, rows.Err()
}
func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error {
if mw.StartTime.IsZero() {
mw.StartTime = time.Now()
}
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"),
mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy)
return err
}
func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error {
_, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id)
return err
}
func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error {
_, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id)
if err != nil {
return err
}
s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows")
return nil
}
func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) {
cutoff := time.Now().Add(-retention)
result, err := s.db.ExecContext(ctx,
s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"),
cutoff,
)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) {
var count int
err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows
WHERE type = 'maintenance'
AND start_time <= CURRENT_TIMESTAMP
AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP)
AND (monitor_id = 0 OR monitor_id = ?
OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`),
monitorID, monitorID).Scan(&count)
if err != nil {
return false, err
}
return count > 0, nil
}
internal/store/sqlstore_maintenance_test.go
+168
View File
@@ -0,0 +1,168 @@
package store
import (
"context"
"testing"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
func TestPruneExpiredMaintenanceWindows(t *testing.T) {
s := newTestStore(t)
now := time.Now()
// Expired 10 days ago — should be pruned with 7d retention.
old := models.MaintenanceWindow{
MonitorID: 0,
Title: "Old Window",
Type: "maintenance",
StartTime: now.Add(-11 * 24 * time.Hour),
EndTime: now.Add(-10 * 24 * time.Hour),
}
if err := s.AddMaintenanceWindow(context.Background(), old); err != nil {
t.Fatalf("AddMaintenanceWindow (old): %v", err)
}
// Expired 1 day ago — within 7d retention, should survive.
recent := models.MaintenanceWindow{
MonitorID: 0,
Title: "Recent Window",
Type: "maintenance",
StartTime: now.Add(-2 * 24 * time.Hour),
EndTime: now.Add(-1 * 24 * time.Hour),
}
if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil {
t.Fatalf("AddMaintenanceWindow (recent): %v", err)
}
// Ongoing — no end time, should survive.
ongoing := models.MaintenanceWindow{
MonitorID: 0,
Title: "Ongoing Window",
Type: "maintenance",
StartTime: now.Add(-1 * time.Hour),
}
if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil {
t.Fatalf("AddMaintenanceWindow (ongoing): %v", err)
}
pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour)
if err != nil {
t.Fatalf("PruneExpiredMaintenanceWindows: %v", err)
}
if pruned != 1 {
t.Errorf("expected 1 pruned, got %d", pruned)
}
all, err := s.GetAllMaintenanceWindows(context.Background(), 100)
if err != nil {
t.Fatalf("GetAllMaintenanceWindows: %v", err)
}
if len(all) != 2 {
t.Fatalf("expected 2 remaining windows, got %d", len(all))
}
for _, w := range all {
if w.Title == "Old Window" {
t.Error("old window should have been pruned")
}
}
}
func TestGetOverlappingMaintenanceWindows(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30}
if err := s.AddSite(ctx, site); err != nil {
t.Fatalf("AddSite: %v", err)
}
now := time.Now()
active := models.MaintenanceWindow{
MonitorID: 1,
Title: "Deploy v2",
Type: "maintenance",
StartTime: now.Add(-30 * time.Minute),
EndTime: now.Add(30 * time.Minute),
}
if err := s.AddMaintenanceWindow(ctx, active); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
ended := models.MaintenanceWindow{
MonitorID: 1,
Title: "Old deploy",
Type: "maintenance",
StartTime: now.Add(-3 * time.Hour),
EndTime: now.Add(-2 * time.Hour),
}
if err := s.AddMaintenanceWindow(ctx, ended); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
t.Run("same monitor overlaps", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 1 {
t.Fatalf("expected 1 overlap, got %d", len(overlaps))
}
if overlaps[0].Title != "Deploy v2" {
t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title)
}
})
t.Run("different monitor no overlap", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 0 {
t.Errorf("expected 0 overlaps, got %d", len(overlaps))
}
})
t.Run("global window overlaps all", func(t *testing.T) {
global := models.MaintenanceWindow{
MonitorID: 0,
Title: "Global freeze",
Type: "maintenance",
StartTime: now.Add(-10 * time.Minute),
EndTime: now.Add(2 * time.Hour),
}
if err := s.AddMaintenanceWindow(ctx, global); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 2 {
t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps))
}
})
t.Run("indefinite window overlaps", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) < 1 {
t.Error("expected at least 1 overlap for indefinite window")
}
})
t.Run("ended window excluded", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 0 {
t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps))
}
})
}
internal/store/sqlstore_test.go
-379
View File
@@ -3,9 +3,7 @@ package store
import (
"context"
"fmt"
"strings"
"testing"
"time"
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
)
@@ -74,60 +72,6 @@ func TestSiteCRUD(t *testing.T) {
}
}
func TestAlertCRUD(t *testing.T) {
s := newTestStore(t)
if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil {
t.Fatalf("AddAlert: %v", err)
}
alerts, err := s.GetAllAlerts(context.Background())
if err != nil {
t.Fatalf("GetAllAlerts: %v", err)
}
if len(alerts) != 1 {
t.Fatalf("expected 1 alert, got %d", len(alerts))
}
if alerts[0].Type != "discord" {
t.Errorf("expected type 'discord', got '%s'", alerts[0].Type)
}
if alerts[0].Settings["url"] != "https://example.com/hook" {
t.Errorf("settings url mismatch")
}
a, err := s.GetAlert(context.Background(), alerts[0].ID)
if err != nil {
t.Fatalf("GetAlert: %v", err)
}
if a.Name != "Discord" {
t.Errorf("expected name 'Discord', got '%s'", a.Name)
}
if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil {
t.Fatalf("UpdateAlert: %v", err)
}
a, err = s.GetAlert(context.Background(), a.ID)
if err != nil {
t.Fatalf("GetAlert: %v", err)
}
if a.Type != "slack" {
t.Errorf("expected type 'slack', got '%s'", a.Type)
}
if err := s.DeleteAlert(context.Background(), a.ID); err != nil {
t.Fatalf("DeleteAlert: %v", err)
}
alerts, err = s.GetAllAlerts(context.Background())
if err != nil {
t.Fatalf("GetAllAlerts: %v", err)
}
if len(alerts) != 0 {
t.Fatalf("expected 0 alerts after delete, got %d", len(alerts))
}
}
func TestUserCRUD(t *testing.T) {
s := newTestStore(t)
@@ -301,101 +245,6 @@ func TestImportData_NilUsersPreservesExisting(t *testing.T) {
}
}
func TestCheckHistory(t *testing.T) {
s := newTestStore(t)
if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil {
t.Fatalf("SaveCheck: %v", err)
}
if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil {
t.Fatalf("SaveCheck: %v", err)
}
if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil {
t.Fatalf("SaveCheck site 2: %v", err)
}
history, err := s.LoadAllHistory(context.Background(), 10)
if err != nil {
t.Fatalf("LoadAllHistory: %v", err)
}
if len(history[1]) != 2 {
t.Fatalf("expected 2 records for site 1, got %d", len(history[1]))
}
if len(history[2]) != 1 {
t.Fatalf("expected 1 record for site 2, got %d", len(history[2]))
}
upCount := 0
for _, r := range history[1] {
if r.IsUp {
upCount++
}
}
if upCount != 1 {
t.Errorf("expected 1 up record for site 1, got %d", upCount)
}
}
func TestDeleteSiteCascade(t *testing.T) {
s := newTestStore(t)
site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30}
if err := s.AddSite(context.Background(), site); err != nil {
t.Fatalf("AddSite: %v", err)
}
sites, err := s.GetSites(context.Background())
if err != nil {
t.Fatalf("GetSites: %v", err)
}
siteID := sites[0].ID
if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil {
t.Fatalf("SaveCheck: %v", err)
}
if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil {
t.Fatalf("SaveStateChange: %v", err)
}
mw := models.MaintenanceWindow{
MonitorID: siteID,
Title: "Test MW",
Type: "maintenance",
StartTime: time.Now(),
}
if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
if err := s.DeleteSite(context.Background(), siteID); err != nil {
t.Fatalf("DeleteSite: %v", err)
}
history, err := s.LoadAllHistory(context.Background(), 100)
if err != nil {
t.Fatalf("LoadAllHistory: %v", err)
}
if len(history[siteID]) != 0 {
t.Errorf("expected 0 check_history rows, got %d", len(history[siteID]))
}
changes, err := s.GetStateChanges(context.Background(), siteID, 100)
if err != nil {
t.Fatalf("GetStateChanges: %v", err)
}
if len(changes) != 0 {
t.Errorf("expected 0 state_changes rows, got %d", len(changes))
}
windows, err := s.GetActiveMaintenanceWindows(context.Background())
if err != nil {
t.Fatalf("GetActiveMaintenanceWindows: %v", err)
}
for _, w := range windows {
if w.MonitorID == siteID {
t.Errorf("orphaned maintenance window found: id=%d", w.ID)
}
}
}
func TestPruneLogs(t *testing.T) {
s := newTestStore(t)
@@ -428,231 +277,3 @@ func TestPruneLogs(t *testing.T) {
t.Error("oldest log survived prune")
}
}
func TestPruneCheckHistory(t *testing.T) {
s := newTestStore(t)
for i := 0; i < maxCheckHistory+5; i++ {
if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil {
t.Fatalf("SaveCheck site 1: %v", err)
}
}
for i := 0; i < 3; i++ {
if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil {
t.Fatalf("SaveCheck site 2: %v", err)
}
}
if err := s.PruneCheckHistory(context.Background()); err != nil {
t.Fatalf("PruneCheckHistory: %v", err)
}
history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2)
if err != nil {
t.Fatalf("LoadAllHistory: %v", err)
}
if len(history[1]) != maxCheckHistory {
t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1]))
}
if len(history[2]) != 3 {
t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2]))
}
}
func TestPruneExpiredMaintenanceWindows(t *testing.T) {
s := newTestStore(t)
now := time.Now()
// Expired 10 days ago — should be pruned with 7d retention.
old := models.MaintenanceWindow{
MonitorID: 0,
Title: "Old Window",
Type: "maintenance",
StartTime: now.Add(-11 * 24 * time.Hour),
EndTime: now.Add(-10 * 24 * time.Hour),
}
if err := s.AddMaintenanceWindow(context.Background(), old); err != nil {
t.Fatalf("AddMaintenanceWindow (old): %v", err)
}
// Expired 1 day ago — within 7d retention, should survive.
recent := models.MaintenanceWindow{
MonitorID: 0,
Title: "Recent Window",
Type: "maintenance",
StartTime: now.Add(-2 * 24 * time.Hour),
EndTime: now.Add(-1 * 24 * time.Hour),
}
if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil {
t.Fatalf("AddMaintenanceWindow (recent): %v", err)
}
// Ongoing — no end time, should survive.
ongoing := models.MaintenanceWindow{
MonitorID: 0,
Title: "Ongoing Window",
Type: "maintenance",
StartTime: now.Add(-1 * time.Hour),
}
if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil {
t.Fatalf("AddMaintenanceWindow (ongoing): %v", err)
}
pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour)
if err != nil {
t.Fatalf("PruneExpiredMaintenanceWindows: %v", err)
}
if pruned != 1 {
t.Errorf("expected 1 pruned, got %d", pruned)
}
all, err := s.GetAllMaintenanceWindows(context.Background(), 100)
if err != nil {
t.Fatalf("GetAllMaintenanceWindows: %v", err)
}
if len(all) != 2 {
t.Fatalf("expected 2 remaining windows, got %d", len(all))
}
for _, w := range all {
if w.Title == "Old Window" {
t.Error("old window should have been pruned")
}
}
}
func TestGetOverlappingMaintenanceWindows(t *testing.T) {
s := newTestStore(t)
ctx := context.Background()
site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30}
if err := s.AddSite(ctx, site); err != nil {
t.Fatalf("AddSite: %v", err)
}
now := time.Now()
active := models.MaintenanceWindow{
MonitorID: 1,
Title: "Deploy v2",
Type: "maintenance",
StartTime: now.Add(-30 * time.Minute),
EndTime: now.Add(30 * time.Minute),
}
if err := s.AddMaintenanceWindow(ctx, active); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
ended := models.MaintenanceWindow{
MonitorID: 1,
Title: "Old deploy",
Type: "maintenance",
StartTime: now.Add(-3 * time.Hour),
EndTime: now.Add(-2 * time.Hour),
}
if err := s.AddMaintenanceWindow(ctx, ended); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
t.Run("same monitor overlaps", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 1 {
t.Fatalf("expected 1 overlap, got %d", len(overlaps))
}
if overlaps[0].Title != "Deploy v2" {
t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title)
}
})
t.Run("different monitor no overlap", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 0 {
t.Errorf("expected 0 overlaps, got %d", len(overlaps))
}
})
t.Run("global window overlaps all", func(t *testing.T) {
global := models.MaintenanceWindow{
MonitorID: 0,
Title: "Global freeze",
Type: "maintenance",
StartTime: now.Add(-10 * time.Minute),
EndTime: now.Add(2 * time.Hour),
}
if err := s.AddMaintenanceWindow(ctx, global); err != nil {
t.Fatalf("AddMaintenanceWindow: %v", err)
}
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 2 {
t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps))
}
})
t.Run("indefinite window overlaps", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) < 1 {
t.Error("expected at least 1 overlap for indefinite window")
}
})
t.Run("ended window excluded", func(t *testing.T) {
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(overlaps) != 0 {
t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps))
}
})
}
// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a
// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext.
func TestImportData_EncryptsAlertSettings(t *testing.T) {
s := newTestStore(t)
enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes
if err != nil {
t.Fatalf("NewEncryptor: %v", err)
}
s.SetEncryptor(enc)
backup := models.Backup{
Alerts: []models.AlertConfig{
{ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}},
},
}
if err := s.ImportData(context.Background(), backup); err != nil {
t.Fatalf("ImportData: %v", err)
}
var raw string
if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil {
t.Fatalf("query settings: %v", err)
}
if !strings.HasPrefix(raw, encryptedPrefix) {
t.Errorf("imported settings not encrypted: %q", raw)
}
if strings.Contains(raw, "SECRET") {
t.Errorf("plaintext secret found in stored column: %q", raw)
}
alerts, err := s.GetAllAlerts(context.Background())
if err != nil {
t.Fatalf("GetAllAlerts: %v", err)
}
if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" {
t.Errorf("decrypt round-trip failed: %+v", alerts)
}
}