refactor: decompose monitor.go and sqlstore.go into focused files

Split monitor.go (1131→350 lines) into alerts.go, checks.go,
sites.go, and maintenance.go by concern. Split sqlstore.go
(833→470 lines) into sqlstore_alerts.go, sqlstore_history.go,
and sqlstore_maintenance.go by domain. Tests move with their
implementation. Pure reorganization — no behavioral changes.

internal/monitor/alerts.go

@@ -0,0 +1,272 @@
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
+)
+
+type AlertHealth struct {
+	LastSendAt time.Time
+	LastSendOK bool
+	LastError  string
+	SendCount  int
+	FailCount  int
+}
+
+// InitAlertHealth restores persisted alert send health so the dashboard shows real
+// "last sent" / health state on startup instead of resetting every channel to "never".
+func (e *Engine) InitAlertHealth() {
+	records, err := e.db.LoadAlertHealth(context.Background())
+	if err != nil {
+		return
+	}
+	e.alertHealthMu.Lock()
+	defer e.alertHealthMu.Unlock()
+	for id, r := range records {
+		e.alertHealth[id] = AlertHealth{
+			LastSendAt: r.LastSendAt,
+			LastSendOK: r.LastSendOK,
+			LastError:  r.LastError,
+			SendCount:  r.SendCount,
+			FailCount:  r.FailCount,
+		}
+	}
+}
+
+// handleStatusChange folds a check result into the live state. snap is the
+// stale snapshot the check ran against; the actual mutation is applied onto the
+// CURRENT live entry via applyState, so a concurrent pause / config edit /
+// heartbeat is never reverted by this write. Logs and alerts are emitted after
+// the lock is released, off the critical section.
+func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
+	if !e.IsActive() {
+		return
+	}
+
+	inMaint := e.isInMaintenance(snap.ID)
+	status := models.Status(rawStatus)
+
+	var (
+		prev, next            models.Status
+		name, typ             string
+		alertID               int
+		failCount, maxRetries int
+		confirmedDown         bool
+		failedCheck           bool
+		downSince             time.Time
+		sslWarnFire           bool
+		sslDays               int
+		skipped               bool
+		changed               bool
+	)
+
+	_, exists := e.applyState(snap.ID, func(s *models.Site) {
+		// A non-UP result computed from a stale snapshot must not override a
+		// heartbeat (or newer check) that landed while we were evaluating.
+		if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
+			skipped = true
+			return
+		}
+
+		prev = s.Status
+		name = s.Name
+		typ = s.Type
+		alertID = s.AlertID
+		maxRetries = s.MaxRetries
+		downSince = s.StatusChangedAt
+
+		// Fresh check results (measured by the run against snap).
+		s.StatusCode = code
+		s.Latency = snap.Latency
+		s.LastCheck = snap.LastCheck
+		s.HasSSL = snap.HasSSL
+		s.CertExpiry = snap.CertExpiry
+		s.LastError = errorReason
+		if status == models.StatusUp {
+			s.LastSuccessAt = time.Now()
+			s.LastError = ""
+		}
+
+		// Status + failure-count transition, based on the CURRENT live status.
+		if status == models.StatusUp {
+			s.FailureCount = 0
+			s.Status = models.StatusUp
+		} else {
+			if s.FailureCount <= s.MaxRetries {
+				s.FailureCount++
+			}
+			if s.FailureCount > s.MaxRetries {
+				if s.Status != status {
+					confirmedDown = true
+				}
+				s.Status = status
+				s.FailureCount = s.MaxRetries + 1
+			} else {
+				failedCheck = true
+			}
+		}
+		failCount = s.FailureCount
+
+		if s.Status != prev && prev != models.StatusPending {
+			s.StatusChangedAt = time.Now()
+		} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
+			s.StatusChangedAt = time.Now()
+		}
+
+		// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
+		if typ == "http" && s.CheckSSL && s.HasSSL {
+			days := int(time.Until(s.CertExpiry).Hours() / 24)
+			if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
+				sslWarnFire = true
+				sslDays = days
+				s.SentSSLWarning = true
+			} else if days > s.ExpiryThreshold {
+				s.SentSSLWarning = false
+			}
+		}
+
+		next = s.Status
+		changed = next != prev
+	})
+
+	if !exists || skipped {
+		return
+	}
+
+	e.recordCheck(snap.ID, latency, status == models.StatusUp)
+
+	if confirmedDown {
+		if errorReason != "" {
+			e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
+		} else {
+			e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
+		}
+	} else if failedCheck {
+		e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
+	}
+
+	if changed && prev != models.StatusPending {
+		e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
+	}
+
+	if sslWarnFire {
+		if !inMaint {
+			e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
+		} else {
+			e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
+		}
+	}
+
+	if prev == models.StatusUp && next == models.StatusLate {
+		e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
+	}
+
+	if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
+		if inMaint {
+			e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
+		} else {
+			msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
+			if errorReason != "" {
+				msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
+			}
+			if typ == "push" {
+				msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
+			}
+			e.triggerAlert(alertID, "🚨 ALERT", msg)
+		}
+	}
+	if prev.IsBroken() && next == models.StatusUp {
+		downDur := ""
+		if !downSince.IsZero() {
+			downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
+		}
+		e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
+		if !inMaint {
+			e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
+		}
+	}
+	if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
+		e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
+	}
+}
+
+func (e *Engine) triggerAlert(alertID int, title, message string) {
+	if alertID <= 0 {
+		return
+	}
+	cfg, err := e.db.GetAlert(context.Background(), alertID)
+	if err != nil {
+		e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
+		return
+	}
+	provider := alert.GetProvider(cfg)
+	if provider != nil {
+		go func() {
+			ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
+			defer cancel()
+			if err := provider.Send(ctx, title, message); err != nil {
+				e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
+				e.recordAlertResult(alertID, false, err.Error())
+			} else {
+				e.recordAlertResult(alertID, true, "")
+			}
+		}()
+	}
+}
+
+func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
+	e.alertHealthMu.Lock()
+	defer e.alertHealthMu.Unlock()
+	h := e.alertHealth[alertID]
+	h.LastSendAt = time.Now()
+	h.LastSendOK = ok
+	h.SendCount++
+	if ok {
+		h.LastError = ""
+	} else {
+		h.LastError = errMsg
+		h.FailCount++
+	}
+	e.alertHealth[alertID] = h
+
+	// Persist so health survives restarts; DB IO off the alert path.
+	e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
+		AlertID:    alertID,
+		LastSendAt: h.LastSendAt,
+		LastSendOK: h.LastSendOK,
+		LastError:  h.LastError,
+		SendCount:  h.SendCount,
+		FailCount:  h.FailCount,
+	}})
+}
+
+func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
+	e.alertHealthMu.RLock()
+	defer e.alertHealthMu.RUnlock()
+	return e.alertHealth[alertID]
+}
+
+func (e *Engine) TestAlert(alertID int) error {
+	cfg, err := e.db.GetAlert(context.Background(), alertID)
+	if err != nil {
+		return fmt.Errorf("failed to load alert: %w", err)
+	}
+	provider := alert.GetProvider(cfg)
+	if provider == nil {
+		return fmt.Errorf("no provider for type %q", cfg.Type)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
+	defer cancel()
+	err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
+	if err != nil {
+		e.recordAlertResult(alertID, false, err.Error())
+		return err
+	}
+	e.recordAlertResult(alertID, true, "")
+	e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
+	return nil
+}

internal/monitor/alerts_test.go

@@ -0,0 +1,487 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
+)
+
+// --- Group 1: State Machine ---
+
+func TestHandleStatusChange_PendingToUp(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3, AlertID: 1},
+		SiteState:  models.SiteState{Status: "PENDING"},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "UP", 200, 10*time.Millisecond, "")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP, got %s", s.Status)
+	}
+	if s.FailureCount != 0 {
+		t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) != 0 {
+		t.Error("expected no alert for PENDING→UP")
+	}
+}
+
+func TestHandleStatusChange_UpIncrementFailure(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3},
+		SiteState:  models.SiteState{Status: "UP", FailureCount: 0},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "DOWN", 500, 0, "test error")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP (under retry threshold), got %s", s.Status)
+	}
+	if s.FailureCount != 1 {
+		t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
+	}
+}
+
+func TestHandleStatusChange_UpToDown_ExceedsRetries(t *testing.T) {
+	ms := newMockStore()
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "discord", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2, AlertID: 1},
+		SiteState:  models.SiteState{Status: "UP", FailureCount: 2},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "DOWN", 500, 0, "test error")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected DOWN, got %s", s.Status)
+	}
+	if s.FailureCount != 3 {
+		t.Errorf("expected FailureCount 3, got %d", s.FailureCount)
+	}
+	waitAsync()
+	calls := ms.getAlertCallsSnapshot()
+	if len(calls) == 0 || calls[0] != 1 {
+		t.Errorf("expected alert call for alertID 1, got %v", calls)
+	}
+}
+
+func TestHandleStatusChange_UpToDown_ZeroRetries(t *testing.T) {
+	ms := newMockStore()
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
+		SiteState:  models.SiteState{Status: "UP", FailureCount: 0},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "DOWN", 0, 0, "test error")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected DOWN, got %s", s.Status)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) == 0 {
+		t.Error("expected alert on immediate DOWN")
+	}
+}
+
+func TestHandleStatusChange_DownToUp_Recovery(t *testing.T) {
+	ms := newMockStore()
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
+		SiteState:  models.SiteState{Status: "DOWN", FailureCount: 4},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP, got %s", s.Status)
+	}
+	if s.FailureCount != 0 {
+		t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) == 0 {
+		t.Error("expected recovery alert")
+	}
+}
+
+func TestHandleStatusChange_DownStaysDown(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2},
+		SiteState:  models.SiteState{Status: "DOWN", FailureCount: 3},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "DOWN", 0, 0, "test error")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected DOWN, got %s", s.Status)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) != 0 {
+		t.Error("expected no re-alert for already DOWN")
+	}
+}
+
+func TestHandleStatusChange_SSLExpired(t *testing.T) {
+	ms := newMockStore()
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "SSL EXP", 0, 0, "SSL certificate expired")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "SSL EXP" {
+		t.Errorf("expected SSL EXP, got %s", s.Status)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) == 0 {
+		t.Error("expected alert on SSL EXP")
+	}
+}
+
+func TestHandleStatusChange_AlertSuppressedMaintenance(t *testing.T) {
+	ms := newMockStore()
+	ms.maintenance[1] = true
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+	e.refreshMaintenanceCache(context.Background())
+
+	e.handleStatusChange(site, "DOWN", 0, 0, "test error")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected DOWN, got %s", s.Status)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) != 0 {
+		t.Error("expected no alert during maintenance")
+	}
+	logs := e.GetLogs()
+	found := false
+	for _, l := range logs {
+		if containsStr(l.Message, "suppressed") {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Error("expected log mentioning suppressed")
+	}
+}
+
+func TestHandleStatusChange_RecoverySuppressedMaintenance(t *testing.T) {
+	ms := newMockStore()
+	ms.maintenance[1] = true
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
+		SiteState:  models.SiteState{Status: "DOWN"},
+	}
+	injectSite(e, site)
+	e.refreshMaintenanceCache(context.Background())
+
+	e.handleStatusChange(site, "UP", 200, 0, "")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP, got %s", s.Status)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) != 0 {
+		t.Error("expected no alert during maintenance recovery")
+	}
+}
+
+func TestHandleStatusChange_SSLWarning(t *testing.T) {
+	ms := newMockStore()
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
+		SiteState:  models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "UP", 200, 0, "")
+
+	s, _ := getSite(e, 1)
+	if !s.SentSSLWarning {
+		t.Error("expected SentSSLWarning=true")
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) == 0 {
+		t.Error("expected SSL warning alert")
+	}
+}
+
+func TestHandleStatusChange_SSLWarningNotRepeated(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
+		SiteState:  models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "UP", 200, 0, "")
+
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) != 0 {
+		t.Error("expected no repeat SSL warning")
+	}
+}
+
+func TestHandleStatusChange_SSLWarningReset(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30},
+		SiteState:  models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(60 * 24 * time.Hour)},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "UP", 200, 0, "")
+
+	s, _ := getSite(e, 1)
+	if s.SentSSLWarning {
+		t.Error("expected SentSSLWarning reset to false")
+	}
+}
+
+func TestHandleStatusChange_SSLWarningSuppressedMaint(t *testing.T) {
+	ms := newMockStore()
+	ms.maintenance[1] = true
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
+		SiteState:  models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
+	}
+	injectSite(e, site)
+	e.refreshMaintenanceCache(context.Background())
+
+	e.handleStatusChange(site, "UP", 200, 0, "")
+
+	s, _ := getSite(e, 1)
+	if !s.SentSSLWarning {
+		t.Error("expected SentSSLWarning=true even in maintenance")
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) != 0 {
+		t.Error("expected no alert during maintenance")
+	}
+}
+
+func TestHandleStatusChange_InactiveEngine(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+	e.SetActive(false)
+
+	e.handleStatusChange(site, "DOWN", 0, 0, "test error")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Error("expected no state change when inactive")
+	}
+}
+
+// --- Group 10: liveState merge (lost-update race) ---
+
+// A pause that lands while a check is in flight must survive the check's
+// write-back. The old code snapshotted the site, ran the check, then wrote the
+// whole stale struct back — reverting the pause.
+func TestHandleStatusChange_PauseDuringCheckSurvives(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	// `site` is the stale snapshot the check ran against (Paused=false).
+	// Meanwhile the user pauses the monitor.
+	e.ToggleSitePause(1)
+
+	// Check completes and folds its result in using the stale snapshot.
+	e.handleStatusChange(site, "DOWN", 500, 0, "boom")
+
+	s, _ := getSite(e, 1)
+	if !s.Paused {
+		t.Error("pause was reverted by a stale check write-back")
+	}
+	if s.Status != "DOWN" {
+		t.Errorf("expected check result still applied (DOWN), got %s", s.Status)
+	}
+}
+
+// A config edit that lands while a check is in flight must survive; the check
+// must not resurrect the old config from its snapshot.
+func TestHandleStatusChange_ConfigEditDuringCheckSurvives(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com", Type: "http", MaxRetries: 0, Interval: 30},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	// Config changes mid-check.
+	e.UpdateSiteConfig(models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Type: "http", Interval: 60})
+
+	// Stale check (ran against http://old.com) folds its result in.
+	e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
+
+	s, _ := getSite(e, 1)
+	if s.URL != "http://new.com" {
+		t.Errorf("config edit reverted: URL=%s", s.URL)
+	}
+	if s.Interval != 60 {
+		t.Errorf("config edit reverted: Interval=%d", s.Interval)
+	}
+}
+
+// The classic push false-DOWN: a heartbeat marks the monitor UP while a
+// staleness evaluation (computed from the older LastCheck) is mid-flight.
+// The stale DOWN must not overwrite the fresh heartbeat.
+func TestHandleStatusChange_HeartbeatNotOverwrittenByStaleDown(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	// Snapshot the engine would have taken before evaluating staleness:
+	// LastCheck is old, so checkPush decided "DOWN".
+	snap := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Token: "tok", Interval: 10},
+		SiteState:  models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
+	}
+	injectSite(e, snap)
+
+	// A heartbeat lands first, advancing LastCheck and confirming UP.
+	if !e.RecordHeartbeat("tok") {
+		t.Fatal("heartbeat rejected")
+	}
+
+	// Now the in-flight stale evaluation tries to write DOWN.
+	e.handleStatusChange(snap, "DOWN", 0, 0, "heartbeat missed")
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("stale DOWN overwrote a fresh heartbeat: status=%s", s.Status)
+	}
+}
+
+// A check result for a site removed mid-check must be dropped, not recreate it.
+func TestHandleStatusChange_RemovedSiteDropped(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	e.RemoveSite(1)
+	e.handleStatusChange(site, "DOWN", 500, 0, "boom")
+
+	if _, ok := getSite(e, 1); ok {
+		t.Error("removed site was recreated by a late check write-back")
+	}
+}
+
+// --- Group 12: Phase 3 engine correctness ---
+
+// PENDING→DOWN must honor MaxRetries instead of alerting on first failure.
+func TestHandleStatusChange_PendingRetriesBeforeDown(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "new-monitor", MaxRetries: 2},
+		SiteState:  models.SiteState{Status: "PENDING"},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "DOWN", 0, 0, "timeout")
+	s, _ := getSite(e, 1)
+	if s.Status != "PENDING" {
+		t.Errorf("expected PENDING during retry, got %s", s.Status)
+	}
+	if s.FailureCount != 1 {
+		t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
+	}
+
+	e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
+	s, _ = getSite(e, 1)
+	if s.Status != "PENDING" {
+		t.Errorf("expected PENDING during retry 2, got %s", s.Status)
+	}
+
+	e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
+	s, _ = getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
+	}
+}
+
+// LATE→DOWN must also honor MaxRetries.
+func TestHandleStatusChange_LateRetriesBeforeDown(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push-mon", MaxRetries: 1},
+		SiteState:  models.SiteState{Status: "LATE"},
+	}
+	injectSite(e, site)
+
+	e.handleStatusChange(site, "DOWN", 0, 0, "missed heartbeat")
+	s, _ := getSite(e, 1)
+	if s.Status != "LATE" {
+		t.Errorf("expected LATE during retry, got %s", s.Status)
+	}
+
+	e.handleStatusChange(s, "DOWN", 0, 0, "missed heartbeat")
+	s, _ = getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
+	}
+}

internal/monitor/checks.go

@@ -0,0 +1,315 @@
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"math/rand/v2"
+	"time"
+
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
+)
+
+func (e *Engine) RecordHeartbeat(token string) bool {
+	if !e.IsActive() {
+		return false
+	}
+
+	e.mu.RLock()
+	targetID, ok := e.tokenIndex[token]
+	e.mu.RUnlock()
+	if !ok {
+		return false
+	}
+
+	var (
+		prevStatus models.Status
+		name       string
+		alertID    int
+		downSince  time.Time
+	)
+	_, exists := e.applyState(targetID, func(s *models.Site) {
+		prevStatus = s.Status
+		name = s.Name
+		alertID = s.AlertID
+		downSince = s.StatusChangedAt // captured before mutation = when it went down
+
+		s.LastCheck = time.Now()
+		s.Status = models.StatusUp
+		s.FailureCount = 0
+		s.Latency = 0
+		s.LastError = ""
+		s.LastSuccessAt = time.Now()
+		if prevStatus != models.StatusUp {
+			s.StatusChangedAt = time.Now()
+		}
+	})
+	if !exists {
+		return false
+	}
+
+	switch prevStatus {
+	case models.StatusPending:
+		e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
+	case models.StatusLate:
+		e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
+	case models.StatusStale:
+		e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
+	case models.StatusDown:
+		downDur := ""
+		if !downSince.IsZero() {
+			downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
+		}
+		e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
+		go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
+	}
+
+	e.recordCheck(targetID, 0, true)
+
+	if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
+		e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
+	}
+
+	return true
+}
+
+func (e *Engine) getRecheckChan(id int) chan struct{} {
+	e.recheckMu.Lock()
+	defer e.recheckMu.Unlock()
+	ch, ok := e.recheck[id]
+	if !ok {
+		ch = make(chan struct{}, 1)
+		e.recheck[id] = ch
+	}
+	return ch
+}
+
+func (e *Engine) signalRecheck(id int) {
+	ch := e.getRecheckChan(id)
+	select {
+	case ch <- struct{}{}:
+	default:
+	}
+}
+
+func (e *Engine) monitorRoutine(ctx context.Context, id int) {
+	recheckCh := e.getRecheckChan(id)
+
+	// Stagger initial check to avoid thundering herd on startup
+	stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
+	select {
+	case <-time.After(stagger):
+	case <-ctx.Done():
+		return
+	}
+
+	e.checkByID(ctx, id)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		if !e.IsActive() {
+			select {
+			case <-time.After(pollInterval):
+			case <-ctx.Done():
+				return
+			case <-recheckCh:
+			}
+			continue
+		}
+
+		e.mu.RLock()
+		site, exists := e.liveState[id]
+		e.mu.RUnlock()
+		if !exists {
+			return
+		}
+
+		if site.Paused {
+			select {
+			case <-time.After(pollInterval):
+			case <-ctx.Done():
+				return
+			case <-recheckCh:
+			}
+			continue
+		}
+
+		interval := site.Interval
+		if interval < minCheckInterval {
+			interval = minCheckInterval
+		}
+		jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
+		select {
+		case <-time.After(time.Duration(interval)*time.Second + jitter):
+		case <-ctx.Done():
+			return
+		case <-recheckCh:
+		}
+		e.checkByID(ctx, id)
+	}
+}
+
+func (e *Engine) checkByID(ctx context.Context, id int) {
+	if !e.IsActive() {
+		return
+	}
+
+	e.mu.RLock()
+	site, exists := e.liveState[id]
+	e.mu.RUnlock()
+	if !exists || site.Paused {
+		return
+	}
+
+	switch site.Type {
+	case "push":
+		e.checkPush(ctx, site)
+	case "group":
+		e.checkGroup(ctx, site)
+	default:
+		result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
+		updatedSite := site
+		updatedSite.HasSSL = result.HasSSL
+		updatedSite.CertExpiry = result.CertExpiry
+		updatedSite.Latency = time.Duration(result.LatencyNs)
+		updatedSite.LastCheck = time.Now()
+		e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
+	}
+}
+
+func (e *Engine) checkPush(_ context.Context, site models.Site) {
+	if site.Status == models.StatusPending {
+		return
+	}
+
+	interval := time.Duration(site.Interval) * time.Second
+	grace := interval / 2
+	if grace < minPushGrace {
+		grace = minPushGrace
+	}
+
+	overdue := site.LastCheck.Add(interval)
+	staleMark := overdue.Add(grace / 2)
+	graceEnd := overdue.Add(grace)
+	now := time.Now()
+
+	if now.After(graceEnd) {
+		if site.Status != models.StatusDown {
+			e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
+		}
+	} else if now.After(staleMark) {
+		if site.Status != models.StatusStale {
+			e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
+		}
+	} else if now.After(overdue) {
+		if site.Status != models.StatusLate {
+			e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
+		}
+	}
+}
+
+func (e *Engine) checkGroup(_ context.Context, site models.Site) {
+	e.mu.RLock()
+	status := models.StatusUp
+	hasChildren := false
+	for _, child := range e.liveState {
+		if child.ParentID != site.ID || child.Type == "group" {
+			continue
+		}
+		hasChildren = true
+		if child.Paused || e.isInMaintenance(child.ID) {
+			continue
+		}
+		if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
+			status = models.StatusDown
+		} else if child.Status == models.StatusStale && status != models.StatusDown {
+			status = models.StatusStale
+		} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
+			status = models.StatusLate
+		} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
+			status = models.StatusPending
+		}
+	}
+	e.mu.RUnlock()
+
+	if !hasChildren {
+		status = models.StatusPending
+	}
+
+	e.applyState(site.ID, func(s *models.Site) {
+		s.Status = status
+	})
+	e.recordCheck(site.ID, 0, !status.IsBroken())
+}
+
+func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
+	e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
+}
+
+// SetAggStrategy must be called before Start: the field is read by the probe
+// aggregation path without synchronization.
+func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
+	e.aggStrategy = strategy
+}
+
+func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
+	e.mu.RLock()
+	site, exists := e.liveState[siteID]
+	e.mu.RUnlock()
+	if !exists {
+		return
+	}
+
+	staleAfter := time.Duration(site.Interval) * time.Second * 3
+	if staleAfter < time.Minute {
+		staleAfter = time.Minute
+	}
+
+	now := time.Now()
+	e.probeResultsMu.Lock()
+	if e.probeResults[siteID] == nil {
+		e.probeResults[siteID] = make(map[string]NodeResult)
+	}
+	e.probeResults[siteID][nodeID] = NodeResult{
+		NodeID:      nodeID,
+		IsUp:        isUp,
+		LatencyNs:   latencyNs,
+		CheckedAt:   now,
+		ErrorReason: errorReason,
+	}
+	results := make([]NodeResult, 0, len(e.probeResults[siteID]))
+	for id, r := range e.probeResults[siteID] {
+		if now.Sub(r.CheckedAt) > staleAfter {
+			delete(e.probeResults[siteID], id)
+			continue
+		}
+		results = append(results, r)
+	}
+	e.probeResultsMu.Unlock()
+
+	aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
+
+	probeStatus := models.StatusUp
+	if !aggUp {
+		probeStatus = models.StatusDown
+	}
+
+	updatedSite := site
+	updatedSite.Latency = time.Duration(avgLatency)
+	updatedSite.LastCheck = time.Now()
+	e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
+}
+
+func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
+	e.probeResultsMu.RLock()
+	defer e.probeResultsMu.RUnlock()
+	src := e.probeResults[siteID]
+	cp := make(map[string]NodeResult, len(src))
+	for k, v := range src {
+		cp[k] = v
+	}
+	return cp
+}

internal/monitor/checks_test.go

@@ -0,0 +1,389 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
+)
+
+// --- Group 2: Heartbeat ---
+
+func TestRecordHeartbeat_ValidToken(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	if !e.RecordHeartbeat("abc123") {
+		t.Error("expected true for valid token")
+	}
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP, got %s", s.Status)
+	}
+	if time.Since(s.LastCheck) > time.Second {
+		t.Error("expected LastCheck to be recent")
+	}
+}
+
+func TestRecordHeartbeat_RecoveryFromDown(t *testing.T) {
+	ms := newMockStore()
+	ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123", AlertID: 1},
+		SiteState:  models.SiteState{Status: "DOWN", FailureCount: 3},
+	}
+	injectSite(e, site)
+
+	if !e.RecordHeartbeat("abc123") {
+		t.Error("expected true")
+	}
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP, got %s", s.Status)
+	}
+	if s.FailureCount != 0 {
+		t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
+	}
+	waitAsync()
+	if len(ms.getAlertCallsSnapshot()) == 0 {
+		t.Error("expected recovery alert")
+	}
+}
+
+func TestRecordHeartbeat_UnknownToken(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+
+	if e.RecordHeartbeat("unknown") {
+		t.Error("expected false for unknown token")
+	}
+}
+
+func TestRecordHeartbeat_InactiveEngine(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Type: "push", Token: "abc123"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+	e.SetActive(false)
+
+	if e.RecordHeartbeat("abc123") {
+		t.Error("expected false when inactive")
+	}
+}
+
+// --- Group 3: Push Deadline ---
+
+func TestCheckPush_DeadlineMissed(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 10, MaxRetries: 0},
+		SiteState:  models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
+	}
+	injectSite(e, site)
+
+	e.checkPush(context.Background(), site)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected DOWN after missed deadline, got %s", s.Status)
+	}
+}
+
+func TestCheckPush_OverdueBecomesLate(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
+		SiteState:  models.SiteState{Status: "UP", LastCheck: time.Now().Add(-310 * time.Second)},
+	}
+	injectSite(e, site)
+
+	e.checkPush(context.Background(), site)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "LATE" {
+		t.Errorf("expected LATE when overdue but within grace, got %s", s.Status)
+	}
+}
+
+func TestCheckPush_OverdueBecomesStale(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	// interval=300, grace=150 (300/2), staleMark=overdue+75
+	// at 380s: past staleMark(375) but before graceEnd(450)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
+		SiteState:  models.SiteState{Status: "UP", LastCheck: time.Now().Add(-380 * time.Second)},
+	}
+	injectSite(e, site)
+
+	e.checkPush(context.Background(), site)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "STALE" {
+		t.Errorf("expected STALE when past midpoint of grace, got %s", s.Status)
+	}
+}
+
+func TestCheckPush_WithinDeadline(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
+		SiteState:  models.SiteState{Status: "UP", LastCheck: time.Now()},
+	}
+	injectSite(e, site)
+
+	e.checkPush(context.Background(), site)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP, got %s", s.Status)
+	}
+}
+
+func TestCheckPush_PendingStaysPending(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
+		SiteState:  models.SiteState{Status: "PENDING"},
+	}
+	injectSite(e, site)
+
+	e.checkPush(context.Background(), site)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "PENDING" {
+		t.Errorf("expected PENDING to stay until first heartbeat, got %s", s.Status)
+	}
+}
+
+// --- Group 4: Group Checks ---
+
+func TestCheckGroup_AllChildrenUp(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	group := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
+		SiteState:  models.SiteState{Status: "PENDING"},
+	}
+	child1 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child2 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, group)
+	injectSite(e, child1)
+	injectSite(e, child2)
+
+	e.checkGroup(context.Background(), group)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected group UP, got %s", s.Status)
+	}
+}
+
+func TestCheckGroup_OneChildDown(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	group := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child1 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child2 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
+		SiteState:  models.SiteState{Status: "DOWN"},
+	}
+	injectSite(e, group)
+	injectSite(e, child1)
+	injectSite(e, child2)
+
+	e.checkGroup(context.Background(), group)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "DOWN" {
+		t.Errorf("expected group DOWN, got %s", s.Status)
+	}
+}
+
+func TestCheckGroup_PausedChildIgnored(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	group := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
+	}
+	child1 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child2 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
+		SiteState:  models.SiteState{Status: "DOWN"},
+	}
+	injectSite(e, group)
+	injectSite(e, child1)
+	injectSite(e, child2)
+
+	e.checkGroup(context.Background(), group)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP (paused child ignored), got %s", s.Status)
+	}
+}
+
+func TestCheckGroup_MaintenanceChildIgnored(t *testing.T) {
+	ms := newMockStore()
+	ms.maintenance[3] = true
+	e := newTestEngine(ms)
+	group := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
+	}
+	child1 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child2 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
+		SiteState:  models.SiteState{Status: "DOWN"},
+	}
+	injectSite(e, group)
+	injectSite(e, child1)
+	injectSite(e, child2)
+	e.refreshMaintenanceCache(context.Background())
+
+	e.checkGroup(context.Background(), group)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "UP" {
+		t.Errorf("expected UP (maint child ignored), got %s", s.Status)
+	}
+}
+
+func TestCheckGroup_NoChildren(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	group := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, group)
+
+	e.checkGroup(context.Background(), group)
+
+	s, _ := getSite(e, 1)
+	if s.Status != "PENDING" {
+		t.Errorf("expected PENDING for no children, got %s", s.Status)
+	}
+}
+
+// Groups must not auto-pause when all children are paused — that creates a
+// one-way trap because monitorRoutine skips paused sites.
+func TestCheckGroup_AllPausedNoAutoFreeze(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	group := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child1 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1, Paused: true},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child2 := models.Site{
+		SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, group)
+	injectSite(e, child1)
+	injectSite(e, child2)
+
+	e.checkGroup(context.Background(), group)
+
+	s, _ := getSite(e, 1)
+	if s.Paused {
+		t.Error("group must not auto-pause when all children are paused")
+	}
+}
+
+// Dead probe results must be expired so they don't poison aggregation.
+func TestIngestProbeResult_ExpiresStaleProbes(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", Interval: 30},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	e.probeResultsMu.Lock()
+	e.probeResults[1] = map[string]NodeResult{
+		"dead-probe": {
+			NodeID:    "dead-probe",
+			IsUp:      false,
+			CheckedAt: time.Now().Add(-10 * time.Minute),
+		},
+	}
+	e.probeResultsMu.Unlock()
+
+	e.IngestProbeResult("live-probe", 1, 5000, true, "")
+
+	e.probeResultsMu.RLock()
+	_, deadExists := e.probeResults[1]["dead-probe"]
+	_, liveExists := e.probeResults[1]["live-probe"]
+	e.probeResultsMu.RUnlock()
+
+	if deadExists {
+		t.Error("stale probe result should have been expired")
+	}
+	if !liveExists {
+		t.Error("live probe result should still exist")
+	}
+}
+
+// RemoveSite must clean up probeResults.
+func TestRemoveSite_CleansProbeResults(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	e.probeResultsMu.Lock()
+	e.probeResults[1] = map[string]NodeResult{
+		"node-a": {NodeID: "node-a", IsUp: true, CheckedAt: time.Now()},
+	}
+	e.probeResultsMu.Unlock()
+
+	e.RemoveSite(1)
+
+	e.probeResultsMu.RLock()
+	defer e.probeResultsMu.RUnlock()
+	if _, exists := e.probeResults[1]; exists {
+		t.Error("probe results should be cleaned up after RemoveSite")
+	}
+}

internal/monitor/maintenance.go

@@ -0,0 +1,70 @@
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+func (e *Engine) maintenancePruner(ctx context.Context) {
+	ticker := time.NewTicker(maintPruneInterval)
+	defer ticker.Stop()
+
+	e.pruneMaintenanceWindows(ctx)
+
+	for {
+		select {
+		case <-ticker.C:
+			e.pruneMaintenanceWindows(ctx)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
+	pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
+	if err != nil {
+		e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
+		return
+	}
+	if pruned > 0 {
+		e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
+	}
+}
+
+func (e *Engine) isInMaintenance(monitorID int) bool {
+	e.maintCacheMu.RLock()
+	defer e.maintCacheMu.RUnlock()
+	return e.maintCache[monitorID]
+}
+
+func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
+	windows, err := e.db.GetActiveMaintenanceWindows(ctx)
+	if err != nil {
+		return
+	}
+
+	directMaint := make(map[int]bool)
+	var globalMaint bool
+	for _, w := range windows {
+		if w.MonitorID == 0 {
+			globalMaint = true
+		} else {
+			directMaint[w.MonitorID] = true
+		}
+	}
+
+	resolved := make(map[int]bool)
+	e.mu.RLock()
+	for id, site := range e.liveState {
+		if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
+			resolved[id] = true
+		}
+	}
+	e.mu.RUnlock()
+
+	e.maintCacheMu.Lock()
+	e.maintCache = resolved
+	e.maintCacheMu.Unlock()
+}

internal/monitor/maintenance_test.go

@@ -0,0 +1,53 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
+)
+
+// Maintenance cache resolves parent relationships correctly.
+func TestIsInMaintenance_UsesCache(t *testing.T) {
+	ms := newMockStore()
+	ms.maintenance[10] = true // direct maintenance on group
+	e := newTestEngine(ms)
+	group := models.Site{
+		SiteConfig: models.SiteConfig{ID: 10, Name: "group", Type: "group"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	child := models.Site{
+		SiteConfig: models.SiteConfig{ID: 20, Name: "child", Type: "http", ParentID: 10},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, group)
+	injectSite(e, child)
+	e.refreshMaintenanceCache(context.Background())
+
+	if !e.isInMaintenance(10) {
+		t.Error("group should be in maintenance (direct)")
+	}
+	if !e.isInMaintenance(20) {
+		t.Error("child should be in maintenance (parent)")
+	}
+	if e.isInMaintenance(99) {
+		t.Error("unknown monitor should not be in maintenance")
+	}
+}
+
+// Global maintenance (monitor_id=0) applies to all monitors.
+func TestIsInMaintenance_GlobalMaintenance(t *testing.T) {
+	ms := newMockStore()
+	ms.maintenance[0] = true
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+	e.refreshMaintenanceCache(context.Background())
+
+	if !e.isInMaintenance(1) {
+		t.Error("all monitors should be in maintenance during global window")
+	}
+}

internal/monitor/monitor.go

@@ -4,14 +4,12 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"math/rand/v2"
 	"net/http"
 	"regexp"
 	"strings"
 	"sync"
 	"time"
 
-	"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
 	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
 	"gitea.lerkolabs.com/lerkolabs/uptop/internal/store"
 )
@@ -28,14 +26,6 @@ const (
 	dbPruneInterval       = 10 * time.Minute
 )
 
-type AlertHealth struct {
-	LastSendAt time.Time
-	LastSendOK bool
-	LastError  string
-	SendCount  int
-	FailCount  int
-}
-
 type Engine struct {
 	mu        sync.RWMutex
 	liveState map[int]models.Site
@@ -261,26 +251,6 @@ func (e *Engine) InitLogs() {
 	e.logStore = entries
 }
 
-// InitAlertHealth restores persisted alert send health so the dashboard shows real
-// "last sent" / health state on startup instead of resetting every channel to "never".
-func (e *Engine) InitAlertHealth() {
-	records, err := e.db.LoadAlertHealth(context.Background())
-	if err != nil {
-		return
-	}
-	e.alertHealthMu.Lock()
-	defer e.alertHealthMu.Unlock()
-	for id, r := range records {
-		e.alertHealth[id] = AlertHealth{
-			LastSendAt: r.LastSendAt,
-			LastSendOK: r.LastSendOK,
-			LastError:  r.LastError,
-			SendCount:  r.SendCount,
-			FailCount:  r.FailCount,
-		}
-	}
-}
-
 func (e *Engine) GetLogs() []models.LogEntry {
 	e.logMu.RLock()
 	defer e.logMu.RUnlock()
@@ -289,123 +259,6 @@ func (e *Engine) GetLogs() []models.LogEntry {
 	return logs
 }
 
-func (e *Engine) SetActive(active bool) {
-	e.activeMu.Lock()
-	defer e.activeMu.Unlock()
-	if e.isActive != active {
-		e.isActive = active
-		status := "RESUMED (Active)"
-		if !active {
-			status = "PAUSED (Passive)"
-		}
-		e.AddLog(fmt.Sprintf("Engine %s", status))
-	}
-}
-
-func (e *Engine) IsActive() bool {
-	e.activeMu.RLock()
-	defer e.activeMu.RUnlock()
-	return e.isActive
-}
-
-func (e *Engine) GetAllSites() []models.Site {
-	e.mu.RLock()
-	defer e.mu.RUnlock()
-	sites := make([]models.Site, 0, len(e.liveState))
-	for _, s := range e.liveState {
-		sites = append(sites, s)
-	}
-	return sites
-}
-
-func (e *Engine) GetLiveState() map[int]models.Site {
-	e.mu.RLock()
-	defer e.mu.RUnlock()
-	cp := make(map[int]models.Site, len(e.liveState))
-	for k, v := range e.liveState {
-		cp[k] = v
-	}
-	return cp
-}
-
-func (e *Engine) RecordHeartbeat(token string) bool {
-	if !e.IsActive() {
-		return false
-	}
-
-	e.mu.RLock()
-	targetID, ok := e.tokenIndex[token]
-	e.mu.RUnlock()
-	if !ok {
-		return false
-	}
-
-	var (
-		prevStatus models.Status
-		name       string
-		alertID    int
-		downSince  time.Time
-	)
-	_, exists := e.applyState(targetID, func(s *models.Site) {
-		prevStatus = s.Status
-		name = s.Name
-		alertID = s.AlertID
-		downSince = s.StatusChangedAt // captured before mutation = when it went down
-
-		s.LastCheck = time.Now()
-		s.Status = models.StatusUp
-		s.FailureCount = 0
-		s.Latency = 0
-		s.LastError = ""
-		s.LastSuccessAt = time.Now()
-		if prevStatus != models.StatusUp {
-			s.StatusChangedAt = time.Now()
-		}
-	})
-	if !exists {
-		return false
-	}
-
-	switch prevStatus {
-	case models.StatusPending:
-		e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
-	case models.StatusLate:
-		e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
-	case models.StatusStale:
-		e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
-	case models.StatusDown:
-		downDur := ""
-		if !downSince.IsZero() {
-			downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
-		}
-		e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
-		go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
-	}
-
-	e.recordCheck(targetID, 0, true)
-
-	if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
-		e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
-	}
-
-	return true
-}
-
-func (e *Engine) addToTokenIndex(site models.Site) {
-	if site.Type == "push" && site.Token != "" {
-		e.tokenIndex[site.Token] = site.ID
-	}
-}
-
-func (e *Engine) removeFromTokenIndex(id int) {
-	for token, sid := range e.tokenIndex {
-		if sid == id {
-			delete(e.tokenIndex, token)
-			return
-		}
-	}
-}
-
 func (e *Engine) Start(ctx context.Context) {
 	// e.cancel is invoked by Stop() to drain and halt the writer; gosec can't
 	// trace the cross-method call, and cancelling the parent reaps this child
@@ -496,637 +349,3 @@ func (e *Engine) Start(ctx context.Context) {
 		e.maintenancePruner(ctx)
 	}()
 }
-
-func (e *Engine) maintenancePruner(ctx context.Context) {
-	ticker := time.NewTicker(maintPruneInterval)
-	defer ticker.Stop()
-
-	e.pruneMaintenanceWindows(ctx)
-
-	for {
-		select {
-		case <-ticker.C:
-			e.pruneMaintenanceWindows(ctx)
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
-func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
-	pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
-	if err != nil {
-		e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
-		return
-	}
-	if pruned > 0 {
-		e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
-	}
-}
-
-func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
-	e.mu.Lock()
-	if existing, ok := e.liveState[cfg.ID]; ok {
-		e.removeFromTokenIndex(cfg.ID)
-		existing.SiteConfig = cfg
-		e.liveState[cfg.ID] = existing
-		e.addToTokenIndex(existing)
-	}
-	e.mu.Unlock()
-
-	e.signalRecheck(cfg.ID)
-}
-
-func (e *Engine) getRecheckChan(id int) chan struct{} {
-	e.recheckMu.Lock()
-	defer e.recheckMu.Unlock()
-	ch, ok := e.recheck[id]
-	if !ok {
-		ch = make(chan struct{}, 1)
-		e.recheck[id] = ch
-	}
-	return ch
-}
-
-func (e *Engine) signalRecheck(id int) {
-	ch := e.getRecheckChan(id)
-	select {
-	case ch <- struct{}{}:
-	default:
-	}
-}
-
-func (e *Engine) RemoveSite(id int) {
-	e.mu.Lock()
-	e.removeFromTokenIndex(id)
-	delete(e.liveState, id)
-	e.mu.Unlock()
-	e.removeHistory(id)
-
-	e.probeResultsMu.Lock()
-	delete(e.probeResults, id)
-	e.probeResultsMu.Unlock()
-
-	e.recheckMu.Lock()
-	delete(e.recheck, id)
-	e.recheckMu.Unlock()
-}
-
-func (e *Engine) ToggleSitePause(id int) bool {
-	var (
-		paused bool
-		name   string
-	)
-	_, ok := e.applyState(id, func(s *models.Site) {
-		s.Paused = !s.Paused
-		paused = s.Paused
-		name = s.Name
-	})
-	if !ok {
-		return false
-	}
-	if paused {
-		e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
-	} else {
-		e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
-	}
-	return paused
-}
-
-func (e *Engine) monitorRoutine(ctx context.Context, id int) {
-	recheckCh := e.getRecheckChan(id)
-
-	// Stagger initial check to avoid thundering herd on startup
-	stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
-	select {
-	case <-time.After(stagger):
-	case <-ctx.Done():
-		return
-	}
-
-	e.checkByID(ctx, id)
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-
-		if !e.IsActive() {
-			select {
-			case <-time.After(pollInterval):
-			case <-ctx.Done():
-				return
-			case <-recheckCh:
-			}
-			continue
-		}
-
-		e.mu.RLock()
-		site, exists := e.liveState[id]
-		e.mu.RUnlock()
-		if !exists {
-			return
-		}
-
-		if site.Paused {
-			select {
-			case <-time.After(pollInterval):
-			case <-ctx.Done():
-				return
-			case <-recheckCh:
-			}
-			continue
-		}
-
-		interval := site.Interval
-		if interval < minCheckInterval {
-			interval = minCheckInterval
-		}
-		jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
-		select {
-		case <-time.After(time.Duration(interval)*time.Second + jitter):
-		case <-ctx.Done():
-			return
-		case <-recheckCh:
-		}
-		e.checkByID(ctx, id)
-	}
-}
-
-// applyState atomically reads, mutates, and writes back the live entry for id.
-// The mutator runs under the engine write lock and receives a pointer to the
-// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
-// never clobbered by a stale snapshot. The mutator must only touch runtime /
-// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
-// are owned by UpdateSiteConfig and must not be written here. Returns the
-// post-mutation copy and whether the site still exists.
-func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	cur, ok := e.liveState[id]
-	if !ok {
-		return models.Site{}, false
-	}
-	mutate(&cur)
-	e.liveState[id] = cur
-	return cur, true
-}
-
-func (e *Engine) checkByID(ctx context.Context, id int) {
-	if !e.IsActive() {
-		return
-	}
-
-	e.mu.RLock()
-	site, exists := e.liveState[id]
-	e.mu.RUnlock()
-	if !exists || site.Paused {
-		return
-	}
-
-	switch site.Type {
-	case "push":
-		e.checkPush(ctx, site)
-	case "group":
-		e.checkGroup(ctx, site)
-	default:
-		result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
-		updatedSite := site
-		updatedSite.HasSSL = result.HasSSL
-		updatedSite.CertExpiry = result.CertExpiry
-		updatedSite.Latency = time.Duration(result.LatencyNs)
-		updatedSite.LastCheck = time.Now()
-		e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
-	}
-}
-
-func (e *Engine) checkPush(_ context.Context, site models.Site) {
-	if site.Status == models.StatusPending {
-		return
-	}
-
-	interval := time.Duration(site.Interval) * time.Second
-	grace := interval / 2
-	if grace < minPushGrace {
-		grace = minPushGrace
-	}
-
-	overdue := site.LastCheck.Add(interval)
-	staleMark := overdue.Add(grace / 2)
-	graceEnd := overdue.Add(grace)
-	now := time.Now()
-
-	if now.After(graceEnd) {
-		if site.Status != models.StatusDown {
-			e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
-		}
-	} else if now.After(staleMark) {
-		if site.Status != models.StatusStale {
-			e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
-		}
-	} else if now.After(overdue) {
-		if site.Status != models.StatusLate {
-			e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
-		}
-	}
-}
-
-// handleStatusChange folds a check result into the live state. snap is the
-// stale snapshot the check ran against; the actual mutation is applied onto the
-// CURRENT live entry via applyState, so a concurrent pause / config edit /
-// heartbeat is never reverted by this write. Logs and alerts are emitted after
-// the lock is released, off the critical section.
-func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
-	if !e.IsActive() {
-		return
-	}
-
-	inMaint := e.isInMaintenance(snap.ID)
-	status := models.Status(rawStatus)
-
-	var (
-		prev, next            models.Status
-		name, typ             string
-		alertID               int
-		failCount, maxRetries int
-		confirmedDown         bool
-		failedCheck           bool
-		downSince             time.Time
-		sslWarnFire           bool
-		sslDays               int
-		skipped               bool
-		changed               bool
-	)
-
-	_, exists := e.applyState(snap.ID, func(s *models.Site) {
-		// A non-UP result computed from a stale snapshot must not override a
-		// heartbeat (or newer check) that landed while we were evaluating.
-		if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
-			skipped = true
-			return
-		}
-
-		prev = s.Status
-		name = s.Name
-		typ = s.Type
-		alertID = s.AlertID
-		maxRetries = s.MaxRetries
-		downSince = s.StatusChangedAt
-
-		// Fresh check results (measured by the run against snap).
-		s.StatusCode = code
-		s.Latency = snap.Latency
-		s.LastCheck = snap.LastCheck
-		s.HasSSL = snap.HasSSL
-		s.CertExpiry = snap.CertExpiry
-		s.LastError = errorReason
-		if status == models.StatusUp {
-			s.LastSuccessAt = time.Now()
-			s.LastError = ""
-		}
-
-		// Status + failure-count transition, based on the CURRENT live status.
-		if status == models.StatusUp {
-			s.FailureCount = 0
-			s.Status = models.StatusUp
-		} else {
-			if s.FailureCount <= s.MaxRetries {
-				s.FailureCount++
-			}
-			if s.FailureCount > s.MaxRetries {
-				if s.Status != status {
-					confirmedDown = true
-				}
-				s.Status = status
-				s.FailureCount = s.MaxRetries + 1
-			} else {
-				failedCheck = true
-			}
-		}
-		failCount = s.FailureCount
-
-		if s.Status != prev && prev != models.StatusPending {
-			s.StatusChangedAt = time.Now()
-		} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
-			s.StatusChangedAt = time.Now()
-		}
-
-		// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
-		if typ == "http" && s.CheckSSL && s.HasSSL {
-			days := int(time.Until(s.CertExpiry).Hours() / 24)
-			if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
-				sslWarnFire = true
-				sslDays = days
-				s.SentSSLWarning = true
-			} else if days > s.ExpiryThreshold {
-				s.SentSSLWarning = false
-			}
-		}
-
-		next = s.Status
-		changed = next != prev
-	})
-
-	if !exists || skipped {
-		return
-	}
-
-	e.recordCheck(snap.ID, latency, status == models.StatusUp)
-
-	if confirmedDown {
-		if errorReason != "" {
-			e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
-		} else {
-			e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
-		}
-	} else if failedCheck {
-		e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
-	}
-
-	if changed && prev != models.StatusPending {
-		e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
-	}
-
-	if sslWarnFire {
-		if !inMaint {
-			e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
-		} else {
-			e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
-		}
-	}
-
-	if prev == models.StatusUp && next == models.StatusLate {
-		e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
-	}
-
-	if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
-		if inMaint {
-			e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
-		} else {
-			msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
-			if errorReason != "" {
-				msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
-			}
-			if typ == "push" {
-				msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
-			}
-			e.triggerAlert(alertID, "🚨 ALERT", msg)
-		}
-	}
-	if prev.IsBroken() && next == models.StatusUp {
-		downDur := ""
-		if !downSince.IsZero() {
-			downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
-		}
-		e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
-		if !inMaint {
-			e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
-		}
-	}
-	if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
-		e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
-	}
-}
-
-func (e *Engine) triggerAlert(alertID int, title, message string) {
-	if alertID <= 0 {
-		return
-	}
-	cfg, err := e.db.GetAlert(context.Background(), alertID)
-	if err != nil {
-		e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
-		return
-	}
-	provider := alert.GetProvider(cfg)
-	if provider != nil {
-		go func() {
-			ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
-			defer cancel()
-			if err := provider.Send(ctx, title, message); err != nil {
-				e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
-				e.recordAlertResult(alertID, false, err.Error())
-			} else {
-				e.recordAlertResult(alertID, true, "")
-			}
-		}()
-	}
-}
-
-func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
-	e.alertHealthMu.Lock()
-	defer e.alertHealthMu.Unlock()
-	h := e.alertHealth[alertID]
-	h.LastSendAt = time.Now()
-	h.LastSendOK = ok
-	h.SendCount++
-	if ok {
-		h.LastError = ""
-	} else {
-		h.LastError = errMsg
-		h.FailCount++
-	}
-	e.alertHealth[alertID] = h
-
-	// Persist so health survives restarts; DB IO off the alert path.
-	e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
-		AlertID:    alertID,
-		LastSendAt: h.LastSendAt,
-		LastSendOK: h.LastSendOK,
-		LastError:  h.LastError,
-		SendCount:  h.SendCount,
-		FailCount:  h.FailCount,
-	}})
-}
-
-func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
-	e.alertHealthMu.RLock()
-	defer e.alertHealthMu.RUnlock()
-	return e.alertHealth[alertID]
-}
-
-func (e *Engine) TestAlert(alertID int) error {
-	cfg, err := e.db.GetAlert(context.Background(), alertID)
-	if err != nil {
-		return fmt.Errorf("failed to load alert: %w", err)
-	}
-	provider := alert.GetProvider(cfg)
-	if provider == nil {
-		return fmt.Errorf("no provider for type %q", cfg.Type)
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
-	defer cancel()
-	err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
-	if err != nil {
-		e.recordAlertResult(alertID, false, err.Error())
-		return err
-	}
-	e.recordAlertResult(alertID, true, "")
-	e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
-	return nil
-}
-
-func (e *Engine) isInMaintenance(monitorID int) bool {
-	e.maintCacheMu.RLock()
-	defer e.maintCacheMu.RUnlock()
-	return e.maintCache[monitorID]
-}
-
-func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
-	windows, err := e.db.GetActiveMaintenanceWindows(ctx)
-	if err != nil {
-		return
-	}
-
-	directMaint := make(map[int]bool)
-	var globalMaint bool
-	for _, w := range windows {
-		if w.MonitorID == 0 {
-			globalMaint = true
-		} else {
-			directMaint[w.MonitorID] = true
-		}
-	}
-
-	resolved := make(map[int]bool)
-	e.mu.RLock()
-	for id, site := range e.liveState {
-		if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
-			resolved[id] = true
-		}
-	}
-	e.mu.RUnlock()
-
-	e.maintCacheMu.Lock()
-	e.maintCache = resolved
-	e.maintCacheMu.Unlock()
-}
-
-func (e *Engine) GetDisplayStatus(site models.Site) string {
-	if site.Paused {
-		return "PAUSED"
-	}
-	if e.isInMaintenance(site.ID) {
-		return "MAINT"
-	}
-	return string(site.Status)
-}
-
-func (e *Engine) checkGroup(_ context.Context, site models.Site) {
-	e.mu.RLock()
-	status := models.StatusUp
-	hasChildren := false
-	for _, child := range e.liveState {
-		if child.ParentID != site.ID || child.Type == "group" {
-			continue
-		}
-		hasChildren = true
-		if child.Paused || e.isInMaintenance(child.ID) {
-			continue
-		}
-		if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
-			status = models.StatusDown
-		} else if child.Status == models.StatusStale && status != models.StatusDown {
-			status = models.StatusStale
-		} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
-			status = models.StatusLate
-		} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
-			status = models.StatusPending
-		}
-	}
-	e.mu.RUnlock()
-
-	if !hasChildren {
-		status = models.StatusPending
-	}
-
-	e.applyState(site.ID, func(s *models.Site) {
-		s.Status = status
-	})
-	e.recordCheck(site.ID, 0, !status.IsBroken())
-}
-
-func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
-	e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
-}
-
-// SetAggStrategy must be called before Start: the field is read by the probe
-// aggregation path without synchronization.
-func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
-	e.aggStrategy = strategy
-}
-
-func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
-	e.mu.RLock()
-	site, exists := e.liveState[siteID]
-	e.mu.RUnlock()
-	if !exists {
-		return
-	}
-
-	staleAfter := time.Duration(site.Interval) * time.Second * 3
-	if staleAfter < time.Minute {
-		staleAfter = time.Minute
-	}
-
-	now := time.Now()
-	e.probeResultsMu.Lock()
-	if e.probeResults[siteID] == nil {
-		e.probeResults[siteID] = make(map[string]NodeResult)
-	}
-	e.probeResults[siteID][nodeID] = NodeResult{
-		NodeID:      nodeID,
-		IsUp:        isUp,
-		LatencyNs:   latencyNs,
-		CheckedAt:   now,
-		ErrorReason: errorReason,
-	}
-	results := make([]NodeResult, 0, len(e.probeResults[siteID]))
-	for id, r := range e.probeResults[siteID] {
-		if now.Sub(r.CheckedAt) > staleAfter {
-			delete(e.probeResults[siteID], id)
-			continue
-		}
-		results = append(results, r)
-	}
-	e.probeResultsMu.Unlock()
-
-	aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
-
-	probeStatus := models.StatusUp
-	if !aggUp {
-		probeStatus = models.StatusDown
-	}
-
-	updatedSite := site
-	updatedSite.Latency = time.Duration(avgLatency)
-	updatedSite.LastCheck = time.Now()
-	e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
-}
-
-func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
-	e.probeResultsMu.RLock()
-	defer e.probeResultsMu.RUnlock()
-	src := e.probeResults[siteID]
-	cp := make(map[string]NodeResult, len(src))
-	for k, v := range src {
-		cp[k] = v
-	}
-	return cp
-}
-
-func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
-	changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
-	if err != nil {
-		return nil
-	}
-	return changes
-}
-
-func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
-	changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
-	if err != nil {
-		return nil
-	}
-	return changes
-}

internal/monitor/sites.go

@@ -0,0 +1,158 @@
+package monitor
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
+)
+
+func (e *Engine) SetActive(active bool) {
+	e.activeMu.Lock()
+	defer e.activeMu.Unlock()
+	if e.isActive != active {
+		e.isActive = active
+		status := "RESUMED (Active)"
+		if !active {
+			status = "PAUSED (Passive)"
+		}
+		e.AddLog(fmt.Sprintf("Engine %s", status))
+	}
+}
+
+func (e *Engine) IsActive() bool {
+	e.activeMu.RLock()
+	defer e.activeMu.RUnlock()
+	return e.isActive
+}
+
+func (e *Engine) GetAllSites() []models.Site {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	sites := make([]models.Site, 0, len(e.liveState))
+	for _, s := range e.liveState {
+		sites = append(sites, s)
+	}
+	return sites
+}
+
+func (e *Engine) GetLiveState() map[int]models.Site {
+	e.mu.RLock()
+	defer e.mu.RUnlock()
+	cp := make(map[int]models.Site, len(e.liveState))
+	for k, v := range e.liveState {
+		cp[k] = v
+	}
+	return cp
+}
+
+func (e *Engine) addToTokenIndex(site models.Site) {
+	if site.Type == "push" && site.Token != "" {
+		e.tokenIndex[site.Token] = site.ID
+	}
+}
+
+func (e *Engine) removeFromTokenIndex(id int) {
+	for token, sid := range e.tokenIndex {
+		if sid == id {
+			delete(e.tokenIndex, token)
+			return
+		}
+	}
+}
+
+func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
+	e.mu.Lock()
+	if existing, ok := e.liveState[cfg.ID]; ok {
+		e.removeFromTokenIndex(cfg.ID)
+		existing.SiteConfig = cfg
+		e.liveState[cfg.ID] = existing
+		e.addToTokenIndex(existing)
+	}
+	e.mu.Unlock()
+
+	e.signalRecheck(cfg.ID)
+}
+
+func (e *Engine) RemoveSite(id int) {
+	e.mu.Lock()
+	e.removeFromTokenIndex(id)
+	delete(e.liveState, id)
+	e.mu.Unlock()
+	e.removeHistory(id)
+
+	e.probeResultsMu.Lock()
+	delete(e.probeResults, id)
+	e.probeResultsMu.Unlock()
+
+	e.recheckMu.Lock()
+	delete(e.recheck, id)
+	e.recheckMu.Unlock()
+}
+
+func (e *Engine) ToggleSitePause(id int) bool {
+	var (
+		paused bool
+		name   string
+	)
+	_, ok := e.applyState(id, func(s *models.Site) {
+		s.Paused = !s.Paused
+		paused = s.Paused
+		name = s.Name
+	})
+	if !ok {
+		return false
+	}
+	if paused {
+		e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
+	} else {
+		e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
+	}
+	return paused
+}
+
+// applyState atomically reads, mutates, and writes back the live entry for id.
+// The mutator runs under the engine write lock and receives a pointer to the
+// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
+// never clobbered by a stale snapshot. The mutator must only touch runtime /
+// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
+// are owned by UpdateSiteConfig and must not be written here. Returns the
+// post-mutation copy and whether the site still exists.
+func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	cur, ok := e.liveState[id]
+	if !ok {
+		return models.Site{}, false
+	}
+	mutate(&cur)
+	e.liveState[id] = cur
+	return cur, true
+}
+
+func (e *Engine) GetDisplayStatus(site models.Site) string {
+	if site.Paused {
+		return "PAUSED"
+	}
+	if e.isInMaintenance(site.ID) {
+		return "MAINT"
+	}
+	return string(site.Status)
+}
+
+func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
+	changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
+	if err != nil {
+		return nil
+	}
+	return changes
+}
+
+func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
+	changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
+	if err != nil {
+		return nil
+	}
+	return changes
+}

internal/monitor/sites_test.go

@@ -0,0 +1,133 @@
+package monitor
+
+import (
+	"testing"
+	"time"
+
+	"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
+)
+
+func TestUpdateSiteConfig_PreservesRuntime(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com"},
+		SiteState:  models.SiteState{Status: "DOWN", FailureCount: 3, Latency: 100 * time.Millisecond},
+	}
+	injectSite(e, site)
+
+	updated := models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Interval: 60}
+	e.UpdateSiteConfig(updated)
+
+	s, _ := getSite(e, 1)
+	if s.URL != "http://new.com" {
+		t.Errorf("expected URL updated, got %s", s.URL)
+	}
+	if s.Status != "DOWN" {
+		t.Errorf("expected Status preserved, got %s", s.Status)
+	}
+	if s.FailureCount != 3 {
+		t.Errorf("expected FailureCount preserved, got %d", s.FailureCount)
+	}
+	if s.Latency != 100*time.Millisecond {
+		t.Errorf("expected Latency preserved, got %v", s.Latency)
+	}
+}
+
+func TestRemoveSite_CleansUp(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "push", Token: "tok1"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+	e.recordCheck(1, 5*time.Millisecond, true)
+
+	e.RemoveSite(1)
+
+	if _, ok := getSite(e, 1); ok {
+		t.Error("expected site removed from liveState")
+	}
+	if e.RecordHeartbeat("tok1") {
+		t.Error("expected token removed from index")
+	}
+	if _, ok := e.GetHistory(1); ok {
+		t.Error("expected history removed")
+	}
+}
+
+func TestToggleSitePause(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	site := models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "test"},
+		SiteState:  models.SiteState{Status: "UP"},
+	}
+	injectSite(e, site)
+
+	paused := e.ToggleSitePause(1)
+	if !paused {
+		t.Error("expected paused=true after first toggle")
+	}
+	s, _ := getSite(e, 1)
+	if !s.Paused {
+		t.Error("expected Paused=true in state")
+	}
+
+	paused = e.ToggleSitePause(1)
+	if paused {
+		t.Error("expected paused=false after second toggle")
+	}
+}
+
+func TestToggleSitePause_NonexistentSite(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	if e.ToggleSitePause(999) {
+		t.Error("expected false for nonexistent site")
+	}
+}
+
+func TestGetAllSites_ReturnsCopy(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	injectSite(e, models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
+		SiteState:  models.SiteState{Status: "UP"},
+	})
+	injectSite(e, models.Site{
+		SiteConfig: models.SiteConfig{ID: 2, Name: "s2"},
+		SiteState:  models.SiteState{Status: "DOWN"},
+	})
+
+	sites := e.GetAllSites()
+	if len(sites) != 2 {
+		t.Fatalf("expected 2 sites, got %d", len(sites))
+	}
+	sites[0].Name = "mutated"
+
+	fresh := e.GetAllSites()
+	for _, s := range fresh {
+		if s.Name == "mutated" {
+			t.Error("GetAllSites returned reference, not copy")
+		}
+	}
+}
+
+func TestGetLiveState_ReturnsCopy(t *testing.T) {
+	ms := newMockStore()
+	e := newTestEngine(ms)
+	injectSite(e, models.Site{
+		SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
+		SiteState:  models.SiteState{Status: "UP"},
+	})
+
+	state := e.GetLiveState()
+	state[1] = models.Site{SiteConfig: models.SiteConfig{Name: "mutated"}}
+
+	fresh := e.GetLiveState()
+	if fresh[1].Name == "mutated" {
+		t.Error("GetLiveState returned reference, not copy")
+	}
+}