refactor: decompose monitor.go and sqlstore.go into focused files
Split monitor.go (1131→350 lines) into alerts.go, checks.go, sites.go, and maintenance.go by concern. Split sqlstore.go (833→470 lines) into sqlstore_alerts.go, sqlstore_history.go, and sqlstore_maintenance.go by domain. Tests move with their implementation. Pure reorganization — no behavioral changes.
This commit was merged in pull request #154.
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
type AlertHealth struct {
|
||||
LastSendAt time.Time
|
||||
LastSendOK bool
|
||||
LastError string
|
||||
SendCount int
|
||||
FailCount int
|
||||
}
|
||||
|
||||
// InitAlertHealth restores persisted alert send health so the dashboard shows real
|
||||
// "last sent" / health state on startup instead of resetting every channel to "never".
|
||||
func (e *Engine) InitAlertHealth() {
|
||||
records, err := e.db.LoadAlertHealth(context.Background())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
e.alertHealthMu.Lock()
|
||||
defer e.alertHealthMu.Unlock()
|
||||
for id, r := range records {
|
||||
e.alertHealth[id] = AlertHealth{
|
||||
LastSendAt: r.LastSendAt,
|
||||
LastSendOK: r.LastSendOK,
|
||||
LastError: r.LastError,
|
||||
SendCount: r.SendCount,
|
||||
FailCount: r.FailCount,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleStatusChange folds a check result into the live state. snap is the
|
||||
// stale snapshot the check ran against; the actual mutation is applied onto the
|
||||
// CURRENT live entry via applyState, so a concurrent pause / config edit /
|
||||
// heartbeat is never reverted by this write. Logs and alerts are emitted after
|
||||
// the lock is released, off the critical section.
|
||||
func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
|
||||
if !e.IsActive() {
|
||||
return
|
||||
}
|
||||
|
||||
inMaint := e.isInMaintenance(snap.ID)
|
||||
status := models.Status(rawStatus)
|
||||
|
||||
var (
|
||||
prev, next models.Status
|
||||
name, typ string
|
||||
alertID int
|
||||
failCount, maxRetries int
|
||||
confirmedDown bool
|
||||
failedCheck bool
|
||||
downSince time.Time
|
||||
sslWarnFire bool
|
||||
sslDays int
|
||||
skipped bool
|
||||
changed bool
|
||||
)
|
||||
|
||||
_, exists := e.applyState(snap.ID, func(s *models.Site) {
|
||||
// A non-UP result computed from a stale snapshot must not override a
|
||||
// heartbeat (or newer check) that landed while we were evaluating.
|
||||
if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
|
||||
skipped = true
|
||||
return
|
||||
}
|
||||
|
||||
prev = s.Status
|
||||
name = s.Name
|
||||
typ = s.Type
|
||||
alertID = s.AlertID
|
||||
maxRetries = s.MaxRetries
|
||||
downSince = s.StatusChangedAt
|
||||
|
||||
// Fresh check results (measured by the run against snap).
|
||||
s.StatusCode = code
|
||||
s.Latency = snap.Latency
|
||||
s.LastCheck = snap.LastCheck
|
||||
s.HasSSL = snap.HasSSL
|
||||
s.CertExpiry = snap.CertExpiry
|
||||
s.LastError = errorReason
|
||||
if status == models.StatusUp {
|
||||
s.LastSuccessAt = time.Now()
|
||||
s.LastError = ""
|
||||
}
|
||||
|
||||
// Status + failure-count transition, based on the CURRENT live status.
|
||||
if status == models.StatusUp {
|
||||
s.FailureCount = 0
|
||||
s.Status = models.StatusUp
|
||||
} else {
|
||||
if s.FailureCount <= s.MaxRetries {
|
||||
s.FailureCount++
|
||||
}
|
||||
if s.FailureCount > s.MaxRetries {
|
||||
if s.Status != status {
|
||||
confirmedDown = true
|
||||
}
|
||||
s.Status = status
|
||||
s.FailureCount = s.MaxRetries + 1
|
||||
} else {
|
||||
failedCheck = true
|
||||
}
|
||||
}
|
||||
failCount = s.FailureCount
|
||||
|
||||
if s.Status != prev && prev != models.StatusPending {
|
||||
s.StatusChangedAt = time.Now()
|
||||
} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
|
||||
s.StatusChangedAt = time.Now()
|
||||
}
|
||||
|
||||
// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
|
||||
if typ == "http" && s.CheckSSL && s.HasSSL {
|
||||
days := int(time.Until(s.CertExpiry).Hours() / 24)
|
||||
if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
|
||||
sslWarnFire = true
|
||||
sslDays = days
|
||||
s.SentSSLWarning = true
|
||||
} else if days > s.ExpiryThreshold {
|
||||
s.SentSSLWarning = false
|
||||
}
|
||||
}
|
||||
|
||||
next = s.Status
|
||||
changed = next != prev
|
||||
})
|
||||
|
||||
if !exists || skipped {
|
||||
return
|
||||
}
|
||||
|
||||
e.recordCheck(snap.ID, latency, status == models.StatusUp)
|
||||
|
||||
if confirmedDown {
|
||||
if errorReason != "" {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
|
||||
} else {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
|
||||
}
|
||||
} else if failedCheck {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
|
||||
}
|
||||
|
||||
if changed && prev != models.StatusPending {
|
||||
e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
|
||||
}
|
||||
|
||||
if sslWarnFire {
|
||||
if !inMaint {
|
||||
e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
|
||||
} else {
|
||||
e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
|
||||
}
|
||||
}
|
||||
|
||||
if prev == models.StatusUp && next == models.StatusLate {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
|
||||
}
|
||||
|
||||
if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
|
||||
if inMaint {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
|
||||
} else {
|
||||
msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
|
||||
if errorReason != "" {
|
||||
msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
|
||||
}
|
||||
if typ == "push" {
|
||||
msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
|
||||
}
|
||||
e.triggerAlert(alertID, "🚨 ALERT", msg)
|
||||
}
|
||||
}
|
||||
if prev.IsBroken() && next == models.StatusUp {
|
||||
downDur := ""
|
||||
if !downSince.IsZero() {
|
||||
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
||||
}
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
|
||||
if !inMaint {
|
||||
e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
|
||||
}
|
||||
}
|
||||
if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) triggerAlert(alertID int, title, message string) {
|
||||
if alertID <= 0 {
|
||||
return
|
||||
}
|
||||
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
||||
if err != nil {
|
||||
e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
|
||||
return
|
||||
}
|
||||
provider := alert.GetProvider(cfg)
|
||||
if provider != nil {
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
||||
defer cancel()
|
||||
if err := provider.Send(ctx, title, message); err != nil {
|
||||
e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
|
||||
e.recordAlertResult(alertID, false, err.Error())
|
||||
} else {
|
||||
e.recordAlertResult(alertID, true, "")
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
|
||||
e.alertHealthMu.Lock()
|
||||
defer e.alertHealthMu.Unlock()
|
||||
h := e.alertHealth[alertID]
|
||||
h.LastSendAt = time.Now()
|
||||
h.LastSendOK = ok
|
||||
h.SendCount++
|
||||
if ok {
|
||||
h.LastError = ""
|
||||
} else {
|
||||
h.LastError = errMsg
|
||||
h.FailCount++
|
||||
}
|
||||
e.alertHealth[alertID] = h
|
||||
|
||||
// Persist so health survives restarts; DB IO off the alert path.
|
||||
e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
|
||||
AlertID: alertID,
|
||||
LastSendAt: h.LastSendAt,
|
||||
LastSendOK: h.LastSendOK,
|
||||
LastError: h.LastError,
|
||||
SendCount: h.SendCount,
|
||||
FailCount: h.FailCount,
|
||||
}})
|
||||
}
|
||||
|
||||
func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
|
||||
e.alertHealthMu.RLock()
|
||||
defer e.alertHealthMu.RUnlock()
|
||||
return e.alertHealth[alertID]
|
||||
}
|
||||
|
||||
func (e *Engine) TestAlert(alertID int) error {
|
||||
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load alert: %w", err)
|
||||
}
|
||||
provider := alert.GetProvider(cfg)
|
||||
if provider == nil {
|
||||
return fmt.Errorf("no provider for type %q", cfg.Type)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
||||
defer cancel()
|
||||
err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
|
||||
if err != nil {
|
||||
e.recordAlertResult(alertID, false, err.Error())
|
||||
return err
|
||||
}
|
||||
e.recordAlertResult(alertID, true, "")
|
||||
e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
// --- Group 1: State Machine ---
|
||||
|
||||
func TestHandleStatusChange_PendingToUp(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "PENDING"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "UP", 200, 10*time.Millisecond, "")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP, got %s", s.Status)
|
||||
}
|
||||
if s.FailureCount != 0 {
|
||||
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||
t.Error("expected no alert for PENDING→UP")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_UpIncrementFailure(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3},
|
||||
SiteState: models.SiteState{Status: "UP", FailureCount: 0},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 500, 0, "test error")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP (under retry threshold), got %s", s.Status)
|
||||
}
|
||||
if s.FailureCount != 1 {
|
||||
t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_UpToDown_ExceedsRetries(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "discord", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "UP", FailureCount: 2},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 500, 0, "test error")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected DOWN, got %s", s.Status)
|
||||
}
|
||||
if s.FailureCount != 3 {
|
||||
t.Errorf("expected FailureCount 3, got %d", s.FailureCount)
|
||||
}
|
||||
waitAsync()
|
||||
calls := ms.getAlertCallsSnapshot()
|
||||
if len(calls) == 0 || calls[0] != 1 {
|
||||
t.Errorf("expected alert call for alertID 1, got %v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_UpToDown_ZeroRetries(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "UP", FailureCount: 0},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected DOWN, got %s", s.Status)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||
t.Error("expected alert on immediate DOWN")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_DownToUp_Recovery(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "DOWN", FailureCount: 4},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP, got %s", s.Status)
|
||||
}
|
||||
if s.FailureCount != 0 {
|
||||
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||
t.Error("expected recovery alert")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_DownStaysDown(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2},
|
||||
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected DOWN, got %s", s.Status)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||
t.Error("expected no re-alert for already DOWN")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_SSLExpired(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "SSL EXP", 0, 0, "SSL certificate expired")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "SSL EXP" {
|
||||
t.Errorf("expected SSL EXP, got %s", s.Status)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||
t.Error("expected alert on SSL EXP")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_AlertSuppressedMaintenance(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.maintenance[1] = true
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
e.refreshMaintenanceCache(context.Background())
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected DOWN, got %s", s.Status)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||
t.Error("expected no alert during maintenance")
|
||||
}
|
||||
logs := e.GetLogs()
|
||||
found := false
|
||||
for _, l := range logs {
|
||||
if containsStr(l.Message, "suppressed") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("expected log mentioning suppressed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_RecoverySuppressedMaintenance(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.maintenance[1] = true
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "DOWN"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
e.refreshMaintenanceCache(context.Background())
|
||||
|
||||
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP, got %s", s.Status)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||
t.Error("expected no alert during maintenance recovery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_SSLWarning(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if !s.SentSSLWarning {
|
||||
t.Error("expected SentSSLWarning=true")
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||
t.Error("expected SSL warning alert")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_SSLWarningNotRepeated(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||
t.Error("expected no repeat SSL warning")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_SSLWarningReset(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30},
|
||||
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(60 * 24 * time.Hour)},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.SentSSLWarning {
|
||||
t.Error("expected SentSSLWarning reset to false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_SSLWarningSuppressedMaint(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.maintenance[1] = true
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
|
||||
}
|
||||
injectSite(e, site)
|
||||
e.refreshMaintenanceCache(context.Background())
|
||||
|
||||
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if !s.SentSSLWarning {
|
||||
t.Error("expected SentSSLWarning=true even in maintenance")
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||
t.Error("expected no alert during maintenance")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStatusChange_InactiveEngine(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
e.SetActive(false)
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Error("expected no state change when inactive")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Group 10: liveState merge (lost-update race) ---
|
||||
|
||||
// A pause that lands while a check is in flight must survive the check's
|
||||
// write-back. The old code snapshotted the site, ran the check, then wrote the
|
||||
// whole stale struct back — reverting the pause.
|
||||
func TestHandleStatusChange_PauseDuringCheckSurvives(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
// `site` is the stale snapshot the check ran against (Paused=false).
|
||||
// Meanwhile the user pauses the monitor.
|
||||
e.ToggleSitePause(1)
|
||||
|
||||
// Check completes and folds its result in using the stale snapshot.
|
||||
e.handleStatusChange(site, "DOWN", 500, 0, "boom")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if !s.Paused {
|
||||
t.Error("pause was reverted by a stale check write-back")
|
||||
}
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected check result still applied (DOWN), got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// A config edit that lands while a check is in flight must survive; the check
|
||||
// must not resurrect the old config from its snapshot.
|
||||
func TestHandleStatusChange_ConfigEditDuringCheckSurvives(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com", Type: "http", MaxRetries: 0, Interval: 30},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
// Config changes mid-check.
|
||||
e.UpdateSiteConfig(models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Type: "http", Interval: 60})
|
||||
|
||||
// Stale check (ran against http://old.com) folds its result in.
|
||||
e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.URL != "http://new.com" {
|
||||
t.Errorf("config edit reverted: URL=%s", s.URL)
|
||||
}
|
||||
if s.Interval != 60 {
|
||||
t.Errorf("config edit reverted: Interval=%d", s.Interval)
|
||||
}
|
||||
}
|
||||
|
||||
// The classic push false-DOWN: a heartbeat marks the monitor UP while a
|
||||
// staleness evaluation (computed from the older LastCheck) is mid-flight.
|
||||
// The stale DOWN must not overwrite the fresh heartbeat.
|
||||
func TestHandleStatusChange_HeartbeatNotOverwrittenByStaleDown(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
// Snapshot the engine would have taken before evaluating staleness:
|
||||
// LastCheck is old, so checkPush decided "DOWN".
|
||||
snap := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Token: "tok", Interval: 10},
|
||||
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
|
||||
}
|
||||
injectSite(e, snap)
|
||||
|
||||
// A heartbeat lands first, advancing LastCheck and confirming UP.
|
||||
if !e.RecordHeartbeat("tok") {
|
||||
t.Fatal("heartbeat rejected")
|
||||
}
|
||||
|
||||
// Now the in-flight stale evaluation tries to write DOWN.
|
||||
e.handleStatusChange(snap, "DOWN", 0, 0, "heartbeat missed")
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("stale DOWN overwrote a fresh heartbeat: status=%s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// A check result for a site removed mid-check must be dropped, not recreate it.
|
||||
func TestHandleStatusChange_RemovedSiteDropped(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.RemoveSite(1)
|
||||
e.handleStatusChange(site, "DOWN", 500, 0, "boom")
|
||||
|
||||
if _, ok := getSite(e, 1); ok {
|
||||
t.Error("removed site was recreated by a late check write-back")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Group 12: Phase 3 engine correctness ---
|
||||
|
||||
// PENDING→DOWN must honor MaxRetries instead of alerting on first failure.
|
||||
func TestHandleStatusChange_PendingRetriesBeforeDown(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "new-monitor", MaxRetries: 2},
|
||||
SiteState: models.SiteState{Status: "PENDING"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 0, 0, "timeout")
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "PENDING" {
|
||||
t.Errorf("expected PENDING during retry, got %s", s.Status)
|
||||
}
|
||||
if s.FailureCount != 1 {
|
||||
t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
|
||||
}
|
||||
|
||||
e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
|
||||
s, _ = getSite(e, 1)
|
||||
if s.Status != "PENDING" {
|
||||
t.Errorf("expected PENDING during retry 2, got %s", s.Status)
|
||||
}
|
||||
|
||||
e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
|
||||
s, _ = getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// LATE→DOWN must also honor MaxRetries.
|
||||
func TestHandleStatusChange_LateRetriesBeforeDown(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push-mon", MaxRetries: 1},
|
||||
SiteState: models.SiteState{Status: "LATE"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.handleStatusChange(site, "DOWN", 0, 0, "missed heartbeat")
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "LATE" {
|
||||
t.Errorf("expected LATE during retry, got %s", s.Status)
|
||||
}
|
||||
|
||||
e.handleStatusChange(s, "DOWN", 0, 0, "missed heartbeat")
|
||||
s, _ = getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func (e *Engine) RecordHeartbeat(token string) bool {
|
||||
if !e.IsActive() {
|
||||
return false
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
targetID, ok := e.tokenIndex[token]
|
||||
e.mu.RUnlock()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
var (
|
||||
prevStatus models.Status
|
||||
name string
|
||||
alertID int
|
||||
downSince time.Time
|
||||
)
|
||||
_, exists := e.applyState(targetID, func(s *models.Site) {
|
||||
prevStatus = s.Status
|
||||
name = s.Name
|
||||
alertID = s.AlertID
|
||||
downSince = s.StatusChangedAt // captured before mutation = when it went down
|
||||
|
||||
s.LastCheck = time.Now()
|
||||
s.Status = models.StatusUp
|
||||
s.FailureCount = 0
|
||||
s.Latency = 0
|
||||
s.LastError = ""
|
||||
s.LastSuccessAt = time.Now()
|
||||
if prevStatus != models.StatusUp {
|
||||
s.StatusChangedAt = time.Now()
|
||||
}
|
||||
})
|
||||
if !exists {
|
||||
return false
|
||||
}
|
||||
|
||||
switch prevStatus {
|
||||
case models.StatusPending:
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
|
||||
case models.StatusLate:
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
|
||||
case models.StatusStale:
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
|
||||
case models.StatusDown:
|
||||
downDur := ""
|
||||
if !downSince.IsZero() {
|
||||
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
||||
}
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
|
||||
go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
|
||||
}
|
||||
|
||||
e.recordCheck(targetID, 0, true)
|
||||
|
||||
if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
|
||||
e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *Engine) getRecheckChan(id int) chan struct{} {
|
||||
e.recheckMu.Lock()
|
||||
defer e.recheckMu.Unlock()
|
||||
ch, ok := e.recheck[id]
|
||||
if !ok {
|
||||
ch = make(chan struct{}, 1)
|
||||
e.recheck[id] = ch
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
func (e *Engine) signalRecheck(id int) {
|
||||
ch := e.getRecheckChan(id)
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) monitorRoutine(ctx context.Context, id int) {
|
||||
recheckCh := e.getRecheckChan(id)
|
||||
|
||||
// Stagger initial check to avoid thundering herd on startup
|
||||
stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
|
||||
select {
|
||||
case <-time.After(stagger):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
e.checkByID(ctx, id)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if !e.IsActive() {
|
||||
select {
|
||||
case <-time.After(pollInterval):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-recheckCh:
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
site, exists := e.liveState[id]
|
||||
e.mu.RUnlock()
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
if site.Paused {
|
||||
select {
|
||||
case <-time.After(pollInterval):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-recheckCh:
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
interval := site.Interval
|
||||
if interval < minCheckInterval {
|
||||
interval = minCheckInterval
|
||||
}
|
||||
jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
|
||||
select {
|
||||
case <-time.After(time.Duration(interval)*time.Second + jitter):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-recheckCh:
|
||||
}
|
||||
e.checkByID(ctx, id)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) checkByID(ctx context.Context, id int) {
|
||||
if !e.IsActive() {
|
||||
return
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
site, exists := e.liveState[id]
|
||||
e.mu.RUnlock()
|
||||
if !exists || site.Paused {
|
||||
return
|
||||
}
|
||||
|
||||
switch site.Type {
|
||||
case "push":
|
||||
e.checkPush(ctx, site)
|
||||
case "group":
|
||||
e.checkGroup(ctx, site)
|
||||
default:
|
||||
result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
|
||||
updatedSite := site
|
||||
updatedSite.HasSSL = result.HasSSL
|
||||
updatedSite.CertExpiry = result.CertExpiry
|
||||
updatedSite.Latency = time.Duration(result.LatencyNs)
|
||||
updatedSite.LastCheck = time.Now()
|
||||
e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) checkPush(_ context.Context, site models.Site) {
|
||||
if site.Status == models.StatusPending {
|
||||
return
|
||||
}
|
||||
|
||||
interval := time.Duration(site.Interval) * time.Second
|
||||
grace := interval / 2
|
||||
if grace < minPushGrace {
|
||||
grace = minPushGrace
|
||||
}
|
||||
|
||||
overdue := site.LastCheck.Add(interval)
|
||||
staleMark := overdue.Add(grace / 2)
|
||||
graceEnd := overdue.Add(grace)
|
||||
now := time.Now()
|
||||
|
||||
if now.After(graceEnd) {
|
||||
if site.Status != models.StatusDown {
|
||||
e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
|
||||
}
|
||||
} else if now.After(staleMark) {
|
||||
if site.Status != models.StatusStale {
|
||||
e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
|
||||
}
|
||||
} else if now.After(overdue) {
|
||||
if site.Status != models.StatusLate {
|
||||
e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) checkGroup(_ context.Context, site models.Site) {
|
||||
e.mu.RLock()
|
||||
status := models.StatusUp
|
||||
hasChildren := false
|
||||
for _, child := range e.liveState {
|
||||
if child.ParentID != site.ID || child.Type == "group" {
|
||||
continue
|
||||
}
|
||||
hasChildren = true
|
||||
if child.Paused || e.isInMaintenance(child.ID) {
|
||||
continue
|
||||
}
|
||||
if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
|
||||
status = models.StatusDown
|
||||
} else if child.Status == models.StatusStale && status != models.StatusDown {
|
||||
status = models.StatusStale
|
||||
} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
|
||||
status = models.StatusLate
|
||||
} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
|
||||
status = models.StatusPending
|
||||
}
|
||||
}
|
||||
e.mu.RUnlock()
|
||||
|
||||
if !hasChildren {
|
||||
status = models.StatusPending
|
||||
}
|
||||
|
||||
e.applyState(site.ID, func(s *models.Site) {
|
||||
s.Status = status
|
||||
})
|
||||
e.recordCheck(site.ID, 0, !status.IsBroken())
|
||||
}
|
||||
|
||||
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
|
||||
e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
|
||||
}
|
||||
|
||||
// SetAggStrategy must be called before Start: the field is read by the probe
|
||||
// aggregation path without synchronization.
|
||||
func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
|
||||
e.aggStrategy = strategy
|
||||
}
|
||||
|
||||
func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
|
||||
e.mu.RLock()
|
||||
site, exists := e.liveState[siteID]
|
||||
e.mu.RUnlock()
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
staleAfter := time.Duration(site.Interval) * time.Second * 3
|
||||
if staleAfter < time.Minute {
|
||||
staleAfter = time.Minute
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
e.probeResultsMu.Lock()
|
||||
if e.probeResults[siteID] == nil {
|
||||
e.probeResults[siteID] = make(map[string]NodeResult)
|
||||
}
|
||||
e.probeResults[siteID][nodeID] = NodeResult{
|
||||
NodeID: nodeID,
|
||||
IsUp: isUp,
|
||||
LatencyNs: latencyNs,
|
||||
CheckedAt: now,
|
||||
ErrorReason: errorReason,
|
||||
}
|
||||
results := make([]NodeResult, 0, len(e.probeResults[siteID]))
|
||||
for id, r := range e.probeResults[siteID] {
|
||||
if now.Sub(r.CheckedAt) > staleAfter {
|
||||
delete(e.probeResults[siteID], id)
|
||||
continue
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
e.probeResultsMu.Unlock()
|
||||
|
||||
aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
|
||||
|
||||
probeStatus := models.StatusUp
|
||||
if !aggUp {
|
||||
probeStatus = models.StatusDown
|
||||
}
|
||||
|
||||
updatedSite := site
|
||||
updatedSite.Latency = time.Duration(avgLatency)
|
||||
updatedSite.LastCheck = time.Now()
|
||||
e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
|
||||
}
|
||||
|
||||
func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
|
||||
e.probeResultsMu.RLock()
|
||||
defer e.probeResultsMu.RUnlock()
|
||||
src := e.probeResults[siteID]
|
||||
cp := make(map[string]NodeResult, len(src))
|
||||
for k, v := range src {
|
||||
cp[k] = v
|
||||
}
|
||||
return cp
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
// --- Group 2: Heartbeat ---
|
||||
|
||||
func TestRecordHeartbeat_ValidToken(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
if !e.RecordHeartbeat("abc123") {
|
||||
t.Error("expected true for valid token")
|
||||
}
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP, got %s", s.Status)
|
||||
}
|
||||
if time.Since(s.LastCheck) > time.Second {
|
||||
t.Error("expected LastCheck to be recent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordHeartbeat_RecoveryFromDown(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123", AlertID: 1},
|
||||
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
if !e.RecordHeartbeat("abc123") {
|
||||
t.Error("expected true")
|
||||
}
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP, got %s", s.Status)
|
||||
}
|
||||
if s.FailureCount != 0 {
|
||||
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
|
||||
}
|
||||
waitAsync()
|
||||
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||
t.Error("expected recovery alert")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordHeartbeat_UnknownToken(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
|
||||
if e.RecordHeartbeat("unknown") {
|
||||
t.Error("expected false for unknown token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordHeartbeat_InactiveEngine(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Type: "push", Token: "abc123"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
e.SetActive(false)
|
||||
|
||||
if e.RecordHeartbeat("abc123") {
|
||||
t.Error("expected false when inactive")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Group 3: Push Deadline ---
|
||||
|
||||
func TestCheckPush_DeadlineMissed(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 10, MaxRetries: 0},
|
||||
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.checkPush(context.Background(), site)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected DOWN after missed deadline, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPush_OverdueBecomesLate(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
|
||||
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-310 * time.Second)},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.checkPush(context.Background(), site)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "LATE" {
|
||||
t.Errorf("expected LATE when overdue but within grace, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPush_OverdueBecomesStale(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
// interval=300, grace=150 (300/2), staleMark=overdue+75
|
||||
// at 380s: past staleMark(375) but before graceEnd(450)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
|
||||
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-380 * time.Second)},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.checkPush(context.Background(), site)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "STALE" {
|
||||
t.Errorf("expected STALE when past midpoint of grace, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPush_WithinDeadline(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
|
||||
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now()},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.checkPush(context.Background(), site)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPush_PendingStaysPending(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
|
||||
SiteState: models.SiteState{Status: "PENDING"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.checkPush(context.Background(), site)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "PENDING" {
|
||||
t.Errorf("expected PENDING to stay until first heartbeat, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Group 4: Group Checks ---
|
||||
|
||||
func TestCheckGroup_AllChildrenUp(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
group := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||
SiteState: models.SiteState{Status: "PENDING"},
|
||||
}
|
||||
child1 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child2 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, group)
|
||||
injectSite(e, child1)
|
||||
injectSite(e, child2)
|
||||
|
||||
e.checkGroup(context.Background(), group)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected group UP, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGroup_OneChildDown(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
group := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child1 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child2 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
|
||||
SiteState: models.SiteState{Status: "DOWN"},
|
||||
}
|
||||
injectSite(e, group)
|
||||
injectSite(e, child1)
|
||||
injectSite(e, child2)
|
||||
|
||||
e.checkGroup(context.Background(), group)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected group DOWN, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGroup_PausedChildIgnored(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
group := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||
}
|
||||
child1 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child2 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
|
||||
SiteState: models.SiteState{Status: "DOWN"},
|
||||
}
|
||||
injectSite(e, group)
|
||||
injectSite(e, child1)
|
||||
injectSite(e, child2)
|
||||
|
||||
e.checkGroup(context.Background(), group)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP (paused child ignored), got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGroup_MaintenanceChildIgnored(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.maintenance[3] = true
|
||||
e := newTestEngine(ms)
|
||||
group := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||
}
|
||||
child1 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child2 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
|
||||
SiteState: models.SiteState{Status: "DOWN"},
|
||||
}
|
||||
injectSite(e, group)
|
||||
injectSite(e, child1)
|
||||
injectSite(e, child2)
|
||||
e.refreshMaintenanceCache(context.Background())
|
||||
|
||||
e.checkGroup(context.Background(), group)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "UP" {
|
||||
t.Errorf("expected UP (maint child ignored), got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGroup_NoChildren(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
group := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, group)
|
||||
|
||||
e.checkGroup(context.Background(), group)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Status != "PENDING" {
|
||||
t.Errorf("expected PENDING for no children, got %s", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// Groups must not auto-pause when all children are paused — that creates a
|
||||
// one-way trap because monitorRoutine skips paused sites.
|
||||
func TestCheckGroup_AllPausedNoAutoFreeze(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
group := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child1 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1, Paused: true},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child2 := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, group)
|
||||
injectSite(e, child1)
|
||||
injectSite(e, child2)
|
||||
|
||||
e.checkGroup(context.Background(), group)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.Paused {
|
||||
t.Error("group must not auto-pause when all children are paused")
|
||||
}
|
||||
}
|
||||
|
||||
// Dead probe results must be expired so they don't poison aggregation.
|
||||
func TestIngestProbeResult_ExpiresStaleProbes(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", Interval: 30},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.probeResultsMu.Lock()
|
||||
e.probeResults[1] = map[string]NodeResult{
|
||||
"dead-probe": {
|
||||
NodeID: "dead-probe",
|
||||
IsUp: false,
|
||||
CheckedAt: time.Now().Add(-10 * time.Minute),
|
||||
},
|
||||
}
|
||||
e.probeResultsMu.Unlock()
|
||||
|
||||
e.IngestProbeResult("live-probe", 1, 5000, true, "")
|
||||
|
||||
e.probeResultsMu.RLock()
|
||||
_, deadExists := e.probeResults[1]["dead-probe"]
|
||||
_, liveExists := e.probeResults[1]["live-probe"]
|
||||
e.probeResultsMu.RUnlock()
|
||||
|
||||
if deadExists {
|
||||
t.Error("stale probe result should have been expired")
|
||||
}
|
||||
if !liveExists {
|
||||
t.Error("live probe result should still exist")
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveSite must clean up probeResults.
|
||||
func TestRemoveSite_CleansProbeResults(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
e.probeResultsMu.Lock()
|
||||
e.probeResults[1] = map[string]NodeResult{
|
||||
"node-a": {NodeID: "node-a", IsUp: true, CheckedAt: time.Now()},
|
||||
}
|
||||
e.probeResultsMu.Unlock()
|
||||
|
||||
e.RemoveSite(1)
|
||||
|
||||
e.probeResultsMu.RLock()
|
||||
defer e.probeResultsMu.RUnlock()
|
||||
if _, exists := e.probeResults[1]; exists {
|
||||
t.Error("probe results should be cleaned up after RemoveSite")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (e *Engine) maintenancePruner(ctx context.Context) {
|
||||
ticker := time.NewTicker(maintPruneInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
e.pruneMaintenanceWindows(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
e.pruneMaintenanceWindows(ctx)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
|
||||
pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
|
||||
if err != nil {
|
||||
e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
|
||||
return
|
||||
}
|
||||
if pruned > 0 {
|
||||
e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) isInMaintenance(monitorID int) bool {
|
||||
e.maintCacheMu.RLock()
|
||||
defer e.maintCacheMu.RUnlock()
|
||||
return e.maintCache[monitorID]
|
||||
}
|
||||
|
||||
func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
|
||||
windows, err := e.db.GetActiveMaintenanceWindows(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
directMaint := make(map[int]bool)
|
||||
var globalMaint bool
|
||||
for _, w := range windows {
|
||||
if w.MonitorID == 0 {
|
||||
globalMaint = true
|
||||
} else {
|
||||
directMaint[w.MonitorID] = true
|
||||
}
|
||||
}
|
||||
|
||||
resolved := make(map[int]bool)
|
||||
e.mu.RLock()
|
||||
for id, site := range e.liveState {
|
||||
if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
|
||||
resolved[id] = true
|
||||
}
|
||||
}
|
||||
e.mu.RUnlock()
|
||||
|
||||
e.maintCacheMu.Lock()
|
||||
e.maintCache = resolved
|
||||
e.maintCacheMu.Unlock()
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
// Maintenance cache resolves parent relationships correctly.
|
||||
func TestIsInMaintenance_UsesCache(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.maintenance[10] = true // direct maintenance on group
|
||||
e := newTestEngine(ms)
|
||||
group := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 10, Name: "group", Type: "group"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
child := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 20, Name: "child", Type: "http", ParentID: 10},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, group)
|
||||
injectSite(e, child)
|
||||
e.refreshMaintenanceCache(context.Background())
|
||||
|
||||
if !e.isInMaintenance(10) {
|
||||
t.Error("group should be in maintenance (direct)")
|
||||
}
|
||||
if !e.isInMaintenance(20) {
|
||||
t.Error("child should be in maintenance (parent)")
|
||||
}
|
||||
if e.isInMaintenance(99) {
|
||||
t.Error("unknown monitor should not be in maintenance")
|
||||
}
|
||||
}
|
||||
|
||||
// Global maintenance (monitor_id=0) applies to all monitors.
|
||||
func TestIsInMaintenance_GlobalMaintenance(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
ms.maintenance[0] = true
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
e.refreshMaintenanceCache(context.Background())
|
||||
|
||||
if !e.isInMaintenance(1) {
|
||||
t.Error("all monitors should be in maintenance during global window")
|
||||
}
|
||||
}
|
||||
@@ -4,14 +4,12 @@ import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/store"
|
||||
)
|
||||
@@ -28,14 +26,6 @@ const (
|
||||
dbPruneInterval = 10 * time.Minute
|
||||
)
|
||||
|
||||
type AlertHealth struct {
|
||||
LastSendAt time.Time
|
||||
LastSendOK bool
|
||||
LastError string
|
||||
SendCount int
|
||||
FailCount int
|
||||
}
|
||||
|
||||
type Engine struct {
|
||||
mu sync.RWMutex
|
||||
liveState map[int]models.Site
|
||||
@@ -261,26 +251,6 @@ func (e *Engine) InitLogs() {
|
||||
e.logStore = entries
|
||||
}
|
||||
|
||||
// InitAlertHealth restores persisted alert send health so the dashboard shows real
|
||||
// "last sent" / health state on startup instead of resetting every channel to "never".
|
||||
func (e *Engine) InitAlertHealth() {
|
||||
records, err := e.db.LoadAlertHealth(context.Background())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
e.alertHealthMu.Lock()
|
||||
defer e.alertHealthMu.Unlock()
|
||||
for id, r := range records {
|
||||
e.alertHealth[id] = AlertHealth{
|
||||
LastSendAt: r.LastSendAt,
|
||||
LastSendOK: r.LastSendOK,
|
||||
LastError: r.LastError,
|
||||
SendCount: r.SendCount,
|
||||
FailCount: r.FailCount,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) GetLogs() []models.LogEntry {
|
||||
e.logMu.RLock()
|
||||
defer e.logMu.RUnlock()
|
||||
@@ -289,123 +259,6 @@ func (e *Engine) GetLogs() []models.LogEntry {
|
||||
return logs
|
||||
}
|
||||
|
||||
func (e *Engine) SetActive(active bool) {
|
||||
e.activeMu.Lock()
|
||||
defer e.activeMu.Unlock()
|
||||
if e.isActive != active {
|
||||
e.isActive = active
|
||||
status := "RESUMED (Active)"
|
||||
if !active {
|
||||
status = "PAUSED (Passive)"
|
||||
}
|
||||
e.AddLog(fmt.Sprintf("Engine %s", status))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) IsActive() bool {
|
||||
e.activeMu.RLock()
|
||||
defer e.activeMu.RUnlock()
|
||||
return e.isActive
|
||||
}
|
||||
|
||||
func (e *Engine) GetAllSites() []models.Site {
|
||||
e.mu.RLock()
|
||||
defer e.mu.RUnlock()
|
||||
sites := make([]models.Site, 0, len(e.liveState))
|
||||
for _, s := range e.liveState {
|
||||
sites = append(sites, s)
|
||||
}
|
||||
return sites
|
||||
}
|
||||
|
||||
func (e *Engine) GetLiveState() map[int]models.Site {
|
||||
e.mu.RLock()
|
||||
defer e.mu.RUnlock()
|
||||
cp := make(map[int]models.Site, len(e.liveState))
|
||||
for k, v := range e.liveState {
|
||||
cp[k] = v
|
||||
}
|
||||
return cp
|
||||
}
|
||||
|
||||
func (e *Engine) RecordHeartbeat(token string) bool {
|
||||
if !e.IsActive() {
|
||||
return false
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
targetID, ok := e.tokenIndex[token]
|
||||
e.mu.RUnlock()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
var (
|
||||
prevStatus models.Status
|
||||
name string
|
||||
alertID int
|
||||
downSince time.Time
|
||||
)
|
||||
_, exists := e.applyState(targetID, func(s *models.Site) {
|
||||
prevStatus = s.Status
|
||||
name = s.Name
|
||||
alertID = s.AlertID
|
||||
downSince = s.StatusChangedAt // captured before mutation = when it went down
|
||||
|
||||
s.LastCheck = time.Now()
|
||||
s.Status = models.StatusUp
|
||||
s.FailureCount = 0
|
||||
s.Latency = 0
|
||||
s.LastError = ""
|
||||
s.LastSuccessAt = time.Now()
|
||||
if prevStatus != models.StatusUp {
|
||||
s.StatusChangedAt = time.Now()
|
||||
}
|
||||
})
|
||||
if !exists {
|
||||
return false
|
||||
}
|
||||
|
||||
switch prevStatus {
|
||||
case models.StatusPending:
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
|
||||
case models.StatusLate:
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
|
||||
case models.StatusStale:
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
|
||||
case models.StatusDown:
|
||||
downDur := ""
|
||||
if !downSince.IsZero() {
|
||||
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
||||
}
|
||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
|
||||
go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
|
||||
}
|
||||
|
||||
e.recordCheck(targetID, 0, true)
|
||||
|
||||
if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
|
||||
e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *Engine) addToTokenIndex(site models.Site) {
|
||||
if site.Type == "push" && site.Token != "" {
|
||||
e.tokenIndex[site.Token] = site.ID
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) removeFromTokenIndex(id int) {
|
||||
for token, sid := range e.tokenIndex {
|
||||
if sid == id {
|
||||
delete(e.tokenIndex, token)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) Start(ctx context.Context) {
|
||||
// e.cancel is invoked by Stop() to drain and halt the writer; gosec can't
|
||||
// trace the cross-method call, and cancelling the parent reaps this child
|
||||
@@ -496,637 +349,3 @@ func (e *Engine) Start(ctx context.Context) {
|
||||
e.maintenancePruner(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
func (e *Engine) maintenancePruner(ctx context.Context) {
|
||||
ticker := time.NewTicker(maintPruneInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
e.pruneMaintenanceWindows(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
e.pruneMaintenanceWindows(ctx)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
|
||||
pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
|
||||
if err != nil {
|
||||
e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
|
||||
return
|
||||
}
|
||||
if pruned > 0 {
|
||||
e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
|
||||
e.mu.Lock()
|
||||
if existing, ok := e.liveState[cfg.ID]; ok {
|
||||
e.removeFromTokenIndex(cfg.ID)
|
||||
existing.SiteConfig = cfg
|
||||
e.liveState[cfg.ID] = existing
|
||||
e.addToTokenIndex(existing)
|
||||
}
|
||||
e.mu.Unlock()
|
||||
|
||||
e.signalRecheck(cfg.ID)
|
||||
}
|
||||
|
||||
func (e *Engine) getRecheckChan(id int) chan struct{} {
|
||||
e.recheckMu.Lock()
|
||||
defer e.recheckMu.Unlock()
|
||||
ch, ok := e.recheck[id]
|
||||
if !ok {
|
||||
ch = make(chan struct{}, 1)
|
||||
e.recheck[id] = ch
|
||||
}
|
||||
return ch
|
||||
}
|
||||
|
||||
func (e *Engine) signalRecheck(id int) {
|
||||
ch := e.getRecheckChan(id)
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) RemoveSite(id int) {
|
||||
e.mu.Lock()
|
||||
e.removeFromTokenIndex(id)
|
||||
delete(e.liveState, id)
|
||||
e.mu.Unlock()
|
||||
e.removeHistory(id)
|
||||
|
||||
e.probeResultsMu.Lock()
|
||||
delete(e.probeResults, id)
|
||||
e.probeResultsMu.Unlock()
|
||||
|
||||
e.recheckMu.Lock()
|
||||
delete(e.recheck, id)
|
||||
e.recheckMu.Unlock()
|
||||
}
|
||||
|
||||
func (e *Engine) ToggleSitePause(id int) bool {
|
||||
var (
|
||||
paused bool
|
||||
name string
|
||||
)
|
||||
_, ok := e.applyState(id, func(s *models.Site) {
|
||||
s.Paused = !s.Paused
|
||||
paused = s.Paused
|
||||
name = s.Name
|
||||
})
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if paused {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
|
||||
} else {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
|
||||
}
|
||||
return paused
|
||||
}
|
||||
|
||||
func (e *Engine) monitorRoutine(ctx context.Context, id int) {
|
||||
recheckCh := e.getRecheckChan(id)
|
||||
|
||||
// Stagger initial check to avoid thundering herd on startup
|
||||
stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
|
||||
select {
|
||||
case <-time.After(stagger):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
e.checkByID(ctx, id)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if !e.IsActive() {
|
||||
select {
|
||||
case <-time.After(pollInterval):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-recheckCh:
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
site, exists := e.liveState[id]
|
||||
e.mu.RUnlock()
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
if site.Paused {
|
||||
select {
|
||||
case <-time.After(pollInterval):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-recheckCh:
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
interval := site.Interval
|
||||
if interval < minCheckInterval {
|
||||
interval = minCheckInterval
|
||||
}
|
||||
jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
|
||||
select {
|
||||
case <-time.After(time.Duration(interval)*time.Second + jitter):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-recheckCh:
|
||||
}
|
||||
e.checkByID(ctx, id)
|
||||
}
|
||||
}
|
||||
|
||||
// applyState atomically reads, mutates, and writes back the live entry for id.
|
||||
// The mutator runs under the engine write lock and receives a pointer to the
|
||||
// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
|
||||
// never clobbered by a stale snapshot. The mutator must only touch runtime /
|
||||
// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
|
||||
// are owned by UpdateSiteConfig and must not be written here. Returns the
|
||||
// post-mutation copy and whether the site still exists.
|
||||
func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
cur, ok := e.liveState[id]
|
||||
if !ok {
|
||||
return models.Site{}, false
|
||||
}
|
||||
mutate(&cur)
|
||||
e.liveState[id] = cur
|
||||
return cur, true
|
||||
}
|
||||
|
||||
func (e *Engine) checkByID(ctx context.Context, id int) {
|
||||
if !e.IsActive() {
|
||||
return
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
site, exists := e.liveState[id]
|
||||
e.mu.RUnlock()
|
||||
if !exists || site.Paused {
|
||||
return
|
||||
}
|
||||
|
||||
switch site.Type {
|
||||
case "push":
|
||||
e.checkPush(ctx, site)
|
||||
case "group":
|
||||
e.checkGroup(ctx, site)
|
||||
default:
|
||||
result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
|
||||
updatedSite := site
|
||||
updatedSite.HasSSL = result.HasSSL
|
||||
updatedSite.CertExpiry = result.CertExpiry
|
||||
updatedSite.Latency = time.Duration(result.LatencyNs)
|
||||
updatedSite.LastCheck = time.Now()
|
||||
e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) checkPush(_ context.Context, site models.Site) {
|
||||
if site.Status == models.StatusPending {
|
||||
return
|
||||
}
|
||||
|
||||
interval := time.Duration(site.Interval) * time.Second
|
||||
grace := interval / 2
|
||||
if grace < minPushGrace {
|
||||
grace = minPushGrace
|
||||
}
|
||||
|
||||
overdue := site.LastCheck.Add(interval)
|
||||
staleMark := overdue.Add(grace / 2)
|
||||
graceEnd := overdue.Add(grace)
|
||||
now := time.Now()
|
||||
|
||||
if now.After(graceEnd) {
|
||||
if site.Status != models.StatusDown {
|
||||
e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
|
||||
}
|
||||
} else if now.After(staleMark) {
|
||||
if site.Status != models.StatusStale {
|
||||
e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
|
||||
}
|
||||
} else if now.After(overdue) {
|
||||
if site.Status != models.StatusLate {
|
||||
e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleStatusChange folds a check result into the live state. snap is the
|
||||
// stale snapshot the check ran against; the actual mutation is applied onto the
|
||||
// CURRENT live entry via applyState, so a concurrent pause / config edit /
|
||||
// heartbeat is never reverted by this write. Logs and alerts are emitted after
|
||||
// the lock is released, off the critical section.
|
||||
func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
|
||||
if !e.IsActive() {
|
||||
return
|
||||
}
|
||||
|
||||
inMaint := e.isInMaintenance(snap.ID)
|
||||
status := models.Status(rawStatus)
|
||||
|
||||
var (
|
||||
prev, next models.Status
|
||||
name, typ string
|
||||
alertID int
|
||||
failCount, maxRetries int
|
||||
confirmedDown bool
|
||||
failedCheck bool
|
||||
downSince time.Time
|
||||
sslWarnFire bool
|
||||
sslDays int
|
||||
skipped bool
|
||||
changed bool
|
||||
)
|
||||
|
||||
_, exists := e.applyState(snap.ID, func(s *models.Site) {
|
||||
// A non-UP result computed from a stale snapshot must not override a
|
||||
// heartbeat (or newer check) that landed while we were evaluating.
|
||||
if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
|
||||
skipped = true
|
||||
return
|
||||
}
|
||||
|
||||
prev = s.Status
|
||||
name = s.Name
|
||||
typ = s.Type
|
||||
alertID = s.AlertID
|
||||
maxRetries = s.MaxRetries
|
||||
downSince = s.StatusChangedAt
|
||||
|
||||
// Fresh check results (measured by the run against snap).
|
||||
s.StatusCode = code
|
||||
s.Latency = snap.Latency
|
||||
s.LastCheck = snap.LastCheck
|
||||
s.HasSSL = snap.HasSSL
|
||||
s.CertExpiry = snap.CertExpiry
|
||||
s.LastError = errorReason
|
||||
if status == models.StatusUp {
|
||||
s.LastSuccessAt = time.Now()
|
||||
s.LastError = ""
|
||||
}
|
||||
|
||||
// Status + failure-count transition, based on the CURRENT live status.
|
||||
if status == models.StatusUp {
|
||||
s.FailureCount = 0
|
||||
s.Status = models.StatusUp
|
||||
} else {
|
||||
if s.FailureCount <= s.MaxRetries {
|
||||
s.FailureCount++
|
||||
}
|
||||
if s.FailureCount > s.MaxRetries {
|
||||
if s.Status != status {
|
||||
confirmedDown = true
|
||||
}
|
||||
s.Status = status
|
||||
s.FailureCount = s.MaxRetries + 1
|
||||
} else {
|
||||
failedCheck = true
|
||||
}
|
||||
}
|
||||
failCount = s.FailureCount
|
||||
|
||||
if s.Status != prev && prev != models.StatusPending {
|
||||
s.StatusChangedAt = time.Now()
|
||||
} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
|
||||
s.StatusChangedAt = time.Now()
|
||||
}
|
||||
|
||||
// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
|
||||
if typ == "http" && s.CheckSSL && s.HasSSL {
|
||||
days := int(time.Until(s.CertExpiry).Hours() / 24)
|
||||
if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
|
||||
sslWarnFire = true
|
||||
sslDays = days
|
||||
s.SentSSLWarning = true
|
||||
} else if days > s.ExpiryThreshold {
|
||||
s.SentSSLWarning = false
|
||||
}
|
||||
}
|
||||
|
||||
next = s.Status
|
||||
changed = next != prev
|
||||
})
|
||||
|
||||
if !exists || skipped {
|
||||
return
|
||||
}
|
||||
|
||||
e.recordCheck(snap.ID, latency, status == models.StatusUp)
|
||||
|
||||
if confirmedDown {
|
||||
if errorReason != "" {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
|
||||
} else {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
|
||||
}
|
||||
} else if failedCheck {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
|
||||
}
|
||||
|
||||
if changed && prev != models.StatusPending {
|
||||
e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
|
||||
}
|
||||
|
||||
if sslWarnFire {
|
||||
if !inMaint {
|
||||
e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
|
||||
} else {
|
||||
e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
|
||||
}
|
||||
}
|
||||
|
||||
if prev == models.StatusUp && next == models.StatusLate {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
|
||||
}
|
||||
|
||||
if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
|
||||
if inMaint {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
|
||||
} else {
|
||||
msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
|
||||
if errorReason != "" {
|
||||
msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
|
||||
}
|
||||
if typ == "push" {
|
||||
msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
|
||||
}
|
||||
e.triggerAlert(alertID, "🚨 ALERT", msg)
|
||||
}
|
||||
}
|
||||
if prev.IsBroken() && next == models.StatusUp {
|
||||
downDur := ""
|
||||
if !downSince.IsZero() {
|
||||
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
||||
}
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
|
||||
if !inMaint {
|
||||
e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
|
||||
}
|
||||
}
|
||||
if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) triggerAlert(alertID int, title, message string) {
|
||||
if alertID <= 0 {
|
||||
return
|
||||
}
|
||||
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
||||
if err != nil {
|
||||
e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
|
||||
return
|
||||
}
|
||||
provider := alert.GetProvider(cfg)
|
||||
if provider != nil {
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
||||
defer cancel()
|
||||
if err := provider.Send(ctx, title, message); err != nil {
|
||||
e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
|
||||
e.recordAlertResult(alertID, false, err.Error())
|
||||
} else {
|
||||
e.recordAlertResult(alertID, true, "")
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
|
||||
e.alertHealthMu.Lock()
|
||||
defer e.alertHealthMu.Unlock()
|
||||
h := e.alertHealth[alertID]
|
||||
h.LastSendAt = time.Now()
|
||||
h.LastSendOK = ok
|
||||
h.SendCount++
|
||||
if ok {
|
||||
h.LastError = ""
|
||||
} else {
|
||||
h.LastError = errMsg
|
||||
h.FailCount++
|
||||
}
|
||||
e.alertHealth[alertID] = h
|
||||
|
||||
// Persist so health survives restarts; DB IO off the alert path.
|
||||
e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
|
||||
AlertID: alertID,
|
||||
LastSendAt: h.LastSendAt,
|
||||
LastSendOK: h.LastSendOK,
|
||||
LastError: h.LastError,
|
||||
SendCount: h.SendCount,
|
||||
FailCount: h.FailCount,
|
||||
}})
|
||||
}
|
||||
|
||||
func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
|
||||
e.alertHealthMu.RLock()
|
||||
defer e.alertHealthMu.RUnlock()
|
||||
return e.alertHealth[alertID]
|
||||
}
|
||||
|
||||
func (e *Engine) TestAlert(alertID int) error {
|
||||
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load alert: %w", err)
|
||||
}
|
||||
provider := alert.GetProvider(cfg)
|
||||
if provider == nil {
|
||||
return fmt.Errorf("no provider for type %q", cfg.Type)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
||||
defer cancel()
|
||||
err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
|
||||
if err != nil {
|
||||
e.recordAlertResult(alertID, false, err.Error())
|
||||
return err
|
||||
}
|
||||
e.recordAlertResult(alertID, true, "")
|
||||
e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) isInMaintenance(monitorID int) bool {
|
||||
e.maintCacheMu.RLock()
|
||||
defer e.maintCacheMu.RUnlock()
|
||||
return e.maintCache[monitorID]
|
||||
}
|
||||
|
||||
func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
|
||||
windows, err := e.db.GetActiveMaintenanceWindows(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
directMaint := make(map[int]bool)
|
||||
var globalMaint bool
|
||||
for _, w := range windows {
|
||||
if w.MonitorID == 0 {
|
||||
globalMaint = true
|
||||
} else {
|
||||
directMaint[w.MonitorID] = true
|
||||
}
|
||||
}
|
||||
|
||||
resolved := make(map[int]bool)
|
||||
e.mu.RLock()
|
||||
for id, site := range e.liveState {
|
||||
if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
|
||||
resolved[id] = true
|
||||
}
|
||||
}
|
||||
e.mu.RUnlock()
|
||||
|
||||
e.maintCacheMu.Lock()
|
||||
e.maintCache = resolved
|
||||
e.maintCacheMu.Unlock()
|
||||
}
|
||||
|
||||
func (e *Engine) GetDisplayStatus(site models.Site) string {
|
||||
if site.Paused {
|
||||
return "PAUSED"
|
||||
}
|
||||
if e.isInMaintenance(site.ID) {
|
||||
return "MAINT"
|
||||
}
|
||||
return string(site.Status)
|
||||
}
|
||||
|
||||
func (e *Engine) checkGroup(_ context.Context, site models.Site) {
|
||||
e.mu.RLock()
|
||||
status := models.StatusUp
|
||||
hasChildren := false
|
||||
for _, child := range e.liveState {
|
||||
if child.ParentID != site.ID || child.Type == "group" {
|
||||
continue
|
||||
}
|
||||
hasChildren = true
|
||||
if child.Paused || e.isInMaintenance(child.ID) {
|
||||
continue
|
||||
}
|
||||
if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
|
||||
status = models.StatusDown
|
||||
} else if child.Status == models.StatusStale && status != models.StatusDown {
|
||||
status = models.StatusStale
|
||||
} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
|
||||
status = models.StatusLate
|
||||
} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
|
||||
status = models.StatusPending
|
||||
}
|
||||
}
|
||||
e.mu.RUnlock()
|
||||
|
||||
if !hasChildren {
|
||||
status = models.StatusPending
|
||||
}
|
||||
|
||||
e.applyState(site.ID, func(s *models.Site) {
|
||||
s.Status = status
|
||||
})
|
||||
e.recordCheck(site.ID, 0, !status.IsBroken())
|
||||
}
|
||||
|
||||
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
|
||||
e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
|
||||
}
|
||||
|
||||
// SetAggStrategy must be called before Start: the field is read by the probe
|
||||
// aggregation path without synchronization.
|
||||
func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
|
||||
e.aggStrategy = strategy
|
||||
}
|
||||
|
||||
func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
|
||||
e.mu.RLock()
|
||||
site, exists := e.liveState[siteID]
|
||||
e.mu.RUnlock()
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
staleAfter := time.Duration(site.Interval) * time.Second * 3
|
||||
if staleAfter < time.Minute {
|
||||
staleAfter = time.Minute
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
e.probeResultsMu.Lock()
|
||||
if e.probeResults[siteID] == nil {
|
||||
e.probeResults[siteID] = make(map[string]NodeResult)
|
||||
}
|
||||
e.probeResults[siteID][nodeID] = NodeResult{
|
||||
NodeID: nodeID,
|
||||
IsUp: isUp,
|
||||
LatencyNs: latencyNs,
|
||||
CheckedAt: now,
|
||||
ErrorReason: errorReason,
|
||||
}
|
||||
results := make([]NodeResult, 0, len(e.probeResults[siteID]))
|
||||
for id, r := range e.probeResults[siteID] {
|
||||
if now.Sub(r.CheckedAt) > staleAfter {
|
||||
delete(e.probeResults[siteID], id)
|
||||
continue
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
e.probeResultsMu.Unlock()
|
||||
|
||||
aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
|
||||
|
||||
probeStatus := models.StatusUp
|
||||
if !aggUp {
|
||||
probeStatus = models.StatusDown
|
||||
}
|
||||
|
||||
updatedSite := site
|
||||
updatedSite.Latency = time.Duration(avgLatency)
|
||||
updatedSite.LastCheck = time.Now()
|
||||
e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
|
||||
}
|
||||
|
||||
func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
|
||||
e.probeResultsMu.RLock()
|
||||
defer e.probeResultsMu.RUnlock()
|
||||
src := e.probeResults[siteID]
|
||||
cp := make(map[string]NodeResult, len(src))
|
||||
for k, v := range src {
|
||||
cp[k] = v
|
||||
}
|
||||
return cp
|
||||
}
|
||||
|
||||
func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
|
||||
changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return changes
|
||||
}
|
||||
|
||||
func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
|
||||
changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return changes
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func (e *Engine) SetActive(active bool) {
|
||||
e.activeMu.Lock()
|
||||
defer e.activeMu.Unlock()
|
||||
if e.isActive != active {
|
||||
e.isActive = active
|
||||
status := "RESUMED (Active)"
|
||||
if !active {
|
||||
status = "PAUSED (Passive)"
|
||||
}
|
||||
e.AddLog(fmt.Sprintf("Engine %s", status))
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) IsActive() bool {
|
||||
e.activeMu.RLock()
|
||||
defer e.activeMu.RUnlock()
|
||||
return e.isActive
|
||||
}
|
||||
|
||||
func (e *Engine) GetAllSites() []models.Site {
|
||||
e.mu.RLock()
|
||||
defer e.mu.RUnlock()
|
||||
sites := make([]models.Site, 0, len(e.liveState))
|
||||
for _, s := range e.liveState {
|
||||
sites = append(sites, s)
|
||||
}
|
||||
return sites
|
||||
}
|
||||
|
||||
func (e *Engine) GetLiveState() map[int]models.Site {
|
||||
e.mu.RLock()
|
||||
defer e.mu.RUnlock()
|
||||
cp := make(map[int]models.Site, len(e.liveState))
|
||||
for k, v := range e.liveState {
|
||||
cp[k] = v
|
||||
}
|
||||
return cp
|
||||
}
|
||||
|
||||
func (e *Engine) addToTokenIndex(site models.Site) {
|
||||
if site.Type == "push" && site.Token != "" {
|
||||
e.tokenIndex[site.Token] = site.ID
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) removeFromTokenIndex(id int) {
|
||||
for token, sid := range e.tokenIndex {
|
||||
if sid == id {
|
||||
delete(e.tokenIndex, token)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
|
||||
e.mu.Lock()
|
||||
if existing, ok := e.liveState[cfg.ID]; ok {
|
||||
e.removeFromTokenIndex(cfg.ID)
|
||||
existing.SiteConfig = cfg
|
||||
e.liveState[cfg.ID] = existing
|
||||
e.addToTokenIndex(existing)
|
||||
}
|
||||
e.mu.Unlock()
|
||||
|
||||
e.signalRecheck(cfg.ID)
|
||||
}
|
||||
|
||||
func (e *Engine) RemoveSite(id int) {
|
||||
e.mu.Lock()
|
||||
e.removeFromTokenIndex(id)
|
||||
delete(e.liveState, id)
|
||||
e.mu.Unlock()
|
||||
e.removeHistory(id)
|
||||
|
||||
e.probeResultsMu.Lock()
|
||||
delete(e.probeResults, id)
|
||||
e.probeResultsMu.Unlock()
|
||||
|
||||
e.recheckMu.Lock()
|
||||
delete(e.recheck, id)
|
||||
e.recheckMu.Unlock()
|
||||
}
|
||||
|
||||
func (e *Engine) ToggleSitePause(id int) bool {
|
||||
var (
|
||||
paused bool
|
||||
name string
|
||||
)
|
||||
_, ok := e.applyState(id, func(s *models.Site) {
|
||||
s.Paused = !s.Paused
|
||||
paused = s.Paused
|
||||
name = s.Name
|
||||
})
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if paused {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
|
||||
} else {
|
||||
e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
|
||||
}
|
||||
return paused
|
||||
}
|
||||
|
||||
// applyState atomically reads, mutates, and writes back the live entry for id.
|
||||
// The mutator runs under the engine write lock and receives a pointer to the
|
||||
// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
|
||||
// never clobbered by a stale snapshot. The mutator must only touch runtime /
|
||||
// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
|
||||
// are owned by UpdateSiteConfig and must not be written here. Returns the
|
||||
// post-mutation copy and whether the site still exists.
|
||||
func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
cur, ok := e.liveState[id]
|
||||
if !ok {
|
||||
return models.Site{}, false
|
||||
}
|
||||
mutate(&cur)
|
||||
e.liveState[id] = cur
|
||||
return cur, true
|
||||
}
|
||||
|
||||
func (e *Engine) GetDisplayStatus(site models.Site) string {
|
||||
if site.Paused {
|
||||
return "PAUSED"
|
||||
}
|
||||
if e.isInMaintenance(site.ID) {
|
||||
return "MAINT"
|
||||
}
|
||||
return string(site.Status)
|
||||
}
|
||||
|
||||
func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
|
||||
changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return changes
|
||||
}
|
||||
|
||||
func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
|
||||
changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return changes
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func TestUpdateSiteConfig_PreservesRuntime(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com"},
|
||||
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3, Latency: 100 * time.Millisecond},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
updated := models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Interval: 60}
|
||||
e.UpdateSiteConfig(updated)
|
||||
|
||||
s, _ := getSite(e, 1)
|
||||
if s.URL != "http://new.com" {
|
||||
t.Errorf("expected URL updated, got %s", s.URL)
|
||||
}
|
||||
if s.Status != "DOWN" {
|
||||
t.Errorf("expected Status preserved, got %s", s.Status)
|
||||
}
|
||||
if s.FailureCount != 3 {
|
||||
t.Errorf("expected FailureCount preserved, got %d", s.FailureCount)
|
||||
}
|
||||
if s.Latency != 100*time.Millisecond {
|
||||
t.Errorf("expected Latency preserved, got %v", s.Latency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveSite_CleansUp(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "push", Token: "tok1"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
e.recordCheck(1, 5*time.Millisecond, true)
|
||||
|
||||
e.RemoveSite(1)
|
||||
|
||||
if _, ok := getSite(e, 1); ok {
|
||||
t.Error("expected site removed from liveState")
|
||||
}
|
||||
if e.RecordHeartbeat("tok1") {
|
||||
t.Error("expected token removed from index")
|
||||
}
|
||||
if _, ok := e.GetHistory(1); ok {
|
||||
t.Error("expected history removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToggleSitePause(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
site := models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "test"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
}
|
||||
injectSite(e, site)
|
||||
|
||||
paused := e.ToggleSitePause(1)
|
||||
if !paused {
|
||||
t.Error("expected paused=true after first toggle")
|
||||
}
|
||||
s, _ := getSite(e, 1)
|
||||
if !s.Paused {
|
||||
t.Error("expected Paused=true in state")
|
||||
}
|
||||
|
||||
paused = e.ToggleSitePause(1)
|
||||
if paused {
|
||||
t.Error("expected paused=false after second toggle")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToggleSitePause_NonexistentSite(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
if e.ToggleSitePause(999) {
|
||||
t.Error("expected false for nonexistent site")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAllSites_ReturnsCopy(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
injectSite(e, models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
})
|
||||
injectSite(e, models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 2, Name: "s2"},
|
||||
SiteState: models.SiteState{Status: "DOWN"},
|
||||
})
|
||||
|
||||
sites := e.GetAllSites()
|
||||
if len(sites) != 2 {
|
||||
t.Fatalf("expected 2 sites, got %d", len(sites))
|
||||
}
|
||||
sites[0].Name = "mutated"
|
||||
|
||||
fresh := e.GetAllSites()
|
||||
for _, s := range fresh {
|
||||
if s.Name == "mutated" {
|
||||
t.Error("GetAllSites returned reference, not copy")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLiveState_ReturnsCopy(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
e := newTestEngine(ms)
|
||||
injectSite(e, models.Site{
|
||||
SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
|
||||
SiteState: models.SiteState{Status: "UP"},
|
||||
})
|
||||
|
||||
state := e.GetLiveState()
|
||||
state[1] = models.Site{SiteConfig: models.SiteConfig{Name: "mutated"}}
|
||||
|
||||
fresh := e.GetLiveState()
|
||||
if fresh[1].Name == "mutated" {
|
||||
t.Error("GetLiveState returned reference, not copy")
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -218,40 +217,6 @@ func (s *SQLStore) GetSiteByName(ctx context.Context, name string) (models.SiteC
|
||||
return st, err
|
||||
}
|
||||
|
||||
func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) {
|
||||
decrypted, err := s.decryptSettings(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt settings: %w", err)
|
||||
}
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(decrypted), &m); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal settings: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) {
|
||||
jsonBytes, err := json.Marshal(settings)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.encryptSettings(string(jsonBytes))
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) {
|
||||
var a models.AlertConfig
|
||||
var settingsRaw string
|
||||
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("alert %q: %w", name, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfig) (int, error) {
|
||||
token := ""
|
||||
if site.Type == "push" {
|
||||
@@ -278,89 +243,6 @@ func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfi
|
||||
return int(id), err
|
||||
}
|
||||
|
||||
func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) {
|
||||
stored, err := s.marshalSettings(settings)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if s.dollar {
|
||||
var id int
|
||||
err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
id, err := result.LastInsertId()
|
||||
return int(id), err
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var alerts []models.AlertConfig
|
||||
for rows.Next() {
|
||||
var a models.AlertConfig
|
||||
var settingsRaw string
|
||||
if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil {
|
||||
return alerts, err
|
||||
}
|
||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||
if err != nil {
|
||||
return alerts, fmt.Errorf("alert %q: %w", a.Name, err)
|
||||
}
|
||||
alerts = append(alerts, a)
|
||||
}
|
||||
return alerts, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) {
|
||||
var a models.AlertConfig
|
||||
var settingsRaw string
|
||||
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("alert %d: %w", id, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error {
|
||||
stored, err := s.marshalSettings(settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error {
|
||||
stored, err := s.marshalSettings(settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error {
|
||||
if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.dialect.ResetSequenceOnEmpty(s.db, "alerts")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAllUsers(ctx context.Context) ([]models.User, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT id, username, public_key, role FROM users")
|
||||
if err != nil {
|
||||
@@ -393,85 +275,6 @@ func (s *SQLStore) DeleteUser(ctx context.Context, id int) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"),
|
||||
siteID, fromStatus, toStatus, errorReason)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var changes []models.StateChange
|
||||
for rows.Next() {
|
||||
var sc models.StateChange
|
||||
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
||||
return changes, err
|
||||
}
|
||||
changes = append(changes, sc)
|
||||
}
|
||||
return changes, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var changes []models.StateChange
|
||||
for rows.Next() {
|
||||
var sc models.StateChange
|
||||
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
||||
return changes, err
|
||||
}
|
||||
changes = append(changes, sc)
|
||||
}
|
||||
return changes, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error {
|
||||
return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp)
|
||||
}
|
||||
|
||||
// SaveCheckFromNode inserts a single check row. Retention is handled out of
|
||||
// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot
|
||||
// path a plain INSERT.
|
||||
func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp)
|
||||
return err
|
||||
}
|
||||
|
||||
// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per
|
||||
// site, across all sites, in one pass. Intended to run periodically.
|
||||
func (s *SQLStore) PruneCheckHistory(ctx context.Context) error {
|
||||
q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn
|
||||
FROM check_history
|
||||
) ranked WHERE rn > %d
|
||||
)`, maxCheckHistory)
|
||||
_, err := s.db.ExecContext(ctx, s.q(q))
|
||||
return err
|
||||
}
|
||||
|
||||
// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite
|
||||
// rows per site. Generous so realistic SLA windows are unaffected; bounds the
|
||||
// otherwise unbounded growth of a flapping monitor's history.
|
||||
func (s *SQLStore) PruneStateChanges(ctx context.Context) error {
|
||||
q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn
|
||||
FROM state_changes
|
||||
) ranked WHERE rn > %d
|
||||
)`, maxStateChangesPerSite)
|
||||
_, err := s.db.ExecContext(ctx, s.q(q))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) RegisterNode(ctx context.Context, node models.ProbeNode) error {
|
||||
_, err := s.db.ExecContext(ctx, s.dialect.UpsertNodeSQL(), node.ID, node.Name, node.Region, node.Version)
|
||||
return err
|
||||
@@ -582,172 +385,6 @@ func (s *SQLStore) LoadLogs(ctx context.Context, limit int) ([]models.LogEntry,
|
||||
return entries, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) {
|
||||
result := make(map[int][]models.CheckRecord)
|
||||
rows, err := s.db.QueryContext(ctx, s.q(`
|
||||
SELECT site_id, latency_ns, is_up FROM (
|
||||
SELECT site_id, latency_ns, is_up,
|
||||
ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn
|
||||
FROM check_history
|
||||
) sub WHERE rn <= ?`), limit)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var r models.CheckRecord
|
||||
if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result[r.SiteID] = append(result[r.SiteID], r)
|
||||
}
|
||||
for id, records := range result {
|
||||
for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 {
|
||||
records[i], records[j] = records[j], records[i]
|
||||
}
|
||||
result[id] = records
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) {
|
||||
var mw models.MaintenanceWindow
|
||||
var endTime sql.NullTime
|
||||
if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil {
|
||||
return mw, err
|
||||
}
|
||||
if endTime.Valid {
|
||||
mw.EndTime = endTime.Time
|
||||
}
|
||||
return mw, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var windows []models.MaintenanceWindow
|
||||
for rows.Next() {
|
||||
mw, err := s.scanMaintenanceWindow(rows)
|
||||
if err != nil {
|
||||
return windows, err
|
||||
}
|
||||
windows = append(windows, mw)
|
||||
}
|
||||
return windows, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var windows []models.MaintenanceWindow
|
||||
for rows.Next() {
|
||||
mw, err := s.scanMaintenanceWindow(rows)
|
||||
if err != nil {
|
||||
return windows, err
|
||||
}
|
||||
windows = append(windows, mw)
|
||||
}
|
||||
return windows, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) {
|
||||
var timeClause string
|
||||
var args []interface{}
|
||||
|
||||
if endTime.IsZero() {
|
||||
timeClause = "(end_time IS NULL OR end_time > ?)"
|
||||
args = append(args, startTime)
|
||||
} else {
|
||||
timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?"
|
||||
args = append(args, startTime, endTime)
|
||||
}
|
||||
|
||||
var scopeClause string
|
||||
if monitorID == 0 {
|
||||
scopeClause = "1=1"
|
||||
} else {
|
||||
scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))"
|
||||
args = append(args, monitorID, monitorID)
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time",
|
||||
timeClause, scopeClause,
|
||||
)
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var windows []models.MaintenanceWindow
|
||||
for rows.Next() {
|
||||
mw, err := s.scanMaintenanceWindow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
windows = append(windows, mw)
|
||||
}
|
||||
return windows, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error {
|
||||
if mw.StartTime.IsZero() {
|
||||
mw.StartTime = time.Now()
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"),
|
||||
mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) {
|
||||
cutoff := time.Now().Add(-retention)
|
||||
result, err := s.db.ExecContext(ctx,
|
||||
s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"),
|
||||
cutoff,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) {
|
||||
var count int
|
||||
err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows
|
||||
WHERE type = 'maintenance'
|
||||
AND start_time <= CURRENT_TIMESTAMP
|
||||
AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP)
|
||||
AND (monitor_id = 0 OR monitor_id = ?
|
||||
OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`),
|
||||
monitorID, monitorID).Scan(&count)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetPreference(ctx context.Context, key string) (string, error) {
|
||||
var value string
|
||||
err := s.db.QueryRowContext(ctx, s.q("SELECT value FROM preferences WHERE key = ?"), key).Scan(&value)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) {
|
||||
decrypted, err := s.decryptSettings(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt settings: %w", err)
|
||||
}
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal([]byte(decrypted), &m); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal settings: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) {
|
||||
jsonBytes, err := json.Marshal(settings)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.encryptSettings(string(jsonBytes))
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) {
|
||||
var a models.AlertConfig
|
||||
var settingsRaw string
|
||||
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("alert %q: %w", name, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) {
|
||||
stored, err := s.marshalSettings(settings)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if s.dollar {
|
||||
var id int
|
||||
err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
id, err := result.LastInsertId()
|
||||
return int(id), err
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var alerts []models.AlertConfig
|
||||
for rows.Next() {
|
||||
var a models.AlertConfig
|
||||
var settingsRaw string
|
||||
if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil {
|
||||
return alerts, err
|
||||
}
|
||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||
if err != nil {
|
||||
return alerts, fmt.Errorf("alert %q: %w", a.Name, err)
|
||||
}
|
||||
alerts = append(alerts, a)
|
||||
}
|
||||
return alerts, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) {
|
||||
var a models.AlertConfig
|
||||
var settingsRaw string
|
||||
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("alert %d: %w", id, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error {
|
||||
stored, err := s.marshalSettings(settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error {
|
||||
stored, err := s.marshalSettings(settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error {
|
||||
if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.dialect.ResetSequenceOnEmpty(s.db, "alerts")
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func TestAlertCRUD(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil {
|
||||
t.Fatalf("AddAlert: %v", err)
|
||||
}
|
||||
|
||||
alerts, err := s.GetAllAlerts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllAlerts: %v", err)
|
||||
}
|
||||
if len(alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(alerts))
|
||||
}
|
||||
if alerts[0].Type != "discord" {
|
||||
t.Errorf("expected type 'discord', got '%s'", alerts[0].Type)
|
||||
}
|
||||
if alerts[0].Settings["url"] != "https://example.com/hook" {
|
||||
t.Errorf("settings url mismatch")
|
||||
}
|
||||
|
||||
a, err := s.GetAlert(context.Background(), alerts[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAlert: %v", err)
|
||||
}
|
||||
if a.Name != "Discord" {
|
||||
t.Errorf("expected name 'Discord', got '%s'", a.Name)
|
||||
}
|
||||
|
||||
if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil {
|
||||
t.Fatalf("UpdateAlert: %v", err)
|
||||
}
|
||||
|
||||
a, err = s.GetAlert(context.Background(), a.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAlert: %v", err)
|
||||
}
|
||||
if a.Type != "slack" {
|
||||
t.Errorf("expected type 'slack', got '%s'", a.Type)
|
||||
}
|
||||
|
||||
if err := s.DeleteAlert(context.Background(), a.ID); err != nil {
|
||||
t.Fatalf("DeleteAlert: %v", err)
|
||||
}
|
||||
|
||||
alerts, err = s.GetAllAlerts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllAlerts: %v", err)
|
||||
}
|
||||
if len(alerts) != 0 {
|
||||
t.Fatalf("expected 0 alerts after delete, got %d", len(alerts))
|
||||
}
|
||||
}
|
||||
|
||||
// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a
|
||||
// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext.
|
||||
func TestImportData_EncryptsAlertSettings(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes
|
||||
if err != nil {
|
||||
t.Fatalf("NewEncryptor: %v", err)
|
||||
}
|
||||
s.SetEncryptor(enc)
|
||||
|
||||
backup := models.Backup{
|
||||
Alerts: []models.AlertConfig{
|
||||
{ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}},
|
||||
},
|
||||
}
|
||||
if err := s.ImportData(context.Background(), backup); err != nil {
|
||||
t.Fatalf("ImportData: %v", err)
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil {
|
||||
t.Fatalf("query settings: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(raw, encryptedPrefix) {
|
||||
t.Errorf("imported settings not encrypted: %q", raw)
|
||||
}
|
||||
if strings.Contains(raw, "SECRET") {
|
||||
t.Errorf("plaintext secret found in stored column: %q", raw)
|
||||
}
|
||||
|
||||
alerts, err := s.GetAllAlerts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllAlerts: %v", err)
|
||||
}
|
||||
if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" {
|
||||
t.Errorf("decrypt round-trip failed: %+v", alerts)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"),
|
||||
siteID, fromStatus, toStatus, errorReason)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var changes []models.StateChange
|
||||
for rows.Next() {
|
||||
var sc models.StateChange
|
||||
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
||||
return changes, err
|
||||
}
|
||||
changes = append(changes, sc)
|
||||
}
|
||||
return changes, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var changes []models.StateChange
|
||||
for rows.Next() {
|
||||
var sc models.StateChange
|
||||
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
||||
return changes, err
|
||||
}
|
||||
changes = append(changes, sc)
|
||||
}
|
||||
return changes, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error {
|
||||
return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp)
|
||||
}
|
||||
|
||||
// SaveCheckFromNode inserts a single check row. Retention is handled out of
|
||||
// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot
|
||||
// path a plain INSERT.
|
||||
func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp)
|
||||
return err
|
||||
}
|
||||
|
||||
// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per
|
||||
// site, across all sites, in one pass. Intended to run periodically.
|
||||
func (s *SQLStore) PruneCheckHistory(ctx context.Context) error {
|
||||
q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn
|
||||
FROM check_history
|
||||
) ranked WHERE rn > %d
|
||||
)`, maxCheckHistory)
|
||||
_, err := s.db.ExecContext(ctx, s.q(q))
|
||||
return err
|
||||
}
|
||||
|
||||
// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite
|
||||
// rows per site. Generous so realistic SLA windows are unaffected; bounds the
|
||||
// otherwise unbounded growth of a flapping monitor's history.
|
||||
func (s *SQLStore) PruneStateChanges(ctx context.Context) error {
|
||||
q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN (
|
||||
SELECT id FROM (
|
||||
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn
|
||||
FROM state_changes
|
||||
) ranked WHERE rn > %d
|
||||
)`, maxStateChangesPerSite)
|
||||
_, err := s.db.ExecContext(ctx, s.q(q))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) {
|
||||
result := make(map[int][]models.CheckRecord)
|
||||
rows, err := s.db.QueryContext(ctx, s.q(`
|
||||
SELECT site_id, latency_ns, is_up FROM (
|
||||
SELECT site_id, latency_ns, is_up,
|
||||
ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn
|
||||
FROM check_history
|
||||
) sub WHERE rn <= ?`), limit)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var r models.CheckRecord
|
||||
if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result[r.SiteID] = append(result[r.SiteID], r)
|
||||
}
|
||||
for id, records := range result {
|
||||
for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 {
|
||||
records[i], records[j] = records[j], records[i]
|
||||
}
|
||||
result[id] = records
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func TestCheckHistory(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil {
|
||||
t.Fatalf("SaveCheck: %v", err)
|
||||
}
|
||||
if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil {
|
||||
t.Fatalf("SaveCheck: %v", err)
|
||||
}
|
||||
if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil {
|
||||
t.Fatalf("SaveCheck site 2: %v", err)
|
||||
}
|
||||
|
||||
history, err := s.LoadAllHistory(context.Background(), 10)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAllHistory: %v", err)
|
||||
}
|
||||
if len(history[1]) != 2 {
|
||||
t.Fatalf("expected 2 records for site 1, got %d", len(history[1]))
|
||||
}
|
||||
if len(history[2]) != 1 {
|
||||
t.Fatalf("expected 1 record for site 2, got %d", len(history[2]))
|
||||
}
|
||||
|
||||
upCount := 0
|
||||
for _, r := range history[1] {
|
||||
if r.IsUp {
|
||||
upCount++
|
||||
}
|
||||
}
|
||||
if upCount != 1 {
|
||||
t.Errorf("expected 1 up record for site 1, got %d", upCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneCheckHistory(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
for i := 0; i < maxCheckHistory+5; i++ {
|
||||
if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil {
|
||||
t.Fatalf("SaveCheck site 1: %v", err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil {
|
||||
t.Fatalf("SaveCheck site 2: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.PruneCheckHistory(context.Background()); err != nil {
|
||||
t.Fatalf("PruneCheckHistory: %v", err)
|
||||
}
|
||||
|
||||
history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAllHistory: %v", err)
|
||||
}
|
||||
if len(history[1]) != maxCheckHistory {
|
||||
t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1]))
|
||||
}
|
||||
if len(history[2]) != 3 {
|
||||
t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2]))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSiteCascade(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30}
|
||||
if err := s.AddSite(context.Background(), site); err != nil {
|
||||
t.Fatalf("AddSite: %v", err)
|
||||
}
|
||||
sites, err := s.GetSites(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetSites: %v", err)
|
||||
}
|
||||
siteID := sites[0].ID
|
||||
|
||||
if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil {
|
||||
t.Fatalf("SaveCheck: %v", err)
|
||||
}
|
||||
if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil {
|
||||
t.Fatalf("SaveStateChange: %v", err)
|
||||
}
|
||||
mw := models.MaintenanceWindow{
|
||||
MonitorID: siteID,
|
||||
Title: "Test MW",
|
||||
Type: "maintenance",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
|
||||
if err := s.DeleteSite(context.Background(), siteID); err != nil {
|
||||
t.Fatalf("DeleteSite: %v", err)
|
||||
}
|
||||
|
||||
history, err := s.LoadAllHistory(context.Background(), 100)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAllHistory: %v", err)
|
||||
}
|
||||
if len(history[siteID]) != 0 {
|
||||
t.Errorf("expected 0 check_history rows, got %d", len(history[siteID]))
|
||||
}
|
||||
|
||||
changes, err := s.GetStateChanges(context.Background(), siteID, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("GetStateChanges: %v", err)
|
||||
}
|
||||
if len(changes) != 0 {
|
||||
t.Errorf("expected 0 state_changes rows, got %d", len(changes))
|
||||
}
|
||||
|
||||
windows, err := s.GetActiveMaintenanceWindows(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetActiveMaintenanceWindows: %v", err)
|
||||
}
|
||||
for _, w := range windows {
|
||||
if w.MonitorID == siteID {
|
||||
t.Errorf("orphaned maintenance window found: id=%d", w.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) {
|
||||
var mw models.MaintenanceWindow
|
||||
var endTime sql.NullTime
|
||||
if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil {
|
||||
return mw, err
|
||||
}
|
||||
if endTime.Valid {
|
||||
mw.EndTime = endTime.Time
|
||||
}
|
||||
return mw, nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var windows []models.MaintenanceWindow
|
||||
for rows.Next() {
|
||||
mw, err := s.scanMaintenanceWindow(rows)
|
||||
if err != nil {
|
||||
return windows, err
|
||||
}
|
||||
windows = append(windows, mw)
|
||||
}
|
||||
return windows, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) {
|
||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var windows []models.MaintenanceWindow
|
||||
for rows.Next() {
|
||||
mw, err := s.scanMaintenanceWindow(rows)
|
||||
if err != nil {
|
||||
return windows, err
|
||||
}
|
||||
windows = append(windows, mw)
|
||||
}
|
||||
return windows, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) {
|
||||
var timeClause string
|
||||
var args []interface{}
|
||||
|
||||
if endTime.IsZero() {
|
||||
timeClause = "(end_time IS NULL OR end_time > ?)"
|
||||
args = append(args, startTime)
|
||||
} else {
|
||||
timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?"
|
||||
args = append(args, startTime, endTime)
|
||||
}
|
||||
|
||||
var scopeClause string
|
||||
if monitorID == 0 {
|
||||
scopeClause = "1=1"
|
||||
} else {
|
||||
scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))"
|
||||
args = append(args, monitorID, monitorID)
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(
|
||||
"SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time",
|
||||
timeClause, scopeClause,
|
||||
)
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var windows []models.MaintenanceWindow
|
||||
for rows.Next() {
|
||||
mw, err := s.scanMaintenanceWindow(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
windows = append(windows, mw)
|
||||
}
|
||||
return windows, rows.Err()
|
||||
}
|
||||
|
||||
func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error {
|
||||
if mw.StartTime.IsZero() {
|
||||
mw.StartTime = time.Now()
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"),
|
||||
mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error {
|
||||
_, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) {
|
||||
cutoff := time.Now().Add(-retention)
|
||||
result, err := s.db.ExecContext(ctx,
|
||||
s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"),
|
||||
cutoff,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) {
|
||||
var count int
|
||||
err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows
|
||||
WHERE type = 'maintenance'
|
||||
AND start_time <= CURRENT_TIMESTAMP
|
||||
AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP)
|
||||
AND (monitor_id = 0 OR monitor_id = ?
|
||||
OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`),
|
||||
monitorID, monitorID).Scan(&count)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
|
||||
func TestPruneExpiredMaintenanceWindows(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Expired 10 days ago — should be pruned with 7d retention.
|
||||
old := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Old Window",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-11 * 24 * time.Hour),
|
||||
EndTime: now.Add(-10 * 24 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), old); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow (old): %v", err)
|
||||
}
|
||||
|
||||
// Expired 1 day ago — within 7d retention, should survive.
|
||||
recent := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Recent Window",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-2 * 24 * time.Hour),
|
||||
EndTime: now.Add(-1 * 24 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow (recent): %v", err)
|
||||
}
|
||||
|
||||
// Ongoing — no end time, should survive.
|
||||
ongoing := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Ongoing Window",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-1 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow (ongoing): %v", err)
|
||||
}
|
||||
|
||||
pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("PruneExpiredMaintenanceWindows: %v", err)
|
||||
}
|
||||
if pruned != 1 {
|
||||
t.Errorf("expected 1 pruned, got %d", pruned)
|
||||
}
|
||||
|
||||
all, err := s.GetAllMaintenanceWindows(context.Background(), 100)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllMaintenanceWindows: %v", err)
|
||||
}
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected 2 remaining windows, got %d", len(all))
|
||||
}
|
||||
for _, w := range all {
|
||||
if w.Title == "Old Window" {
|
||||
t.Error("old window should have been pruned")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOverlappingMaintenanceWindows(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30}
|
||||
if err := s.AddSite(ctx, site); err != nil {
|
||||
t.Fatalf("AddSite: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
active := models.MaintenanceWindow{
|
||||
MonitorID: 1,
|
||||
Title: "Deploy v2",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-30 * time.Minute),
|
||||
EndTime: now.Add(30 * time.Minute),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(ctx, active); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
|
||||
ended := models.MaintenanceWindow{
|
||||
MonitorID: 1,
|
||||
Title: "Old deploy",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-3 * time.Hour),
|
||||
EndTime: now.Add(-2 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(ctx, ended); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
|
||||
t.Run("same monitor overlaps", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 1 {
|
||||
t.Fatalf("expected 1 overlap, got %d", len(overlaps))
|
||||
}
|
||||
if overlaps[0].Title != "Deploy v2" {
|
||||
t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("different monitor no overlap", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 0 {
|
||||
t.Errorf("expected 0 overlaps, got %d", len(overlaps))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("global window overlaps all", func(t *testing.T) {
|
||||
global := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Global freeze",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
EndTime: now.Add(2 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(ctx, global); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 2 {
|
||||
t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("indefinite window overlaps", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) < 1 {
|
||||
t.Error("expected at least 1 overlap for indefinite window")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ended window excluded", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 0 {
|
||||
t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -3,9 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||
)
|
||||
@@ -74,60 +72,6 @@ func TestSiteCRUD(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertCRUD(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil {
|
||||
t.Fatalf("AddAlert: %v", err)
|
||||
}
|
||||
|
||||
alerts, err := s.GetAllAlerts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllAlerts: %v", err)
|
||||
}
|
||||
if len(alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(alerts))
|
||||
}
|
||||
if alerts[0].Type != "discord" {
|
||||
t.Errorf("expected type 'discord', got '%s'", alerts[0].Type)
|
||||
}
|
||||
if alerts[0].Settings["url"] != "https://example.com/hook" {
|
||||
t.Errorf("settings url mismatch")
|
||||
}
|
||||
|
||||
a, err := s.GetAlert(context.Background(), alerts[0].ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAlert: %v", err)
|
||||
}
|
||||
if a.Name != "Discord" {
|
||||
t.Errorf("expected name 'Discord', got '%s'", a.Name)
|
||||
}
|
||||
|
||||
if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil {
|
||||
t.Fatalf("UpdateAlert: %v", err)
|
||||
}
|
||||
|
||||
a, err = s.GetAlert(context.Background(), a.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAlert: %v", err)
|
||||
}
|
||||
if a.Type != "slack" {
|
||||
t.Errorf("expected type 'slack', got '%s'", a.Type)
|
||||
}
|
||||
|
||||
if err := s.DeleteAlert(context.Background(), a.ID); err != nil {
|
||||
t.Fatalf("DeleteAlert: %v", err)
|
||||
}
|
||||
|
||||
alerts, err = s.GetAllAlerts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllAlerts: %v", err)
|
||||
}
|
||||
if len(alerts) != 0 {
|
||||
t.Fatalf("expected 0 alerts after delete, got %d", len(alerts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserCRUD(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
@@ -301,101 +245,6 @@ func TestImportData_NilUsersPreservesExisting(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHistory(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil {
|
||||
t.Fatalf("SaveCheck: %v", err)
|
||||
}
|
||||
if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil {
|
||||
t.Fatalf("SaveCheck: %v", err)
|
||||
}
|
||||
if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil {
|
||||
t.Fatalf("SaveCheck site 2: %v", err)
|
||||
}
|
||||
|
||||
history, err := s.LoadAllHistory(context.Background(), 10)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAllHistory: %v", err)
|
||||
}
|
||||
if len(history[1]) != 2 {
|
||||
t.Fatalf("expected 2 records for site 1, got %d", len(history[1]))
|
||||
}
|
||||
if len(history[2]) != 1 {
|
||||
t.Fatalf("expected 1 record for site 2, got %d", len(history[2]))
|
||||
}
|
||||
|
||||
upCount := 0
|
||||
for _, r := range history[1] {
|
||||
if r.IsUp {
|
||||
upCount++
|
||||
}
|
||||
}
|
||||
if upCount != 1 {
|
||||
t.Errorf("expected 1 up record for site 1, got %d", upCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteSiteCascade(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30}
|
||||
if err := s.AddSite(context.Background(), site); err != nil {
|
||||
t.Fatalf("AddSite: %v", err)
|
||||
}
|
||||
sites, err := s.GetSites(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetSites: %v", err)
|
||||
}
|
||||
siteID := sites[0].ID
|
||||
|
||||
if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil {
|
||||
t.Fatalf("SaveCheck: %v", err)
|
||||
}
|
||||
if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil {
|
||||
t.Fatalf("SaveStateChange: %v", err)
|
||||
}
|
||||
mw := models.MaintenanceWindow{
|
||||
MonitorID: siteID,
|
||||
Title: "Test MW",
|
||||
Type: "maintenance",
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
|
||||
if err := s.DeleteSite(context.Background(), siteID); err != nil {
|
||||
t.Fatalf("DeleteSite: %v", err)
|
||||
}
|
||||
|
||||
history, err := s.LoadAllHistory(context.Background(), 100)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAllHistory: %v", err)
|
||||
}
|
||||
if len(history[siteID]) != 0 {
|
||||
t.Errorf("expected 0 check_history rows, got %d", len(history[siteID]))
|
||||
}
|
||||
|
||||
changes, err := s.GetStateChanges(context.Background(), siteID, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("GetStateChanges: %v", err)
|
||||
}
|
||||
if len(changes) != 0 {
|
||||
t.Errorf("expected 0 state_changes rows, got %d", len(changes))
|
||||
}
|
||||
|
||||
windows, err := s.GetActiveMaintenanceWindows(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetActiveMaintenanceWindows: %v", err)
|
||||
}
|
||||
for _, w := range windows {
|
||||
if w.MonitorID == siteID {
|
||||
t.Errorf("orphaned maintenance window found: id=%d", w.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneLogs(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
@@ -428,231 +277,3 @@ func TestPruneLogs(t *testing.T) {
|
||||
t.Error("oldest log survived prune")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneCheckHistory(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
for i := 0; i < maxCheckHistory+5; i++ {
|
||||
if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil {
|
||||
t.Fatalf("SaveCheck site 1: %v", err)
|
||||
}
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil {
|
||||
t.Fatalf("SaveCheck site 2: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.PruneCheckHistory(context.Background()); err != nil {
|
||||
t.Fatalf("PruneCheckHistory: %v", err)
|
||||
}
|
||||
|
||||
history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAllHistory: %v", err)
|
||||
}
|
||||
if len(history[1]) != maxCheckHistory {
|
||||
t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1]))
|
||||
}
|
||||
if len(history[2]) != 3 {
|
||||
t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2]))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneExpiredMaintenanceWindows(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Expired 10 days ago — should be pruned with 7d retention.
|
||||
old := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Old Window",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-11 * 24 * time.Hour),
|
||||
EndTime: now.Add(-10 * 24 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), old); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow (old): %v", err)
|
||||
}
|
||||
|
||||
// Expired 1 day ago — within 7d retention, should survive.
|
||||
recent := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Recent Window",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-2 * 24 * time.Hour),
|
||||
EndTime: now.Add(-1 * 24 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow (recent): %v", err)
|
||||
}
|
||||
|
||||
// Ongoing — no end time, should survive.
|
||||
ongoing := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Ongoing Window",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-1 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow (ongoing): %v", err)
|
||||
}
|
||||
|
||||
pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("PruneExpiredMaintenanceWindows: %v", err)
|
||||
}
|
||||
if pruned != 1 {
|
||||
t.Errorf("expected 1 pruned, got %d", pruned)
|
||||
}
|
||||
|
||||
all, err := s.GetAllMaintenanceWindows(context.Background(), 100)
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllMaintenanceWindows: %v", err)
|
||||
}
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected 2 remaining windows, got %d", len(all))
|
||||
}
|
||||
for _, w := range all {
|
||||
if w.Title == "Old Window" {
|
||||
t.Error("old window should have been pruned")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetOverlappingMaintenanceWindows(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30}
|
||||
if err := s.AddSite(ctx, site); err != nil {
|
||||
t.Fatalf("AddSite: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
active := models.MaintenanceWindow{
|
||||
MonitorID: 1,
|
||||
Title: "Deploy v2",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-30 * time.Minute),
|
||||
EndTime: now.Add(30 * time.Minute),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(ctx, active); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
|
||||
ended := models.MaintenanceWindow{
|
||||
MonitorID: 1,
|
||||
Title: "Old deploy",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-3 * time.Hour),
|
||||
EndTime: now.Add(-2 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(ctx, ended); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
|
||||
t.Run("same monitor overlaps", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 1 {
|
||||
t.Fatalf("expected 1 overlap, got %d", len(overlaps))
|
||||
}
|
||||
if overlaps[0].Title != "Deploy v2" {
|
||||
t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("different monitor no overlap", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 0 {
|
||||
t.Errorf("expected 0 overlaps, got %d", len(overlaps))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("global window overlaps all", func(t *testing.T) {
|
||||
global := models.MaintenanceWindow{
|
||||
MonitorID: 0,
|
||||
Title: "Global freeze",
|
||||
Type: "maintenance",
|
||||
StartTime: now.Add(-10 * time.Minute),
|
||||
EndTime: now.Add(2 * time.Hour),
|
||||
}
|
||||
if err := s.AddMaintenanceWindow(ctx, global); err != nil {
|
||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||
}
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 2 {
|
||||
t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("indefinite window overlaps", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) < 1 {
|
||||
t.Error("expected at least 1 overlap for indefinite window")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ended window excluded", func(t *testing.T) {
|
||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(overlaps) != 0 {
|
||||
t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a
|
||||
// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext.
|
||||
func TestImportData_EncryptsAlertSettings(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes
|
||||
if err != nil {
|
||||
t.Fatalf("NewEncryptor: %v", err)
|
||||
}
|
||||
s.SetEncryptor(enc)
|
||||
|
||||
backup := models.Backup{
|
||||
Alerts: []models.AlertConfig{
|
||||
{ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}},
|
||||
},
|
||||
}
|
||||
if err := s.ImportData(context.Background(), backup); err != nil {
|
||||
t.Fatalf("ImportData: %v", err)
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil {
|
||||
t.Fatalf("query settings: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(raw, encryptedPrefix) {
|
||||
t.Errorf("imported settings not encrypted: %q", raw)
|
||||
}
|
||||
if strings.Contains(raw, "SECRET") {
|
||||
t.Errorf("plaintext secret found in stored column: %q", raw)
|
||||
}
|
||||
|
||||
alerts, err := s.GetAllAlerts(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("GetAllAlerts: %v", err)
|
||||
}
|
||||
if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" {
|
||||
t.Errorf("decrypt round-trip failed: %+v", alerts)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user