refactor: decompose monitor.go and sqlstore.go into focused files #154
@@ -0,0 +1,272 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AlertHealth struct {
|
||||||
|
LastSendAt time.Time
|
||||||
|
LastSendOK bool
|
||||||
|
LastError string
|
||||||
|
SendCount int
|
||||||
|
FailCount int
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitAlertHealth restores persisted alert send health so the dashboard shows real
|
||||||
|
// "last sent" / health state on startup instead of resetting every channel to "never".
|
||||||
|
func (e *Engine) InitAlertHealth() {
|
||||||
|
records, err := e.db.LoadAlertHealth(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
e.alertHealthMu.Lock()
|
||||||
|
defer e.alertHealthMu.Unlock()
|
||||||
|
for id, r := range records {
|
||||||
|
e.alertHealth[id] = AlertHealth{
|
||||||
|
LastSendAt: r.LastSendAt,
|
||||||
|
LastSendOK: r.LastSendOK,
|
||||||
|
LastError: r.LastError,
|
||||||
|
SendCount: r.SendCount,
|
||||||
|
FailCount: r.FailCount,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleStatusChange folds a check result into the live state. snap is the
|
||||||
|
// stale snapshot the check ran against; the actual mutation is applied onto the
|
||||||
|
// CURRENT live entry via applyState, so a concurrent pause / config edit /
|
||||||
|
// heartbeat is never reverted by this write. Logs and alerts are emitted after
|
||||||
|
// the lock is released, off the critical section.
|
||||||
|
func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
|
||||||
|
if !e.IsActive() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
inMaint := e.isInMaintenance(snap.ID)
|
||||||
|
status := models.Status(rawStatus)
|
||||||
|
|
||||||
|
var (
|
||||||
|
prev, next models.Status
|
||||||
|
name, typ string
|
||||||
|
alertID int
|
||||||
|
failCount, maxRetries int
|
||||||
|
confirmedDown bool
|
||||||
|
failedCheck bool
|
||||||
|
downSince time.Time
|
||||||
|
sslWarnFire bool
|
||||||
|
sslDays int
|
||||||
|
skipped bool
|
||||||
|
changed bool
|
||||||
|
)
|
||||||
|
|
||||||
|
_, exists := e.applyState(snap.ID, func(s *models.Site) {
|
||||||
|
// A non-UP result computed from a stale snapshot must not override a
|
||||||
|
// heartbeat (or newer check) that landed while we were evaluating.
|
||||||
|
if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
|
||||||
|
skipped = true
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
prev = s.Status
|
||||||
|
name = s.Name
|
||||||
|
typ = s.Type
|
||||||
|
alertID = s.AlertID
|
||||||
|
maxRetries = s.MaxRetries
|
||||||
|
downSince = s.StatusChangedAt
|
||||||
|
|
||||||
|
// Fresh check results (measured by the run against snap).
|
||||||
|
s.StatusCode = code
|
||||||
|
s.Latency = snap.Latency
|
||||||
|
s.LastCheck = snap.LastCheck
|
||||||
|
s.HasSSL = snap.HasSSL
|
||||||
|
s.CertExpiry = snap.CertExpiry
|
||||||
|
s.LastError = errorReason
|
||||||
|
if status == models.StatusUp {
|
||||||
|
s.LastSuccessAt = time.Now()
|
||||||
|
s.LastError = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status + failure-count transition, based on the CURRENT live status.
|
||||||
|
if status == models.StatusUp {
|
||||||
|
s.FailureCount = 0
|
||||||
|
s.Status = models.StatusUp
|
||||||
|
} else {
|
||||||
|
if s.FailureCount <= s.MaxRetries {
|
||||||
|
s.FailureCount++
|
||||||
|
}
|
||||||
|
if s.FailureCount > s.MaxRetries {
|
||||||
|
if s.Status != status {
|
||||||
|
confirmedDown = true
|
||||||
|
}
|
||||||
|
s.Status = status
|
||||||
|
s.FailureCount = s.MaxRetries + 1
|
||||||
|
} else {
|
||||||
|
failedCheck = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failCount = s.FailureCount
|
||||||
|
|
||||||
|
if s.Status != prev && prev != models.StatusPending {
|
||||||
|
s.StatusChangedAt = time.Now()
|
||||||
|
} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
|
||||||
|
s.StatusChangedAt = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
|
||||||
|
if typ == "http" && s.CheckSSL && s.HasSSL {
|
||||||
|
days := int(time.Until(s.CertExpiry).Hours() / 24)
|
||||||
|
if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
|
||||||
|
sslWarnFire = true
|
||||||
|
sslDays = days
|
||||||
|
s.SentSSLWarning = true
|
||||||
|
} else if days > s.ExpiryThreshold {
|
||||||
|
s.SentSSLWarning = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
next = s.Status
|
||||||
|
changed = next != prev
|
||||||
|
})
|
||||||
|
|
||||||
|
if !exists || skipped {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
e.recordCheck(snap.ID, latency, status == models.StatusUp)
|
||||||
|
|
||||||
|
if confirmedDown {
|
||||||
|
if errorReason != "" {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
|
||||||
|
} else {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
|
||||||
|
}
|
||||||
|
} else if failedCheck {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
|
||||||
|
}
|
||||||
|
|
||||||
|
if changed && prev != models.StatusPending {
|
||||||
|
e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
|
||||||
|
}
|
||||||
|
|
||||||
|
if sslWarnFire {
|
||||||
|
if !inMaint {
|
||||||
|
e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
|
||||||
|
} else {
|
||||||
|
e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if prev == models.StatusUp && next == models.StatusLate {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
|
||||||
|
}
|
||||||
|
|
||||||
|
if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
|
||||||
|
if inMaint {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
|
||||||
|
} else {
|
||||||
|
msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
|
||||||
|
if errorReason != "" {
|
||||||
|
msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
|
||||||
|
}
|
||||||
|
if typ == "push" {
|
||||||
|
msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
|
||||||
|
}
|
||||||
|
e.triggerAlert(alertID, "🚨 ALERT", msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if prev.IsBroken() && next == models.StatusUp {
|
||||||
|
downDur := ""
|
||||||
|
if !downSince.IsZero() {
|
||||||
|
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
||||||
|
}
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
|
||||||
|
if !inMaint {
|
||||||
|
e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) triggerAlert(alertID int, title, message string) {
|
||||||
|
if alertID <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
||||||
|
if err != nil {
|
||||||
|
e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
provider := alert.GetProvider(cfg)
|
||||||
|
if provider != nil {
|
||||||
|
go func() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := provider.Send(ctx, title, message); err != nil {
|
||||||
|
e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
|
||||||
|
e.recordAlertResult(alertID, false, err.Error())
|
||||||
|
} else {
|
||||||
|
e.recordAlertResult(alertID, true, "")
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
|
||||||
|
e.alertHealthMu.Lock()
|
||||||
|
defer e.alertHealthMu.Unlock()
|
||||||
|
h := e.alertHealth[alertID]
|
||||||
|
h.LastSendAt = time.Now()
|
||||||
|
h.LastSendOK = ok
|
||||||
|
h.SendCount++
|
||||||
|
if ok {
|
||||||
|
h.LastError = ""
|
||||||
|
} else {
|
||||||
|
h.LastError = errMsg
|
||||||
|
h.FailCount++
|
||||||
|
}
|
||||||
|
e.alertHealth[alertID] = h
|
||||||
|
|
||||||
|
// Persist so health survives restarts; DB IO off the alert path.
|
||||||
|
e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
|
||||||
|
AlertID: alertID,
|
||||||
|
LastSendAt: h.LastSendAt,
|
||||||
|
LastSendOK: h.LastSendOK,
|
||||||
|
LastError: h.LastError,
|
||||||
|
SendCount: h.SendCount,
|
||||||
|
FailCount: h.FailCount,
|
||||||
|
}})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
|
||||||
|
e.alertHealthMu.RLock()
|
||||||
|
defer e.alertHealthMu.RUnlock()
|
||||||
|
return e.alertHealth[alertID]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) TestAlert(alertID int) error {
|
||||||
|
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to load alert: %w", err)
|
||||||
|
}
|
||||||
|
provider := alert.GetProvider(cfg)
|
||||||
|
if provider == nil {
|
||||||
|
return fmt.Errorf("no provider for type %q", cfg.Type)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
||||||
|
defer cancel()
|
||||||
|
err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
|
||||||
|
if err != nil {
|
||||||
|
e.recordAlertResult(alertID, false, err.Error())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
e.recordAlertResult(alertID, true, "")
|
||||||
|
e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,487 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// --- Group 1: State Machine ---
|
||||||
|
|
||||||
|
func TestHandleStatusChange_PendingToUp(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "PENDING"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "UP", 200, 10*time.Millisecond, "")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.FailureCount != 0 {
|
||||||
|
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||||
|
t.Error("expected no alert for PENDING→UP")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_UpIncrementFailure(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 3},
|
||||||
|
SiteState: models.SiteState{Status: "UP", FailureCount: 0},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 500, 0, "test error")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP (under retry threshold), got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.FailureCount != 1 {
|
||||||
|
t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_UpToDown_ExceedsRetries(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "discord", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP", FailureCount: 2},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 500, 0, "test error")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected DOWN, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.FailureCount != 3 {
|
||||||
|
t.Errorf("expected FailureCount 3, got %d", s.FailureCount)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
calls := ms.getAlertCallsSnapshot()
|
||||||
|
if len(calls) == 0 || calls[0] != 1 {
|
||||||
|
t.Errorf("expected alert call for alertID 1, got %v", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_UpToDown_ZeroRetries(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP", FailureCount: 0},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected DOWN, got %s", s.Status)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||||
|
t.Error("expected alert on immediate DOWN")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_DownToUp_Recovery(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN", FailureCount: 4},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.FailureCount != 0 {
|
||||||
|
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||||
|
t.Error("expected recovery alert")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_DownStaysDown(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 2},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected DOWN, got %s", s.Status)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||||
|
t.Error("expected no re-alert for already DOWN")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_SSLExpired(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "SSL EXP", 0, 0, "SSL certificate expired")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "SSL EXP" {
|
||||||
|
t.Errorf("expected SSL EXP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||||
|
t.Error("expected alert on SSL EXP")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_AlertSuppressedMaintenance(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.maintenance[1] = true
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
e.refreshMaintenanceCache(context.Background())
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected DOWN, got %s", s.Status)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||||
|
t.Error("expected no alert during maintenance")
|
||||||
|
}
|
||||||
|
logs := e.GetLogs()
|
||||||
|
found := false
|
||||||
|
for _, l := range logs {
|
||||||
|
if containsStr(l.Message, "suppressed") {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("expected log mentioning suppressed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_RecoverySuppressedMaintenance(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.maintenance[1] = true
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
e.refreshMaintenanceCache(context.Background())
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||||
|
t.Error("expected no alert during maintenance recovery")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_SSLWarning(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if !s.SentSSLWarning {
|
||||||
|
t.Error("expected SentSSLWarning=true")
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||||
|
t.Error("expected SSL warning alert")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_SSLWarningNotRepeated(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||||
|
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||||
|
t.Error("expected no repeat SSL warning")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_SSLWarningReset(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30},
|
||||||
|
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: true, CertExpiry: time.Now().Add(60 * 24 * time.Hour)},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.SentSSLWarning {
|
||||||
|
t.Error("expected SentSSLWarning reset to false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_SSLWarningSuppressedMaint(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.maintenance[1] = true
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", CheckSSL: true, ExpiryThreshold: 30, AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP", HasSSL: true, SentSSLWarning: false, CertExpiry: time.Now().Add(15 * 24 * time.Hour)},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
e.refreshMaintenanceCache(context.Background())
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "UP", 200, 0, "")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if !s.SentSSLWarning {
|
||||||
|
t.Error("expected SentSSLWarning=true even in maintenance")
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) != 0 {
|
||||||
|
t.Error("expected no alert during maintenance")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleStatusChange_InactiveEngine(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
e.SetActive(false)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 0, 0, "test error")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Error("expected no state change when inactive")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Group 10: liveState merge (lost-update race) ---
|
||||||
|
|
||||||
|
// A pause that lands while a check is in flight must survive the check's
|
||||||
|
// write-back. The old code snapshotted the site, ran the check, then wrote the
|
||||||
|
// whole stale struct back — reverting the pause.
|
||||||
|
func TestHandleStatusChange_PauseDuringCheckSurvives(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
// `site` is the stale snapshot the check ran against (Paused=false).
|
||||||
|
// Meanwhile the user pauses the monitor.
|
||||||
|
e.ToggleSitePause(1)
|
||||||
|
|
||||||
|
// Check completes and folds its result in using the stale snapshot.
|
||||||
|
e.handleStatusChange(site, "DOWN", 500, 0, "boom")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if !s.Paused {
|
||||||
|
t.Error("pause was reverted by a stale check write-back")
|
||||||
|
}
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected check result still applied (DOWN), got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A config edit that lands while a check is in flight must survive; the check
|
||||||
|
// must not resurrect the old config from its snapshot.
|
||||||
|
func TestHandleStatusChange_ConfigEditDuringCheckSurvives(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com", Type: "http", MaxRetries: 0, Interval: 30},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
// Config changes mid-check.
|
||||||
|
e.UpdateSiteConfig(models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Type: "http", Interval: 60})
|
||||||
|
|
||||||
|
// Stale check (ran against http://old.com) folds its result in.
|
||||||
|
e.handleStatusChange(site, "UP", 200, 5*time.Millisecond, "")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.URL != "http://new.com" {
|
||||||
|
t.Errorf("config edit reverted: URL=%s", s.URL)
|
||||||
|
}
|
||||||
|
if s.Interval != 60 {
|
||||||
|
t.Errorf("config edit reverted: Interval=%d", s.Interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The classic push false-DOWN: a heartbeat marks the monitor UP while a
|
||||||
|
// staleness evaluation (computed from the older LastCheck) is mid-flight.
|
||||||
|
// The stale DOWN must not overwrite the fresh heartbeat.
|
||||||
|
func TestHandleStatusChange_HeartbeatNotOverwrittenByStaleDown(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
// Snapshot the engine would have taken before evaluating staleness:
|
||||||
|
// LastCheck is old, so checkPush decided "DOWN".
|
||||||
|
snap := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Token: "tok", Interval: 10},
|
||||||
|
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
|
||||||
|
}
|
||||||
|
injectSite(e, snap)
|
||||||
|
|
||||||
|
// A heartbeat lands first, advancing LastCheck and confirming UP.
|
||||||
|
if !e.RecordHeartbeat("tok") {
|
||||||
|
t.Fatal("heartbeat rejected")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now the in-flight stale evaluation tries to write DOWN.
|
||||||
|
e.handleStatusChange(snap, "DOWN", 0, 0, "heartbeat missed")
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("stale DOWN overwrote a fresh heartbeat: status=%s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A check result for a site removed mid-check must be dropped, not recreate it.
|
||||||
|
func TestHandleStatusChange_RemovedSiteDropped(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", MaxRetries: 0},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.RemoveSite(1)
|
||||||
|
e.handleStatusChange(site, "DOWN", 500, 0, "boom")
|
||||||
|
|
||||||
|
if _, ok := getSite(e, 1); ok {
|
||||||
|
t.Error("removed site was recreated by a late check write-back")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Group 12: Phase 3 engine correctness ---
|
||||||
|
|
||||||
|
// PENDING→DOWN must honor MaxRetries instead of alerting on first failure.
|
||||||
|
func TestHandleStatusChange_PendingRetriesBeforeDown(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "new-monitor", MaxRetries: 2},
|
||||||
|
SiteState: models.SiteState{Status: "PENDING"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 0, 0, "timeout")
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "PENDING" {
|
||||||
|
t.Errorf("expected PENDING during retry, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.FailureCount != 1 {
|
||||||
|
t.Errorf("expected FailureCount 1, got %d", s.FailureCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
|
||||||
|
s, _ = getSite(e, 1)
|
||||||
|
if s.Status != "PENDING" {
|
||||||
|
t.Errorf("expected PENDING during retry 2, got %s", s.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
e.handleStatusChange(s, "DOWN", 0, 0, "timeout")
|
||||||
|
s, _ = getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LATE→DOWN must also honor MaxRetries.
|
||||||
|
func TestHandleStatusChange_LateRetriesBeforeDown(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push-mon", MaxRetries: 1},
|
||||||
|
SiteState: models.SiteState{Status: "LATE"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.handleStatusChange(site, "DOWN", 0, 0, "missed heartbeat")
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "LATE" {
|
||||||
|
t.Errorf("expected LATE during retry, got %s", s.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
e.handleStatusChange(s, "DOWN", 0, 0, "missed heartbeat")
|
||||||
|
s, _ = getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected DOWN after retries exhausted, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,315 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"math/rand/v2"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (e *Engine) RecordHeartbeat(token string) bool {
|
||||||
|
if !e.IsActive() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
e.mu.RLock()
|
||||||
|
targetID, ok := e.tokenIndex[token]
|
||||||
|
e.mu.RUnlock()
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
prevStatus models.Status
|
||||||
|
name string
|
||||||
|
alertID int
|
||||||
|
downSince time.Time
|
||||||
|
)
|
||||||
|
_, exists := e.applyState(targetID, func(s *models.Site) {
|
||||||
|
prevStatus = s.Status
|
||||||
|
name = s.Name
|
||||||
|
alertID = s.AlertID
|
||||||
|
downSince = s.StatusChangedAt // captured before mutation = when it went down
|
||||||
|
|
||||||
|
s.LastCheck = time.Now()
|
||||||
|
s.Status = models.StatusUp
|
||||||
|
s.FailureCount = 0
|
||||||
|
s.Latency = 0
|
||||||
|
s.LastError = ""
|
||||||
|
s.LastSuccessAt = time.Now()
|
||||||
|
if prevStatus != models.StatusUp {
|
||||||
|
s.StatusChangedAt = time.Now()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if !exists {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
switch prevStatus {
|
||||||
|
case models.StatusPending:
|
||||||
|
e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
|
||||||
|
case models.StatusLate:
|
||||||
|
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
|
||||||
|
case models.StatusStale:
|
||||||
|
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
|
||||||
|
case models.StatusDown:
|
||||||
|
downDur := ""
|
||||||
|
if !downSince.IsZero() {
|
||||||
|
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
||||||
|
}
|
||||||
|
e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
|
||||||
|
go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
|
||||||
|
}
|
||||||
|
|
||||||
|
e.recordCheck(targetID, 0, true)
|
||||||
|
|
||||||
|
if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
|
||||||
|
e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) getRecheckChan(id int) chan struct{} {
|
||||||
|
e.recheckMu.Lock()
|
||||||
|
defer e.recheckMu.Unlock()
|
||||||
|
ch, ok := e.recheck[id]
|
||||||
|
if !ok {
|
||||||
|
ch = make(chan struct{}, 1)
|
||||||
|
e.recheck[id] = ch
|
||||||
|
}
|
||||||
|
return ch
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) signalRecheck(id int) {
|
||||||
|
ch := e.getRecheckChan(id)
|
||||||
|
select {
|
||||||
|
case ch <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) monitorRoutine(ctx context.Context, id int) {
|
||||||
|
recheckCh := e.getRecheckChan(id)
|
||||||
|
|
||||||
|
// Stagger initial check to avoid thundering herd on startup
|
||||||
|
stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
|
||||||
|
select {
|
||||||
|
case <-time.After(stagger):
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
e.checkByID(ctx, id)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if !e.IsActive() {
|
||||||
|
select {
|
||||||
|
case <-time.After(pollInterval):
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-recheckCh:
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
e.mu.RLock()
|
||||||
|
site, exists := e.liveState[id]
|
||||||
|
e.mu.RUnlock()
|
||||||
|
if !exists {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if site.Paused {
|
||||||
|
select {
|
||||||
|
case <-time.After(pollInterval):
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-recheckCh:
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
interval := site.Interval
|
||||||
|
if interval < minCheckInterval {
|
||||||
|
interval = minCheckInterval
|
||||||
|
}
|
||||||
|
jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
|
||||||
|
select {
|
||||||
|
case <-time.After(time.Duration(interval)*time.Second + jitter):
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-recheckCh:
|
||||||
|
}
|
||||||
|
e.checkByID(ctx, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) checkByID(ctx context.Context, id int) {
|
||||||
|
if !e.IsActive() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
e.mu.RLock()
|
||||||
|
site, exists := e.liveState[id]
|
||||||
|
e.mu.RUnlock()
|
||||||
|
if !exists || site.Paused {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
switch site.Type {
|
||||||
|
case "push":
|
||||||
|
e.checkPush(ctx, site)
|
||||||
|
case "group":
|
||||||
|
e.checkGroup(ctx, site)
|
||||||
|
default:
|
||||||
|
result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
|
||||||
|
updatedSite := site
|
||||||
|
updatedSite.HasSSL = result.HasSSL
|
||||||
|
updatedSite.CertExpiry = result.CertExpiry
|
||||||
|
updatedSite.Latency = time.Duration(result.LatencyNs)
|
||||||
|
updatedSite.LastCheck = time.Now()
|
||||||
|
e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) checkPush(_ context.Context, site models.Site) {
|
||||||
|
if site.Status == models.StatusPending {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
interval := time.Duration(site.Interval) * time.Second
|
||||||
|
grace := interval / 2
|
||||||
|
if grace < minPushGrace {
|
||||||
|
grace = minPushGrace
|
||||||
|
}
|
||||||
|
|
||||||
|
overdue := site.LastCheck.Add(interval)
|
||||||
|
staleMark := overdue.Add(grace / 2)
|
||||||
|
graceEnd := overdue.Add(grace)
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
if now.After(graceEnd) {
|
||||||
|
if site.Status != models.StatusDown {
|
||||||
|
e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
|
||||||
|
}
|
||||||
|
} else if now.After(staleMark) {
|
||||||
|
if site.Status != models.StatusStale {
|
||||||
|
e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
|
||||||
|
}
|
||||||
|
} else if now.After(overdue) {
|
||||||
|
if site.Status != models.StatusLate {
|
||||||
|
e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) checkGroup(_ context.Context, site models.Site) {
|
||||||
|
e.mu.RLock()
|
||||||
|
status := models.StatusUp
|
||||||
|
hasChildren := false
|
||||||
|
for _, child := range e.liveState {
|
||||||
|
if child.ParentID != site.ID || child.Type == "group" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hasChildren = true
|
||||||
|
if child.Paused || e.isInMaintenance(child.ID) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
|
||||||
|
status = models.StatusDown
|
||||||
|
} else if child.Status == models.StatusStale && status != models.StatusDown {
|
||||||
|
status = models.StatusStale
|
||||||
|
} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
|
||||||
|
status = models.StatusLate
|
||||||
|
} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
|
||||||
|
status = models.StatusPending
|
||||||
|
}
|
||||||
|
}
|
||||||
|
e.mu.RUnlock()
|
||||||
|
|
||||||
|
if !hasChildren {
|
||||||
|
status = models.StatusPending
|
||||||
|
}
|
||||||
|
|
||||||
|
e.applyState(site.ID, func(s *models.Site) {
|
||||||
|
s.Status = status
|
||||||
|
})
|
||||||
|
e.recordCheck(site.ID, 0, !status.IsBroken())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
|
||||||
|
e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetAggStrategy must be called before Start: the field is read by the probe
|
||||||
|
// aggregation path without synchronization.
|
||||||
|
func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
|
||||||
|
e.aggStrategy = strategy
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
|
||||||
|
e.mu.RLock()
|
||||||
|
site, exists := e.liveState[siteID]
|
||||||
|
e.mu.RUnlock()
|
||||||
|
if !exists {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
staleAfter := time.Duration(site.Interval) * time.Second * 3
|
||||||
|
if staleAfter < time.Minute {
|
||||||
|
staleAfter = time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
e.probeResultsMu.Lock()
|
||||||
|
if e.probeResults[siteID] == nil {
|
||||||
|
e.probeResults[siteID] = make(map[string]NodeResult)
|
||||||
|
}
|
||||||
|
e.probeResults[siteID][nodeID] = NodeResult{
|
||||||
|
NodeID: nodeID,
|
||||||
|
IsUp: isUp,
|
||||||
|
LatencyNs: latencyNs,
|
||||||
|
CheckedAt: now,
|
||||||
|
ErrorReason: errorReason,
|
||||||
|
}
|
||||||
|
results := make([]NodeResult, 0, len(e.probeResults[siteID]))
|
||||||
|
for id, r := range e.probeResults[siteID] {
|
||||||
|
if now.Sub(r.CheckedAt) > staleAfter {
|
||||||
|
delete(e.probeResults[siteID], id)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
results = append(results, r)
|
||||||
|
}
|
||||||
|
e.probeResultsMu.Unlock()
|
||||||
|
|
||||||
|
aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
|
||||||
|
|
||||||
|
probeStatus := models.StatusUp
|
||||||
|
if !aggUp {
|
||||||
|
probeStatus = models.StatusDown
|
||||||
|
}
|
||||||
|
|
||||||
|
updatedSite := site
|
||||||
|
updatedSite.Latency = time.Duration(avgLatency)
|
||||||
|
updatedSite.LastCheck = time.Now()
|
||||||
|
e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
|
||||||
|
e.probeResultsMu.RLock()
|
||||||
|
defer e.probeResultsMu.RUnlock()
|
||||||
|
src := e.probeResults[siteID]
|
||||||
|
cp := make(map[string]NodeResult, len(src))
|
||||||
|
for k, v := range src {
|
||||||
|
cp[k] = v
|
||||||
|
}
|
||||||
|
return cp
|
||||||
|
}
|
||||||
@@ -0,0 +1,389 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// --- Group 2: Heartbeat ---
|
||||||
|
|
||||||
|
func TestRecordHeartbeat_ValidToken(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
if !e.RecordHeartbeat("abc123") {
|
||||||
|
t.Error("expected true for valid token")
|
||||||
|
}
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if time.Since(s.LastCheck) > time.Second {
|
||||||
|
t.Error("expected LastCheck to be recent")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecordHeartbeat_RecoveryFromDown(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.alerts[1] = models.AlertConfig{ID: 1, Name: "test", Type: "webhook", Settings: map[string]string{"url": "http://example.com"}}
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push-test", Type: "push", Token: "abc123", AlertID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
if !e.RecordHeartbeat("abc123") {
|
||||||
|
t.Error("expected true")
|
||||||
|
}
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.FailureCount != 0 {
|
||||||
|
t.Errorf("expected FailureCount 0, got %d", s.FailureCount)
|
||||||
|
}
|
||||||
|
waitAsync()
|
||||||
|
if len(ms.getAlertCallsSnapshot()) == 0 {
|
||||||
|
t.Error("expected recovery alert")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecordHeartbeat_UnknownToken(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
|
||||||
|
if e.RecordHeartbeat("unknown") {
|
||||||
|
t.Error("expected false for unknown token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecordHeartbeat_InactiveEngine(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Type: "push", Token: "abc123"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
e.SetActive(false)
|
||||||
|
|
||||||
|
if e.RecordHeartbeat("abc123") {
|
||||||
|
t.Error("expected false when inactive")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Group 3: Push Deadline ---
|
||||||
|
|
||||||
|
func TestCheckPush_DeadlineMissed(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 10, MaxRetries: 0},
|
||||||
|
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-120 * time.Second)},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.checkPush(context.Background(), site)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected DOWN after missed deadline, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckPush_OverdueBecomesLate(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
|
||||||
|
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-310 * time.Second)},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.checkPush(context.Background(), site)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "LATE" {
|
||||||
|
t.Errorf("expected LATE when overdue but within grace, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckPush_OverdueBecomesStale(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
// interval=300, grace=150 (300/2), staleMark=overdue+75
|
||||||
|
// at 380s: past staleMark(375) but before graceEnd(450)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 300},
|
||||||
|
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now().Add(-380 * time.Second)},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.checkPush(context.Background(), site)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "STALE" {
|
||||||
|
t.Errorf("expected STALE when past midpoint of grace, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckPush_WithinDeadline(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
|
||||||
|
SiteState: models.SiteState{Status: "UP", LastCheck: time.Now()},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.checkPush(context.Background(), site)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckPush_PendingStaysPending(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "push", Type: "push", Interval: 60},
|
||||||
|
SiteState: models.SiteState{Status: "PENDING"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.checkPush(context.Background(), site)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "PENDING" {
|
||||||
|
t.Errorf("expected PENDING to stay until first heartbeat, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Group 4: Group Checks ---
|
||||||
|
|
||||||
|
func TestCheckGroup_AllChildrenUp(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: "PENDING"},
|
||||||
|
}
|
||||||
|
child1 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child2 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child1)
|
||||||
|
injectSite(e, child2)
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected group UP, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGroup_OneChildDown(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child1 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child2 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN"},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child1)
|
||||||
|
injectSite(e, child2)
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected group DOWN, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGroup_PausedChildIgnored(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||||
|
}
|
||||||
|
child1 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child2 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN"},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child1)
|
||||||
|
injectSite(e, child2)
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP (paused child ignored), got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGroup_MaintenanceChildIgnored(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.maintenance[3] = true
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||||
|
}
|
||||||
|
child1 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child2 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN"},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child1)
|
||||||
|
injectSite(e, child2)
|
||||||
|
e.refreshMaintenanceCache(context.Background())
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "UP" {
|
||||||
|
t.Errorf("expected UP (maint child ignored), got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGroup_NoChildren(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Status != "PENDING" {
|
||||||
|
t.Errorf("expected PENDING for no children, got %s", s.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Groups must not auto-pause when all children are paused — that creates a
|
||||||
|
// one-way trap because monitorRoutine skips paused sites.
|
||||||
|
func TestCheckGroup_AllPausedNoAutoFreeze(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "group", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child1 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 2, Name: "child1", Type: "http", ParentID: 1, Paused: true},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child2 := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 3, Name: "child2", Type: "http", ParentID: 1, Paused: true},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child1)
|
||||||
|
injectSite(e, child2)
|
||||||
|
|
||||||
|
e.checkGroup(context.Background(), group)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.Paused {
|
||||||
|
t.Error("group must not auto-pause when all children are paused")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dead probe results must be expired so they don't poison aggregation.
|
||||||
|
func TestIngestProbeResult_ExpiresStaleProbes(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http", Interval: 30},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.probeResultsMu.Lock()
|
||||||
|
e.probeResults[1] = map[string]NodeResult{
|
||||||
|
"dead-probe": {
|
||||||
|
NodeID: "dead-probe",
|
||||||
|
IsUp: false,
|
||||||
|
CheckedAt: time.Now().Add(-10 * time.Minute),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
e.probeResultsMu.Unlock()
|
||||||
|
|
||||||
|
e.IngestProbeResult("live-probe", 1, 5000, true, "")
|
||||||
|
|
||||||
|
e.probeResultsMu.RLock()
|
||||||
|
_, deadExists := e.probeResults[1]["dead-probe"]
|
||||||
|
_, liveExists := e.probeResults[1]["live-probe"]
|
||||||
|
e.probeResultsMu.RUnlock()
|
||||||
|
|
||||||
|
if deadExists {
|
||||||
|
t.Error("stale probe result should have been expired")
|
||||||
|
}
|
||||||
|
if !liveExists {
|
||||||
|
t.Error("live probe result should still exist")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoveSite must clean up probeResults.
|
||||||
|
func TestRemoveSite_CleansProbeResults(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
e.probeResultsMu.Lock()
|
||||||
|
e.probeResults[1] = map[string]NodeResult{
|
||||||
|
"node-a": {NodeID: "node-a", IsUp: true, CheckedAt: time.Now()},
|
||||||
|
}
|
||||||
|
e.probeResultsMu.Unlock()
|
||||||
|
|
||||||
|
e.RemoveSite(1)
|
||||||
|
|
||||||
|
e.probeResultsMu.RLock()
|
||||||
|
defer e.probeResultsMu.RUnlock()
|
||||||
|
if _, exists := e.probeResults[1]; exists {
|
||||||
|
t.Error("probe results should be cleaned up after RemoveSite")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (e *Engine) maintenancePruner(ctx context.Context) {
|
||||||
|
ticker := time.NewTicker(maintPruneInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
e.pruneMaintenanceWindows(ctx)
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
e.pruneMaintenanceWindows(ctx)
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
|
||||||
|
pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
|
||||||
|
if err != nil {
|
||||||
|
e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if pruned > 0 {
|
||||||
|
e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) isInMaintenance(monitorID int) bool {
|
||||||
|
e.maintCacheMu.RLock()
|
||||||
|
defer e.maintCacheMu.RUnlock()
|
||||||
|
return e.maintCache[monitorID]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
|
||||||
|
windows, err := e.db.GetActiveMaintenanceWindows(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
directMaint := make(map[int]bool)
|
||||||
|
var globalMaint bool
|
||||||
|
for _, w := range windows {
|
||||||
|
if w.MonitorID == 0 {
|
||||||
|
globalMaint = true
|
||||||
|
} else {
|
||||||
|
directMaint[w.MonitorID] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved := make(map[int]bool)
|
||||||
|
e.mu.RLock()
|
||||||
|
for id, site := range e.liveState {
|
||||||
|
if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
|
||||||
|
resolved[id] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
e.mu.RUnlock()
|
||||||
|
|
||||||
|
e.maintCacheMu.Lock()
|
||||||
|
e.maintCache = resolved
|
||||||
|
e.maintCacheMu.Unlock()
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Maintenance cache resolves parent relationships correctly.
|
||||||
|
func TestIsInMaintenance_UsesCache(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.maintenance[10] = true // direct maintenance on group
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
group := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 10, Name: "group", Type: "group"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
child := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 20, Name: "child", Type: "http", ParentID: 10},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, group)
|
||||||
|
injectSite(e, child)
|
||||||
|
e.refreshMaintenanceCache(context.Background())
|
||||||
|
|
||||||
|
if !e.isInMaintenance(10) {
|
||||||
|
t.Error("group should be in maintenance (direct)")
|
||||||
|
}
|
||||||
|
if !e.isInMaintenance(20) {
|
||||||
|
t.Error("child should be in maintenance (parent)")
|
||||||
|
}
|
||||||
|
if e.isInMaintenance(99) {
|
||||||
|
t.Error("unknown monitor should not be in maintenance")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Global maintenance (monitor_id=0) applies to all monitors.
|
||||||
|
func TestIsInMaintenance_GlobalMaintenance(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
ms.maintenance[0] = true
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "http"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
e.refreshMaintenanceCache(context.Background())
|
||||||
|
|
||||||
|
if !e.isInMaintenance(1) {
|
||||||
|
t.Error("all monitors should be in maintenance during global window")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,14 +4,12 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math/rand/v2"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/alert"
|
|
||||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/store"
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/store"
|
||||||
)
|
)
|
||||||
@@ -28,14 +26,6 @@ const (
|
|||||||
dbPruneInterval = 10 * time.Minute
|
dbPruneInterval = 10 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
type AlertHealth struct {
|
|
||||||
LastSendAt time.Time
|
|
||||||
LastSendOK bool
|
|
||||||
LastError string
|
|
||||||
SendCount int
|
|
||||||
FailCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
liveState map[int]models.Site
|
liveState map[int]models.Site
|
||||||
@@ -261,26 +251,6 @@ func (e *Engine) InitLogs() {
|
|||||||
e.logStore = entries
|
e.logStore = entries
|
||||||
}
|
}
|
||||||
|
|
||||||
// InitAlertHealth restores persisted alert send health so the dashboard shows real
|
|
||||||
// "last sent" / health state on startup instead of resetting every channel to "never".
|
|
||||||
func (e *Engine) InitAlertHealth() {
|
|
||||||
records, err := e.db.LoadAlertHealth(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
e.alertHealthMu.Lock()
|
|
||||||
defer e.alertHealthMu.Unlock()
|
|
||||||
for id, r := range records {
|
|
||||||
e.alertHealth[id] = AlertHealth{
|
|
||||||
LastSendAt: r.LastSendAt,
|
|
||||||
LastSendOK: r.LastSendOK,
|
|
||||||
LastError: r.LastError,
|
|
||||||
SendCount: r.SendCount,
|
|
||||||
FailCount: r.FailCount,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetLogs() []models.LogEntry {
|
func (e *Engine) GetLogs() []models.LogEntry {
|
||||||
e.logMu.RLock()
|
e.logMu.RLock()
|
||||||
defer e.logMu.RUnlock()
|
defer e.logMu.RUnlock()
|
||||||
@@ -289,123 +259,6 @@ func (e *Engine) GetLogs() []models.LogEntry {
|
|||||||
return logs
|
return logs
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) SetActive(active bool) {
|
|
||||||
e.activeMu.Lock()
|
|
||||||
defer e.activeMu.Unlock()
|
|
||||||
if e.isActive != active {
|
|
||||||
e.isActive = active
|
|
||||||
status := "RESUMED (Active)"
|
|
||||||
if !active {
|
|
||||||
status = "PAUSED (Passive)"
|
|
||||||
}
|
|
||||||
e.AddLog(fmt.Sprintf("Engine %s", status))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) IsActive() bool {
|
|
||||||
e.activeMu.RLock()
|
|
||||||
defer e.activeMu.RUnlock()
|
|
||||||
return e.isActive
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetAllSites() []models.Site {
|
|
||||||
e.mu.RLock()
|
|
||||||
defer e.mu.RUnlock()
|
|
||||||
sites := make([]models.Site, 0, len(e.liveState))
|
|
||||||
for _, s := range e.liveState {
|
|
||||||
sites = append(sites, s)
|
|
||||||
}
|
|
||||||
return sites
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetLiveState() map[int]models.Site {
|
|
||||||
e.mu.RLock()
|
|
||||||
defer e.mu.RUnlock()
|
|
||||||
cp := make(map[int]models.Site, len(e.liveState))
|
|
||||||
for k, v := range e.liveState {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
return cp
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) RecordHeartbeat(token string) bool {
|
|
||||||
if !e.IsActive() {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
e.mu.RLock()
|
|
||||||
targetID, ok := e.tokenIndex[token]
|
|
||||||
e.mu.RUnlock()
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
prevStatus models.Status
|
|
||||||
name string
|
|
||||||
alertID int
|
|
||||||
downSince time.Time
|
|
||||||
)
|
|
||||||
_, exists := e.applyState(targetID, func(s *models.Site) {
|
|
||||||
prevStatus = s.Status
|
|
||||||
name = s.Name
|
|
||||||
alertID = s.AlertID
|
|
||||||
downSince = s.StatusChangedAt // captured before mutation = when it went down
|
|
||||||
|
|
||||||
s.LastCheck = time.Now()
|
|
||||||
s.Status = models.StatusUp
|
|
||||||
s.FailureCount = 0
|
|
||||||
s.Latency = 0
|
|
||||||
s.LastError = ""
|
|
||||||
s.LastSuccessAt = time.Now()
|
|
||||||
if prevStatus != models.StatusUp {
|
|
||||||
s.StatusChangedAt = time.Now()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if !exists {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
switch prevStatus {
|
|
||||||
case models.StatusPending:
|
|
||||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", name))
|
|
||||||
case models.StatusLate:
|
|
||||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", name))
|
|
||||||
case models.StatusStale:
|
|
||||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was stale)", name))
|
|
||||||
case models.StatusDown:
|
|
||||||
downDur := ""
|
|
||||||
if !downSince.IsZero() {
|
|
||||||
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
|
||||||
}
|
|
||||||
e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered%s", name, downDur))
|
|
||||||
go e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.%s", name, downDur))
|
|
||||||
}
|
|
||||||
|
|
||||||
e.recordCheck(targetID, 0, true)
|
|
||||||
|
|
||||||
if prevStatus != models.StatusUp && prevStatus != models.StatusPending {
|
|
||||||
e.enqueueWrite(writeStateChange{siteID: targetID, fromStatus: string(prevStatus), toStatus: string(models.StatusUp)})
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) addToTokenIndex(site models.Site) {
|
|
||||||
if site.Type == "push" && site.Token != "" {
|
|
||||||
e.tokenIndex[site.Token] = site.ID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) removeFromTokenIndex(id int) {
|
|
||||||
for token, sid := range e.tokenIndex {
|
|
||||||
if sid == id {
|
|
||||||
delete(e.tokenIndex, token)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) Start(ctx context.Context) {
|
func (e *Engine) Start(ctx context.Context) {
|
||||||
// e.cancel is invoked by Stop() to drain and halt the writer; gosec can't
|
// e.cancel is invoked by Stop() to drain and halt the writer; gosec can't
|
||||||
// trace the cross-method call, and cancelling the parent reaps this child
|
// trace the cross-method call, and cancelling the parent reaps this child
|
||||||
@@ -496,637 +349,3 @@ func (e *Engine) Start(ctx context.Context) {
|
|||||||
e.maintenancePruner(ctx)
|
e.maintenancePruner(ctx)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) maintenancePruner(ctx context.Context) {
|
|
||||||
ticker := time.NewTicker(maintPruneInterval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
e.pruneMaintenanceWindows(ctx)
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ticker.C:
|
|
||||||
e.pruneMaintenanceWindows(ctx)
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) pruneMaintenanceWindows(ctx context.Context) {
|
|
||||||
pruned, err := e.db.PruneExpiredMaintenanceWindows(ctx, e.maintRetention)
|
|
||||||
if err != nil {
|
|
||||||
e.AddLog(fmt.Sprintf("Maintenance prune error: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if pruned > 0 {
|
|
||||||
e.AddLog(fmt.Sprintf("Pruned %d expired maintenance window(s)", pruned))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
|
|
||||||
e.mu.Lock()
|
|
||||||
if existing, ok := e.liveState[cfg.ID]; ok {
|
|
||||||
e.removeFromTokenIndex(cfg.ID)
|
|
||||||
existing.SiteConfig = cfg
|
|
||||||
e.liveState[cfg.ID] = existing
|
|
||||||
e.addToTokenIndex(existing)
|
|
||||||
}
|
|
||||||
e.mu.Unlock()
|
|
||||||
|
|
||||||
e.signalRecheck(cfg.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) getRecheckChan(id int) chan struct{} {
|
|
||||||
e.recheckMu.Lock()
|
|
||||||
defer e.recheckMu.Unlock()
|
|
||||||
ch, ok := e.recheck[id]
|
|
||||||
if !ok {
|
|
||||||
ch = make(chan struct{}, 1)
|
|
||||||
e.recheck[id] = ch
|
|
||||||
}
|
|
||||||
return ch
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) signalRecheck(id int) {
|
|
||||||
ch := e.getRecheckChan(id)
|
|
||||||
select {
|
|
||||||
case ch <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) RemoveSite(id int) {
|
|
||||||
e.mu.Lock()
|
|
||||||
e.removeFromTokenIndex(id)
|
|
||||||
delete(e.liveState, id)
|
|
||||||
e.mu.Unlock()
|
|
||||||
e.removeHistory(id)
|
|
||||||
|
|
||||||
e.probeResultsMu.Lock()
|
|
||||||
delete(e.probeResults, id)
|
|
||||||
e.probeResultsMu.Unlock()
|
|
||||||
|
|
||||||
e.recheckMu.Lock()
|
|
||||||
delete(e.recheck, id)
|
|
||||||
e.recheckMu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) ToggleSitePause(id int) bool {
|
|
||||||
var (
|
|
||||||
paused bool
|
|
||||||
name string
|
|
||||||
)
|
|
||||||
_, ok := e.applyState(id, func(s *models.Site) {
|
|
||||||
s.Paused = !s.Paused
|
|
||||||
paused = s.Paused
|
|
||||||
name = s.Name
|
|
||||||
})
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if paused {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
|
|
||||||
} else {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
|
|
||||||
}
|
|
||||||
return paused
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) monitorRoutine(ctx context.Context, id int) {
|
|
||||||
recheckCh := e.getRecheckChan(id)
|
|
||||||
|
|
||||||
// Stagger initial check to avoid thundering herd on startup
|
|
||||||
stagger := time.Duration(rand.IntN(3000)) * time.Millisecond //nolint:gosec // non-security jitter
|
|
||||||
select {
|
|
||||||
case <-time.After(stagger):
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
e.checkByID(ctx, id)
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
if !e.IsActive() {
|
|
||||||
select {
|
|
||||||
case <-time.After(pollInterval):
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-recheckCh:
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
e.mu.RLock()
|
|
||||||
site, exists := e.liveState[id]
|
|
||||||
e.mu.RUnlock()
|
|
||||||
if !exists {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if site.Paused {
|
|
||||||
select {
|
|
||||||
case <-time.After(pollInterval):
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-recheckCh:
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
interval := site.Interval
|
|
||||||
if interval < minCheckInterval {
|
|
||||||
interval = minCheckInterval
|
|
||||||
}
|
|
||||||
jitter := time.Duration(rand.IntN(interval*100)) * time.Millisecond //nolint:gosec // non-security jitter
|
|
||||||
select {
|
|
||||||
case <-time.After(time.Duration(interval)*time.Second + jitter):
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-recheckCh:
|
|
||||||
}
|
|
||||||
e.checkByID(ctx, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyState atomically reads, mutates, and writes back the live entry for id.
|
|
||||||
// The mutator runs under the engine write lock and receives a pointer to the
|
|
||||||
// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
|
|
||||||
// never clobbered by a stale snapshot. The mutator must only touch runtime /
|
|
||||||
// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
|
|
||||||
// are owned by UpdateSiteConfig and must not be written here. Returns the
|
|
||||||
// post-mutation copy and whether the site still exists.
|
|
||||||
func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
|
|
||||||
e.mu.Lock()
|
|
||||||
defer e.mu.Unlock()
|
|
||||||
cur, ok := e.liveState[id]
|
|
||||||
if !ok {
|
|
||||||
return models.Site{}, false
|
|
||||||
}
|
|
||||||
mutate(&cur)
|
|
||||||
e.liveState[id] = cur
|
|
||||||
return cur, true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) checkByID(ctx context.Context, id int) {
|
|
||||||
if !e.IsActive() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
e.mu.RLock()
|
|
||||||
site, exists := e.liveState[id]
|
|
||||||
e.mu.RUnlock()
|
|
||||||
if !exists || site.Paused {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
switch site.Type {
|
|
||||||
case "push":
|
|
||||||
e.checkPush(ctx, site)
|
|
||||||
case "group":
|
|
||||||
e.checkGroup(ctx, site)
|
|
||||||
default:
|
|
||||||
result := RunCheck(ctx, site.SiteConfig, e.strictClient, e.insecureClient, e.insecureSkipVerify, e.allowPrivateTargets)
|
|
||||||
updatedSite := site
|
|
||||||
updatedSite.HasSSL = result.HasSSL
|
|
||||||
updatedSite.CertExpiry = result.CertExpiry
|
|
||||||
updatedSite.Latency = time.Duration(result.LatencyNs)
|
|
||||||
updatedSite.LastCheck = time.Now()
|
|
||||||
e.handleStatusChange(updatedSite, result.Status, result.StatusCode, time.Duration(result.LatencyNs), result.ErrorReason)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) checkPush(_ context.Context, site models.Site) {
|
|
||||||
if site.Status == models.StatusPending {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
interval := time.Duration(site.Interval) * time.Second
|
|
||||||
grace := interval / 2
|
|
||||||
if grace < minPushGrace {
|
|
||||||
grace = minPushGrace
|
|
||||||
}
|
|
||||||
|
|
||||||
overdue := site.LastCheck.Add(interval)
|
|
||||||
staleMark := overdue.Add(grace / 2)
|
|
||||||
graceEnd := overdue.Add(grace)
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
if now.After(graceEnd) {
|
|
||||||
if site.Status != models.StatusDown {
|
|
||||||
e.handleStatusChange(site, string(models.StatusDown), 0, 0, "heartbeat missed")
|
|
||||||
}
|
|
||||||
} else if now.After(staleMark) {
|
|
||||||
if site.Status != models.StatusStale {
|
|
||||||
e.handleStatusChange(site, string(models.StatusStale), 0, 0, "heartbeat stale")
|
|
||||||
}
|
|
||||||
} else if now.After(overdue) {
|
|
||||||
if site.Status != models.StatusLate {
|
|
||||||
e.handleStatusChange(site, string(models.StatusLate), 0, 0, "heartbeat overdue")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleStatusChange folds a check result into the live state. snap is the
|
|
||||||
// stale snapshot the check ran against; the actual mutation is applied onto the
|
|
||||||
// CURRENT live entry via applyState, so a concurrent pause / config edit /
|
|
||||||
// heartbeat is never reverted by this write. Logs and alerts are emitted after
|
|
||||||
// the lock is released, off the critical section.
|
|
||||||
func (e *Engine) handleStatusChange(snap models.Site, rawStatus string, code int, latency time.Duration, errorReason string) {
|
|
||||||
if !e.IsActive() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
inMaint := e.isInMaintenance(snap.ID)
|
|
||||||
status := models.Status(rawStatus)
|
|
||||||
|
|
||||||
var (
|
|
||||||
prev, next models.Status
|
|
||||||
name, typ string
|
|
||||||
alertID int
|
|
||||||
failCount, maxRetries int
|
|
||||||
confirmedDown bool
|
|
||||||
failedCheck bool
|
|
||||||
downSince time.Time
|
|
||||||
sslWarnFire bool
|
|
||||||
sslDays int
|
|
||||||
skipped bool
|
|
||||||
changed bool
|
|
||||||
)
|
|
||||||
|
|
||||||
_, exists := e.applyState(snap.ID, func(s *models.Site) {
|
|
||||||
// A non-UP result computed from a stale snapshot must not override a
|
|
||||||
// heartbeat (or newer check) that landed while we were evaluating.
|
|
||||||
if status != models.StatusUp && s.LastCheck.After(snap.LastCheck) {
|
|
||||||
skipped = true
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prev = s.Status
|
|
||||||
name = s.Name
|
|
||||||
typ = s.Type
|
|
||||||
alertID = s.AlertID
|
|
||||||
maxRetries = s.MaxRetries
|
|
||||||
downSince = s.StatusChangedAt
|
|
||||||
|
|
||||||
// Fresh check results (measured by the run against snap).
|
|
||||||
s.StatusCode = code
|
|
||||||
s.Latency = snap.Latency
|
|
||||||
s.LastCheck = snap.LastCheck
|
|
||||||
s.HasSSL = snap.HasSSL
|
|
||||||
s.CertExpiry = snap.CertExpiry
|
|
||||||
s.LastError = errorReason
|
|
||||||
if status == models.StatusUp {
|
|
||||||
s.LastSuccessAt = time.Now()
|
|
||||||
s.LastError = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Status + failure-count transition, based on the CURRENT live status.
|
|
||||||
if status == models.StatusUp {
|
|
||||||
s.FailureCount = 0
|
|
||||||
s.Status = models.StatusUp
|
|
||||||
} else {
|
|
||||||
if s.FailureCount <= s.MaxRetries {
|
|
||||||
s.FailureCount++
|
|
||||||
}
|
|
||||||
if s.FailureCount > s.MaxRetries {
|
|
||||||
if s.Status != status {
|
|
||||||
confirmedDown = true
|
|
||||||
}
|
|
||||||
s.Status = status
|
|
||||||
s.FailureCount = s.MaxRetries + 1
|
|
||||||
} else {
|
|
||||||
failedCheck = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
failCount = s.FailureCount
|
|
||||||
|
|
||||||
if s.Status != prev && prev != models.StatusPending {
|
|
||||||
s.StatusChangedAt = time.Now()
|
|
||||||
} else if s.StatusChangedAt.IsZero() && s.Status != models.StatusPending {
|
|
||||||
s.StatusChangedAt = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
// SSL expiry warning (fresh HasSSL/CertExpiry + config threshold).
|
|
||||||
if typ == "http" && s.CheckSSL && s.HasSSL {
|
|
||||||
days := int(time.Until(s.CertExpiry).Hours() / 24)
|
|
||||||
if days <= s.ExpiryThreshold && !s.SentSSLWarning && status != models.StatusSSLExp {
|
|
||||||
sslWarnFire = true
|
|
||||||
sslDays = days
|
|
||||||
s.SentSSLWarning = true
|
|
||||||
} else if days > s.ExpiryThreshold {
|
|
||||||
s.SentSSLWarning = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
next = s.Status
|
|
||||||
changed = next != prev
|
|
||||||
})
|
|
||||||
|
|
||||||
if !exists || skipped {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
e.recordCheck(snap.ID, latency, status == models.StatusUp)
|
|
||||||
|
|
||||||
if confirmedDown {
|
|
||||||
if errorReason != "" {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN: %s", name, errorReason))
|
|
||||||
} else {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' confirmed DOWN", name))
|
|
||||||
}
|
|
||||||
} else if failedCheck {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' failed check %d/%d", name, failCount, maxRetries))
|
|
||||||
}
|
|
||||||
|
|
||||||
if changed && prev != models.StatusPending {
|
|
||||||
e.enqueueWrite(writeStateChange{siteID: snap.ID, fromStatus: string(prev), toStatus: string(next), reason: errorReason})
|
|
||||||
}
|
|
||||||
|
|
||||||
if sslWarnFire {
|
|
||||||
if !inMaint {
|
|
||||||
e.triggerAlert(alertID, "SSL WARNING", fmt.Sprintf("SSL for '%s' expires in %d days", name, sslDays))
|
|
||||||
} else {
|
|
||||||
e.AddLog(fmt.Sprintf("SSL warning for '%s' suppressed (maintenance)", name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if prev == models.StatusUp && next == models.StatusLate {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat overdue", name))
|
|
||||||
}
|
|
||||||
|
|
||||||
if !prev.IsBroken() && next.IsBroken() && next != models.StatusPending {
|
|
||||||
if inMaint {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' is DOWN (alerts suppressed — maintenance)", name))
|
|
||||||
} else {
|
|
||||||
msg := fmt.Sprintf("Monitor '%s' is DOWN (%s)", name, rawStatus)
|
|
||||||
if errorReason != "" {
|
|
||||||
msg = fmt.Sprintf("Monitor '%s' is DOWN: %s", name, errorReason)
|
|
||||||
}
|
|
||||||
if typ == "push" {
|
|
||||||
msg = fmt.Sprintf("Push Monitor '%s' missed heartbeat.", name)
|
|
||||||
}
|
|
||||||
e.triggerAlert(alertID, "🚨 ALERT", msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if prev.IsBroken() && next == models.StatusUp {
|
|
||||||
downDur := ""
|
|
||||||
if !downSince.IsZero() {
|
|
||||||
downDur = fmt.Sprintf(" (was down %s)", fmtDurationShort(time.Since(downSince)))
|
|
||||||
}
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' recovered%s", name, downDur))
|
|
||||||
if !inMaint {
|
|
||||||
e.triggerAlert(alertID, "✅ RECOVERY", fmt.Sprintf("Monitor '%s' is UP%s", name, downDur))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if prev == models.StatusLate && next == models.StatusUp && !prev.IsBroken() {
|
|
||||||
e.AddLog(fmt.Sprintf("Monitor '%s' heartbeat arrived (was late)", name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) triggerAlert(alertID int, title, message string) {
|
|
||||||
if alertID <= 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
|
||||||
if err != nil {
|
|
||||||
e.AddLog(fmt.Sprintf("Failed to load alert config %d: %v", alertID, err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
provider := alert.GetProvider(cfg)
|
|
||||||
if provider != nil {
|
|
||||||
go func() {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
|
||||||
defer cancel()
|
|
||||||
if err := provider.Send(ctx, title, message); err != nil {
|
|
||||||
e.AddLog(fmt.Sprintf("Alert send failed (%s): %v", cfg.Name, err))
|
|
||||||
e.recordAlertResult(alertID, false, err.Error())
|
|
||||||
} else {
|
|
||||||
e.recordAlertResult(alertID, true, "")
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) recordAlertResult(alertID int, ok bool, errMsg string) {
|
|
||||||
e.alertHealthMu.Lock()
|
|
||||||
defer e.alertHealthMu.Unlock()
|
|
||||||
h := e.alertHealth[alertID]
|
|
||||||
h.LastSendAt = time.Now()
|
|
||||||
h.LastSendOK = ok
|
|
||||||
h.SendCount++
|
|
||||||
if ok {
|
|
||||||
h.LastError = ""
|
|
||||||
} else {
|
|
||||||
h.LastError = errMsg
|
|
||||||
h.FailCount++
|
|
||||||
}
|
|
||||||
e.alertHealth[alertID] = h
|
|
||||||
|
|
||||||
// Persist so health survives restarts; DB IO off the alert path.
|
|
||||||
e.enqueueWrite(writeAlertHealth{rec: models.AlertHealthRecord{
|
|
||||||
AlertID: alertID,
|
|
||||||
LastSendAt: h.LastSendAt,
|
|
||||||
LastSendOK: h.LastSendOK,
|
|
||||||
LastError: h.LastError,
|
|
||||||
SendCount: h.SendCount,
|
|
||||||
FailCount: h.FailCount,
|
|
||||||
}})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetAlertHealth(alertID int) AlertHealth {
|
|
||||||
e.alertHealthMu.RLock()
|
|
||||||
defer e.alertHealthMu.RUnlock()
|
|
||||||
return e.alertHealth[alertID]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) TestAlert(alertID int) error {
|
|
||||||
cfg, err := e.db.GetAlert(context.Background(), alertID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to load alert: %w", err)
|
|
||||||
}
|
|
||||||
provider := alert.GetProvider(cfg)
|
|
||||||
if provider == nil {
|
|
||||||
return fmt.Errorf("no provider for type %q", cfg.Type)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), alertSendTimeout)
|
|
||||||
defer cancel()
|
|
||||||
err = provider.Send(ctx, "🧪 Test Alert", fmt.Sprintf("Test notification from uptop for channel '%s'.", cfg.Name))
|
|
||||||
if err != nil {
|
|
||||||
e.recordAlertResult(alertID, false, err.Error())
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
e.recordAlertResult(alertID, true, "")
|
|
||||||
e.AddLog(fmt.Sprintf("Test alert sent to '%s'", cfg.Name))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) isInMaintenance(monitorID int) bool {
|
|
||||||
e.maintCacheMu.RLock()
|
|
||||||
defer e.maintCacheMu.RUnlock()
|
|
||||||
return e.maintCache[monitorID]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) refreshMaintenanceCache(ctx context.Context) {
|
|
||||||
windows, err := e.db.GetActiveMaintenanceWindows(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
directMaint := make(map[int]bool)
|
|
||||||
var globalMaint bool
|
|
||||||
for _, w := range windows {
|
|
||||||
if w.MonitorID == 0 {
|
|
||||||
globalMaint = true
|
|
||||||
} else {
|
|
||||||
directMaint[w.MonitorID] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved := make(map[int]bool)
|
|
||||||
e.mu.RLock()
|
|
||||||
for id, site := range e.liveState {
|
|
||||||
if globalMaint || directMaint[id] || (site.ParentID > 0 && directMaint[site.ParentID]) {
|
|
||||||
resolved[id] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
e.mu.RUnlock()
|
|
||||||
|
|
||||||
e.maintCacheMu.Lock()
|
|
||||||
e.maintCache = resolved
|
|
||||||
e.maintCacheMu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetDisplayStatus(site models.Site) string {
|
|
||||||
if site.Paused {
|
|
||||||
return "PAUSED"
|
|
||||||
}
|
|
||||||
if e.isInMaintenance(site.ID) {
|
|
||||||
return "MAINT"
|
|
||||||
}
|
|
||||||
return string(site.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) checkGroup(_ context.Context, site models.Site) {
|
|
||||||
e.mu.RLock()
|
|
||||||
status := models.StatusUp
|
|
||||||
hasChildren := false
|
|
||||||
for _, child := range e.liveState {
|
|
||||||
if child.ParentID != site.ID || child.Type == "group" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
hasChildren = true
|
|
||||||
if child.Paused || e.isInMaintenance(child.ID) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if child.Status == models.StatusDown || child.Status == models.StatusSSLExp {
|
|
||||||
status = models.StatusDown
|
|
||||||
} else if child.Status == models.StatusStale && status != models.StatusDown {
|
|
||||||
status = models.StatusStale
|
|
||||||
} else if child.Status == models.StatusLate && status != models.StatusDown && status != models.StatusStale {
|
|
||||||
status = models.StatusLate
|
|
||||||
} else if child.Status == models.StatusPending && status != models.StatusDown && status != models.StatusStale && status != models.StatusLate {
|
|
||||||
status = models.StatusPending
|
|
||||||
}
|
|
||||||
}
|
|
||||||
e.mu.RUnlock()
|
|
||||||
|
|
||||||
if !hasChildren {
|
|
||||||
status = models.StatusPending
|
|
||||||
}
|
|
||||||
|
|
||||||
e.applyState(site.ID, func(s *models.Site) {
|
|
||||||
s.Status = status
|
|
||||||
})
|
|
||||||
e.recordCheck(site.ID, 0, !status.IsBroken())
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) EnqueueProbeCheck(siteID int, nodeID string, latencyNs int64, isUp bool) {
|
|
||||||
e.enqueueWrite(writeProbeCheck{siteID: siteID, nodeID: nodeID, latencyNs: latencyNs, isUp: isUp})
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetAggStrategy must be called before Start: the field is read by the probe
|
|
||||||
// aggregation path without synchronization.
|
|
||||||
func (e *Engine) SetAggStrategy(strategy AggregationStrategy) {
|
|
||||||
e.aggStrategy = strategy
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) IngestProbeResult(nodeID string, siteID int, latencyNs int64, isUp bool, errorReason string) {
|
|
||||||
e.mu.RLock()
|
|
||||||
site, exists := e.liveState[siteID]
|
|
||||||
e.mu.RUnlock()
|
|
||||||
if !exists {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
staleAfter := time.Duration(site.Interval) * time.Second * 3
|
|
||||||
if staleAfter < time.Minute {
|
|
||||||
staleAfter = time.Minute
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
e.probeResultsMu.Lock()
|
|
||||||
if e.probeResults[siteID] == nil {
|
|
||||||
e.probeResults[siteID] = make(map[string]NodeResult)
|
|
||||||
}
|
|
||||||
e.probeResults[siteID][nodeID] = NodeResult{
|
|
||||||
NodeID: nodeID,
|
|
||||||
IsUp: isUp,
|
|
||||||
LatencyNs: latencyNs,
|
|
||||||
CheckedAt: now,
|
|
||||||
ErrorReason: errorReason,
|
|
||||||
}
|
|
||||||
results := make([]NodeResult, 0, len(e.probeResults[siteID]))
|
|
||||||
for id, r := range e.probeResults[siteID] {
|
|
||||||
if now.Sub(r.CheckedAt) > staleAfter {
|
|
||||||
delete(e.probeResults[siteID], id)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
results = append(results, r)
|
|
||||||
}
|
|
||||||
e.probeResultsMu.Unlock()
|
|
||||||
|
|
||||||
aggUp, avgLatency := AggregateStatus(results, e.aggStrategy)
|
|
||||||
|
|
||||||
probeStatus := models.StatusUp
|
|
||||||
if !aggUp {
|
|
||||||
probeStatus = models.StatusDown
|
|
||||||
}
|
|
||||||
|
|
||||||
updatedSite := site
|
|
||||||
updatedSite.Latency = time.Duration(avgLatency)
|
|
||||||
updatedSite.LastCheck = time.Now()
|
|
||||||
e.handleStatusChange(updatedSite, string(probeStatus), 0, time.Duration(avgLatency), errorReason)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetProbeResults(siteID int) map[string]NodeResult {
|
|
||||||
e.probeResultsMu.RLock()
|
|
||||||
defer e.probeResultsMu.RUnlock()
|
|
||||||
src := e.probeResults[siteID]
|
|
||||||
cp := make(map[string]NodeResult, len(src))
|
|
||||||
for k, v := range src {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
return cp
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
|
|
||||||
changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return changes
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
|
|
||||||
changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return changes
|
|
||||||
}
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,158 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (e *Engine) SetActive(active bool) {
|
||||||
|
e.activeMu.Lock()
|
||||||
|
defer e.activeMu.Unlock()
|
||||||
|
if e.isActive != active {
|
||||||
|
e.isActive = active
|
||||||
|
status := "RESUMED (Active)"
|
||||||
|
if !active {
|
||||||
|
status = "PAUSED (Passive)"
|
||||||
|
}
|
||||||
|
e.AddLog(fmt.Sprintf("Engine %s", status))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) IsActive() bool {
|
||||||
|
e.activeMu.RLock()
|
||||||
|
defer e.activeMu.RUnlock()
|
||||||
|
return e.isActive
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) GetAllSites() []models.Site {
|
||||||
|
e.mu.RLock()
|
||||||
|
defer e.mu.RUnlock()
|
||||||
|
sites := make([]models.Site, 0, len(e.liveState))
|
||||||
|
for _, s := range e.liveState {
|
||||||
|
sites = append(sites, s)
|
||||||
|
}
|
||||||
|
return sites
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) GetLiveState() map[int]models.Site {
|
||||||
|
e.mu.RLock()
|
||||||
|
defer e.mu.RUnlock()
|
||||||
|
cp := make(map[int]models.Site, len(e.liveState))
|
||||||
|
for k, v := range e.liveState {
|
||||||
|
cp[k] = v
|
||||||
|
}
|
||||||
|
return cp
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) addToTokenIndex(site models.Site) {
|
||||||
|
if site.Type == "push" && site.Token != "" {
|
||||||
|
e.tokenIndex[site.Token] = site.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) removeFromTokenIndex(id int) {
|
||||||
|
for token, sid := range e.tokenIndex {
|
||||||
|
if sid == id {
|
||||||
|
delete(e.tokenIndex, token)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) UpdateSiteConfig(cfg models.SiteConfig) {
|
||||||
|
e.mu.Lock()
|
||||||
|
if existing, ok := e.liveState[cfg.ID]; ok {
|
||||||
|
e.removeFromTokenIndex(cfg.ID)
|
||||||
|
existing.SiteConfig = cfg
|
||||||
|
e.liveState[cfg.ID] = existing
|
||||||
|
e.addToTokenIndex(existing)
|
||||||
|
}
|
||||||
|
e.mu.Unlock()
|
||||||
|
|
||||||
|
e.signalRecheck(cfg.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) RemoveSite(id int) {
|
||||||
|
e.mu.Lock()
|
||||||
|
e.removeFromTokenIndex(id)
|
||||||
|
delete(e.liveState, id)
|
||||||
|
e.mu.Unlock()
|
||||||
|
e.removeHistory(id)
|
||||||
|
|
||||||
|
e.probeResultsMu.Lock()
|
||||||
|
delete(e.probeResults, id)
|
||||||
|
e.probeResultsMu.Unlock()
|
||||||
|
|
||||||
|
e.recheckMu.Lock()
|
||||||
|
delete(e.recheck, id)
|
||||||
|
e.recheckMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ToggleSitePause(id int) bool {
|
||||||
|
var (
|
||||||
|
paused bool
|
||||||
|
name string
|
||||||
|
)
|
||||||
|
_, ok := e.applyState(id, func(s *models.Site) {
|
||||||
|
s.Paused = !s.Paused
|
||||||
|
paused = s.Paused
|
||||||
|
name = s.Name
|
||||||
|
})
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if paused {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' paused", name))
|
||||||
|
} else {
|
||||||
|
e.AddLog(fmt.Sprintf("Monitor '%s' resumed", name))
|
||||||
|
}
|
||||||
|
return paused
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyState atomically reads, mutates, and writes back the live entry for id.
|
||||||
|
// The mutator runs under the engine write lock and receives a pointer to the
|
||||||
|
// CURRENT live state, so concurrent config edits, pauses, and heartbeats are
|
||||||
|
// never clobbered by a stale snapshot. The mutator must only touch runtime /
|
||||||
|
// check-result fields — config fields (Name/URL/Type/Token/Interval/AlertID/…)
|
||||||
|
// are owned by UpdateSiteConfig and must not be written here. Returns the
|
||||||
|
// post-mutation copy and whether the site still exists.
|
||||||
|
func (e *Engine) applyState(id int, mutate func(s *models.Site)) (models.Site, bool) {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
cur, ok := e.liveState[id]
|
||||||
|
if !ok {
|
||||||
|
return models.Site{}, false
|
||||||
|
}
|
||||||
|
mutate(&cur)
|
||||||
|
e.liveState[id] = cur
|
||||||
|
return cur, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) GetDisplayStatus(site models.Site) string {
|
||||||
|
if site.Paused {
|
||||||
|
return "PAUSED"
|
||||||
|
}
|
||||||
|
if e.isInMaintenance(site.ID) {
|
||||||
|
return "MAINT"
|
||||||
|
}
|
||||||
|
return string(site.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) GetStateChanges(siteID int, limit int) []models.StateChange {
|
||||||
|
changes, err := e.db.GetStateChanges(context.Background(), siteID, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return changes
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) GetStateChangesSince(siteID int, since time.Time) []models.StateChange {
|
||||||
|
changes, err := e.db.GetStateChangesSince(context.Background(), siteID, since)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return changes
|
||||||
|
}
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUpdateSiteConfig_PreservesRuntime(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", URL: "http://old.com"},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN", FailureCount: 3, Latency: 100 * time.Millisecond},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
updated := models.SiteConfig{ID: 1, Name: "test", URL: "http://new.com", Interval: 60}
|
||||||
|
e.UpdateSiteConfig(updated)
|
||||||
|
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if s.URL != "http://new.com" {
|
||||||
|
t.Errorf("expected URL updated, got %s", s.URL)
|
||||||
|
}
|
||||||
|
if s.Status != "DOWN" {
|
||||||
|
t.Errorf("expected Status preserved, got %s", s.Status)
|
||||||
|
}
|
||||||
|
if s.FailureCount != 3 {
|
||||||
|
t.Errorf("expected FailureCount preserved, got %d", s.FailureCount)
|
||||||
|
}
|
||||||
|
if s.Latency != 100*time.Millisecond {
|
||||||
|
t.Errorf("expected Latency preserved, got %v", s.Latency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoveSite_CleansUp(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test", Type: "push", Token: "tok1"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
e.recordCheck(1, 5*time.Millisecond, true)
|
||||||
|
|
||||||
|
e.RemoveSite(1)
|
||||||
|
|
||||||
|
if _, ok := getSite(e, 1); ok {
|
||||||
|
t.Error("expected site removed from liveState")
|
||||||
|
}
|
||||||
|
if e.RecordHeartbeat("tok1") {
|
||||||
|
t.Error("expected token removed from index")
|
||||||
|
}
|
||||||
|
if _, ok := e.GetHistory(1); ok {
|
||||||
|
t.Error("expected history removed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToggleSitePause(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
site := models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "test"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
}
|
||||||
|
injectSite(e, site)
|
||||||
|
|
||||||
|
paused := e.ToggleSitePause(1)
|
||||||
|
if !paused {
|
||||||
|
t.Error("expected paused=true after first toggle")
|
||||||
|
}
|
||||||
|
s, _ := getSite(e, 1)
|
||||||
|
if !s.Paused {
|
||||||
|
t.Error("expected Paused=true in state")
|
||||||
|
}
|
||||||
|
|
||||||
|
paused = e.ToggleSitePause(1)
|
||||||
|
if paused {
|
||||||
|
t.Error("expected paused=false after second toggle")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToggleSitePause_NonexistentSite(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
if e.ToggleSitePause(999) {
|
||||||
|
t.Error("expected false for nonexistent site")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetAllSites_ReturnsCopy(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
injectSite(e, models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
})
|
||||||
|
injectSite(e, models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 2, Name: "s2"},
|
||||||
|
SiteState: models.SiteState{Status: "DOWN"},
|
||||||
|
})
|
||||||
|
|
||||||
|
sites := e.GetAllSites()
|
||||||
|
if len(sites) != 2 {
|
||||||
|
t.Fatalf("expected 2 sites, got %d", len(sites))
|
||||||
|
}
|
||||||
|
sites[0].Name = "mutated"
|
||||||
|
|
||||||
|
fresh := e.GetAllSites()
|
||||||
|
for _, s := range fresh {
|
||||||
|
if s.Name == "mutated" {
|
||||||
|
t.Error("GetAllSites returned reference, not copy")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetLiveState_ReturnsCopy(t *testing.T) {
|
||||||
|
ms := newMockStore()
|
||||||
|
e := newTestEngine(ms)
|
||||||
|
injectSite(e, models.Site{
|
||||||
|
SiteConfig: models.SiteConfig{ID: 1, Name: "s1"},
|
||||||
|
SiteState: models.SiteState{Status: "UP"},
|
||||||
|
})
|
||||||
|
|
||||||
|
state := e.GetLiveState()
|
||||||
|
state[1] = models.Site{SiteConfig: models.SiteConfig{Name: "mutated"}}
|
||||||
|
|
||||||
|
fresh := e.GetLiveState()
|
||||||
|
if fresh[1].Name == "mutated" {
|
||||||
|
t.Error("GetLiveState returned reference, not copy")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -218,40 +217,6 @@ func (s *SQLStore) GetSiteByName(ctx context.Context, name string) (models.SiteC
|
|||||||
return st, err
|
return st, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) {
|
|
||||||
decrypted, err := s.decryptSettings(raw)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("decrypt settings: %w", err)
|
|
||||||
}
|
|
||||||
var m map[string]string
|
|
||||||
if err := json.Unmarshal([]byte(decrypted), &m); err != nil {
|
|
||||||
return nil, fmt.Errorf("unmarshal settings: %w", err)
|
|
||||||
}
|
|
||||||
return m, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) {
|
|
||||||
jsonBytes, err := json.Marshal(settings)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return s.encryptSettings(string(jsonBytes))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) {
|
|
||||||
var a models.AlertConfig
|
|
||||||
var settingsRaw string
|
|
||||||
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
|
||||||
if err != nil {
|
|
||||||
return a, err
|
|
||||||
}
|
|
||||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
|
||||||
if err != nil {
|
|
||||||
return a, fmt.Errorf("alert %q: %w", name, err)
|
|
||||||
}
|
|
||||||
return a, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfig) (int, error) {
|
func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfig) (int, error) {
|
||||||
token := ""
|
token := ""
|
||||||
if site.Type == "push" {
|
if site.Type == "push" {
|
||||||
@@ -278,89 +243,6 @@ func (s *SQLStore) AddSiteReturningID(ctx context.Context, site models.SiteConfi
|
|||||||
return int(id), err
|
return int(id), err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) {
|
|
||||||
stored, err := s.marshalSettings(settings)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if s.dollar {
|
|
||||||
var id int
|
|
||||||
err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id)
|
|
||||||
return id, err
|
|
||||||
}
|
|
||||||
result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
id, err := result.LastInsertId()
|
|
||||||
return int(id), err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) {
|
|
||||||
rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var alerts []models.AlertConfig
|
|
||||||
for rows.Next() {
|
|
||||||
var a models.AlertConfig
|
|
||||||
var settingsRaw string
|
|
||||||
if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil {
|
|
||||||
return alerts, err
|
|
||||||
}
|
|
||||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
|
||||||
if err != nil {
|
|
||||||
return alerts, fmt.Errorf("alert %q: %w", a.Name, err)
|
|
||||||
}
|
|
||||||
alerts = append(alerts, a)
|
|
||||||
}
|
|
||||||
return alerts, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) {
|
|
||||||
var a models.AlertConfig
|
|
||||||
var settingsRaw string
|
|
||||||
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
|
||||||
if err != nil {
|
|
||||||
return a, err
|
|
||||||
}
|
|
||||||
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
|
||||||
if err != nil {
|
|
||||||
return a, fmt.Errorf("alert %d: %w", id, err)
|
|
||||||
}
|
|
||||||
return a, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error {
|
|
||||||
stored, err := s.marshalSettings(settings)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error {
|
|
||||||
stored, err := s.marshalSettings(settings)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error {
|
|
||||||
if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
s.dialect.ResetSequenceOnEmpty(s.db, "alerts")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetAllUsers(ctx context.Context) ([]models.User, error) {
|
func (s *SQLStore) GetAllUsers(ctx context.Context) ([]models.User, error) {
|
||||||
rows, err := s.db.QueryContext(ctx, "SELECT id, username, public_key, role FROM users")
|
rows, err := s.db.QueryContext(ctx, "SELECT id, username, public_key, role FROM users")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -393,85 +275,6 @@ func (s *SQLStore) DeleteUser(ctx context.Context, id int) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error {
|
|
||||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"),
|
|
||||||
siteID, fromStatus, toStatus, errorReason)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) {
|
|
||||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var changes []models.StateChange
|
|
||||||
for rows.Next() {
|
|
||||||
var sc models.StateChange
|
|
||||||
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
|
||||||
return changes, err
|
|
||||||
}
|
|
||||||
changes = append(changes, sc)
|
|
||||||
}
|
|
||||||
return changes, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) {
|
|
||||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var changes []models.StateChange
|
|
||||||
for rows.Next() {
|
|
||||||
var sc models.StateChange
|
|
||||||
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
|
||||||
return changes, err
|
|
||||||
}
|
|
||||||
changes = append(changes, sc)
|
|
||||||
}
|
|
||||||
return changes, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error {
|
|
||||||
return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SaveCheckFromNode inserts a single check row. Retention is handled out of
|
|
||||||
// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot
|
|
||||||
// path a plain INSERT.
|
|
||||||
func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error {
|
|
||||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per
|
|
||||||
// site, across all sites, in one pass. Intended to run periodically.
|
|
||||||
func (s *SQLStore) PruneCheckHistory(ctx context.Context) error {
|
|
||||||
q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN (
|
|
||||||
SELECT id FROM (
|
|
||||||
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn
|
|
||||||
FROM check_history
|
|
||||||
) ranked WHERE rn > %d
|
|
||||||
)`, maxCheckHistory)
|
|
||||||
_, err := s.db.ExecContext(ctx, s.q(q))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite
|
|
||||||
// rows per site. Generous so realistic SLA windows are unaffected; bounds the
|
|
||||||
// otherwise unbounded growth of a flapping monitor's history.
|
|
||||||
func (s *SQLStore) PruneStateChanges(ctx context.Context) error {
|
|
||||||
q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN (
|
|
||||||
SELECT id FROM (
|
|
||||||
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn
|
|
||||||
FROM state_changes
|
|
||||||
) ranked WHERE rn > %d
|
|
||||||
)`, maxStateChangesPerSite)
|
|
||||||
_, err := s.db.ExecContext(ctx, s.q(q))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) RegisterNode(ctx context.Context, node models.ProbeNode) error {
|
func (s *SQLStore) RegisterNode(ctx context.Context, node models.ProbeNode) error {
|
||||||
_, err := s.db.ExecContext(ctx, s.dialect.UpsertNodeSQL(), node.ID, node.Name, node.Region, node.Version)
|
_, err := s.db.ExecContext(ctx, s.dialect.UpsertNodeSQL(), node.ID, node.Name, node.Region, node.Version)
|
||||||
return err
|
return err
|
||||||
@@ -582,172 +385,6 @@ func (s *SQLStore) LoadLogs(ctx context.Context, limit int) ([]models.LogEntry,
|
|||||||
return entries, rows.Err()
|
return entries, rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) {
|
|
||||||
result := make(map[int][]models.CheckRecord)
|
|
||||||
rows, err := s.db.QueryContext(ctx, s.q(`
|
|
||||||
SELECT site_id, latency_ns, is_up FROM (
|
|
||||||
SELECT site_id, latency_ns, is_up,
|
|
||||||
ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn
|
|
||||||
FROM check_history
|
|
||||||
) sub WHERE rn <= ?`), limit)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
for rows.Next() {
|
|
||||||
var r models.CheckRecord
|
|
||||||
if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
result[r.SiteID] = append(result[r.SiteID], r)
|
|
||||||
}
|
|
||||||
for id, records := range result {
|
|
||||||
for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 {
|
|
||||||
records[i], records[j] = records[j], records[i]
|
|
||||||
}
|
|
||||||
result[id] = records
|
|
||||||
}
|
|
||||||
return result, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) {
|
|
||||||
var mw models.MaintenanceWindow
|
|
||||||
var endTime sql.NullTime
|
|
||||||
if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil {
|
|
||||||
return mw, err
|
|
||||||
}
|
|
||||||
if endTime.Valid {
|
|
||||||
mw.EndTime = endTime.Time
|
|
||||||
}
|
|
||||||
return mw, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) {
|
|
||||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC"))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var windows []models.MaintenanceWindow
|
|
||||||
for rows.Next() {
|
|
||||||
mw, err := s.scanMaintenanceWindow(rows)
|
|
||||||
if err != nil {
|
|
||||||
return windows, err
|
|
||||||
}
|
|
||||||
windows = append(windows, mw)
|
|
||||||
}
|
|
||||||
return windows, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) {
|
|
||||||
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
var windows []models.MaintenanceWindow
|
|
||||||
for rows.Next() {
|
|
||||||
mw, err := s.scanMaintenanceWindow(rows)
|
|
||||||
if err != nil {
|
|
||||||
return windows, err
|
|
||||||
}
|
|
||||||
windows = append(windows, mw)
|
|
||||||
}
|
|
||||||
return windows, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) {
|
|
||||||
var timeClause string
|
|
||||||
var args []interface{}
|
|
||||||
|
|
||||||
if endTime.IsZero() {
|
|
||||||
timeClause = "(end_time IS NULL OR end_time > ?)"
|
|
||||||
args = append(args, startTime)
|
|
||||||
} else {
|
|
||||||
timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?"
|
|
||||||
args = append(args, startTime, endTime)
|
|
||||||
}
|
|
||||||
|
|
||||||
var scopeClause string
|
|
||||||
if monitorID == 0 {
|
|
||||||
scopeClause = "1=1"
|
|
||||||
} else {
|
|
||||||
scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))"
|
|
||||||
args = append(args, monitorID, monitorID)
|
|
||||||
}
|
|
||||||
|
|
||||||
query := fmt.Sprintf(
|
|
||||||
"SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time",
|
|
||||||
timeClause, scopeClause,
|
|
||||||
)
|
|
||||||
|
|
||||||
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
var windows []models.MaintenanceWindow
|
|
||||||
for rows.Next() {
|
|
||||||
mw, err := s.scanMaintenanceWindow(rows)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
windows = append(windows, mw)
|
|
||||||
}
|
|
||||||
return windows, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error {
|
|
||||||
if mw.StartTime.IsZero() {
|
|
||||||
mw.StartTime = time.Now()
|
|
||||||
}
|
|
||||||
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"),
|
|
||||||
mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error {
|
|
||||||
_, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error {
|
|
||||||
_, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) {
|
|
||||||
cutoff := time.Now().Add(-retention)
|
|
||||||
result, err := s.db.ExecContext(ctx,
|
|
||||||
s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"),
|
|
||||||
cutoff,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return result.RowsAffected()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) {
|
|
||||||
var count int
|
|
||||||
err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows
|
|
||||||
WHERE type = 'maintenance'
|
|
||||||
AND start_time <= CURRENT_TIMESTAMP
|
|
||||||
AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP)
|
|
||||||
AND (monitor_id = 0 OR monitor_id = ?
|
|
||||||
OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`),
|
|
||||||
monitorID, monitorID).Scan(&count)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return count > 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *SQLStore) GetPreference(ctx context.Context, key string) (string, error) {
|
func (s *SQLStore) GetPreference(ctx context.Context, key string) (string, error) {
|
||||||
var value string
|
var value string
|
||||||
err := s.db.QueryRowContext(ctx, s.q("SELECT value FROM preferences WHERE key = ?"), key).Scan(&value)
|
err := s.db.QueryRowContext(ctx, s.q("SELECT value FROM preferences WHERE key = ?"), key).Scan(&value)
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *SQLStore) unmarshalSettings(raw string) (map[string]string, error) {
|
||||||
|
decrypted, err := s.decryptSettings(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decrypt settings: %w", err)
|
||||||
|
}
|
||||||
|
var m map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(decrypted), &m); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal settings: %w", err)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) marshalSettings(settings map[string]string) (string, error) {
|
||||||
|
jsonBytes, err := json.Marshal(settings)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return s.encryptSettings(string(jsonBytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetAlertByName(ctx context.Context, name string) (models.AlertConfig, error) {
|
||||||
|
var a models.AlertConfig
|
||||||
|
var settingsRaw string
|
||||||
|
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE name = ?"), name).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
||||||
|
if err != nil {
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
|
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||||
|
if err != nil {
|
||||||
|
return a, fmt.Errorf("alert %q: %w", name, err)
|
||||||
|
}
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) AddAlertReturningID(ctx context.Context, name, aType string, settings map[string]string) (int, error) {
|
||||||
|
stored, err := s.marshalSettings(settings)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if s.dollar {
|
||||||
|
var id int
|
||||||
|
err := s.db.QueryRowContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?) RETURNING id"), name, aType, stored).Scan(&id)
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
|
result, err := s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
id, err := result.LastInsertId()
|
||||||
|
return int(id), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetAllAlerts(ctx context.Context) ([]models.AlertConfig, error) {
|
||||||
|
rows, err := s.db.QueryContext(ctx, "SELECT id, name, type, settings FROM alerts")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var alerts []models.AlertConfig
|
||||||
|
for rows.Next() {
|
||||||
|
var a models.AlertConfig
|
||||||
|
var settingsRaw string
|
||||||
|
if err := rows.Scan(&a.ID, &a.Name, &a.Type, &settingsRaw); err != nil {
|
||||||
|
return alerts, err
|
||||||
|
}
|
||||||
|
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||||
|
if err != nil {
|
||||||
|
return alerts, fmt.Errorf("alert %q: %w", a.Name, err)
|
||||||
|
}
|
||||||
|
alerts = append(alerts, a)
|
||||||
|
}
|
||||||
|
return alerts, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetAlert(ctx context.Context, id int) (models.AlertConfig, error) {
|
||||||
|
var a models.AlertConfig
|
||||||
|
var settingsRaw string
|
||||||
|
err := s.db.QueryRowContext(ctx, s.q("SELECT id, name, type, settings FROM alerts WHERE id = ?"), id).Scan(&a.ID, &a.Name, &a.Type, &settingsRaw)
|
||||||
|
if err != nil {
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
|
a.Settings, err = s.unmarshalSettings(settingsRaw)
|
||||||
|
if err != nil {
|
||||||
|
return a, fmt.Errorf("alert %d: %w", id, err)
|
||||||
|
}
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) AddAlert(ctx context.Context, name, aType string, settings map[string]string) error {
|
||||||
|
stored, err := s.marshalSettings(settings)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = s.db.ExecContext(ctx, s.q("INSERT INTO alerts (name, type, settings) VALUES (?, ?, ?)"), name, aType, stored)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) UpdateAlert(ctx context.Context, id int, name, aType string, settings map[string]string) error {
|
||||||
|
stored, err := s.marshalSettings(settings)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = s.db.ExecContext(ctx, s.q("UPDATE alerts SET name=?, type=?, settings=? WHERE id=?"), name, aType, stored, id)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) DeleteAlert(ctx context.Context, id int) error {
|
||||||
|
if _, err := s.db.ExecContext(ctx, s.q("UPDATE sites SET alert_id = 0 WHERE alert_id = ?"), id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := s.db.ExecContext(ctx, s.q("DELETE FROM alerts WHERE id=?"), id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.dialect.ResetSequenceOnEmpty(s.db, "alerts")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAlertCRUD(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil {
|
||||||
|
t.Fatalf("AddAlert: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
alerts, err := s.GetAllAlerts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetAllAlerts: %v", err)
|
||||||
|
}
|
||||||
|
if len(alerts) != 1 {
|
||||||
|
t.Fatalf("expected 1 alert, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
if alerts[0].Type != "discord" {
|
||||||
|
t.Errorf("expected type 'discord', got '%s'", alerts[0].Type)
|
||||||
|
}
|
||||||
|
if alerts[0].Settings["url"] != "https://example.com/hook" {
|
||||||
|
t.Errorf("settings url mismatch")
|
||||||
|
}
|
||||||
|
|
||||||
|
a, err := s.GetAlert(context.Background(), alerts[0].ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetAlert: %v", err)
|
||||||
|
}
|
||||||
|
if a.Name != "Discord" {
|
||||||
|
t.Errorf("expected name 'Discord', got '%s'", a.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil {
|
||||||
|
t.Fatalf("UpdateAlert: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
a, err = s.GetAlert(context.Background(), a.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetAlert: %v", err)
|
||||||
|
}
|
||||||
|
if a.Type != "slack" {
|
||||||
|
t.Errorf("expected type 'slack', got '%s'", a.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.DeleteAlert(context.Background(), a.ID); err != nil {
|
||||||
|
t.Fatalf("DeleteAlert: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
alerts, err = s.GetAllAlerts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetAllAlerts: %v", err)
|
||||||
|
}
|
||||||
|
if len(alerts) != 0 {
|
||||||
|
t.Fatalf("expected 0 alerts after delete, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a
|
||||||
|
// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext.
|
||||||
|
func TestImportData_EncryptsAlertSettings(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewEncryptor: %v", err)
|
||||||
|
}
|
||||||
|
s.SetEncryptor(enc)
|
||||||
|
|
||||||
|
backup := models.Backup{
|
||||||
|
Alerts: []models.AlertConfig{
|
||||||
|
{ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := s.ImportData(context.Background(), backup); err != nil {
|
||||||
|
t.Fatalf("ImportData: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var raw string
|
||||||
|
if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil {
|
||||||
|
t.Fatalf("query settings: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(raw, encryptedPrefix) {
|
||||||
|
t.Errorf("imported settings not encrypted: %q", raw)
|
||||||
|
}
|
||||||
|
if strings.Contains(raw, "SECRET") {
|
||||||
|
t.Errorf("plaintext secret found in stored column: %q", raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
alerts, err := s.GetAllAlerts(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetAllAlerts: %v", err)
|
||||||
|
}
|
||||||
|
if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" {
|
||||||
|
t.Errorf("decrypt round-trip failed: %+v", alerts)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *SQLStore) SaveStateChange(ctx context.Context, siteID int, fromStatus, toStatus, errorReason string) error {
|
||||||
|
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO state_changes (site_id, from_status, to_status, error_reason) VALUES (?, ?, ?, ?)"),
|
||||||
|
siteID, fromStatus, toStatus, errorReason)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetStateChanges(ctx context.Context, siteID int, limit int) ([]models.StateChange, error) {
|
||||||
|
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? ORDER BY changed_at DESC LIMIT ?"), siteID, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var changes []models.StateChange
|
||||||
|
for rows.Next() {
|
||||||
|
var sc models.StateChange
|
||||||
|
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
||||||
|
return changes, err
|
||||||
|
}
|
||||||
|
changes = append(changes, sc)
|
||||||
|
}
|
||||||
|
return changes, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetStateChangesSince(ctx context.Context, siteID int, since time.Time) ([]models.StateChange, error) {
|
||||||
|
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, site_id, from_status, to_status, error_reason, changed_at FROM state_changes WHERE site_id = ? AND changed_at >= ? ORDER BY changed_at DESC"), siteID, since)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var changes []models.StateChange
|
||||||
|
for rows.Next() {
|
||||||
|
var sc models.StateChange
|
||||||
|
if err := rows.Scan(&sc.ID, &sc.SiteID, &sc.FromStatus, &sc.ToStatus, &sc.ErrorReason, &sc.ChangedAt); err != nil {
|
||||||
|
return changes, err
|
||||||
|
}
|
||||||
|
changes = append(changes, sc)
|
||||||
|
}
|
||||||
|
return changes, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) SaveCheck(ctx context.Context, siteID int, latencyNs int64, isUp bool) error {
|
||||||
|
return s.SaveCheckFromNode(ctx, siteID, "", latencyNs, isUp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveCheckFromNode inserts a single check row. Retention is handled out of
|
||||||
|
// band by PruneCheckHistory on a timer, not per-insert, to keep the write hot
|
||||||
|
// path a plain INSERT.
|
||||||
|
func (s *SQLStore) SaveCheckFromNode(ctx context.Context, siteID int, nodeID string, latencyNs int64, isUp bool) error {
|
||||||
|
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO check_history (site_id, node_id, latency_ns, is_up) VALUES (?, ?, ?, ?)"), siteID, nodeID, latencyNs, isUp)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// PruneCheckHistory trims check_history to the newest maxCheckHistory rows per
|
||||||
|
// site, across all sites, in one pass. Intended to run periodically.
|
||||||
|
func (s *SQLStore) PruneCheckHistory(ctx context.Context) error {
|
||||||
|
q := fmt.Sprintf(`DELETE FROM check_history WHERE id IN (
|
||||||
|
SELECT id FROM (
|
||||||
|
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC, id DESC) AS rn
|
||||||
|
FROM check_history
|
||||||
|
) ranked WHERE rn > %d
|
||||||
|
)`, maxCheckHistory)
|
||||||
|
_, err := s.db.ExecContext(ctx, s.q(q))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// PruneStateChanges trims state_changes to the newest maxStateChangesPerSite
|
||||||
|
// rows per site. Generous so realistic SLA windows are unaffected; bounds the
|
||||||
|
// otherwise unbounded growth of a flapping monitor's history.
|
||||||
|
func (s *SQLStore) PruneStateChanges(ctx context.Context) error {
|
||||||
|
q := fmt.Sprintf(`DELETE FROM state_changes WHERE id IN (
|
||||||
|
SELECT id FROM (
|
||||||
|
SELECT id, ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY changed_at DESC, id DESC) AS rn
|
||||||
|
FROM state_changes
|
||||||
|
) ranked WHERE rn > %d
|
||||||
|
)`, maxStateChangesPerSite)
|
||||||
|
_, err := s.db.ExecContext(ctx, s.q(q))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) LoadAllHistory(ctx context.Context, limit int) (map[int][]models.CheckRecord, error) {
|
||||||
|
result := make(map[int][]models.CheckRecord)
|
||||||
|
rows, err := s.db.QueryContext(ctx, s.q(`
|
||||||
|
SELECT site_id, latency_ns, is_up FROM (
|
||||||
|
SELECT site_id, latency_ns, is_up,
|
||||||
|
ROW_NUMBER() OVER (PARTITION BY site_id ORDER BY checked_at DESC) AS rn
|
||||||
|
FROM check_history
|
||||||
|
) sub WHERE rn <= ?`), limit)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var r models.CheckRecord
|
||||||
|
if err := rows.Scan(&r.SiteID, &r.LatencyNs, &r.IsUp); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result[r.SiteID] = append(result[r.SiteID], r)
|
||||||
|
}
|
||||||
|
for id, records := range result {
|
||||||
|
for i, j := 0, len(records)-1; i < j; i, j = i+1, j-1 {
|
||||||
|
records[i], records[j] = records[j], records[i]
|
||||||
|
}
|
||||||
|
result[id] = records
|
||||||
|
}
|
||||||
|
return result, rows.Err()
|
||||||
|
}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCheckHistory(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil {
|
||||||
|
t.Fatalf("SaveCheck: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil {
|
||||||
|
t.Fatalf("SaveCheck: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil {
|
||||||
|
t.Fatalf("SaveCheck site 2: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
history, err := s.LoadAllHistory(context.Background(), 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadAllHistory: %v", err)
|
||||||
|
}
|
||||||
|
if len(history[1]) != 2 {
|
||||||
|
t.Fatalf("expected 2 records for site 1, got %d", len(history[1]))
|
||||||
|
}
|
||||||
|
if len(history[2]) != 1 {
|
||||||
|
t.Fatalf("expected 1 record for site 2, got %d", len(history[2]))
|
||||||
|
}
|
||||||
|
|
||||||
|
upCount := 0
|
||||||
|
for _, r := range history[1] {
|
||||||
|
if r.IsUp {
|
||||||
|
upCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if upCount != 1 {
|
||||||
|
t.Errorf("expected 1 up record for site 1, got %d", upCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPruneCheckHistory(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
for i := 0; i < maxCheckHistory+5; i++ {
|
||||||
|
if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil {
|
||||||
|
t.Fatalf("SaveCheck site 1: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil {
|
||||||
|
t.Fatalf("SaveCheck site 2: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.PruneCheckHistory(context.Background()); err != nil {
|
||||||
|
t.Fatalf("PruneCheckHistory: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadAllHistory: %v", err)
|
||||||
|
}
|
||||||
|
if len(history[1]) != maxCheckHistory {
|
||||||
|
t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1]))
|
||||||
|
}
|
||||||
|
if len(history[2]) != 3 {
|
||||||
|
t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeleteSiteCascade(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30}
|
||||||
|
if err := s.AddSite(context.Background(), site); err != nil {
|
||||||
|
t.Fatalf("AddSite: %v", err)
|
||||||
|
}
|
||||||
|
sites, err := s.GetSites(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetSites: %v", err)
|
||||||
|
}
|
||||||
|
siteID := sites[0].ID
|
||||||
|
|
||||||
|
if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil {
|
||||||
|
t.Fatalf("SaveCheck: %v", err)
|
||||||
|
}
|
||||||
|
if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil {
|
||||||
|
t.Fatalf("SaveStateChange: %v", err)
|
||||||
|
}
|
||||||
|
mw := models.MaintenanceWindow{
|
||||||
|
MonitorID: siteID,
|
||||||
|
Title: "Test MW",
|
||||||
|
Type: "maintenance",
|
||||||
|
StartTime: time.Now(),
|
||||||
|
}
|
||||||
|
if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil {
|
||||||
|
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.DeleteSite(context.Background(), siteID); err != nil {
|
||||||
|
t.Fatalf("DeleteSite: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
history, err := s.LoadAllHistory(context.Background(), 100)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LoadAllHistory: %v", err)
|
||||||
|
}
|
||||||
|
if len(history[siteID]) != 0 {
|
||||||
|
t.Errorf("expected 0 check_history rows, got %d", len(history[siteID]))
|
||||||
|
}
|
||||||
|
|
||||||
|
changes, err := s.GetStateChanges(context.Background(), siteID, 100)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetStateChanges: %v", err)
|
||||||
|
}
|
||||||
|
if len(changes) != 0 {
|
||||||
|
t.Errorf("expected 0 state_changes rows, got %d", len(changes))
|
||||||
|
}
|
||||||
|
|
||||||
|
windows, err := s.GetActiveMaintenanceWindows(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetActiveMaintenanceWindows: %v", err)
|
||||||
|
}
|
||||||
|
for _, w := range windows {
|
||||||
|
if w.MonitorID == siteID {
|
||||||
|
t.Errorf("orphaned maintenance window found: id=%d", w.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *SQLStore) scanMaintenanceWindow(rows *sql.Rows) (models.MaintenanceWindow, error) {
|
||||||
|
var mw models.MaintenanceWindow
|
||||||
|
var endTime sql.NullTime
|
||||||
|
if err := rows.Scan(&mw.ID, &mw.MonitorID, &mw.Title, &mw.Description, &mw.Type, &mw.StartTime, &endTime, &mw.CreatedBy, &mw.CreatedAt); err != nil {
|
||||||
|
return mw, err
|
||||||
|
}
|
||||||
|
if endTime.Valid {
|
||||||
|
mw.EndTime = endTime.Time
|
||||||
|
}
|
||||||
|
return mw, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetActiveMaintenanceWindows(ctx context.Context) ([]models.MaintenanceWindow, error) {
|
||||||
|
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE start_time <= CURRENT_TIMESTAMP AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP) ORDER BY start_time DESC"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var windows []models.MaintenanceWindow
|
||||||
|
for rows.Next() {
|
||||||
|
mw, err := s.scanMaintenanceWindow(rows)
|
||||||
|
if err != nil {
|
||||||
|
return windows, err
|
||||||
|
}
|
||||||
|
windows = append(windows, mw)
|
||||||
|
}
|
||||||
|
return windows, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetAllMaintenanceWindows(ctx context.Context, limit int) ([]models.MaintenanceWindow, error) {
|
||||||
|
rows, err := s.db.QueryContext(ctx, s.q("SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows ORDER BY created_at DESC LIMIT ?"), limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var windows []models.MaintenanceWindow
|
||||||
|
for rows.Next() {
|
||||||
|
mw, err := s.scanMaintenanceWindow(rows)
|
||||||
|
if err != nil {
|
||||||
|
return windows, err
|
||||||
|
}
|
||||||
|
windows = append(windows, mw)
|
||||||
|
}
|
||||||
|
return windows, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) GetOverlappingMaintenanceWindows(ctx context.Context, monitorID int, startTime, endTime time.Time) ([]models.MaintenanceWindow, error) {
|
||||||
|
var timeClause string
|
||||||
|
var args []interface{}
|
||||||
|
|
||||||
|
if endTime.IsZero() {
|
||||||
|
timeClause = "(end_time IS NULL OR end_time > ?)"
|
||||||
|
args = append(args, startTime)
|
||||||
|
} else {
|
||||||
|
timeClause = "(end_time IS NULL OR end_time > ?) AND start_time < ?"
|
||||||
|
args = append(args, startTime, endTime)
|
||||||
|
}
|
||||||
|
|
||||||
|
var scopeClause string
|
||||||
|
if monitorID == 0 {
|
||||||
|
scopeClause = "1=1"
|
||||||
|
} else {
|
||||||
|
scopeClause = "(monitor_id = ? OR monitor_id = 0 OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))"
|
||||||
|
args = append(args, monitorID, monitorID)
|
||||||
|
}
|
||||||
|
|
||||||
|
query := fmt.Sprintf(
|
||||||
|
"SELECT id, monitor_id, title, description, type, start_time, end_time, created_by, created_at FROM maintenance_windows WHERE %s AND %s ORDER BY start_time",
|
||||||
|
timeClause, scopeClause,
|
||||||
|
)
|
||||||
|
|
||||||
|
rows, err := s.db.QueryContext(ctx, s.q(query), args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var windows []models.MaintenanceWindow
|
||||||
|
for rows.Next() {
|
||||||
|
mw, err := s.scanMaintenanceWindow(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
windows = append(windows, mw)
|
||||||
|
}
|
||||||
|
return windows, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) AddMaintenanceWindow(ctx context.Context, mw models.MaintenanceWindow) error {
|
||||||
|
if mw.StartTime.IsZero() {
|
||||||
|
mw.StartTime = time.Now()
|
||||||
|
}
|
||||||
|
_, err := s.db.ExecContext(ctx, s.q("INSERT INTO maintenance_windows (monitor_id, title, description, type, start_time, end_time, created_by) VALUES (?, ?, ?, ?, ?, ?, ?)"),
|
||||||
|
mw.MonitorID, mw.Title, mw.Description, mw.Type, mw.StartTime, sql.NullTime{Time: mw.EndTime, Valid: !mw.EndTime.IsZero()}, mw.CreatedBy)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) EndMaintenanceWindow(ctx context.Context, id int) error {
|
||||||
|
_, err := s.db.ExecContext(ctx, s.q("UPDATE maintenance_windows SET end_time = CURRENT_TIMESTAMP WHERE id = ?"), id)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) DeleteMaintenanceWindow(ctx context.Context, id int) error {
|
||||||
|
_, err := s.db.ExecContext(ctx, s.q("DELETE FROM maintenance_windows WHERE id = ?"), id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.dialect.ResetSequenceOnEmpty(s.db, "maintenance_windows")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) PruneExpiredMaintenanceWindows(ctx context.Context, retention time.Duration) (int64, error) {
|
||||||
|
cutoff := time.Now().Add(-retention)
|
||||||
|
result, err := s.db.ExecContext(ctx,
|
||||||
|
s.q("DELETE FROM maintenance_windows WHERE end_time IS NOT NULL AND end_time < ?"),
|
||||||
|
cutoff,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.RowsAffected()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *SQLStore) IsMonitorInMaintenance(ctx context.Context, monitorID int) (bool, error) {
|
||||||
|
var count int
|
||||||
|
err := s.db.QueryRowContext(ctx, s.q(`SELECT COUNT(*) FROM maintenance_windows
|
||||||
|
WHERE type = 'maintenance'
|
||||||
|
AND start_time <= CURRENT_TIMESTAMP
|
||||||
|
AND (end_time IS NULL OR end_time > CURRENT_TIMESTAMP)
|
||||||
|
AND (monitor_id = 0 OR monitor_id = ?
|
||||||
|
OR monitor_id IN (SELECT parent_id FROM sites WHERE id = ? AND parent_id > 0))`),
|
||||||
|
monitorID, monitorID).Scan(&count)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return count > 0, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPruneExpiredMaintenanceWindows(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
// Expired 10 days ago — should be pruned with 7d retention.
|
||||||
|
old := models.MaintenanceWindow{
|
||||||
|
MonitorID: 0,
|
||||||
|
Title: "Old Window",
|
||||||
|
Type: "maintenance",
|
||||||
|
StartTime: now.Add(-11 * 24 * time.Hour),
|
||||||
|
EndTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
if err := s.AddMaintenanceWindow(context.Background(), old); err != nil {
|
||||||
|
t.Fatalf("AddMaintenanceWindow (old): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expired 1 day ago — within 7d retention, should survive.
|
||||||
|
recent := models.MaintenanceWindow{
|
||||||
|
MonitorID: 0,
|
||||||
|
Title: "Recent Window",
|
||||||
|
Type: "maintenance",
|
||||||
|
StartTime: now.Add(-2 * 24 * time.Hour),
|
||||||
|
EndTime: now.Add(-1 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil {
|
||||||
|
t.Fatalf("AddMaintenanceWindow (recent): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ongoing — no end time, should survive.
|
||||||
|
ongoing := models.MaintenanceWindow{
|
||||||
|
MonitorID: 0,
|
||||||
|
Title: "Ongoing Window",
|
||||||
|
Type: "maintenance",
|
||||||
|
StartTime: now.Add(-1 * time.Hour),
|
||||||
|
}
|
||||||
|
if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil {
|
||||||
|
t.Fatalf("AddMaintenanceWindow (ongoing): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PruneExpiredMaintenanceWindows: %v", err)
|
||||||
|
}
|
||||||
|
if pruned != 1 {
|
||||||
|
t.Errorf("expected 1 pruned, got %d", pruned)
|
||||||
|
}
|
||||||
|
|
||||||
|
all, err := s.GetAllMaintenanceWindows(context.Background(), 100)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetAllMaintenanceWindows: %v", err)
|
||||||
|
}
|
||||||
|
if len(all) != 2 {
|
||||||
|
t.Fatalf("expected 2 remaining windows, got %d", len(all))
|
||||||
|
}
|
||||||
|
for _, w := range all {
|
||||||
|
if w.Title == "Old Window" {
|
||||||
|
t.Error("old window should have been pruned")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetOverlappingMaintenanceWindows(t *testing.T) {
|
||||||
|
s := newTestStore(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30}
|
||||||
|
if err := s.AddSite(ctx, site); err != nil {
|
||||||
|
t.Fatalf("AddSite: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
active := models.MaintenanceWindow{
|
||||||
|
MonitorID: 1,
|
||||||
|
Title: "Deploy v2",
|
||||||
|
Type: "maintenance",
|
||||||
|
StartTime: now.Add(-30 * time.Minute),
|
||||||
|
EndTime: now.Add(30 * time.Minute),
|
||||||
|
}
|
||||||
|
if err := s.AddMaintenanceWindow(ctx, active); err != nil {
|
||||||
|
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ended := models.MaintenanceWindow{
|
||||||
|
MonitorID: 1,
|
||||||
|
Title: "Old deploy",
|
||||||
|
Type: "maintenance",
|
||||||
|
StartTime: now.Add(-3 * time.Hour),
|
||||||
|
EndTime: now.Add(-2 * time.Hour),
|
||||||
|
}
|
||||||
|
if err := s.AddMaintenanceWindow(ctx, ended); err != nil {
|
||||||
|
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("same monitor overlaps", func(t *testing.T) {
|
||||||
|
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(overlaps) != 1 {
|
||||||
|
t.Fatalf("expected 1 overlap, got %d", len(overlaps))
|
||||||
|
}
|
||||||
|
if overlaps[0].Title != "Deploy v2" {
|
||||||
|
t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("different monitor no overlap", func(t *testing.T) {
|
||||||
|
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(overlaps) != 0 {
|
||||||
|
t.Errorf("expected 0 overlaps, got %d", len(overlaps))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("global window overlaps all", func(t *testing.T) {
|
||||||
|
global := models.MaintenanceWindow{
|
||||||
|
MonitorID: 0,
|
||||||
|
Title: "Global freeze",
|
||||||
|
Type: "maintenance",
|
||||||
|
StartTime: now.Add(-10 * time.Minute),
|
||||||
|
EndTime: now.Add(2 * time.Hour),
|
||||||
|
}
|
||||||
|
if err := s.AddMaintenanceWindow(ctx, global); err != nil {
|
||||||
|
t.Fatalf("AddMaintenanceWindow: %v", err)
|
||||||
|
}
|
||||||
|
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(overlaps) != 2 {
|
||||||
|
t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("indefinite window overlaps", func(t *testing.T) {
|
||||||
|
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(overlaps) < 1 {
|
||||||
|
t.Error("expected at least 1 overlap for indefinite window")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ended window excluded", func(t *testing.T) {
|
||||||
|
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(overlaps) != 0 {
|
||||||
|
t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -3,9 +3,7 @@ package store
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
"gitea.lerkolabs.com/lerkolabs/uptop/internal/models"
|
||||||
)
|
)
|
||||||
@@ -74,60 +72,6 @@ func TestSiteCRUD(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAlertCRUD(t *testing.T) {
|
|
||||||
s := newTestStore(t)
|
|
||||||
|
|
||||||
if err := s.AddAlert(context.Background(), "Discord", "discord", map[string]string{"url": "https://example.com/hook"}); err != nil {
|
|
||||||
t.Fatalf("AddAlert: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
alerts, err := s.GetAllAlerts(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetAllAlerts: %v", err)
|
|
||||||
}
|
|
||||||
if len(alerts) != 1 {
|
|
||||||
t.Fatalf("expected 1 alert, got %d", len(alerts))
|
|
||||||
}
|
|
||||||
if alerts[0].Type != "discord" {
|
|
||||||
t.Errorf("expected type 'discord', got '%s'", alerts[0].Type)
|
|
||||||
}
|
|
||||||
if alerts[0].Settings["url"] != "https://example.com/hook" {
|
|
||||||
t.Errorf("settings url mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
a, err := s.GetAlert(context.Background(), alerts[0].ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetAlert: %v", err)
|
|
||||||
}
|
|
||||||
if a.Name != "Discord" {
|
|
||||||
t.Errorf("expected name 'Discord', got '%s'", a.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.UpdateAlert(context.Background(), a.ID, "Slack", "slack", map[string]string{"url": "https://slack.com/hook"}); err != nil {
|
|
||||||
t.Fatalf("UpdateAlert: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
a, err = s.GetAlert(context.Background(), a.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetAlert: %v", err)
|
|
||||||
}
|
|
||||||
if a.Type != "slack" {
|
|
||||||
t.Errorf("expected type 'slack', got '%s'", a.Type)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.DeleteAlert(context.Background(), a.ID); err != nil {
|
|
||||||
t.Fatalf("DeleteAlert: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
alerts, err = s.GetAllAlerts(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetAllAlerts: %v", err)
|
|
||||||
}
|
|
||||||
if len(alerts) != 0 {
|
|
||||||
t.Fatalf("expected 0 alerts after delete, got %d", len(alerts))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUserCRUD(t *testing.T) {
|
func TestUserCRUD(t *testing.T) {
|
||||||
s := newTestStore(t)
|
s := newTestStore(t)
|
||||||
|
|
||||||
@@ -301,101 +245,6 @@ func TestImportData_NilUsersPreservesExisting(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCheckHistory(t *testing.T) {
|
|
||||||
s := newTestStore(t)
|
|
||||||
|
|
||||||
if err := s.SaveCheck(context.Background(), 1, 5000000, true); err != nil {
|
|
||||||
t.Fatalf("SaveCheck: %v", err)
|
|
||||||
}
|
|
||||||
if err := s.SaveCheck(context.Background(), 1, 10000000, false); err != nil {
|
|
||||||
t.Fatalf("SaveCheck: %v", err)
|
|
||||||
}
|
|
||||||
if err := s.SaveCheck(context.Background(), 2, 3000000, true); err != nil {
|
|
||||||
t.Fatalf("SaveCheck site 2: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
history, err := s.LoadAllHistory(context.Background(), 10)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadAllHistory: %v", err)
|
|
||||||
}
|
|
||||||
if len(history[1]) != 2 {
|
|
||||||
t.Fatalf("expected 2 records for site 1, got %d", len(history[1]))
|
|
||||||
}
|
|
||||||
if len(history[2]) != 1 {
|
|
||||||
t.Fatalf("expected 1 record for site 2, got %d", len(history[2]))
|
|
||||||
}
|
|
||||||
|
|
||||||
upCount := 0
|
|
||||||
for _, r := range history[1] {
|
|
||||||
if r.IsUp {
|
|
||||||
upCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if upCount != 1 {
|
|
||||||
t.Errorf("expected 1 up record for site 1, got %d", upCount)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDeleteSiteCascade(t *testing.T) {
|
|
||||||
s := newTestStore(t)
|
|
||||||
|
|
||||||
site := models.SiteConfig{Name: "Cascade Test", URL: "https://example.com", Interval: 30}
|
|
||||||
if err := s.AddSite(context.Background(), site); err != nil {
|
|
||||||
t.Fatalf("AddSite: %v", err)
|
|
||||||
}
|
|
||||||
sites, err := s.GetSites(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetSites: %v", err)
|
|
||||||
}
|
|
||||||
siteID := sites[0].ID
|
|
||||||
|
|
||||||
if err := s.SaveCheck(context.Background(), siteID, 1000, true); err != nil {
|
|
||||||
t.Fatalf("SaveCheck: %v", err)
|
|
||||||
}
|
|
||||||
if err := s.SaveStateChange(context.Background(), siteID, "UP", "DOWN", "timeout"); err != nil {
|
|
||||||
t.Fatalf("SaveStateChange: %v", err)
|
|
||||||
}
|
|
||||||
mw := models.MaintenanceWindow{
|
|
||||||
MonitorID: siteID,
|
|
||||||
Title: "Test MW",
|
|
||||||
Type: "maintenance",
|
|
||||||
StartTime: time.Now(),
|
|
||||||
}
|
|
||||||
if err := s.AddMaintenanceWindow(context.Background(), mw); err != nil {
|
|
||||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.DeleteSite(context.Background(), siteID); err != nil {
|
|
||||||
t.Fatalf("DeleteSite: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
history, err := s.LoadAllHistory(context.Background(), 100)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadAllHistory: %v", err)
|
|
||||||
}
|
|
||||||
if len(history[siteID]) != 0 {
|
|
||||||
t.Errorf("expected 0 check_history rows, got %d", len(history[siteID]))
|
|
||||||
}
|
|
||||||
|
|
||||||
changes, err := s.GetStateChanges(context.Background(), siteID, 100)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStateChanges: %v", err)
|
|
||||||
}
|
|
||||||
if len(changes) != 0 {
|
|
||||||
t.Errorf("expected 0 state_changes rows, got %d", len(changes))
|
|
||||||
}
|
|
||||||
|
|
||||||
windows, err := s.GetActiveMaintenanceWindows(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetActiveMaintenanceWindows: %v", err)
|
|
||||||
}
|
|
||||||
for _, w := range windows {
|
|
||||||
if w.MonitorID == siteID {
|
|
||||||
t.Errorf("orphaned maintenance window found: id=%d", w.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPruneLogs(t *testing.T) {
|
func TestPruneLogs(t *testing.T) {
|
||||||
s := newTestStore(t)
|
s := newTestStore(t)
|
||||||
|
|
||||||
@@ -428,231 +277,3 @@ func TestPruneLogs(t *testing.T) {
|
|||||||
t.Error("oldest log survived prune")
|
t.Error("oldest log survived prune")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPruneCheckHistory(t *testing.T) {
|
|
||||||
s := newTestStore(t)
|
|
||||||
|
|
||||||
for i := 0; i < maxCheckHistory+5; i++ {
|
|
||||||
if err := s.SaveCheck(context.Background(), 1, int64(i), true); err != nil {
|
|
||||||
t.Fatalf("SaveCheck site 1: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 0; i < 3; i++ {
|
|
||||||
if err := s.SaveCheck(context.Background(), 2, int64(i), true); err != nil {
|
|
||||||
t.Fatalf("SaveCheck site 2: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.PruneCheckHistory(context.Background()); err != nil {
|
|
||||||
t.Fatalf("PruneCheckHistory: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
history, err := s.LoadAllHistory(context.Background(), maxCheckHistory*2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LoadAllHistory: %v", err)
|
|
||||||
}
|
|
||||||
if len(history[1]) != maxCheckHistory {
|
|
||||||
t.Errorf("site 1: expected %d rows after prune, got %d", maxCheckHistory, len(history[1]))
|
|
||||||
}
|
|
||||||
if len(history[2]) != 3 {
|
|
||||||
t.Errorf("site 2: expected 3 rows untouched, got %d", len(history[2]))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPruneExpiredMaintenanceWindows(t *testing.T) {
|
|
||||||
s := newTestStore(t)
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
// Expired 10 days ago — should be pruned with 7d retention.
|
|
||||||
old := models.MaintenanceWindow{
|
|
||||||
MonitorID: 0,
|
|
||||||
Title: "Old Window",
|
|
||||||
Type: "maintenance",
|
|
||||||
StartTime: now.Add(-11 * 24 * time.Hour),
|
|
||||||
EndTime: now.Add(-10 * 24 * time.Hour),
|
|
||||||
}
|
|
||||||
if err := s.AddMaintenanceWindow(context.Background(), old); err != nil {
|
|
||||||
t.Fatalf("AddMaintenanceWindow (old): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Expired 1 day ago — within 7d retention, should survive.
|
|
||||||
recent := models.MaintenanceWindow{
|
|
||||||
MonitorID: 0,
|
|
||||||
Title: "Recent Window",
|
|
||||||
Type: "maintenance",
|
|
||||||
StartTime: now.Add(-2 * 24 * time.Hour),
|
|
||||||
EndTime: now.Add(-1 * 24 * time.Hour),
|
|
||||||
}
|
|
||||||
if err := s.AddMaintenanceWindow(context.Background(), recent); err != nil {
|
|
||||||
t.Fatalf("AddMaintenanceWindow (recent): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ongoing — no end time, should survive.
|
|
||||||
ongoing := models.MaintenanceWindow{
|
|
||||||
MonitorID: 0,
|
|
||||||
Title: "Ongoing Window",
|
|
||||||
Type: "maintenance",
|
|
||||||
StartTime: now.Add(-1 * time.Hour),
|
|
||||||
}
|
|
||||||
if err := s.AddMaintenanceWindow(context.Background(), ongoing); err != nil {
|
|
||||||
t.Fatalf("AddMaintenanceWindow (ongoing): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
pruned, err := s.PruneExpiredMaintenanceWindows(context.Background(), 7*24*time.Hour)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("PruneExpiredMaintenanceWindows: %v", err)
|
|
||||||
}
|
|
||||||
if pruned != 1 {
|
|
||||||
t.Errorf("expected 1 pruned, got %d", pruned)
|
|
||||||
}
|
|
||||||
|
|
||||||
all, err := s.GetAllMaintenanceWindows(context.Background(), 100)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetAllMaintenanceWindows: %v", err)
|
|
||||||
}
|
|
||||||
if len(all) != 2 {
|
|
||||||
t.Fatalf("expected 2 remaining windows, got %d", len(all))
|
|
||||||
}
|
|
||||||
for _, w := range all {
|
|
||||||
if w.Title == "Old Window" {
|
|
||||||
t.Error("old window should have been pruned")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetOverlappingMaintenanceWindows(t *testing.T) {
|
|
||||||
s := newTestStore(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
site := models.SiteConfig{Name: "web", URL: "https://example.com", Interval: 30}
|
|
||||||
if err := s.AddSite(ctx, site); err != nil {
|
|
||||||
t.Fatalf("AddSite: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
active := models.MaintenanceWindow{
|
|
||||||
MonitorID: 1,
|
|
||||||
Title: "Deploy v2",
|
|
||||||
Type: "maintenance",
|
|
||||||
StartTime: now.Add(-30 * time.Minute),
|
|
||||||
EndTime: now.Add(30 * time.Minute),
|
|
||||||
}
|
|
||||||
if err := s.AddMaintenanceWindow(ctx, active); err != nil {
|
|
||||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ended := models.MaintenanceWindow{
|
|
||||||
MonitorID: 1,
|
|
||||||
Title: "Old deploy",
|
|
||||||
Type: "maintenance",
|
|
||||||
StartTime: now.Add(-3 * time.Hour),
|
|
||||||
EndTime: now.Add(-2 * time.Hour),
|
|
||||||
}
|
|
||||||
if err := s.AddMaintenanceWindow(ctx, ended); err != nil {
|
|
||||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("same monitor overlaps", func(t *testing.T) {
|
|
||||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(overlaps) != 1 {
|
|
||||||
t.Fatalf("expected 1 overlap, got %d", len(overlaps))
|
|
||||||
}
|
|
||||||
if overlaps[0].Title != "Deploy v2" {
|
|
||||||
t.Errorf("expected 'Deploy v2', got %q", overlaps[0].Title)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("different monitor no overlap", func(t *testing.T) {
|
|
||||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 99, now, now.Add(1*time.Hour))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(overlaps) != 0 {
|
|
||||||
t.Errorf("expected 0 overlaps, got %d", len(overlaps))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("global window overlaps all", func(t *testing.T) {
|
|
||||||
global := models.MaintenanceWindow{
|
|
||||||
MonitorID: 0,
|
|
||||||
Title: "Global freeze",
|
|
||||||
Type: "maintenance",
|
|
||||||
StartTime: now.Add(-10 * time.Minute),
|
|
||||||
EndTime: now.Add(2 * time.Hour),
|
|
||||||
}
|
|
||||||
if err := s.AddMaintenanceWindow(ctx, global); err != nil {
|
|
||||||
t.Fatalf("AddMaintenanceWindow: %v", err)
|
|
||||||
}
|
|
||||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, now.Add(1*time.Hour))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(overlaps) != 2 {
|
|
||||||
t.Errorf("expected 2 overlaps (specific + global), got %d", len(overlaps))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("indefinite window overlaps", func(t *testing.T) {
|
|
||||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now, time.Time{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(overlaps) < 1 {
|
|
||||||
t.Error("expected at least 1 overlap for indefinite window")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("ended window excluded", func(t *testing.T) {
|
|
||||||
overlaps, err := s.GetOverlappingMaintenanceWindows(ctx, 1, now.Add(-4*time.Hour), now.Add(-3*time.Hour))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(overlaps) != 0 {
|
|
||||||
t.Errorf("expected 0 overlaps for past range, got %d", len(overlaps))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// ImportData must encrypt alert settings (like AddAlert/UpdateAlert) so a
|
|
||||||
// restore with UPTOP_ENCRYPTION_KEY set never lands secrets in plaintext.
|
|
||||||
func TestImportData_EncryptsAlertSettings(t *testing.T) {
|
|
||||||
s := newTestStore(t)
|
|
||||||
enc, err := NewEncryptor(strings.Repeat("ab", 32)) // 64 hex chars = 32 bytes
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("NewEncryptor: %v", err)
|
|
||||||
}
|
|
||||||
s.SetEncryptor(enc)
|
|
||||||
|
|
||||||
backup := models.Backup{
|
|
||||||
Alerts: []models.AlertConfig{
|
|
||||||
{ID: 1, Name: "tg", Type: "telegram", Settings: map[string]string{"token": "123:SECRET", "chat_id": "42"}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
if err := s.ImportData(context.Background(), backup); err != nil {
|
|
||||||
t.Fatalf("ImportData: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var raw string
|
|
||||||
if err := s.db.QueryRow("SELECT settings FROM alerts WHERE id = 1").Scan(&raw); err != nil {
|
|
||||||
t.Fatalf("query settings: %v", err)
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(raw, encryptedPrefix) {
|
|
||||||
t.Errorf("imported settings not encrypted: %q", raw)
|
|
||||||
}
|
|
||||||
if strings.Contains(raw, "SECRET") {
|
|
||||||
t.Errorf("plaintext secret found in stored column: %q", raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
alerts, err := s.GetAllAlerts(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetAllAlerts: %v", err)
|
|
||||||
}
|
|
||||||
if len(alerts) != 1 || alerts[0].Settings["token"] != "123:SECRET" {
|
|
||||||
t.Errorf("decrypt round-trip failed: %+v", alerts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user