feat: proper push monitor lifecycle — PENDING, LATE, DOWN #34

Merged
lerko merged 1 commits from feat/push-monitor-states into main 2026-05-28 00:01:57 +00:00
5 changed files with 92 additions and 24 deletions
Showing only changes of commit 5dc31108f8 - Show all commits
+42 -11
View File
@@ -19,8 +19,8 @@ import (
const (
maxLogEntries = 100
pollInterval = 5 * time.Second
pushGracePeriod = 5 * time.Second
minCheckInterval = 5
minPushGrace = 60 * time.Second
)
type Engine struct {
@@ -186,17 +186,34 @@ func (e *Engine) RecordHeartbeat(token string) bool {
return false
}
prevStatus := site.Status
site.LastCheck = time.Now()
wasDown := site.Status == "DOWN"
site.Status = "UP"
site.FailureCount = 0
site.Latency = 0
site.LastError = ""
site.LastSuccessAt = time.Now()
if prevStatus != "UP" {
site.StatusChangedAt = time.Now()
}
e.liveState[targetID] = site
if wasDown {
switch prevStatus {
case "PENDING":
e.AddLog(fmt.Sprintf("Push Monitor '%s' received first heartbeat", site.Name))
case "LATE":
e.AddLog(fmt.Sprintf("Push Monitor '%s' heartbeat arrived (was late)", site.Name))
case "DOWN":
e.AddLog(fmt.Sprintf("Push Monitor '%s' recovered", site.Name))
e.triggerAlert(site.AlertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.", site.Name))
go e.triggerAlert(site.AlertID, "✅ RECOVERY", fmt.Sprintf("Push Monitor '%s' is receiving heartbeats.", site.Name))
}
if prevStatus != "UP" && prevStatus != "PENDING" {
go func() { _ = e.db.SaveStateChange(targetID, prevStatus, "UP", "") }()
}
return true
}
@@ -241,9 +258,6 @@ func (e *Engine) Start(ctx context.Context) {
if !exists {
e.mu.Lock()
s.Status = "PENDING"
if s.Type == "push" {
s.LastCheck = time.Now()
}
if h, ok := e.GetHistory(s.ID); ok && len(h.Statuses) > 0 {
if h.Statuses[len(h.Statuses)-1] {
s.Status = "UP"
@@ -401,11 +415,28 @@ func (e *Engine) checkByID(id int) {
}
func (e *Engine) checkPush(site models.Site) {
deadline := site.LastCheck.Add(time.Duration(site.Interval) * time.Second).Add(pushGracePeriod)
if time.Now().After(deadline) {
if site.Status == "PENDING" {
return
}
interval := time.Duration(site.Interval) * time.Second
grace := interval / 2
if grace < minPushGrace {
grace = minPushGrace
}
overdue := site.LastCheck.Add(interval)
graceEnd := overdue.Add(grace)
now := time.Now()
if now.After(graceEnd) {
if site.Status != "DOWN" {
e.handleStatusChange(site, "DOWN", 0, 0, "heartbeat missed")
} else if site.Status != "UP" {
e.handleStatusChange(site, "UP", 200, 0, "")
}
} else if now.After(overdue) {
if site.Status != "LATE" {
e.handleStatusChange(site, "LATE", 0, 0, "heartbeat overdue")
}
}
}
+23 -5
View File
@@ -537,7 +537,7 @@ func TestCheckPush_DeadlineMissed(t *testing.T) {
site := models.Site{
ID: 1, Name: "push", Type: "push", Status: "UP",
Interval: 10, MaxRetries: 0,
LastCheck: time.Now().Add(-20 * time.Second),
LastCheck: time.Now().Add(-120 * time.Second),
}
injectSite(e, site)
@@ -549,6 +549,24 @@ func TestCheckPush_DeadlineMissed(t *testing.T) {
}
}
func TestCheckPush_OverdueBecomesLate(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
ID: 1, Name: "push", Type: "push", Status: "UP",
Interval: 300,
LastCheck: time.Now().Add(-310 * time.Second),
}
injectSite(e, site)
e.checkPush(site)
s, _ := getSite(e, 1)
if s.Status != "LATE" {
t.Errorf("expected LATE when overdue but within grace, got %s", s.Status)
}
}
func TestCheckPush_WithinDeadline(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
@@ -566,20 +584,20 @@ func TestCheckPush_WithinDeadline(t *testing.T) {
}
}
func TestCheckPush_PendingToUp(t *testing.T) {
func TestCheckPush_PendingStaysPending(t *testing.T) {
ms := newMockStore()
e := newTestEngine(ms)
site := models.Site{
ID: 1, Name: "push", Type: "push", Status: "PENDING",
Interval: 60, LastCheck: time.Now(),
Interval: 60,
}
injectSite(e, site)
e.checkPush(site)
s, _ := getSite(e, 1)
if s.Status != "UP" {
t.Errorf("expected UP, got %s", s.Status)
if s.Status != "PENDING" {
t.Errorf("expected PENDING to stay until first heartbeat, got %s", s.Status)
}
}
+1
View File
@@ -67,6 +67,7 @@ var statusTpl = template.Must(template.New("status").Parse(`
.UP { background: #9ece6a; color: #1a1b26; }
.DOWN { background: #f7768e; color: #1a1b26; }
.PENDING { background: #e0af68; color: #1a1b26; }
.LATE { background: #e0af68; color: #1a1b26; }
.SSL-EXP { background: #e0af68; color: #1a1b26; }
.PAUSED { background: #565f89; color: #c0caf5; }
.MAINT { background: #bb9af7; color: #1a1b26; }
+4 -2
View File
@@ -302,6 +302,8 @@ func fmtStatus(status string, paused bool, inMaint bool) string {
switch status {
case "DOWN", "SSL EXP":
return dangerStyle.Render(status)
case "LATE":
return warnStyle.Render(status)
case "PENDING":
return subtleStyle.Render(status)
default:
@@ -412,7 +414,7 @@ func (m Model) viewSitesTab() string {
name = limitStr(name, nameW)
}
if (site.Status == "DOWN" || site.Status == "SSL EXP") && site.LastError != "" {
if (site.Status == "DOWN" || site.Status == "SSL EXP" || site.Status == "LATE") && site.LastError != "" {
nameLen := len([]rune(name))
errSpace := nameW - nameLen - 1
if errSpace > 10 {
@@ -764,7 +766,7 @@ func (m Model) viewDetailPanel() string {
row("Status", fmtStatus(site.Status, site.Paused, m.isMonitorInMaintenance(site.ID)))
if (site.Status == "DOWN" || site.Status == "SSL EXP") && site.LastError != "" {
if (site.Status == "DOWN" || site.Status == "SSL EXP" || site.Status == "LATE") && site.LastError != "" {
row("Error", dangerStyle.Render(limitStr(site.LastError, 60)))
}
+21 -5
View File
@@ -811,13 +811,20 @@ func (m Model) viewDashboard() string {
allSites := m.engine.GetAllSites()
totalMonitors := 0
downCount := 0
lateCount := 0
for _, s := range allSites {
if s.Type == "group" {
continue
}
totalMonitors++
if !s.Paused && !m.isMonitorInMaintenance(s.ID) && (s.Status == "DOWN" || s.Status == "SSL EXP") {
if s.Paused || m.isMonitorInMaintenance(s.ID) {
continue
}
switch s.Status {
case "DOWN", "SSL EXP":
downCount++
case "LATE":
lateCount++
}
}
offlineNodes := 0
@@ -830,6 +837,8 @@ func (m Model) viewDashboard() string {
var sitesLabel string
if downCount > 0 {
sitesLabel = fmt.Sprintf("Sites (%d↓)", downCount)
} else if lateCount > 0 {
sitesLabel = fmt.Sprintf("Sites (%d⚠)", lateCount)
} else if totalMonitors > 0 {
sitesLabel = fmt.Sprintf("Sites (%d)", totalMonitors)
} else {
@@ -895,14 +904,19 @@ func (m Model) viewDashboard() string {
}
}
upCount := totalMonitors - downCount
upCount := totalMonitors - downCount - lateCount
var upStr string
if downCount > 0 {
upStr = dangerStyle.Render(fmt.Sprintf("%d/%d UP", upCount, totalMonitors))
} else if lateCount > 0 {
upStr = warnStyle.Render(fmt.Sprintf("%d/%d UP", upCount, totalMonitors))
} else {
upStr = specialStyle.Render(fmt.Sprintf("%d/%d UP", upCount, totalMonitors))
}
statusParts := []string{upStr}
if lateCount > 0 {
statusParts = append(statusParts, warnStyle.Render(fmt.Sprintf("%d LATE", lateCount)))
}
if len(m.nodes) > 0 {
online := 0
for _, n := range m.nodes {
@@ -949,10 +963,12 @@ func siteOrder(s models.Site) int {
switch s.Status {
case "DOWN", "SSL EXP":
return 0
case "PENDING":
return 2
default:
case "LATE":
return 1
case "PENDING":
return 3
default:
return 2
}
}