Initial structure
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/jasper/quptime/internal/config"
|
||||
)
|
||||
|
||||
// discordTimeout caps how long a single webhook POST is allowed to
|
||||
// take.
|
||||
const discordTimeout = 10 * time.Second
|
||||
|
||||
// discordPayload is the minimum shape the Discord webhook API
|
||||
// accepts. We do not use embeds — plain text keeps the payload
|
||||
// trivial to read in operator-side logs.
|
||||
type discordPayload struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
// sendDiscord posts msg.Subject + body to the configured webhook URL.
|
||||
func sendDiscord(a *config.Alert, msg Message) error {
|
||||
if a.DiscordWebhook == "" {
|
||||
return errors.New("discord webhook url not set")
|
||||
}
|
||||
|
||||
content := msg.Subject + "\n```\n" + msg.Body + "\n```"
|
||||
raw, err := json.Marshal(discordPayload{Content: content})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), discordTimeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.DiscordWebhook, bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("discord webhook: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
return fmt.Errorf("discord webhook status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/jasper/quptime/internal/checks"
|
||||
"github.com/jasper/quptime/internal/config"
|
||||
)
|
||||
|
||||
// Dispatcher fans an aggregator transition out to every alert listed
|
||||
// on the check. Errors are logged but never propagated: alerting must
|
||||
// not block the aggregation pipeline.
|
||||
type Dispatcher struct {
|
||||
cluster *config.ClusterConfig
|
||||
selfID string
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// New constructs a Dispatcher.
|
||||
func New(cluster *config.ClusterConfig, selfID string, logger *log.Logger) *Dispatcher {
|
||||
if logger == nil {
|
||||
logger = log.Default()
|
||||
}
|
||||
return &Dispatcher{cluster: cluster, selfID: selfID, logger: logger}
|
||||
}
|
||||
|
||||
// OnTransition is wired as checks.TransitionFn.
|
||||
func (d *Dispatcher) OnTransition(check *config.Check, from, to checks.State, snap checks.Snapshot) {
|
||||
if to == checks.StateUnknown {
|
||||
return
|
||||
}
|
||||
msg := Render(d.selfID, check, from, to, snap)
|
||||
for _, alertID := range check.AlertIDs {
|
||||
alert, _ := d.cluster.FindAlert(alertID)
|
||||
if alert == nil {
|
||||
d.logger.Printf("alerts: check %q references unknown alert %q", check.Name, alertID)
|
||||
continue
|
||||
}
|
||||
if err := d.dispatchOne(alert, msg); err != nil {
|
||||
d.logger.Printf("alerts: %q via %s: %v", alert.Name, alert.Type, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test sends a one-shot test message to the named alert. Returns an
|
||||
// error so the CLI can surface failures interactively.
|
||||
func (d *Dispatcher) Test(alertID string) error {
|
||||
alert, _ := d.cluster.FindAlert(alertID)
|
||||
if alert == nil {
|
||||
return fmt.Errorf("alert %q not found", alertID)
|
||||
}
|
||||
msg := Message{
|
||||
Subject: "[quptime] test alert",
|
||||
Body: fmt.Sprintf("This is a test of alert %q from node %s.\nIf you see this, the alert channel is wired correctly.\n", alert.Name, d.selfID),
|
||||
}
|
||||
return d.dispatchOne(alert, msg)
|
||||
}
|
||||
|
||||
func (d *Dispatcher) dispatchOne(a *config.Alert, msg Message) error {
|
||||
switch a.Type {
|
||||
case config.AlertSMTP:
|
||||
return sendSMTP(a, msg)
|
||||
case config.AlertDiscord:
|
||||
return sendDiscord(a, msg)
|
||||
default:
|
||||
return fmt.Errorf("unknown alert type %q", a.Type)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// Package alerts dispatches state-transition notifications to the
|
||||
// configured channels (SMTP, Discord). The aggregator owns hysteresis
|
||||
// so this package fires exactly one message per UP↔DOWN flip.
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jasper/quptime/internal/checks"
|
||||
"github.com/jasper/quptime/internal/config"
|
||||
)
|
||||
|
||||
// Message is the rendered notification ready to ship across any
|
||||
// channel. Channels may format Subject + Body differently (SMTP uses
|
||||
// both; Discord renders a single string).
|
||||
type Message struct {
|
||||
Subject string
|
||||
Body string
|
||||
}
|
||||
|
||||
// Render produces a human-readable message from one state transition.
|
||||
func Render(nodeID string, check *config.Check, from, to checks.State, snap checks.Snapshot) Message {
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
verb := transitionVerb(from, to)
|
||||
subject := fmt.Sprintf("[quptime] %s %s — %s", check.Name, verb, check.Target)
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Check %q is now %s.\n", check.Name, strings.ToUpper(string(to)))
|
||||
fmt.Fprintf(&b, "Previous state: %s\n", from)
|
||||
fmt.Fprintf(&b, "Target: %s (%s)\n", check.Target, check.Type)
|
||||
fmt.Fprintf(&b, "Reports: %d (ok=%d, fail=%d)\n", snap.Reports, snap.OKCount, snap.NotOK)
|
||||
if snap.Detail != "" {
|
||||
fmt.Fprintf(&b, "Detail: %s\n", snap.Detail)
|
||||
}
|
||||
fmt.Fprintf(&b, "Master: %s\n", nodeID)
|
||||
fmt.Fprintf(&b, "When: %s\n", now)
|
||||
return Message{Subject: subject, Body: b.String()}
|
||||
}
|
||||
|
||||
func transitionVerb(from, to checks.State) string {
|
||||
switch to {
|
||||
case checks.StateDown:
|
||||
return "DOWN"
|
||||
case checks.StateUp:
|
||||
if from == checks.StateDown {
|
||||
return "RECOVERED"
|
||||
}
|
||||
return "UP"
|
||||
}
|
||||
return strings.ToUpper(string(to))
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/smtp"
|
||||
"strings"
|
||||
|
||||
"github.com/jasper/quptime/internal/config"
|
||||
)
|
||||
|
||||
// sendSMTP delivers msg through the alert's SMTP relay. STARTTLS is
|
||||
// negotiated whenever the alert has SMTPStartTLS true; the smtp
|
||||
// server is responsible for advertising the extension.
|
||||
func sendSMTP(a *config.Alert, msg Message) error {
|
||||
if a.SMTPHost == "" || a.SMTPPort == 0 {
|
||||
return errors.New("smtp host/port not set")
|
||||
}
|
||||
if a.SMTPFrom == "" || len(a.SMTPTo) == 0 {
|
||||
return errors.New("smtp from/to not set")
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", a.SMTPHost, a.SMTPPort)
|
||||
client, err := smtp.Dial(addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial smtp: %w", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
if a.SMTPStartTLS {
|
||||
if ok, _ := client.Extension("STARTTLS"); !ok {
|
||||
return errors.New("server does not support STARTTLS")
|
||||
}
|
||||
if err := client.StartTLS(&tls.Config{ServerName: a.SMTPHost, MinVersion: tls.VersionTLS12}); err != nil {
|
||||
return fmt.Errorf("starttls: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if a.SMTPUser != "" {
|
||||
auth := smtp.PlainAuth("", a.SMTPUser, a.SMTPPassword, a.SMTPHost)
|
||||
if err := client.Auth(auth); err != nil {
|
||||
return fmt.Errorf("smtp auth: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := client.Mail(a.SMTPFrom); err != nil {
|
||||
return fmt.Errorf("mail from: %w", err)
|
||||
}
|
||||
for _, rcpt := range a.SMTPTo {
|
||||
if err := client.Rcpt(rcpt); err != nil {
|
||||
return fmt.Errorf("rcpt %s: %w", rcpt, err)
|
||||
}
|
||||
}
|
||||
w, err := client.Data()
|
||||
if err != nil {
|
||||
return fmt.Errorf("data: %w", err)
|
||||
}
|
||||
if _, err := w.Write(buildRFC822(a.SMTPFrom, a.SMTPTo, msg)); err != nil {
|
||||
return fmt.Errorf("write body: %w", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return fmt.Errorf("close body: %w", err)
|
||||
}
|
||||
return client.Quit()
|
||||
}
|
||||
|
||||
func buildRFC822(from string, to []string, msg Message) []byte {
|
||||
var sb strings.Builder
|
||||
fmt.Fprintf(&sb, "From: %s\r\n", from)
|
||||
fmt.Fprintf(&sb, "To: %s\r\n", strings.Join(to, ", "))
|
||||
fmt.Fprintf(&sb, "Subject: %s\r\n", msg.Subject)
|
||||
fmt.Fprintf(&sb, "MIME-Version: 1.0\r\n")
|
||||
fmt.Fprintf(&sb, "Content-Type: text/plain; charset=UTF-8\r\n")
|
||||
fmt.Fprintf(&sb, "\r\n")
|
||||
sb.WriteString(msg.Body)
|
||||
return []byte(sb.String())
|
||||
}
|
||||
Reference in New Issue
Block a user