Caddyview/internal/parser/caddy.go
SW 2be12c24cd perf: server-side awk aggregation for history views
High-traffic servers (100+ req/s) generated millions of log lines
per day. Transferring raw lines over SSH was impractical (GB of data).

Instead, run an awk script on the server that aggregates all matching
log files (including .gz rotated ones) into time buckets and returns
only a ~50-line summary. For a server doing 180 req/s this reduces
SSH transfer from ~7GB to <1KB per history request.

The awk script filters by timestamp, groups by bucket, and outputs:
- per-bucket request counts, bytes, and status code groups
- top hosts and IPs with counts
- overall totals

Files are pre-filtered with find -mtime to skip archives older than
the requested window.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-11 14:50:51 +02:00

260 lines
6.1 KiB
Go

package parser
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
)
// LogEntry represents one Caddy JSON access log line.
type LogEntry struct {
Timestamp float64 `json:"ts"`
Level string `json:"level"`
Logger string `json:"logger"`
Request struct {
Method string `json:"method"`
URI string `json:"uri"`
Host string `json:"host"`
Proto string `json:"proto"`
RemoteIP string `json:"remote_ip"`
} `json:"request"`
Duration float64 `json:"duration"`
Size int64 `json:"size"`
Status int `json:"status"`
}
func (e *LogEntry) Time() time.Time {
sec := int64(e.Timestamp)
nsec := int64((e.Timestamp - float64(sec)) * 1e9)
return time.Unix(sec, nsec)
}
// ParseLine parses a single Caddy JSON log line. Returns nil if not a request log.
func ParseLine(line string) *LogEntry {
line = strings.TrimSpace(line)
if line == "" || !strings.HasPrefix(line, "{") {
return nil
}
var entry LogEntry
if err := json.Unmarshal([]byte(line), &entry); err != nil {
return nil
}
if entry.Request.Host == "" || entry.Status == 0 {
return nil
}
return &entry
}
// ParseLines parses multiple lines and returns valid entries.
func ParseLines(lines []string) []*LogEntry {
entries := make([]*LogEntry, 0, len(lines))
for _, l := range lines {
if e := ParseLine(l); e != nil {
entries = append(entries, e)
}
}
return entries
}
// -- Stats (used for live view and derived from AggregatedHistory for history) --
type Stats struct {
TotalRequests int64
TotalBytes int64
StatusCodes map[int]int64
TopHosts map[string]int64
TopIPs map[string]int64
AvgDurationMs float64
durationSum float64
durationN int64
}
func NewStats() *Stats {
return &Stats{
StatusCodes: make(map[int]int64),
TopHosts: make(map[string]int64),
TopIPs: make(map[string]int64),
}
}
func (s *Stats) Add(e *LogEntry) {
s.TotalRequests++
s.TotalBytes += e.Size
s.StatusCodes[e.Status]++
s.TopHosts[e.Request.Host]++
s.TopIPs[e.Request.RemoteIP]++
s.durationSum += e.Duration * 1000
s.durationN++
if s.durationN > 0 {
s.AvgDurationMs = s.durationSum / float64(s.durationN)
}
}
func (s *Stats) BytesPerSec(windowSecs float64) float64 {
if windowSecs <= 0 {
return 0
}
return float64(s.TotalBytes) / windowSecs
}
func ComputeStats(entries []*LogEntry) *Stats {
s := NewStats()
for _, e := range entries {
s.Add(e)
}
return s
}
// -- Bucket (time-series bar chart data) --------------------------------------
type Bucket struct {
Time time.Time
Requests int64
Bytes int64
}
func Bucketize(entries []*LogEntry, bucketSize time.Duration) []Bucket {
if len(entries) == 0 {
return nil
}
buckets := map[int64]*Bucket{}
for _, e := range entries {
t := e.Time().Truncate(bucketSize)
key := t.Unix()
if _, ok := buckets[key]; !ok {
buckets[key] = &Bucket{Time: t}
}
buckets[key].Requests++
buckets[key].Bytes += e.Size
}
result := make([]Bucket, 0, len(buckets))
for _, b := range buckets {
result = append(result, *b)
}
return result
}
// -- AggregatedHistory (server-side aggregation result) -----------------------
// AggregatedBucket is one time bucket returned by server-side awk aggregation.
type AggregatedBucket struct {
Time time.Time
Requests, Bytes int64
Status2xx, Status3xx, Status4xx, Status5xx int64
}
// AggregatedHistory holds the full result of a server-side history aggregation.
type AggregatedHistory struct {
Buckets []AggregatedBucket
TopHosts map[string]int64
TopIPs map[string]int64
TotalReqs int64
TotalBytes int64
}
// ToStats converts aggregated history to a Stats object for use in the TUI panels.
func (h *AggregatedHistory) ToStats() *Stats {
s := NewStats()
s.TotalRequests = h.TotalReqs
s.TotalBytes = h.TotalBytes
s.TopHosts = h.TopHosts
s.TopIPs = h.TopIPs
for _, b := range h.Buckets {
s.StatusCodes[200] += b.Status2xx
s.StatusCodes[301] += b.Status3xx
s.StatusCodes[404] += b.Status4xx
s.StatusCodes[500] += b.Status5xx
}
return s
}
// ToBuckets converts aggregated buckets to the Bucket slice used by the bar chart.
func (h *AggregatedHistory) ToBuckets() []Bucket {
out := make([]Bucket, len(h.Buckets))
for i, ab := range h.Buckets {
out[i] = Bucket{Time: ab.Time, Requests: ab.Requests, Bytes: ab.Bytes}
}
return out
}
// ParseAggregatedOutput parses the output of the server-side awk aggregation script.
//
// Format:
//
// BUCKETS
// <unix_ts> <count> <bytes> <2xx> <3xx> <4xx> <5xx>
// HOSTS
// <count> <host>
// IPS
// <count> <ip>
// TOTALS
// <total_reqs> <total_bytes>
func ParseAggregatedOutput(raw string) (*AggregatedHistory, error) {
h := &AggregatedHistory{
TopHosts: make(map[string]int64),
TopIPs: make(map[string]int64),
}
section := ""
for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
switch line {
case "BUCKETS", "HOSTS", "IPS", "TOTALS":
section = line
continue
}
fields := strings.Fields(line)
switch section {
case "BUCKETS":
if len(fields) < 7 {
continue
}
ts, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil {
continue
}
ab := AggregatedBucket{Time: time.Unix(ts, 0)}
ab.Requests, _ = strconv.ParseInt(fields[1], 10, 64)
ab.Bytes, _ = strconv.ParseInt(fields[2], 10, 64)
ab.Status2xx, _ = strconv.ParseInt(fields[3], 10, 64)
ab.Status3xx, _ = strconv.ParseInt(fields[4], 10, 64)
ab.Status4xx, _ = strconv.ParseInt(fields[5], 10, 64)
ab.Status5xx, _ = strconv.ParseInt(fields[6], 10, 64)
h.Buckets = append(h.Buckets, ab)
case "HOSTS":
if len(fields) < 2 {
continue
}
count, _ := strconv.ParseInt(fields[0], 10, 64)
host := strings.Join(fields[1:], " ")
h.TopHosts[host] = count
case "IPS":
if len(fields) < 2 {
continue
}
count, _ := strconv.ParseInt(fields[0], 10, 64)
h.TopIPs[fields[1]] = count
case "TOTALS":
if len(fields) < 2 {
continue
}
h.TotalReqs, _ = strconv.ParseInt(fields[0], 10, 64)
h.TotalBytes, _ = strconv.ParseInt(fields[1], 10, 64)
}
}
if section == "" {
return nil, fmt.Errorf("no aggregation output received from server")
}
return h, nil
}