mirror of
https://github.com/coredns/coredns.git
synced 2026-08-20 23:08:28 -04:00
76056dd2e5
* plugin/shed: add UDP overload protection plugin
UDP responses written back through one listener socket serialize on the
Go runtime's internal fdMutex, which allows at most 2^20-1 concurrent
operations per file descriptor and panics the process when exceeded.
CoreDNS serves UDP with one goroutine per query, all writing through the
shared packet connection, so a sustained overload parks every excess
in-flight query in that wait queue until the process dies with
"too many concurrent operations on a single file or socket". Observed
in production: ~2.8M goroutines and 60GiB RSS before the panic.
The shed plugin makes the panic structurally unreachable. It installs,
via Config.UDPDecorateWriterFunc, a per-socket bounded evict-oldest
stack drained newest-first by a single writer goroutine, so the fd
never sees more than one writer and residual capacity under overload
always goes to the freshest response. While a socket's stack is full,
arriving queries are dropped before any plugin runs. Drops are silent
(the client's resolver retries elsewhere) and counted in
coredns_shed_dropped_total{server, reason}.
plugin/shed/fdmutex_test.go demonstrates the failure and the fix with
one shared flood harness. Two subprocess tests reproduce the exact
runtime panic without the plugin's write discipline - one deterministic
(a held write plus >2^20 queued writers), one with nothing held or
mocked; both exercise the Go runtime rather than the plugin, so they
are gated behind SHED_FLOOD_TEST=1. The counterfactual - the same load
through the plugin's stack, completing with every response accounted
for as written or dropped - runs in every test invocation, including
-race, at 50k responders, and at the full 1.5M with SHED_FLOOD_TEST=1:
SHED_FLOOD_TEST=1 go test ./plugin/shed/
Signed-off-by: Ryan Brewster <rpb@anthropic.com>
* test: add shed e2e test
Query a shed-enabled server over UDP (the plugin's deferred
single-writer path) and TCP (which shed passes through), and check
that coredns_shed_dropped_total is exported with its reason label.
No-Verification-Needed: test-only change
Signed-off-by: Ryan Brewster <rpb@anthropic.com>
---------
Signed-off-by: Ryan Brewster <rpb@anthropic.com>
69 lines
1.8 KiB
Go
69 lines
1.8 KiB
Go
package shed
|
|
|
|
import (
|
|
"github.com/coredns/caddy"
|
|
"github.com/coredns/coredns/core/dnsserver"
|
|
"github.com/coredns/coredns/plugin"
|
|
clog "github.com/coredns/coredns/plugin/pkg/log"
|
|
pkgparse "github.com/coredns/coredns/plugin/pkg/parse"
|
|
"github.com/coredns/coredns/plugin/pkg/transport"
|
|
)
|
|
|
|
const pluginName = "shed"
|
|
|
|
var log = clog.NewWithPlugin(pluginName)
|
|
|
|
// stackDepth is a burst budget, not a knob: ~12-16ms at a typical socket's
|
|
// serialized drain rate; a larger value would only hold staler responses.
|
|
const stackDepth = 1024
|
|
|
|
func init() { plugin.Register(pluginName, setup) }
|
|
|
|
func setup(c *caddy.Controller) error {
|
|
s, err := parse(c)
|
|
if err != nil {
|
|
return plugin.Error(pluginName, err)
|
|
}
|
|
|
|
// The hook only exists on the plain-DNS UDP path; on any other transport
|
|
// shed would silently protect nothing, so refuse at parse time. Every
|
|
// block key is checked: caddy propagates the plugin list to all keys of
|
|
// a server block, not just the one setup runs for.
|
|
for _, key := range c.ServerBlockKeys {
|
|
if tr, _ := pkgparse.Transport(key); tr != transport.DNS {
|
|
return plugin.Error(pluginName, c.Errf("only plain DNS server blocks are supported; %q uses transport %q", key, tr))
|
|
}
|
|
}
|
|
|
|
cfg := dnsserver.GetConfig(c)
|
|
cfg.UDPDecorateWriterFunc = s.decorateWriterFactory
|
|
|
|
// On a graceful reload the retiring instance must stop its writer
|
|
// goroutines; shutdown removes only this instance's sockets.
|
|
c.OnShutdown(s.shutdown)
|
|
|
|
cfg.AddPlugin(func(next plugin.Handler) plugin.Handler {
|
|
s.Next = next
|
|
return s
|
|
})
|
|
return nil
|
|
}
|
|
|
|
func parse(c *caddy.Controller) (*Shed, error) {
|
|
s := &Shed{}
|
|
i := 0
|
|
for c.Next() {
|
|
if i > 0 {
|
|
return nil, plugin.ErrOnce
|
|
}
|
|
i++
|
|
if len(c.RemainingArgs()) != 0 {
|
|
return nil, c.ArgErr()
|
|
}
|
|
if c.NextBlock() {
|
|
return nil, c.Errf("shed takes no options")
|
|
}
|
|
}
|
|
return s, nil
|
|
}
|