test: poll instead of fixed sleeps across several tests (#8275)

Several tests slept a fixed duration then took a single snapshot of an
async result, racing whatever they waited on:
- forward health tests waited 20ms for the health-check goroutine to bump
  an atomic counter,
- the auto plugin tests (dns + metrics) waited 50-110ms for a file-watch
  reload to be picked up,
- the file ZoneReload test waited 30ms (self-described as could still be
  racy) for a reload,
- the overloaded health test slept 1s for its background goroutine to fire
  its first request.

Replace each with a bounded poll of the actual condition (the atomic
counter, a dns.Exchange response, a metrics scrape, z.ApexIfDefined, or a
channel signalled by the test's own handler) so they pass as soon as the
awaited state is reached and no longer flake when it is slower than the
fixed wait. Test-only.

Signed-off-by: Nikolaus Schuetz <nikolauspschuetz@gmail.com>

plugin/file/reload_test.go

@@ -58,12 +58,15 @@ func TestZoneReload(t *testing.T) {
 	if err := os.WriteFile(fileName, []byte(reloadZone2Test), 0644); err != nil {
 		t.Fatalf("Failed to write new zone data: %s", err)
 	}
-	// Could still be racy, but we need to wait a bit for the event to be seen
-	time.Sleep(30 * time.Millisecond)
-
-	rrs, err = z.ApexIfDefined()
-	if err != nil {
-		t.Fatal(err)
+	for start := time.Now(); time.Since(start) < 2*time.Second; {
+		rrs, err = z.ApexIfDefined()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(rrs) == 3 {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
 	}
 	if len(rrs) != 3 {
 		t.Fatalf("Expected 3 RRs, got %d", len(rrs))

plugin/forward/health_test.go

@@ -47,8 +47,14 @@ func TestHealth(t *testing.T) {
 
 	f.ServeDNS(context.TODO(), &test.ResponseWriter{}, req)
 
-	time.Sleep(20 * time.Millisecond)
-	i1 := atomic.LoadUint32(&i)
+	i1 := uint32(0)
+	for start := time.Now(); time.Since(start) < 2*time.Second; {
+		i1 = atomic.LoadUint32(&i)
+		if i1 == 1 {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
 	if i1 != 1 {
 		t.Errorf("Expected number of health checks with RecursionDesired==true to be %d, got %d", 1, i1)
 	}
@@ -88,8 +94,14 @@ func TestHealthTCP(t *testing.T) {
 
 	f.ServeDNS(context.TODO(), &test.ResponseWriter{TCP: true}, req)
 
-	time.Sleep(20 * time.Millisecond)
-	i1 := atomic.LoadUint32(&i)
+	i1 := uint32(0)
+	for start := time.Now(); time.Since(start) < 2*time.Second; {
+		i1 = atomic.LoadUint32(&i)
+		if i1 == 1 {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
 	if i1 != 1 {
 		t.Errorf("Expected number of health checks with RecursionDesired==true to be %d, got %d", 1, i1)
 	}
@@ -129,8 +141,14 @@ func TestHealthNoRecursion(t *testing.T) {
 
 	f.ServeDNS(context.TODO(), &test.ResponseWriter{}, req)
 
-	time.Sleep(20 * time.Millisecond)
-	i1 := atomic.LoadUint32(&i)
+	i1 := uint32(0)
+	for start := time.Now(); time.Since(start) < 2*time.Second; {
+		i1 = atomic.LoadUint32(&i)
+		if i1 == 1 {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
 	if i1 != 1 {
 		t.Errorf("Expected number of health checks with RecursionDesired==false to be %d, got %d", 1, i1)
 	}

plugin/health/overloaded_test.go

@@ -10,7 +10,12 @@ import (
 )
 
 func Test_health_overloaded_cancellation(t *testing.T) {
+	started := make(chan struct{}, 1)
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _r *http.Request) {
+		select {
+		case started <- struct{}{}:
+		default:
+		}
 		time.Sleep(1 * time.Second)
 		w.WriteHeader(http.StatusOK)
 	}))
@@ -36,8 +41,11 @@ func Test_health_overloaded_cancellation(t *testing.T) {
 		stopped <- struct{}{}
 	}()
 
-	// wait for overloaded function to start atleast once
-	time.Sleep(1 * time.Second)
+	select {
+	case <-started:
+	case <-time.After(5 * time.Second):
+		t.Fatal("overloaded function did not start")
+	}
 
 	cancel()