Add assert func for verifying status, netmap and netcheck (#1723)

.github/workflows/test-integration-v2-TestPingAllByIPPublicDERP.yaml

@@ -0,0 +1,67 @@
+# DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
+# To regenerate, run "go generate" in cmd/gh-action-integration-generator/
+
+name: Integration Test v2 - TestPingAllByIPPublicDERP
+
+on: [pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  TestPingAllByIPPublicDERP:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - uses: satackey/action-docker-layer-caching@main
+        continue-on-error: true
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v34
+        with:
+          files: |
+            *.nix
+            go.*
+            **/*.go
+            integration_test/
+            config-example.yaml
+
+      - name: Run TestPingAllByIPPublicDERP
+        uses: Wandalen/wretry.action@master
+        if: steps.changed-files.outputs.any_changed == 'true'
+        with:
+          attempt_limit: 5
+          command: |
+            nix develop --command -- docker run \
+              --tty --rm \
+              --volume ~/.cache/hs-integration-go:/go \
+              --name headscale-test-suite \
+              --volume $PWD:$PWD -w $PWD/integration \
+              --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
+              golang:1 \
+                go run gotest.tools/gotestsum@latest -- ./... \
+                  -failfast \
+                  -timeout 120m \
+                  -parallel 1 \
+                  -run "^TestPingAllByIPPublicDERP$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: pprof
+          path: "control_logs/*.pprof.tar"

hscontrol/poll.go

@@ -201,9 +201,15 @@ func (h *Headscale) handlePoll(
 			return
 		}
 
+		// TODO(kradalby): Figure out why patch changes does
+		// not show up in output from `tailscale debug netmap`.
+		// stateUpdate := types.StateUpdate{
+		// 	Type:          types.StatePeerChangedPatch,
+		// 	ChangePatches: []*tailcfg.PeerChange{&change},
+		// }
 		stateUpdate := types.StateUpdate{
-			Type:          types.StatePeerChangedPatch,
+			Type:        types.StatePeerChanged,
-			ChangePatches: []*tailcfg.PeerChange{&change},
+			ChangeNodes: types.Nodes{node},
 		}
 		if stateUpdate.Valid() {
 			ctx := types.NotifyCtx(context.Background(), "poll-nodeupdate-peers-patch", node.Hostname)

hscontrol/types/node_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/juanfont/headscale/hscontrol/util"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
@@ -366,3 +367,110 @@ func TestPeerChangeFromMapRequest(t *testing.T) {
 		})
 	}
 }
+
+func TestApplyPeerChange(t *testing.T) {
+	tests := []struct {
+		name       string
+		nodeBefore Node
+		change     *tailcfg.PeerChange
+		want       Node
+	}{
+		{
+			name:       "hostinfo-and-netinfo-not-exists",
+			nodeBefore: Node{},
+			change: &tailcfg.PeerChange{
+				DERPRegion: 1,
+			},
+			want: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 1,
+					},
+				},
+			},
+		},
+		{
+			name: "hostinfo-netinfo-not-exists",
+			nodeBefore: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+				},
+			},
+			change: &tailcfg.PeerChange{
+				DERPRegion: 3,
+			},
+			want: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 3,
+					},
+				},
+			},
+		},
+		{
+			name: "hostinfo-netinfo-exists-derp-set",
+			nodeBefore: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 999,
+					},
+				},
+			},
+			change: &tailcfg.PeerChange{
+				DERPRegion: 2,
+			},
+			want: Node{
+				Hostinfo: &tailcfg.Hostinfo{
+					Hostname: "test",
+					NetInfo: &tailcfg.NetInfo{
+						PreferredDERP: 2,
+					},
+				},
+			},
+		},
+		{
+			name:       "endpoints-not-set",
+			nodeBefore: Node{},
+			change: &tailcfg.PeerChange{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+			want: Node{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+		},
+		{
+			name: "endpoints-set",
+			nodeBefore: Node{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("6.6.6.6:66"),
+				},
+			},
+			change: &tailcfg.PeerChange{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+			want: Node{
+				Endpoints: []netip.AddrPort{
+					netip.MustParseAddrPort("8.8.8.8:88"),
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.nodeBefore.ApplyPeerChange(tt.change)
+
+			if diff := cmp.Diff(tt.want, tt.nodeBefore, util.Comparers...); diff != "" {
+				t.Errorf("Patch unexpected result (-want +got):\n%s", diff)
+			}
+		})
+	}
+}

hscontrol/util/test.go

@@ -15,6 +15,10 @@ var IPComparer = cmp.Comparer(func(x, y netip.Addr) bool {
 	return x.Compare(y) == 0
 })
 
+var AddrPortComparer = cmp.Comparer(func(x, y netip.AddrPort) bool {
+	return x == y
+})
+
 var MkeyComparer = cmp.Comparer(func(x, y key.MachinePublic) bool {
 	return x.String() == y.String()
 })
@@ -28,5 +32,5 @@ var DkeyComparer = cmp.Comparer(func(x, y key.DiscoPublic) bool {
 })
 
 var Comparers []cmp.Option = []cmp.Option{
-	IPComparer, PrefixComparer, MkeyComparer, NkeyComparer, DkeyComparer,
+	IPComparer, PrefixComparer, AddrPortComparer, MkeyComparer, NkeyComparer, DkeyComparer,
 }

integration/auth_oidc_test.go

@@ -83,6 +83,8 @@ func TestOIDCAuthenticationPingAll(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})
@@ -140,6 +142,8 @@ func TestOIDCExpireNodesBasedOnTokenExpiry(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})

integration/auth_web_flow_test.go

@@ -53,6 +53,8 @@ func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})
@@ -90,6 +92,8 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})

integration/embedded_derp_test.go

@@ -33,20 +33,23 @@ func TestDERPServerScenario(t *testing.T) {
 	defer scenario.Shutdown()
 
 	spec := map[string]int{
-		"user1": len(MustTestVersions),
+		"user1": 10,
+		// "user1": len(MustTestVersions),
 	}
 
-	headscaleConfig := map[string]string{}
+	headscaleConfig := map[string]string{
-	headscaleConfig["HEADSCALE_DERP_URLS"] = ""
+		"HEADSCALE_DERP_URLS":                    "",
-	headscaleConfig["HEADSCALE_DERP_SERVER_ENABLED"] = "true"
+		"HEADSCALE_DERP_SERVER_ENABLED":          "true",
-	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_ID"] = "999"
+		"HEADSCALE_DERP_SERVER_REGION_ID":        "999",
-	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_CODE"] = "headscale"
+		"HEADSCALE_DERP_SERVER_REGION_CODE":      "headscale",
-	headscaleConfig["HEADSCALE_DERP_SERVER_REGION_NAME"] = "Headscale Embedded DERP"
+		"HEADSCALE_DERP_SERVER_REGION_NAME":      "Headscale Embedded DERP",
-	headscaleConfig["HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR"] = "0.0.0.0:3478"
+		"HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478",
-	headscaleConfig["HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH"] = "/tmp/derp.key"
+		"HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key",
-	// Envknob for enabling DERP debug logs
+
-	headscaleConfig["DERP_DEBUG_LOGS"] = "true"
+		// Envknob for enabling DERP debug logs
-	headscaleConfig["DERP_PROBER_DEBUG_LOGS"] = "true"
+		"DERP_DEBUG_LOGS":        "true",
+		"DERP_PROBER_DEBUG_LOGS": "true",
+	}
 
 	err = scenario.CreateHeadscaleEnv(
 		spec,
@@ -67,6 +70,8 @@ func TestDERPServerScenario(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allHostnames, err := scenario.ListTailscaleClientsFQDNs()
 	assertNoErrListFQDN(t, err)
 

integration/general_test.go

@@ -33,7 +33,27 @@ func TestPingAllByIP(t *testing.T) {
 		"user2": len(MustTestVersions),
 	}
 
-	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("pingallbyip"))
+	headscaleConfig := map[string]string{
+		"HEADSCALE_DERP_URLS":                    "",
+		"HEADSCALE_DERP_SERVER_ENABLED":          "true",
+		"HEADSCALE_DERP_SERVER_REGION_ID":        "999",
+		"HEADSCALE_DERP_SERVER_REGION_CODE":      "headscale",
+		"HEADSCALE_DERP_SERVER_REGION_NAME":      "Headscale Embedded DERP",
+		"HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478",
+		"HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key",
+
+		// Envknob for enabling DERP debug logs
+		"DERP_DEBUG_LOGS":        "true",
+		"DERP_PROBER_DEBUG_LOGS": "true",
+	}
+
+	err = scenario.CreateHeadscaleEnv(spec,
+		[]tsic.Option{},
+		hsic.WithTestName("pingallbyip"),
+		hsic.WithConfigEnv(headscaleConfig),
+		hsic.WithTLS(),
+		hsic.WithHostnameAsServerURL(),
+	)
 	assertNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
@@ -45,6 +65,46 @@ func TestPingAllByIP(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
+	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
+		return x.String()
+	})
+
+	success := pingAllHelper(t, allClients, allAddrs)
+	t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
+}
+
+func TestPingAllByIPPublicDERP(t *testing.T) {
+	IntegrationSkip(t)
+	t.Parallel()
+
+	scenario, err := NewScenario()
+	assertNoErr(t, err)
+	defer scenario.Shutdown()
+
+	spec := map[string]int{
+		"user1": len(MustTestVersions),
+		"user2": len(MustTestVersions),
+	}
+
+	err = scenario.CreateHeadscaleEnv(spec,
+		[]tsic.Option{},
+		hsic.WithTestName("pingallbyippubderp"),
+	)
+	assertNoErrHeadscaleEnv(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	assertNoErrListClients(t, err)
+
+	allIps, err := scenario.ListTailscaleClientsIPs()
+	assertNoErrListClientIPs(t, err)
+
+	err = scenario.WaitForTailscaleSync()
+	assertNoErrSync(t, err)
+
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})
@@ -75,6 +135,8 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	clientIPs := make(map[TailscaleClient][]netip.Addr)
 	for _, client := range allClients {
 		ips, err := client.IPs()
@@ -114,6 +176,8 @@ func TestAuthKeyLogoutAndRelogin(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allClients, err = scenario.ListTailscaleClients()
 	assertNoErrListClients(t, err)
 
@@ -265,6 +329,8 @@ func TestPingAllByHostname(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allHostnames, err := scenario.ListTailscaleClientsFQDNs()
 	assertNoErrListFQDN(t, err)
 
@@ -451,6 +517,74 @@ func TestTaildrop(t *testing.T) {
 	}
 }
 
+func TestResolveMagicDNS(t *testing.T) {
+	IntegrationSkip(t)
+	t.Parallel()
+
+	scenario, err := NewScenario()
+	assertNoErr(t, err)
+	defer scenario.Shutdown()
+
+	spec := map[string]int{
+		"magicdns1": len(MustTestVersions),
+		"magicdns2": len(MustTestVersions),
+	}
+
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{}, hsic.WithTestName("magicdns"))
+	assertNoErrHeadscaleEnv(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	assertNoErrListClients(t, err)
+
+	err = scenario.WaitForTailscaleSync()
+	assertNoErrSync(t, err)
+
+	assertClientsState(t, allClients)
+
+	// Poor mans cache
+	_, err = scenario.ListTailscaleClientsFQDNs()
+	assertNoErrListFQDN(t, err)
+
+	_, err = scenario.ListTailscaleClientsIPs()
+	assertNoErrListClientIPs(t, err)
+
+	for _, client := range allClients {
+		for _, peer := range allClients {
+			// It is safe to ignore this error as we handled it when caching it
+			peerFQDN, _ := peer.FQDN()
+
+			command := []string{
+				"tailscale",
+				"ip", peerFQDN,
+			}
+			result, _, err := client.Execute(command)
+			if err != nil {
+				t.Fatalf(
+					"failed to execute resolve/ip command %s from %s: %s",
+					peerFQDN,
+					client.Hostname(),
+					err,
+				)
+			}
+
+			ips, err := peer.IPs()
+			if err != nil {
+				t.Fatalf(
+					"failed to get ips for %s: %s",
+					peer.Hostname(),
+					err,
+				)
+			}
+
+			for _, ip := range ips {
+				if !strings.Contains(result, ip.String()) {
+					t.Fatalf("ip %s is not found in \n%s\n", ip.String(), result)
+				}
+			}
+		}
+	}
+}
+
 func TestExpireNode(t *testing.T) {
 	IntegrationSkip(t)
 	t.Parallel()
@@ -475,6 +609,8 @@ func TestExpireNode(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})
@@ -599,6 +735,8 @@ func TestNodeOnlineLastSeenStatus(t *testing.T) {
 	err = scenario.WaitForTailscaleSync()
 	assertNoErrSync(t, err)
 
+	assertClientsState(t, allClients)
+
 	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})

integration/tailscale.go

@@ -7,6 +7,7 @@ import (
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/tsic"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/net/netcheck"
 	"tailscale.com/types/netmap"
 )
 
@@ -28,6 +29,7 @@ type TailscaleClient interface {
 	FQDN() (string, error)
 	Status() (*ipnstate.Status, error)
 	Netmap() (*netmap.NetworkMap, error)
+	Netcheck() (*netcheck.Report, error)
 	WaitForNeedsLogin() error
 	WaitForRunning() error
 	WaitForPeers(expected int) error

integration/tsic/tsic.go

@@ -17,6 +17,7 @@ import (
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
 	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/net/netcheck"
 	"tailscale.com/types/netmap"
 )
 
@@ -544,6 +545,29 @@ func (t *TailscaleInContainer) Netmap() (*netmap.NetworkMap, error) {
 	return &nm, err
 }
 
+// Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance.
+func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) {
+	command := []string{
+		"tailscale",
+		"netcheck",
+		"--format=json",
+	}
+
+	result, stderr, err := t.Execute(command)
+	if err != nil {
+		fmt.Printf("stderr: %s\n", stderr)
+		return nil, fmt.Errorf("failed to execute tailscale debug netcheck command: %w", err)
+	}
+
+	var nm netcheck.Report
+	err = json.Unmarshal([]byte(result), &nm)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal tailscale netcheck: %w", err)
+	}
+
+	return &nm, err
+}
+
 // FQDN returns the FQDN as a string of the Tailscale instance.
 func (t *TailscaleInContainer) FQDN() (string, error) {
 	if t.fqdn != "" {
@@ -648,12 +672,22 @@ func (t *TailscaleInContainer) WaitForPeers(expected int) error {
 				len(peers),
 			)
 		} else {
+			// Verify that the peers of a given node is Online
+			// has a hostname and a DERP relay.
 			for _, peerKey := range peers {
 				peer := status.Peer[peerKey]
 
 				if !peer.Online {
 					return fmt.Errorf("[%s] peer count correct, but %s is not online", t.hostname, peer.HostName)
 				}
+
+				if peer.HostName == "" {
+					return fmt.Errorf("[%s] peer count correct, but %s does not have a Hostname", t.hostname, peer.HostName)
+				}
+
+				if peer.Relay == "" {
+					return fmt.Errorf("[%s] peer count correct, but %s does not have a DERP", t.hostname, peer.HostName)
+				}
 			}
 		}
 

integration/utils.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/stretchr/testify/assert"
+	"tailscale.com/util/cmpver"
 )
 
 const (
@@ -83,7 +85,7 @@ func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string, opts
 		for _, addr := range addrs {
 			err := client.Ping(addr, opts...)
 			if err != nil {
-				t.Fatalf("failed to ping %s from %s: %s", addr, client.Hostname(), err)
+				t.Errorf("failed to ping %s from %s: %s", addr, client.Hostname(), err)
 			} else {
 				success++
 			}
@@ -120,6 +122,148 @@ func pingDerpAllHelper(t *testing.T, clients []TailscaleClient, addrs []string)
 	return success
 }
 
+// assertClientsState validates the status and netmap of a list of
+// clients for the general case of all to all connectivity.
+func assertClientsState(t *testing.T, clients []TailscaleClient) {
+	t.Helper()
+
+	for _, client := range clients {
+		assertValidStatus(t, client)
+		assertValidNetmap(t, client)
+		assertValidNetcheck(t, client)
+	}
+}
+
+// assertValidNetmap asserts that the netmap of a client has all
+// the minimum required fields set to a known working config for
+// the general case. Fields are checked on self, then all peers.
+// This test is not suitable for ACL/partial connection tests.
+// This test can only be run on clients from 1.56.1. It will
+// automatically pass all clients below that and is safe to call
+// for all versions.
+func assertValidNetmap(t *testing.T, client TailscaleClient) {
+	t.Helper()
+
+	if cmpver.Compare("1.56.1", client.Version()) <= 0 ||
+		!strings.Contains(client.Hostname(), "unstable") ||
+		!strings.Contains(client.Hostname(), "head") {
+		return
+	}
+
+	netmap, err := client.Netmap()
+	if err != nil {
+		t.Fatalf("getting netmap for %q: %s", client.Hostname(), err)
+	}
+
+	assert.Truef(t, netmap.SelfNode.Hostinfo().Valid(), "%q does not have Hostinfo", client.Hostname())
+	if hi := netmap.SelfNode.Hostinfo(); hi.Valid() {
+		assert.LessOrEqual(t, 1, netmap.SelfNode.Hostinfo().Services().Len(), "%q does not have enough services, got: %v", client.Hostname(), netmap.SelfNode.Hostinfo().Services())
+	}
+
+	assert.NotEmptyf(t, netmap.SelfNode.AllowedIPs(), "%q does not have any allowed IPs", client.Hostname())
+	assert.NotEmptyf(t, netmap.SelfNode.Addresses(), "%q does not have any addresses", client.Hostname())
+
+	assert.Truef(t, *netmap.SelfNode.Online(), "%q is not online", client.Hostname())
+
+	assert.Falsef(t, netmap.SelfNode.Key().IsZero(), "%q does not have a valid NodeKey", client.Hostname())
+	assert.Falsef(t, netmap.SelfNode.Machine().IsZero(), "%q does not have a valid MachineKey", client.Hostname())
+	assert.Falsef(t, netmap.SelfNode.DiscoKey().IsZero(), "%q does not have a valid DiscoKey", client.Hostname())
+
+	for _, peer := range netmap.Peers {
+		assert.NotEqualf(t, "127.3.3.40:0", peer.DERP(), "peer (%s) has no home DERP in %q's netmap, got: %s", peer.ComputedName(), client.Hostname(), peer.DERP())
+
+		assert.Truef(t, peer.Hostinfo().Valid(), "peer (%s) of %q does not have Hostinfo", peer.ComputedName(), client.Hostname())
+		if hi := peer.Hostinfo(); hi.Valid() {
+			assert.LessOrEqualf(t, 3, peer.Hostinfo().Services().Len(), "peer (%s) of %q does not have enough services, got: %v", peer.ComputedName(), client.Hostname(), peer.Hostinfo().Services())
+
+			// Netinfo is not always set
+			assert.Truef(t, hi.NetInfo().Valid(), "peer (%s) of %q does not have NetInfo", peer.ComputedName(), client.Hostname())
+			if ni := hi.NetInfo(); ni.Valid() {
+				assert.NotEqualf(t, 0, ni.PreferredDERP(), "peer (%s) has no home DERP in %q's netmap, got: %s", peer.ComputedName(), client.Hostname(), peer.Hostinfo().NetInfo().PreferredDERP())
+			}
+
+		}
+
+		assert.NotEmptyf(t, peer.Endpoints(), "peer (%s) of %q does not have any endpoints", peer.ComputedName(), client.Hostname())
+		assert.NotEmptyf(t, peer.AllowedIPs(), "peer (%s) of %q does not have any allowed IPs", peer.ComputedName(), client.Hostname())
+		assert.NotEmptyf(t, peer.Addresses(), "peer (%s) of %q does not have any addresses", peer.ComputedName(), client.Hostname())
+
+		assert.Truef(t, *peer.Online(), "peer (%s) of %q is not online", peer.ComputedName(), client.Hostname())
+
+		assert.Falsef(t, peer.Key().IsZero(), "peer (%s) of %q does not have a valid NodeKey", peer.ComputedName(), client.Hostname())
+		assert.Falsef(t, peer.Machine().IsZero(), "peer (%s) of %q does not have a valid MachineKey", peer.ComputedName(), client.Hostname())
+		assert.Falsef(t, peer.DiscoKey().IsZero(), "peer (%s) of %q does not have a valid DiscoKey", peer.ComputedName(), client.Hostname())
+	}
+}
+
+// assertValidStatus asserts that the status of a client has all
+// the minimum required fields set to a known working config for
+// the general case. Fields are checked on self, then all peers.
+// This test is not suitable for ACL/partial connection tests.
+func assertValidStatus(t *testing.T, client TailscaleClient) {
+	t.Helper()
+	status, err := client.Status()
+	if err != nil {
+		t.Fatalf("getting status for %q: %s", client.Hostname(), err)
+	}
+
+	assert.NotEmptyf(t, status.Self.HostName, "%q does not have HostName set, likely missing Hostinfo", client.Hostname())
+	assert.NotEmptyf(t, status.Self.OS, "%q does not have OS set, likely missing Hostinfo", client.Hostname())
+	assert.NotEmptyf(t, status.Self.Relay, "%q does not have a relay, likely missing Hostinfo/Netinfo", client.Hostname())
+
+	assert.NotEmptyf(t, status.Self.TailscaleIPs, "%q does not have Tailscale IPs", client.Hostname())
+
+	// This seem to not appear until version 1.56
+	if status.Self.AllowedIPs != nil {
+		assert.NotEmptyf(t, status.Self.AllowedIPs, "%q does not have any allowed IPs", client.Hostname())
+	}
+
+	assert.NotEmptyf(t, status.Self.Addrs, "%q does not have any endpoints", client.Hostname())
+
+	assert.Truef(t, status.Self.Online, "%q is not online", client.Hostname())
+
+	assert.Truef(t, status.Self.InNetworkMap, "%q is not in network map", client.Hostname())
+
+	// This isnt really relevant for Self as it wont be in its own socket/wireguard.
+	// assert.Truef(t, status.Self.InMagicSock, "%q is not tracked by magicsock", client.Hostname())
+	// assert.Truef(t, status.Self.InEngine, "%q is not in in wireguard engine", client.Hostname())
+
+	for _, peer := range status.Peer {
+		assert.NotEmptyf(t, peer.HostName, "peer (%s) of %q does not have HostName set, likely missing Hostinfo", peer.DNSName, client.Hostname())
+		assert.NotEmptyf(t, peer.OS, "peer (%s) of %q does not have OS set, likely missing Hostinfo", peer.DNSName, client.Hostname())
+		assert.NotEmptyf(t, peer.Relay, "peer (%s) of %q does not have a relay, likely missing Hostinfo/Netinfo", peer.DNSName, client.Hostname())
+
+		assert.NotEmptyf(t, peer.TailscaleIPs, "peer (%s) of %q does not have Tailscale IPs", peer.DNSName, client.Hostname())
+
+		// This seem to not appear until version 1.56
+		if peer.AllowedIPs != nil {
+			assert.NotEmptyf(t, peer.AllowedIPs, "peer (%s) of %q does not have any allowed IPs", peer.DNSName, client.Hostname())
+		}
+
+		// Addrs does not seem to appear in the status from peers.
+		// assert.NotEmptyf(t, peer.Addrs, "peer (%s) of %q does not have any endpoints", peer.DNSName, client.Hostname())
+
+		assert.Truef(t, peer.Online, "peer (%s) of %q is not online", peer.DNSName, client.Hostname())
+
+		assert.Truef(t, peer.InNetworkMap, "peer (%s) of %q is not in network map", peer.DNSName, client.Hostname())
+		assert.Truef(t, peer.InMagicSock, "peer (%s) of %q is not tracked by magicsock", peer.DNSName, client.Hostname())
+
+		// TODO(kradalby): InEngine is only true when a proper tunnel is set up,
+		// there might be some interesting stuff to test here in the future.
+		// assert.Truef(t, peer.InEngine, "peer (%s) of %q is not in wireguard engine", peer.DNSName, client.Hostname())
+	}
+}
+
+func assertValidNetcheck(t *testing.T, client TailscaleClient) {
+	t.Helper()
+	report, err := client.Netcheck()
+	if err != nil {
+		t.Fatalf("getting status for %q: %s", client.Hostname(), err)
+	}
+
+	assert.NotEqualf(t, 0, report.PreferredDERP, "%q does not have a DERP relay", client.Hostname())
+}
+
 func isSelfClient(client TailscaleClient, addr string) bool {
 	if addr == client.Hostname() {
 		return true
@@ -152,7 +296,7 @@ func isCI() bool {
 }
 
 func dockertestMaxWait() time.Duration {
-	wait := 60 * time.Second //nolint
+	wait := 120 * time.Second //nolint
 
 	if isCI() {
 		wait = 300 * time.Second //nolint