Merge f8ec54d816 into edf9e25001

* feat: support client verify for derp

* docs: fix doc for integration test

* tests: add integration test for DERP verify endpoint

* tests: use `tailcfg.DERPMap` instead of `[]byte`

* refactor: introduce func `ContainsNodeKey`

* tests(dsic): use string builder for cmd args

* ci: fix tests order

* tests: fix derper failure

* chore: cleanup

* tests(verify-client): perfer to use `CreateHeadscaleEnv`

* refactor(verify-client): simplify error handling

* tests: fix `TestDERPVerifyEndpoint`

* refactor: make `doVerify` a seperated func

---------

Co-authored-by: 117503445 <t117503445@gmail.com>

.github/workflows/test-integration.yaml

@@ -38,6 +38,7 @@ jobs:
           - TestNodeMoveCommand
           - TestPolicyCommand
           - TestPolicyBrokenConfigCommand
+          - TestDERPVerifyEndpoint
           - TestResolveMagicDNS
           - TestValidateResolvConf
           - TestDERPServerScenario

CHANGELOG.md

@@ -23,6 +23,7 @@
 - Added conversion of 'Hostname' to 'givenName' in a node with FQDN rules applied [#2198](https://github.com/juanfont/headscale/pull/2198)
 - Fixed updating of hostname and givenName when it is updated in HostInfo [#2199](https://github.com/juanfont/headscale/pull/2199)
 - Fixed missing `stable-debug` container tag [#2232](https://github.com/juanfont/headscale/pr/2232)
+- Loosened up `server_url` and `base_domain` check. It was overly strict in some cases.
 
 ## 0.23.0 (2024-09-18)
 

Dockerfile.derper

@@ -0,0 +1,19 @@
+# For testing purposes only
+
+FROM golang:alpine AS build-env
+
+WORKDIR /go/src
+
+RUN apk add --no-cache git
+ARG VERSION_BRANCH=main
+RUN git clone https://github.com/tailscale/tailscale.git --branch=$VERSION_BRANCH --depth=1
+WORKDIR /go/src/tailscale
+
+ARG TARGETARCH
+RUN GOARCH=$TARGETARCH go install -v ./cmd/derper
+
+FROM alpine:3.18
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl
+
+COPY --from=build-env /go/bin/* /usr/local/bin/
+ENTRYPOINT [ "/usr/local/bin/derper" ]

Dockerfile.tailscale-HEAD

@@ -28,7 +28,9 @@ ARG VERSION_GIT_HASH=""
 ENV VERSION_GIT_HASH=$VERSION_GIT_HASH
 ARG TARGETARCH
 
-RUN GOARCH=$TARGETARCH go install -ldflags="\
+ARG BUILD_TAGS=""
+
+RUN GOARCH=$TARGETARCH go install -tags="${BUILD_TAGS}" -ldflags="\
       -X tailscale.com/version.longStamp=$VERSION_LONG \
       -X tailscale.com/version.shortStamp=$VERSION_SHORT \
       -X tailscale.com/version.gitCommitStamp=$VERSION_GIT_HASH" \

hscontrol/app.go

@@ -457,6 +457,8 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router {
 	router.HandleFunc("/swagger/v1/openapiv2.json", headscale.SwaggerAPIv1).
 		Methods(http.MethodGet)
 
+	router.HandleFunc("/verify", h.VerifyHandler).Methods(http.MethodPost)
+
 	if h.cfg.DERP.ServerEnabled {
 		router.HandleFunc("/derp", h.DERPServer.DERPHandler)
 		router.HandleFunc("/derp/probe", derpServer.DERPProbeHandler)
@@ -1027,14 +1029,18 @@ func (h *Headscale) loadACLPolicy() error {
 		if err != nil {
 			return fmt.Errorf("loading nodes from database to validate policy: %w", err)
 		}
+		users, err := h.db.ListUsers()
+		if err != nil {
+			return fmt.Errorf("loading users from database to validate policy: %w", err)
+		}
 
-		_, err = pol.CompileFilterRules(nodes)
+		_, err = pol.CompileFilterRules(users, nodes)
 		if err != nil {
 			return fmt.Errorf("verifying policy rules: %w", err)
 		}
 
 		if len(nodes) > 0 {
-			_, err = pol.CompileSSHPolicy(nodes[0], nodes)
+			_, err = pol.CompileSSHPolicy(nodes[0], users, nodes)
 			if err != nil {
 				return fmt.Errorf("verifying SSH rules: %w", err)
 			}

hscontrol/db/node_test.go

@@ -255,10 +255,10 @@ func (s *Suite) TestGetACLFilteredPeers(c *check.C) {
 	c.Assert(err, check.IsNil)
 	c.Assert(len(testPeers), check.Equals, 9)
 
-	adminRules, _, err := policy.GenerateFilterAndSSHRulesForTests(aclPolicy, adminNode, adminPeers)
+	adminRules, _, err := policy.GenerateFilterAndSSHRulesForTests(aclPolicy, adminNode, adminPeers, []types.User{*stor[0].user, *stor[1].user})
 	c.Assert(err, check.IsNil)
 
-	testRules, _, err := policy.GenerateFilterAndSSHRulesForTests(aclPolicy, testNode, testPeers)
+	testRules, _, err := policy.GenerateFilterAndSSHRulesForTests(aclPolicy, testNode, testPeers, []types.User{*stor[0].user, *stor[1].user})
 	c.Assert(err, check.IsNil)
 
 	peersOfAdminNode := policy.FilterNodesByACL(adminNode, adminPeers, adminRules)

hscontrol/db/routes.go

@@ -648,8 +648,13 @@ func EnableAutoApprovedRoutes(
 			if approvedAlias == node.User.Username() {
 				approvedRoutes = append(approvedRoutes, advertisedRoute)
 			} else {
+				users, err := ListUsers(tx)
+				if err != nil {
+					return fmt.Errorf("looking up users to expand route alias: %w", err)
+				}
+
 				// TODO(kradalby): figure out how to get this to depend on less stuff
-				approvedIps, err := aclPolicy.ExpandAlias(types.Nodes{node}, approvedAlias)
+				approvedIps, err := aclPolicy.ExpandAlias(types.Nodes{node}, users, approvedAlias)
 				if err != nil {
 					return fmt.Errorf("expanding alias %q for autoApprovers: %w", approvedAlias, err)
 				}

hscontrol/grpcv1.go

@@ -737,14 +737,18 @@ func (api headscaleV1APIServer) SetPolicy(
 	if err != nil {
 		return nil, fmt.Errorf("loading nodes from database to validate policy: %w", err)
 	}
+	users, err := api.h.db.ListUsers()
+	if err != nil {
+		return nil, fmt.Errorf("loading users from database to validate policy: %w", err)
+	}
 
-	_, err = pol.CompileFilterRules(nodes)
+	_, err = pol.CompileFilterRules(users, nodes)
 	if err != nil {
 		return nil, fmt.Errorf("verifying policy rules: %w", err)
 	}
 
 	if len(nodes) > 0 {
-		_, err = pol.CompileSSHPolicy(nodes[0], nodes)
+		_, err = pol.CompileSSHPolicy(nodes[0], users, nodes)
 		if err != nil {
 			return nil, fmt.Errorf("verifying SSH rules: %w", err)
 		}

hscontrol/handlers.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -56,6 +57,65 @@ func parseCabailityVersion(req *http.Request) (tailcfg.CapabilityVersion, error)
 	return tailcfg.CapabilityVersion(clientCapabilityVersion), nil
 }
 
+func (h *Headscale) handleVerifyRequest(
+	req *http.Request,
+) (bool, error) {
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		return false, fmt.Errorf("cannot read request body: %w", err)
+	}
+
+	var derpAdmitClientRequest tailcfg.DERPAdmitClientRequest
+	if err := json.Unmarshal(body, &derpAdmitClientRequest); err != nil {
+		return false, fmt.Errorf("cannot parse derpAdmitClientRequest: %w", err)
+	}
+
+	nodes, err := h.db.ListNodes()
+	if err != nil {
+		return false, fmt.Errorf("cannot list nodes: %w", err)
+	}
+
+	return nodes.ContainsNodeKey(derpAdmitClientRequest.NodePublic), nil
+}
+
+// see https://github.com/tailscale/tailscale/blob/964282d34f06ecc06ce644769c66b0b31d118340/derp/derp_server.go#L1159, Derp use verifyClientsURL to verify whether a client is allowed to connect to the DERP server.
+func (h *Headscale) VerifyHandler(
+	writer http.ResponseWriter,
+	req *http.Request,
+) {
+	if req.Method != http.MethodPost {
+		http.Error(writer, "Wrong method", http.StatusMethodNotAllowed)
+
+		return
+	}
+	log.Debug().
+		Str("handler", "/verify").
+		Msg("verify client")
+
+	allow, err := h.handleVerifyRequest(req)
+	if err != nil {
+		log.Error().
+			Caller().
+			Err(err).
+			Msg("Failed to verify client")
+		http.Error(writer, "Internal error", http.StatusInternalServerError)
+	}
+
+	resp := tailcfg.DERPAdmitClientResponse{
+		Allow: allow,
+	}
+
+	writer.Header().Set("Content-Type", "application/json")
+	writer.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(writer).Encode(resp)
+	if err != nil {
+		log.Error().
+			Caller().
+			Err(err).
+			Msg("Failed to write response")
+	}
+}
+
 // KeyHandler provides the Headscale pub key
 // Listens in /key.
 func (h *Headscale) KeyHandler(

hscontrol/mapper/mapper.go

@@ -153,6 +153,7 @@ func addNextDNSMetadata(resolvers []*dnstype.Resolver, node *types.Node) {
 func (m *Mapper) fullMapResponse(
 	node *types.Node,
 	peers types.Nodes,
+	users []types.User,
 	pol *policy.ACLPolicy,
 	capVer tailcfg.CapabilityVersion,
 ) (*tailcfg.MapResponse, error) {
@@ -167,6 +168,7 @@ func (m *Mapper) fullMapResponse(
 		pol,
 		node,
 		capVer,
+		users,
 		peers,
 		peers,
 		m.cfg,
@@ -189,8 +191,12 @@ func (m *Mapper) FullMapResponse(
 	if err != nil {
 		return nil, err
 	}
+	users, err := m.db.ListUsers()
+	if err != nil {
+		return nil, err
+	}
 
-	resp, err := m.fullMapResponse(node, peers, pol, mapRequest.Version)
+	resp, err := m.fullMapResponse(node, peers, users, pol, mapRequest.Version)
 	if err != nil {
 		return nil, err
 	}
@@ -253,6 +259,11 @@ func (m *Mapper) PeerChangedResponse(
 		return nil, err
 	}
 
+	users, err := m.db.ListUsers()
+	if err != nil {
+		return nil, fmt.Errorf("listing users for map response: %w", err)
+	}
+
 	var removedIDs []tailcfg.NodeID
 	var changedIDs []types.NodeID
 	for nodeID, nodeChanged := range changed {
@@ -276,6 +287,7 @@ func (m *Mapper) PeerChangedResponse(
 		pol,
 		node,
 		mapRequest.Version,
+		users,
 		peers,
 		changedNodes,
 		m.cfg,
@@ -508,16 +520,17 @@ func appendPeerChanges(
 	pol *policy.ACLPolicy,
 	node *types.Node,
 	capVer tailcfg.CapabilityVersion,
+	users []types.User,
 	peers types.Nodes,
 	changed types.Nodes,
 	cfg *types.Config,
 ) error {
-	packetFilter, err := pol.CompileFilterRules(append(peers, node))
+	packetFilter, err := pol.CompileFilterRules(users, append(peers, node))
 	if err != nil {
 		return err
 	}
 
-	sshPolicy, err := pol.CompileSSHPolicy(node, peers)
+	sshPolicy, err := pol.CompileSSHPolicy(node, users, peers)
 	if err != nil {
 		return err
 	}

hscontrol/mapper/mapper_test.go

@@ -159,6 +159,9 @@ func Test_fullMapResponse(t *testing.T) {
 	lastSeen := time.Date(2009, time.November, 10, 23, 9, 0, 0, time.UTC)
 	expire := time.Date(2500, time.November, 11, 23, 0, 0, 0, time.UTC)
 
+	user1 := types.User{Model: gorm.Model{ID: 0}, Name: "mini"}
+	user2 := types.User{Model: gorm.Model{ID: 1}, Name: "peer2"}
+
 	mini := &types.Node{
 		ID: 0,
 		MachineKey: mustMK(
@@ -173,8 +176,8 @@ func Test_fullMapResponse(t *testing.T) {
 		IPv4:       iap("100.64.0.1"),
 		Hostname:   "mini",
 		GivenName:  "mini",
-		UserID:     0,
-		User:       types.User{Name: "mini"},
+		UserID:     user1.ID,
+		User:       user1,
 		ForcedTags: []string{},
 		AuthKey:    &types.PreAuthKey{},
 		LastSeen:   &lastSeen,
@@ -253,8 +256,8 @@ func Test_fullMapResponse(t *testing.T) {
 		IPv4:       iap("100.64.0.2"),
 		Hostname:   "peer1",
 		GivenName:  "peer1",
-		UserID:     0,
-		User:       types.User{Name: "mini"},
+		UserID:     user1.ID,
+		User:       user1,
 		ForcedTags: []string{},
 		LastSeen:   &lastSeen,
 		Expiry:     &expire,
@@ -308,8 +311,8 @@ func Test_fullMapResponse(t *testing.T) {
 		IPv4:       iap("100.64.0.3"),
 		Hostname:   "peer2",
 		GivenName:  "peer2",
-		UserID:     1,
-		User:       types.User{Name: "peer2"},
+		UserID:     user2.ID,
+		User:       user2,
 		ForcedTags: []string{},
 		LastSeen:   &lastSeen,
 		Expiry:     &expire,
@@ -468,6 +471,7 @@ func Test_fullMapResponse(t *testing.T) {
 			got, err := mappy.fullMapResponse(
 				tt.node,
 				tt.peers,
+				[]types.User{user1, user2},
 				tt.pol,
 				0,
 			)

hscontrol/policy/acls.go

@@ -137,20 +137,21 @@ func GenerateFilterAndSSHRulesForTests(
 	policy *ACLPolicy,
 	node *types.Node,
 	peers types.Nodes,
+	users []types.User,
 ) ([]tailcfg.FilterRule, *tailcfg.SSHPolicy, error) {
 	// If there is no policy defined, we default to allow all
 	if policy == nil {
 		return tailcfg.FilterAllowAll, &tailcfg.SSHPolicy{}, nil
 	}
 
-	rules, err := policy.CompileFilterRules(append(peers, node))
+	rules, err := policy.CompileFilterRules(users, append(peers, node))
 	if err != nil {
 		return []tailcfg.FilterRule{}, &tailcfg.SSHPolicy{}, err
 	}
 
 	log.Trace().Interface("ACL", rules).Str("node", node.GivenName).Msg("ACL rules")
 
-	sshPolicy, err := policy.CompileSSHPolicy(node, peers)
+	sshPolicy, err := policy.CompileSSHPolicy(node, users, peers)
 	if err != nil {
 		return []tailcfg.FilterRule{}, &tailcfg.SSHPolicy{}, err
 	}
@@ -161,6 +162,7 @@ func GenerateFilterAndSSHRulesForTests(
 // CompileFilterRules takes a set of nodes and an ACLPolicy and generates a
 // set of Tailscale compatible FilterRules used to allow traffic on clients.
 func (pol *ACLPolicy) CompileFilterRules(
+	users []types.User,
 	nodes types.Nodes,
 ) ([]tailcfg.FilterRule, error) {
 	if pol == nil {
@@ -176,9 +178,14 @@ func (pol *ACLPolicy) CompileFilterRules(
 
 		var srcIPs []string
 		for srcIndex, src := range acl.Sources {
-			srcs, err := pol.expandSource(src, nodes)
+			srcs, err := pol.expandSource(src, users, nodes)
 			if err != nil {
-				return nil, fmt.Errorf("parsing policy, acl index: %d->%d: %w", index, srcIndex, err)
+				return nil, fmt.Errorf(
+					"parsing policy, acl index: %d->%d: %w",
+					index,
+					srcIndex,
+					err,
+				)
 			}
 			srcIPs = append(srcIPs, srcs...)
 		}
@@ -197,6 +204,7 @@ func (pol *ACLPolicy) CompileFilterRules(
 
 			expanded, err := pol.ExpandAlias(
 				nodes,
+				users,
 				alias,
 			)
 			if err != nil {
@@ -281,6 +289,7 @@ func ReduceFilterRules(node *types.Node, rules []tailcfg.FilterRule) []tailcfg.F
 
 func (pol *ACLPolicy) CompileSSHPolicy(
 	node *types.Node,
+	users []types.User,
 	peers types.Nodes,
 ) (*tailcfg.SSHPolicy, error) {
 	if pol == nil {
@@ -312,7 +321,7 @@ func (pol *ACLPolicy) CompileSSHPolicy(
 	for index, sshACL := range pol.SSHs {
 		var dest netipx.IPSetBuilder
 		for _, src := range sshACL.Destinations {
-			expanded, err := pol.ExpandAlias(append(peers, node), src)
+			expanded, err := pol.ExpandAlias(append(peers, node), users, src)
 			if err != nil {
 				return nil, err
 			}
@@ -335,12 +344,21 @@ func (pol *ACLPolicy) CompileSSHPolicy(
 		case "check":
 			checkAction, err := sshCheckAction(sshACL.CheckPeriod)
 			if err != nil {
-				return nil, fmt.Errorf("parsing SSH policy, parsing check duration, index: %d: %w", index, err)
+				return nil, fmt.Errorf(
+					"parsing SSH policy, parsing check duration, index: %d: %w",
+					index,
+					err,
+				)
 			} else {
 				action = *checkAction
 			}
 		default:
-			return nil, fmt.Errorf("parsing SSH policy, unknown action %q, index: %d: %w", sshACL.Action, index, err)
+			return nil, fmt.Errorf(
+				"parsing SSH policy, unknown action %q, index: %d: %w",
+				sshACL.Action,
+				index,
+				err,
+			)
 		}
 
 		principals := make([]*tailcfg.SSHPrincipal, 0, len(sshACL.Sources))
@@ -363,6 +381,7 @@ func (pol *ACLPolicy) CompileSSHPolicy(
 			} else {
 				expandedSrcs, err := pol.ExpandAlias(
 					peers,
+					users,
 					rawSrc,
 				)
 				if err != nil {
@@ -512,9 +531,10 @@ func parseProtocol(protocol string) ([]int, bool, error) {
 // with the given src alias.
 func (pol *ACLPolicy) expandSource(
 	src string,
+	users []types.User,
 	nodes types.Nodes,
 ) ([]string, error) {
-	ipSet, err := pol.ExpandAlias(nodes, src)
+	ipSet, err := pol.ExpandAlias(nodes, users, src)
 	if err != nil {
 		return []string{}, err
 	}
@@ -538,6 +558,7 @@ func (pol *ACLPolicy) expandSource(
 // and transform these in IPAddresses.
 func (pol *ACLPolicy) ExpandAlias(
 	nodes types.Nodes,
+	users []types.User,
 	alias string,
 ) (*netipx.IPSet, error) {
 	if isWildcard(alias) {
@@ -552,12 +573,12 @@ func (pol *ACLPolicy) ExpandAlias(
 
 	// if alias is a group
 	if isGroup(alias) {
-		return pol.expandIPsFromGroup(alias, nodes)
+		return pol.expandIPsFromGroup(alias, users, nodes)
 	}
 
 	// if alias is a tag
 	if isTag(alias) {
-		return pol.expandIPsFromTag(alias, nodes)
+		return pol.expandIPsFromTag(alias, users, nodes)
 	}
 
 	if isAutoGroup(alias) {
@@ -565,7 +586,7 @@ func (pol *ACLPolicy) ExpandAlias(
 	}
 
 	// if alias is a user
-	if ips, err := pol.expandIPsFromUser(alias, nodes); ips != nil {
+	if ips, err := pol.expandIPsFromUser(alias, users, nodes); ips != nil {
 		return ips, err
 	}
 
@@ -574,7 +595,7 @@ func (pol *ACLPolicy) ExpandAlias(
 	if h, ok := pol.Hosts[alias]; ok {
 		log.Trace().Str("host", h.String()).Msg("ExpandAlias got hosts entry")
 
-		return pol.ExpandAlias(nodes, h.String())
+		return pol.ExpandAlias(nodes, users, h.String())
 	}
 
 	// if alias is an IP
@@ -751,16 +772,17 @@ func (pol *ACLPolicy) expandUsersFromGroup(
 
 func (pol *ACLPolicy) expandIPsFromGroup(
 	group string,
+	users []types.User,
 	nodes types.Nodes,
 ) (*netipx.IPSet, error) {
 	var build netipx.IPSetBuilder
 
-	users, err := pol.expandUsersFromGroup(group)
+	userTokens, err := pol.expandUsersFromGroup(group)
 	if err != nil {
 		return &netipx.IPSet{}, err
 	}
-	for _, user := range users {
-		filteredNodes := filterNodesByUser(nodes, user)
+	for _, user := range userTokens {
+		filteredNodes := filterNodesByUser(nodes, users, user)
 		for _, node := range filteredNodes {
 			node.AppendToIPSet(&build)
 		}
@@ -771,6 +793,7 @@ func (pol *ACLPolicy) expandIPsFromGroup(
 
 func (pol *ACLPolicy) expandIPsFromTag(
 	alias string,
+	users []types.User,
 	nodes types.Nodes,
 ) (*netipx.IPSet, error) {
 	var build netipx.IPSetBuilder
@@ -803,7 +826,7 @@ func (pol *ACLPolicy) expandIPsFromTag(
 
 	// filter out nodes per tag owner
 	for _, user := range owners {
-		nodes := filterNodesByUser(nodes, user)
+		nodes := filterNodesByUser(nodes, users, user)
 		for _, node := range nodes {
 			if node.Hostinfo == nil {
 				continue
@@ -820,11 +843,12 @@ func (pol *ACLPolicy) expandIPsFromTag(
 
 func (pol *ACLPolicy) expandIPsFromUser(
 	user string,
+	users []types.User,
 	nodes types.Nodes,
 ) (*netipx.IPSet, error) {
 	var build netipx.IPSetBuilder
 
-	filteredNodes := filterNodesByUser(nodes, user)
+	filteredNodes := filterNodesByUser(nodes, users, user)
 	filteredNodes = excludeCorrectlyTaggedNodes(pol, filteredNodes, user)
 
 	// shortcurcuit if we have no nodes to get ips from.
@@ -953,10 +977,43 @@ func (pol *ACLPolicy) TagsOfNode(
 	return validTags, invalidTags
 }
 
-func filterNodesByUser(nodes types.Nodes, user string) types.Nodes {
+// filterNodesByUser returns a list of nodes that match the given userToken from a
+// policy.
+// Matching nodes are determined by first matching the user token to a user by checking:
+// - If it is an ID that mactches the user database ID
+// - It is the Provider Identifier from OIDC
+// - It matches the username or email of a user
+//
+// If the token matches more than one user, zero nodes will returned.
+func filterNodesByUser(nodes types.Nodes, users []types.User, userToken string) types.Nodes {
 	var out types.Nodes
+
+	var potentialUsers []types.User
+	for _, user := range users {
+		if user.ProviderIdentifier.Valid && user.ProviderIdentifier.String == userToken {
+			// If a user is matching with a known unique field,
+			// disgard all other users and only keep the current
+			// user.
+			potentialUsers = []types.User{user}
+
+			break
+		}
+		if user.Email == userToken {
+			potentialUsers = append(potentialUsers, user)
+		}
+		if user.Name == userToken {
+			potentialUsers = append(potentialUsers, user)
+		}
+	}
+
+	if len(potentialUsers) != 1 {
+		return nil
+	}
+
+	user := potentialUsers[0]
+
 	for _, node := range nodes {
-		if node.User.Username() == user {
+		if node.User.ID == user.ID {
 			out = append(out, node)
 		}
 	}

hscontrol/policy/acls_test.go

@@ -1,9 +1,12 @@
 package policy
 
 import (
+	"database/sql"
 	"errors"
+	"math/rand/v2"
 	"net/netip"
 	"slices"
+	"sort"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -14,6 +17,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"go4.org/netipx"
 	"gopkg.in/check.v1"
+	"gorm.io/gorm"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/tailcfg"
 )
@@ -375,18 +379,24 @@ func TestParsing(t *testing.T) {
 				return
 			}
 
-			rules, err := pol.CompileFilterRules(types.Nodes{
-				&types.Node{
-					IPv4: iap("100.100.100.100"),
+			user := types.User{
+				Model: gorm.Model{ID: 1},
+				Name:  "testuser",
+			}
+			rules, err := pol.CompileFilterRules(
+				[]types.User{
+					user,
 				},
-				&types.Node{
-					IPv4: iap("200.200.200.200"),
-					User: types.User{
-						Name: "testuser",
+				types.Nodes{
+					&types.Node{
+						IPv4: iap("100.100.100.100"),
 					},
-					Hostinfo: &tailcfg.Hostinfo{},
-				},
-			})
+					&types.Node{
+						IPv4:     iap("200.200.200.200"),
+						User:     user,
+						Hostinfo: &tailcfg.Hostinfo{},
+					},
+				})
 
 			if (err != nil) != tt.wantErr {
 				t.Errorf("parsing() error = %v, wantErr %v", err, tt.wantErr)
@@ -533,7 +543,7 @@ func (s *Suite) TestRuleInvalidGeneration(c *check.C) {
 	c.Assert(pol.ACLs, check.HasLen, 6)
 	c.Assert(err, check.IsNil)
 
-	rules, err := pol.CompileFilterRules(types.Nodes{})
+	rules, err := pol.CompileFilterRules([]types.User{}, types.Nodes{})
 	c.Assert(err, check.NotNil)
 	c.Assert(rules, check.IsNil)
 }
@@ -549,7 +559,7 @@ func (s *Suite) TestInvalidAction(c *check.C) {
 			},
 		},
 	}
-	_, _, err := GenerateFilterAndSSHRulesForTests(pol, &types.Node{}, types.Nodes{})
+	_, _, err := GenerateFilterAndSSHRulesForTests(pol, &types.Node{}, types.Nodes{}, []types.User{})
 	c.Assert(errors.Is(err, ErrInvalidAction), check.Equals, true)
 }
 
@@ -568,7 +578,7 @@ func (s *Suite) TestInvalidGroupInGroup(c *check.C) {
 			},
 		},
 	}
-	_, _, err := GenerateFilterAndSSHRulesForTests(pol, &types.Node{}, types.Nodes{})
+	_, _, err := GenerateFilterAndSSHRulesForTests(pol, &types.Node{}, types.Nodes{}, []types.User{})
 	c.Assert(errors.Is(err, ErrInvalidGroup), check.Equals, true)
 }
 
@@ -584,7 +594,7 @@ func (s *Suite) TestInvalidTagOwners(c *check.C) {
 		},
 	}
 
-	_, _, err := GenerateFilterAndSSHRulesForTests(pol, &types.Node{}, types.Nodes{})
+	_, _, err := GenerateFilterAndSSHRulesForTests(pol, &types.Node{}, types.Nodes{}, []types.User{})
 	c.Assert(errors.Is(err, ErrInvalidTag), check.Equals, true)
 }
 
@@ -860,7 +870,20 @@ func Test_expandPorts(t *testing.T) {
 	}
 }
 
-func Test_listNodesInUser(t *testing.T) {
+func Test_filterNodesByUser(t *testing.T) {
+	users := []types.User{
+		{Model: gorm.Model{ID: 1}, Name: "marc"},
+		{Model: gorm.Model{ID: 2}, Name: "joe", Email: "joe@headscale.net"},
+		{Model: gorm.Model{ID: 3}, Name: "mikael", Email: "mikael@headscale.net", ProviderIdentifier: sql.NullString{String: "http://oidc.org/1234", Valid: true}},
+		{Model: gorm.Model{ID: 4}, Name: "mikael2", Email: "mikael@headscale.net"},
+		{Model: gorm.Model{ID: 5}, Name: "mikael", Email: "mikael2@headscale.net"},
+		{Model: gorm.Model{ID: 6}, Name: "http://oidc.org/1234", Email: "mikael@headscale.net"},
+		{Model: gorm.Model{ID: 7}, Name: "1"},
+		{Model: gorm.Model{ID: 8}, Name: "alex", Email: "alex@headscale.net"},
+		{Model: gorm.Model{ID: 9}, Name: "alex@headscale.net"},
+		{Model: gorm.Model{ID: 10}, Email: "http://oidc.org/1234"},
+	}
+
 	type args struct {
 		nodes types.Nodes
 		user  string
@@ -874,50 +897,258 @@ func Test_listNodesInUser(t *testing.T) {
 			name: "1 node in user",
 			args: args{
 				nodes: types.Nodes{
-					&types.Node{User: types.User{Name: "joe"}},
+					&types.Node{User: users[1]},
 				},
 				user: "joe",
 			},
 			want: types.Nodes{
-				&types.Node{User: types.User{Name: "joe"}},
+				&types.Node{User: users[1]},
 			},
 		},
 		{
 			name: "3 nodes, 2 in user",
 			args: args{
 				nodes: types.Nodes{
-					&types.Node{ID: 1, User: types.User{Name: "joe"}},
-					&types.Node{ID: 2, User: types.User{Name: "marc"}},
-					&types.Node{ID: 3, User: types.User{Name: "marc"}},
+					&types.Node{ID: 1, User: users[1]},
+					&types.Node{ID: 2, User: users[0]},
+					&types.Node{ID: 3, User: users[0]},
 				},
 				user: "marc",
 			},
 			want: types.Nodes{
-				&types.Node{ID: 2, User: types.User{Name: "marc"}},
-				&types.Node{ID: 3, User: types.User{Name: "marc"}},
+				&types.Node{ID: 2, User: users[0]},
+				&types.Node{ID: 3, User: users[0]},
 			},
 		},
 		{
 			name: "5 nodes, 0 in user",
 			args: args{
 				nodes: types.Nodes{
-					&types.Node{ID: 1, User: types.User{Name: "joe"}},
-					&types.Node{ID: 2, User: types.User{Name: "marc"}},
-					&types.Node{ID: 3, User: types.User{Name: "marc"}},
-					&types.Node{ID: 4, User: types.User{Name: "marc"}},
-					&types.Node{ID: 5, User: types.User{Name: "marc"}},
+					&types.Node{ID: 1, User: users[1]},
+					&types.Node{ID: 2, User: users[0]},
+					&types.Node{ID: 3, User: users[0]},
+					&types.Node{ID: 4, User: users[0]},
+					&types.Node{ID: 5, User: users[0]},
 				},
 				user: "mickael",
 			},
 			want: nil,
 		},
+		{
+			name: "match-by-provider-ident",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[1]},
+					&types.Node{ID: 2, User: users[2]},
+				},
+				user: "http://oidc.org/1234",
+			},
+			want: types.Nodes{
+				&types.Node{ID: 2, User: users[2]},
+			},
+		},
+		{
+			name: "match-by-email",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[1]},
+					&types.Node{ID: 2, User: users[2]},
+					&types.Node{ID: 8, User: users[7]},
+				},
+				user: "joe@headscale.net",
+			},
+			want: types.Nodes{
+				&types.Node{ID: 1, User: users[1]},
+			},
+		},
+		{
+			name: "multi-match-is-zero",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[1]},
+					&types.Node{ID: 2, User: users[2]},
+					&types.Node{ID: 3, User: users[3]},
+				},
+				user: "mikael@headscale.net",
+			},
+			want: nil,
+		},
+		{
+			name: "multi-email-first-match-is-zero",
+			args: args{
+				nodes: types.Nodes{
+					// First match email, then provider id
+					&types.Node{ID: 3, User: users[3]},
+					&types.Node{ID: 2, User: users[2]},
+				},
+				user: "mikael@headscale.net",
+			},
+			want: nil,
+		},
+		{
+			name: "multi-username-first-match-is-zero",
+			args: args{
+				nodes: types.Nodes{
+					// First match username, then provider id
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 2, User: users[2]},
+				},
+				user: "mikael",
+			},
+			want: nil,
+		},
+		{
+			name: "all-users-duplicate-username-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+				},
+				user: "mikael",
+			},
+			want: nil,
+		},
+		{
+			name: "all-users-unique-username-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+				},
+				user: "marc",
+			},
+			want: types.Nodes{
+				&types.Node{ID: 1, User: users[0]},
+			},
+		},
+		{
+			name: "all-users-no-username-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+				},
+				user: "not-working",
+			},
+			want: nil,
+		},
+		{
+			name: "all-users-duplicate-email-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+				},
+				user: "mikael@headscale.net",
+			},
+			want: nil,
+		},
+		{
+			name: "all-users-duplicate-email-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+					&types.Node{ID: 8, User: users[7]},
+				},
+				user: "joe@headscale.net",
+			},
+			want: types.Nodes{
+				&types.Node{ID: 2, User: users[1]},
+			},
+		},
+		{
+			name: "email-as-username-duplicate",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[7]},
+					&types.Node{ID: 2, User: users[8]},
+				},
+				user: "alex@headscale.net",
+			},
+			want: nil,
+		},
+		{
+			name: "all-users-no-email-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+				},
+				user: "not-working@headscale.net",
+			},
+			want: nil,
+		},
+		{
+			name: "all-users-provider-id-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+					&types.Node{ID: 6, User: users[5]},
+				},
+				user: "http://oidc.org/1234",
+			},
+			want: types.Nodes{
+				&types.Node{ID: 3, User: users[2]},
+			},
+		},
+		{
+			name: "all-users-no-provider-id-random-order",
+			args: args{
+				nodes: types.Nodes{
+					&types.Node{ID: 1, User: users[0]},
+					&types.Node{ID: 2, User: users[1]},
+					&types.Node{ID: 3, User: users[2]},
+					&types.Node{ID: 4, User: users[3]},
+					&types.Node{ID: 5, User: users[4]},
+					&types.Node{ID: 6, User: users[5]},
+				},
+				user: "http://oidc.org/4321",
+			},
+			want: nil,
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			got := filterNodesByUser(test.args.nodes, test.args.user)
+			for range 1000 {
+				ns := test.args.nodes
+				rand.Shuffle(len(ns), func(i, j int) {
+					ns[i], ns[j] = ns[j], ns[i]
+				})
+				us := users
+				rand.Shuffle(len(us), func(i, j int) {
+					us[i], us[j] = us[j], us[i]
+				})
+				got := filterNodesByUser(ns, us, test.args.user)
+				sort.Slice(got, func(i, j int) bool {
+					return got[i].ID < got[j].ID
+				})
 
-			if diff := cmp.Diff(test.want, got, util.Comparers...); diff != "" {
-				t.Errorf("listNodesInUser() = (-want +got):\n%s", diff)
+				if diff := cmp.Diff(test.want, got, util.Comparers...); diff != "" {
+					t.Errorf("filterNodesByUser() = (-want +got):\n%s", diff)
+				}
 			}
 		})
 	}
@@ -940,6 +1171,12 @@ func Test_expandAlias(t *testing.T) {
 		return s
 	}
 
+	users := []types.User{
+		{Model: gorm.Model{ID: 1}, Name: "joe"},
+		{Model: gorm.Model{ID: 2}, Name: "marc"},
+		{Model: gorm.Model{ID: 3}, Name: "mickael"},
+	}
+
 	type field struct {
 		pol ACLPolicy
 	}
@@ -989,19 +1226,19 @@ func Test_expandAlias(t *testing.T) {
 				nodes: types.Nodes{
 					&types.Node{
 						IPv4: iap("100.64.0.1"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.2"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.3"),
-						User: types.User{Name: "marc"},
+						User: users[1],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.4"),
-						User: types.User{Name: "mickael"},
+						User: users[2],
 					},
 				},
 			},
@@ -1022,19 +1259,19 @@ func Test_expandAlias(t *testing.T) {
 				nodes: types.Nodes{
 					&types.Node{
 						IPv4: iap("100.64.0.1"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.2"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.3"),
-						User: types.User{Name: "marc"},
+						User: users[1],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.4"),
-						User: types.User{Name: "mickael"},
+						User: users[2],
 					},
 				},
 			},
@@ -1185,7 +1422,7 @@ func Test_expandAlias(t *testing.T) {
 				nodes: types.Nodes{
 					&types.Node{
 						IPv4: iap("100.64.0.1"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 						Hostinfo: &tailcfg.Hostinfo{
 							OS:          "centos",
 							Hostname:    "foo",
@@ -1194,7 +1431,7 @@ func Test_expandAlias(t *testing.T) {
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.2"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 						Hostinfo: &tailcfg.Hostinfo{
 							OS:          "centos",
 							Hostname:    "foo",
@@ -1203,11 +1440,11 @@ func Test_expandAlias(t *testing.T) {
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.3"),
-						User: types.User{Name: "marc"},
+						User: users[1],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.4"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 					},
 				},
 			},
@@ -1260,21 +1497,21 @@ func Test_expandAlias(t *testing.T) {
 				nodes: types.Nodes{
 					&types.Node{
 						IPv4:       iap("100.64.0.1"),
-						User:       types.User{Name: "joe"},
+						User:       users[0],
 						ForcedTags: []string{"tag:hr-webserver"},
 					},
 					&types.Node{
 						IPv4:       iap("100.64.0.2"),
-						User:       types.User{Name: "joe"},
+						User:       users[0],
 						ForcedTags: []string{"tag:hr-webserver"},
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.3"),
-						User: types.User{Name: "marc"},
+						User: users[1],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.4"),
-						User: types.User{Name: "mickael"},
+						User: users[2],
 					},
 				},
 			},
@@ -1295,12 +1532,12 @@ func Test_expandAlias(t *testing.T) {
 				nodes: types.Nodes{
 					&types.Node{
 						IPv4:       iap("100.64.0.1"),
-						User:       types.User{Name: "joe"},
+						User:       users[0],
 						ForcedTags: []string{"tag:hr-webserver"},
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.2"),
-						User: types.User{Name: "joe"},
+						User: users[0],
 						Hostinfo: &tailcfg.Hostinfo{
 							OS:          "centos",
 							Hostname:    "foo",
@@ -1309,11 +1546,11 @@ func Test_expandAlias(t *testing.T) {
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.3"),
-						User: types.User{Name: "marc"},
+						User: users[1],
 					},
 					&types.Node{
 						IPv4: iap("100.64.0.4"),
-						User: types.User{Name: "mickael"},
+						User: users[2],
 					},
 				},
 			},
@@ -1350,12 +1587,12 @@ func Test_expandAlias(t *testing.T) {
 					},
 					&types.Node{
 						IPv4:     iap("100.64.0.3"),
-						User:     types.User{Name: "marc"},
+						User:     users[1],
 						Hostinfo: &tailcfg.Hostinfo{},
 					},
 					&types.Node{
 						IPv4:     iap("100.64.0.4"),
-						User:     types.User{Name: "joe"},
+						User:     users[0],
 						Hostinfo: &tailcfg.Hostinfo{},
 					},
 				},
@@ -1368,6 +1605,7 @@ func Test_expandAlias(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			got, err := test.field.pol.ExpandAlias(
 				test.args.nodes,
+				users,
 				test.args.alias,
 			)
 			if (err != nil) != test.wantErr {
@@ -1715,6 +1953,7 @@ func TestACLPolicy_generateFilterRules(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, err := tt.field.pol.CompileFilterRules(
+				[]types.User{},
 				tt.args.nodes,
 			)
 			if (err != nil) != tt.wantErr {
@@ -1834,6 +2073,13 @@ func TestTheInternet(t *testing.T) {
 }
 
 func TestReduceFilterRules(t *testing.T) {
+	users := []types.User{
+		{Model: gorm.Model{ID: 1}, Name: "mickael"},
+		{Model: gorm.Model{ID: 2}, Name: "user1"},
+		{Model: gorm.Model{ID: 3}, Name: "user2"},
+		{Model: gorm.Model{ID: 4}, Name: "user100"},
+	}
+
 	tests := []struct {
 		name  string
 		node  *types.Node
@@ -1855,13 +2101,13 @@ func TestReduceFilterRules(t *testing.T) {
 			node: &types.Node{
 				IPv4: iap("100.64.0.1"),
 				IPv6: iap("fd7a:115c:a1e0:ab12:4843:2222:6273:2221"),
-				User: types.User{Name: "mickael"},
+				User: users[0],
 			},
 			peers: types.Nodes{
 				&types.Node{
 					IPv4: iap("100.64.0.2"),
 					IPv6: iap("fd7a:115c:a1e0:ab12:4843:2222:6273:2222"),
-					User: types.User{Name: "mickael"},
+					User: users[0],
 				},
 			},
 			want: []tailcfg.FilterRule{},
@@ -1888,7 +2134,7 @@ func TestReduceFilterRules(t *testing.T) {
 			node: &types.Node{
 				IPv4: iap("100.64.0.1"),
 				IPv6: iap("fd7a:115c:a1e0::1"),
-				User: types.User{Name: "user1"},
+				User: users[1],
 				Hostinfo: &tailcfg.Hostinfo{
 					RoutableIPs: []netip.Prefix{
 						netip.MustParsePrefix("10.33.0.0/16"),
@@ -1899,7 +2145,7 @@ func TestReduceFilterRules(t *testing.T) {
 				&types.Node{
 					IPv4: iap("100.64.0.2"),
 					IPv6: iap("fd7a:115c:a1e0::2"),
-					User: types.User{Name: "user1"},
+					User: users[1],
 				},
 			},
 			want: []tailcfg.FilterRule{
@@ -1967,19 +2213,19 @@ func TestReduceFilterRules(t *testing.T) {
 			node: &types.Node{
 				IPv4: iap("100.64.0.1"),
 				IPv6: iap("fd7a:115c:a1e0::1"),
-				User: types.User{Name: "user1"},
+				User: users[1],
 			},
 			peers: types.Nodes{
 				&types.Node{
 					IPv4: iap("100.64.0.2"),
 					IPv6: iap("fd7a:115c:a1e0::2"),
-					User: types.User{Name: "user2"},
+					User: users[2],
 				},
 				// "internal" exit node
 				&types.Node{
 					IPv4: iap("100.64.0.100"),
 					IPv6: iap("fd7a:115c:a1e0::100"),
-					User: types.User{Name: "user100"},
+					User: users[3],
 					Hostinfo: &tailcfg.Hostinfo{
 						RoutableIPs: tsaddr.ExitRoutes(),
 					},
@@ -2026,12 +2272,12 @@ func TestReduceFilterRules(t *testing.T) {
 				&types.Node{
 					IPv4: iap("100.64.0.2"),
 					IPv6: iap("fd7a:115c:a1e0::2"),
-					User: types.User{Name: "user2"},
+					User: users[2],
 				},
 				&types.Node{
 					IPv4: iap("100.64.0.1"),
 					IPv6: iap("fd7a:115c:a1e0::1"),
-					User: types.User{Name: "user1"},
+					User: users[1],
 				},
 			},
 			want: []tailcfg.FilterRule{
@@ -2113,7 +2359,7 @@ func TestReduceFilterRules(t *testing.T) {
 			node: &types.Node{
 				IPv4: iap("100.64.0.100"),
 				IPv6: iap("fd7a:115c:a1e0::100"),
-				User: types.User{Name: "user100"},
+				User: users[3],
 				Hostinfo: &tailcfg.Hostinfo{
 					RoutableIPs: tsaddr.ExitRoutes(),
 				},
@@ -2122,12 +2368,12 @@ func TestReduceFilterRules(t *testing.T) {
 				&types.Node{
 					IPv4: iap("100.64.0.2"),
 					IPv6: iap("fd7a:115c:a1e0::2"),
-					User: types.User{Name: "user2"},
+					User: users[2],
 				},
 				&types.Node{
 					IPv4: iap("100.64.0.1"),
 					IPv6: iap("fd7a:115c:a1e0::1"),
-					User: types.User{Name: "user1"},
+					User: users[1],
 				},
 			},
 			want: []tailcfg.FilterRule{
@@ -2215,7 +2461,7 @@ func TestReduceFilterRules(t *testing.T) {
 			node: &types.Node{
 				IPv4: iap("100.64.0.100"),
 				IPv6: iap("fd7a:115c:a1e0::100"),
-				User: types.User{Name: "user100"},
+				User: users[3],
 				Hostinfo: &tailcfg.Hostinfo{
 					RoutableIPs: []netip.Prefix{netip.MustParsePrefix("8.0.0.0/16"), netip.MustParsePrefix("16.0.0.0/16")},
 				},
@@ -2224,12 +2470,12 @@ func TestReduceFilterRules(t *testing.T) {
 				&types.Node{
 					IPv4: iap("100.64.0.2"),
 					IPv6: iap("fd7a:115c:a1e0::2"),
-					User: types.User{Name: "user2"},
+					User: users[2],
 				},
 				&types.Node{
 					IPv4: iap("100.64.0.1"),
 					IPv6: iap("fd7a:115c:a1e0::1"),
-					User: types.User{Name: "user1"},
+					User: users[1],
 				},
 			},
 			want: []tailcfg.FilterRule{
@@ -2292,7 +2538,7 @@ func TestReduceFilterRules(t *testing.T) {
 			node: &types.Node{
 				IPv4: iap("100.64.0.100"),
 				IPv6: iap("fd7a:115c:a1e0::100"),
-				User: types.User{Name: "user100"},
+				User: users[3],
 				Hostinfo: &tailcfg.Hostinfo{
 					RoutableIPs: []netip.Prefix{netip.MustParsePrefix("8.0.0.0/8"), netip.MustParsePrefix("16.0.0.0/8")},
 				},
@@ -2301,12 +2547,12 @@ func TestReduceFilterRules(t *testing.T) {
 				&types.Node{
 					IPv4: iap("100.64.0.2"),
 					IPv6: iap("fd7a:115c:a1e0::2"),
-					User: types.User{Name: "user2"},
+					User: users[2],
 				},
 				&types.Node{
 					IPv4: iap("100.64.0.1"),
 					IPv6: iap("fd7a:115c:a1e0::1"),
-					User: types.User{Name: "user1"},
+					User: users[1],
 				},
 			},
 			want: []tailcfg.FilterRule{
@@ -2362,7 +2608,7 @@ func TestReduceFilterRules(t *testing.T) {
 			node: &types.Node{
 				IPv4: iap("100.64.0.100"),
 				IPv6: iap("fd7a:115c:a1e0::100"),
-				User: types.User{Name: "user100"},
+				User: users[3],
 				Hostinfo: &tailcfg.Hostinfo{
 					RoutableIPs: []netip.Prefix{netip.MustParsePrefix("172.16.0.0/24")},
 				},
@@ -2372,7 +2618,7 @@ func TestReduceFilterRules(t *testing.T) {
 				&types.Node{
 					IPv4: iap("100.64.0.1"),
 					IPv6: iap("fd7a:115c:a1e0::1"),
-					User: types.User{Name: "user1"},
+					User: users[1],
 				},
 			},
 			want: []tailcfg.FilterRule{
@@ -2400,6 +2646,7 @@ func TestReduceFilterRules(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			got, _ := tt.pol.CompileFilterRules(
+				users,
 				append(tt.peers, tt.node),
 			)
 
@@ -3391,7 +3638,7 @@ func TestSSHRules(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.pol.CompileSSHPolicy(&tt.node, tt.peers)
+			got, err := tt.pol.CompileSSHPolicy(&tt.node, []types.User{}, tt.peers)
 			assert.NoError(t, err)
 
 			if diff := cmp.Diff(tt.want, got); diff != "" {
@@ -3474,14 +3721,17 @@ func TestValidExpandTagOwnersInSources(t *testing.T) {
 		RequestTags: []string{"tag:test"},
 	}
 
+	user := types.User{
+		Model: gorm.Model{ID: 1},
+		Name:  "user1",
+	}
+
 	node := &types.Node{
-		ID:       0,
-		Hostname: "testnodes",
-		IPv4:     iap("100.64.0.1"),
-		UserID:   0,
-		User: types.User{
-			Name: "user1",
-		},
+		ID:             0,
+		Hostname:       "testnodes",
+		IPv4:           iap("100.64.0.1"),
+		UserID:         0,
+		User:           user,
 		RegisterMethod: util.RegisterMethodAuthKey,
 		Hostinfo:       &hostInfo,
 	}
@@ -3498,7 +3748,7 @@ func TestValidExpandTagOwnersInSources(t *testing.T) {
 		},
 	}
 
-	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{})
+	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{}, []types.User{user})
 	assert.NoError(t, err)
 
 	want := []tailcfg.FilterRule{
@@ -3532,7 +3782,8 @@ func TestInvalidTagValidUser(t *testing.T) {
 		IPv4:     iap("100.64.0.1"),
 		UserID:   1,
 		User: types.User{
-			Name: "user1",
+			Model: gorm.Model{ID: 1},
+			Name:  "user1",
 		},
 		RegisterMethod: util.RegisterMethodAuthKey,
 		Hostinfo:       &hostInfo,
@@ -3549,7 +3800,7 @@ func TestInvalidTagValidUser(t *testing.T) {
 		},
 	}
 
-	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{})
+	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{}, []types.User{node.User})
 	assert.NoError(t, err)
 
 	want := []tailcfg.FilterRule{
@@ -3583,7 +3834,8 @@ func TestValidExpandTagOwnersInDestinations(t *testing.T) {
 		IPv4:     iap("100.64.0.1"),
 		UserID:   1,
 		User: types.User{
-			Name: "user1",
+			Model: gorm.Model{ID: 1},
+			Name:  "user1",
 		},
 		RegisterMethod: util.RegisterMethodAuthKey,
 		Hostinfo:       &hostInfo,
@@ -3608,7 +3860,7 @@ func TestValidExpandTagOwnersInDestinations(t *testing.T) {
 	// c.Assert(rules[0].DstPorts, check.HasLen, 1)
 	// c.Assert(rules[0].DstPorts[0].IP, check.Equals, "100.64.0.1/32")
 
-	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{})
+	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{}, []types.User{node.User})
 	assert.NoError(t, err)
 
 	want := []tailcfg.FilterRule{
@@ -3637,15 +3889,17 @@ func TestValidTagInvalidUser(t *testing.T) {
 		Hostname:    "webserver",
 		RequestTags: []string{"tag:webapp"},
 	}
+	user := types.User{
+		Model: gorm.Model{ID: 1},
+		Name:  "user1",
+	}
 
 	node := &types.Node{
-		ID:       1,
-		Hostname: "webserver",
-		IPv4:     iap("100.64.0.1"),
-		UserID:   1,
-		User: types.User{
-			Name: "user1",
-		},
+		ID:             1,
+		Hostname:       "webserver",
+		IPv4:           iap("100.64.0.1"),
+		UserID:         1,
+		User:           user,
 		RegisterMethod: util.RegisterMethodAuthKey,
 		Hostinfo:       &hostInfo,
 	}
@@ -3656,13 +3910,11 @@ func TestValidTagInvalidUser(t *testing.T) {
 	}
 
 	nodes2 := &types.Node{
-		ID:       2,
-		Hostname: "user",
-		IPv4:     iap("100.64.0.2"),
-		UserID:   1,
-		User: types.User{
-			Name: "user1",
-		},
+		ID:             2,
+		Hostname:       "user",
+		IPv4:           iap("100.64.0.2"),
+		UserID:         1,
+		User:           user,
 		RegisterMethod: util.RegisterMethodAuthKey,
 		Hostinfo:       &hostInfo2,
 	}
@@ -3678,7 +3930,7 @@ func TestValidTagInvalidUser(t *testing.T) {
 		},
 	}
 
-	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{nodes2})
+	got, _, err := GenerateFilterAndSSHRulesForTests(pol, node, types.Nodes{nodes2}, []types.User{user})
 	assert.NoError(t, err)
 
 	want := []tailcfg.FilterRule{

hscontrol/types/config.go

@@ -28,8 +28,9 @@ const (
 	maxDuration           time.Duration = 1<<63 - 1
 )
 
-var errOidcMutuallyExclusive = errors.New(
-	"oidc_client_secret and oidc_client_secret_path are mutually exclusive",
+var (
+	errOidcMutuallyExclusive = errors.New("oidc_client_secret and oidc_client_secret_path are mutually exclusive")
+	errServerURLSuffix       = errors.New("server_url cannot be part of base_domain in a way that could make the DERP and headscale server unreachable")
 )
 
 type IPAllocationStrategy string
@@ -827,11 +828,10 @@ func LoadServerConfig() (*Config, error) {
 	// - DERP run on their own domains
 	// - Control plane runs on login.tailscale.com/controlplane.tailscale.com
 	// - MagicDNS (BaseDomain) for users is on a *.ts.net domain per tailnet (e.g. tail-scale.ts.net)
-	if dnsConfig.BaseDomain != "" &&
-		strings.Contains(serverURL, dnsConfig.BaseDomain) {
-		return nil, errors.New(
-			"server_url cannot contain the base_domain, this will cause the headscale server and embedded DERP to become unreachable from the Tailscale node.",
-		)
+	if dnsConfig.BaseDomain != "" {
+		if err := isSafeServerURL(serverURL, dnsConfig.BaseDomain); err != nil {
+			return nil, err
+		}
 	}
 
 	return &Config{
@@ -924,6 +924,37 @@ func LoadServerConfig() (*Config, error) {
 	}, nil
 }
 
+// BaseDomain cannot be a suffix of the server URL.
+// This is because Tailscale takes over the domain in BaseDomain,
+// causing the headscale server and DERP to be unreachable.
+// For Tailscale upstream, the following is true:
+// - DERP run on their own domains.
+// - Control plane runs on login.tailscale.com/controlplane.tailscale.com.
+// - MagicDNS (BaseDomain) for users is on a *.ts.net domain per tailnet (e.g. tail-scale.ts.net).
+func isSafeServerURL(serverURL, baseDomain string) error {
+	server, err := url.Parse(serverURL)
+	if err != nil {
+		return err
+	}
+
+	serverDomainParts := strings.Split(server.Host, ".")
+	baseDomainParts := strings.Split(baseDomain, ".")
+
+	if len(serverDomainParts) <= len(baseDomainParts) {
+		return nil
+	}
+
+	s := len(serverDomainParts)
+	b := len(baseDomainParts)
+	for i := range len(baseDomainParts) {
+		if serverDomainParts[s-i-1] != baseDomainParts[b-i-1] {
+			return nil
+		}
+	}
+
+	return errServerURLSuffix
+}
+
 type deprecator struct {
 	warns  set.Set[string]
 	fatals set.Set[string]

hscontrol/types/config_test.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -139,7 +140,7 @@ func TestReadConfig(t *testing.T) {
 				return LoadServerConfig()
 			},
 			want:    nil,
-			wantErr: "server_url cannot contain the base_domain, this will cause the headscale server and embedded DERP to become unreachable from the Tailscale node.",
+			wantErr: errServerURLSuffix.Error(),
 		},
 		{
 			name:       "base-domain-not-in-server-url",
@@ -333,3 +334,64 @@ tls_letsencrypt_challenge_type: TLS-ALPN-01
 	err = LoadConfig(tmpDir, false)
 	assert.NoError(t, err)
 }
+
+// OK
+// server_url: headscale.com, base: clients.headscale.com
+// server_url: headscale.com, base: headscale.net
+//
+// NOT OK
+// server_url: server.headscale.com, base: headscale.com.
+func TestSafeServerURL(t *testing.T) {
+	tests := []struct {
+		serverURL, baseDomain,
+		wantErr string
+	}{
+		{
+			serverURL:  "https://example.com",
+			baseDomain: "example.org",
+		},
+		{
+			serverURL:  "https://headscale.com",
+			baseDomain: "headscale.com",
+		},
+		{
+			serverURL:  "https://headscale.com",
+			baseDomain: "clients.headscale.com",
+		},
+		{
+			serverURL:  "https://headscale.com",
+			baseDomain: "clients.subdomain.headscale.com",
+		},
+		{
+			serverURL:  "https://headscale.kristoffer.com",
+			baseDomain: "mybase",
+		},
+		{
+			serverURL:  "https://server.headscale.com",
+			baseDomain: "headscale.com",
+			wantErr:    errServerURLSuffix.Error(),
+		},
+		{
+			serverURL:  "https://server.subdomain.headscale.com",
+			baseDomain: "headscale.com",
+			wantErr:    errServerURLSuffix.Error(),
+		},
+		{
+			serverURL: "http://foo\x00",
+			wantErr:   `parse "http://foo\x00": net/url: invalid control character in URL`,
+		},
+	}
+
+	for _, tt := range tests {
+		testName := fmt.Sprintf("server=%s domain=%s", tt.serverURL, tt.baseDomain)
+		t.Run(testName, func(t *testing.T) {
+			err := isSafeServerURL(tt.serverURL, tt.baseDomain)
+			if tt.wantErr != "" {
+				assert.EqualError(t, err, tt.wantErr)
+
+				return
+			}
+			assert.NoError(t, err)
+		})
+	}
+}

hscontrol/types/node.go

@@ -223,6 +223,16 @@ func (nodes Nodes) FilterByIP(ip netip.Addr) Nodes {
 	return found
 }
 
+func (nodes Nodes) ContainsNodeKey(nodeKey key.NodePublic) bool {
+	for _, node := range nodes {
+		if node.NodeKey == nodeKey {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (node *Node) Proto() *v1.Node {
 	nodeProto := &v1.Node{
 		Id:         uint64(node.ID),

hscontrol/types/testdata/base-domain-in-server-url.yaml

@@ -8,7 +8,7 @@ prefixes:
 database:
   type: sqlite3
 
-server_url: "https://derp.no"
+server_url: "https://server.derp.no"
 
 dns:
   magic_dns: true

hscontrol/types/users.go

@@ -2,6 +2,8 @@ package types
 
 import (
 	"cmp"
+	"database/sql"
+	"net/mail"
 	"strconv"
 
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
@@ -34,7 +36,7 @@ type User struct {
 	// Unique identifier of the user from OIDC,
 	// comes from `sub` claim in the OIDC token
 	// and is used to lookup the user.
-	ProviderIdentifier string `gorm:"index"`
+	ProviderIdentifier sql.NullString `gorm:"index"`
 
 	// Provider is the origin of the user account,
 	// same as RegistrationMethod, without authkey.
@@ -51,7 +53,7 @@ type User struct {
 // should be used throughout headscale, in information returned to the
 // user and the Policy engine.
 func (u *User) Username() string {
-	return cmp.Or(u.Email, u.Name, u.ProviderIdentifier, strconv.FormatUint(uint64(u.ID), 10))
+	return cmp.Or(u.Email, u.Name, u.ProviderIdentifier.String, strconv.FormatUint(uint64(u.ID), 10))
 }
 
 // DisplayNameOrUsername returns the DisplayName if it exists, otherwise
@@ -107,7 +109,7 @@ func (u *User) Proto() *v1.User {
 		CreatedAt:     timestamppb.New(u.CreatedAt),
 		DisplayName:   u.DisplayName,
 		Email:         u.Email,
-		ProviderId:    u.ProviderIdentifier,
+		ProviderId:    u.ProviderIdentifier.String,
 		Provider:      u.Provider,
 		ProfilePicUrl: u.ProfilePicURL,
 	}
@@ -129,10 +131,20 @@ type OIDCClaims struct {
 // FromClaim overrides a User from OIDC claims.
 // All fields will be updated, except for the ID.
 func (u *User) FromClaim(claims *OIDCClaims) {
-	u.ProviderIdentifier = claims.Sub
+	err := util.CheckForFQDNRules(claims.Username)
+	if err == nil {
+		u.Name = claims.Username
+	}
+
+	if claims.EmailVerified {
+		_, err = mail.ParseAddress(claims.Email)
+		if err == nil {
+			u.Email = claims.Email
+		}
+	}
+
+	u.ProviderIdentifier.String = claims.Sub
 	u.DisplayName = claims.Name
-	u.Email = claims.Email
-	u.Name = claims.Username
 	u.ProfilePicURL = claims.ProfilePictureURL
 	u.Provider = util.RegisterMethodOIDC
 }

integration/README.md

@@ -11,10 +11,10 @@ Tests are located in files ending with `_test.go` and the framework are located
 
 ## Running integration tests locally
 
-The easiest way to run tests locally is to use `[act](INSERT LINK)`, a local GitHub Actions runner:
+The easiest way to run tests locally is to use [act](https://github.com/nektos/act), a local GitHub Actions runner:
 
 ```
-act pull_request -W .github/workflows/test-integration-v2-TestPingAllByIP.yaml
+act pull_request -W .github/workflows/test-integration.yaml
 ```
 
 Alternatively, the `docker run` command in each GitHub workflow file can be used.

integration/derp_verify_endpoint_test.go

@@ -0,0 +1,96 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dsic"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/juanfont/headscale/integration/tsic"
+	"tailscale.com/tailcfg"
+)
+
+func TestDERPVerifyEndpoint(t *testing.T) {
+	IntegrationSkip(t)
+
+	// Generate random hostname for the headscale instance
+	hash, err := util.GenerateRandomStringDNSSafe(6)
+	assertNoErr(t, err)
+	testName := "derpverify"
+	hostname := fmt.Sprintf("hs-%s-%s", testName, hash)
+
+	headscalePort := 8080
+
+	// Create cert for headscale
+	certHeadscale, keyHeadscale, err := integrationutil.CreateCertificate(hostname)
+	assertNoErr(t, err)
+
+	scenario, err := NewScenario(dockertestMaxWait())
+	assertNoErr(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	spec := map[string]int{
+		"user1": len(MustTestVersions),
+	}
+
+	derper, err := scenario.CreateDERPServer("head",
+		dsic.WithCACert(certHeadscale),
+		dsic.WithVerifyClientURL(fmt.Sprintf("https://%s/verify", net.JoinHostPort(hostname, strconv.Itoa(headscalePort)))),
+	)
+	assertNoErr(t, err)
+
+	derpMap := tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			900: {
+				RegionID:   900,
+				RegionCode: "test-derpverify",
+				RegionName: "TestDerpVerify",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:     "TestDerpVerify",
+						RegionID: 900,
+						HostName: derper.GetHostname(),
+						STUNPort: derper.GetSTUNPort(),
+						STUNOnly: false,
+						DERPPort: derper.GetDERPPort(),
+					},
+				},
+			},
+		},
+	}
+
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{tsic.WithCACert(derper.GetCert())},
+		hsic.WithHostname(hostname),
+		hsic.WithPort(headscalePort),
+		hsic.WithCustomTLS(certHeadscale, keyHeadscale),
+		hsic.WithHostnameAsServerURL(),
+		hsic.WithDERPConfig(derpMap))
+	assertNoErrHeadscaleEnv(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	assertNoErrListClients(t, err)
+
+	for _, client := range allClients {
+		report, err := client.DebugDERPRegion("test-derpverify")
+		assertNoErr(t, err)
+		successful := false
+		for _, line := range report.Info {
+			if strings.Contains(line, "Successfully established a DERP connection with node") {
+				successful = true
+
+				break
+			}
+		}
+		if !successful {
+			stJSON, err := json.Marshal(report)
+			assertNoErr(t, err)
+			t.Errorf("Client %s could not establish a DERP connection: %s", client.Hostname(), string(stJSON))
+		}
+	}
+}

integration/dsic/dsic.go

@@ -0,0 +1,321 @@
+package dsic
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+const (
+	dsicHashLength       = 6
+	dockerContextPath    = "../."
+	caCertRoot           = "/usr/local/share/ca-certificates"
+	DERPerCertRoot       = "/usr/local/share/derper-certs"
+	dockerExecuteTimeout = 60 * time.Second
+)
+
+var errDERPerStatusCodeNotOk = errors.New("DERPer status code not OK")
+
+// DERPServerInContainer represents DERP Server in Container (DSIC).
+type DERPServerInContainer struct {
+	version  string
+	hostname string
+
+	pool      *dockertest.Pool
+	container *dockertest.Resource
+	network   *dockertest.Network
+
+	stunPort            int
+	derpPort            int
+	caCerts             [][]byte
+	tlsCert             []byte
+	tlsKey              []byte
+	withExtraHosts      []string
+	withVerifyClientURL string
+	workdir             string
+}
+
+// Option represent optional settings that can be given to a
+// DERPer instance.
+type Option = func(c *DERPServerInContainer)
+
+// WithCACert adds it to the trusted surtificate of the Tailscale container.
+func WithCACert(cert []byte) Option {
+	return func(dsic *DERPServerInContainer) {
+		dsic.caCerts = append(dsic.caCerts, cert)
+	}
+}
+
+// WithOrCreateNetwork sets the Docker container network to use with
+// the DERPer instance, if the parameter is nil, a new network,
+// isolating the DERPer, will be created. If a network is
+// passed, the DERPer instance will join the given network.
+func WithOrCreateNetwork(network *dockertest.Network) Option {
+	return func(tsic *DERPServerInContainer) {
+		if network != nil {
+			tsic.network = network
+
+			return
+		}
+
+		network, err := dockertestutil.GetFirstOrCreateNetwork(
+			tsic.pool,
+			tsic.hostname+"-network",
+		)
+		if err != nil {
+			log.Fatalf("failed to create network: %s", err)
+		}
+
+		tsic.network = network
+	}
+}
+
+// WithDockerWorkdir allows the docker working directory to be set.
+func WithDockerWorkdir(dir string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.workdir = dir
+	}
+}
+
+// WithVerifyClientURL sets the URL to verify the client.
+func WithVerifyClientURL(url string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.withVerifyClientURL = url
+	}
+}
+
+// WithExtraHosts adds extra hosts to the container.
+func WithExtraHosts(hosts []string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.withExtraHosts = hosts
+	}
+}
+
+// New returns a new TailscaleInContainer instance.
+func New(
+	pool *dockertest.Pool,
+	version string,
+	network *dockertest.Network,
+	opts ...Option,
+) (*DERPServerInContainer, error) {
+	hash, err := util.GenerateRandomStringDNSSafe(dsicHashLength)
+	if err != nil {
+		return nil, err
+	}
+
+	hostname := fmt.Sprintf("derp-%s-%s", strings.ReplaceAll(version, ".", "-"), hash)
+	tlsCert, tlsKey, err := integrationutil.CreateCertificate(hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create certificates for headscale test: %w", err)
+	}
+	dsic := &DERPServerInContainer{
+		version:  version,
+		hostname: hostname,
+		pool:     pool,
+		network:  network,
+		tlsCert:  tlsCert,
+		tlsKey:   tlsKey,
+		stunPort: 3478, //nolint
+		derpPort: 443,  //nolint
+	}
+
+	for _, opt := range opts {
+		opt(dsic)
+	}
+
+	var cmdArgs strings.Builder
+	fmt.Fprintf(&cmdArgs, "--hostname=%s", hostname)
+	fmt.Fprintf(&cmdArgs, " --certmode=manual")
+	fmt.Fprintf(&cmdArgs, " --certdir=%s", DERPerCertRoot)
+	fmt.Fprintf(&cmdArgs, " --a=:%d", dsic.derpPort)
+	fmt.Fprintf(&cmdArgs, " --stun=true")
+	fmt.Fprintf(&cmdArgs, " --stun-port=%d", dsic.stunPort)
+	if dsic.withVerifyClientURL != "" {
+		fmt.Fprintf(&cmdArgs, " --verify-client-url=%s", dsic.withVerifyClientURL)
+	}
+
+	runOptions := &dockertest.RunOptions{
+		Name:       hostname,
+		Networks:   []*dockertest.Network{dsic.network},
+		ExtraHosts: dsic.withExtraHosts,
+		// we currently need to give us some time to inject the certificate further down.
+		Entrypoint: []string{"/bin/sh", "-c", "/bin/sleep 3 ; update-ca-certificates ; derper " + cmdArgs.String()},
+		ExposedPorts: []string{
+			"80/tcp",
+			fmt.Sprintf("%d/tcp", dsic.derpPort),
+			fmt.Sprintf("%d/udp", dsic.stunPort),
+		},
+	}
+
+	if dsic.workdir != "" {
+		runOptions.WorkingDir = dsic.workdir
+	}
+
+	// dockertest isnt very good at handling containers that has already
+	// been created, this is an attempt to make sure this container isnt
+	// present.
+	err = pool.RemoveContainerByName(hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	var container *dockertest.Resource
+	buildOptions := &dockertest.BuildOptions{
+		Dockerfile: "Dockerfile.derper",
+		ContextDir: dockerContextPath,
+		BuildArgs:  []docker.BuildArg{},
+	}
+	switch version {
+	case "head":
+		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+			Name:  "VERSION_BRANCH",
+			Value: "main",
+		})
+	default:
+		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+			Name:  "VERSION_BRANCH",
+			Value: "v" + version,
+		})
+	}
+	container, err = pool.BuildAndRunWithBuildOptions(
+		buildOptions,
+		runOptions,
+		dockertestutil.DockerRestartPolicy,
+		dockertestutil.DockerAllowLocalIPv6,
+		dockertestutil.DockerAllowNetworkAdministration,
+	)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%s could not start tailscale DERPer container (version: %s): %w",
+			hostname,
+			version,
+			err,
+		)
+	}
+	log.Printf("Created %s container\n", hostname)
+
+	dsic.container = container
+
+	for i, cert := range dsic.caCerts {
+		err = dsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+	if len(dsic.tlsCert) != 0 {
+		err = dsic.WriteFile(fmt.Sprintf("%s/%s.crt", DERPerCertRoot, dsic.hostname), dsic.tlsCert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+	if len(dsic.tlsKey) != 0 {
+		err = dsic.WriteFile(fmt.Sprintf("%s/%s.key", DERPerCertRoot, dsic.hostname), dsic.tlsKey)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS key to container: %w", err)
+		}
+	}
+
+	return dsic, nil
+}
+
+// Shutdown stops and cleans up the DERPer container.
+func (t *DERPServerInContainer) Shutdown() error {
+	err := t.SaveLog("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"Failed to save log from %s: %s",
+			t.hostname,
+			fmt.Errorf("failed to save log: %w", err),
+		)
+	}
+
+	return t.pool.Purge(t.container)
+}
+
+// GetCert returns the TLS certificate of the DERPer instance.
+func (t *DERPServerInContainer) GetCert() []byte {
+	return t.tlsCert
+}
+
+// Hostname returns the hostname of the DERPer instance.
+func (t *DERPServerInContainer) Hostname() string {
+	return t.hostname
+}
+
+// Version returns the running DERPer version of the instance.
+func (t *DERPServerInContainer) Version() string {
+	return t.version
+}
+
+// ID returns the Docker container ID of the DERPServerInContainer
+// instance.
+func (t *DERPServerInContainer) ID() string {
+	return t.container.Container.ID
+}
+
+func (t *DERPServerInContainer) GetHostname() string {
+	return t.hostname
+}
+
+// GetSTUNPort returns the STUN port of the DERPer instance.
+func (t *DERPServerInContainer) GetSTUNPort() int {
+	return t.stunPort
+}
+
+// GetDERPPort returns the DERP port of the DERPer instance.
+func (t *DERPServerInContainer) GetDERPPort() int {
+	return t.derpPort
+}
+
+// WaitForRunning blocks until the DERPer instance is ready to be used.
+func (t *DERPServerInContainer) WaitForRunning() error {
+	url := "https://" + net.JoinHostPort(t.GetHostname(), strconv.Itoa(t.GetDERPPort())) + "/"
+	log.Printf("waiting for DERPer to be ready at %s", url)
+
+	insecureTransport := http.DefaultTransport.(*http.Transport).Clone()      //nolint
+	insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint
+	client := &http.Client{Transport: insecureTransport}
+
+	return t.pool.Retry(func() error {
+		resp, err := client.Get(url) //nolint
+		if err != nil {
+			return fmt.Errorf("headscale is not ready: %w", err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return errDERPerStatusCodeNotOk
+		}
+
+		return nil
+	})
+}
+
+// ConnectToNetwork connects the DERPer instance to a network.
+func (t *DERPServerInContainer) ConnectToNetwork(network *dockertest.Network) error {
+	return t.container.ConnectToNetwork(network)
+}
+
+// WriteFile save file inside the container.
+func (t *DERPServerInContainer) WriteFile(path string, data []byte) error {
+	return integrationutil.WriteFileToContainer(t.pool, t.container, path, data)
+}
+
+// SaveLog saves the current stdout log of the container to a path
+// on the host system.
+func (t *DERPServerInContainer) SaveLog(path string) error {
+	_, _, err := dockertestutil.SaveLog(t.pool, t.container, path)
+
+	return err
+}

integration/embedded_derp_test.go

@@ -55,7 +55,7 @@ func TestDERPServerWebsocketScenario(t *testing.T) {
 	spec := map[string]ClientsSpec{
 		"user1": {
 			Plain:         0,
-			WebsocketDERP: len(MustTestVersions),
+			WebsocketDERP: 2,
 		},
 	}
 
@@ -239,10 +239,13 @@ func (s *EmbeddedDERPServerScenario) CreateHeadscaleEnv(
 
 		if clientCount.WebsocketDERP > 0 {
 			// Containers that use DERP-over-WebSocket
+			// Note that these clients *must* be built
+			// from source, which is currently
+			// only done for HEAD.
 			err = s.CreateTailscaleIsolatedNodesInUser(
 				hash,
 				userName,
-				"all",
+				tsic.VersionHead,
 				clientCount.WebsocketDERP,
 				tsic.WithWebsocketDERP(true),
 			)
@@ -307,7 +310,7 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser(
 			cert := hsServer.GetCert()
 
 			opts = append(opts,
-				tsic.WithHeadscaleTLS(cert),
+				tsic.WithCACert(cert),
 			)
 
 			user.createWaitGroup.Go(func() error {

integration/hsic/hsic.go

@@ -1,19 +1,12 @@
 package hsic
 
 import (
-	"bytes"
-	"crypto/rand"
-	"crypto/rsa"
 	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
 	"encoding/json"
-	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
 	"log"
-	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -32,11 +25,14 @@ import (
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
+	"gopkg.in/yaml.v3"
+	"tailscale.com/tailcfg"
 )
 
 const (
 	hsicHashLength       = 6
 	dockerContextPath    = "../."
+	caCertRoot           = "/usr/local/share/ca-certificates"
 	aclPolicyPath        = "/etc/headscale/acl.hujson"
 	tlsCertPath          = "/etc/headscale/tls.cert"
 	tlsKeyPath           = "/etc/headscale/tls.key"
@@ -64,6 +60,7 @@ type HeadscaleInContainer struct {
 	// optional config
 	port             int
 	extraPorts       []string
+	caCerts          [][]byte
 	hostPortBindings map[string][]string
 	aclPolicy        *policy.ACLPolicy
 	env              map[string]string
@@ -88,18 +85,29 @@ func WithACLPolicy(acl *policy.ACLPolicy) Option {
 	}
 }
 
+// WithCACert adds it to the trusted surtificate of the container.
+func WithCACert(cert []byte) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.caCerts = append(hsic.caCerts, cert)
+	}
+}
+
 // WithTLS creates certificates and enables HTTPS.
 func WithTLS() Option {
 	return func(hsic *HeadscaleInContainer) {
-		cert, key, err := createCertificate(hsic.hostname)
+		cert, key, err := integrationutil.CreateCertificate(hsic.hostname)
 		if err != nil {
 			log.Fatalf("failed to create certificates for headscale test: %s", err)
 		}
 
-		// TODO(kradalby): Move somewhere appropriate
-		hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath
-		hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath
+		hsic.tlsCert = cert
+		hsic.tlsKey = key
+	}
+}
 
+// WithCustomTLS uses the given certificates for the Headscale instance.
+func WithCustomTLS(cert, key []byte) Option {
+	return func(hsic *HeadscaleInContainer) {
 		hsic.tlsCert = cert
 		hsic.tlsKey = key
 	}
@@ -146,6 +154,13 @@ func WithTestName(testName string) Option {
 	}
 }
 
+// WithHostname sets the hostname of the Headscale instance.
+func WithHostname(hostname string) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.hostname = hostname
+	}
+}
+
 // WithHostnameAsServerURL sets the Headscale ServerURL based on
 // the Hostname.
 func WithHostnameAsServerURL() Option {
@@ -203,6 +218,34 @@ func WithEmbeddedDERPServerOnly() Option {
 	}
 }
 
+// WithDERPConfig configures Headscale use a custom
+// DERP server only.
+func WithDERPConfig(derpMap tailcfg.DERPMap) Option {
+	return func(hsic *HeadscaleInContainer) {
+		contents, err := yaml.Marshal(derpMap)
+		if err != nil {
+			log.Fatalf("failed to marshal DERP map: %s", err)
+
+			return
+		}
+
+		hsic.env["HEADSCALE_DERP_PATHS"] = "/etc/headscale/derp.yml"
+		hsic.filesInContainer = append(hsic.filesInContainer,
+			fileInContainer{
+				path:     "/etc/headscale/derp.yml",
+				contents: contents,
+			})
+
+		// Disable global DERP server and embedded DERP server
+		hsic.env["HEADSCALE_DERP_URLS"] = ""
+		hsic.env["HEADSCALE_DERP_SERVER_ENABLED"] = "false"
+
+		// Envknob for enabling DERP debug logs
+		hsic.env["DERP_DEBUG_LOGS"] = "true"
+		hsic.env["DERP_PROBER_DEBUG_LOGS"] = "true"
+	}
+}
+
 // WithTuning allows changing the tuning settings easily.
 func WithTuning(batchTimeout time.Duration, mapSessionChanSize int) Option {
 	return func(hsic *HeadscaleInContainer) {
@@ -300,6 +343,10 @@ func New(
 		"HEADSCALE_DEBUG_HIGH_CARDINALITY_METRICS=1",
 		"HEADSCALE_DEBUG_DUMP_CONFIG=1",
 	}
+	if hsic.hasTLS() {
+		hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath
+		hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath
+	}
 	for key, value := range hsic.env {
 		env = append(env, fmt.Sprintf("%s=%s", key, value))
 	}
@@ -313,7 +360,7 @@ func New(
 		// Cmd:          []string{"headscale", "serve"},
 		// TODO(kradalby): Get rid of this hack, we currently need to give us some
 		// to inject the headscale configuration further down.
-		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"},
+		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; update-ca-certificates ; headscale serve ; /bin/sleep 30"},
 		Env:        env,
 	}
 
@@ -351,6 +398,14 @@ func New(
 
 	hsic.container = container
 
+	// Write the CA certificates to the container
+	for i, cert := range hsic.caCerts {
+		err = hsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+
 	err = hsic.WriteFile("/etc/headscale/config.yaml", []byte(MinimumConfigYAML()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to write headscale config to container: %w", err)
@@ -749,86 +804,3 @@ func (t *HeadscaleInContainer) SendInterrupt() error {
 
 	return nil
 }
-
-// nolint
-func createCertificate(hostname string) ([]byte, []byte, error) {
-	// From:
-	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
-
-	ca := &x509.Certificate{
-		SerialNumber: big.NewInt(2019),
-		Subject: pkix.Name{
-			Organization: []string{"Headscale testing INC"},
-			Country:      []string{"NL"},
-			Locality:     []string{"Leiden"},
-		},
-		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(60 * time.Hour),
-		IsCA:      true,
-		ExtKeyUsage: []x509.ExtKeyUsage{
-			x509.ExtKeyUsageClientAuth,
-			x509.ExtKeyUsageServerAuth,
-		},
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
-		BasicConstraintsValid: true,
-	}
-
-	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cert := &x509.Certificate{
-		SerialNumber: big.NewInt(1658),
-		Subject: pkix.Name{
-			CommonName:   hostname,
-			Organization: []string{"Headscale testing INC"},
-			Country:      []string{"NL"},
-			Locality:     []string{"Leiden"},
-		},
-		NotBefore:    time.Now(),
-		NotAfter:     time.Now().Add(60 * time.Minute),
-		SubjectKeyId: []byte{1, 2, 3, 4, 6},
-		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		KeyUsage:     x509.KeyUsageDigitalSignature,
-		DNSNames:     []string{hostname},
-	}
-
-	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certBytes, err := x509.CreateCertificate(
-		rand.Reader,
-		cert,
-		ca,
-		&certPrivKey.PublicKey,
-		caPrivKey,
-	)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certPEM := new(bytes.Buffer)
-
-	err = pem.Encode(certPEM, &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: certBytes,
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certPrivKeyPEM := new(bytes.Buffer)
-
-	err = pem.Encode(certPrivKeyPEM, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
-}

integration/integrationutil/util.go

@@ -3,9 +3,16 @@ package integrationutil
 import (
 	"archive/tar"
 	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"math/big"
 	"path/filepath"
+	"time"
 
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/ory/dockertest/v3"
@@ -93,3 +100,86 @@ func FetchPathFromContainer(
 
 	return buf.Bytes(), nil
 }
+
+// nolint
+func CreateCertificate(hostname string) ([]byte, []byte, error) {
+	// From:
+	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
+
+	ca := &x509.Certificate{
+		SerialNumber: big.NewInt(2019),
+		Subject: pkix.Name{
+			Organization: []string{"Headscale testing INC"},
+			Country:      []string{"NL"},
+			Locality:     []string{"Leiden"},
+		},
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().Add(60 * time.Hour),
+		IsCA:      true,
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+		},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+
+	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cert := &x509.Certificate{
+		SerialNumber: big.NewInt(1658),
+		Subject: pkix.Name{
+			CommonName:   hostname,
+			Organization: []string{"Headscale testing INC"},
+			Country:      []string{"NL"},
+			Locality:     []string{"Leiden"},
+		},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().Add(60 * time.Minute),
+		SubjectKeyId: []byte{1, 2, 3, 4, 6},
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		DNSNames:     []string{hostname},
+	}
+
+	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certBytes, err := x509.CreateCertificate(
+		rand.Reader,
+		cert,
+		ca,
+		&certPrivKey.PublicKey,
+		caPrivKey,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certPEM := new(bytes.Buffer)
+
+	err = pem.Encode(certPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certPrivKeyPEM := new(bytes.Buffer)
+
+	err = pem.Encode(certPrivKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
+}

integration/scenario.go

@@ -14,6 +14,7 @@ import (
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/dsic"
 	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/ory/dockertest/v3"
@@ -140,6 +141,7 @@ type Scenario struct {
 	// TODO(kradalby): support multiple headcales for later, currently only
 	// use one.
 	controlServers *xsync.MapOf[string, ControlServer]
+	derpServers    []*dsic.DERPServerInContainer
 
 	users map[string]*User
 
@@ -224,6 +226,13 @@ func (s *Scenario) ShutdownAssertNoPanics(t *testing.T) {
 		}
 	}
 
+	for _, derp := range s.derpServers {
+		err := derp.Shutdown()
+		if err != nil {
+			log.Printf("failed to tear down derp server: %s", err)
+		}
+	}
+
 	if err := s.pool.RemoveNetwork(s.network); err != nil {
 		log.Printf("failed to remove network: %s", err)
 	}
@@ -352,7 +361,7 @@ func (s *Scenario) CreateTailscaleNodesInUser(
 			hostname := headscale.GetHostname()
 
 			opts = append(opts,
-				tsic.WithHeadscaleTLS(cert),
+				tsic.WithCACert(cert),
 				tsic.WithHeadscaleName(hostname),
 			)
 
@@ -651,3 +660,20 @@ func (s *Scenario) WaitForTailscaleLogout() error {
 
 	return nil
 }
+
+// CreateDERPServer creates a new DERP server in a container.
+func (s *Scenario) CreateDERPServer(version string, opts ...dsic.Option) (*dsic.DERPServerInContainer, error) {
+	derp, err := dsic.New(s.pool, version, s.network, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DERP server: %w", err)
+	}
+
+	err = derp.WaitForRunning()
+	if err != nil {
+		return nil, fmt.Errorf("failed to reach DERP server: %w", err)
+	}
+
+	s.derpServers = append(s.derpServers, derp)
+
+	return derp, nil
+}

integration/tailscale.go

@@ -30,6 +30,7 @@ type TailscaleClient interface {
 	FQDN() (string, error)
 	Status(...bool) (*ipnstate.Status, error)
 	Netmap() (*netmap.NetworkMap, error)
+	DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error)
 	Netcheck() (*netcheck.Report, error)
 	WaitForNeedsLogin() error
 	WaitForRunning() error

integration/tsic/tsic.go

@@ -12,6 +12,7 @@ import (
 	"net/netip"
 	"net/url"
 	"os"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -32,7 +33,7 @@ const (
 	defaultPingTimeout   = 300 * time.Millisecond
 	defaultPingCount     = 10
 	dockerContextPath    = "../."
-	headscaleCertPath    = "/usr/local/share/ca-certificates/headscale.crt"
+	caCertRoot           = "/usr/local/share/ca-certificates"
 	dockerExecuteTimeout = 60 * time.Second
 )
 
@@ -44,6 +45,11 @@ var (
 	errTailscaleCannotUpWithoutAuthkey = errors.New("cannot up without authkey")
 	errTailscaleNotConnected           = errors.New("tailscale not connected")
 	errTailscaledNotReadyForLogin      = errors.New("tailscaled not ready for login")
+	errInvalidClientConfig             = errors.New("verifiably invalid client config requested")
+)
+
+const (
+	VersionHead = "head"
 )
 
 func errTailscaleStatus(hostname string, err error) error {
@@ -65,7 +71,7 @@ type TailscaleInContainer struct {
 	fqdn string
 
 	// optional config
-	headscaleCert     []byte
+	caCerts           [][]byte
 	headscaleHostname string
 	withWebsocketDERP bool
 	withSSH           bool
@@ -74,17 +80,23 @@ type TailscaleInContainer struct {
 	withExtraHosts    []string
 	workdir           string
 	netfilter         string
+
+	// build options, solely for HEAD
+	buildConfig TailscaleInContainerBuildConfig
+}
+
+type TailscaleInContainerBuildConfig struct {
+	tags []string
 }
 
 // Option represent optional settings that can be given to a
 // Tailscale instance.
 type Option = func(c *TailscaleInContainer)
 
-// WithHeadscaleTLS takes the certificate of the Headscale instance
-// and adds it to the trusted surtificate of the Tailscale container.
-func WithHeadscaleTLS(cert []byte) Option {
+// WithCACert adds it to the trusted surtificate of the Tailscale container.
+func WithCACert(cert []byte) Option {
 	return func(tsic *TailscaleInContainer) {
-		tsic.headscaleCert = cert
+		tsic.caCerts = append(tsic.caCerts, cert)
 	}
 }
 
@@ -113,7 +125,7 @@ func WithOrCreateNetwork(network *dockertest.Network) Option {
 }
 
 // WithHeadscaleName set the name of the headscale instance,
-// mostly useful in combination with TLS and WithHeadscaleTLS.
+// mostly useful in combination with TLS and WithCACert.
 func WithHeadscaleName(hsName string) Option {
 	return func(tsic *TailscaleInContainer) {
 		tsic.headscaleHostname = hsName
@@ -175,6 +187,22 @@ func WithNetfilter(state string) Option {
 	}
 }
 
+// WithBuildTag adds an additional value to the `-tags=` parameter
+// of the Go compiler, allowing callers to customize the Tailscale client build.
+// This option is only meaningful when invoked on **HEAD** versions of the client.
+// Attempts to use it with any other version is a bug in the calling code.
+func WithBuildTag(tag string) Option {
+	return func(tsic *TailscaleInContainer) {
+		if tsic.version != VersionHead {
+			panic(errInvalidClientConfig)
+		}
+
+		tsic.buildConfig.tags = append(
+			tsic.buildConfig.tags, tag,
+		)
+	}
+}
+
 // New returns a new TailscaleInContainer instance.
 func New(
 	pool *dockertest.Pool,
@@ -219,18 +247,20 @@ func New(
 	}
 
 	if tsic.withWebsocketDERP {
+		if version != VersionHead {
+			return tsic, errInvalidClientConfig
+		}
+
+		WithBuildTag("ts_debug_websockets")(tsic)
+
 		tailscaleOptions.Env = append(
 			tailscaleOptions.Env,
 			fmt.Sprintf("TS_DEBUG_DERP_WS_CLIENT=%t", tsic.withWebsocketDERP),
 		)
 	}
 
-	if tsic.headscaleHostname != "" {
-		tailscaleOptions.ExtraHosts = []string{
-			"host.docker.internal:host-gateway",
-			fmt.Sprintf("%s:host-gateway", tsic.headscaleHostname),
-		}
-	}
+	tailscaleOptions.ExtraHosts = append(tailscaleOptions.ExtraHosts,
+		"host.docker.internal:host-gateway")
 
 	if tsic.workdir != "" {
 		tailscaleOptions.WorkingDir = tsic.workdir
@@ -245,14 +275,36 @@ func New(
 	}
 
 	var container *dockertest.Resource
+
+	if version != VersionHead {
+		// build options are not meaningful with pre-existing images,
+		// let's not lead anyone astray by pretending otherwise.
+		defaultBuildConfig := TailscaleInContainerBuildConfig{}
+		hasBuildConfig := !reflect.DeepEqual(defaultBuildConfig, tsic.buildConfig)
+		if hasBuildConfig {
+			return tsic, errInvalidClientConfig
+		}
+	}
+
 	switch version {
-	case "head":
+	case VersionHead:
 		buildOptions := &dockertest.BuildOptions{
 			Dockerfile: "Dockerfile.tailscale-HEAD",
 			ContextDir: dockerContextPath,
 			BuildArgs:  []docker.BuildArg{},
 		}
 
+		buildTags := strings.Join(tsic.buildConfig.tags, ",")
+		if len(buildTags) > 0 {
+			buildOptions.BuildArgs = append(
+				buildOptions.BuildArgs,
+				docker.BuildArg{
+					Name:  "BUILD_TAGS",
+					Value: buildTags,
+				},
+			)
+		}
+
 		container, err = pool.BuildAndRunWithBuildOptions(
 			buildOptions,
 			tailscaleOptions,
@@ -294,8 +346,8 @@ func New(
 
 	tsic.container = container
 
-	if tsic.hasTLS() {
-		err = tsic.WriteFile(headscaleCertPath, tsic.headscaleCert)
+	for i, cert := range tsic.caCerts {
+		err = tsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
 		if err != nil {
 			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
 		}
@@ -304,10 +356,6 @@ func New(
 	return tsic, nil
 }
 
-func (t *TailscaleInContainer) hasTLS() bool {
-	return len(t.headscaleCert) != 0
-}
-
 // Shutdown stops and cleans up the Tailscale container.
 func (t *TailscaleInContainer) Shutdown() error {
 	err := t.SaveLog("/tmp/control")
@@ -682,6 +730,34 @@ func (t *TailscaleInContainer) watchIPN(ctx context.Context) (*ipn.Notify, error
 	}
 }
 
+func (t *TailscaleInContainer) DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error) {
+	if !util.TailscaleVersionNewerOrEqual("1.34", t.version) {
+		panic("tsic.DebugDERPRegion() called with unsupported version: " + t.version)
+	}
+
+	command := []string{
+		"tailscale",
+		"debug",
+		"derp",
+		region,
+	}
+
+	result, stderr, err := t.Execute(command)
+	if err != nil {
+		fmt.Printf("stderr: %s\n", stderr) // nolint
+
+		return nil, fmt.Errorf("failed to execute tailscale debug derp command: %w", err)
+	}
+
+	var report ipnstate.DebugDERPRegionReport
+	err = json.Unmarshal([]byte(result), &report)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal tailscale derp region report: %w", err)
+	}
+
+	return &report, err
+}
+
 // Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance.
 func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) {
 	command := []string{