tests: add integration test for DERP verify endpoint

.github/workflows/test-integration.yaml

@@ -37,6 +37,7 @@ jobs:
           - TestNodeRenameCommand
           - TestNodeMoveCommand
           - TestPolicyCommand
+          - TestDERPVerifyEndpoint
           - TestDERPServerScenario
           - TestPingAllByIP
           - TestPingAllByIPPublicDERP

Dockerfile.derper

@@ -0,0 +1,19 @@
+# For testing purposes only
+
+FROM golang:1.22-alpine AS build-env
+
+WORKDIR /go/src
+
+RUN apk add --no-cache git
+ARG VERSION_BRANCH=main
+RUN git clone https://github.com/tailscale/tailscale.git --branch=$VERSION_BRANCH --depth=1
+WORKDIR /go/src/tailscale
+
+ARG TARGETARCH
+RUN GOARCH=$TARGETARCH go install -v ./cmd/derper
+
+FROM alpine:3.18
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl
+
+COPY --from=build-env /go/bin/* /usr/local/bin/
+ENTRYPOINT [ "/usr/local/bin/derper" ]

hscontrol/app.go

@@ -445,6 +445,8 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router {
 	router.HandleFunc("/swagger/v1/openapiv2.json", headscale.SwaggerAPIv1).
 		Methods(http.MethodGet)
 
+	router.HandleFunc("/verify", h.VerifyHandler).Methods(http.MethodPost)
+
 	if h.cfg.DERP.ServerEnabled {
 		router.HandleFunc("/derp", h.DERPServer.DERPHandler)
 		router.HandleFunc("/derp/probe", derpServer.DERPProbeHandler)

hscontrol/handlers.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"html/template"
+	"io"
 	"net/http"
 	"strconv"
 	"time"
@@ -53,6 +54,76 @@ func parseCabailityVersion(req *http.Request) (tailcfg.CapabilityVersion, error)
 	return tailcfg.CapabilityVersion(clientCapabilityVersion), nil
 }
 
+// see https://github.com/tailscale/tailscale/blob/964282d34f06ecc06ce644769c66b0b31d118340/derp/derp_server.go#L1159, Derp use verifyClientsURL to verify whether a client is allowed to connect to the DERP server.
+func (h *Headscale) VerifyHandler(
+	writer http.ResponseWriter,
+	req *http.Request,
+) {
+	if req.Method != http.MethodPost {
+		http.Error(writer, "Wrong method", http.StatusMethodNotAllowed)
+		return
+	}
+	log.Debug().
+		Str("handler", "/verify").
+		Msg("verify client")
+
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		log.Error().
+			Str("handler", "/verify").
+			Err(err).
+			Msg("Cannot read request body")
+		http.Error(writer, "Internal error", http.StatusInternalServerError)
+		return
+	}
+
+	var derpAdmitClientRequest tailcfg.DERPAdmitClientRequest
+	if err := json.Unmarshal(body, &derpAdmitClientRequest); err != nil {
+		log.Error().
+			Caller().
+			Err(err).
+			Msg("Cannot parse derpAdmitClientRequest")
+		http.Error(writer, "Internal error", http.StatusInternalServerError)
+		return
+	}
+
+	nodes, err := h.db.ListNodes()
+	if err != nil {
+		log.Error().
+			Caller().
+			Err(err).
+			Msg("Cannot list nodes")
+		http.Error(writer, "Internal error", http.StatusInternalServerError)
+	}
+
+	for _, node := range nodes {
+		log.Debug().Str("node", node.NodeKey.String()).Msg("Node")
+	}
+
+	allow := false
+	// Check if the node is in the list of nodes
+	for _, node := range nodes {
+		if node.NodeKey == derpAdmitClientRequest.NodePublic {
+			allow = true
+			break
+		}
+	}
+
+	resp := tailcfg.DERPAdmitClientResponse{
+		Allow: allow,
+	}
+
+	writer.Header().Set("Content-Type", "application/json")
+	writer.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(writer).Encode(resp)
+	if err != nil {
+		log.Error().
+			Caller().
+			Err(err).
+			Msg("Failed to write response")
+	}
+}
+
 // KeyHandler provides the Headscale pub key
 // Listens in /key.
 func (h *Headscale) KeyHandler(

integration/README.md

@@ -11,10 +11,10 @@ Tests are located in files ending with `_test.go` and the framework are located
 
 ## Running integration tests locally
 
-The easiest way to run tests locally is to use `[act](INSERT LINK)`, a local GitHub Actions runner:
+The easiest way to run tests locally is to use [act](https://github.com/nektos/act), a local GitHub Actions runner:
 
 ```
-act pull_request -W .github/workflows/test-integration-v2-TestPingAllByIP.yaml
+act pull_request -W .github/workflows/test-integration.yaml
 ```
 
 Alternatively, the `docker run` command in each GitHub workflow file can be used.

integration/derp_verify_endpoint_test.go

@@ -0,0 +1,111 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dsic"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/juanfont/headscale/integration/tsic"
+)
+
+func TestDERPVerifyEndpoint(t *testing.T) {
+	IntegrationSkip(t)
+
+	// Generate random hostname for the headscale instance
+	hash, err := util.GenerateRandomStringDNSSafe(6)
+	assertNoErr(t, err)
+	testName := "derpverify"
+	hostname := fmt.Sprintf("hs-%s-%s", testName, hash)
+
+	headscalePort := 8080
+
+	// Create cert for headscale
+	certHeadscale, keyHeadscale, err := integrationutil.CreateCertificate(hostname)
+	assertNoErr(t, err)
+
+	scenario, err := NewScenario(dockertestMaxWait())
+	assertNoErr(t, err)
+	defer scenario.Shutdown()
+
+	spec := map[string]int{
+		"user1": 10,
+	}
+
+	derper, err := scenario.CreateDERPServer("head",
+		dsic.WithCACert(certHeadscale),
+		dsic.WithVerifyClientURL(fmt.Sprintf("https://%s/verify", net.JoinHostPort(hostname, strconv.Itoa(headscalePort)))),
+	)
+	assertNoErr(t, err)
+
+	derpConfig := "regions:\n"
+	derpConfig += "  900:\n"
+	derpConfig += "    regionid: 900\n"
+	derpConfig += "    regioncode: test-derpverify\n"
+	derpConfig += "    regionname: TestDerpVerify\n"
+	derpConfig += "    nodes:\n"
+	derpConfig += "      - name: TestDerpVerify\n"
+	derpConfig += "        regionid: 900\n"
+	derpConfig += "        hostname: " + derper.GetHostname() + "\n"
+	derpConfig += "        stunport: " + derper.GetSTUNPort() + "\n"
+	derpConfig += "        stunonly: false\n"
+	derpConfig += "        derpport: " + derper.GetDERPPort() + "\n"
+
+	headscale, err := scenario.Headscale(
+		hsic.WithHostname(hostname),
+		hsic.WithPort(headscalePort),
+		hsic.WithCustomTLS(certHeadscale, keyHeadscale),
+		hsic.WithHostnameAsServerURL(),
+		hsic.WithCustomDERPServerOnly([]byte(derpConfig)),
+	)
+	assertNoErrHeadscaleEnv(t, err)
+
+	for userName, clientCount := range spec {
+		err = scenario.CreateUser(userName)
+		if err != nil {
+			t.Fatalf("failed to create user %s: %s", userName, err)
+		}
+
+		err = scenario.CreateTailscaleNodesInUser(userName, "all", clientCount, tsic.WithCACert(derper.GetCert()))
+		if err != nil {
+			t.Fatalf("failed to create tailscale nodes in user %s: %s", userName, err)
+		}
+
+		key, err := scenario.CreatePreAuthKey(userName, true, true)
+		if err != nil {
+			t.Fatalf("failed to create pre-auth key for user %s: %s", userName, err)
+		}
+
+		err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey())
+		if err != nil {
+			t.Fatalf("failed to run tailscale up for user %s: %s", userName, err)
+		}
+	}
+
+	allClients, err := scenario.ListTailscaleClients()
+	assertNoErrListClients(t, err)
+
+	for _, client := range allClients {
+		report, err := client.DebugDERPRegion("test-derpverify")
+		assertNoErr(t, err)
+		successful := false
+		for _, line := range report.Info {
+			if strings.Contains(line, "Successfully established a DERP connection with node") {
+				successful = true
+
+				break
+			}
+		}
+		if !successful {
+			stJSON, err := json.Marshal(report)
+			assertNoErr(t, err)
+			t.Errorf("Client %s could not establish a DERP connection: %s", client.Hostname(), string(stJSON))
+		}
+	}
+}

integration/dsic/dsic.go

@@ -0,0 +1,316 @@
+package dsic
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+const (
+	dsicHashLength       = 6
+	dockerContextPath    = "../."
+	caCertRoot           = "/usr/local/share/ca-certificates"
+	DERPerCertRoot       = "/usr/local/share/derper-certs"
+	dockerExecuteTimeout = 60 * time.Second
+)
+
+var errDERPerStatusCodeNotOk = errors.New("DERPer status code not OK")
+
+// DERPServerInContainer represents DERP Server in Container (DSIC).
+type DERPServerInContainer struct {
+	version  string
+	hostname string
+
+	pool      *dockertest.Pool
+	container *dockertest.Resource
+	network   *dockertest.Network
+
+	stunPort            int
+	derpPort            int
+	caCerts             [][]byte
+	tlsCert             []byte
+	tlsKey              []byte
+	withExtraHosts      []string
+	withVerifyClientURL string
+	workdir             string
+}
+
+// Option represent optional settings that can be given to a
+// DERPer instance.
+type Option = func(c *DERPServerInContainer)
+
+// WithCACert adds it to the trusted surtificate of the Tailscale container.
+func WithCACert(cert []byte) Option {
+	return func(dsic *DERPServerInContainer) {
+		dsic.caCerts = append(dsic.caCerts, cert)
+	}
+}
+
+// WithOrCreateNetwork sets the Docker container network to use with
+// the DERPer instance, if the parameter is nil, a new network,
+// isolating the DERPer, will be created. If a network is
+// passed, the DERPer instance will join the given network.
+func WithOrCreateNetwork(network *dockertest.Network) Option {
+	return func(tsic *DERPServerInContainer) {
+		if network != nil {
+			tsic.network = network
+
+			return
+		}
+
+		network, err := dockertestutil.GetFirstOrCreateNetwork(
+			tsic.pool,
+			fmt.Sprintf("%s-network", tsic.hostname),
+		)
+		if err != nil {
+			log.Fatalf("failed to create network: %s", err)
+		}
+
+		tsic.network = network
+	}
+}
+
+// WithDockerWorkdir allows the docker working directory to be set.
+func WithDockerWorkdir(dir string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.workdir = dir
+	}
+}
+
+// WithVerifyClientURL sets the URL to verify the client.
+func WithVerifyClientURL(url string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.withVerifyClientURL = url
+	}
+}
+
+// WithExtraHosts adds extra hosts to the container.
+func WithExtraHosts(hosts []string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.withExtraHosts = hosts
+	}
+}
+
+// New returns a new TailscaleInContainer instance.
+func New(
+	pool *dockertest.Pool,
+	version string,
+	network *dockertest.Network,
+	opts ...Option,
+) (*DERPServerInContainer, error) {
+	hash, err := util.GenerateRandomStringDNSSafe(dsicHashLength)
+	if err != nil {
+		return nil, err
+	}
+
+	hostname := fmt.Sprintf("derp-%s-%s", strings.ReplaceAll(version, ".", "-"), hash)
+	tlsCert, tlsKey, err := integrationutil.CreateCertificate(hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create certificates for headscale test: %w", err)
+	}
+	dsic := &DERPServerInContainer{
+		version:  version,
+		hostname: hostname,
+		pool:     pool,
+		network:  network,
+		tlsCert:  tlsCert,
+		tlsKey:   tlsKey,
+		stunPort: 3478, //nolint
+		derpPort: 443,  //nolint
+	}
+
+	for _, opt := range opts {
+		opt(dsic)
+	}
+
+	cmdArgs := "--hostname=" + hostname
+	cmdArgs += " --certmode=manual"
+	cmdArgs += " --certdir=" + DERPerCertRoot
+	cmdArgs += " --a=:" + strconv.Itoa(dsic.derpPort)
+	cmdArgs += " --stun=true"
+	cmdArgs += " --stun-port=" + strconv.Itoa(dsic.stunPort)
+	if dsic.withVerifyClientURL != "" {
+		cmdArgs += " --verify-client-url=" + dsic.withVerifyClientURL
+	}
+
+	runOptions := &dockertest.RunOptions{
+		Name:       hostname,
+		Networks:   []*dockertest.Network{dsic.network},
+		ExtraHosts: dsic.withExtraHosts,
+		// we currently need to give us some time to inject the certificate further down.
+		Entrypoint: []string{"/bin/sh", "-c", "/bin/sleep 3 ; update-ca-certificates ; derper " + cmdArgs},
+		ExposedPorts: []string{
+			"80/tcp",
+			fmt.Sprintf("%d/tcp", dsic.derpPort),
+			fmt.Sprintf("%d/udp", dsic.stunPort),
+		},
+	}
+
+	if dsic.workdir != "" {
+		runOptions.WorkingDir = dsic.workdir
+	}
+
+	// dockertest isnt very good at handling containers that has already
+	// been created, this is an attempt to make sure this container isnt
+	// present.
+	err = pool.RemoveContainerByName(hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	var container *dockertest.Resource
+	buildOptions := &dockertest.BuildOptions{
+		Dockerfile: "Dockerfile.derper",
+		ContextDir: dockerContextPath,
+		BuildArgs:  []docker.BuildArg{},
+	}
+	switch version {
+	case "head":
+		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+			Name:  "VERSION_BRANCH",
+			Value: "main",
+		})
+	default:
+		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+			Name:  "VERSION_BRANCH",
+			Value: "v" + version,
+		})
+	}
+	container, err = pool.BuildAndRunWithBuildOptions(
+		buildOptions,
+		runOptions,
+		dockertestutil.DockerRestartPolicy,
+		dockertestutil.DockerAllowLocalIPv6,
+		dockertestutil.DockerAllowNetworkAdministration,
+	)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%s could not start tailscale DERPer container (version: %s): %w",
+			hostname,
+			version,
+			err,
+		)
+	}
+	log.Printf("Created %s container\n", hostname)
+
+	dsic.container = container
+
+	for i, cert := range dsic.caCerts {
+		err = dsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+	if len(dsic.tlsCert) != 0 {
+		err = dsic.WriteFile(fmt.Sprintf("%s/%s.crt", DERPerCertRoot, dsic.hostname), dsic.tlsCert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+	if len(dsic.tlsKey) != 0 {
+		err = dsic.WriteFile(fmt.Sprintf("%s/%s.key", DERPerCertRoot, dsic.hostname), dsic.tlsKey)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS key to container: %w", err)
+		}
+	}
+	return dsic, nil
+}
+
+// Shutdown stops and cleans up the DERPer container.
+func (t *DERPServerInContainer) Shutdown() error {
+	err := t.SaveLog("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"Failed to save log from %s: %s",
+			t.hostname,
+			fmt.Errorf("failed to save log: %w", err),
+		)
+	}
+	return t.pool.Purge(t.container)
+}
+
+// GetCert returns the TLS certificate of the DERPer instance.
+func (t *DERPServerInContainer) GetCert() []byte {
+	return t.tlsCert
+}
+
+// Hostname returns the hostname of the DERPer instance.
+func (t *DERPServerInContainer) Hostname() string {
+	return t.hostname
+}
+
+// Version returns the running DERPer version of the instance.
+func (t *DERPServerInContainer) Version() string {
+	return t.version
+}
+
+// ID returns the Docker container ID of the DERPServerInContainer
+// instance.
+func (t *DERPServerInContainer) ID() string {
+	return t.container.Container.ID
+}
+
+func (t *DERPServerInContainer) GetHostname() string {
+	return t.hostname
+}
+
+// GetSTUNPort returns the STUN port of the DERPer instance.
+func (t *DERPServerInContainer) GetSTUNPort() string {
+	return strconv.Itoa(t.stunPort)
+}
+
+// GetDERPPort returns the DERP port of the DERPer instance.
+func (t *DERPServerInContainer) GetDERPPort() string {
+	return strconv.Itoa(t.derpPort)
+}
+
+// WaitForRunning blocks until the DERPer instance is ready to be used.
+func (t *DERPServerInContainer) WaitForRunning() error {
+	url := "https://" + net.JoinHostPort(t.GetHostname(), t.GetDERPPort()) + "/"
+	log.Printf("waiting for DERPer to be ready at %s", url)
+
+	insecureTransport := http.DefaultTransport.(*http.Transport).Clone()      //nolint
+	insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint
+	client := &http.Client{Transport: insecureTransport}
+
+	return t.pool.Retry(func() error {
+		resp, err := client.Get(url) //nolint
+		if err != nil {
+			return fmt.Errorf("headscale is not ready: %w", err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return errDERPerStatusCodeNotOk
+		}
+
+		return nil
+	})
+}
+
+// ConnectToNetwork connects the DERPer instance to a network.
+func (t *DERPServerInContainer) ConnectToNetwork(network *dockertest.Network) error {
+	return t.container.ConnectToNetwork(network)
+}
+
+// WriteFile save file inside the container.
+func (t *DERPServerInContainer) WriteFile(path string, data []byte) error {
+	return integrationutil.WriteFileToContainer(t.pool, t.container, path, data)
+}
+
+// SaveLog saves the current stdout log of the container to a path
+// on the host system.
+func (t *DERPServerInContainer) SaveLog(path string) error {
+	return dockertestutil.SaveLog(t.pool, t.container, path)
+}

integration/embedded_derp_test.go

@@ -163,7 +163,7 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser(
 			cert := hsServer.GetCert()
 
 			opts = append(opts,
-				tsic.WithHeadscaleTLS(cert),
+				tsic.WithCACert(cert),
 			)
 
 			user.createWaitGroup.Go(func() error {

integration/hsic/hsic.go

@@ -1,19 +1,12 @@
 package hsic
 
 import (
-	"bytes"
-	"crypto/rand"
-	"crypto/rsa"
 	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
 	"encoding/json"
-	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
 	"log"
-	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -37,6 +30,7 @@ import (
 const (
 	hsicHashLength       = 6
 	dockerContextPath    = "../."
+	caCertRoot           = "/usr/local/share/ca-certificates"
 	aclPolicyPath        = "/etc/headscale/acl.hujson"
 	tlsCertPath          = "/etc/headscale/tls.cert"
 	tlsKeyPath           = "/etc/headscale/tls.key"
@@ -64,6 +58,7 @@ type HeadscaleInContainer struct {
 	// optional config
 	port             int
 	extraPorts       []string
+	caCerts          [][]byte
 	hostPortBindings map[string][]string
 	aclPolicy        *policy.ACLPolicy
 	env              map[string]string
@@ -88,18 +83,29 @@ func WithACLPolicy(acl *policy.ACLPolicy) Option {
 	}
 }
 
+// WithCACert adds it to the trusted surtificate of the container.
+func WithCACert(cert []byte) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.caCerts = append(hsic.caCerts, cert)
+	}
+}
+
 // WithTLS creates certificates and enables HTTPS.
 func WithTLS() Option {
 	return func(hsic *HeadscaleInContainer) {
-		cert, key, err := createCertificate(hsic.hostname)
+		cert, key, err := integrationutil.CreateCertificate(hsic.hostname)
 		if err != nil {
 			log.Fatalf("failed to create certificates for headscale test: %s", err)
 		}
 
-		// TODO(kradalby): Move somewhere appropriate
-		hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath
-		hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath
+		hsic.tlsCert = cert
+		hsic.tlsKey = key
+	}
+}
 
+// WithCustomTLS uses the given certificates for the Headscale instance.
+func WithCustomTLS(cert, key []byte) Option {
+	return func(hsic *HeadscaleInContainer) {
 		hsic.tlsCert = cert
 		hsic.tlsKey = key
 	}
@@ -146,6 +152,13 @@ func WithTestName(testName string) Option {
 	}
 }
 
+// WithHostname sets the hostname of the Headscale instance.
+func WithHostname(hostname string) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.hostname = hostname
+	}
+}
+
 // WithHostnameAsServerURL sets the Headscale ServerURL based on
 // the Hostname.
 func WithHostnameAsServerURL() Option {
@@ -203,6 +216,27 @@ func WithEmbeddedDERPServerOnly() Option {
 	}
 }
 
+// WithCustomDERPServerOnly configures Headscale use a custom
+// DERP server only.
+func WithCustomDERPServerOnly(contents []byte) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.env["HEADSCALE_DERP_PATHS"] = "/etc/headscale/derp.yml"
+		hsic.filesInContainer = append(hsic.filesInContainer,
+			fileInContainer{
+				path:     "/etc/headscale/derp.yml",
+				contents: contents,
+			})
+
+		// Disable global DERP server and embedded DERP server
+		hsic.env["HEADSCALE_DERP_URLS"] = ""
+		hsic.env["HEADSCALE_DERP_SERVER_ENABLED"] = "false"
+
+		// Envknob for enabling DERP debug logs
+		hsic.env["DERP_DEBUG_LOGS"] = "true"
+		hsic.env["DERP_PROBER_DEBUG_LOGS"] = "true"
+	}
+}
+
 // WithTuning allows changing the tuning settings easily.
 func WithTuning(batchTimeout time.Duration, mapSessionChanSize int) Option {
 	return func(hsic *HeadscaleInContainer) {
@@ -294,6 +328,10 @@ func New(
 		"HEADSCALE_DEBUG_HIGH_CARDINALITY_METRICS=1",
 		"HEADSCALE_DEBUG_DUMP_CONFIG=1",
 	}
+	if hsic.hasTLS() {
+		hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath
+		hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath
+	}
 	for key, value := range hsic.env {
 		env = append(env, fmt.Sprintf("%s=%s", key, value))
 	}
@@ -307,7 +345,7 @@ func New(
 		// Cmd:          []string{"headscale", "serve"},
 		// TODO(kradalby): Get rid of this hack, we currently need to give us some
 		// to inject the headscale configuration further down.
-		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"},
+		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; update-ca-certificates ; headscale serve ; /bin/sleep 30"},
 		Env:        env,
 	}
 
@@ -345,6 +383,14 @@ func New(
 
 	hsic.container = container
 
+	// Write the CA certificates to the container
+	for i, cert := range hsic.caCerts {
+		err = hsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+
 	err = hsic.WriteFile("/etc/headscale/config.yaml", []byte(MinimumConfigYAML()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to write headscale config to container: %w", err)
@@ -737,86 +783,3 @@ func (t *HeadscaleInContainer) SendInterrupt() error {
 
 	return nil
 }
-
-// nolint
-func createCertificate(hostname string) ([]byte, []byte, error) {
-	// From:
-	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
-
-	ca := &x509.Certificate{
-		SerialNumber: big.NewInt(2019),
-		Subject: pkix.Name{
-			Organization: []string{"Headscale testing INC"},
-			Country:      []string{"NL"},
-			Locality:     []string{"Leiden"},
-		},
-		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(60 * time.Hour),
-		IsCA:      true,
-		ExtKeyUsage: []x509.ExtKeyUsage{
-			x509.ExtKeyUsageClientAuth,
-			x509.ExtKeyUsageServerAuth,
-		},
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
-		BasicConstraintsValid: true,
-	}
-
-	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cert := &x509.Certificate{
-		SerialNumber: big.NewInt(1658),
-		Subject: pkix.Name{
-			CommonName:   hostname,
-			Organization: []string{"Headscale testing INC"},
-			Country:      []string{"NL"},
-			Locality:     []string{"Leiden"},
-		},
-		NotBefore:    time.Now(),
-		NotAfter:     time.Now().Add(60 * time.Minute),
-		SubjectKeyId: []byte{1, 2, 3, 4, 6},
-		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		KeyUsage:     x509.KeyUsageDigitalSignature,
-		DNSNames:     []string{hostname},
-	}
-
-	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certBytes, err := x509.CreateCertificate(
-		rand.Reader,
-		cert,
-		ca,
-		&certPrivKey.PublicKey,
-		caPrivKey,
-	)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certPEM := new(bytes.Buffer)
-
-	err = pem.Encode(certPEM, &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: certBytes,
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certPrivKeyPEM := new(bytes.Buffer)
-
-	err = pem.Encode(certPrivKeyPEM, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
-}

integration/integrationutil/util.go

@@ -3,9 +3,16 @@ package integrationutil
 import (
 	"archive/tar"
 	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"math/big"
 	"path/filepath"
+	"time"
 
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/ory/dockertest/v3"
@@ -93,3 +100,86 @@ func FetchPathFromContainer(
 
 	return buf.Bytes(), nil
 }
+
+// nolint
+func CreateCertificate(hostname string) ([]byte, []byte, error) {
+	// From:
+	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
+
+	ca := &x509.Certificate{
+		SerialNumber: big.NewInt(2019),
+		Subject: pkix.Name{
+			Organization: []string{"Headscale testing INC"},
+			Country:      []string{"NL"},
+			Locality:     []string{"Leiden"},
+		},
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().Add(60 * time.Hour),
+		IsCA:      true,
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+		},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+
+	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cert := &x509.Certificate{
+		SerialNumber: big.NewInt(1658),
+		Subject: pkix.Name{
+			CommonName:   hostname,
+			Organization: []string{"Headscale testing INC"},
+			Country:      []string{"NL"},
+			Locality:     []string{"Leiden"},
+		},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().Add(60 * time.Minute),
+		SubjectKeyId: []byte{1, 2, 3, 4, 6},
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		DNSNames:     []string{hostname},
+	}
+
+	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certBytes, err := x509.CreateCertificate(
+		rand.Reader,
+		cert,
+		ca,
+		&certPrivKey.PublicKey,
+		caPrivKey,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certPEM := new(bytes.Buffer)
+
+	err = pem.Encode(certPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certPrivKeyPEM := new(bytes.Buffer)
+
+	err = pem.Encode(certPrivKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
+}

integration/scenario.go

@@ -13,6 +13,7 @@ import (
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/dsic"
 	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/ory/dockertest/v3"
@@ -134,6 +135,7 @@ type Scenario struct {
 	// TODO(kradalby): support multiple headcales for later, currently only
 	// use one.
 	controlServers *xsync.MapOf[string, ControlServer]
+	derpServers    []*dsic.DERPServerInContainer
 
 	users map[string]*User
 
@@ -212,6 +214,13 @@ func (s *Scenario) Shutdown() {
 		}
 	}
 
+	for _, derp := range s.derpServers {
+		err := derp.Shutdown()
+		if err != nil {
+			log.Printf("failed to tear down derp server: %s", err)
+		}
+	}
+
 	if err := s.pool.RemoveNetwork(s.network); err != nil {
 		log.Printf("failed to remove network: %s", err)
 	}
@@ -328,7 +337,7 @@ func (s *Scenario) CreateTailscaleNodesInUser(
 			hostname := headscale.GetHostname()
 
 			opts = append(opts,
-				tsic.WithHeadscaleTLS(cert),
+				tsic.WithCACert(cert),
 				tsic.WithHeadscaleName(hostname),
 			)
 
@@ -631,3 +640,20 @@ func (s *Scenario) WaitForTailscaleLogout() error {
 
 	return nil
 }
+
+// CreateDERPServer creates a new DERP server in a container.
+func (s *Scenario) CreateDERPServer(version string, opts ...dsic.Option) (*dsic.DERPServerInContainer, error) {
+	derp, err := dsic.New(s.pool, version, s.network, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DERP server: %w", err)
+	}
+
+	err = derp.WaitForRunning()
+	if err != nil {
+		return nil, fmt.Errorf("failed to reach DERP server: %w", err)
+	}
+
+	s.derpServers = append(s.derpServers, derp)
+
+	return derp, nil
+}

integration/tailscale.go

@@ -29,6 +29,7 @@ type TailscaleClient interface {
 	FQDN() (string, error)
 	Status(...bool) (*ipnstate.Status, error)
 	Netmap() (*netmap.NetworkMap, error)
+	DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error)
 	Netcheck() (*netcheck.Report, error)
 	WaitForNeedsLogin() error
 	WaitForRunning() error

integration/tsic/tsic.go

@@ -30,7 +30,7 @@ const (
 	defaultPingTimeout   = 300 * time.Millisecond
 	defaultPingCount     = 10
 	dockerContextPath    = "../."
-	headscaleCertPath    = "/usr/local/share/ca-certificates/headscale.crt"
+	caCertRoot           = "/usr/local/share/ca-certificates"
 	dockerExecuteTimeout = 60 * time.Second
 )
 
@@ -63,7 +63,7 @@ type TailscaleInContainer struct {
 	fqdn string
 
 	// optional config
-	headscaleCert     []byte
+	caCerts           [][]byte
 	headscaleHostname string
 	withSSH           bool
 	withTags          []string
@@ -77,11 +77,10 @@ type TailscaleInContainer struct {
 // Tailscale instance.
 type Option = func(c *TailscaleInContainer)
 
-// WithHeadscaleTLS takes the certificate of the Headscale instance
-// and adds it to the trusted surtificate of the Tailscale container.
-func WithHeadscaleTLS(cert []byte) Option {
+// WithCACert adds it to the trusted surtificate of the Tailscale container.
+func WithCACert(cert []byte) Option {
 	return func(tsic *TailscaleInContainer) {
-		tsic.headscaleCert = cert
+		tsic.caCerts = append(tsic.caCerts, cert)
 	}
 }
 
@@ -110,7 +109,7 @@ func WithOrCreateNetwork(network *dockertest.Network) Option {
 }
 
 // WithHeadscaleName set the name of the headscale instance,
-// mostly useful in combination with TLS and WithHeadscaleTLS.
+// mostly useful in combination with TLS and WithCACert.
 func WithHeadscaleName(hsName string) Option {
 	return func(tsic *TailscaleInContainer) {
 		tsic.headscaleHostname = hsName
@@ -206,12 +205,8 @@ func New(
 		ExtraHosts: tsic.withExtraHosts,
 	}
 
-	if tsic.headscaleHostname != "" {
-		tailscaleOptions.ExtraHosts = []string{
-			"host.docker.internal:host-gateway",
-			fmt.Sprintf("%s:host-gateway", tsic.headscaleHostname),
-		}
-	}
+	tailscaleOptions.ExtraHosts = append(tailscaleOptions.ExtraHosts,
+		"host.docker.internal:host-gateway")
 
 	if tsic.workdir != "" {
 		tailscaleOptions.WorkingDir = tsic.workdir
@@ -275,8 +270,8 @@ func New(
 
 	tsic.container = container
 
-	if tsic.hasTLS() {
-		err = tsic.WriteFile(headscaleCertPath, tsic.headscaleCert)
+	for i, cert := range tsic.caCerts {
+		err = tsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
 		if err != nil {
 			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
 		}
@@ -285,10 +280,6 @@ func New(
 	return tsic, nil
 }
 
-func (t *TailscaleInContainer) hasTLS() bool {
-	return len(t.headscaleCert) != 0
-}
-
 // Shutdown stops and cleans up the Tailscale container.
 func (t *TailscaleInContainer) Shutdown() error {
 	err := t.SaveLog("/tmp/control")
@@ -654,6 +645,33 @@ func (t *TailscaleInContainer) watchIPN(ctx context.Context) (*ipn.Notify, error
 	}
 }
 
+func (t *TailscaleInContainer) DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error) {
+	if !util.TailscaleVersionNewerOrEqual("1.34", t.version) {
+		panic(fmt.Sprintf("tsic.DebugDERPRegion() called with unsupported version: %s", t.version))
+	}
+
+	command := []string{
+		"tailscale",
+		"debug",
+		"derp",
+		region,
+	}
+
+	result, stderr, err := t.Execute(command)
+	if err != nil {
+		fmt.Printf("stderr: %s\n", stderr)
+		return nil, fmt.Errorf("failed to execute tailscale debug derp command: %w", err)
+	}
+
+	var st ipnstate.DebugDERPRegionReport
+	err = json.Unmarshal([]byte(result), &st)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal tailscale derp region report: %w", err)
+	}
+
+	return &st, err
+}
+
 // Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance.
 func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) {
 	command := []string{