refactor(verify-client): simplify error handling

* Document to either use a minimal configuration file or environment
  variables to connect with a remote headscale instance.
* Document a workaround specific for headscale 0.23.0.
* Remove reference to ancient headscale version.
* Use `cli.insecure: true` or `HEADSCALE_CLI_INSECURE=1` to skip
  certificate verification.
* Style and typo fixes

Ref: #2193

.github/CODEOWNERS

@@ -1,10 +1,10 @@
 * @juanfont @kradalby
 
-*.md @ohdearaugustin
-*.yml @ohdearaugustin
-*.yaml @ohdearaugustin
-Dockerfile* @ohdearaugustin
-.goreleaser.yaml @ohdearaugustin
-/docs/ @ohdearaugustin
-/.github/workflows/ @ohdearaugustin
-/.github/renovate.json @ohdearaugustin
+*.md @ohdearaugustin @nblock
+*.yml @ohdearaugustin @nblock
+*.yaml @ohdearaugustin @nblock
+Dockerfile* @ohdearaugustin @nblock
+.goreleaser.yaml @ohdearaugustin @nblock
+/docs/ @ohdearaugustin @nblock
+/.github/workflows/ @ohdearaugustin @nblock
+/.github/renovate.json @ohdearaugustin @nblock

.github/workflows/test-integration.yaml

@@ -38,6 +38,7 @@ jobs:
           - TestNodeMoveCommand
           - TestPolicyCommand
           - TestPolicyBrokenConfigCommand
+          - TestDERPVerifyEndpoint
           - TestResolveMagicDNS
           - TestValidateResolvConf
           - TestDERPServerScenario

.goreleaser.yml

@@ -154,7 +154,7 @@ kos:
       - "{{ if not .Prerelease }}v{{ .Major }}.{{ .Minor }}.{{ .Patch }}-debug{{ end }}"
       - "{{ if not .Prerelease }}v{{ .Major }}.{{ .Minor }}-debug{{ end }}"
       - "{{ if not .Prerelease }}v{{ .Major }}-debug{{ end }}"
-      - "{{ if not .Prerelease }}stable{{ else }}unstable-debug{{ end }}"
+      - "{{ if not .Prerelease }}stable-debug{{ else }}unstable-debug{{ end }}"
       - "{{ .Tag }}-debug"
       - '{{ trimprefix .Tag "v" }}-debug'
       - "sha-{{ .ShortCommit }}-debug"
@@ -177,7 +177,7 @@ kos:
       - "{{ if not .Prerelease }}v{{ .Major }}.{{ .Minor }}.{{ .Patch }}-debug{{ end }}"
       - "{{ if not .Prerelease }}v{{ .Major }}.{{ .Minor }}-debug{{ end }}"
       - "{{ if not .Prerelease }}v{{ .Major }}-debug{{ end }}"
-      - "{{ if not .Prerelease }}stable{{ else }}unstable-debug{{ end }}"
+      - "{{ if not .Prerelease }}stable-debug{{ else }}unstable-debug{{ end }}"
       - "{{ .Tag }}-debug"
       - '{{ trimprefix .Tag "v" }}-debug'
       - "sha-{{ .ShortCommit }}-debug"

.prettierignore

@@ -1,2 +1,3 @@
 .github/workflows/test-integration-v2*
 docs/about/features.md
+docs/ref/remote-cli.md

CHANGELOG.md

@@ -22,6 +22,7 @@
 - Fixed processing of fields in post request in MoveNode rpc [#2179](https://github.com/juanfont/headscale/pull/2179)
 - Added conversion of 'Hostname' to 'givenName' in a node with FQDN rules applied [#2198](https://github.com/juanfont/headscale/pull/2198)
 - Fixed updating of hostname and givenName when it is updated in HostInfo [#2199](https://github.com/juanfont/headscale/pull/2199)
+- Fixed missing `stable-debug` container tag [#2232](https://github.com/juanfont/headscale/pr/2232)
 
 ## 0.23.0 (2024-09-18)
 

Dockerfile.derper

@@ -0,0 +1,19 @@
+# For testing purposes only
+
+FROM golang:alpine AS build-env
+
+WORKDIR /go/src
+
+RUN apk add --no-cache git
+ARG VERSION_BRANCH=main
+RUN git clone https://github.com/tailscale/tailscale.git --branch=$VERSION_BRANCH --depth=1
+WORKDIR /go/src/tailscale
+
+ARG TARGETARCH
+RUN GOARCH=$TARGETARCH go install -v ./cmd/derper
+
+FROM alpine:3.18
+RUN apk add --no-cache ca-certificates iptables iproute2 ip6tables curl
+
+COPY --from=build-env /go/bin/* /usr/local/bin/
+ENTRYPOINT [ "/usr/local/bin/derper" ]

docs/about/clients.md

@@ -13,3 +13,4 @@ headscale.
 | Android | Yes (see [docs](../usage/connect/android.md))                                                         |
 | macOS   | Yes (see [docs](../usage/connect/apple.md#macos) and `/apple` on your headscale for more information) |
 | iOS     | Yes (see [docs](../usage/connect/apple.md#ios) and `/apple` on your headscale for more information)   |
+| tvOS    | Yes (see [docs](../usage/connect/apple.md#tvos) and `/apple` on your headscale for more information)  |

docs/ref/integration/tools.md

@@ -0,0 +1,12 @@
+# Tools related to headscale
+
+!!! warning "Community contributions"
+
+    This page contains community contributions. The projects listed here are not
+    maintained by the headscale authors and are written by community members.
+
+This page collects third-party tools and scripts related to headscale.
+
+| Name              | Repository Link                                                 | Description                                       |
+| ----------------- | --------------------------------------------------------------- | ------------------------------------------------- |
+| tailscale-manager | [Github](https://github.com/singlestore-labs/tailscale-manager) | Dynamically manage Tailscale route advertisements |

docs/ref/integration/web-ui.md

@@ -1,17 +1,19 @@
-# Headscale web interface
+# Web interfaces for headscale
 
 !!! warning "Community contributions"
 
     This page contains community contributions. The projects listed here are not
     maintained by the headscale authors and are written by community members.
 
-| Name            | Repository Link                                         | Description                                                                         | Status |
-| --------------- | ------------------------------------------------------- | ----------------------------------------------------------------------------------- | ------ |
-| headscale-webui | [Github](https://github.com/ifargle/headscale-webui)    | A simple headscale web UI for small-scale deployments.                              | Alpha  |
-| headscale-ui    | [Github](https://github.com/gurucomputing/headscale-ui) | A web frontend for the headscale Tailscale-compatible coordination server           | Alpha  |
-| HeadscaleUi     | [GitHub](https://github.com/simcu/headscale-ui)         | A static headscale admin ui, no backend enviroment required                         | Alpha  |
-| Headplane       | [GitHub](https://github.com/tale/headplane)             | An advanced Tailscale inspired frontend for headscale                               | Alpha  |
-| headscale-admin | [Github](https://github.com/GoodiesHQ/headscale-admin)  | Headscale-Admin is meant to be a simple, modern web interface for headscale         | Beta   |
-| ouroboros       | [Github](https://github.com/yellowsink/ouroboros)       | Ouroboros is designed for users to manage their own devices, rather than for admins | Stable |
+Headscale doesn't provide a built-in web interface but users may pick one from the available options.
+
+| Name            | Repository Link                                         | Description                                                                         |
+| --------------- | ------------------------------------------------------- | ----------------------------------------------------------------------------------- |
+| headscale-webui | [Github](https://github.com/ifargle/headscale-webui)    | A simple headscale web UI for small-scale deployments.                              |
+| headscale-ui    | [Github](https://github.com/gurucomputing/headscale-ui) | A web frontend for the headscale Tailscale-compatible coordination server           |
+| HeadscaleUi     | [GitHub](https://github.com/simcu/headscale-ui)         | A static headscale admin ui, no backend enviroment required                         |
+| Headplane       | [GitHub](https://github.com/tale/headplane)             | An advanced Tailscale inspired frontend for headscale                               |
+| headscale-admin | [Github](https://github.com/GoodiesHQ/headscale-admin)  | Headscale-Admin is meant to be a simple, modern web interface for headscale         |
+| ouroboros       | [Github](https://github.com/yellowsink/ouroboros)       | Ouroboros is designed for users to manage their own devices, rather than for admins |
 
 You can ask for support on our dedicated [Discord channel](https://discord.com/channels/896711691637780480/1105842846386356294).

docs/ref/remote-cli.md

@@ -1,22 +1,21 @@
 # Controlling headscale with remote CLI
 
-This documentation has the goal of showing a user how-to set control a headscale instance
+This documentation has the goal of showing a user how-to control a headscale instance
 from a remote machine with the `headscale` command line binary.
 
 ## Prerequisite
 
-- A workstation to run headscale (could be Linux, macOS, other supported platforms)
-- A headscale server (version `0.13.0` or newer)
-- Access to create API keys (local access to the headscale server)
-- headscale _must_ be served over TLS/HTTPS
-  - Remote access does _not_ support unencrypted traffic.
-- Port `50443` must be open in the firewall (or port overridden by `grpc_listen_addr` option)
+- A workstation to run `headscale` (any supported platform, e.g. Linux).
+- A headscale server with gRPC enabled.
+- Connections to the gRPC port (default: `50443`) are allowed.
+- Remote access requires an encrypted connection via TLS.
+- An API key to authenticate with the headscale server.
 
 ## Create an API key
 
-We need to create an API key to authenticate our remote headscale when using it from our workstation.
+We need to create an API key to authenticate with the remote headscale server when using it from our workstation.
 
-To create a API key, log into your headscale server and generate a key:
+To create an API key, log into your headscale server and generate a key:
 
 ```shell
 headscale apikeys create --expiration 90d
@@ -25,7 +24,7 @@ headscale apikeys create --expiration 90d
 Copy the output of the command and save it for later. Please note that you can not retrieve a key again,
 if the key is lost, expire the old one, and create a new key.
 
-To list the keys currently assosicated with the server:
+To list the keys currently associated with the server:
 
 ```shell
 headscale apikeys list
@@ -39,7 +38,8 @@ headscale apikeys expire --prefix "<PREFIX>"
 
 ## Download and configure headscale
 
-1.  Download the latest [`headscale` binary from GitHub's release page](https://github.com/juanfont/headscale/releases):
+1.  Download the [`headscale` binary from GitHub's release page](https://github.com/juanfont/headscale/releases). Make
+    sure to use the same version as on the server.
 
 1.  Put the binary somewhere in your `PATH`, e.g. `/usr/local/bin/headscale`
 
@@ -49,25 +49,32 @@ headscale apikeys expire --prefix "<PREFIX>"
     chmod +x /usr/local/bin/headscale
     ```
 
-1.  Configure the CLI through environment variables
+1.  Provide the connection parameters for the remote headscale server either via a minimal YAML configuration file or via
+    environment variables:
 
-    ```shell
-    export HEADSCALE_CLI_ADDRESS="<HEADSCALE ADDRESS>:<PORT>"
-    export HEADSCALE_CLI_API_KEY="<API KEY FROM PREVIOUS STAGE>"
-    ```
+    === "Minimal YAML configuration file"
 
-    for example:
+        ```yaml
+        cli:
+            address: <HEADSCALE_ADDRESS>:<PORT>
+            api_key: <API_KEY_FROM_PREVIOUS_STEP>
+        ```
 
-    ```shell
-    export HEADSCALE_CLI_ADDRESS="headscale.example.com:50443"
-    export HEADSCALE_CLI_API_KEY="abcde12345"
-    ```
+    === "Environment variables"
 
-    This will tell the `headscale` binary to connect to a remote instance, instead of looking
-    for a local instance (which is what it does on the server).
+        ```shell
+        export HEADSCALE_CLI_ADDRESS="<HEADSCALE_ADDRESS>:<PORT>"
+        export HEADSCALE_CLI_API_KEY="<API_KEY_FROM_PREVIOUS_STEP>"
+        ```
 
-    The API key is needed to make sure that you are allowed to access the server. The key is _not_
-    needed when running directly on the server, as the connection is local.
+        !!! bug
+
+            Headscale 0.23.0 requires at least an empty configuration file when environment variables are used to
+            specify connection details. See [issue 2193](https://github.com/juanfont/headscale/issues/2193) for more
+            information.
+
+    This instructs the `headscale` binary to connect to a remote instance at `<HEADSCALE_ADDRESS>:<PORT>`, instead of
+    connecting to the local instance.
 
 1.  Test the connection
 
@@ -89,10 +96,10 @@ While this is _not a supported_ feature, an example on how this can be set up on
 
 ## Troubleshooting
 
-Checklist:
-
-- Make sure you have the _same_ headscale version on your server and workstation
-- Make sure you use version `0.13.0` or newer.
-- Verify that your TLS certificate is valid and trusted
-  - If you do not have access to a trusted certificate (e.g. from Let's Encrypt), add your self signed certificate to the trust store of your OS or
-  - Set `HEADSCALE_CLI_INSECURE` to 0 in your environment
+- Make sure you have the _same_ headscale version on your server and workstation.
+- Ensure that connections to the gRPC port are allowed.
+- Verify that your TLS certificate is valid and trusted.
+- If you don't have access to a trusted certificate (e.g. from Let's Encrypt), either:
+    - Add your self-signed certificate to the trust store of your OS _or_
+    - Disable certificate verification by either setting `cli.insecure: true` in the configuration file or by setting
+      `HEADSCALE_CLI_INSECURE=1` via an environment variable. We do **not** recommend to disable certificate validation.

docs/setup/install/community.md

@@ -28,7 +28,7 @@ development version.
 
 ## Fedora, RHEL, CentOS
 
-A 3rd-party repository for various RPM based distributions is available at:
+A third-party repository for various RPM based distributions is available at:
 <https://copr.fedorainfracloud.org/coprs/jonathanspw/headscale/>. The site provides detailed setup and installation
 instructions.
 

docs/usage/connect/apple.md

@@ -47,3 +47,23 @@ tailscale login --login-server <YOUR_HEADSCALE_URL>
 - Under `Custom Login Server`, select `Add Account...`
 - Enter the URL of your headscale instance (e.g `https://headscale.example.com`) and press `Add Account`
 - Follow the login procedure in the browser
+
+## tvOS
+
+### Installation
+
+Install the official Tailscale tvOS client from the [App Store](https://apps.apple.com/app/tailscale/id1470499037).
+
+!!! danger
+
+    **Don't** open the Tailscale App after installation!
+
+### Configuring the headscale URL
+
+- Go Settings (the apple tvOS settings) > Apps > Tailscale
+- Under `ALTERNATE COORDINATION SERVER URL`, select `URL`
+- Enter the URL of your headscale instance (e.g `https://headscale.example.com`) and press `OK`
+- Return to the tvOS Home screen
+- Open Tailscale
+- Click the button `Install VPN configuration` and confirm the appearing popup by clicking the `Allow` button
+- Scan the QR code and follow the login procedure

flake.lock

@@ -20,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1729850857,
-        "narHash": "sha256-WvLXzNNnnw+qpFOmgaM3JUlNEH+T4s22b5i2oyyCpXE=",
+        "lastModified": 1730958623,
+        "narHash": "sha256-JwQZIGSYnRNOgDDoIgqKITrPVil+RMWHsZH1eE1VGN0=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "41dea55321e5a999b17033296ac05fe8a8b5a257",
+        "rev": "85f7e662eda4fa3a995556527c87b2524b691933",
         "type": "github"
       },
       "original": {

hscontrol/app.go

@@ -457,9 +457,12 @@ func (h *Headscale) createRouter(grpcMux *grpcRuntime.ServeMux) *mux.Router {
 	router.HandleFunc("/swagger/v1/openapiv2.json", headscale.SwaggerAPIv1).
 		Methods(http.MethodGet)
 
+	router.HandleFunc("/verify", h.VerifyHandler).Methods(http.MethodPost)
+
 	if h.cfg.DERP.ServerEnabled {
 		router.HandleFunc("/derp", h.DERPServer.DERPHandler)
 		router.HandleFunc("/derp/probe", derpServer.DERPProbeHandler)
+		router.HandleFunc("/derp/latency-check", derpServer.DERPProbeHandler)
 		router.HandleFunc("/bootstrap-dns", derpServer.DERPBootstrapDNSHandler(h.DERPMap))
 	}
 

hscontrol/handlers.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
@@ -56,6 +57,63 @@ func parseCabailityVersion(req *http.Request) (tailcfg.CapabilityVersion, error)
 	return tailcfg.CapabilityVersion(clientCapabilityVersion), nil
 }
 
+// see https://github.com/tailscale/tailscale/blob/964282d34f06ecc06ce644769c66b0b31d118340/derp/derp_server.go#L1159, Derp use verifyClientsURL to verify whether a client is allowed to connect to the DERP server.
+func (h *Headscale) VerifyHandler(
+	writer http.ResponseWriter,
+	req *http.Request,
+) {
+	if req.Method != http.MethodPost {
+		http.Error(writer, "Wrong method", http.StatusMethodNotAllowed)
+
+		return
+	}
+	log.Debug().
+		Str("handler", "/verify").
+		Msg("verify client")
+
+	doVerify := func() (bool, error) {
+		body, err := io.ReadAll(req.Body)
+		if err != nil {
+			return false, fmt.Errorf("cannot read request body: %w", err)
+		}
+
+		var derpAdmitClientRequest tailcfg.DERPAdmitClientRequest
+		if err := json.Unmarshal(body, &derpAdmitClientRequest); err != nil {
+			return false, fmt.Errorf("cannot parse derpAdmitClientRequest: %w", err)
+		}
+
+		nodes, err := h.db.ListNodes()
+		if err != nil {
+			return false, fmt.Errorf("cannot list nodes: %w", err)
+		}
+
+		return nodes.ContainsNodeKey(derpAdmitClientRequest.NodePublic), nil
+	}
+
+	allow, err := doVerify()
+	if err != nil {
+		log.Error().
+			Caller().
+			Err(err).
+			Msg("Failed to verify client")
+		http.Error(writer, "Internal error", http.StatusInternalServerError)
+	}
+
+	resp := tailcfg.DERPAdmitClientResponse{
+		Allow: allow,
+	}
+
+	writer.Header().Set("Content-Type", "application/json")
+	writer.WriteHeader(http.StatusOK)
+	err = json.NewEncoder(writer).Encode(resp)
+	if err != nil {
+		log.Error().
+			Caller().
+			Err(err).
+			Msg("Failed to write response")
+	}
+}
+
 // KeyHandler provides the Headscale pub key
 // Listens in /key.
 func (h *Headscale) KeyHandler(

hscontrol/metrics.go

@@ -78,7 +78,7 @@ func prometheusMiddleware(next http.Handler) http.Handler {
 
 		// Ignore streaming and noise sessions
 		// it has its own router further down.
-		if path == "/ts2021" || path == "/machine/map" || path == "/derp" || path == "/derp/probe" || path == "/bootstrap-dns" {
+		if path == "/ts2021" || path == "/machine/map" || path == "/derp" || path == "/derp/probe" || path == "/derp/latency-check" || path == "/bootstrap-dns" {
 			next.ServeHTTP(w, r)
 			return
 		}

hscontrol/templates/apple.go

@@ -17,9 +17,13 @@ func Apple(url string) *elem.Element {
 			headerOne("headscale: iOS configuration"),
 			headerTwo("GUI"),
 			elem.Ol(nil,
-				elem.Li(nil,
+				elem.Li(
+					nil,
 					elem.Text("Install the official Tailscale iOS client from the "),
-					elem.A(attrs.Props{attrs.Href: "https://apps.apple.com/app/tailscale/id1470499037"},
+					elem.A(
+						attrs.Props{
+							attrs.Href: "https://apps.apple.com/app/tailscale/id1470499037",
+						},
 						elem.Text("App store"),
 					),
 				),
@@ -31,27 +35,47 @@ func Apple(url string) *elem.Element {
 				elem.Li(nil,
 					elem.Text("Open Settings on the iOS device"),
 				),
-				elem.Li(nil,
-					elem.Text(`Scroll down to the "third party apps" section, under "Game Center" or "TV Provider"`),
+				elem.Li(
+					nil,
+					elem.Text(
+						`Scroll down to the "third party apps" section, under "Game Center" or "TV Provider"`,
+					),
 				),
 				elem.Li(nil,
 					elem.Text("Find Tailscale and select it"),
 					elem.Ul(nil,
-						elem.Li(nil,
-							elem.Text(`If the iOS device was previously logged into Tailscale, switch the "Reset Keychain" toggle to "on"`),
+						elem.Li(
+							nil,
+							elem.Text(
+								`If the iOS device was previously logged into Tailscale, switch the "Reset Keychain" toggle to "on"`,
+							),
 						),
 					),
 				),
-				elem.Li(nil,
-					elem.Text(fmt.Sprintf(`Enter "%s" under "Alternate Coordination Server URL"`,url)),
+				elem.Li(
+					nil,
+					elem.Text(
+						fmt.Sprintf(
+							`Enter "%s" under "Alternate Coordination Server URL"`,
+							url,
+						),
+					),
 				),
-				elem.Li(nil,
-					elem.Text("Restart the app by closing it from the iOS app switcher, open the app and select the regular sign in option "),
+				elem.Li(
+					nil,
+					elem.Text(
+						"Restart the app by closing it from the iOS app switcher, open the app and select the regular sign in option ",
+					),
 					elem.I(nil, elem.Text("(non-SSO)")),
-					elem.Text(". It should open up to the headscale authentication page."),
+					elem.Text(
+						". It should open up to the headscale authentication page.",
+					),
 				),
-				elem.Li(nil,
-					elem.Text("Enter your credentials and log in. Headscale should now be working on your iOS device"),
+				elem.Li(
+					nil,
+					elem.Text(
+						"Enter your credentials and log in. Headscale should now be working on your iOS device",
+					),
 				),
 			),
 			headerOne("headscale: macOS configuration"),
@@ -61,39 +85,63 @@ func Apple(url string) *elem.Element {
 			),
 			elem.Pre(nil,
 				elem.Code(nil,
-					elem.Text(fmt.Sprintf("tailscale login --login-server %s",url)),
+					elem.Text(fmt.Sprintf("tailscale login --login-server %s", url)),
 				),
 			),
 			headerTwo("GUI"),
 			elem.Ol(nil,
-				elem.Li(nil,
-					elem.Text("ALT + Click the Tailscale icon in the menu and hover over the Debug menu"),
+				elem.Li(
+					nil,
+					elem.Text(
+						"ALT + Click the Tailscale icon in the menu and hover over the Debug menu",
+					),
 				),
 				elem.Li(nil,
 					elem.Text(`Under "Custom Login Server", select "Add Account..."`),
 				),
-				elem.Li(nil,
-					elem.Text(fmt.Sprintf(`Enter "%s" of the headscale instance and press "Add Account"`,url)),
+				elem.Li(
+					nil,
+					elem.Text(
+						fmt.Sprintf(
+							`Enter "%s" of the headscale instance and press "Add Account"`,
+							url,
+						),
+					),
 				),
 				elem.Li(nil,
 					elem.Text(`Follow the login procedure in the browser`),
 				),
 			),
 			headerTwo("Profiles"),
-			elem.P(nil,
-				elem.Text("Headscale can be set to the default server by installing a Headscale configuration profile:"),
+			elem.P(
+				nil,
+				elem.Text(
+					"Headscale can be set to the default server by installing a Headscale configuration profile:",
+				),
 			),
-			elem.P(nil,
-				elem.A(attrs.Props{attrs.Href: "/apple/macos-app-store", attrs.Download: "headscale_macos.mobileconfig"},
+			elem.P(
+				nil,
+				elem.A(
+					attrs.Props{
+						attrs.Href:     "/apple/macos-app-store",
+						attrs.Download: "headscale_macos.mobileconfig",
+					},
 					elem.Text("macOS AppStore profile "),
 				),
-				elem.A(attrs.Props{attrs.Href: "/apple/macos-standalone", attrs.Download: "headscale_macos.mobileconfig"},
+				elem.A(
+					attrs.Props{
+						attrs.Href:     "/apple/macos-standalone",
+						attrs.Download: "headscale_macos.mobileconfig",
+					},
 					elem.Text("macOS Standalone profile"),
 				),
 			),
 			elem.Ol(nil,
-				elem.Li(nil,
-					elem.Text("Download the profile, then open it. When it has been opened, there should be a notification that a profile can be installed"),
+				elem.Li(
+					nil,
+					elem.Text(
+						"Download the profile, then open it. When it has been opened, there should be a notification that a profile can be installed",
+					),
 				),
 				elem.Li(nil,
 					elem.Text(`Open System Preferences and go to "Profiles"`),
@@ -106,20 +154,35 @@ func Apple(url string) *elem.Element {
 				),
 			),
 			elem.P(nil, elem.Text("Or")),
-			elem.P(nil,
-				elem.Text("Use your terminal to configure the default setting for Tailscale by issuing:"),
+			elem.P(
+				nil,
+				elem.Text(
+					"Use your terminal to configure the default setting for Tailscale by issuing:",
+				),
 			),
 			elem.Ul(nil,
 				elem.Li(nil,
 					elem.Text(`for app store client:`),
-					elem.Code(nil,
-						elem.Text(fmt.Sprintf(`defaults write io.tailscale.ipn.macos ControlURL %s`,url)),
+					elem.Code(
+						nil,
+						elem.Text(
+							fmt.Sprintf(
+								`defaults write io.tailscale.ipn.macos ControlURL %s`,
+								url,
+							),
+						),
 					),
 				),
 				elem.Li(nil,
 					elem.Text(`for standalone client:`),
-					elem.Code(nil,
-						elem.Text(fmt.Sprintf(`defaults write io.tailscale.ipn.macsys ControlURL %s`,url)),
+					elem.Code(
+						nil,
+						elem.Text(
+							fmt.Sprintf(
+								`defaults write io.tailscale.ipn.macsys ControlURL %s`,
+								url,
+							),
+						),
 					),
 				),
 			),
@@ -127,23 +190,73 @@ func Apple(url string) *elem.Element {
 				elem.Text("Restart Tailscale.app and log in."),
 			),
 			headerThree("Caution"),
-			elem.P(nil,
-				elem.Text("You should always download and inspect the profile before installing it:"),
+			elem.P(
+				nil,
+				elem.Text(
+					"You should always download and inspect the profile before installing it:",
+				),
 			),
 			elem.Ul(nil,
 				elem.Li(nil,
 					elem.Text(`for app store client: `),
 					elem.Code(nil,
-						elem.Text(fmt.Sprintf(`curl %s/apple/macos-app-store`,url)),
+						elem.Text(fmt.Sprintf(`curl %s/apple/macos-app-store`, url)),
 					),
 				),
 				elem.Li(nil,
 					elem.Text(`for standalone client: `),
 					elem.Code(nil,
-						elem.Text(fmt.Sprintf(`curl %s/apple/macos-standalone`,url)),
+						elem.Text(fmt.Sprintf(`curl %s/apple/macos-standalone`, url)),
 					),
 				),
 			),
+			headerOne("headscale: tvOS configuration"),
+			headerTwo("GUI"),
+			elem.Ol(nil,
+				elem.Li(
+					nil,
+					elem.Text("Install the official Tailscale tvOS client from the "),
+					elem.A(
+						attrs.Props{
+							attrs.Href: "https://apps.apple.com/app/tailscale/id1470499037",
+						},
+						elem.Text("App store"),
+					),
+				),
+				elem.Li(
+					nil,
+					elem.Text(
+						"Open Settings (the Apple tvOS settings) > Apps > Tailscale",
+					),
+				),
+				elem.Li(
+					nil,
+					elem.Text(
+						fmt.Sprintf(
+							`Enter "%s" under "ALTERNATE COORDINATION SERVER URL"`,
+							url,
+						),
+					),
+				),
+				elem.Li(nil,
+					elem.Text("Return to the tvOS Home screen"),
+				),
+				elem.Li(nil,
+					elem.Text("Open Tailscale"),
+				),
+				elem.Li(nil,
+					elem.Text(`Select "Install VPN configuration"`),
+				),
+				elem.Li(nil,
+					elem.Text(`Select "Allow"`),
+				),
+				elem.Li(nil,
+					elem.Text("Scan the QR code and follow the login procedure"),
+				),
+				elem.Li(nil,
+					elem.Text("Headscale should now be working on your tvOS device"),
+				),
+			),
 		),
 	)
 }

hscontrol/types/node.go

@@ -223,6 +223,16 @@ func (nodes Nodes) FilterByIP(ip netip.Addr) Nodes {
 	return found
 }
 
+func (nodes Nodes) ContainsNodeKey(nodeKey key.NodePublic) bool {
+	for _, node := range nodes {
+		if node.NodeKey == nodeKey {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (node *Node) Proto() *v1.Node {
 	nodeProto := &v1.Node{
 		Id:         uint64(node.ID),

integration/README.md

@@ -11,10 +11,10 @@ Tests are located in files ending with `_test.go` and the framework are located
 
 ## Running integration tests locally
 
-The easiest way to run tests locally is to use `[act](INSERT LINK)`, a local GitHub Actions runner:
+The easiest way to run tests locally is to use [act](https://github.com/nektos/act), a local GitHub Actions runner:
 
 ```
-act pull_request -W .github/workflows/test-integration-v2-TestPingAllByIP.yaml
+act pull_request -W .github/workflows/test-integration.yaml
 ```
 
 Alternatively, the `docker run` command in each GitHub workflow file can be used.

integration/derp_verify_endpoint_test.go

@@ -0,0 +1,96 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dsic"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/juanfont/headscale/integration/tsic"
+	"tailscale.com/tailcfg"
+)
+
+func TestDERPVerifyEndpoint(t *testing.T) {
+	IntegrationSkip(t)
+
+	// Generate random hostname for the headscale instance
+	hash, err := util.GenerateRandomStringDNSSafe(6)
+	assertNoErr(t, err)
+	testName := "derpverify"
+	hostname := fmt.Sprintf("hs-%s-%s", testName, hash)
+
+	headscalePort := 8080
+
+	// Create cert for headscale
+	certHeadscale, keyHeadscale, err := integrationutil.CreateCertificate(hostname)
+	assertNoErr(t, err)
+
+	scenario, err := NewScenario(dockertestMaxWait())
+	assertNoErr(t, err)
+	defer scenario.Shutdown()
+
+	spec := map[string]int{
+		"user1": 10,
+	}
+
+	derper, err := scenario.CreateDERPServer("head",
+		dsic.WithCACert(certHeadscale),
+		dsic.WithVerifyClientURL(fmt.Sprintf("https://%s/verify", net.JoinHostPort(hostname, strconv.Itoa(headscalePort)))),
+	)
+	assertNoErr(t, err)
+
+	derpMap := tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			900: {
+				RegionID:   900,
+				RegionCode: "test-derpverify",
+				RegionName: "TestDerpVerify",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:     "TestDerpVerify",
+						RegionID: 900,
+						HostName: derper.GetHostname(),
+						STUNPort: derper.GetSTUNPort(),
+						STUNOnly: false,
+						DERPPort: derper.GetDERPPort(),
+					},
+				},
+			},
+		},
+	}
+
+	err = scenario.CreateHeadscaleEnv(spec, []tsic.Option{tsic.WithCACert(derper.GetCert())},
+		hsic.WithHostname(hostname),
+		hsic.WithPort(headscalePort),
+		hsic.WithCustomTLS(certHeadscale, keyHeadscale),
+		hsic.WithHostnameAsServerURL(),
+		hsic.WithDERPConfig(derpMap))
+	assertNoErrHeadscaleEnv(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	assertNoErrListClients(t, err)
+
+	for _, client := range allClients {
+		report, err := client.DebugDERPRegion("test-derpverify")
+		assertNoErr(t, err)
+		successful := false
+		for _, line := range report.Info {
+			if strings.Contains(line, "Successfully established a DERP connection with node") {
+				successful = true
+
+				break
+			}
+		}
+		if !successful {
+			stJSON, err := json.Marshal(report)
+			assertNoErr(t, err)
+			t.Errorf("Client %s could not establish a DERP connection: %s", client.Hostname(), string(stJSON))
+		}
+	}
+}

integration/dsic/dsic.go

@@ -0,0 +1,321 @@
+package dsic
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+const (
+	dsicHashLength       = 6
+	dockerContextPath    = "../."
+	caCertRoot           = "/usr/local/share/ca-certificates"
+	DERPerCertRoot       = "/usr/local/share/derper-certs"
+	dockerExecuteTimeout = 60 * time.Second
+)
+
+var errDERPerStatusCodeNotOk = errors.New("DERPer status code not OK")
+
+// DERPServerInContainer represents DERP Server in Container (DSIC).
+type DERPServerInContainer struct {
+	version  string
+	hostname string
+
+	pool      *dockertest.Pool
+	container *dockertest.Resource
+	network   *dockertest.Network
+
+	stunPort            int
+	derpPort            int
+	caCerts             [][]byte
+	tlsCert             []byte
+	tlsKey              []byte
+	withExtraHosts      []string
+	withVerifyClientURL string
+	workdir             string
+}
+
+// Option represent optional settings that can be given to a
+// DERPer instance.
+type Option = func(c *DERPServerInContainer)
+
+// WithCACert adds it to the trusted surtificate of the Tailscale container.
+func WithCACert(cert []byte) Option {
+	return func(dsic *DERPServerInContainer) {
+		dsic.caCerts = append(dsic.caCerts, cert)
+	}
+}
+
+// WithOrCreateNetwork sets the Docker container network to use with
+// the DERPer instance, if the parameter is nil, a new network,
+// isolating the DERPer, will be created. If a network is
+// passed, the DERPer instance will join the given network.
+func WithOrCreateNetwork(network *dockertest.Network) Option {
+	return func(tsic *DERPServerInContainer) {
+		if network != nil {
+			tsic.network = network
+
+			return
+		}
+
+		network, err := dockertestutil.GetFirstOrCreateNetwork(
+			tsic.pool,
+			tsic.hostname+"-network",
+		)
+		if err != nil {
+			log.Fatalf("failed to create network: %s", err)
+		}
+
+		tsic.network = network
+	}
+}
+
+// WithDockerWorkdir allows the docker working directory to be set.
+func WithDockerWorkdir(dir string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.workdir = dir
+	}
+}
+
+// WithVerifyClientURL sets the URL to verify the client.
+func WithVerifyClientURL(url string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.withVerifyClientURL = url
+	}
+}
+
+// WithExtraHosts adds extra hosts to the container.
+func WithExtraHosts(hosts []string) Option {
+	return func(tsic *DERPServerInContainer) {
+		tsic.withExtraHosts = hosts
+	}
+}
+
+// New returns a new TailscaleInContainer instance.
+func New(
+	pool *dockertest.Pool,
+	version string,
+	network *dockertest.Network,
+	opts ...Option,
+) (*DERPServerInContainer, error) {
+	hash, err := util.GenerateRandomStringDNSSafe(dsicHashLength)
+	if err != nil {
+		return nil, err
+	}
+
+	hostname := fmt.Sprintf("derp-%s-%s", strings.ReplaceAll(version, ".", "-"), hash)
+	tlsCert, tlsKey, err := integrationutil.CreateCertificate(hostname)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create certificates for headscale test: %w", err)
+	}
+	dsic := &DERPServerInContainer{
+		version:  version,
+		hostname: hostname,
+		pool:     pool,
+		network:  network,
+		tlsCert:  tlsCert,
+		tlsKey:   tlsKey,
+		stunPort: 3478, //nolint
+		derpPort: 443,  //nolint
+	}
+
+	for _, opt := range opts {
+		opt(dsic)
+	}
+
+	var cmdArgs strings.Builder
+	fmt.Fprintf(&cmdArgs, "--hostname=%s", hostname)
+	fmt.Fprintf(&cmdArgs, " --certmode=manual")
+	fmt.Fprintf(&cmdArgs, " --certdir=%s", DERPerCertRoot)
+	fmt.Fprintf(&cmdArgs, " --a=:%d", dsic.derpPort)
+	fmt.Fprintf(&cmdArgs, " --stun=true")
+	fmt.Fprintf(&cmdArgs, " --stun-port=%d", dsic.stunPort)
+	if dsic.withVerifyClientURL != "" {
+		fmt.Fprintf(&cmdArgs, " --verify-client-url=%s", dsic.withVerifyClientURL)
+	}
+
+	runOptions := &dockertest.RunOptions{
+		Name:       hostname,
+		Networks:   []*dockertest.Network{dsic.network},
+		ExtraHosts: dsic.withExtraHosts,
+		// we currently need to give us some time to inject the certificate further down.
+		Entrypoint: []string{"/bin/sh", "-c", "/bin/sleep 3 ; update-ca-certificates ; derper " + cmdArgs.String()},
+		ExposedPorts: []string{
+			"80/tcp",
+			fmt.Sprintf("%d/tcp", dsic.derpPort),
+			fmt.Sprintf("%d/udp", dsic.stunPort),
+		},
+	}
+
+	if dsic.workdir != "" {
+		runOptions.WorkingDir = dsic.workdir
+	}
+
+	// dockertest isnt very good at handling containers that has already
+	// been created, this is an attempt to make sure this container isnt
+	// present.
+	err = pool.RemoveContainerByName(hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	var container *dockertest.Resource
+	buildOptions := &dockertest.BuildOptions{
+		Dockerfile: "Dockerfile.derper",
+		ContextDir: dockerContextPath,
+		BuildArgs:  []docker.BuildArg{},
+	}
+	switch version {
+	case "head":
+		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+			Name:  "VERSION_BRANCH",
+			Value: "main",
+		})
+	default:
+		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
+			Name:  "VERSION_BRANCH",
+			Value: "v" + version,
+		})
+	}
+	container, err = pool.BuildAndRunWithBuildOptions(
+		buildOptions,
+		runOptions,
+		dockertestutil.DockerRestartPolicy,
+		dockertestutil.DockerAllowLocalIPv6,
+		dockertestutil.DockerAllowNetworkAdministration,
+	)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"%s could not start tailscale DERPer container (version: %s): %w",
+			hostname,
+			version,
+			err,
+		)
+	}
+	log.Printf("Created %s container\n", hostname)
+
+	dsic.container = container
+
+	for i, cert := range dsic.caCerts {
+		err = dsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+	if len(dsic.tlsCert) != 0 {
+		err = dsic.WriteFile(fmt.Sprintf("%s/%s.crt", DERPerCertRoot, dsic.hostname), dsic.tlsCert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+	if len(dsic.tlsKey) != 0 {
+		err = dsic.WriteFile(fmt.Sprintf("%s/%s.key", DERPerCertRoot, dsic.hostname), dsic.tlsKey)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS key to container: %w", err)
+		}
+	}
+
+	return dsic, nil
+}
+
+// Shutdown stops and cleans up the DERPer container.
+func (t *DERPServerInContainer) Shutdown() error {
+	err := t.SaveLog("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"Failed to save log from %s: %s",
+			t.hostname,
+			fmt.Errorf("failed to save log: %w", err),
+		)
+	}
+
+	return t.pool.Purge(t.container)
+}
+
+// GetCert returns the TLS certificate of the DERPer instance.
+func (t *DERPServerInContainer) GetCert() []byte {
+	return t.tlsCert
+}
+
+// Hostname returns the hostname of the DERPer instance.
+func (t *DERPServerInContainer) Hostname() string {
+	return t.hostname
+}
+
+// Version returns the running DERPer version of the instance.
+func (t *DERPServerInContainer) Version() string {
+	return t.version
+}
+
+// ID returns the Docker container ID of the DERPServerInContainer
+// instance.
+func (t *DERPServerInContainer) ID() string {
+	return t.container.Container.ID
+}
+
+func (t *DERPServerInContainer) GetHostname() string {
+	return t.hostname
+}
+
+// GetSTUNPort returns the STUN port of the DERPer instance.
+func (t *DERPServerInContainer) GetSTUNPort() int {
+	return t.stunPort
+}
+
+// GetDERPPort returns the DERP port of the DERPer instance.
+func (t *DERPServerInContainer) GetDERPPort() int {
+	return t.derpPort
+}
+
+// WaitForRunning blocks until the DERPer instance is ready to be used.
+func (t *DERPServerInContainer) WaitForRunning() error {
+	url := "https://" + net.JoinHostPort(t.GetHostname(), strconv.Itoa(t.GetDERPPort())) + "/"
+	log.Printf("waiting for DERPer to be ready at %s", url)
+
+	insecureTransport := http.DefaultTransport.(*http.Transport).Clone()      //nolint
+	insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint
+	client := &http.Client{Transport: insecureTransport}
+
+	return t.pool.Retry(func() error {
+		resp, err := client.Get(url) //nolint
+		if err != nil {
+			return fmt.Errorf("headscale is not ready: %w", err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return errDERPerStatusCodeNotOk
+		}
+
+		return nil
+	})
+}
+
+// ConnectToNetwork connects the DERPer instance to a network.
+func (t *DERPServerInContainer) ConnectToNetwork(network *dockertest.Network) error {
+	return t.container.ConnectToNetwork(network)
+}
+
+// WriteFile save file inside the container.
+func (t *DERPServerInContainer) WriteFile(path string, data []byte) error {
+	return integrationutil.WriteFileToContainer(t.pool, t.container, path, data)
+}
+
+// SaveLog saves the current stdout log of the container to a path
+// on the host system.
+func (t *DERPServerInContainer) SaveLog(path string) error {
+	_, _, err := dockertestutil.SaveLog(t.pool, t.container, path)
+
+	return err
+}

integration/embedded_derp_test.go

@@ -307,7 +307,7 @@ func (s *EmbeddedDERPServerScenario) CreateTailscaleIsolatedNodesInUser(
 			cert := hsServer.GetCert()
 
 			opts = append(opts,
-				tsic.WithHeadscaleTLS(cert),
+				tsic.WithCACert(cert),
 			)
 
 			user.createWaitGroup.Go(func() error {

integration/hsic/hsic.go

@@ -1,19 +1,12 @@
 package hsic
 
 import (
-	"bytes"
-	"crypto/rand"
-	"crypto/rsa"
 	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
 	"encoding/json"
-	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
 	"log"
-	"math/big"
 	"net"
 	"net/http"
 	"net/url"
@@ -32,11 +25,13 @@ import (
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
+	"tailscale.com/tailcfg"
 )
 
 const (
 	hsicHashLength       = 6
 	dockerContextPath    = "../."
+	caCertRoot           = "/usr/local/share/ca-certificates"
 	aclPolicyPath        = "/etc/headscale/acl.hujson"
 	tlsCertPath          = "/etc/headscale/tls.cert"
 	tlsKeyPath           = "/etc/headscale/tls.key"
@@ -64,6 +59,7 @@ type HeadscaleInContainer struct {
 	// optional config
 	port             int
 	extraPorts       []string
+	caCerts          [][]byte
 	hostPortBindings map[string][]string
 	aclPolicy        *policy.ACLPolicy
 	env              map[string]string
@@ -88,18 +84,29 @@ func WithACLPolicy(acl *policy.ACLPolicy) Option {
 	}
 }
 
+// WithCACert adds it to the trusted surtificate of the container.
+func WithCACert(cert []byte) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.caCerts = append(hsic.caCerts, cert)
+	}
+}
+
 // WithTLS creates certificates and enables HTTPS.
 func WithTLS() Option {
 	return func(hsic *HeadscaleInContainer) {
-		cert, key, err := createCertificate(hsic.hostname)
+		cert, key, err := integrationutil.CreateCertificate(hsic.hostname)
 		if err != nil {
 			log.Fatalf("failed to create certificates for headscale test: %s", err)
 		}
 
-		// TODO(kradalby): Move somewhere appropriate
-		hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath
-		hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath
+		hsic.tlsCert = cert
+		hsic.tlsKey = key
+	}
+}
 
+// WithCustomTLS uses the given certificates for the Headscale instance.
+func WithCustomTLS(cert, key []byte) Option {
+	return func(hsic *HeadscaleInContainer) {
 		hsic.tlsCert = cert
 		hsic.tlsKey = key
 	}
@@ -146,6 +153,13 @@ func WithTestName(testName string) Option {
 	}
 }
 
+// WithHostname sets the hostname of the Headscale instance.
+func WithHostname(hostname string) Option {
+	return func(hsic *HeadscaleInContainer) {
+		hsic.hostname = hostname
+	}
+}
+
 // WithHostnameAsServerURL sets the Headscale ServerURL based on
 // the Hostname.
 func WithHostnameAsServerURL() Option {
@@ -203,6 +217,34 @@ func WithEmbeddedDERPServerOnly() Option {
 	}
 }
 
+// WithDERPConfig configures Headscale use a custom
+// DERP server only.
+func WithDERPConfig(derpMap tailcfg.DERPMap) Option {
+	return func(hsic *HeadscaleInContainer) {
+		contents, err := json.Marshal(derpMap)
+		if err != nil {
+			log.Fatalf("failed to marshal DERP map: %s", err)
+
+			return
+		}
+
+		hsic.env["HEADSCALE_DERP_PATHS"] = "/etc/headscale/derp.yml"
+		hsic.filesInContainer = append(hsic.filesInContainer,
+			fileInContainer{
+				path:     "/etc/headscale/derp.yml",
+				contents: contents,
+			})
+
+		// Disable global DERP server and embedded DERP server
+		hsic.env["HEADSCALE_DERP_URLS"] = ""
+		hsic.env["HEADSCALE_DERP_SERVER_ENABLED"] = "false"
+
+		// Envknob for enabling DERP debug logs
+		hsic.env["DERP_DEBUG_LOGS"] = "true"
+		hsic.env["DERP_PROBER_DEBUG_LOGS"] = "true"
+	}
+}
+
 // WithTuning allows changing the tuning settings easily.
 func WithTuning(batchTimeout time.Duration, mapSessionChanSize int) Option {
 	return func(hsic *HeadscaleInContainer) {
@@ -300,6 +342,10 @@ func New(
 		"HEADSCALE_DEBUG_HIGH_CARDINALITY_METRICS=1",
 		"HEADSCALE_DEBUG_DUMP_CONFIG=1",
 	}
+	if hsic.hasTLS() {
+		hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath
+		hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath
+	}
 	for key, value := range hsic.env {
 		env = append(env, fmt.Sprintf("%s=%s", key, value))
 	}
@@ -313,7 +359,7 @@ func New(
 		// Cmd:          []string{"headscale", "serve"},
 		// TODO(kradalby): Get rid of this hack, we currently need to give us some
 		// to inject the headscale configuration further down.
-		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; headscale serve ; /bin/sleep 30"},
+		Entrypoint: []string{"/bin/bash", "-c", "/bin/sleep 3 ; update-ca-certificates ; headscale serve ; /bin/sleep 30"},
 		Env:        env,
 	}
 
@@ -351,6 +397,14 @@ func New(
 
 	hsic.container = container
 
+	// Write the CA certificates to the container
+	for i, cert := range hsic.caCerts {
+		err = hsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+		}
+	}
+
 	err = hsic.WriteFile("/etc/headscale/config.yaml", []byte(MinimumConfigYAML()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to write headscale config to container: %w", err)
@@ -749,86 +803,3 @@ func (t *HeadscaleInContainer) SendInterrupt() error {
 
 	return nil
 }
-
-// nolint
-func createCertificate(hostname string) ([]byte, []byte, error) {
-	// From:
-	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
-
-	ca := &x509.Certificate{
-		SerialNumber: big.NewInt(2019),
-		Subject: pkix.Name{
-			Organization: []string{"Headscale testing INC"},
-			Country:      []string{"NL"},
-			Locality:     []string{"Leiden"},
-		},
-		NotBefore: time.Now(),
-		NotAfter:  time.Now().Add(60 * time.Hour),
-		IsCA:      true,
-		ExtKeyUsage: []x509.ExtKeyUsage{
-			x509.ExtKeyUsageClientAuth,
-			x509.ExtKeyUsageServerAuth,
-		},
-		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
-		BasicConstraintsValid: true,
-	}
-
-	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cert := &x509.Certificate{
-		SerialNumber: big.NewInt(1658),
-		Subject: pkix.Name{
-			CommonName:   hostname,
-			Organization: []string{"Headscale testing INC"},
-			Country:      []string{"NL"},
-			Locality:     []string{"Leiden"},
-		},
-		NotBefore:    time.Now(),
-		NotAfter:     time.Now().Add(60 * time.Minute),
-		SubjectKeyId: []byte{1, 2, 3, 4, 6},
-		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
-		KeyUsage:     x509.KeyUsageDigitalSignature,
-		DNSNames:     []string{hostname},
-	}
-
-	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certBytes, err := x509.CreateCertificate(
-		rand.Reader,
-		cert,
-		ca,
-		&certPrivKey.PublicKey,
-		caPrivKey,
-	)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certPEM := new(bytes.Buffer)
-
-	err = pem.Encode(certPEM, &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: certBytes,
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	certPrivKeyPEM := new(bytes.Buffer)
-
-	err = pem.Encode(certPrivKeyPEM, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
-	})
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
-}

integration/integrationutil/util.go

@@ -3,9 +3,16 @@ package integrationutil
 import (
 	"archive/tar"
 	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"math/big"
 	"path/filepath"
+	"time"
 
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/ory/dockertest/v3"
@@ -93,3 +100,86 @@ func FetchPathFromContainer(
 
 	return buf.Bytes(), nil
 }
+
+// nolint
+func CreateCertificate(hostname string) ([]byte, []byte, error) {
+	// From:
+	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
+
+	ca := &x509.Certificate{
+		SerialNumber: big.NewInt(2019),
+		Subject: pkix.Name{
+			Organization: []string{"Headscale testing INC"},
+			Country:      []string{"NL"},
+			Locality:     []string{"Leiden"},
+		},
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().Add(60 * time.Hour),
+		IsCA:      true,
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+		},
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+	}
+
+	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cert := &x509.Certificate{
+		SerialNumber: big.NewInt(1658),
+		Subject: pkix.Name{
+			CommonName:   hostname,
+			Organization: []string{"Headscale testing INC"},
+			Country:      []string{"NL"},
+			Locality:     []string{"Leiden"},
+		},
+		NotBefore:    time.Now(),
+		NotAfter:     time.Now().Add(60 * time.Minute),
+		SubjectKeyId: []byte{1, 2, 3, 4, 6},
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		DNSNames:     []string{hostname},
+	}
+
+	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certBytes, err := x509.CreateCertificate(
+		rand.Reader,
+		cert,
+		ca,
+		&certPrivKey.PublicKey,
+		caPrivKey,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certPEM := new(bytes.Buffer)
+
+	err = pem.Encode(certPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: certBytes,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certPrivKeyPEM := new(bytes.Buffer)
+
+	err = pem.Encode(certPrivKeyPEM, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
+}

integration/scenario.go

@@ -14,6 +14,7 @@ import (
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/dsic"
 	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/ory/dockertest/v3"
@@ -140,6 +141,7 @@ type Scenario struct {
 	// TODO(kradalby): support multiple headcales for later, currently only
 	// use one.
 	controlServers *xsync.MapOf[string, ControlServer]
+	derpServers    []*dsic.DERPServerInContainer
 
 	users map[string]*User
 
@@ -224,6 +226,13 @@ func (s *Scenario) ShutdownAssertNoPanics(t *testing.T) {
 		}
 	}
 
+	for _, derp := range s.derpServers {
+		err := derp.Shutdown()
+		if err != nil {
+			log.Printf("failed to tear down derp server: %s", err)
+		}
+	}
+
 	if err := s.pool.RemoveNetwork(s.network); err != nil {
 		log.Printf("failed to remove network: %s", err)
 	}
@@ -352,7 +361,7 @@ func (s *Scenario) CreateTailscaleNodesInUser(
 			hostname := headscale.GetHostname()
 
 			opts = append(opts,
-				tsic.WithHeadscaleTLS(cert),
+				tsic.WithCACert(cert),
 				tsic.WithHeadscaleName(hostname),
 			)
 
@@ -651,3 +660,20 @@ func (s *Scenario) WaitForTailscaleLogout() error {
 
 	return nil
 }
+
+// CreateDERPServer creates a new DERP server in a container.
+func (s *Scenario) CreateDERPServer(version string, opts ...dsic.Option) (*dsic.DERPServerInContainer, error) {
+	derp, err := dsic.New(s.pool, version, s.network, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DERP server: %w", err)
+	}
+
+	err = derp.WaitForRunning()
+	if err != nil {
+		return nil, fmt.Errorf("failed to reach DERP server: %w", err)
+	}
+
+	s.derpServers = append(s.derpServers, derp)
+
+	return derp, nil
+}

integration/tailscale.go

@@ -30,6 +30,7 @@ type TailscaleClient interface {
 	FQDN() (string, error)
 	Status(...bool) (*ipnstate.Status, error)
 	Netmap() (*netmap.NetworkMap, error)
+	DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error)
 	Netcheck() (*netcheck.Report, error)
 	WaitForNeedsLogin() error
 	WaitForRunning() error

integration/tsic/tsic.go

@@ -32,7 +32,7 @@ const (
 	defaultPingTimeout   = 300 * time.Millisecond
 	defaultPingCount     = 10
 	dockerContextPath    = "../."
-	headscaleCertPath    = "/usr/local/share/ca-certificates/headscale.crt"
+	caCertRoot           = "/usr/local/share/ca-certificates"
 	dockerExecuteTimeout = 60 * time.Second
 )
 
@@ -65,7 +65,7 @@ type TailscaleInContainer struct {
 	fqdn string
 
 	// optional config
-	headscaleCert     []byte
+	caCerts           [][]byte
 	headscaleHostname string
 	withWebsocketDERP bool
 	withSSH           bool
@@ -80,11 +80,10 @@ type TailscaleInContainer struct {
 // Tailscale instance.
 type Option = func(c *TailscaleInContainer)
 
-// WithHeadscaleTLS takes the certificate of the Headscale instance
-// and adds it to the trusted surtificate of the Tailscale container.
-func WithHeadscaleTLS(cert []byte) Option {
+// WithCACert adds it to the trusted surtificate of the Tailscale container.
+func WithCACert(cert []byte) Option {
 	return func(tsic *TailscaleInContainer) {
-		tsic.headscaleCert = cert
+		tsic.caCerts = append(tsic.caCerts, cert)
 	}
 }
 
@@ -113,7 +112,7 @@ func WithOrCreateNetwork(network *dockertest.Network) Option {
 }
 
 // WithHeadscaleName set the name of the headscale instance,
-// mostly useful in combination with TLS and WithHeadscaleTLS.
+// mostly useful in combination with TLS and WithCACert.
 func WithHeadscaleName(hsName string) Option {
 	return func(tsic *TailscaleInContainer) {
 		tsic.headscaleHostname = hsName
@@ -225,12 +224,8 @@ func New(
 		)
 	}
 
-	if tsic.headscaleHostname != "" {
-		tailscaleOptions.ExtraHosts = []string{
-			"host.docker.internal:host-gateway",
-			fmt.Sprintf("%s:host-gateway", tsic.headscaleHostname),
-		}
-	}
+	tailscaleOptions.ExtraHosts = append(tailscaleOptions.ExtraHosts,
+		"host.docker.internal:host-gateway")
 
 	if tsic.workdir != "" {
 		tailscaleOptions.WorkingDir = tsic.workdir
@@ -294,8 +289,8 @@ func New(
 
 	tsic.container = container
 
-	if tsic.hasTLS() {
-		err = tsic.WriteFile(headscaleCertPath, tsic.headscaleCert)
+	for i, cert := range tsic.caCerts {
+		err = tsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
 		if err != nil {
 			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
 		}
@@ -304,10 +299,6 @@ func New(
 	return tsic, nil
 }
 
-func (t *TailscaleInContainer) hasTLS() bool {
-	return len(t.headscaleCert) != 0
-}
-
 // Shutdown stops and cleans up the Tailscale container.
 func (t *TailscaleInContainer) Shutdown() error {
 	err := t.SaveLog("/tmp/control")
@@ -682,6 +673,34 @@ func (t *TailscaleInContainer) watchIPN(ctx context.Context) (*ipn.Notify, error
 	}
 }
 
+func (t *TailscaleInContainer) DebugDERPRegion(region string) (*ipnstate.DebugDERPRegionReport, error) {
+	if !util.TailscaleVersionNewerOrEqual("1.34", t.version) {
+		panic("tsic.DebugDERPRegion() called with unsupported version: " + t.version)
+	}
+
+	command := []string{
+		"tailscale",
+		"debug",
+		"derp",
+		region,
+	}
+
+	result, stderr, err := t.Execute(command)
+	if err != nil {
+		fmt.Printf("stderr: %s\n", stderr) // nolint
+
+		return nil, fmt.Errorf("failed to execute tailscale debug derp command: %w", err)
+	}
+
+	var report ipnstate.DebugDERPRegionReport
+	err = json.Unmarshal([]byte(result), &report)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal tailscale derp region report: %w", err)
+	}
+
+	return &report, err
+}
+
 // Netcheck returns the current Netcheck Report (netcheck.Report) of the Tailscale instance.
 func (t *TailscaleInContainer) Netcheck() (*netcheck.Report, error) {
 	command := []string{

mkdocs.yml

@@ -183,3 +183,4 @@ nav:
       - Integration:
           - Reverse proxy: ref/integration/reverse-proxy.md
           - Web UI: ref/integration/web-ui.md
+          - Tools: ref/integration/tools.md