Split derp into its own config struct

app.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
 	"os"
 	"sort"
 	"strings"
@@ -28,11 +29,12 @@ type Config struct {
 	ServerURL                      string
 	Addr                           string
 	PrivateKeyPath                 string
-	DerpMap                        *tailcfg.DERPMap
 	EphemeralNodeInactivityTimeout time.Duration
 	IPPrefix                       netaddr.IPPrefix
 	BaseDomain                     string
 
+	DERP DERPConfig
+
 	DBtype string
 	DBpath string
 	DBhost string
@@ -55,6 +57,13 @@ type Config struct {
 	DNSConfig *tailcfg.DNSConfig
 }
 
+type DERPConfig struct {
+	URLs            []url.URL
+	Paths           []string
+	AutoUpdate      bool
+	UpdateFrequency time.Duration
+}
+
 // Headscale represents the base app of the service
 type Headscale struct {
 	cfg        Config
@@ -65,6 +74,8 @@ type Headscale struct {
 	publicKey  *wgkey.Key
 	privateKey *wgkey.Private
 
+	DERPMap *tailcfg.DERPMap
+
 	aclPolicy *ACLPolicy
 	aclRules  *[]tailcfg.FilterRule
 
@@ -153,11 +164,15 @@ func (h *Headscale) expireEphemeralNodesWorker() {
 			return
 		}
 		for _, m := range *machines {
-			if m.AuthKey != nil && m.LastSeen != nil && m.AuthKey.Ephemeral && time.Now().After(m.LastSeen.Add(h.cfg.EphemeralNodeInactivityTimeout)) {
+			if m.AuthKey != nil && m.LastSeen != nil && m.AuthKey.Ephemeral &&
+				time.Now().After(m.LastSeen.Add(h.cfg.EphemeralNodeInactivityTimeout)) {
 				log.Info().Str("machine", m.Name).Msg("Ephemeral client removed from database")
 				err = h.db.Unscoped().Delete(m).Error
 				if err != nil {
-					log.Error().Err(err).Str("machine", m.Name).Msg("🤮 Cannot delete ephemeral machine from the database")
+					log.Error().
+						Err(err).
+						Str("machine", m.Name).
+						Msg("🤮 Cannot delete ephemeral machine from the database")
 				}
 			}
 		}
@@ -198,6 +213,15 @@ func (h *Headscale) Serve() error {
 	go h.watchForKVUpdates(5000)
 	go h.expireEphemeralNodes(5000)
 
+	// Fetch an initial DERP Map before we start serving
+	h.DERPMap = GetDERPMap(h.cfg.DERP)
+
+	if h.cfg.DERP.AutoUpdate {
+		derpMapCancelChannel := make(chan struct{})
+		defer func() { derpMapCancelChannel <- struct{}{} }()
+		go h.scheduledDERPMapUpdateWorker(derpMapCancelChannel)
+	}
+
 	s := &http.Server{
 		Addr:        h.cfg.Addr,
 		Handler:     r,
@@ -273,7 +297,6 @@ func (h *Headscale) getLastStateChange(namespaces ...string) time.Time {
 
 			times = append(times, lastChange)
 		}
-
 	}
 
 	sort.Slice(times, func(i, j int) bool {
@@ -284,7 +307,6 @@ func (h *Headscale) getLastStateChange(namespaces ...string) time.Time {
 
 	if len(times) == 0 {
 		return time.Now().UTC()
-
 	} else {
 		return times[0]
 	}

cmd/headscale/cli/utils.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,7 +13,6 @@ import (
 	"github.com/juanfont/headscale"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
-	"gopkg.in/yaml.v2"
 	"inet.af/netaddr"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/dnstype"
@@ -51,21 +50,26 @@ func LoadConfig(path string) error {
 
 	// Collect any validation errors and return them all at once
 	var errorText string
-	if (viper.GetString("tls_letsencrypt_hostname") != "") && ((viper.GetString("tls_cert_path") != "") || (viper.GetString("tls_key_path") != "")) {
+	if (viper.GetString("tls_letsencrypt_hostname") != "") &&
+		((viper.GetString("tls_cert_path") != "") || (viper.GetString("tls_key_path") != "")) {
 		errorText += "Fatal config error: set either tls_letsencrypt_hostname or tls_cert_path/tls_key_path, not both\n"
 	}
 
-	if (viper.GetString("tls_letsencrypt_hostname") != "") && (viper.GetString("tls_letsencrypt_challenge_type") == "TLS-ALPN-01") && (!strings.HasSuffix(viper.GetString("listen_addr"), ":443")) {
+	if (viper.GetString("tls_letsencrypt_hostname") != "") &&
+		(viper.GetString("tls_letsencrypt_challenge_type") == "TLS-ALPN-01") &&
+		(!strings.HasSuffix(viper.GetString("listen_addr"), ":443")) {
 		// this is only a warning because there could be something sitting in front of headscale that redirects the traffic (e.g. an iptables rule)
 		log.Warn().
 			Msg("Warning: when using tls_letsencrypt_hostname with TLS-ALPN-01 as challenge type, headscale must be reachable on port 443, i.e. listen_addr should probably end in :443")
 	}
 
-	if (viper.GetString("tls_letsencrypt_challenge_type") != "HTTP-01") && (viper.GetString("tls_letsencrypt_challenge_type") != "TLS-ALPN-01") {
+	if (viper.GetString("tls_letsencrypt_challenge_type") != "HTTP-01") &&
+		(viper.GetString("tls_letsencrypt_challenge_type") != "TLS-ALPN-01") {
 		errorText += "Fatal config error: the only supported values for tls_letsencrypt_challenge_type are HTTP-01 and TLS-ALPN-01\n"
 	}
 
-	if !strings.HasPrefix(viper.GetString("server_url"), "http://") && !strings.HasPrefix(viper.GetString("server_url"), "https://") {
+	if !strings.HasPrefix(viper.GetString("server_url"), "http://") &&
+		!strings.HasPrefix(viper.GetString("server_url"), "https://") {
 		errorText += "Fatal config error: server_url must start with https:// or http://\n"
 	}
 	if errorText != "" {
@@ -73,7 +77,35 @@ func LoadConfig(path string) error {
 	} else {
 		return nil
 	}
+}
 
+func GetDERPConfig() headscale.DERPConfig {
+	urlStrs := viper.GetStringSlice("derp.urls")
+
+	urls := make([]url.URL, len(urlStrs))
+	for index, urlStr := range urlStrs {
+		urlAddr, err := url.Parse(urlStr)
+		if err != nil {
+			log.Error().
+				Str("url", urlStr).
+				Err(err).
+				Msg("Failed to parse url, ignoring...")
+		}
+
+		urls[index] = *urlAddr
+	}
+
+	paths := viper.GetStringSlice("derp.paths")
+
+	autoUpdate := viper.GetBool("derp.auto_update_enabled")
+	updateFrequency := viper.GetDuration("derp.update_frequency")
+
+	return headscale.DERPConfig{
+		URLs:            urls,
+		Paths:           paths,
+		AutoUpdate:      autoUpdate,
+		UpdateFrequency: updateFrequency,
+	}
 }
 
 func GetDNSConfig() (*tailcfg.DNSConfig, string) {
@@ -171,33 +203,30 @@ func absPath(path string) string {
 }
 
 func getHeadscaleApp() (*headscale.Headscale, error) {
-	derpPath := absPath(viper.GetString("derp_map_path"))
-	derpMap, err := loadDerpMap(derpPath)
-	if err != nil {
-		log.Error().
-			Str("path", derpPath).
-			Err(err).
-			Msg("Could not load DERP servers map file")
-	}
-
 	// Minimum inactivity time out is keepalive timeout (60s) plus a few seconds
 	// to avoid races
 	minInactivityTimeout, _ := time.ParseDuration("65s")
 	if viper.GetDuration("ephemeral_node_inactivity_timeout") <= minInactivityTimeout {
-		err = fmt.Errorf("ephemeral_node_inactivity_timeout (%s) is set too low, must be more than %s\n", viper.GetString("ephemeral_node_inactivity_timeout"), minInactivityTimeout)
+		err := fmt.Errorf(
+			"ephemeral_node_inactivity_timeout (%s) is set too low, must be more than %s\n",
+			viper.GetString("ephemeral_node_inactivity_timeout"),
+			minInactivityTimeout,
+		)
 		return nil, err
 	}
 
 	dnsConfig, baseDomain := GetDNSConfig()
+	derpConfig := GetDERPConfig()
 
 	cfg := headscale.Config{
 		ServerURL:      viper.GetString("server_url"),
 		Addr:           viper.GetString("listen_addr"),
 		PrivateKeyPath: absPath(viper.GetString("private_key_path")),
-		DerpMap:        derpMap,
 		IPPrefix:       netaddr.MustParseIPPrefix(viper.GetString("ip_prefix")),
 		BaseDomain:     baseDomain,
 
+		DERP: derpConfig,
+
 		EphemeralNodeInactivityTimeout: viper.GetDuration("ephemeral_node_inactivity_timeout"),
 
 		DBtype: viper.GetString("db_type"),
@@ -243,21 +272,6 @@ func getHeadscaleApp() (*headscale.Headscale, error) {
 	return h, nil
 }
 
-func loadDerpMap(path string) (*tailcfg.DERPMap, error) {
-	derpFile, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer derpFile.Close()
-	var derpMap tailcfg.DERPMap
-	b, err := io.ReadAll(derpFile)
-	if err != nil {
-		return nil, err
-	}
-	err = yaml.Unmarshal(b, &derpMap)
-	return &derpMap, err
-}
-
 func JsonOutput(result interface{}, errResult error, outputFormat string) {
 	var j []byte
 	var err error

cmd/headscale/headscale_test.go

@@ -25,7 +25,6 @@ func (s *Suite) SetUpSuite(c *check.C) {
 }
 
 func (s *Suite) TearDownSuite(c *check.C) {
-
 }
 
 func (*Suite) TestPostgresConfigLoading(c *check.C) {
@@ -53,7 +52,6 @@ func (*Suite) TestPostgresConfigLoading(c *check.C) {
 	// Test that config file was interpreted correctly
 	c.Assert(viper.GetString("server_url"), check.Equals, "http://127.0.0.1:8080")
 	c.Assert(viper.GetString("listen_addr"), check.Equals, "0.0.0.0:8080")
-	c.Assert(viper.GetString("derp_map_path"), check.Equals, "derp.yaml")
 	c.Assert(viper.GetString("db_type"), check.Equals, "postgres")
 	c.Assert(viper.GetString("db_port"), check.Equals, "5432")
 	c.Assert(viper.GetString("tls_letsencrypt_hostname"), check.Equals, "")
@@ -86,7 +84,7 @@ func (*Suite) TestSqliteConfigLoading(c *check.C) {
 	// Test that config file was interpreted correctly
 	c.Assert(viper.GetString("server_url"), check.Equals, "http://127.0.0.1:8080")
 	c.Assert(viper.GetString("listen_addr"), check.Equals, "0.0.0.0:8080")
-	c.Assert(viper.GetString("derp_map_path"), check.Equals, "derp.yaml")
+	c.Assert(viper.GetStringSlice("derp.paths")[0], check.Equals, "derp-example.yaml")
 	c.Assert(viper.GetString("db_type"), check.Equals, "sqlite3")
 	c.Assert(viper.GetString("db_path"), check.Equals, "db.sqlite")
 	c.Assert(viper.GetString("tls_letsencrypt_hostname"), check.Equals, "")
@@ -128,7 +126,7 @@ func (*Suite) TestDNSConfigLoading(c *check.C) {
 func writeConfig(c *check.C, tmpDir string, configYaml []byte) {
 	// Populate a custom config file
 	configFile := filepath.Join(tmpDir, "config.yaml")
-	err := ioutil.WriteFile(configFile, configYaml, 0644)
+	err := ioutil.WriteFile(configFile, configYaml, 0o644)
 	if err != nil {
 		c.Fatalf("Couldn't write file %s", configFile)
 	}
@@ -139,10 +137,12 @@ func (*Suite) TestTLSConfigValidation(c *check.C) {
 	if err != nil {
 		c.Fatal(err)
 	}
-	//defer os.RemoveAll(tmpDir)
+	// defer os.RemoveAll(tmpDir)
 	fmt.Println(tmpDir)
 
-	configYaml := []byte("---\ntls_letsencrypt_hostname: \"example.com\"\ntls_letsencrypt_challenge_type: \"\"\ntls_cert_path: \"abc.pem\"")
+	configYaml := []byte(
+		"---\ntls_letsencrypt_hostname: \"example.com\"\ntls_letsencrypt_challenge_type: \"\"\ntls_cert_path: \"abc.pem\"",
+	)
 	writeConfig(c, tmpDir, configYaml)
 
 	// Check configuration validation errors (1)
@@ -150,13 +150,23 @@ func (*Suite) TestTLSConfigValidation(c *check.C) {
 	c.Assert(err, check.NotNil)
 	// check.Matches can not handle multiline strings
 	tmp := strings.ReplaceAll(err.Error(), "\n", "***")
-	c.Assert(tmp, check.Matches, ".*Fatal config error: set either tls_letsencrypt_hostname or tls_cert_path/tls_key_path, not both.*")
+	c.Assert(
-	c.Assert(tmp, check.Matches, ".*Fatal config error: the only supported values for tls_letsencrypt_challenge_type are.*")
+		tmp,
+		check.Matches,
+		".*Fatal config error: set either tls_letsencrypt_hostname or tls_cert_path/tls_key_path, not both.*",
+	)
+	c.Assert(
+		tmp,
+		check.Matches,
+		".*Fatal config error: the only supported values for tls_letsencrypt_challenge_type are.*",
+	)
 	c.Assert(tmp, check.Matches, ".*Fatal config error: server_url must start with https:// or http://.*")
 	fmt.Println(tmp)
 
 	// Check configuration validation errors (2)
-	configYaml = []byte("---\nserver_url: \"http://127.0.0.1:8080\"\ntls_letsencrypt_hostname: \"example.com\"\ntls_letsencrypt_challenge_type: \"TLS-ALPN-01\"")
+	configYaml = []byte(
+		"---\nserver_url: \"http://127.0.0.1:8080\"\ntls_letsencrypt_hostname: \"example.com\"\ntls_letsencrypt_challenge_type: \"TLS-ALPN-01\"",
+	)
 	writeConfig(c, tmpDir, configYaml)
 	err = cli.LoadConfig(tmpDir)
 	c.Assert(err, check.IsNil)

config.yaml.postgres.example

@@ -2,7 +2,6 @@
 server_url: http://127.0.0.1:8080
 listen_addr: 0.0.0.0:8080
 private_key_path: private.key
-derp_map_path: derp.yaml
 ephemeral_node_inactivity_timeout: 30m
 
 # Postgres config

config.yaml.sqlite.example

@@ -1,26 +1,43 @@
 ---
+log_level: info
 server_url: http://127.0.0.1:8080
 listen_addr: 0.0.0.0:8080
 private_key_path: private.key
-derp_map_path: derp.yaml
 ephemeral_node_inactivity_timeout: 30m
 
 # SQLite config (uncomment it if you want to use SQLite)
 db_type: sqlite3
 db_path: db.sqlite
 
+derp:
+  # List of externally available DERP maps encoded in JSON
+  urls:
+    - https://controlplane.tailscale.com/derpmap/default
+
+  # Locally available DERP map files encoded in YAML
+  paths:
+    - derp-example.yaml
+
+  # If enabled, a worker will be set up to periodically
+  # refresh the given sources and update the derpmap
+  # will be set up.
+  auto_update_enabled: true
+
+  # How often should we check for updates?
+  update_frequency: 24h
+
 acme_url: https://acme-v02.api.letsencrypt.org/directory
-acme_email: ''
+acme_email: ""
-tls_letsencrypt_hostname: ''
+tls_letsencrypt_hostname: ""
 tls_letsencrypt_listen: ":http"
 tls_letsencrypt_cache_dir: ".cache"
 tls_letsencrypt_challenge_type: HTTP-01
-tls_cert_path: ''
+tls_cert_path: ""
-tls_key_path: ''
+tls_key_path: ""
-acl_policy_path: ''
+acl_policy_path: ""
 dns_config:
   nameservers:
-  - 1.1.1.1
+    - 1.1.1.1
   domains: []
   magic_dns: true
   base_domain: example.com