From 9bc6ac0f35ca7e9087289899940818f04f6401e5 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Sun, 6 Nov 2022 20:22:21 +0100 Subject: [PATCH] Make TLS setup work automatically This commit injects the per-test-generated tls certs into the tailscale container and makes sure all can ping all. It does not test any of the DERP isolation yet. Signed-off-by: Kristoffer Dalby --- integration/hsic/hsic.go | 236 ++++++++++++++++++++-------- integration/integrationutil/util.go | 77 +++++++++ integration/scenario.go | 36 ++--- integration/tsic/tsic.go | 110 +++++++++++-- 4 files changed, 363 insertions(+), 96 deletions(-) create mode 100644 integration/integrationutil/util.go diff --git a/integration/hsic/hsic.go b/integration/hsic/hsic.go index 5e500d59..e8e8cfbf 100644 --- a/integration/hsic/hsic.go +++ b/integration/hsic/hsic.go @@ -1,52 +1,83 @@ package hsic import ( - "archive/tar" "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "errors" "fmt" - "io" "log" + "math/big" + "net" "net/http" - "path/filepath" + "time" "github.com/juanfont/headscale" v1 "github.com/juanfont/headscale/gen/go/headscale/v1" "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" - "github.com/ory/dockertest/v3/docker" ) const ( - hsicHashLength = 6 - dockerContextPath = "../." - aclPolicyPath = "/etc/headscale/acl.hujson" + hsicHashLength = 6 + dockerContextPath = "../." + aclPolicyPath = "/etc/headscale/acl.hujson" + tlsCertPath = "/etc/headscale/tls.cert" + tlsKeyPath = "/etc/headscale/tls.key" + headscaleDefaultPort = 8080 ) var errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok") type HeadscaleInContainer struct { hostname string - port int pool *dockertest.Pool container *dockertest.Resource network *dockertest.Network // optional config + port int aclPolicy *headscale.ACLPolicy env []string + tlsCert []byte + tlsKey []byte } type Option = func(c *HeadscaleInContainer) func WithACLPolicy(acl *headscale.ACLPolicy) Option { return func(hsic *HeadscaleInContainer) { + // TODO(kradalby): Move somewhere appropriate + hsic.env = append(hsic.env, fmt.Sprintf("HEADSCALE_ACL_POLICY_PATH=%s", aclPolicyPath)) + hsic.aclPolicy = acl } } +func WithTLS() Option { + return func(hsic *HeadscaleInContainer) { + cert, key, err := createCertificate() + if err != nil { + log.Fatalf("failed to create certificates for headscale test: %s", err) + } + + // TODO(kradalby): Move somewhere appropriate + hsic.env = append(hsic.env, fmt.Sprintf("HEADSCALE_TLS_CERT_PATH=%s", tlsCertPath)) + hsic.env = append(hsic.env, fmt.Sprintf("HEADSCALE_TLS_KEY_PATH=%s", tlsKeyPath)) + hsic.env = append(hsic.env, "HEADSCALE_TLS_CLIENT_AUTH_MODE=disabled") + + hsic.tlsCert = cert + hsic.tlsKey = key + } +} + func WithConfigEnv(configEnv map[string]string) Option { return func(hsic *HeadscaleInContainer) { env := []string{} @@ -59,9 +90,14 @@ func WithConfigEnv(configEnv map[string]string) Option { } } +func WithPort(port int) Option { + return func(hsic *HeadscaleInContainer) { + hsic.port = port + } +} + func New( pool *dockertest.Pool, - port int, network *dockertest.Network, opts ...Option, ) (*HeadscaleInContainer, error) { @@ -71,11 +107,10 @@ func New( } hostname := fmt.Sprintf("hs-%s", hash) - portProto := fmt.Sprintf("%d/tcp", port) hsic := &HeadscaleInContainer{ hostname: hostname, - port: port, + port: headscaleDefaultPort, pool: pool, network: network, @@ -85,9 +120,7 @@ func New( opt(hsic) } - if hsic.aclPolicy != nil { - hsic.env = append(hsic.env, fmt.Sprintf("HEADSCALE_ACL_POLICY_PATH=%s", aclPolicyPath)) - } + portProto := fmt.Sprintf("%d/tcp", hsic.port) headscaleBuildOptions := &dockertest.BuildOptions{ Dockerfile: "Dockerfile.debug", @@ -144,9 +177,25 @@ func New( } } + if hsic.hasTLS() { + err = hsic.WriteFile(tlsCertPath, hsic.tlsCert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + + err = hsic.WriteFile(tlsKeyPath, hsic.tlsKey) + if err != nil { + return nil, fmt.Errorf("failed to write TLS key to container: %w", err) + } + } + return hsic, nil } +func (t *HeadscaleInContainer) hasTLS() bool { + return len(t.tlsCert) != 0 && len(t.tlsKey) != 0 +} + func (t *HeadscaleInContainer) Shutdown() error { return t.pool.Purge(t.container) } @@ -183,11 +232,7 @@ func (t *HeadscaleInContainer) GetPort() string { } func (t *HeadscaleInContainer) GetHealthEndpoint() string { - hostEndpoint := fmt.Sprintf("%s:%d", - t.GetIP(), - t.port) - - return fmt.Sprintf("http://%s/health", hostEndpoint) + return fmt.Sprintf("%s/health", t.GetEndpoint()) } func (t *HeadscaleInContainer) GetEndpoint() string { @@ -195,17 +240,39 @@ func (t *HeadscaleInContainer) GetEndpoint() string { t.GetIP(), t.port) + if t.hasTLS() { + return fmt.Sprintf("https://%s", hostEndpoint) + } + return fmt.Sprintf("http://%s", hostEndpoint) } +func (t *HeadscaleInContainer) GetCert() []byte { + return t.tlsCert +} + +func (t *HeadscaleInContainer) GetHostname() string { + return t.hostname +} + func (t *HeadscaleInContainer) WaitForReady() error { url := t.GetHealthEndpoint() log.Printf("waiting for headscale to be ready at %s", url) + client := &http.Client{} + + if t.hasTLS() { + insecureTransport := http.DefaultTransport.(*http.Transport).Clone() + insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + client = &http.Client{Transport: insecureTransport} + } + return t.pool.Retry(func() error { - resp, err := http.Get(url) //nolint + resp, err := client.Get(url) //nolint if err != nil { + log.Printf("ready err: %s", err) + return fmt.Errorf("headscale is not ready: %w", err) } @@ -292,55 +359,96 @@ func (t *HeadscaleInContainer) ListMachinesInNamespace( } func (t *HeadscaleInContainer) WriteFile(path string, data []byte) error { - dirPath, fileName := filepath.Split(path) + return integrationutil.WriteFileToContainer(t.pool, t.container, path, data) +} - file := bytes.NewReader(data) +func createCertificate() ([]byte, []byte, error) { + // From: + // https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ - buf := bytes.NewBuffer([]byte{}) - - tarWriter := tar.NewWriter(buf) - - header := &tar.Header{ - Name: fileName, - Size: file.Size(), - // Mode: int64(stat.Mode()), - // ModTime: stat.ModTime(), - } - - err := tarWriter.WriteHeader(header) - if err != nil { - return fmt.Errorf("failed write file header to tar: %w", err) - } - - _, err = io.Copy(tarWriter, file) - if err != nil { - return fmt.Errorf("failed to copy file to tar: %w", err) - } - - err = tarWriter.Close() - if err != nil { - return fmt.Errorf("failed to close tar: %w", err) - } - - log.Printf("tar: %s", buf.String()) - - // Ensure the directory is present inside the container - _, err = t.Execute([]string{"mkdir", "-p", dirPath}) - if err != nil { - return fmt.Errorf("failed to ensure directory: %w", err) - } - - err = t.pool.Client.UploadToContainer( - t.container.Container.ID, - docker.UploadToContainerOptions{ - NoOverwriteDirNonDir: false, - Path: dirPath, - InputStream: bytes.NewReader(buf.Bytes()), + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"Headscale testing INC"}, + Country: []string{"NL"}, + Locality: []string{"Leiden"}, }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(30 * time.Minute), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + }, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + // caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) + // if err != nil { + // return nil, err + // } + + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1658), + Subject: pkix.Name{ + Organization: []string{"Headscale testing INC"}, + Country: []string{"NL"}, + Locality: []string{"Leiden"}, + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(30 * time.Minute), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + certBytes, err := x509.CreateCertificate( + rand.Reader, + cert, + ca, + &certPrivKey.PublicKey, + caPrivKey, ) if err != nil { - return err + return nil, nil, err } - return nil + certPEM := new(bytes.Buffer) + + err = pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + if err != nil { + return nil, nil, err + } + + certPrivKeyPEM := new(bytes.Buffer) + + err = pem.Encode(certPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + if err != nil { + return nil, nil, err + } + + // serverCert, err := tls.X509KeyPair(certPEM.Bytes(), certPrivKeyPEM.Bytes()) + // if err != nil { + // return nil, err + // } + + return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil } diff --git a/integration/integrationutil/util.go b/integration/integrationutil/util.go new file mode 100644 index 00000000..a0fefaa5 --- /dev/null +++ b/integration/integrationutil/util.go @@ -0,0 +1,77 @@ +package integrationutil + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "log" + "path/filepath" + + "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" +) + +func WriteFileToContainer( + pool *dockertest.Pool, + container *dockertest.Resource, + path string, + data []byte, +) error { + dirPath, fileName := filepath.Split(path) + + file := bytes.NewReader(data) + + buf := bytes.NewBuffer([]byte{}) + + tarWriter := tar.NewWriter(buf) + + header := &tar.Header{ + Name: fileName, + Size: file.Size(), + // Mode: int64(stat.Mode()), + // ModTime: stat.ModTime(), + } + + err := tarWriter.WriteHeader(header) + if err != nil { + return fmt.Errorf("failed write file header to tar: %w", err) + } + + _, err = io.Copy(tarWriter, file) + if err != nil { + return fmt.Errorf("failed to copy file to tar: %w", err) + } + + err = tarWriter.Close() + if err != nil { + return fmt.Errorf("failed to close tar: %w", err) + } + + log.Printf("tar: %s", buf.String()) + + // Ensure the directory is present inside the container + _, _, err = dockertestutil.ExecuteCommand( + container, + []string{"mkdir", "-p", dirPath}, + []string{}, + ) + if err != nil { + return fmt.Errorf("failed to ensure directory: %w", err) + } + + err = pool.Client.UploadToContainer( + container.Container.ID, + docker.UploadToContainerOptions{ + NoOverwriteDirNonDir: false, + Path: dirPath, + InputStream: bytes.NewReader(buf.Bytes()), + }, + ) + if err != nil { + return err + } + + return nil +} diff --git a/integration/scenario.go b/integration/scenario.go index 8ebcadb8..714c0584 100644 --- a/integration/scenario.go +++ b/integration/scenario.go @@ -150,20 +150,8 @@ func (s *Scenario) Namespaces() []string { // Note: These functions assume that there is a _single_ headscale instance for now // TODO(kradalby): make port and headscale configurable, multiple instances support? -func (s *Scenario) StartHeadscale() error { - headscale, err := hsic.New(s.pool, headscalePort, s.network, - hsic.WithACLPolicy( - &headscale.ACLPolicy{ - ACLs: []headscale.ACL{ - { - Action: "accept", - Sources: []string{"*"}, - Destinations: []string{"*:*"}, - }, - }, - }, - ), - ) +func (s *Scenario) StartHeadscale(opts ...hsic.Option) error { + headscale, err := hsic.New(s.pool, s.network, opts...) if err != nil { return fmt.Errorf("failed to create headscale container: %w", err) } @@ -228,10 +216,22 @@ func (s *Scenario) CreateTailscaleNodesInNamespace( defer namespace.createWaitGroup.Done() // TODO(kradalby): error handle this - tsClient, err := tsic.New(s.pool, version, s.network) + tsClient, err := tsic.New( + s.pool, + version, + s.network, + tsic.WithHeadscaleTLS(s.Headscale().GetCert()), + tsic.WithHeadscaleName(s.Headscale().GetHostname()), + ) if err != nil { // return fmt.Errorf("failed to add tailscale node: %w", err) - log.Printf("failed to add tailscale node: %s", err) + log.Printf("failed to create tailscale node: %s", err) + } + + err = tsClient.WaitForReady() + if err != nil { + // return fmt.Errorf("failed to add tailscale node: %w", err) + log.Printf("failed to wait for tailscaled: %s", err) } namespace.Clients[tsClient.Hostname()] = tsClient @@ -306,8 +306,8 @@ func (s *Scenario) WaitForTailscaleSync() error { // CreateHeadscaleEnv is a conventient method returning a set up Headcale // test environment with nodes of all versions, joined to the server with X // namespaces. -func (s *Scenario) CreateHeadscaleEnv(namespaces map[string]int) error { - err := s.StartHeadscale() +func (s *Scenario) CreateHeadscaleEnv(namespaces map[string]int, opts ...hsic.Option) error { + err := s.StartHeadscale(opts...) if err != nil { return err } diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 930eb301..712ad34d 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -12,6 +12,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/juanfont/headscale" "github.com/juanfont/headscale/integration/dockertestutil" + "github.com/juanfont/headscale/integration/integrationutil" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "tailscale.com/ipn/ipnstate" @@ -20,6 +21,7 @@ import ( const ( tsicHashLength = 6 dockerContextPath = "../." + headscaleCertPath = "/usr/local/share/ca-certificates/headscale.crt" ) var ( @@ -41,12 +43,51 @@ type TailscaleInContainer struct { // "cache" ips []netip.Addr fqdn string + + // optional config + headscaleCert []byte + headscaleHostname string +} + +type Option = func(c *TailscaleInContainer) + +func WithHeadscaleTLS(cert []byte) Option { + return func(tsic *TailscaleInContainer) { + tsic.headscaleCert = cert + } +} + +func WithOrCreateNetwork(network *dockertest.Network) Option { + return func(tsic *TailscaleInContainer) { + if network != nil { + tsic.network = network + + return + } + + network, err := dockertestutil.GetFirstOrCreateNetwork( + tsic.pool, + fmt.Sprintf("%s-network", tsic.hostname), + ) + if err != nil { + log.Fatalf("failed to create network: %s", err) + } + + tsic.network = network + } +} + +func WithHeadscaleName(hsName string) Option { + return func(tsic *TailscaleInContainer) { + tsic.headscaleHostname = hsName + } } func New( pool *dockertest.Pool, version string, network *dockertest.Network, + opts ...Option, ) (*TailscaleInContainer, error) { hash, err := headscale.GenerateRandomStringDNSSafe(tsicHashLength) if err != nil { @@ -55,20 +96,38 @@ func New( hostname := fmt.Sprintf("ts-%s-%s", strings.ReplaceAll(version, ".", "-"), hash) - // TODO(kradalby): figure out why we need to "refresh" the network here. - // network, err = dockertestutil.GetFirstOrCreateNetwork(pool, network.Network.Name) - // if err != nil { - // return nil, err - // } + tsic := &TailscaleInContainer{ + version: version, + hostname: hostname, + + pool: pool, + network: network, + } + + for _, opt := range opts { + opt(tsic) + } tailscaleOptions := &dockertest.RunOptions{ Name: hostname, Networks: []*dockertest.Network{network}, - Cmd: []string{ - "tailscaled", "--tun=tsdev", + // Cmd: []string{ + // "tailscaled", "--tun=tsdev", + // }, + Entrypoint: []string{ + "/bin/bash", + "-c", + "/bin/sleep 3 ; update-ca-certificates ; tailscaled --tun=tsdev", }, } + if tsic.headscaleHostname != "" { + tailscaleOptions.ExtraHosts = []string{ + "host.docker.internal:host-gateway", + fmt.Sprintf("%s:host-gateway", tsic.headscaleHostname), + } + } + // dockertest isnt very good at handling containers that has already // been created, this is an attempt to make sure this container isnt // present. @@ -89,14 +148,20 @@ func New( } log.Printf("Created %s container\n", hostname) - return &TailscaleInContainer{ - version: version, - hostname: hostname, + tsic.container = container - pool: pool, - container: container, - network: network, - }, nil + if tsic.hasTLS() { + err = tsic.WriteFile(headscaleCertPath, tsic.headscaleCert) + if err != nil { + return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err) + } + } + + return tsic, nil +} + +func (t *TailscaleInContainer) hasTLS() bool { + return len(t.headscaleCert) != 0 } func (t *TailscaleInContainer) Shutdown() error { @@ -111,6 +176,19 @@ func (t *TailscaleInContainer) Version() string { return t.version } +func (t *TailscaleInContainer) WaitForReady() error { + return t.pool.Retry(func() error { + // If tailscaled has not started yet, this will return a non-zero + // status code + _, err := t.Execute([]string{"tailscale", "status"}) + if err != nil { + return err + } + + return nil + }) +} + func (t *TailscaleInContainer) Execute( command []string, ) (string, string, error) { @@ -318,6 +396,10 @@ func (t *TailscaleInContainer) Ping(hostnameOrIP string) error { }) } +func (t *TailscaleInContainer) WriteFile(path string, data []byte) error { + return integrationutil.WriteFileToContainer(t.pool, t.container, path, data) +} + func createTailscaleBuildOptions(version string) *dockertest.BuildOptions { var tailscaleBuildOptions *dockertest.BuildOptions switch version {