Improve generated integration tests

- Save logs from control(headscale) on every run to tmp
- Upgrade nix-actions
- Cancel builds if new commit is pushed
- Fix a sorting bug in user command test

Signed-off-by: Kristoffer Dalby <kristoffer@tailscale.com>

.github/workflows/build.yml

@@ -35,7 +35,7 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: nix build
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
           name: headscale-linux

.github/workflows/test-integration-v2-TestAuthKeyLogoutAndRelogin.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestAuthKeyLogoutAndRelogin
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestAuthKeyLogoutAndRelogin$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestAuthWebFlowAuthenticationPingAll.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestAuthWebFlowAuthenticationPingAll
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestAuthWebFlowAuthenticationPingAll$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestAuthWebFlowLogoutAndRelogin.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestAuthWebFlowLogoutAndRelogin
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestAuthWebFlowLogoutAndRelogin$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestCreateTailscale.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestCreateTailscale
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestCreateTailscale$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestEnablingRoutes.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestEnablingRoutes
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestEnablingRoutes$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestHeadscale.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestHeadscale
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestHeadscale$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestOIDCAuthenticationPingAll.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestOIDCAuthenticationPingAll
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestOIDCAuthenticationPingAll$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestOIDCExpireNodes.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestOIDCExpireNodes
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestOIDCExpireNodes$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestPingAllByHostname.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestPingAllByHostname
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestPingAllByHostname$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestPingAllByIP.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestPingAllByIP
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestPingAllByIP$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestPreAuthKeyCommand.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestPreAuthKeyCommand
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestPreAuthKeyCommand$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestPreAuthKeyCommandReusableEphemeral.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestPreAuthKeyCommandReusableEphemeral
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestPreAuthKeyCommandReusableEphemeral$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestPreAuthKeyCommandWithoutExpiry.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestPreAuthKeyCommandWithoutExpiry
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestPreAuthKeyCommandWithoutExpiry$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestResolveMagicDNS.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestResolveMagicDNS
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestResolveMagicDNS$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestSSHIsBlockedInACL.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestSSHIsBlockedInACL
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestSSHIsBlockedInACL$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestSSHMultipleUsersAllToAll.yaml

@@ -2,10 +2,14 @@
 # DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
 # To regenerate, run "go generate" in cmd/gh-action-integration-generator/
 
-name: Integration Test v2 - TestSSNamespaceOnlyIsolation
+name: Integration Test v2 - TestSSHMultipleUsersAllToAll
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,10 +42,17 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
                   -failfast \
                   -timeout 120m \
                   -parallel 1 \
-                  -run "^TestSSNamespaceOnlyIsolation$"
+                  -run "^TestSSHMultipleUsersAllToAll$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestSSHNoSSHConfigured.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestSSHNoSSHConfigured
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestSSHNoSSHConfigured$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestSSHOneUserAllToAll.yaml

@@ -2,10 +2,14 @@
 # DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
 # To regenerate, run "go generate" in cmd/gh-action-integration-generator/
 
-name: Integration Test v2 - TestSSHMultipleNamespacesAllToAll
+name: Integration Test v2 - TestSSHOneUserAllToAll
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,10 +42,17 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
                   -failfast \
                   -timeout 120m \
                   -parallel 1 \
-                  -run "^TestSSHMultipleNamespacesAllToAll$"
+                  -run "^TestSSHOneUserAllToAll$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestSSUserOnlyIsolation.yaml

@@ -2,10 +2,14 @@
 # DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
 # To regenerate, run "go generate" in cmd/gh-action-integration-generator/
 
-name: Integration Test v2 - TestSSHOneNamespaceAllToAll
+name: Integration Test v2 - TestSSUserOnlyIsolation
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,10 +42,17 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
                   -failfast \
                   -timeout 120m \
                   -parallel 1 \
-                  -run "^TestSSHOneNamespaceAllToAll$"
+                  -run "^TestSSUserOnlyIsolation$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestTaildrop.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestTaildrop
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestTaildrop$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestTailscaleNodesJoiningHeadcale.yaml

@@ -6,6 +6,10 @@ name: Integration Test v2 - TestTailscaleNodesJoiningHeadcale
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,6 +42,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -45,3 +50,9 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^TestTailscaleNodesJoiningHeadcale$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.github/workflows/test-integration-v2-TestUserCommand.yaml

@@ -2,10 +2,14 @@
 # DO NOT EDIT, generated with cmd/gh-action-integration-generator/main.go
 # To regenerate, run "go generate" in cmd/gh-action-integration-generator/
 
-name: Integration Test v2 - TestNamespaceCommand
+name: Integration Test v2 - TestUserCommand
 
 on: [pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-$${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -26,8 +30,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: ${{ env.ACT }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -38,10 +42,17 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
                   -failfast \
                   -timeout 120m \
                   -parallel 1 \
-                  -run "^TestNamespaceCommand$"
+                  -run "^TestUserCommand$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"

.gitignore

@@ -26,7 +26,8 @@ derp.yaml
 # Exclude Jetbrains Editors
 .idea
 
-test_output/ 
+test_output/
+control_logs/
 
 # Nix build output
 result

cmd/gh-action-integration-generator/main.go

@@ -20,6 +20,10 @@ name: Integration Test v2 - {{.Name}}
 
 on: [pull_request]
 
+concurrency:
+  group: {{ "${{ github.workflow }}-$${{ github.head_ref || github.run_id }}" }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -40,8 +44,8 @@ jobs:
             integration_test/
             config-example.yaml
 
-      - uses: cachix/install-nix-action@v16
-        if: steps.changed-files.outputs.any_changed == 'true'
+      - uses: cachix/install-nix-action@v18
+        if: {{ "${{ env.ACT }}" }} || steps.changed-files.outputs.any_changed == 'true'
 
       - name: Run general integration tests
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -52,6 +56,7 @@ jobs:
               --name headscale-test-suite \
               --volume $PWD:$PWD -w $PWD/integration \
               --volume /var/run/docker.sock:/var/run/docker.sock \
+              --volume $PWD/control_logs:/tmp/control \
               golang:1 \
                 go test ./... \
                   -tags ts2019 \
@@ -59,6 +64,12 @@ jobs:
                   -timeout 120m \
                   -parallel 1 \
                   -run "^{{.Name}}$"
+
+      - uses: actions/upload-artifact@v3
+        if: always() && steps.changed-files.outputs.any_changed == 'true'
+        with:
+          name: logs
+          path: "control_logs/*.log"
 `))
 )
 

flake.nix

@@ -6,157 +6,163 @@
     flake-utils.url = "github:numtide/flake-utils";
   };
 
-  outputs = {
-    self,
-    nixpkgs,
-    flake-utils,
-    ...
-  }: let
-    headscaleVersion =
-      if (self ? shortRev)
-      then self.shortRev
-      else "dev";
-  in
+  outputs =
+    { self
+    , nixpkgs
+    , flake-utils
+    , ...
+    }:
+    let
+      headscaleVersion =
+        if (self ? shortRev)
+        then self.shortRev
+        else "dev";
+    in
     {
-      overlay = _: prev: let
-        pkgs = nixpkgs.legacyPackages.${prev.system};
-      in rec {
-        headscale = pkgs.buildGo119Module rec {
-          pname = "headscale";
-          version = headscaleVersion;
-          src = pkgs.lib.cleanSource self;
+      overlay = _: prev:
+        let
+          pkgs = nixpkgs.legacyPackages.${prev.system};
+        in
+        rec {
+          headscale = pkgs.buildGo119Module rec {
+            pname = "headscale";
+            version = headscaleVersion;
+            src = pkgs.lib.cleanSource self;
 
-          tags = ["ts2019"];
+            tags = [ "ts2019" ];
 
-          # Only run unit tests when testing a build
-          checkFlags = ["-short"];
+            # Only run unit tests when testing a build
+            checkFlags = [ "-short" ];
 
-          # When updating go.mod or go.sum, a new sha will need to be calculated,
-          # update this if you have a mismatch after doing a change to thos files.
-          vendorSha256 = "sha256-8p5NFxXKaZPsW4B6NMzfi0pqfVroIahSgA0fukvB3JI=";
+            # When updating go.mod or go.sum, a new sha will need to be calculated,
+            # update this if you have a mismatch after doing a change to thos files.
+            vendorSha256 = "sha256-8p5NFxXKaZPsW4B6NMzfi0pqfVroIahSgA0fukvB3JI=";
 
-          ldflags = ["-s" "-w" "-X github.com/juanfont/headscale/cmd/headscale/cli.Version=v${version}"];
-        };
-
-        golines = pkgs.buildGoModule rec {
-          pname = "golines";
-          version = "0.11.0";
-
-          src = pkgs.fetchFromGitHub {
-            owner = "segmentio";
-            repo = "golines";
-            rev = "v${version}";
-            sha256 = "sha256-2K9KAg8iSubiTbujyFGN3yggrL+EDyeUCs9OOta/19A=";
+            ldflags = [ "-s" "-w" "-X github.com/juanfont/headscale/cmd/headscale/cli.Version=v${version}" ];
           };
 
-          vendorSha256 = "sha256-rxYuzn4ezAxaeDhxd8qdOzt+CKYIh03A9zKNdzILq18=";
+          golines = pkgs.buildGoModule rec {
+            pname = "golines";
+            version = "0.11.0";
 
-          nativeBuildInputs = [pkgs.installShellFiles];
-        };
+            src = pkgs.fetchFromGitHub {
+              owner = "segmentio";
+              repo = "golines";
+              rev = "v${version}";
+              sha256 = "sha256-2K9KAg8iSubiTbujyFGN3yggrL+EDyeUCs9OOta/19A=";
+            };
 
-        golangci-lint = prev.golangci-lint.override {
-          # Override https://github.com/NixOS/nixpkgs/pull/166801 which changed this
-          # to buildGo118Module because it does not build on Darwin.
-          inherit (prev) buildGoModule;
-        };
+            vendorSha256 = "sha256-rxYuzn4ezAxaeDhxd8qdOzt+CKYIh03A9zKNdzILq18=";
 
-        protoc-gen-grpc-gateway = pkgs.buildGoModule rec {
-          pname = "grpc-gateway";
-          version = "2.14.0";
-
-          src = pkgs.fetchFromGitHub {
-            owner = "grpc-ecosystem";
-            repo = "grpc-gateway";
-            rev = "v${version}";
-            sha256 = "sha256-lnNdsDCpeSHtl2lC1IhUw11t3cnGF+37qSM7HDvKLls=";
+            nativeBuildInputs = [ pkgs.installShellFiles ];
           };
 
-          vendorSha256 = "sha256-dGdnDuRbwg8fU7uB5GaHEWa/zI3w06onqjturvooJQA=";
+          golangci-lint = prev.golangci-lint.override {
+            # Override https://github.com/NixOS/nixpkgs/pull/166801 which changed this
+            # to buildGo118Module because it does not build on Darwin.
+            inherit (prev) buildGoModule;
+          };
 
-          nativeBuildInputs = [pkgs.installShellFiles];
+          protoc-gen-grpc-gateway = pkgs.buildGoModule rec {
+            pname = "grpc-gateway";
+            version = "2.14.0";
 
-          subPackages = ["protoc-gen-grpc-gateway" "protoc-gen-openapiv2"];
+            src = pkgs.fetchFromGitHub {
+              owner = "grpc-ecosystem";
+              repo = "grpc-gateway";
+              rev = "v${version}";
+              sha256 = "sha256-lnNdsDCpeSHtl2lC1IhUw11t3cnGF+37qSM7HDvKLls=";
+            };
+
+            vendorSha256 = "sha256-dGdnDuRbwg8fU7uB5GaHEWa/zI3w06onqjturvooJQA=";
+
+            nativeBuildInputs = [ pkgs.installShellFiles ];
+
+            subPackages = [ "protoc-gen-grpc-gateway" "protoc-gen-openapiv2" ];
+          };
         };
-      };
     }
     // flake-utils.lib.eachDefaultSystem
-    (system: let
-      pkgs = import nixpkgs {
-        overlays = [self.overlay];
-        inherit system;
-      };
-      buildDeps = with pkgs; [git go_1_19 gnumake];
-      devDeps = with pkgs;
-        buildDeps
-        ++ [
-          golangci-lint
-          golines
-          nodePackages.prettier
-          goreleaser
+      (system:
+      let
+        pkgs = import nixpkgs {
+          overlays = [ self.overlay ];
+          inherit system;
+        };
+        buildDeps = with pkgs; [ git go_1_19 gnumake ];
+        devDeps = with pkgs;
+          buildDeps
+          ++ [
+            golangci-lint
+            golines
+            nodePackages.prettier
+            goreleaser
+            gotestsum
 
-          # Protobuf dependencies
-          protobuf
-          protoc-gen-go
-          protoc-gen-go-grpc
-          protoc-gen-grpc-gateway
-          buf
-          clang-tools # clang-format
-        ];
+            # Protobuf dependencies
+            protobuf
+            protoc-gen-go
+            protoc-gen-go-grpc
+            protoc-gen-grpc-gateway
+            buf
+            clang-tools # clang-format
+          ];
 
-      # Add entry to build a docker image with headscale
-      # caveat: only works on Linux
-      #
-      # Usage:
-      # nix build .#headscale-docker
-      # docker load < result
-      headscale-docker = pkgs.dockerTools.buildLayeredImage {
-        name = "headscale";
-        tag = headscaleVersion;
-        contents = [pkgs.headscale];
-        config.Entrypoint = [(pkgs.headscale + "/bin/headscale")];
-      };
-    in rec {
-      # `nix develop`
-      devShell = pkgs.mkShell {
-        buildInputs = devDeps;
+        # Add entry to build a docker image with headscale
+        # caveat: only works on Linux
+        #
+        # Usage:
+        # nix build .#headscale-docker
+        # docker load < result
+        headscale-docker = pkgs.dockerTools.buildLayeredImage {
+          name = "headscale";
+          tag = headscaleVersion;
+          contents = [ pkgs.headscale ];
+          config.Entrypoint = [ (pkgs.headscale + "/bin/headscale") ];
+        };
+      in
+      rec {
+        # `nix develop`
+        devShell = pkgs.mkShell {
+          buildInputs = devDeps;
 
-        shellHook = ''
-          export GOFLAGS=-tags="ts2019"
-        '';
-      };
-
-      # `nix build`
-      packages = with pkgs; {
-        inherit headscale;
-        inherit headscale-docker;
-      };
-      defaultPackage = pkgs.headscale;
-
-      # `nix run`
-      apps.headscale = flake-utils.lib.mkApp {
-        drv = packages.headscale;
-      };
-      apps.default = apps.headscale;
-
-      checks = {
-        format =
-          pkgs.runCommand "check-format"
-          {
-            buildInputs = with pkgs; [
-              gnumake
-              nixpkgs-fmt
-              golangci-lint
-              nodePackages.prettier
-              golines
-              clang-tools
-            ];
-          } ''
-            ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt ${./.}
-            ${pkgs.golangci-lint}/bin/golangci-lint run --fix --timeout 10m
-            ${pkgs.nodePackages.prettier}/bin/prettier --write '**/**.{ts,js,md,yaml,yml,sass,css,scss,html}'
-            ${pkgs.golines}/bin/golines --max-len=88 --base-formatter=gofumpt -w ${./.}
-            ${pkgs.clang-tools}/bin/clang-format -style="{BasedOnStyle: Google, IndentWidth: 4, AlignConsecutiveDeclarations: true, AlignConsecutiveAssignments: true, ColumnLimit: 0}" -i ${./.}
+          shellHook = ''
+            export GOFLAGS=-tags="ts2019"
           '';
-      };
-    });
+        };
+
+        # `nix build`
+        packages = with pkgs; {
+          inherit headscale;
+          inherit headscale-docker;
+        };
+        defaultPackage = pkgs.headscale;
+
+        # `nix run`
+        apps.headscale = flake-utils.lib.mkApp {
+          drv = packages.headscale;
+        };
+        apps.default = apps.headscale;
+
+        checks = {
+          format =
+            pkgs.runCommand "check-format"
+              {
+                buildInputs = with pkgs; [
+                  gnumake
+                  nixpkgs-fmt
+                  golangci-lint
+                  nodePackages.prettier
+                  golines
+                  clang-tools
+                ];
+              } ''
+              ${pkgs.nixpkgs-fmt}/bin/nixpkgs-fmt ${./.}
+              ${pkgs.golangci-lint}/bin/golangci-lint run --fix --timeout 10m
+              ${pkgs.nodePackages.prettier}/bin/prettier --write '**/**.{ts,js,md,yaml,yml,sass,css,scss,html}'
+              ${pkgs.golines}/bin/golines --max-len=88 --base-formatter=gofumpt -w ${./.}
+              ${pkgs.clang-tools}/bin/clang-format -style="{BasedOnStyle: Google, IndentWidth: 4, AlignConsecutiveDeclarations: true, AlignConsecutiveAssignments: true, ColumnLimit: 0}" -i ${./.}
+            '';
+        };
+      });
 }

integration/cli_test.go

@@ -99,7 +99,7 @@ func TestUserCommand(t *testing.T) {
 
 	assert.Equal(
 		t,
-		[]string{"user1", "newname"},
+		[]string{"newname", "user1"},
 		result,
 	)
 

integration/control.go

@@ -6,6 +6,7 @@ import (
 
 type ControlServer interface {
 	Shutdown() error
+	SaveLog(string) error
 	Execute(command []string) (string, error)
 	GetHealthEndpoint() string
 	GetEndpoint() string

integration/dockertestutil/logs.go

@@ -0,0 +1,68 @@
+package dockertestutil
+
+import (
+	"bytes"
+	"context"
+	"log"
+	"os"
+	"path"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+const filePerm = 0o644
+
+func SaveLog(
+	pool *dockertest.Pool,
+	resource *dockertest.Resource,
+	basePath string,
+) error {
+	err := os.MkdirAll(basePath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	err = pool.Client.Logs(
+		docker.LogsOptions{
+			Context:      context.TODO(),
+			Container:    resource.Container.ID,
+			OutputStream: &stdout,
+			ErrorStream:  &stderr,
+			Tail:         "all",
+			RawTerminal:  false,
+			Stdout:       true,
+			Stderr:       true,
+			Follow:       false,
+			Timestamps:   false,
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Saving logs for %s to %s\n", resource.Container.Name, basePath)
+
+	err = os.WriteFile(
+		path.Join(basePath, resource.Container.Name+".stdout.log"),
+		stdout.Bytes(),
+		filePerm,
+	)
+	if err != nil {
+		return err
+	}
+
+	err = os.WriteFile(
+		path.Join(basePath, resource.Container.Name+".stderr.log"),
+		stderr.Bytes(),
+		filePerm,
+	)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

integration/hsic/hsic.go

@@ -248,6 +248,10 @@ func (t *HeadscaleInContainer) Shutdown() error {
 	return t.pool.Purge(t.container)
 }
 
+func (t *HeadscaleInContainer) SaveLog(path string) error {
+	return dockertestutil.SaveLog(t.pool, t.container, path)
+}
+
 func (t *HeadscaleInContainer) Execute(
 	command []string,
 ) (string, error) {

integration/scenario.go

@@ -126,7 +126,15 @@ func NewScenario() (*Scenario, error) {
 
 func (s *Scenario) Shutdown() error {
 	s.controlServers.Range(func(_ string, control ControlServer) bool {
-		err := control.Shutdown()
+		err := control.SaveLog("/tmp/control")
+		if err != nil {
+			log.Printf(
+				"Failed to save log from control: %s",
+				fmt.Errorf("failed to save log from control: %w", err),
+			)
+		}
+
+		err = control.Shutdown()
 		if err != nil {
 			log.Printf(
 				"Failed to shut down control: %s",