diff --git a/integration/ssh_test.go b/integration/ssh_test.go
index d480f8df..57d85980 100644
--- a/integration/ssh_test.go
+++ b/integration/ssh_test.go
@@ -28,6 +28,13 @@ var retry = func(times int, sleepInterval time.Duration,
 		if err == nil {
 			return result, stderr, nil
 		}
+
+		// If we get a permission denied error, we can fail immediately
+		// since that is something we wont recover from by retrying.
+		if err != nil && strings.Contains(stderr, "Permission denied (tailscale)") {
+			return result, stderr, err
+		}
+
 		time.Sleep(sleepInterval)
 	}