add linux audio screensharing (#130)

Co-authored-by: V <vendicated@riseup.net>
Co-authored-by: kaitlynkitty <87152313+kaitlynkittyy@users.noreply.github.com>
Co-authored-by: Curve <fynnbwdt@gmail.com>

.env.example

@@ -4,3 +4,5 @@
 # all permissions at the defaults (public repos read only, 0 permissions):
 # https://github.com/settings/personal-access-tokens/new
 GITHUB_TOKEN=
+
+ELECTRON_LAUNCH_FLAGS="--ozone-platform-hint=auto --enable-webrtc-pipewire-capturer --enable-features=WaylandWindowDecorations"

package.json

@@ -25,6 +25,9 @@
     "dependencies": {
         "arrpc": "github:OpenAsar/arrpc#89f4da610ccfac93f461826a446a17cd3b23953d"
     },
+    "optionalDependencies": {
+        "@vencord/venmic": "^1.4.0"
+    },
     "devDependencies": {
         "@fal-works/esbuild-plugin-global-externals": "^2.1.2",
         "@types/node": "^20.8.4",

scripts/build/build.mts

@@ -5,6 +5,7 @@
  */
 
 import { BuildContext, BuildOptions, context } from "esbuild";
+import { copyFile } from "fs/promises";
 
 import vencordDep from "./vencordDep.mjs";
 
@@ -34,6 +35,11 @@ async function createContext(options: BuildOptions) {
 }
 
 await Promise.all([
+    process.platform === "linux" &&
+        copyFile(
+            "./node_modules/@vencord/venmic/prebuilds/venmic-addon-linux-x64/node-napi-v7.node",
+            "./static/dist/venmic.node"
+        ).catch(() => console.warn("Failed to copy venmic. Building without venmic support")),
     createContext({
         ...NodeCommonOpts,
         entryPoints: ["src/main/index.ts"],

scripts/start.ts

@@ -8,4 +8,4 @@ import "./utils/dotenv";
 
 import { spawnNodeModuleBin } from "./utils/spawn.mjs";
 
-spawnNodeModuleBin("electron", ["."]);
+spawnNodeModuleBin("electron", [".", ...(process.env.ELECTRON_LAUNCH_FLAGS?.split(" ") ?? [])]);

src/main/ipc.ts

@@ -4,6 +4,8 @@
  * Copyright (c) 2023 Vendicated and Vencord contributors
  */
 
+if (process.platform === "linux") import("./virtmic");
+
 import { execFile } from "child_process";
 import { app, dialog, RelaunchOptions, session, shell } from "electron";
 import { mkdirSync, readFileSync, watch } from "fs";

src/main/screenShare.ts

@@ -10,6 +10,9 @@ import { IpcEvents } from "shared/IpcEvents";
 
 import { handle } from "./utils/ipcWrappers";
 
+const isWayland =
+    process.platform === "linux" && (process.env.XDG_SESSION_TYPE === "wayland" || !!process.env.WAYLAND_DISPLAY);
+
 export function registerScreenShareHandler() {
     handle(IpcEvents.CAPTURER_GET_LARGE_THUMBNAIL, async (_, id: string) => {
         const sources = await desktopCapturer.getSources({
@@ -23,17 +26,19 @@ export function registerScreenShareHandler() {
     });
 
     session.defaultSession.setDisplayMediaRequestHandler(async (request, callback) => {
-        const sources = await desktopCapturer.getSources({
+        // request full resolution on wayland right away because we always only end up with one result anyway
+        const width = isWayland ? 1920 : 176;
+        const sources = await desktopCapturer
+            .getSources({
                 types: ["window", "screen"],
                 thumbnailSize: {
-                width: 176,
+                    width,
-                height: 99
+                    height: width * (9 / 16)
                 }
-        });
+            })
+            .catch(err => console.error("Error during screenshare picker", err));
 
-        const isWayland =
+        if (!sources) return callback({});
-            process.platform === "linux" &&
-            (process.env.XDG_SESSION_TYPE === "wayland" || !!process.env.WAYLAND_DISPLAY);
 
         const data = sources.map(({ id, name, thumbnail }) => ({
             id,
@@ -43,14 +48,26 @@ export function registerScreenShareHandler() {
 
         if (isWayland) {
             const video = data[0];
-            callback(video ? { video } : {});
+            if (video) {
+                const stream = await request.frame
+                    .executeJavaScript(
+                        `Vesktop.Components.ScreenShare.openScreenSharePicker(${JSON.stringify([video])},true)`
+                    )
+                    .catch(() => null);
+                if (stream === null) return callback({});
+            }
+
+            callback(video ? { video: sources[0] } : {});
             return;
         }
 
         const choice = await request.frame
             .executeJavaScript(`Vesktop.Components.ScreenShare.openScreenSharePicker(${JSON.stringify(data)})`)
             .then(e => e as StreamPick)
-            .catch(() => null);
+            .catch(e => {
+                console.error("Error during screenshare picker", e);
+                return null;
+            });
 
         if (!choice) return callback({});
 

src/main/virtmic.ts

@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: GPL-3.0
+ * Vesktop, a desktop app aiming to give you a snappier Discord Experience
+ * Copyright (c) 2023 Vendicated and Vencord contributors
+ */
+
+import { app, ipcMain } from "electron";
+import { join } from "path";
+import { IpcEvents } from "shared/IpcEvents";
+import { STATIC_DIR } from "shared/paths";
+
+let initialized = false;
+let patchBay: import("@vencord/venmic").PatchBay | undefined;
+
+function getRendererAudioServicePid() {
+    return (
+        app
+            .getAppMetrics()
+            .find(proc => proc.name === "Audio Service")
+            ?.pid?.toString() ?? "owo"
+    );
+}
+
+function obtainVenmic() {
+    if (!initialized) {
+        initialized = true;
+        try {
+            const { PatchBay } = require(join(STATIC_DIR, "dist/venmic.node")) as typeof import("@vencord/venmic");
+            patchBay = new PatchBay();
+        } catch (e) {
+            console.error("Failed to initialise venmic. Make sure you're using pipewire", e);
+        }
+    }
+
+    return patchBay;
+}
+
+ipcMain.handle(IpcEvents.VIRT_MIC_LIST, () => {
+    const audioPid = getRendererAudioServicePid();
+    return obtainVenmic()
+        ?.list()
+        .filter(s => s["application.process.id"] !== audioPid)
+        .map(s => s["application.name"]);
+});
+
+ipcMain.handle(
+    IpcEvents.VIRT_MIC_START,
+    (_, target: string) =>
+        obtainVenmic()?.link({
+            key: "application.name",
+            value: target,
+            mode: "include"
+        })
+);
+
+ipcMain.handle(
+    IpcEvents.VIRT_MIC_START_SYSTEM,
+    () =>
+        obtainVenmic()?.link({
+            key: "application.process.id",
+            value: getRendererAudioServicePid(),
+            mode: "exclude"
+        })
+);
+
+ipcMain.handle(IpcEvents.VIRT_MIC_STOP, () => obtainVenmic()?.unlink());

src/preload/VesktopNative.ts

@@ -59,6 +59,13 @@ export const VesktopNative = {
     capturer: {
         getLargeThumbnail: (id: string) => invoke<string>(IpcEvents.CAPTURER_GET_LARGE_THUMBNAIL, id)
     },
+    /** only available on Linux. */
+    virtmic: {
+        list: () => invoke<string[] | null>(IpcEvents.VIRT_MIC_LIST),
+        start: (target: string) => invoke<void>(IpcEvents.VIRT_MIC_START, target),
+        startSystem: () => invoke<void>(IpcEvents.VIRT_MIC_START_SYSTEM),
+        stop: () => invoke<void>(IpcEvents.VIRT_MIC_STOP)
+    },
     arrpc: {
         onActivity(cb: (data: string) => void) {
             ipcRenderer.on(IpcEvents.ARRPC_ACTIVITY, (_, data: string) => cb(data));

src/renderer/components/ScreenSharePicker.tsx

@@ -7,11 +7,21 @@
 import "./screenSharePicker.css";
 
 import { closeModal, Modals, openModal, useAwaiter } from "@vencord/types/utils";
-import { findStoreLazy } from "@vencord/types/webpack";
+import { findStoreLazy, onceReady } from "@vencord/types/webpack";
-import { Button, Card, Forms, Switch, Text, useState } from "@vencord/types/webpack/common";
+import {
+    Button,
+    Card,
+    FluxDispatcher,
+    Forms,
+    Select,
+    Switch,
+    Text,
+    UserStore,
+    useState
+} from "@vencord/types/webpack/common";
 import type { Dispatch, SetStateAction } from "react";
 import { addPatch } from "renderer/patches/shared";
-import { isWindows } from "renderer/utils";
+import { isLinux, isWindows } from "renderer/utils";
 
 const StreamResolutions = ["480", "720", "1080", "1440"] as const;
 const StreamFps = ["15", "30", "60"] as const;
@@ -25,6 +35,7 @@ interface StreamSettings {
     resolution: StreamResolution;
     fps: StreamFps;
     audio: boolean;
+    audioSource?: string;
 }
 
 export interface StreamPick extends StreamSettings {
@@ -70,18 +81,41 @@ addPatch({
     }
 });
 
-export function openScreenSharePicker(screens: Source[]) {
+if (isLinux) {
+    onceReady.then(() => {
+        FluxDispatcher.subscribe("VOICE_STATE_UPDATES", e => {
+            for (const state of e.voiceStates) {
+                if (state.userId === UserStore.getCurrentUser().id && state.oldChannelId && !state.channelId)
+                    VesktopNative.virtmic.stop();
+            }
+        });
+    });
+}
+
+export function openScreenSharePicker(screens: Source[], skipPicker: boolean) {
+    let didSubmit = false;
     return new Promise<StreamPick>((resolve, reject) => {
         const key = openModal(
             props => (
                 <ModalComponent
                     screens={screens}
                     modalProps={props}
-                    submit={resolve}
+                    submit={async v => {
+                        didSubmit = true;
+                        if (v.audioSource && v.audioSource !== "None") {
+                            if (v.audioSource === "Entire System") {
+                                await VesktopNative.virtmic.startSystem();
+                            } else {
+                                await VesktopNative.virtmic.start(v.audioSource);
+                            }
+                        }
+                        resolve(v);
+                    }}
                     close={() => {
                         props.onClose();
-                        reject("Aborted");
+                        if (!didSubmit) reject("Aborted");
                     }}
+                    skipPicker={skipPicker}
                 />
             ),
             {
@@ -112,16 +146,21 @@ function ScreenPicker({ screens, chooseScreen }: { screens: Source[]; chooseScre
 function StreamSettings({
     source,
     settings,
-    setSettings
+    setSettings,
+    skipPicker
 }: {
     source: Source;
     settings: StreamSettings;
     setSettings: Dispatch<SetStateAction<StreamSettings>>;
+    skipPicker: boolean;
 }) {
-    const [thumb] = useAwaiter(() => VesktopNative.capturer.getLargeThumbnail(source.id), {
+    const [thumb] = useAwaiter(
+        () => (skipPicker ? Promise.resolve(source.url) : VesktopNative.capturer.getLargeThumbnail(source.id)),
+        {
             fallbackValue: source.url,
             deps: [source.id]
-    });
+        }
+    );
 
     return (
         <div>
@@ -182,23 +221,65 @@ function StreamSettings({
                         Stream With Audio
                     </Switch>
                 )}
+
+                {isLinux && (
+                    <AudioSourcePickerLinux
+                        audioSource={settings.audioSource}
+                        setAudioSource={source => setSettings(s => ({ ...s, audioSource: source }))}
+                    />
+                )}
             </Card>
         </div>
     );
 }
 
+function AudioSourcePickerLinux({
+    audioSource,
+    setAudioSource
+}: {
+    audioSource?: string;
+    setAudioSource(s: string): void;
+}) {
+    const [sources, _, loading] = useAwaiter(() => VesktopNative.virtmic.list(), { fallbackValue: [] });
+    const allSources = sources ? ["None", "Entire System", ...sources] : null;
+
+    return (
+        <section>
+            <Forms.FormTitle>Audio</Forms.FormTitle>
+            {loading && <Forms.FormTitle>Loading Audio sources...</Forms.FormTitle>}
+            {allSources === null && (
+                <Forms.FormTitle>
+                    Failed to retrieve Audio Sources. If you would like to stream with Audio, make sure you're using
+                    Pipewire, not Pulseaudio
+                </Forms.FormTitle>
+            )}
+
+            {allSources && (
+                <Select
+                    options={allSources.map(s => ({ label: s, value: s, default: s === "None" }))}
+                    isSelected={s => s === audioSource}
+                    select={setAudioSource}
+                    serialize={String}
+                />
+            )}
+        </section>
+    );
+}
+
 function ModalComponent({
     screens,
     modalProps,
     submit,
-    close
+    close,
+    skipPicker
 }: {
     screens: Source[];
     modalProps: any;
     submit: (data: StreamPick) => void;
     close: () => void;
+    skipPicker: boolean;
 }) {
-    const [selected, setSelected] = useState<string>();
+    const [selected, setSelected] = useState<string | undefined>(skipPicker ? screens[0].id : void 0);
     const [settings, setSettings] = useState<StreamSettings>({
         resolution: "1080",
         fps: "60",
@@ -220,6 +301,7 @@ function ModalComponent({
                         source={screens.find(s => s.id === selected)!}
                         settings={settings}
                         setSettings={setSettings}
+                        skipPicker={skipPicker}
                     />
                 )}
             </Modals.ModalContent>
@@ -259,7 +341,7 @@ function ModalComponent({
                     Go Live
                 </Button>
 
-                {selected ? (
+                {selected && !skipPicker ? (
                     <Button color={Button.Colors.TRANSPARENT} onClick={() => setSelected(void 0)}>
                         Back
                     </Button>

src/renderer/patches/index.ts

@@ -8,3 +8,4 @@
 import "./spellCheck";
 import "./platformClass";
 import "./windowsTitleBar";
+import "./screenShareAudio";

src/renderer/patches/screenShareAudio.ts

@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: GPL-3.0
+ * Vesktop, a desktop app aiming to give you a snappier Discord Experience
+ * Copyright (c) 2023 Vendicated and Vencord contributors
+ */
+
+import { isLinux } from "renderer/utils";
+
+if (isLinux) {
+    const original = navigator.mediaDevices.getDisplayMedia;
+
+    async function getVirtmic() {
+        try {
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            const audioDevice = devices.find(({ label }) => label === "vencord-screen-share");
+            return audioDevice?.deviceId;
+        } catch (error) {
+            return null;
+        }
+    }
+
+    navigator.mediaDevices.getDisplayMedia = async function (opts) {
+        const stream = await original.call(this, opts);
+        const id = await getVirtmic();
+
+        if (id) {
+            const audio = await navigator.mediaDevices.getUserMedia({
+                audio: {
+                    deviceId: {
+                        exact: id
+                    },
+                    autoGainControl: false,
+                    echoCancellation: false,
+                    noiseSuppression: false
+                }
+            });
+            audio.getAudioTracks().forEach(t => stream.addTrack(t));
+        }
+
+        return stream;
+    };
+}

src/renderer/utils.ts

@@ -17,3 +17,4 @@ const { platform } = navigator;
 
 export const isWindows = platform.startsWith("Win");
 export const isMac = platform.startsWith("Mac");
+export const isLinux = platform.startsWith("Linux");

src/shared/IpcEvents.ts

@@ -42,5 +42,10 @@ export const enum IpcEvents {
     ENABLE_AUTOSTART = "VCD_ENABLE_AUTOSTART",
     DISABLE_AUTOSTART = "VCD_DISABLE_AUTOSTART",
 
+    VIRT_MIC_LIST = "VCD_VIRT_MIC_LIST",
+    VIRT_MIC_START = "VCD_VIRT_MIC_START",
+    VIRT_MIC_START_SYSTEM = "VCD_VIRT_MIC_START_ALL",
+    VIRT_MIC_STOP = "VCD_VIRT_MIC_STOP",
+
     ARRPC_ACTIVITY = "VCD_ARRPC_ACTIVITY"
 }

static/dist/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

static/views/about.html

@@ -51,6 +51,10 @@
                 <a href="https://github.com/OpenAsar/arrpc" target="_blank">arrpc</a>
                 - An open implementation of Discord's Rich Presence server
             </li>
+            <li>
+                <a href="https://github.com/Soundux/rohrkabel" target="_blank">rohrkabel</a>
+                - A C++ RAII Pipewire-API Wrapper 
+            </li>
             <li>
                 And many
                 <a href="https://github.com/Vencord/Vesktop/blob/main/pnpm-lock.yaml" target="_blank"