overhaul & improve Linux screenshare (#489)

Fixes fps/resolution not properly being applied
Enables hardware encoding via vaapi
Redesigns stream picker modal 

Co-authored-by: kaitlynkittyy <kaitlynyaadev@kaitlynyaa.dev>
Co-authored-by: Oleh Polisan <polisanoleg@gmail.com>
Co-authored-by: Vendicated <vendicated@riseup.net>

src/main/index.ts

@@ -27,14 +27,18 @@ process.env.VENCORD_USER_DATA_DIR = DATA_DIR;
 function init() {
     const { disableSmoothScroll, hardwareAcceleration } = Settings.store;
 
-    if (hardwareAcceleration === false) app.disableHardwareAcceleration();
+    if (hardwareAcceleration === false) {
+        app.disableHardwareAcceleration();
+    } else {
+        app.commandLine.appendSwitch("enable-features", "VaapiVideoDecodeLinuxGL,VaapiVideoEncoder,VaapiVideoDecoder");
+    }
+
     if (disableSmoothScroll) {
         app.commandLine.appendSwitch("disable-smooth-scrolling");
     }
 
     // work around chrome 66 disabling autoplay by default
     app.commandLine.appendSwitch("autoplay-policy", "no-user-gesture-required");
-
     // WinRetrieveSuggestionsOnlyOnDemand: Work around electron 13 bug w/ async spellchecking on Windows.
     // HardwareMediaKeyHandling,MediaSessionService: Prevent Discord from registering as a media service.
     //

src/renderer/components/ScreenSharePicker.tsx

@@ -6,7 +6,7 @@
 
 import "./screenSharePicker.css";
 
-import { closeModal, Margins, Modals, ModalSize, openModal, useAwaiter } from "@vencord/types/utils";
+import { closeModal, Logger, Margins, Modals, ModalSize, openModal, useAwaiter } from "@vencord/types/utils";
 import { findStoreLazy, onceReady } from "@vencord/types/webpack";
 import {
     Button,
@@ -36,6 +36,7 @@ interface StreamSettings {
     fps: StreamFps;
     audio: boolean;
     audioSource?: string;
+    contentHint?: string;
     workaround?: boolean;
     onlyDefaultSpeakers?: boolean;
 }
@@ -50,7 +51,9 @@ interface Source {
     url: string;
 }
 
-let currentSettings: StreamSettings | null = null;
+export let currentSettings: StreamSettings | null = null;
+
+const logger = new Logger("VesktopScreenShare");
 
 addPatch({
     patches: [
@@ -60,6 +63,20 @@ addPatch({
                 match: /this.localWant=/,
                 replace: "$self.patchStreamQuality(this);$&"
             }
+        },
+        {
+            find: "x-google-max-bitrate",
+            replacement: [
+                {
+                    // eslint-disable-next-line no-useless-escape
+                    match: /"x-google-max-bitrate=".concat\(\i\)/,
+                    replace: '"x-google-max-bitrate=".concat("80_000")'
+                },
+                {
+                    match: /;level-asymmetry-allowed=1/,
+                    replace: ";b=AS:800000;level-asymmetry-allowed=1"
+                }
+            ]
         }
     ],
     patchStreamQuality(opts: any) {
@@ -74,6 +91,14 @@ addPatch({
             bitrateMax: 8000000,
             bitrateTarget: 600000
         });
+        if (opts?.encode) {
+            Object.assign(opts.encode, {
+                framerate,
+                width,
+                height,
+                pixelCount: height * width
+            });
+        }
         Object.assign(opts.capture, {
             framerate,
             width,
@@ -219,6 +244,47 @@ function StreamSettings({
                             </div>
                         </section>
                     </div>
+                    <div className="vcd-screen-picker-quality">
+                        <section>
+                            <Forms.FormTitle>Content Type</Forms.FormTitle>
+                            <div>
+                                <div className="vcd-screen-picker-radios">
+                                    <label
+                                        className="vcd-screen-picker-radio"
+                                        data-checked={settings.contentHint === "motion"}
+                                    >
+                                        <Text variant="text-sm/bold">Prefer Smoothness</Text>
+                                        <input
+                                            type="radio"
+                                            name="contenthint"
+                                            value="motion"
+                                            checked={settings.contentHint === "motion"}
+                                            onChange={() => setSettings(s => ({ ...s, contentHint: "motion" }))}
+                                        />
+                                    </label>
+                                    <label
+                                        className="vcd-screen-picker-radio"
+                                        data-checked={settings.contentHint === "detail"}
+                                    >
+                                        <Text variant="text-sm/bold">Prefer Clarity</Text>
+                                        <input
+                                            type="radio"
+                                            name="contenthint"
+                                            value="detail"
+                                            checked={settings.contentHint === "detail"}
+                                            onChange={() => setSettings(s => ({ ...s, contentHint: "detail" }))}
+                                        />
+                                    </label>
+                                </div>
+                                <div className="vcd-screen-picker-hint-description">
+                                    <p>
+                                        Choosing "Prefer Clarity" will result in a significantly lower framerate in
+                                        exchange for a much sharper and clearer image.
+                                    </p>
+                                </div>
+                            </div>
+                        </section>
+                    </div>
                 </Card>
             </div>
 
@@ -358,6 +424,7 @@ function ModalComponent({
     const [settings, setSettings] = useState<StreamSettings>({
         resolution: "1080",
         fps: "60",
+        contentHint: "motion",
         audio: true
     });
 
@@ -367,7 +434,6 @@ function ModalComponent({
                 <Forms.FormTitle tag="h2">ScreenShare</Forms.FormTitle>
                 <Modals.ModalCloseButton onClick={close} />
             </Modals.ModalHeader>
-
             <Modals.ModalContent className="vcd-screen-picker-modal">
                 {!selected ? (
                     <ScreenPicker screens={screens} chooseScreen={setSelected} />
@@ -380,29 +446,24 @@ function ModalComponent({
                     />
                 )}
             </Modals.ModalContent>
-
             <Modals.ModalFooter className="vcd-screen-picker-footer">
                 <Button
                     disabled={!selected}
                     onClick={() => {
                         currentSettings = settings;
-
+                        try {
-                        // If there are 2 connections, the second one is the existing stream.
+                            const frameRate = Number(settings.fps);
-                        // In that case, we patch its quality
-                        const conn = [...MediaEngineStore.getMediaEngine().connections][1];
-                        if (conn && conn.videoStreamParameters.length > 0) {
                             const height = Number(settings.resolution);
                             const width = Math.round(height * (16 / 9));
-                            Object.assign(conn.videoStreamParameters[0], {
+
-                                maxFrameRate: Number(settings.fps),
+                            const conn = [...MediaEngineStore.getMediaEngine().connections].find(
-                                maxPixelCount: width * height,
+                                connection => connection.streamUserId === UserStore.getCurrentUser().id
-                                maxBitrate: 8000000,
+                            );
-                                maxResolution: {
+
-                                    type: "fixed",
+                            if (conn) {
-                                    width,
+                                conn.videoStreamParameters[0].maxFrameRate = frameRate;
-                                    height
+                                conn.videoStreamParameters[0].maxResolution.height = height;
-                                }
+                                conn.videoStreamParameters[0].maxResolution.width = width;
-                            });
                             }
 
                             submit({
@@ -410,6 +471,38 @@ function ModalComponent({
                                 ...settings
                             });
 
+                            setTimeout(async () => {
+                                const conn = [...MediaEngineStore.getMediaEngine().connections].find(
+                                    connection => connection.streamUserId === UserStore.getCurrentUser().id
+                                );
+                                if (!conn) return;
+
+                                const track = conn.input.stream.getVideoTracks()[0];
+
+                                const constraints = {
+                                    ...track.getConstraints(),
+                                    frameRate,
+                                    width: { min: 640, ideal: width, max: width },
+                                    height: { min: 480, ideal: height, max: height },
+                                    advanced: [{ width: width, height: height }],
+                                    resizeMode: "none"
+                                };
+
+                                try {
+                                    await track.applyConstraints(constraints);
+
+                                    logger.info(
+                                        "Applied constraints successfully. New constraints:",
+                                        track.getConstraints()
+                                    );
+                                } catch (e) {
+                                    logger.error("Failed to apply constraints.", e);
+                                }
+                            }, 100);
+                        } catch (error) {
+                            logger.error("Error while submitting stream.", error);
+                        }
+
                         close();
                     }}
                 >

src/renderer/components/screenSharePicker.css

@@ -17,6 +17,15 @@
     grid-template-columns: 1fr 1fr;
 }
 
+.vcd-screen-picker-settings-grid > div {
+    display: flex;
+    flex-direction: column;
+}
+
+.vcd-screen-picker-card {
+    flex-grow: 1;
+}
+
 .vcd-screen-picker-grid {
     display: grid;
     grid-template-columns: 1fr 1fr;
@@ -128,3 +137,10 @@
 .vcd-screen-picker-audio {
     margin-bottom: 0;
 }
+
+.vcd-screen-picker-hint-description {
+    color: var(--header-secondary);
+    font-size: 14px;
+    line-height: 20px;
+    font-weight: 400;
+}

src/renderer/patches/index.ts

@@ -7,6 +7,6 @@
 // TODO: Possibly auto generate glob if we have more patches in the future
 import "./enableNotificationsByDefault";
 import "./platformClass";
-import "./screenShareAudio";
+import "./screenShareFixes";
 import "./spellCheck";
 import "./windowsTitleBar";

src/renderer/patches/screenShareFixes.ts

@@ -4,8 +4,12 @@
  * Copyright (c) 2023 Vendicated and Vencord contributors
  */
 
+import { Logger } from "@vencord/types/utils";
+import { currentSettings } from "renderer/components/ScreenSharePicker";
 import { isLinux } from "renderer/utils";
 
+const logger = new Logger("VesktopStreamFixes");
+
 if (isLinux) {
     const original = navigator.mediaDevices.getDisplayMedia;
 
@@ -23,6 +27,29 @@ if (isLinux) {
         const stream = await original.call(this, opts);
         const id = await getVirtmic();
 
+        const frameRate = Number(currentSettings?.fps);
+        const height = Number(currentSettings?.resolution);
+        const width = Math.round(height * (16 / 9));
+        const track = stream.getVideoTracks()[0];
+
+        track.contentHint = String(currentSettings?.contentHint);
+
+        const constraints = {
+            ...track.getConstraints(),
+            frameRate,
+            width: { min: 640, ideal: width, max: width },
+            height: { min: 480, ideal: height, max: height },
+            advanced: [{ width: width, height: height }],
+            resizeMode: "none"
+        };
+
+        track
+            .applyConstraints(constraints)
+            .then(() => {
+                logger.info("Applied constraints successfully. New constraints: ", track.getConstraints());
+            })
+            .catch(e => logger.error("Failed to apply constraints.", e));
+
         if (id) {
             const audio = await navigator.mediaDevices.getUserMedia({
                 audio: {