Repository navigation
feat(linux): auto-sample X11 cursor telemetry when portal metadata is missing #1416
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
0e7af55
fcb97b1
85cdf37
e256831
27dbb14
5fd254a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| import fs from "node:fs"; | ||
| import { runInNewContext } from "node:vm"; | ||
| import { describe, expect, it, vi } from "vitest"; | ||
|
|
||
| const source = fs.readFileSync(new URL("./handlers.ts", import.meta.url), "utf8"); | ||
|
|
||
| // Execute the small registration in isolation: importing handlers.ts requires | ||
| // the complete Electron runtime, but a source-only assertion misses broken cleanup. | ||
| function fixture() { | ||
| const start = source.indexOf('ipcMain.handle("discard-cursor-telemetry"'); | ||
| expect(start).toBeGreaterThan(-1); | ||
| const end = source.indexOf("\n\t});", start) + 7; | ||
| const handle = vi.fn(); | ||
| const context = { | ||
| ipcMain: { handle }, | ||
| cursorTelemetryRecordingId: 7 as number | null, | ||
| pendingCursorRecordingData: { samples: [1] } as unknown, | ||
| stopCursorRecording: vi.fn(async () => undefined), | ||
| }; | ||
| runInNewContext(source.slice(start, end).replace("recordingId: number", "recordingId"), context); | ||
| return { | ||
| context, | ||
| discard: handle.mock.calls[0][1] as (event: unknown, id: number) => Promise<void>, | ||
| }; | ||
| } | ||
|
|
||
| describe("discard cursor telemetry IPC", () => { | ||
| it("registers the preload channel and clears only the matching take after stopping", async () => { | ||
| const { context, discard } = fixture(); | ||
| await discard({}, 7); | ||
| expect(context.stopCursorRecording).toHaveBeenCalledTimes(1); | ||
| expect(context.pendingCursorRecordingData).toBeNull(); | ||
| expect(context.cursorTelemetryRecordingId).toBeNull(); | ||
| }); | ||
| it("does not clear a newer take if stopping the old sampler yields", async () => { | ||
| const { context, discard } = fixture(); | ||
| context.stopCursorRecording.mockImplementation(async () => { | ||
| context.cursorTelemetryRecordingId = 8; | ||
| context.pendingCursorRecordingData = { samples: [2] }; | ||
| }); | ||
| await discard({}, 7); | ||
| expect(context.cursorTelemetryRecordingId).toBe(8); | ||
| expect(context.pendingCursorRecordingData).toEqual({ samples: [2] }); | ||
| }); | ||
|
|
||
| it.each([6, NaN, Infinity])("ignores stale or invalid discard id %s", async (id) => { | ||
| const { context, discard } = fixture(); | ||
| await discard({}, id); | ||
| expect(context.stopCursorRecording).not.toHaveBeenCalled(); | ||
| expect(context.pendingCursorRecordingData).not.toBeNull(); | ||
| expect(context.cursorTelemetryRecordingId).toBe(7); | ||
| }); | ||
| it("compacts and bounds the browser track before publishing its sidecar", () => { | ||
| const body = source.slice(source.indexOf("async function storeRecordedSessionFiles")); | ||
| expect(body.indexOf("finalizeBrowserCursorTelemetry(")).toBeLessThan( | ||
| body.indexOf("await writePendingCursorTelemetry(screenVideoPath)"), | ||
| ); | ||
| expect(body).toContain("payload.cursorPauseRanges"); | ||
| expect(body).toContain("payload.durationMs"); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -111,14 +111,22 @@ import { | |
| type LinuxCaptureSourceKind, | ||
| LinuxNativeCaptureSession, | ||
| } from "../native-bridge/capture/linuxNativeCaptureSession"; | ||
| import { compactCursorTelemetryPauseRanges } from "../native-bridge/cursor/recording/cursorPauseRanges"; | ||
| import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; | ||
| import { | ||
| compactCursorTelemetryPauseRanges, | ||
| finalizeBrowserCursorTelemetry, | ||
| } from "../native-bridge/cursor/recording/cursorPauseRanges"; | ||
| import { | ||
| createCursorRecordingSession, | ||
| createX11PointerRecordingSession, | ||
| isX11PointerFallbackEnabled, | ||
| } from "../native-bridge/cursor/recording/factory"; | ||
| import { | ||
| isMacCursorHelperUnavailable, | ||
| requestMacCursorAccessibilityAccess, | ||
| } from "../native-bridge/cursor/recording/macNativeCursorRecordingSession"; | ||
| import { findLinuxCaptureHelperPath } from "../native-bridge/cursor/recording/pipeWireCursorRecordingSession"; | ||
| import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; | ||
| import { resolveX11CaptureBounds } from "../native-bridge/cursor/recording/x11CaptureBounds"; | ||
| import { toHelperRect } from "../native-bridge/helperCoordinates"; | ||
| import { | ||
| isMacPickerSourceId, | ||
|
|
@@ -798,6 +806,7 @@ const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz | |
|
|
||
| let cursorRecordingSession: CursorRecordingSession | null = null; | ||
| let pendingCursorRecordingData: CursorRecordingData | null = null; | ||
| let cursorTelemetryRecordingId: number | null = null; | ||
| let nativeWindowsCaptureProcess: ChildProcessWithoutNullStreams | null = null; | ||
| let nativeWindowsCaptureOutput = ""; | ||
| let nativeWindowsCaptureTargetPath: string | null = null; | ||
|
|
@@ -1179,6 +1188,25 @@ function isMacWindowCaptureSourceId(sourceId: string | null | undefined) { | |
| ); | ||
| } | ||
|
|
||
| // Electron pointer positions and display bounds are DIP; helper origins may | ||
| // instead be X11 pixels. Resolve conservatively, never mix those coordinate spaces. | ||
| function getX11CaptureBounds() { | ||
| const displays = screen.getAllDisplays(); | ||
| if (linuxNativeCaptureSession) { | ||
| return resolveX11CaptureBounds({ | ||
| displays, | ||
| sourceKind: linuxNativeCaptureSession.grantedSourceKind, | ||
| origin: linuxNativeCaptureSession.grantedCaptureOrigin, | ||
| size: linuxNativeCaptureSession.grantedCaptureSize, | ||
| }); | ||
| } | ||
| // The browser path has an explicit desktopCapturer selection, not a portal | ||
| // grant. Window geometry cannot be inferred from a monitor id. | ||
| const source = selectedSource; | ||
| if (!source?.id?.startsWith("screen:") || !source.display_id) return null; | ||
| return displays.find((display) => String(display.id) === source.display_id)?.bounds ?? null; | ||
| } | ||
|
|
||
| function getSelectedSourceBounds() { | ||
| // A window take is normalised against the rect the helper is recording. That | ||
| // rect is seeded from the source frame before the first cursor sample and | ||
|
|
@@ -1374,13 +1402,44 @@ async function resolveDirectShowWebcamClsid(deviceName?: string) { | |
| } | ||
|
|
||
| async function startCursorRecording(recordingId?: number) { | ||
| cursorTelemetryRecordingId = recordingId ?? null; | ||
| // On Linux the native capture helper already produces cursor samples, from | ||
| // the SAME portal session that produces the pixels. Spawning the cursor-only | ||
| // helper alongside it would open a SECOND portal session — and since | ||
| // PipeWire helper alongside it would open a SECOND portal session — and since | ||
| // SelectSources may be called once per session, that means a second picker | ||
| // in front of the user, for a stream nothing consumes. The double prompt is | ||
| // precisely what the native path exists to remove. | ||
| // | ||
| // X11 is the exception: the portal often has no METADATA cursor mode at all, | ||
| // so the helper records pixels with an empty cursor track. Sampling the | ||
| // pointer through X11 (Electron / xinput) does not talk to the portal, | ||
| // so it can run in parallel and fill `<video>.cursor.json` when native samples | ||
| // come back empty. | ||
| if (linuxNativeCaptureSession) { | ||
| if (!isX11PointerFallbackEnabled()) { | ||
| return; | ||
| } | ||
| if (cursorRecordingSession) { | ||
| pendingCursorRecordingData = await cursorRecordingSession.stop(); | ||
| cursorRecordingSession = null; | ||
| } | ||
| pendingCursorRecordingData = null; | ||
| // Clock origin is sampler start, which is after waitUntilCapturing() — | ||
| // i.e. at or shortly after the first encoded frame. Passing recordingId | ||
| // here used to pin t=0 to prepare time (picker + countdown), so every | ||
| // X11 sample landed late relative to the video. | ||
| cursorRecordingSession = createX11PointerRecordingSession({ | ||
| getDisplayBounds: getX11CaptureBounds, | ||
| maxSamples: MAX_CURSOR_SAMPLES, | ||
| sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, | ||
| }); | ||
| try { | ||
| await cursorRecordingSession.start(); | ||
| console.info("[cursor-x11] sampling the pointer alongside native Linux capture"); | ||
| } catch (error) { | ||
| console.error("Failed to start the X11 pointer sampler:", error); | ||
| cursorRecordingSession = null; | ||
| } | ||
| return; | ||
| } | ||
|
|
||
|
|
@@ -1391,7 +1450,7 @@ async function startCursorRecording(recordingId?: number) { | |
|
|
||
| pendingCursorRecordingData = null; | ||
| cursorRecordingSession = createCursorRecordingSession({ | ||
| getDisplayBounds: getSelectedSourceBounds, | ||
| getDisplayBounds: process.platform === "linux" ? getX11CaptureBounds : getSelectedSourceBounds, | ||
| maxSamples: MAX_CURSOR_SAMPLES, | ||
| platform: process.platform, | ||
| sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, | ||
|
|
@@ -1409,17 +1468,16 @@ async function startCursorRecording(recordingId?: number) { | |
| } | ||
|
|
||
| async function stopCursorRecording() { | ||
| if (!cursorRecordingSession) { | ||
| return; | ||
| } | ||
|
|
||
| const session = cursorRecordingSession; | ||
| if (!session) return; | ||
| try { | ||
| pendingCursorRecordingData = await cursorRecordingSession.stop(); | ||
| const data = await session.stop(); | ||
| if (cursorRecordingSession === session) pendingCursorRecordingData = data; | ||
| } catch (error) { | ||
| console.error("Failed to stop cursor recording session:", error); | ||
| pendingCursorRecordingData = null; | ||
| if (cursorRecordingSession === session) pendingCursorRecordingData = null; | ||
| } finally { | ||
| cursorRecordingSession = null; | ||
| if (cursorRecordingSession === session) cursorRecordingSession = null; | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -1429,6 +1487,7 @@ async function writePendingCursorTelemetry(videoPath: string) { | |
| await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8"); | ||
| } | ||
| pendingCursorRecordingData = null; | ||
| cursorTelemetryRecordingId = null; | ||
| } | ||
|
|
||
| // P4 — proactively seeds the media-links registry for a fresh recording so | ||
|
|
@@ -2957,6 +3016,10 @@ export function registerIpcHandlers( | |
| linuxNativeCaptureSourceLabel = linuxSourceLabel(session.grantedSourceKind); | ||
| setAppRecording(true, linuxNativeCaptureSourceLabel, recordingId); | ||
|
|
||
| if (cursorCaptureMode === "editable-overlay") { | ||
| await startCursorRecording(recordingId); | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| } | ||
|
|
||
| return { success: true, recordingId, path: outputPath }; | ||
| } catch (error) { | ||
| console.error("Failed to start native Linux recording:", error); | ||
|
|
@@ -3003,6 +3066,8 @@ export function registerIpcHandlers( | |
|
|
||
| try { | ||
| if (discard) { | ||
| await stopCursorRecording(); | ||
| pendingCursorRecordingData = null; | ||
| session.discard(); | ||
| const discarded = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`); | ||
| await Promise.all([ | ||
|
|
@@ -3040,15 +3105,30 @@ export function registerIpcHandlers( | |
| ? { screenVideoPath: result.path, message: result.warning } | ||
| : null; | ||
|
|
||
| // The helper collects cursor samples itself, from the same portal | ||
| // session that produced the pixels, so there is no separate sampler | ||
| // to stop and no clock offset to correct — the two are one recording. | ||
| if (cursorCaptureMode === "editable-overlay" && result.cursor.samples.length > 0) { | ||
| await fs.writeFile( | ||
| `${result.path}.cursor.json`, | ||
| JSON.stringify(result.cursor, null, 2), | ||
| "utf-8", | ||
| ); | ||
| if (cursorCaptureMode === "editable-overlay") { | ||
| await stopCursorRecording(); | ||
|
Comment on lines
+3108
to
+3109
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Exclude fallback samples collected during helper shutdown. The sampler continues while Stop sampling at the capture-stop boundary, or filter the compacted fallback samples against
|
||
| } else { | ||
| pendingCursorRecordingData = null; | ||
| } | ||
|
|
||
| // Prefer the helper's own samples — they share the portal session that | ||
| // produced the pixels, so there is no clock offset to correct. When | ||
| // the portal had no METADATA cursor mode the track is empty; the X11 | ||
| // sampler started alongside the take then fills the sidecar so | ||
| // `--auto-zoom` still has clicks to work from. Those fallback samples | ||
| // still include paused time, which the video does not. | ||
| const cursor = | ||
| result.cursor.samples.length > 0 | ||
| ? result.cursor | ||
| : pendingCursorRecordingData | ||
| ? compactCursorTelemetryPauseRanges( | ||
| pendingCursorRecordingData, | ||
| result.cursorPauseRanges, | ||
|
Comment on lines
+3124
to
+3126
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Align fallback sample times before applying native pause ranges. The helper measures pause ranges from its For example, with a 200 ms startup gap and a native pause range of 1000–2000 ms, a click 50 ms after resume has sampler time 1850 ms. This compaction deletes that valid click. Use the capture-start timestamp for the sampler, or rebase fallback samples to that timestamp before compaction. Preserve the zero-offset case.
|
||
| ) | ||
| : null; | ||
| pendingCursorRecordingData = null; | ||
| if (cursorCaptureMode === "editable-overlay" && cursor && cursor.samples.length > 0) { | ||
| await fs.writeFile(`${result.path}.cursor.json`, JSON.stringify(cursor, null, 2), "utf-8"); | ||
| } | ||
|
|
||
| const session_: RecordingSession = { | ||
|
|
@@ -3086,6 +3166,8 @@ export function registerIpcHandlers( | |
| console.error("Failed to stop native Linux recording:", error); | ||
| return { success: false, error: String(error) }; | ||
| } finally { | ||
| await stopCursorRecording(); | ||
| pendingCursorRecordingData = null; | ||
| linuxNativeCaptureSession = null; | ||
| linuxNativeCaptureRecordingId = null; | ||
| linuxNativeCaptureCursorMode = "editable-overlay"; | ||
|
|
@@ -4347,6 +4429,13 @@ export function registerIpcHandlers( | |
| // fresh-take auto-zoom reads that file the moment it imports -- an empty read there | ||
| // is indistinguishable from a take with no click, so the zooms are silently | ||
| // skipped. | ||
| if (pendingCursorRecordingData) { | ||
| pendingCursorRecordingData = finalizeBrowserCursorTelemetry( | ||
| pendingCursorRecordingData, | ||
| payload.cursorPauseRanges, | ||
| payload.durationMs, | ||
| ); | ||
| } | ||
| await writePendingCursorTelemetry(screenVideoPath); | ||
| setCurrentRecordingSessionState(session); | ||
| currentProjectPath = null; | ||
|
|
@@ -4442,6 +4531,15 @@ export function registerIpcHandlers( | |
| }, | ||
| ); | ||
|
|
||
| ipcMain.handle("discard-cursor-telemetry", async (_, recordingId: number) => { | ||
| // An old take's asynchronous finalizer must not discard a newer take. | ||
| if (!Number.isFinite(recordingId) || cursorTelemetryRecordingId !== recordingId) return; | ||
| await stopCursorRecording(); | ||
| if (cursorTelemetryRecordingId !== recordingId) return; | ||
| pendingCursorRecordingData = null; | ||
| cursorTelemetryRecordingId = null; | ||
| }); | ||
|
|
||
| ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { | ||
| const targetVideoPath = resolveApprovedVideoPath( | ||
| videoPath ?? currentRecordingSession?.screenVideoPath, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Guard the cursor telemetry clears against a newer take.
writePendingCursorTelemetry()awaitsfs.writeFile. A new take can start during that await, becausestartCursorRecording()does not wait for it. After the write, the function clearscursorTelemetryRecordingIdandpendingCursorRecordingDatawithout checking their owner. A new take's ID can then be cleared, sodiscard-cursor-telemetryfor that take returns early and leaves its sampler running. A new take's pending data can also be dropped.Capture the owner and data before the write. Clear each shared value only when it still matches the captured value.
Suggested fix
async function writePendingCursorTelemetry(videoPath: string) { const telemetryPath = `${videoPath}.cursor.json`; - if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { - await fs.writeFile(telemetryPath, JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8"); + const owner = cursorTelemetryRecordingId; + const data = pendingCursorRecordingData; + if (data && data.samples.length > 0) { + await fs.writeFile(telemetryPath, JSON.stringify(data, null, 2), "utf-8"); } - pendingCursorRecordingData = null; - cursorTelemetryRecordingId = null; + if (pendingCursorRecordingData === data) pendingCursorRecordingData = null; + if (cursorTelemetryRecordingId === owner) cursorTelemetryRecordingId = null; }