From 7bf9d0aa9c242571f0769474daa5156f5ac854e0 Mon Sep 17 00:00:00 2001 From: Mark Date: Mon, 29 Jun 2026 19:53:23 +1000 Subject: [PATCH] fix(cursor): align editable cursor overlay with window-mode recordings When recording a single window on macOS, the editable cursor overlay was offset from the real cursor by a significant amount on both axes, so clicks in playback landed on elements the visible cursor wasn't over. Root cause: cursor telemetry was normalized against the whole display's bounds, but a window recording's video only contains the window. The overlay was therefore placed at the cursor's position relative to the full screen instead of relative to the captured window, offset by the window's on-screen origin (and scaled by the display/window size ratio). Fix (mirrors the existing Windows cursor sampler, which already reports per-sample window bounds): - The macOS cursor helper now accepts the target windowId and, each sample, reads that window's live on-screen frame via CGWindowListCopyWindowInfo (top-left points, same space as Electron's getCursorScreenPoint) and emits it as `bounds`. Reading it per sample also keeps alignment correct if the window is moved mid-recording. - MacNativeCursorRecordingSession normalizes against the emitted window bounds when present, falling back to the display bounds otherwise, via a new pure `normalizeCursorToBounds` helper. - The source id is threaded through the factory so the session can detect window capture and pass the windowId to the helper. Adds unit tests for normalizeCursorToBounds, including a regression test showing the same physical point maps correctly to the window vs. the (previously used, misaligned) display-relative coordinates. Co-authored-by: Cursor --- .../native-bridge/cursor/recording/factory.ts | 1 + .../macNativeCursorRecordingSession.test.ts | 57 +++++++++++++++++++ .../macNativeCursorRecordingSession.ts | 57 ++++++++++++++++--- .../OpenScreenMacOSCursorHelper/main.swift | 37 +++++++++++- 4 files changed, 142 insertions(+), 10 deletions(-) create mode 100644 electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 9a82ffd55..a574f957d 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -31,6 +31,7 @@ export function createCursorRecordingSession( getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, sampleIntervalMs: options.sampleIntervalMs, + sourceId: options.sourceId, startTimeMs: options.startTimeMs, }); } diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts new file mode 100644 index 000000000..598021f6c --- /dev/null +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { normalizeCursorToBounds } from "./macNativeCursorRecordingSession"; + +describe("normalizeCursorToBounds", () => { + it("maps a point to [0,1] within the given bounds", () => { + const bounds = { x: 0, y: 0, width: 1000, height: 500 }; + expect(normalizeCursorToBounds({ x: 0, y: 0 }, bounds)).toMatchObject({ + normalizedX: 0, + normalizedY: 0, + isOutsideBounds: false, + }); + expect(normalizeCursorToBounds({ x: 500, y: 250 }, bounds)).toMatchObject({ + normalizedX: 0.5, + normalizedY: 0.5, + isOutsideBounds: false, + }); + }); + + it("accounts for the bounds origin offset", () => { + const bounds = { x: 200, y: 100, width: 800, height: 600 }; + const { normalizedX, normalizedY, isOutsideBounds } = normalizeCursorToBounds( + { x: 600, y: 400 }, + bounds, + ); + expect(normalizedX).toBeCloseTo(0.5); + expect(normalizedY).toBeCloseTo(0.5); + expect(isOutsideBounds).toBe(false); + }); + + it("flags points outside the bounds", () => { + const bounds = { x: 200, y: 100, width: 800, height: 600 }; + expect(normalizeCursorToBounds({ x: 100, y: 50 }, bounds).isOutsideBounds).toBe(true); + expect(normalizeCursorToBounds({ x: 1200, y: 800 }, bounds).isOutsideBounds).toBe(true); + }); + + // The window-capture bug: the recorded video is just the window, but the cursor + // used to be normalized against the whole display, putting the overlay cursor far + // from where the click actually landed. Normalizing against the window frame fixes it. + it("normalizes against the window frame, not the display, for window capture", () => { + // A 1920x1080 display with a window offset to (600, 300) sized 800x600. + const display = { x: 0, y: 0, width: 1920, height: 1080 }; + const windowFrame = { x: 600, y: 300, width: 800, height: 600 }; + // Cursor sitting at the exact center of the window. + const cursor = { x: windowFrame.x + 400, y: windowFrame.y + 300 }; + + const againstWindow = normalizeCursorToBounds(cursor, windowFrame); + expect(againstWindow.normalizedX).toBeCloseTo(0.5); + expect(againstWindow.normalizedY).toBeCloseTo(0.5); + + const againstDisplay = normalizeCursorToBounds(cursor, display); + // Against the display the same physical point maps to a very different spot, + // which is the visible misalignment users reported. + expect(againstDisplay.normalizedX).toBeCloseTo(1000 / 1920); + expect(againstDisplay.normalizedY).toBeCloseTo(600 / 1080); + expect(Math.abs(againstWindow.normalizedX - againstDisplay.normalizedX)).toBeGreaterThan(0.02); + }); +}); diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts index 27a8a870c..623cf587a 100644 --- a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -3,6 +3,7 @@ import { accessSync, constants as fsConstants } from "node:fs"; import path from "node:path"; import type { Readable } from "node:stream"; import { type Rectangle, screen, systemPreferences } from "electron"; +import { parseMacWindowIdFromSourceId } from "../../../../src/lib/nativeMacRecording"; import type { CursorRecordingData, CursorRecordingSample, @@ -11,6 +12,13 @@ import type { } from "../../../../src/native/contracts"; import type { CursorRecordingSession } from "./session"; +interface MacCursorBounds { + x: number; + y: number; + width: number; + height: number; +} + interface MacCursorAssetPayload { id: string; imageDataUrl: string; @@ -26,6 +34,8 @@ interface MacNativeCursorRecordingSessionOptions { maxSamples: number; sampleIntervalMs: number; startTimeMs?: number; + /** Source id of the recording target; used to detect window capture. */ + sourceId?: string | null; } type MacCursorEvent = @@ -41,6 +51,8 @@ type MacCursorEvent = cursorType?: NativeCursorType | null; assetId?: string | null; asset?: MacCursorAssetPayload | null; + /** On-screen frame of the captured window, when recording a single window. */ + bounds?: MacCursorBounds | null; leftButtonDown?: boolean; leftButtonPressed?: boolean; leftButtonReleased?: boolean; @@ -181,6 +193,24 @@ function normalizeCursorType(value: unknown): NativeCursorType | null { return value === "arrow" || value === "pointer" || value === "text" ? value : null; } +/** + * Normalize an absolute screen cursor point to [0,1] within a captured region's + * bounds. For window capture the region is the window's frame; for full-screen + * it's the display. Returned values may fall outside [0,1] when the cursor is + * outside the region, which the caller uses for visibility/clipping decisions. + */ +export function normalizeCursorToBounds( + cursor: { x: number; y: number }, + bounds: { x: number; y: number; width: number; height: number }, +): { normalizedX: number; normalizedY: number; isOutsideBounds: boolean } { + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (cursor.x - bounds.x) / width; + const normalizedY = (cursor.y - bounds.y) / height; + const isOutsideBounds = normalizedX < 0 || normalizedX > 1 || normalizedY < 0 || normalizedY > 1; + return { normalizedX, normalizedY, isOutsideBounds }; +} + export class MacNativeCursorRecordingSession implements CursorRecordingSession { private samples: CursorRecordingSample[] = []; private assets = new Map(); @@ -220,11 +250,13 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { return; } + const windowId = parseMacWindowIdFromSourceId(this.options.sourceId); const child = spawn( helperPath, [ JSON.stringify({ sampleIntervalMs: this.options.sampleIntervalMs, + ...(windowId != null ? { windowId } : {}), }), ], { @@ -286,9 +318,9 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { } private startPositionOnlyFallback() { - this.captureSample(Date.now(), null, null, false, false, false); + this.captureSample(Date.now(), null, null, false, false, false, null); this.fallbackInterval = setInterval(() => { - this.captureSample(Date.now(), null, null, false, false, false); + this.captureSample(Date.now(), null, null, false, false, false, null); }, this.options.sampleIntervalMs); } @@ -350,6 +382,7 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { payload.leftButtonDown === true, payload.leftButtonPressed === true, payload.leftButtonReleased === true, + payload.bounds ?? null, ); } } @@ -361,15 +394,21 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { leftButtonDown: boolean, leftButtonPressed: boolean, leftButtonReleased: boolean, + windowBounds: MacCursorBounds | null, ) { const cursor = screen.getCursorScreenPoint(); - const bounds = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); - const normalizedX = (cursor.x - bounds.x) / width; - const normalizedY = (cursor.y - bounds.y) / height; - const isOutsideDisplay = - normalizedX < 0 || normalizedX > 1 || normalizedY < 0 || normalizedY > 1; + // For window capture, the helper reports the window's on-screen frame so we + // normalize against the window (the recorded video is just that window). + // Otherwise fall back to the captured display's bounds. + const bounds = + windowBounds ?? + this.options.getDisplayBounds() ?? + screen.getDisplayNearestPoint(cursor).bounds; + const { + normalizedX, + normalizedY, + isOutsideBounds: isOutsideDisplay, + } = normalizeCursorToBounds(cursor, bounds); // Brief exits (under THRESHOLD samples) clip to the canvas edge via clip-path instead // of snapping invisible. Sustained exits (>=THRESHOLD, ~100ms) mark visible=false to // avoid ghost cursors and motion trails from multi-display movement. diff --git a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift index ace6827ca..b348a6436 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift @@ -5,6 +5,31 @@ import Foundation struct CursorHelperRequest: Decodable { let sampleIntervalMs: Int? + // When capturing a single window, the CGWindowID of that window. Cursor + // positions are then normalized against the window's on-screen frame instead + // of the whole display, so the editable cursor overlay lines up with the + // window-only recording. + let windowId: UInt32? +} + +/// Current on-screen frame of a window in global display points (top-left origin, +/// y increasing downward) — the same coordinate space as Electron's +/// `screen.getCursorScreenPoint()`. Re-read every sample so a window that is moved +/// mid-recording stays aligned. Returns nil if the window can't be found. +func windowBounds(for windowId: UInt32) -> CGRect? { + guard + let infoList = CGWindowListCopyWindowInfo([.optionIncludingWindow], windowId) + as? [[String: Any]], + let info = infoList.first, + let boundsDict = info[kCGWindowBounds as String] as? NSDictionary, + let rect = CGRect(dictionaryRepresentation: boundsDict as CFDictionary), + rect.width > 0, + rect.height > 0 + else { + return nil + } + + return rect } struct CapturedCursorAsset { @@ -300,7 +325,7 @@ if CommandLine.arguments.count >= 2, { request = decoded } else { - request = CursorHelperRequest(sampleIntervalMs: nil) + request = CursorHelperRequest(sampleIntervalMs: nil, windowId: nil) } let intervalMs = max(8, request.sampleIntervalMs ?? 33) @@ -337,12 +362,22 @@ while true { "scaleFactor": asset.scaleFactor, ] } + var boundsPayload: [String: Double]? + if let windowId = request.windowId, let frame = windowBounds(for: windowId) { + boundsPayload = [ + "x": Double(frame.origin.x), + "y": Double(frame.origin.y), + "width": Double(frame.size.width), + "height": Double(frame.size.height), + ] + } emit([ "type": "sample", "timestampMs": timestampMs(), "cursorType": currentCursorType(), "assetId": asset?.id, "asset": assetPayload, + "bounds": boundsPayload, "leftButtonDown": leftButtonDown(), "leftButtonPressed": mouseEvents.leftDownCount > 0, "leftButtonReleased": mouseEvents.leftUpCount > 0,