diff --git a/.changeset/fix-dev-build-dir-race.md b/.changeset/fix-dev-build-dir-race.md new file mode 100644 index 00000000000..4033f0fa7ec --- /dev/null +++ b/.changeset/fix-dev-build-dir-race.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Fixes intermittent `trigger dev` run crashes where a run could fail at boot with a cryptic `Cannot find module .../dev-run-worker.mjs` after a rebuild had cleaned up the build directory the run was launched against. Dev runs now retry cleanly instead of hard-crashing when their build directory is missing, the dev watchdog no longer removes the build tree of a still-running session, and a run assigned to a worker version that was superseded by a rebuild now fails fast with a clear message instead of silently hanging until it times out. diff --git a/packages/cli-v3/src/dev/devSupervisor.ts b/packages/cli-v3/src/dev/devSupervisor.ts index 7335f947a5a..6a0d1888afd 100644 --- a/packages/cli-v3/src/dev/devSupervisor.ts +++ b/packages/cli-v3/src/dev/devSupervisor.ts @@ -1,7 +1,9 @@ import { tryCatch } from "@trigger.dev/core/utils"; +import { TaskRunErrorCodes } from "@trigger.dev/core/v3"; import type { BuildManifest, CreateBackgroundWorkerRequestBody, + DequeuedMessage, DevConfigResponseBody, WorkerManifest, } from "@trigger.dev/core/v3"; @@ -224,6 +226,7 @@ class DevSupervisor implements WorkerRuntime { this.activeRunsPath = join(triggerDir, `active-runs${suffix}.json`); this.watchdogPidPath = join(triggerDir, `watchdog${suffix}.pid`); + const lockFilePath = join(triggerDir, safeBranch ? `dev.${safeBranch}.lock` : "dev.lock"); // Write empty active-runs file this.#updateActiveRunsFile(); @@ -249,6 +252,7 @@ class DevSupervisor implements WorkerRuntime { WATCHDOG_ACTIVE_RUNS: this.activeRunsPath, WATCHDOG_PID_FILE: this.watchdogPidPath, WATCHDOG_TMP_DIR: getTmpRoot(this.options.config.workingDir, this.options.branch), + WATCHDOG_LOCK_FILE: lockFilePath, }, }); @@ -478,7 +482,9 @@ class DevSupervisor implements WorkerRuntime { } ); - //todo call the API to crash the run with a good message + this.#failRunWithMissingWorker(message).catch((error) => { + logger.debug("[DevSupervisor] Failed to fail run with missing worker", { error }); + }); continue; } @@ -588,6 +594,33 @@ class DevSupervisor implements WorkerRuntime { } } + async #failRunWithMissingWorker(message: DequeuedMessage) { + const start = await this.options.client.dev.startRunAttempt( + message.run.friendlyId, + message.snapshot.friendlyId + ); + + if (!start.success) { + return; + } + + const { run, snapshot, execution } = start.data; + + await this.options.client.dev.completeRunAttempt(run.friendlyId, snapshot.friendlyId, { + completion: { + id: execution.run.id, + ok: false, + retry: undefined, + error: { + type: "INTERNAL_ERROR", + code: TaskRunErrorCodes.COULD_NOT_FIND_EXECUTOR, + message: + "This run was assigned to a background worker version that is no longer available in the dev session because it was superseded by a rebuild. Trigger the run again to use the current version.", + }, + }, + }); + } + async #startPresenceConnection() { try { const eventSource = this.options.client.dev.presenceConnection(); diff --git a/packages/cli-v3/src/dev/devWatchdog.ts b/packages/cli-v3/src/dev/devWatchdog.ts index 15dfae929f3..906da7d1803 100644 --- a/packages/cli-v3/src/dev/devWatchdog.ts +++ b/packages/cli-v3/src/dev/devWatchdog.ts @@ -34,6 +34,7 @@ const apiKey = process.env.WATCHDOG_API_KEY!; const activeRunsPath = process.env.WATCHDOG_ACTIVE_RUNS!; const pidFilePath = process.env.WATCHDOG_PID_FILE!; const tmpDir = process.env.WATCHDOG_TMP_DIR; +const lockFilePath = process.env.WATCHDOG_LOCK_FILE; if (!parentPid || !apiUrl || !apiKey || !activeRunsPath || !pidFilePath) { process.exit(1); @@ -77,8 +78,25 @@ function cleanup() { } catch {} } +function tmpDirOwnedByLiveSession(): boolean { + if (!lockFilePath) return false; + try { + const pid = Number(readFileSync(lockFilePath, "utf8").trim()); + if (!pid || pid === parentPid) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + } catch { + return false; + } +} + function cleanupTmpDir() { if (!tmpDir) return; + if (tmpDirOwnedByLiveSession()) return; try { rmSync(tmpDir, { recursive: true, force: true }); } catch { diff --git a/packages/cli-v3/src/entryPoints/dev-run-controller.ts b/packages/cli-v3/src/entryPoints/dev-run-controller.ts index 9ef8f2007c9..7b679b6490f 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-controller.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-controller.ts @@ -14,6 +14,7 @@ import { isOOMRunError, SuspendedProcessError, } from "@trigger.dev/core/v3"; +import { UnexpectedExitError } from "@trigger.dev/core/v3/errors"; import { type WorkloadRunAttemptStartResponseBody } from "@trigger.dev/core/v3/workers"; import { setTimeout as sleep } from "timers/promises"; import type { CliApiClient } from "../apiClient.js"; @@ -22,6 +23,7 @@ import { assertExhaustive } from "../utilities/assertExhaustive.js"; import { logger } from "../utilities/logger.js"; import { sanitizeEnvVars } from "../utilities/sanitizeEnvVars.js"; import { join } from "node:path"; +import { existsSync } from "node:fs"; import type { BackgroundWorker } from "../dev/backgroundWorker.js"; import { eventBus } from "../utilities/eventBus.js"; import type { TaskRunProcessPool } from "../dev/taskRunProcessPool.js"; @@ -600,6 +602,15 @@ export class DevRunController { throw new Error(`No worker manifest for Dev ${run.friendlyId}`); } + const workerEntryPoint = this.opts.worker.manifest.workerEntryPoint; + if (!existsSync(workerEntryPoint)) { + throw new UnexpectedExitError( + 1, + null, + `Dev worker build directory was removed before the run could start, likely cleaned up by a concurrent rebuild. Missing worker entry: ${workerEntryPoint}` + ); + } + this.snapshotPoller.start(); logger.debug("getProcess", {