From eb2fb7434bdc79cc2ed641a2ccf87e3bed89b54d Mon Sep 17 00:00:00 2001 From: jottakka Date: Thu, 16 Jul 2026 11:15:40 -0300 Subject: [PATCH 1/4] fix: preserve last-known-good toolkit docs on generation failures Reject incomplete or unsafe inputs before they can erase generated artifacts, while retaining actionable verification evidence for the workflow. Co-authored-by: Cursor --- toolkit-docs-generator/ARCHITECTURE.md | 25 +- toolkit-docs-generator/README.md | 21 +- .../src/cli/generate-flow.ts | 15 ++ toolkit-docs-generator/src/cli/index.ts | 213 ++++++++---------- .../src/generator/json-generator.ts | 100 ++++++-- .../src/merger/data-merger.ts | 22 +- .../src/sources/custom-sections-file.ts | 7 +- .../src/sources/toolkit-data-source.ts | 24 ++ toolkit-docs-generator/src/types/index.ts | 15 +- .../src/utils/output-dir.ts | 42 +++- .../tests/cli/generate-flow.test.ts | 14 ++ .../tests/generator/output-verifier.test.ts | 79 ++++++- .../tests/merger/data-merger.test.ts | 25 +- .../sources/custom-sections-file.test.ts | 34 +++ .../tests/sources/toolkit-data-source.test.ts | 45 +++- .../tests/utils/output-dir.test.ts | 66 +++++- toolkit-docs-generator/vitest.config.ts | 2 + 17 files changed, 576 insertions(+), 173 deletions(-) diff --git a/toolkit-docs-generator/ARCHITECTURE.md b/toolkit-docs-generator/ARCHITECTURE.md index 14efe20db..67022ac8c 100644 --- a/toolkit-docs-generator/ARCHITECTURE.md +++ b/toolkit-docs-generator/ARCHITECTURE.md @@ -58,16 +58,33 @@ It can also verify output consistency with `OutputVerifier`. The generator output is consumed by the Next.js app: - The app loads JSON from `toolkit-docs-generator/data/toolkits/`. -- Toolkit pages are statically generated using those files. +- `generateStaticParams` enumerates the toolkit routes and disables unknown dynamic parameters. - Custom documentation chunks are rendered as MDX in the UI. If you need HTML output, add a separate build step in the app. The generator intentionally avoids HTML to keep the pipeline deterministic. -## Static page generation +## Vercel build -Static generation happens in the app, not in this generator. +The generator does not run during a Vercel build. The generation workflow commits +the JSON files and navigation changes through a pull request. Vercel then runs the +root `pnpm build` command and compiles the committed files with Next.js. -The generator only provides JSON and markdown content. The app turns those into static HTML during its build. +`app/_lib/toolkit-static-params.ts` enumerates routes from `index.json` and the +per-toolkit files. `app/_lib/toolkit-data.ts` reads the same files for page +rendering and the `/api/toolkit-data/[toolkitId]` route. The root layout reads +request headers, so Vercel reports the docs routes as dynamic even though the +toolkit parameter set is fixed at build time. + +## Search indexing + +Search uses an external Algolia crawler. There is no Pagefind or local search +index build step in this repository. After deployment, the crawler indexes the +rendered site. `app/_components/algolia-search.tsx` queries that index with the +public, read-only values configured through these Vercel environment variables: + +- `NEXT_PUBLIC_ALGOLIA_APP_ID` +- `NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY` +- `NEXT_PUBLIC_ALGOLIA_INDEX_NAME` ## Key files diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index 7b3a65c0e..08289c621 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -19,30 +19,29 @@ The generator merges three inputs into one JSON output per toolkit: It also reads the previous output when you use `--skip-unchanged` or `--previous-output`. -When `--skip-unchanged` runs against the tool metadata API, the generator calls -the summary endpoint (`/v1/tool_metadata_summary`) first to decide which toolkits -need updates. It only fetches full tool metadata from `/v1/tool_metadata` for -toolkits with version changes. The Engine API guarantees that tool definitions -do not change unless the toolkit version changes. +When `--skip-unchanged` runs against the tool metadata API, the generator fetches +one complete snapshot from `/v1/tool_metadata`. It reuses that snapshot for +change detection, progress calculation, and generation so a run cannot compare +different API states. Only changed toolkits are regenerated. ## GitHub workflow: generate and sync The workflow file is `/.github/workflows/generate-toolkit-docs.yml`. It runs these steps: -1. Generate toolkit JSON using `toolkit-docs-generator` and the Engine API. -2. Sync sidebar navigation from `toolkit-docs-generator/data/toolkits` to the `_meta.tsx` files. -3. Create a pull request if there are changes. +1. Type-check and test the toolkit docs generator. +2. Generate toolkit JSON using `toolkit-docs-generator` and the Engine API. +3. Sync sidebar navigation from `toolkit-docs-generator/data/toolkits` to the `_meta.tsx` files. +4. Create or update a pull request if there are changes. Required secrets: - `ENGINE_API_URL`, `ENGINE_API_KEY` -- `OPENAI_API_KEY` for LLM examples and summaries +- `ANTHROPIC_API_KEY` for examples, summaries, and secret-coherence edits Optional secrets: -- `OPENAI_MODEL` (defaults in the workflow) -- `ANTHROPIC_API_KEY` enables the secret-coherence editor (see below). Without it the workflow still runs; the scanners emit warnings but no LLM edits are applied. +- `ANTHROPIC_MODEL` (defaults to `claude-sonnet-4-6` in the workflow) - `ANTHROPIC_EDITOR_MODEL` (defaults to `claude-sonnet-4-6` in the workflow) ## Rendering pipeline (docs site) diff --git a/toolkit-docs-generator/src/cli/generate-flow.ts b/toolkit-docs-generator/src/cli/generate-flow.ts index 96f2885f0..194634e19 100644 --- a/toolkit-docs-generator/src/cli/generate-flow.ts +++ b/toolkit-docs-generator/src/cli/generate-flow.ts @@ -15,6 +15,21 @@ export const collectRemovedToolkitIds = ( .map((c) => c.toolkitId.toLowerCase()) ); +/** + * Protect incremental runs from treating a transient empty API response as a + * request to delete every previously generated toolkit. + */ +export const assertSafeCurrentToolkitSnapshot = ( + currentToolkitCount: number, + previousToolkitCount: number +): void => { + if (currentToolkitCount === 0 && previousToolkitCount > 0) { + throw new Error( + `Current toolkit snapshot is empty; refusing to remove all ${previousToolkitCount} existing toolkits.` + ); + } +}; + export interface ProcessingStats { totalToolkits: number; effectiveSkipped: number; diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index bb05939f6..bf30fdd63 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -13,7 +13,7 @@ import chalk from "chalk"; import { Command } from "commander"; -import { readdir, readFile, rm } from "fs/promises"; +import { readdir, readFile } from "fs/promises"; import ora from "ora"; import { join, resolve } from "path"; import { @@ -46,13 +46,23 @@ import { createMockMetadataSource } from "../sources/mock-metadata.js"; import { createDesignSystemProviderIdResolver } from "../sources/oauth-provider-resolver.js"; import { createArcadeToolkitDataSource, + createCachedToolkitDataSource, createEngineToolkitDataSource, createMockToolkitDataSource, type IToolkitDataSource, type ToolkitData, } from "../sources/toolkit-data-source.js"; +import { + type MergedToolkit, + type ProviderVersion, + ProviderVersionSchema, +} from "../types/index.js"; import { readExclusionList } from "../utils/exclusion-list.js"; import { readIgnoreList } from "../utils/ignore-list.js"; +import { + clearSafeOutputDir, + resolveDefaultOutputDir, +} from "../utils/output-dir.js"; import { createProgressTracker, formatToolkitComplete, @@ -63,27 +73,15 @@ import { readFailedToolsReport, writeFailedToolsReport, } from "../utils/run-logs.js"; +import { type ApiSource, resolveApiSource } from "./api-source.js"; import { cleanupExcludedToolkitOutput } from "./exclusion-cleanup.js"; import { + assertSafeCurrentToolkitSnapshot, collectRemovedToolkitIds, computeProcessingStats, filterProvidersBySkipIds, } from "./generate-flow.js"; -/** - * Supported API sources: - * - "list-tools": Uses /v1/tools endpoint (production Arcade API) - * - "tool-metadata": Uses /v1/tool_metadata endpoint (Engine API) - * - "mock": Uses local mock data files - */ -type ApiSource = "list-tools" | "tool-metadata" | "mock"; - -import { - type MergedToolkit, - type ProviderVersion, - ProviderVersionSchema, -} from "../types/index.js"; - const program = new Command(); /** @@ -118,17 +116,6 @@ const parseProviders = (input: string): ProviderVersion[] => { */ const getDefaultMockDataDir = (): string => join(process.cwd(), "mock-data"); -/** - * Get the default output directory for docs JSON. - */ -const getDefaultOutputDir = (): string => { - const cwd = process.cwd(); - if (cwd.endsWith("toolkit-docs-generator")) { - return resolve(cwd, "..", "data", "toolkits"); - } - return resolve(cwd, "data", "toolkits"); -}; - const getDefaultVerificationDir = (): string => { const cwd = process.cwd(); if (cwd.endsWith("toolkit-docs-generator")) { @@ -265,12 +252,7 @@ const clearOutputDir = async ( outputDir: string, verbose: boolean ): Promise => { - const resolvedDir = resolve(outputDir); - const repoRoot = resolve(process.cwd()); - if (resolvedDir === "/" || resolvedDir === repoRoot) { - throw new Error(`Refusing to overwrite output directory: ${resolvedDir}`); - } - await rm(resolvedDir, { recursive: true, force: true }); + const resolvedDir = await clearSafeOutputDir(outputDir); if (verbose) { console.log(chalk.dim(`Cleared output directory: ${resolvedDir}`)); } @@ -485,51 +467,6 @@ const resolveSecretEditGenerator = ( }); }; -const resolveApiSource = (options: { - apiSource?: string; - toolMetadataUrl?: string; - toolMetadataKey?: string; - listToolsUrl?: string; - listToolsKey?: string; -}): ApiSource => { - // Explicit source takes precedence - if (options.apiSource) { - const source = options.apiSource.toLowerCase(); - // Support both old and new names for backwards compatibility - if (source === "list-tools" || source === "arcade") { - return "list-tools"; - } - if (source === "tool-metadata" || source === "engine") { - return "tool-metadata"; - } - if (source === "mock") { - return "mock"; - } - throw new Error( - `Invalid --api-source "${options.apiSource}". Use "list-tools", "tool-metadata", or "mock".` - ); - } - - // Auto-detect based on provided credentials - const hasListToolsKey = !!( - options.listToolsKey ?? process.env.ARCADE_API_KEY - ); - const hasToolMetadataKey = !!( - options.toolMetadataKey ?? process.env.ENGINE_API_KEY - ); - const hasToolMetadataUrl = !!( - options.toolMetadataUrl ?? process.env.ENGINE_API_URL - ); - - if (hasListToolsKey) { - return "list-tools"; - } - if (hasToolMetadataKey && hasToolMetadataUrl) { - return "tool-metadata"; - } - return "mock"; -}; - interface ToolkitDataSourceOptions { apiSource?: string; listToolsUrl?: string; @@ -855,6 +792,10 @@ const processProviders = async ( } } catch (error) { spinner.fail(`${pv.provider}: ${error}`); + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to process ${pv.provider}: ${message}`, { + cause: error, + }); } } @@ -878,7 +819,7 @@ program "Path to failed tools report to rerun only impacted toolkits" ) .option("--all", "Generate documentation for all toolkits", false) - .option("-o, --output ", "Output directory", getDefaultOutputDir()) + .option("-o, --output ", "Output directory", resolveDefaultOutputDir()) .option("--log-dir ", "Directory for run logs", getDefaultLogDir()) .option("--mock-data-dir ", "Path to mock data directory") .option("--metadata-file ", "Path to metadata JSON file") @@ -1209,13 +1150,15 @@ program // Create toolkit data source based on API source const apiSource = resolveApiSource(options); - const toolkitDataSource = createToolkitDataSourceForApi( - apiSource, - options, - metadataSource, - mockDataDir, - options.verbose, - spinner + const toolkitDataSource = createCachedToolkitDataSource( + createToolkitDataSourceForApi( + apiSource, + options, + metadataSource, + mockDataDir, + options.verbose, + spinner + ) ); const needsExamples = !options.skipExamples; @@ -1386,6 +1329,10 @@ program } currentToolkitDataForDiff.set(id, data); } + assertSafeCurrentToolkitSnapshot( + currentToolkitDataForDiff.size, + previousToolkits?.size ?? 0 + ); // Detect changes const compareStartedAt = Date.now(); @@ -1714,11 +1661,14 @@ program } } + const mergeFailures = allResults.filter((result) => result.error); + const writableResults = allResults.filter((result) => !result.error); + // Generate output files (batch mode if not incremental) - if (!useIncremental && allResults.length > 0) { + if (!useIncremental && writableResults.length > 0) { spinner.start("Writing output files..."); - const toolkits = allResults.map((r) => r.toolkit); + const toolkits = writableResults.map((result) => result.toolkit); const genResult = await generator.generateAll(toolkits); filesWritten.push(...genResult.filesWritten); @@ -1731,13 +1681,15 @@ program } } else if (useIncremental) { // Generate index file for incremental mode - if (allResults.length > 0 || skipToolkitIds.size > 0) { + if (writableResults.length > 0 || skipToolkitIds.size > 0) { spinner.start("Generating index file..."); try { const existingToolkits = await generator.getCompletedToolkitIds(); const allToolkitIds = new Set([ ...existingToolkits, - ...allResults.map((r) => r.toolkit.id.toLowerCase()), + ...writableResults.map((result) => + result.toolkit.id.toLowerCase() + ), ]); // Load all toolkits for index @@ -1797,7 +1749,11 @@ program } // Print summary - if (filesWritten.length > 0 || writeErrors.length > 0) { + if ( + filesWritten.length > 0 || + writeErrors.length > 0 || + mergeFailures.length > 0 + ) { console.log(chalk.green("\n✓ Generation complete\n")); if (options.verbose) { console.log(chalk.dim("Files:")); @@ -1814,6 +1770,14 @@ program console.log(chalk.yellow(` ${error}`)); } } + if (mergeFailures.length > 0) { + console.log(chalk.red("\nFailed toolkits:")); + for (const failure of mergeFailures) { + console.log( + chalk.red(` ${failure.toolkit.id}: ${failure.error}`) + ); + } + } } if (options.verifyOutput) { @@ -1846,13 +1810,7 @@ program (count, result) => count + result.warnings.length, 0 ); - const failedToolkits = allResults - .filter((result) => - result.warnings.some((warning) => - warning.startsWith("Error processing toolkit") - ) - ) - .map((result) => result.toolkit.id); + const failedToolkits = mergeFailures.map((result) => result.toolkit.id); const failedTools = allResults.flatMap((result) => result.failedTools); const failedToolkitsFromTools = Array.from( new Set(failedTools.map((tool) => tool.toolkitId)) @@ -1905,6 +1863,9 @@ program title: "generate", details: runDetails, }); + if (mergeFailures.length > 0 || writeErrors.length > 0) { + process.exitCode = 1; + } } catch (error) { spinner.fail( `Error: ${error instanceof Error ? error.message : String(error)}` @@ -1924,7 +1885,7 @@ program program .command("generate-all") .description("Generate documentation for all toolkits in mock data") - .option("-o, --output ", "Output directory", getDefaultOutputDir()) + .option("-o, --output ", "Output directory", resolveDefaultOutputDir()) .option("--mock-data-dir ", "Path to mock data directory") .option("--metadata-file ", "Path to metadata JSON file") .option( @@ -2144,13 +2105,15 @@ program // Create toolkit data source based on API source const apiSource = resolveApiSource(options); - const toolkitDataSource = createToolkitDataSourceForApi( - apiSource, - options, - metadataSource, - mockDataDir, - options.verbose, - spinner + const toolkitDataSource = createCachedToolkitDataSource( + createToolkitDataSourceForApi( + apiSource, + options, + metadataSource, + mockDataDir, + options.verbose, + spinner + ) ); const needsExamples = !options.skipExamples; @@ -2415,6 +2378,8 @@ program spinner.start(progressTracker.getProgressString()); const results = await merger.mergeAllToolkits(); + const mergeFailures = results.filter((result) => result.error); + const writableResults = results.filter((result) => !result.error); const summary = progressTracker.getSummary(); spinner.succeed( `Processed ${summary.completed} toolkit(s) with ${summary.totalTools} tools in ${summary.elapsed}` @@ -2435,11 +2400,20 @@ program console.log(chalk.dim(` - ${warning}`)); } } + if (mergeFailures.length > 0) { + console.log(chalk.red("\nFailed toolkits:")); + for (const failure of mergeFailures) { + console.log( + chalk.red(` ${failure.toolkit.id}: ${failure.error}`) + ); + } + process.exitCode = 1; + } // Generate output (batch mode if not incremental) - if (!useIncremental && results.length > 0) { + if (!useIncremental && writableResults.length > 0) { spinner.start("Writing output files..."); - const toolkits = results.map((r) => r.toolkit); + const toolkits = writableResults.map((result) => result.toolkit); const genResult = await generator.generateAll(toolkits); filesWritten.push(...genResult.filesWritten); @@ -2461,7 +2435,9 @@ program const existingToolkits = await generator.getCompletedToolkitIds(); const allToolkitIds = new Set([ ...existingToolkits, - ...results.map((r) => r.toolkit.id.toLowerCase()), + ...writableResults.map((result) => + result.toolkit.id.toLowerCase() + ), ]); const toolkitsForIndex: MergedToolkit[] = []; @@ -2553,6 +2529,7 @@ program for (const warning of writeErrors) { console.log(chalk.yellow(` ${warning}`)); } + process.exitCode = 1; } } catch (error) { spinner.fail( @@ -2627,7 +2604,7 @@ program program .command("verify-output") .description("Verify output directory structure and schema") - .option("-o, --output ", "Output directory", getDefaultOutputDir()) + .option("-o, --output ", "Output directory", resolveDefaultOutputDir()) .option("--verbose", "Show detailed progress", false) .action(async (options: { output: string; verbose: boolean }) => { const spinner = ora("Verifying output...").start(); @@ -2667,7 +2644,7 @@ program .option( "-o, --output ", "Previous output directory", - getDefaultOutputDir() + resolveDefaultOutputDir() ) .option("--log-dir ", "Directory for run logs", getDefaultLogDir()) .option("--mock-data-dir ", "Path to mock data directory") @@ -2727,13 +2704,15 @@ program }); const apiSource = resolveApiSource(options); - const toolkitDataSource = createToolkitDataSourceForApi( - apiSource, - options, - metadataSource, - mockDataDir, - false, // not verbose during fetch - spinner + const toolkitDataSource = createCachedToolkitDataSource( + createToolkitDataSourceForApi( + apiSource, + options, + metadataSource, + mockDataDir, + false, // not verbose during fetch + spinner + ) ); // Fetch current data from API diff --git a/toolkit-docs-generator/src/generator/json-generator.ts b/toolkit-docs-generator/src/generator/json-generator.ts index 2991d3b6a..e032ab76d 100644 --- a/toolkit-docs-generator/src/generator/json-generator.ts +++ b/toolkit-docs-generator/src/generator/json-generator.ts @@ -3,7 +3,8 @@ * * Outputs merged toolkit data as JSON files. */ -import { mkdir, readFile, stat, writeFile } from "fs/promises"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, rename, rm, stat, writeFile } from "fs/promises"; import { dirname, join } from "path"; import { parsePreviousToolkitForDiff } from "../diff/previous-output.js"; import type { @@ -17,6 +18,60 @@ import { type ToolkitReadResult, } from "./output-verifier.js"; +const SAFE_TOOLKIT_ID = /^[a-z0-9][a-z0-9_-]*$/i; +const RESERVED_TOOLKIT_ID = "index"; + +const getToolkitFileName = (toolkitId: string): string => { + const normalizedId = toolkitId.toLowerCase(); + if (normalizedId === RESERVED_TOOLKIT_ID) { + throw new Error(`"${toolkitId}" is a reserved toolkit ID`); + } + if (!SAFE_TOOLKIT_ID.test(toolkitId)) { + throw new Error(`Unsafe toolkit ID: "${toolkitId}"`); + } + return `${normalizedId}.json`; +}; + +const validateToolkitFileNames = ( + toolkits: readonly MergedToolkit[] +): string[] => { + const errors: string[] = []; + const toolkitIdsByFile = new Map(); + + for (const toolkit of toolkits) { + try { + const fileName = getToolkitFileName(toolkit.id); + const toolkitIds = toolkitIdsByFile.get(fileName) ?? []; + toolkitIdsByFile.set(fileName, [...toolkitIds, toolkit.id]); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + } + + for (const [fileName, toolkitIds] of toolkitIdsByFile) { + if (toolkitIds.length > 1) { + errors.push( + `Toolkit IDs collide on ${fileName}: ${toolkitIds.join(", ")}` + ); + } + } + + return errors; +}; + +const writeFileAtomically = async ( + filePath: string, + content: string +): Promise => { + const tempPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + try { + await writeFile(tempPath, content, "utf-8"); + await rename(tempPath, filePath); + } finally { + await rm(tempPath, { force: true }); + } +}; + // ============================================================================ // Generator Configuration // ============================================================================ @@ -86,7 +141,7 @@ export class JsonGenerator { } } - const fileName = `${toolkit.id.toLowerCase()}.json`; + const fileName = getToolkitFileName(toolkit.id); const filePath = join(this.outputDir, fileName); // Ensure directory exists @@ -97,7 +152,7 @@ export class JsonGenerator { ? JSON.stringify(toolkit, null, 2) : JSON.stringify(toolkit); - await writeFile(filePath, content, "utf-8"); + await writeFileAtomically(filePath, content); return filePath; } @@ -105,9 +160,9 @@ export class JsonGenerator { * Check if a toolkit file already exists in the output directory */ async hasToolkitFile(toolkitId: string): Promise { - const fileName = `${toolkitId.toLowerCase()}.json`; - const filePath = join(this.outputDir, fileName); try { + const fileName = getToolkitFileName(toolkitId); + const filePath = join(this.outputDir, fileName); const stats = await stat(filePath); return stats.isFile(); } catch { @@ -137,9 +192,9 @@ export class JsonGenerator { * Load an existing toolkit file */ async loadToolkitFile(toolkitId: string): Promise { - const fileName = `${toolkitId.toLowerCase()}.json`; - const filePath = join(this.outputDir, fileName); try { + const fileName = getToolkitFileName(toolkitId); + const filePath = join(this.outputDir, fileName); const content = await readFile(filePath, "utf-8"); const parsed = JSON.parse(content) as unknown; const result = MergedToolkitSchema.safeParse(parsed); @@ -160,7 +215,10 @@ export class JsonGenerator { toolkits: readonly MergedToolkit[] ): Promise { const filesWritten: string[] = []; - const errors: string[] = []; + const errors = validateToolkitFileNames(toolkits); + if (errors.length > 0) { + return { filesWritten, errors }; + } // Generate per-toolkit files for (const toolkit of toolkits) { @@ -199,15 +257,21 @@ export class JsonGenerator { private async generateIndexFile( toolkits: readonly MergedToolkit[] ): Promise { - const entries: ToolkitIndexEntry[] = toolkits.map((t) => ({ - id: t.id, - label: t.label, - version: t.version, - category: t.metadata.category, - type: t.metadata.type, - toolCount: t.tools.length, - authType: t.auth?.type ?? "none", - })); + const entries: ToolkitIndexEntry[] = toolkits + .map((t) => ({ + id: t.id, + label: t.label, + version: t.version, + category: t.metadata.category, + type: t.metadata.type, + toolCount: t.tools.length, + authType: t.auth?.type ?? "none", + })) + .sort( + (left, right) => + left.id.toLowerCase().localeCompare(right.id.toLowerCase()) || + left.id.localeCompare(right.id) + ); const index: ToolkitIndex = { generatedAt: new Date().toISOString(), @@ -223,7 +287,7 @@ export class JsonGenerator { ? JSON.stringify(index, null, 2) : JSON.stringify(index); - await writeFile(filePath, content, "utf-8"); + await writeFileAtomically(filePath, content); return filePath; } diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts index 6696dfd89..b73f9139c 100644 --- a/toolkit-docs-generator/src/merger/data-merger.ts +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -93,6 +93,7 @@ export interface MergeResult { toolkit: MergedToolkit; warnings: string[]; failedTools: FailedTool[]; + error?: string; } export interface ToolExampleResult { @@ -987,6 +988,15 @@ export class DataMerger { message: string, previousToolkit?: MergedToolkit ): MergeResult { + if (previousToolkit) { + return { + toolkit: previousToolkit, + warnings: [`Error processing toolkit: ${message}`], + failedTools: [], + error: message, + }; + } + return { toolkit: { id: toolkitId, @@ -1005,13 +1015,14 @@ export class DataMerger { }, auth: null, tools: [], - documentationChunks: previousToolkit?.documentationChunks ?? [], - customImports: previousToolkit?.customImports ?? [], - subPages: previousToolkit?.subPages ?? [], + documentationChunks: [], + customImports: [], + subPages: [], generatedAt: new Date().toISOString(), }, warnings: [`Error processing toolkit: ${message}`], failedTools: [], + error: message, }; } @@ -1049,6 +1060,11 @@ export class DataMerger { return result; } catch (error) { const message = error instanceof Error ? error.message : String(error); + if (this.requireCompleteData) { + throw new Error(`Failed to process ${toolkitId}: ${message}`, { + cause: error, + }); + } const previousToolkit = this.getPreviousToolkit(toolkitId); return this.buildMergeErrorResult(toolkitId, message, previousToolkit); } diff --git a/toolkit-docs-generator/src/sources/custom-sections-file.ts b/toolkit-docs-generator/src/sources/custom-sections-file.ts index 636b195ad..2ed843062 100644 --- a/toolkit-docs-generator/src/sources/custom-sections-file.ts +++ b/toolkit-docs-generator/src/sources/custom-sections-file.ts @@ -7,7 +7,10 @@ import { access, readFile } from "fs/promises"; import { z } from "zod"; import type { CustomSections } from "../types/index.js"; -import { DocumentationChunkSchema } from "../types/index.js"; +import { + DocumentationChunkSchema, + ToolkitSubPageSchema, +} from "../types/index.js"; import { normalizeId } from "../utils/fp.js"; import type { ICustomSectionsSource } from "./interfaces.js"; @@ -20,7 +23,7 @@ const CustomSectionsFileSchema = z.record( z.object({ documentationChunks: z.array(DocumentationChunkSchema).default([]), customImports: z.array(z.string()).default([]), - subPages: z.array(z.string()).default([]), + subPages: z.array(ToolkitSubPageSchema).default([]), toolChunks: z .record(z.string(), z.array(DocumentationChunkSchema)) .default({}), diff --git a/toolkit-docs-generator/src/sources/toolkit-data-source.ts b/toolkit-docs-generator/src/sources/toolkit-data-source.ts index 3ee4e1fcf..1b6b191a6 100644 --- a/toolkit-docs-generator/src/sources/toolkit-data-source.ts +++ b/toolkit-docs-generator/src/sources/toolkit-data-source.ts @@ -75,6 +75,30 @@ export interface IToolkitDataSource { readonly isAvailable: () => Promise; } +/** + * Reuse one all-toolkit snapshot for the lifetime of a generation run. + * + * Change detection, progress setup, and merging all consume the same data + * instead of issuing independent API reads that can disagree mid-run. + */ +export const createCachedToolkitDataSource = ( + source: IToolkitDataSource +): IToolkitDataSource => { + let allToolkitsSnapshot: + | Promise> + | undefined; + + return { + fetchToolkitData: (toolkitId, version) => + source.fetchToolkitData(toolkitId, version), + fetchAllToolkitsData: () => { + allToolkitsSnapshot ??= source.fetchAllToolkitsData(); + return allToolkitsSnapshot; + }, + isAvailable: () => source.isAvailable(), + }; +}; + // ============================================================================ // Combined Implementation (Current: Separate Sources) // ============================================================================ diff --git a/toolkit-docs-generator/src/types/index.ts b/toolkit-docs-generator/src/types/index.ts index 366dde7aa..c8dba520c 100644 --- a/toolkit-docs-generator/src/types/index.ts +++ b/toolkit-docs-generator/src/types/index.ts @@ -316,13 +316,22 @@ export type ToolCodeExample = z.infer; // Custom Sections Schema (extracted from MDX) // ============================================================================ +export const ToolkitSubPageSchema = z.union([ + z.string(), + z.object({ + type: z.string().min(1), + content: z.string(), + relativePath: z.string().min(1), + }), +]); + export const CustomSectionsSchema = z.object({ /** Toolkit-level documentation chunks */ documentationChunks: z.array(DocumentationChunkSchema).default([]), /** Custom imports needed for the MDX file */ customImports: z.array(z.string()).default([]), /** Sub-pages that exist for this toolkit */ - subPages: z.array(z.string()).default([]), + subPages: z.array(ToolkitSubPageSchema).default([]), /** Per-tool documentation chunks (keyed by tool name) */ toolChunks: z .record(z.string(), z.array(DocumentationChunkSchema)) @@ -428,9 +437,7 @@ export const MergedToolkitSchema = z.object({ * Each entry is either a string (legacy slug) or a rich object with * { type, content, relativePath } for inline MDX sub-page content. */ - subPages: z - .array(z.union([z.string(), z.record(z.string(), z.unknown())])) - .default([]), + subPages: z.array(ToolkitSubPageSchema).default([]), /** Generation metadata */ generatedAt: z.string().optional(), }); diff --git a/toolkit-docs-generator/src/utils/output-dir.ts b/toolkit-docs-generator/src/utils/output-dir.ts index 22b925a36..50a244e63 100644 --- a/toolkit-docs-generator/src/utils/output-dir.ts +++ b/toolkit-docs-generator/src/utils/output-dir.ts @@ -1,11 +1,27 @@ -import { realpath } from "fs/promises"; -import { isAbsolute, resolve, sep } from "path"; +import { realpath, rm } from "fs/promises"; +import { basename, isAbsolute, resolve, sep } from "path"; type ResolveOptions = { repoRoot?: string; homeDir?: string; }; +export const resolveRepositoryRoot = (cwd: string = process.cwd()): string => + basename(cwd) === "toolkit-docs-generator" + ? resolve(cwd, "..") + : resolve(cwd); + +export const resolveDefaultOutputDir = ( + cwd: string = process.cwd() +): string => { + const repoRoot = resolveRepositoryRoot(cwd); + const generatorRoot = + basename(cwd) === "toolkit-docs-generator" + ? cwd + : resolve(repoRoot, "toolkit-docs-generator"); + return resolve(generatorRoot, "data", "toolkits"); +}; + const isSubpath = (parent: string, child: string): boolean => { const normalizedParent = parent.endsWith(sep) ? parent : `${parent}${sep}`; return child === parent || child.startsWith(normalizedParent); @@ -15,7 +31,9 @@ export const resolveSafeOutputDir = async ( outputDir: string, options: ResolveOptions = {} ): Promise => { - const resolvedRepoRoot = resolve(options.repoRoot ?? process.cwd()); + const resolvedRepoRoot = resolve( + options.repoRoot ?? resolveRepositoryRoot(process.cwd()) + ); const repoRoot = await realpath(resolvedRepoRoot).catch( () => resolvedRepoRoot ); @@ -30,12 +48,9 @@ export const resolveSafeOutputDir = async ( const resolvedDir = resolve(outputDir); const realDir = await realpath(resolvedDir).catch(() => resolvedDir); - const forbidden = new Set(["/", repoRoot]); - if (homeDir) { - forbidden.add(homeDir); - } - - if (forbidden.has(realDir)) { + const containsRepoRoot = isSubpath(realDir, repoRoot); + const containsHomeDir = homeDir ? isSubpath(realDir, homeDir) : false; + if (containsRepoRoot || containsHomeDir) { throw new Error(`Refusing to delete unsafe output directory: ${realDir}`); } @@ -48,3 +63,12 @@ export const resolveSafeOutputDir = async ( return realDir; }; + +export const clearSafeOutputDir = async ( + outputDir: string, + options: ResolveOptions = {} +): Promise => { + const safeDir = await resolveSafeOutputDir(outputDir, options); + await rm(safeDir, { recursive: true, force: true }); + return safeDir; +}; diff --git a/toolkit-docs-generator/tests/cli/generate-flow.test.ts b/toolkit-docs-generator/tests/cli/generate-flow.test.ts index bdb2044ea..12440b99a 100644 --- a/toolkit-docs-generator/tests/cli/generate-flow.test.ts +++ b/toolkit-docs-generator/tests/cli/generate-flow.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + assertSafeCurrentToolkitSnapshot, collectRemovedToolkitIds, computeProcessingStats, filterProvidersBySkipIds, @@ -72,6 +73,19 @@ describe("collectRemovedToolkitIds", () => { }); }); +describe("assertSafeCurrentToolkitSnapshot", () => { + it("rejects an empty current snapshot when previous output exists", () => { + expect(() => assertSafeCurrentToolkitSnapshot(0, 115)).toThrow( + "refusing to remove all 115 existing toolkits" + ); + }); + + it("allows an empty initial snapshot and non-empty updates", () => { + expect(() => assertSafeCurrentToolkitSnapshot(0, 0)).not.toThrow(); + expect(() => assertSafeCurrentToolkitSnapshot(114, 115)).not.toThrow(); + }); +}); + describe("computeProcessingStats", () => { it("ignores skip IDs that do not exist in the fetched toolkit list", () => { const toolkitList = new Map([ diff --git a/toolkit-docs-generator/tests/generator/output-verifier.test.ts b/toolkit-docs-generator/tests/generator/output-verifier.test.ts index ada7865fb..4322e817b 100644 --- a/toolkit-docs-generator/tests/generator/output-verifier.test.ts +++ b/toolkit-docs-generator/tests/generator/output-verifier.test.ts @@ -1,6 +1,6 @@ import { mkdtemp, readFile, rename, rm, writeFile } from "fs/promises"; import { tmpdir } from "os"; -import { join } from "path"; +import { basename, join } from "path"; import { describe, expect, it } from "vitest"; import { @@ -410,3 +410,80 @@ describe("JsonGenerator.rebuildIndexFromOutput", () => { }); }); }); + +describe("JsonGenerator toolkit file paths", () => { + it("rejects toolkit IDs that escape the output directory", async () => { + await withTempDir(async (dir) => { + const toolkit = await loadFixture("github-toolkit.json"); + const escapeName = `${basename(dir)}-escape`; + const escapePath = join(dir, "..", `${escapeName}.json`); + const generator = createJsonGenerator({ outputDir: dir }); + + try { + await expect( + generator.generateToolkitFile({ + ...toolkit, + id: `../${escapeName}`, + }) + ).rejects.toThrow("Unsafe toolkit ID"); + await expect(readFile(escapePath, "utf-8")).rejects.toThrow(); + } finally { + await rm(escapePath, { force: true }); + } + }); + }); + + it('reserves "index" for the generated index file', async () => { + await withTempDir(async (dir) => { + const toolkit = await loadFixture("github-toolkit.json"); + const generator = createJsonGenerator({ outputDir: dir }); + + await expect( + generator.generateToolkitFile({ ...toolkit, id: "INDEX" }) + ).rejects.toThrow("reserved toolkit ID"); + await expect( + readFile(join(dir, "index.json"), "utf-8") + ).rejects.toThrow(); + }); + }); + + it("rejects case-insensitive toolkit ID collisions before writing", async () => { + await withTempDir(async (dir) => { + const toolkit = await loadFixture("github-toolkit.json"); + const generator = createJsonGenerator({ outputDir: dir }); + + const result = await generator.generateAll([ + toolkit, + { ...toolkit, id: toolkit.id.toUpperCase() }, + ]); + + expect(result.filesWritten).toEqual([]); + expect(result.errors).toEqual([ + `Toolkit IDs collide on github.json: ${toolkit.id}, ${toolkit.id.toUpperCase()}`, + ]); + await expect( + readFile(join(dir, "github.json"), "utf-8") + ).rejects.toThrow(); + await expect( + readFile(join(dir, "index.json"), "utf-8") + ).rejects.toThrow(); + }); + }); + + it("sorts index entries by normalized toolkit ID", async () => { + await withTempDir(async (dir) => { + const [githubToolkit, slackToolkit] = await Promise.all([ + loadFixture("github-toolkit.json"), + loadFixture("slack-toolkit.json"), + ]); + const generator = createJsonGenerator({ outputDir: dir }); + + await generator.generateAll([slackToolkit, githubToolkit]); + + const index = JSON.parse( + await readFile(join(dir, "index.json"), "utf-8") + ) as { toolkits: Array<{ id: string }> }; + expect(index.toolkits.map(({ id }) => id)).toEqual(["Github", "Slack"]); + }); + }); +}); diff --git a/toolkit-docs-generator/tests/merger/data-merger.test.ts b/toolkit-docs-generator/tests/merger/data-merger.test.ts index 70b0afe0c..eb4d69177 100644 --- a/toolkit-docs-generator/tests/merger/data-merger.test.ts +++ b/toolkit-docs-generator/tests/merger/data-merger.test.ts @@ -1928,7 +1928,8 @@ describe("DataMerger", () => { const results = await merger.mergeAllToolkits(); const result = results[0]; - // Error result must preserve previous custom sections, not return empty arrays + expect(result?.error).toBe("Custom sections source unavailable"); + expect(result?.toolkit).toEqual(previousResult.toolkit); expect(result?.toolkit.documentationChunks).toHaveLength(1); expect(result?.toolkit.documentationChunks[0]?.content).toBe( "Critical: GitHub Apps only." @@ -1955,6 +1956,7 @@ describe("DataMerger", () => { const results = await merger.mergeAllToolkits(); const result = results[0]; + expect(result?.error).toBe("Custom sections source unavailable"); expect(result?.toolkit.documentationChunks).toHaveLength(0); expect(result?.toolkit.customImports).toHaveLength(0); expect(result?.toolkit.subPages).toHaveLength(0); @@ -2053,6 +2055,27 @@ describe("DataMerger", () => { expect(results[0]?.toolkit.id).toBe("Github"); }); + it("fails strict runs when a complete toolkit cannot be merged", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([githubTool1]), + metadataSource: new InMemoryMetadataSource([githubMetadata]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: { + getCustomSections: async () => { + throw new Error("Custom sections source unavailable"); + }, + }, + toolExampleGenerator: createStubGenerator(), + requireCompleteData: true, + }); + + await expect(merger.mergeAllToolkits()).rejects.toThrow( + "Failed to process Github: Custom sections source unavailable" + ); + }); + it("should return empty array when no tools", async () => { const toolkitDataSource = createCombinedToolkitDataSource({ toolSource: new InMemoryToolDataSource([]), diff --git a/toolkit-docs-generator/tests/sources/custom-sections-file.test.ts b/toolkit-docs-generator/tests/sources/custom-sections-file.test.ts index c4a68192c..1e4af1936 100644 --- a/toolkit-docs-generator/tests/sources/custom-sections-file.test.ts +++ b/toolkit-docs-generator/tests/sources/custom-sections-file.test.ts @@ -53,6 +53,25 @@ describe("CustomSectionsFileSource", () => { expect(result?.toolChunks).toEqual({}); }); + it("loads rich subpage entries supported by generated toolkit output", async () => { + tempDir = await createTempDir(); + const filePath = join(tempDir, "custom-sections.json"); + const subPage = { + type: "mdx", + content: "# Setup", + relativePath: "setup/page.mdx", + }; + await writeFile( + filePath, + JSON.stringify({ Github: { subPages: [subPage] } }, null, 2) + ); + + const source = createCustomSectionsFileSource(filePath); + const result = await source.getCustomSections("Github"); + + expect(result?.subPages).toEqual([subPage]); + }); + it("throws a helpful error when JSON is invalid", async () => { tempDir = await createTempDir(); const filePath = join(tempDir, "invalid.json"); @@ -87,4 +106,19 @@ describe("CustomSectionsFileSource", () => { `Custom sections file has invalid schema (${filePath})` ); }); + + it("rejects malformed rich subpage entries", async () => { + tempDir = await createTempDir(); + const filePath = join(tempDir, "invalid-subpage.json"); + await writeFile( + filePath, + JSON.stringify({ Github: { subPages: [{ type: "mdx" }] } }, null, 2) + ); + + const source = createCustomSectionsFileSource(filePath); + + await expect(source.getAllCustomSections()).rejects.toThrow( + `Custom sections file has invalid schema (${filePath})` + ); + }); }); diff --git a/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts b/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts index 6e75e049d..99d93ee31 100644 --- a/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts +++ b/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts @@ -11,7 +11,11 @@ import { InMemoryToolDataSource, } from "../../src/sources/in-memory.js"; import type { IMetadataSource } from "../../src/sources/internal.js"; -import { createCombinedToolkitDataSource } from "../../src/sources/toolkit-data-source.js"; +import { + createCachedToolkitDataSource, + createCombinedToolkitDataSource, + type IToolkitDataSource, +} from "../../src/sources/toolkit-data-source.js"; import type { ToolDefinition, ToolkitMetadata } from "../../src/types/index.js"; const createTool = ( @@ -406,3 +410,42 @@ describe("CombinedToolkitDataSource", () => { expect(lookedUpIds).toEqual([]); }); }); + +describe("createCachedToolkitDataSource", () => { + it("reuses one immutable all-toolkit snapshot within a run", async () => { + const snapshot = new Map([ + [ + "Github", + { + tools: [createTool()], + metadata: createMetadata(), + }, + ], + ]); + const githubData = snapshot.get("Github"); + if (!githubData) { + throw new Error("Expected GitHub fixture"); + } + let fetchAllCalls = 0; + const source: IToolkitDataSource = { + fetchToolkitData: async () => githubData, + fetchAllToolkitsData: async () => { + fetchAllCalls += 1; + return snapshot; + }, + isAvailable: async () => true, + }; + const cached = createCachedToolkitDataSource(source); + + const [first, second, third] = await Promise.all([ + cached.fetchAllToolkitsData(), + cached.fetchAllToolkitsData(), + cached.fetchAllToolkitsData(), + ]); + + expect(fetchAllCalls).toBe(1); + expect(first).toBe(snapshot); + expect(second).toBe(first); + expect(third).toBe(first); + }); +}); diff --git a/toolkit-docs-generator/tests/utils/output-dir.test.ts b/toolkit-docs-generator/tests/utils/output-dir.test.ts index 6f58f95c2..44a85e416 100644 --- a/toolkit-docs-generator/tests/utils/output-dir.test.ts +++ b/toolkit-docs-generator/tests/utils/output-dir.test.ts @@ -1,8 +1,12 @@ import { mkdir, mkdtemp, realpath, rm } from "fs/promises"; import { tmpdir } from "os"; -import { join } from "path"; +import { basename, join } from "path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { resolveSafeOutputDir } from "../../src/utils/output-dir.js"; +import { + clearSafeOutputDir, + resolveDefaultOutputDir, + resolveSafeOutputDir, +} from "../../src/utils/output-dir.js"; describe("resolveSafeOutputDir", () => { const originalCwd = process.cwd(); @@ -65,4 +69,62 @@ describe("resolveSafeOutputDir", () => { }) ).rejects.toThrow("unsafe output directory"); }); + + it("rejects the repository root when run from the generator directory", async () => { + const generatorDir = join(repoRoot ?? "", "toolkit-docs-generator"); + await mkdir(generatorDir); + process.chdir(generatorDir); + + await expect(clearSafeOutputDir(repoRoot ?? "")).rejects.toThrow( + "unsafe output directory" + ); + expect(await realpath(repoRoot ?? "")).toContain( + basename(repoRoot ?? "repo") + ); + }); + + it("clears a safe output directory", async () => { + const outputDir = join(repoRoot ?? "", "output"); + await mkdir(outputDir, { recursive: true }); + const expected = await realpath(outputDir); + + const cleared = await clearSafeOutputDir(outputDir, { + repoRoot: repoRoot ?? undefined, + homeDir: homeDir ?? undefined, + }); + + expect(cleared).toBe(expected); + await expect(realpath(outputDir)).rejects.toThrow(); + }); + + it("does not clear a relative directory outside the repository", async () => { + const outsideName = `${basename(repoRoot ?? "repo")}-outside`; + const outsideDir = join(repoRoot ?? "", "..", outsideName); + await mkdir(outsideDir); + try { + await expect( + clearSafeOutputDir(`../${outsideName}`, { + repoRoot: repoRoot ?? undefined, + homeDir: homeDir ?? undefined, + }) + ).rejects.toThrow("outside repo root"); + expect(await realpath(outsideDir)).toContain(outsideName); + } finally { + await rm(outsideDir, { recursive: true, force: true }); + } + }); +}); + +describe("resolveDefaultOutputDir", () => { + it("uses the generator data directory from the repository root", () => { + expect(resolveDefaultOutputDir("/repo")).toBe( + "/repo/toolkit-docs-generator/data/toolkits" + ); + }); + + it("uses the local data directory from the generator directory", () => { + expect(resolveDefaultOutputDir("/repo/toolkit-docs-generator")).toBe( + "/repo/toolkit-docs-generator/data/toolkits" + ); + }); }); diff --git a/toolkit-docs-generator/vitest.config.ts b/toolkit-docs-generator/vitest.config.ts index f7e5aeeda..3ff060c9f 100644 --- a/toolkit-docs-generator/vitest.config.ts +++ b/toolkit-docs-generator/vitest.config.ts @@ -1,6 +1,8 @@ +import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; export default defineConfig({ + root: fileURLToPath(new URL(".", import.meta.url)), test: { // Enable globals like describe, it, expect without imports globals: true, From 176815f56a28aa5f6f4335ca13e4acdaf9bb8efb Mon Sep 17 00:00:00 2001 From: jottakka Date: Thu, 16 Jul 2026 17:24:20 -0300 Subject: [PATCH 2/4] fix: preserve fallback toolkits in batch output Keep last-known-good merge fallbacks in generated output when an individual toolkit merge fails. Co-authored-by: Cursor --- toolkit-docs-generator/src/cli/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index bf30fdd63..3ced003ab 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -1662,7 +1662,10 @@ program } const mergeFailures = allResults.filter((result) => result.error); - const writableResults = allResults.filter((result) => !result.error); + // Error results can still carry a last-known-good toolkit fallback from + // the merger. Keep them in the batch output so one failed merge cannot + // silently remove that toolkit from index.json. + const writableResults = allResults; // Generate output files (batch mode if not incremental) if (!useIncremental && writableResults.length > 0) { From 70a55114535733395094ddf81f66d5b2fa61c51d Mon Sep 17 00:00:00 2001 From: jottakka Date: Thu, 16 Jul 2026 17:26:45 -0300 Subject: [PATCH 3/4] fix: preserve fallback toolkits in all generation modes Write last-known-good merge fallbacks during generate-all as well as provider batch generation. Co-authored-by: Cursor --- toolkit-docs-generator/src/cli/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index 3ced003ab..522899b4f 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -2382,7 +2382,10 @@ program spinner.start(progressTracker.getProgressString()); const results = await merger.mergeAllToolkits(); const mergeFailures = results.filter((result) => result.error); - const writableResults = results.filter((result) => !result.error); + // Error results can still carry a last-known-good toolkit fallback. + // Preserve it in batch output rather than letting --clear-output + // remove the prior JSON artifact. + const writableResults = results; const summary = progressTracker.getSummary(); spinner.succeed( `Processed ${summary.completed} toolkit(s) with ${summary.totalTools} tools in ${summary.elapsed}` From 68be507d6fda269cfadda7f5833cccfe1bdde296 Mon Sep 17 00:00:00 2001 From: jottakka Date: Thu, 16 Jul 2026 17:29:58 -0300 Subject: [PATCH 4/4] fix: reject empty snapshots during change checks Prevent check-changes from reporting a transient empty toolkit API response as every toolkit being removed. Co-authored-by: Cursor --- toolkit-docs-generator/src/cli/index.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index 522899b4f..17c11e302 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -2744,6 +2744,11 @@ program const loadPreviousDurationMs = Date.now() - loadPreviousStartedAt; const previousToolkits = previousToolkitLoad.toolkits; + assertSafeCurrentToolkitSnapshot( + currentToolkitDataForDiff.size, + previousToolkits.size + ); + // Detect changes spinner.text = "Comparing tool definitions..."; const compareStartedAt = Date.now();