diff --git a/docs/cli.md b/docs/cli.md index 653f073bc..35f686aa2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1048,8 +1048,8 @@ Malformed format values and token windows fail with `INVALID_OUTPUT_VALUE` in th selected structured format. Token limits and offsets must be nonnegative safe integers; zero selects an empty window. These checks run before command execution. `--json` is indented on a terminal and compact when piped or redirected, which keeps large results smaller for agents and scripts; pipe -through `jq .` for indented files. Output selection and formatting do not change -operation status. +through `jq .` for indented files. A formatting refusal exits nonzero even when +the underlying operation succeeded. `--format md` escapes backslashes, pipes, and inline Markdown syntax in table headers and cells so source text is displayed literally, and renders cell line @@ -1061,6 +1061,12 @@ values instead of replacing them with an object label. Table widths are measured iteratively, so an admitted row page is not limited by the JavaScript function argument count. +Markdown refuses selected NUL characters and unpaired UTF-16 surrogates because +renderers or UTF-8 output would substitute them. TOON refuses selected unpaired +surrogates for the same UTF-8 reason. Use JSON or YAML to retain those values. +An authored replacement character and valid supplementary characters are not +treated as malformed. + `--format csv` and `--format tsv` export one selected row collection. Select an exact array path with `--filter-output` and provide comma-separated direct scalar columns with `--columns`; column order is the output column order: @@ -1080,7 +1086,8 @@ delimiter, quotes, or line breaks are quoted and embedded quotes are doubled. Values are emitted as UTF-8 without Unicode normalization. Missing and `null` cells are empty; booleans and finite numbers use their ordinary text forms. String cells and headers that begin with spreadsheet formula syntax are refused -with a structured error; JSON retains those values unchanged. +with a structured error; unpaired surrogates in selected cells or headers are +also refused. JSON retains those values unchanged. Delimited output does not flatten nested Evidence or nested field values. It returns a structured error and a nonzero status if the filter does not diff --git a/package-lock.json b/package-lock.json index 41bb9eb5b..bf410ac47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,6 +61,7 @@ "@stryker-mutator/core": "10.0.0", "@stryker-mutator/typescript-checker": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", + "@toon-format/toon": "2.3.1", "@types/cross-spawn": "6.0.6", "@types/node": "26.6.4", "@types/semver": "7.7.1", diff --git a/package.json b/package.json index 4d9481c57..051fad65c 100644 --- a/package.json +++ b/package.json @@ -399,6 +399,7 @@ "@stryker-mutator/core": "10.0.0", "@stryker-mutator/typescript-checker": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", + "@toon-format/toon": "2.3.1", "@types/cross-spawn": "6.0.6", "@types/node": "26.6.4", "@types/semver": "7.7.1", diff --git a/src/cli/cliOutputFormatter.ts b/src/cli/cliOutputFormatter.ts index ab02231c8..049600d4e 100644 --- a/src/cli/cliOutputFormatter.ts +++ b/src/cli/cliOutputFormatter.ts @@ -33,6 +33,58 @@ export interface CliDelimitedOutputOptions { type DelimitedScalar = string | number | boolean | null | undefined; +/** UTF-8 cannot encode a lone UTF-16 surrogate without replacing it. */ +export const hasUnpairedSurrogate = (value: string): boolean => + /[\ud800-\udfff]/u.test(value); + +/** Check selected output, including keys, before using a non-JSON text format. */ +export const containsUnpairedSurrogate = (value: unknown): boolean => { + return containsText(value, hasUnpairedSurrogate); +}; + +/** Markdown renderers replace NUL as well as UTF-8-invalid code units. */ +export const containsUnrepresentableMarkdown = (value: unknown): boolean => + containsText( + value, + (text) => text.includes("\u0000") || hasUnpairedSurrogate(text), + ); + +const containsText = ( + value: unknown, + predicate: (text: string) => boolean, +): boolean => { + const pending: Iterator[] = [[value][Symbol.iterator]()]; + const seen = new WeakSet(); + while (pending.length > 0) { + const next = pending.at(-1)!.next(); + if (next.done === true) { + pending.pop(); + continue; + } + const current: unknown = next.value; + if (typeof current === "string") { + if (predicate(current)) return true; + continue; + } + if (typeof current !== "object" || current === null || seen.has(current)) + continue; + seen.add(current); + pending.push(outputValues(current)); + } + return false; +}; + +function* outputValues(value: object): Generator { + if (Array.isArray(value)) { + yield* value; + return; + } + for (const key of Object.keys(value)) { + yield key; + yield (value as Record)[key]; + } +} + export interface PreparedDelimitedOutput { readonly encodedHeaders: readonly string[]; readonly rows: readonly (readonly DelimitedScalar[])[]; @@ -97,6 +149,14 @@ export const prepareDelimitedOutput = ( const encodedHeaders: string[] = []; for (const field of fields) { + if (hasUnpairedSurrogate(field)) + return err( + delimitedOutputInputError( + ["--columns"], + "CSV/TSV cannot preserve unpaired UTF-16 surrogates in UTF-8 output; use JSON.", + operation, + ), + ); if (isSpreadsheetFormula(field)) return err( delimitedOutputInputError( @@ -135,6 +195,14 @@ export const prepareDelimitedOutput = ( operation, ), ); + if (typeof value === "string" && hasUnpairedSurrogate(value)) + return err( + delimitedOutputInputError( + ["--columns", field], + "CSV/TSV cannot preserve unpaired UTF-16 surrogates in UTF-8 output; use JSON.", + operation, + ), + ); preparedRow.push(value); } preparedRows.push(preparedRow); @@ -250,6 +318,11 @@ const escapeInlineSyntax = ( const escapeTableCell = (value: string): string => value + .replace( + /\u0000|[\ud800-\udfff]/gu, + (character) => + `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`, + ) .replace(/&/g, "&") .replace(//g, ">") diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index f23ba115b..ea29d27cb 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -29,6 +29,9 @@ import { formatCliOutput, prepareDelimitedOutput, delimitedOutputInputError, + containsUnpairedSurrogate, + containsUnrepresentableMarkdown, + hasUnpairedSurrogate, } from "./cliOutputFormatter.js"; import { cliEvidencePublicationReceipt, @@ -477,6 +480,62 @@ export const createCliResultOutput = ( return true; } const selected = selection.value; + const refuseUnrepresentableText = async (): Promise => { + error = projectAnalysisError( + new AnalysisInputError( + metadata.command, + { + userMessage: + format === "md" + ? "Markdown cannot preserve NUL or unpaired UTF-16 surrogates in rendered output." + : "TOON cannot preserve unpaired UTF-16 surrogates in UTF-8 output.", + remediationAction: + "Use --format json or --format yaml to retain the exact code units.", + ...(completed?.originalFailure === undefined + ? {} + : { + diagnostics: { original_failure: completed.originalFailure }, + }), + }, + [ + { + path: ["--format"], + reason: "invalid_value", + message: "The selected format cannot preserve this value.", + }, + ], + ), + ); + const document = options.fullOutput + ? { + ok: false, + error, + meta: { command: metadata.command, duration: metadata.duration }, + } + : error; + handled = true; + try { + await writeOutputText( + destination, + `${formatCliOutput(document, format)}\n`, + ); + } catch (cause: unknown) { + failed = true; + throw cause; + } + return true; + }; + if ( + (format === "toon" || format === "md") && + !options.tokenCount && + (!options.tokenWindow || + format === "md" || + operationFailure !== undefined) && + (format === "md" + ? containsUnrepresentableMarkdown(selected) + : containsUnpairedSurrogate(selected)) + ) + return refuseUnrepresentableText(); const filtered = (format === "json" || format === "jsonl") && ownsFilteredOutput && @@ -649,6 +708,8 @@ export const createCliResultOutput = ( format, ) : sliced; + if (format === "toon" && hasUnpairedSurrogate(outputDocument)) + return refuseUnrepresentableText(); handled = true; try { await writeOutputText( diff --git a/tests/acceptance/applications/keyedArchiveTextUnicode.test.ts b/tests/acceptance/applications/keyedArchiveTextUnicode.test.ts new file mode 100644 index 000000000..58e9c90d6 --- /dev/null +++ b/tests/acceptance/applications/keyedArchiveTextUnicode.test.ts @@ -0,0 +1,145 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { decode } from "@toon-format/toon"; +import { buildBinary } from "plist"; +import { parseFragment, type DefaultTreeAdapterTypes } from "parse5"; +import { createMarkdownRenderer, disposeMdItInstance } from "vitepress"; +import { afterAll, expect } from "vitest"; +import { parse as parseYaml } from "yaml"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +afterAll(() => disposeMdItInstance()); + +const renderedText = (node: DefaultTreeAdapterTypes.Node): string => + node.nodeName === "#text" && "value" in node + ? node.value + : "childNodes" in node + ? node.childNodes.map(renderedText).join("") + : ""; + +cliTest.for([ + ["high", "probe:\ud800:end"], + ["low", "probe:\udc00:end"], + ["pair", "probe:😀:end"], + ["replacement", "probe:�:end"], + ["nul", "probe:\u0000:end"], + ["literal", "probe:\\u0000:end"], +] as const)( + "preserves binary keyed-archive code-unit identity across output formats (%s)", + async ([label, value], { cli }) => { + const root = await createTestTempDirectory("rea-keyed-text-unicode-"); + const renderer = await createMarkdownRenderer(root, { linkify: false }); + const path = join(root, `${label}.bplist`); + await writeFile( + path, + buildBinary({ + $version: 100000, + $archiver: "NSKeyedArchiver", + $top: { root: { UID: 1 } }, + $objects: ["$null", value], + }), + ); + const run = (args: readonly string[]) => + cli.run({ + arguments: ["inspect-keyed-archive", path, ...args], + }); + for (const format of ["json", "yaml"] as const) { + const output = await run(["--format", format]); + expect(output.exitCode, output.stderr).toBe(0); + const document = + format === "json" + ? JSON.parse(output.stdout) + : parseYaml(output.stdout); + expect(document.normalized_result.objects[1].value).toBe(value); + } + for (const args of [[], ["--format", "toon"]] as const) { + const output = await run(args); + const document = decode(output.stdout) as { + code?: string; + normalized_result?: { objects: { value?: string }[] }; + }; + if (label === "high" || label === "low") { + expect(output.exitCode).toBe(1); + expect(document.code).toBe("invalid_request"); + expect(output.stdout).toContain("--format json"); + expect(output.stdout).not.toContain("�"); + } else { + expect(output.exitCode, output.stderr).toBe(0); + expect(document.normalized_result?.objects[1]?.value).toBe(value); + } + } + const markdown = await run(["--format", "md"]); + if (label === "high" || label === "low" || label === "nul") { + expect(markdown.exitCode).toBe(1); + expect(markdown.stdout).toContain("--format json"); + expect(markdown.stdout).not.toContain("�"); + return; + } + expect(markdown.exitCode, markdown.stderr).toBe(0); + const rendered = renderedText( + parseFragment(renderer.render(markdown.stdout)), + ); + expect(rendered).toContain(value); + }, +); + +cliTest( + "keeps NUL-bearing class names distinct in rendered Markdown", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-keyed-class-unicode-"); + const renderer = await createMarkdownRenderer(root, { linkify: false }); + for (const [label, name, expected] of [ + ["nul", "My\u0000Class", ""], + ["replacement", "My�Class", "My�Class"], + ["literal", "My\\u0000Class", "My\\u0000Class"], + ] as const) { + const path = join(root, `${label}.bplist`); + await writeFile( + path, + buildBinary({ + $version: 100000, + $archiver: "NSKeyedArchiver", + $top: { root: { UID: 1 } }, + $objects: [ + "$null", + { $class: { UID: 2 } }, + { $classname: name, $classes: [name, "NSObject"] }, + name, + [name, "NSObject"], + "NSObject", + ], + }), + ); + const json = await cli.run({ + arguments: ["inspect-keyed-archive", path, "--json"], + }); + expect(json.exitCode, json.stderr).toBe(0); + expect( + JSON.parse(json.stdout).normalized_result.objects[1].class_name, + ).toBe(name); + const toon = await cli.run({ + arguments: ["inspect-keyed-archive", path, "--format", "toon"], + }); + expect(toon.exitCode, toon.stderr).toBe(0); + const toonDocument = decode(toon.stdout) as { + normalized_result: { objects: { class_name?: string }[] }; + }; + expect(toonDocument.normalized_result.objects[1]?.class_name).toBe(name); + const markdown = await cli.run({ + arguments: ["inspect-keyed-archive", path, "--format", "md"], + }); + if (label === "nul") { + expect(markdown.exitCode).toBe(1); + expect(markdown.stdout).toContain("--format json"); + continue; + } + expect(markdown.exitCode, markdown.stderr).toBe(0); + const rendered = renderedText( + parseFragment(renderer.render(markdown.stdout)), + ); + expect(rendered).toContain(expected); + } + }, +); diff --git a/tests/boundary/cli/textUnicodeOutput.test.ts b/tests/boundary/cli/textUnicodeOutput.test.ts new file mode 100644 index 000000000..6097dc16a --- /dev/null +++ b/tests/boundary/cli/textUnicodeOutput.test.ts @@ -0,0 +1,192 @@ +import { Writable } from "node:stream"; +import { expect, it } from "vitest"; +import { decode } from "@toon-format/toon"; +import { createCliResultOutput } from "../../../src/cli/cliResultOutput.js"; +import { parseCliOutputArguments } from "../../../src/cliOutput.js"; +import { logCliCommand } from "../../../src/cliLogging.js"; +import { silentLogger } from "../../../src/logger.js"; +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { + formatCliOutput, + prepareDelimitedOutput, +} from "../../../src/cli/cliOutputFormatter.js"; + +it("refuses TOON values or keys that UTF-8 cannot preserve", async () => { + for (const value of [{ name: "a\ud800b" }, { ["a\udc00b"]: "value" }]) { + const chunks: Buffer[] = []; + const output = createCliResultOutput( + parseCliOutputArguments(["--format", "toon"]), + new Writable({ + write(chunk: Buffer, _encoding, done) { + chunks.push(chunk); + done(); + }, + }), + ); + if (output === undefined) throw new Error("invalid output settings"); + expect( + await output.write(value, { + command: "fixture", + duration: "1ms", + format: "toon", + }), + ).toBe(true); + expect(output.error).toMatchObject({ code: "invalid_request" }); + expect(decode(Buffer.concat(chunks).toString("utf8"))).toMatchObject({ + code: "invalid_request", + remediation: { action: expect.stringContaining("--format json") }, + }); + } +}); + +it.each([false, true])( + "retains a classified operation failure when TOON cannot encode its diagnostic (token window: %s)", + async (tokenWindow) => { + const previousExitCode = process.exitCode; + try { + const value = await logCliCommand(silentLogger, "fixture", async () => { + throw new AnalysisInputError("fixture", { userMessage: "bad \ud800" }); + }); + const output = createCliResultOutput( + parseCliOutputArguments([ + "--format", + "toon", + "--full-output", + ...(tokenWindow ? ["--token-limit", "1000"] : []), + ]), + new Writable({ + write(_chunk, _encoding, done) { + done(); + }, + }), + ); + if (output === undefined) throw new Error("invalid output settings"); + expect( + await output.write(value, { + command: "fixture", + duration: "1ms", + format: "toon", + }), + ).toBe(true); + expect(output.error).toMatchObject({ + code: "invalid_request", + details: { + diagnostics: { + original_failure: { + code: "invalid_request", + category: "invalid_input", + retryable: false, + }, + }, + }, + }); + } finally { + process.exitCode = previousExitCode; + } + }, +); + +it("keeps valid supplementary characters and an authored replacement character", async () => { + const output = createCliResultOutput( + parseCliOutputArguments(["--format", "toon"]), + new Writable({ + write(_chunk, _encoding, done) { + done(); + }, + }), + ); + if (output === undefined) throw new Error("invalid output settings"); + expect( + await output.write( + { value: "😀�" }, + { command: "fixture", duration: "1ms", format: "toon" }, + ), + ).toBe(false); +}); + +it("only refuses malformed code units in selected TOON output", async () => { + const chunks: Buffer[] = []; + const output = createCliResultOutput( + parseCliOutputArguments(["--format", "toon", "--filter-output", "safe"]), + new Writable({ + write(chunk: Buffer, _encoding, done) { + chunks.push(chunk); + done(); + }, + }), + ); + if (output === undefined) throw new Error("invalid output settings"); + expect( + await output.write( + { safe: "retained", other: "\ud800" }, + { command: "fixture", duration: "1ms", format: "toon" }, + ), + ).toBe(true); + expect(output.error).toBeUndefined(); + expect(decode(Buffer.concat(chunks).toString("utf8"))).toBe("retained"); +}); + +it("refuses malformed code units retained in a TOON token window", async () => { + const chunks: Buffer[] = []; + const output = createCliResultOutput( + parseCliOutputArguments(["--format", "toon", "--token-limit", "1000"]), + new Writable({ + write(chunk: Buffer, _encoding, done) { + chunks.push(chunk); + done(); + }, + }), + ); + if (output === undefined) throw new Error("invalid output settings"); + expect( + await output.write( + { value: "\ud800" }, + { command: "fixture", duration: "1ms", format: "toon" }, + ), + ).toBe(true); + expect(output.error).toMatchObject({ code: "invalid_request" }); +}); + +it("shows NUL visibly in Markdown", () => { + const markdown = formatCliOutput({ rows: [{ value: "a\u0000b�😀" }] }, "md"); + expect(markdown).toContain("a\\\\u0000b�😀"); + expect(markdown).not.toContain("\u0000"); +}); + +it.each(["\ud800", "\u0000"])( + "refuses unrepresentable Markdown code units without substituting them", + async (value) => { + const chunks: Buffer[] = []; + const output = createCliResultOutput( + parseCliOutputArguments(["--format", "md"]), + new Writable({ + write(chunk: Buffer, _encoding, done) { + chunks.push(chunk); + done(); + }, + }), + ); + if (output === undefined) throw new Error("invalid output settings"); + expect( + await output.write( + { value }, + { command: "fixture", duration: "1ms", format: "md" }, + ), + ).toBe(true); + expect(output.error).toMatchObject({ code: "invalid_request" }); + expect(Buffer.concat(chunks).toString("utf8")).not.toContain("�"); + }, +); + +it("refuses unrepresentable delimited cells and column names before output", () => { + for (const [rows, fields] of [ + [[{ x: "\ud800" }], ["x"]], + [[{ ["\udc00"]: "x" }], ["\udc00"]], + ] as const) { + const result = prepareDelimitedOutput(rows, "csv", { + fields, + metadata: {}, + }); + expect(result.ok).toBe(false); + } +});