diff --git a/scripts/markdown.spec.ts b/scripts/markdown.spec.ts new file mode 100644 index 0000000000..e0056db639 --- /dev/null +++ b/scripts/markdown.spec.ts @@ -0,0 +1,38 @@ +/** Regression coverage for reusing one parse across the Markdown gate helpers. */ + +import { describe, expect, it } from 'vitest' +import { markdownHeadingLines, markdownProseLines, parseMarkdown } from './markdown.ts' + +const SOURCE = [ + '# Title', + '', + 'Prose kept.', + '', + '', + '```ts', + '# not a heading', + '```', + '', + 'Setext heading', + '--------------', + '', +].join('\n') + +describe('Markdown parse reuse', () => { + it('derives prose lines and headings from a supplied parse exactly as from its own parse', () => { + const tree = parseMarkdown(SOURCE) + const unchanged = structuredClone(tree) + + const prose = markdownProseLines(SOURCE, tree) + const headings = markdownHeadingLines(SOURCE, tree) + + expect(prose).toEqual(markdownProseLines(SOURCE)) + expect(headings).toEqual(markdownHeadingLines(SOURCE)) + expect(prose.map(line => line.index)).toEqual([1, 2, 3, 5, 9, 10, 11, 12]) + expect(headings).toEqual([ + { depth: 1, index: 1, raw: '# Title', text: 'Title' }, + { depth: 2, index: 10, raw: 'Setext heading', text: 'Setext heading' }, + ]) + expect(tree).toEqual(unchanged) + }) +}) diff --git a/scripts/markdown.ts b/scripts/markdown.ts index f221d1c342..d16d88cc9b 100644 --- a/scripts/markdown.ts +++ b/scripts/markdown.ts @@ -180,11 +180,16 @@ function renderedText(node: Nodes): string { return '' } -/** Return every parsed Markdown heading with its rendered text and source line. */ -export function markdownHeadingLines(source: string): MarkdownHeadingLine[] { +/** + * Return every parsed Markdown heading with its rendered text and source line. + * @param source - Markdown source whose headings are returned. + * @param tree - the {@link parseMarkdown} result for exactly `source`, when the caller already has it. + * @returns headings in document order. + */ +export function markdownHeadingLines(source: string, tree: Nodes = parseMarkdown(source)): MarkdownHeadingLine[] { const rawLines = source.split('\n') const headings: MarkdownHeadingLine[] = [] - visitMarkdown(parseMarkdown(source), (node) => { + visitMarkdown(tree, (node) => { if (node.type !== 'heading' || node.position === undefined) return headings.push({ depth: node.depth, @@ -200,9 +205,9 @@ type ColumnRange = readonly [start: number, end: number] type OffsetRange = readonly [start: number, end: number] /** Source-column ranges occupied by parsed HTML comments, keyed by source line. */ -function htmlCommentRanges(source: string, rawLines: readonly string[]): Map { +function htmlCommentRanges(tree: Nodes, rawLines: readonly string[]): Map { const comments: OffsetRange[] = [] - visitMarkdown(parseMarkdown(source), (node) => { + visitMarkdown(tree, (node) => { if (node.type !== 'html' || node.position?.start.offset === undefined) return let cursor = 0 while (true) { @@ -250,13 +255,14 @@ function hasRenderedTextOutsideComments(raw: string, ranges: readonly ColumnRang /** * Return source lines outside code blocks and HTML comments. * @param source - Markdown source whose prose should be retained verbatim. + * @param tree - the {@link parseMarkdown} result for exactly `source`, when the caller already has it. * @returns unfenced lines with their original 1-based locations. */ -export function markdownProseLines(source: string): MarkdownProseLine[] { +export function markdownProseLines(source: string, tree: Nodes = parseMarkdown(source)): MarkdownProseLine[] { const rawLines = source.split('\n') - const comments = htmlCommentRanges(source, rawLines) + const comments = htmlCommentRanges(tree, rawLines) const fenced = new Set() - visitMarkdown(parseMarkdown(source), (node) => { + visitMarkdown(tree, (node) => { if (node.type !== 'code' || node.position === undefined) return for (let line = node.position.start.line; line <= node.position.end.line; line += 1) fenced.add(line) }) diff --git a/scripts/persistence-artifacts.ts b/scripts/persistence-artifacts.ts index ec2fe8fd54..2f61a66112 100644 --- a/scripts/persistence-artifacts.ts +++ b/scripts/persistence-artifacts.ts @@ -46,9 +46,10 @@ export function renderPersistencePair(root: string, source: string, en: string, translationStructureSignature(zhTree, sourceTargets, { ...context, sourcePath: paths.zh, markdown: zh }), ) if (errors.length > 0) throw new Error(`${source}: bilingual structure mismatch: ${errors.join('; ')}`) + const record = computeTranslationPairingRecord(paths, en, zh, context, { en: sourceTree, zh: zhTree }) return [ { path: paths.source, content: en }, { path: paths.zh, content: zh }, - { path: paths.meta, content: renderTranslationPairingRecord(paths, computeTranslationPairingRecord(paths, en, zh, context)) }, + { path: paths.meta, content: renderTranslationPairingRecord(paths, record) }, ] } diff --git a/scripts/translation-links.ts b/scripts/translation-links.ts index d149d44372..b882803c2a 100644 --- a/scripts/translation-links.ts +++ b/scripts/translation-links.ts @@ -58,7 +58,8 @@ interface Replacement { value: string } -type LinkNode = Extract +/** Markdown nodes whose destinations translation links resolve. */ +export type LinkNode = Extract /** Offset of the one top-level switcher link immediately following the H1. */ export function languageSwitcherLinkOffset( @@ -306,6 +307,51 @@ export function rewriteTranslationLinkLocales( return { content: applyReplacements(markdown, replacements), rewritten: replacements.length } } +/** One authored destination that link normalization replaced. */ +export interface TranslationLinkDestinationEdit { + /** Inline link or referenced definition whose destination was replaced. */ + node: LinkNode + /** Absolute source offset where the authored destination text starts. */ + start: number + /** Absolute source offset where the authored destination text ends. */ + end: number + /** Semantic target text that replaces `markdown.slice(start, end)`. */ + value: string +} + +/** Normalized Markdown text and the destination edits that produced it. */ +export interface TranslationLinkNormalization { + /** Markdown with paired-document destinations replaced by semantic targets. */ + content: string + /** Applied edits in ascending source order; empty when `content` equals the input. */ + edits: TranslationLinkDestinationEdit[] +} + +/** + * Normalize only paired-document locale paths and report each replaced destination. + * @param markdown - Exact authored Markdown text used for link offsets. + * @param context - Repository and locale used to resolve relative links. + * @param skipTargets - Accepted language-switcher destinations. + * @param tree - Optional GFM parse of this exact Markdown text; never a normalized or edited document. + * @returns The normalized text plus the authored destination ranges it replaced. + */ +export function normalizeTranslationMarkdownLinkEdits( + markdown: string, + context: TranslationLinkContext, + skipTargets: readonly string[] = [], + tree?: Nodes, +): TranslationLinkNormalization { + const edits: TranslationLinkDestinationEdit[] = [] + visitResolvedDocumentLinks(markdown, context, skipTargets, (node, destination, resolved) => { + edits.push({ + node, + ...replacementFor(destination, `dsh-translation-target:${resolved.pair.source}${resolved.suffix}`), + }) + }, tree) + const content = applyReplacements(markdown, [...edits]) + return { content, edits: edits.sort((left, right) => left.start - right.start) } +} + /** * Normalize only paired-document locale paths while retaining every other byte and URL suffix. * @param markdown - Exact authored Markdown text used for link offsets. @@ -320,14 +366,7 @@ export function normalizeTranslationMarkdownLinks( skipTargets: readonly string[] = [], tree?: Nodes, ): string { - const replacements: Replacement[] = [] - visitResolvedDocumentLinks(markdown, context, skipTargets, (_node, destination, resolved) => { - replacements.push(replacementFor( - destination, - `dsh-translation-target:${resolved.pair.source}${resolved.suffix}`, - )) - }, tree) - return applyReplacements(markdown, replacements) + return normalizeTranslationMarkdownLinkEdits(markdown, context, skipTargets, tree).content } /** Semantic target of one authored inline link or referenced definition. */ diff --git a/scripts/translation-pairing-record.ts b/scripts/translation-pairing-record.ts index 09dd1c8f41..89b2bd5a46 100644 --- a/scripts/translation-pairing-record.ts +++ b/scripts/translation-pairing-record.ts @@ -7,8 +7,13 @@ import { createHash } from 'node:crypto' import { basename } from 'node:path' -import type { Nodes } from 'mdast' -import { normalizeTranslationMarkdownLinks, type TranslationLinkContext } from './translation-links.ts' +import type { Nodes, RootContent } from 'mdast' +import { + normalizeTranslationMarkdownLinkEdits, + type TranslationLinkContext, + type TranslationLinkDestinationEdit, + type TranslationLinkNormalization, +} from './translation-links.ts' import { generatedRegions, parseTranslationMarkdown } from './translation-pairing.ts' /** The three repository-relative paths that form one bilingual pair. */ @@ -58,31 +63,112 @@ interface Section { blocks: string[] } +/** One top-level Markdown node, located in the text that {@link sections} slices. */ +export interface TopLevelBlock { + node: RootContent + start: number | undefined + end: number | undefined + line: number +} + function headingText(node: Nodes): string { if ('value' in node && (node.type === 'text' || node.type === 'inlineCode')) return node.value return 'children' in node ? node.children.map(headingText).join('') : '' } +function topLevelBlocks(tree: Nodes): TopLevelBlock[] { + if (tree.type !== 'root') throw new Error('Markdown parser returned a non-root tree') + return tree.children.map(node => ({ + node, + start: node.position?.start.offset, + end: node.position?.end.offset, + line: node.position?.start.line ?? 0, + })) +} + +/** Characters that read only as destination text in every raw or angle-bracket destination slot. */ +const PARSE_NEUTRAL_DESTINATION = /^[A-Za-z0-9._/:#%-]+$/ + +/** + * Whether one edit replaces a complete, title-free destination that starts on its opener's line, + * where both the authored and the replacement text match {@link PARSE_NEUTRAL_DESTINATION}. + */ +function isParseNeutralEdit(markdown: string, edit: TranslationLinkDestinationEdit): boolean { + const authored = markdown.slice(edit.start, edit.end) + const nodeStart = edit.node.position?.start.offset + const nodeEnd = edit.node.position?.end.offset + if (nodeStart === undefined || nodeEnd === undefined || edit.node.url !== authored) return false + if (!PARSE_NEUTRAL_DESTINATION.test(authored) || !PARSE_NEUTRAL_DESTINATION.test(edit.value)) return false + const before = markdown.slice(nodeStart, edit.start) + const after = markdown.slice(edit.end, nodeEnd) + const angle = before.endsWith('<') + return edit.node.type === 'link' + ? /\]\([ \t]*[ \t]*\)$/ : /^[ \t]*\)$/).test(after) + : /\]:[ \t]*[ \t]*$/ : /^[ \t]*$/).test(after) +} + +/** + * Locate the authored tree's top-level nodes in normalized text without reparsing it. + * + * An accepted edit only exchanges letters, digits, and `._/:#%-` for others of + * the same set inside one destination that neither starts a line nor carries a + * title. Those characters cannot end the destination, start or end a code span, + * HTML, autolink, autolink literal, emphasis, table cell, title, escape, or + * character reference, or change how a line starts, so the normalized text + * parses to the same nodes with offsets moved by the preceding edits. + * + * @param markdown - Exact authored text that `tree` parses. + * @param tree - GFM parse of `markdown`. + * @param edits - Normalization edits of `markdown` in ascending source order. + * @returns The moved blocks, or `undefined` when an edit is not provably parse-neutral or lies outside every top-level node. + */ +export function normalizedTopLevelBlocks( + markdown: string, + tree: Nodes, + edits: readonly TranslationLinkDestinationEdit[], +): TopLevelBlock[] | undefined { + if (!edits.every(edit => isParseNeutralEdit(markdown, edit))) return undefined + let next = 0 + let shift = 0 + const moved: TopLevelBlock[] = [] + for (const block of topLevelBlocks(tree)) { + if (block.start === undefined || block.end === undefined) return undefined + const start = block.start + shift + for (let edit = edits[next]; edit !== undefined && edit.start < block.end; edit = edits[next]) { + if (edit.start < block.start || edit.end > block.end) return undefined + shift += edit.value.length - (edit.end - edit.start) + next += 1 + } + moved.push({ ...block, start, end: block.end + shift }) + } + return next === edits.length ? moved : undefined +} + /** * Split normalized Markdown into the preamble plus one section per heading. * Fenced code blocks and generated regions, which the pairing gate requires to * be identical on both sides, are left out of each section's blocks. */ -function sections(markdown: string, tree: Nodes = parseTranslationMarkdown(markdown)): Section[] { +function sections(markdown: string, blocks: readonly TopLevelBlock[]): Section[] { const frontmatter = /^---\n[\s\S]*?\n---(?:\n|$)/.exec(markdown)?.[0].length ?? 0 const result: Section[] = [{ blocks: frontmatter > 0 ? [markdown.slice(0, frontmatter)] : [] }] const regionLines = generatedRegions(markdown).map(region => [region.begin + 1, region.end + 1] as const) - if (tree.type !== 'root') throw new Error('Markdown parser returned a non-root tree') - for (const node of tree.children) { - if ((node.position?.start.offset ?? 0) < frontmatter) continue + for (const { node, start, end, line } of blocks) { + if ((start ?? 0) < frontmatter) continue if (node.type === 'heading') result.push({ heading: node, blocks: [] }) - const line = node.position?.start.line ?? 0 - if (node.type === 'code' || regionLines.some(([begin, end]) => line >= begin && line <= end)) continue - result.at(-1)?.blocks.push(markdown.slice(node.position?.start.offset, node.position?.end.offset)) + if (node.type === 'code' || regionLines.some(([begin, regionEnd]) => line >= begin && line <= regionEnd)) continue + result.at(-1)?.blocks.push(markdown.slice(start, end)) } return result } +/** Sections of one normalized side, reusing its authored parse whenever normalization provably keeps the node layout. */ +function sideSections(markdown: string, normalized: TranslationLinkNormalization, tree: Nodes | undefined): Section[] { + if (tree !== undefined && normalized.content === markdown) return sections(markdown, topLevelBlocks(tree)) + const moved = tree === undefined ? undefined : normalizedTopLevelBlocks(markdown, tree, normalized.edits) + return sections(normalized.content, moved ?? topLevelBlocks(parseTranslationMarkdown(normalized.content))) +} + function slug(text: string): string { return text.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'section' } @@ -105,7 +191,8 @@ function sectionHash(blocks: string[]): string { * @param en - English document text. * @param zh - Chinese document text. * @param context - Link-resolution inputs for both sides. - * @param trees - Optional GFM parses of the exact English and Chinese inputs; normalized text is parsed separately when it changes. + * @param trees - Optional GFM parses of the exact English and Chinese inputs; changed normalized text + * is parsed again only when {@link normalizedTopLevelBlocks} cannot place its nodes. * @returns Entries in document order. * @throws Error when the two sides have different heading counts or malformed generated regions. */ @@ -116,10 +203,10 @@ export function computeTranslationPairingRecord( context: TranslationPairingRecordContext, trees?: { readonly en: Nodes; readonly zh: Nodes }, ): TranslationPairingRecord { - const normalizedEn = normalizeTranslationMarkdownLinks(en, { ...context, sourcePath: paths.source }, [], trees?.en) - const normalizedZh = normalizeTranslationMarkdownLinks(zh, { ...context, sourcePath: paths.zh }, [], trees?.zh) - const enSections = sections(normalizedEn, normalizedEn === en ? trees?.en : undefined) - const zhSections = sections(normalizedZh, normalizedZh === zh ? trees?.zh : undefined) + const normalizedEn = normalizeTranslationMarkdownLinkEdits(en, { ...context, sourcePath: paths.source }, [], trees?.en) + const normalizedZh = normalizeTranslationMarkdownLinkEdits(zh, { ...context, sourcePath: paths.zh }, [], trees?.zh) + const enSections = sideSections(en, normalizedEn, trees?.en) + const zhSections = sideSections(zh, normalizedZh, trees?.zh) if (enSections.length !== zhSections.length) { throw new Error(`${paths.source} has ${enSections.length - 1} heading(s) but ${paths.zh} has ${zhSections.length - 1}`) } diff --git a/scripts/translation-pairing.spec.ts b/scripts/translation-pairing.spec.ts index d22156b7e6..4db59110b2 100644 --- a/scripts/translation-pairing.spec.ts +++ b/scripts/translation-pairing.spec.ts @@ -6,8 +6,10 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { gitIndexPaths, readGitIndexBlob } from './translation-pairing-git.ts' +import { normalizeTranslationMarkdownLinkEdits } from './translation-links.ts' import { computeTranslationPairingRecord, + normalizedTopLevelBlocks, parseTranslationPairingRecord, renderTranslationPairingRecord, translationPairingRecordDiff, @@ -276,6 +278,45 @@ describe('translation pairing records', () => { ]) }) + it('places normalized nodes from the authored parse when every rewritten destination is parse-neutral', () => { + const english = [ + '# Guide', '', 'See [bar]() and [baz][b], [q](./bar.md#x_y).', '', '[b]: baz.md', '', + '## Tail', '', '| [a](bar.md) | b |', '| --- | --- |', '', + ].join('\n') + const chinese = [ + '# 指南', '', '参见[bar]()和[baz][b]、[q](./bar.zh.md#x_y)。', '', '[b]: baz.zh.md', '', + '## 尾部', '', '| [a](bar.zh.md) | b |', '| --- | --- |', '', + ].join('\n') + const tree = parseTranslationMarkdown(english) + const normalized = normalizeTranslationMarkdownLinkEdits(english, { ...context, sourcePath: paths.source }, [], tree) + const reparsed = parseTranslationMarkdown(normalized.content) + + expect(normalized.edits).toHaveLength(4) + expect(normalizedTopLevelBlocks(english, tree, normalized.edits)?.map(({ node, start, end, line }) => [node.type, start, end, line])) + .toEqual('children' in reparsed && reparsed.children.map(node => [ + node.type, node.position?.start.offset, node.position?.end.offset, node.position?.start.line, + ])) + expect(computeTranslationPairingRecord(paths, english, chinese, context, { en: tree, zh: parseTranslationMarkdown(chinese) })) + .toEqual(record(english, chinese)) + }) + + it.each([ + ['a space that unlinks a heading', '# [Guide](a%20b.md)\n\nBody.\n', '# [指南](a%20b.zh.md)\n\n正文。\n', ['/guide-dsh-translation-target-docs-a-b-md']], + ['a pipe that turns a paragraph into a table', '# Guide\n\nSee [a](a%7Cb.md)\n| --- | --- |\n', '# 指南\n\n参见 [a](a%7Cb.zh.md)\n| --- | --- |\n', ['/guide']], + ['a title', '# Guide\n\nSee [bar](bar.md "Bar").\n', '# 指南\n\n参见[bar](bar.zh.md "Bar")。\n', ['/guide']], + ['a destination on the next line', '# Guide\n\nSee [bar](\nbar.md).\n', '# 指南\n\n参见[bar](\nbar.zh.md)。\n', ['/guide']], + ['an entity in the query', '# Guide\n\n[bar](bar.md?x=1&y=2)\n', '# 指南\n\n[bar](bar.zh.md?x=1&y=2)\n', ['/guide']], + ])('reparses normalized text for %s', (_case, english, chinese, keys) => { + const tree = parseTranslationMarkdown(english) + const normalized = normalizeTranslationMarkdownLinkEdits(english, { ...context, sourcePath: paths.source }, [], tree) + const computed = computeTranslationPairingRecord(paths, english, chinese, context, { en: tree, zh: parseTranslationMarkdown(chinese) }) + + expect(normalized.edits).toHaveLength(1) + expect(normalizedTopLevelBlocks(english, tree, normalized.edits)).toBeUndefined() + expect([...computed.keys()]).toEqual(keys) + expect(computed).toEqual(record(english, chinese)) + }) + it('refuses sides with different heading counts', () => { expect(() => record(en, `${zh}\n## 额外\n`)).toThrow('docs/foo.md has 3 heading(s) but docs/foo.zh.md has 4') }) diff --git a/scripts/type-equiv-prefilter.spec.ts b/scripts/type-equiv-prefilter.spec.ts new file mode 100644 index 0000000000..8cd540bad3 --- /dev/null +++ b/scripts/type-equiv-prefilter.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' +import { markdownFences } from './markdown.ts' +import { mayContainEquivFence } from './type-equiv-prefilter.ts' + +const EQUIV_INFOS = new Set(['ts type-equiv', 'ts public-api', 'ts type-equiv public-api']) + +function parsesEquivFence(source: string): boolean { + return markdownFences(source).some(fence => EQUIV_INFOS.has(fence.info)) +} + +describe('type-equivalence prefilter', () => { + it.each([ + '```ts type-equiv\ninterface A {}\n```\n', + '~~~ts public-api\nclass A {}\n~~~\n', + '> ```ts type-equiv\n> interface A {}\n> ```\n', + '```ts type-equiv\ninterface A {}\n```\n', + '```ts type\\-equiv\ninterface A {}\n```\n', + '```ts type-equiv\ninterface A {}\n```\n', + '~~~ts public-api\nclass A {}\n~~~\n', + ])('keeps a document whose fence parses as a source-equivalence form: %j', (source) => { + expect(parsesEquivFence(source)).toBe(true) + expect(mayContainEquivFence(source)).toBe(true) + }) + + it.each([ + '# Title\n\nPlain prose with a `type` word and an & sign.\n', + '```ts\ninterface A {}\n```\n', + '```sh\npnpm run build\n```\n', + ])('skips a document without such a fence: %j', (source) => { + expect(parsesEquivFence(source)).toBe(false) + expect(mayContainEquivFence(source)).toBe(false) + }) +}) diff --git a/scripts/type-equiv-prefilter.ts b/scripts/type-equiv-prefilter.ts new file mode 100644 index 0000000000..d9289bcd60 --- /dev/null +++ b/scripts/type-equiv-prefilter.ts @@ -0,0 +1,19 @@ +/** Text prefilter that lets `verify-type-equiv` skip Markdown parsing for documents without a source-equivalence fence. */ + +/** + * Whether `source` can contain a fence whose parsed info string is a + * source-equivalence form; other documents skip Markdown parsing. Each such + * info string contains `type-equiv` or `public-api`, and it is the parsed rest + * of one opening fence line, which contains three backticks or tildes. Parsing + * changes that text only by decoding backslash escapes and character + * references, so an opening line without `\` or `&` contains the word + * literally, and a fence line with either character keeps the document: + * `ts type-equiv` and `ts type\-equiv` both parse as `ts type-equiv`. + * @param source - Markdown document text. + * @returns `false` only when no fence in `source` can parse to a `type-equiv` or `public-api` info string. + */ +export function mayContainEquivFence(source: string): boolean { + if (source.includes('type-equiv') || source.includes('public-api')) return true + return source.split('\n').some(line => + (line.includes('```') || line.includes('~~~')) && (line.includes('&') || line.includes('\\'))) +} diff --git a/scripts/verify-md-links.spec.ts b/scripts/verify-md-links.spec.ts index 9e492d650c..e38707fed9 100644 --- a/scripts/verify-md-links.spec.ts +++ b/scripts/verify-md-links.spec.ts @@ -9,7 +9,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' -import { anchorCache, documentAnchors, findViolations, githubSlug, markdownLinkSourcePaths } from './verify-md-links.ts' +import { documentAnchors, findViolations, githubSlug, markdownLinkCache, markdownLinkSourcePaths } from './verify-md-links.ts' const roots: string[] = [] afterEach(() => { @@ -27,7 +27,7 @@ function layout(files: Record): string { } function violationsIn(root: string, rel: string): { url: string; reason: string }[] { - return findViolations(join(root, rel), anchorCache(), root).map(({ url, reason }) => ({ url, reason })) + return findViolations(join(root, rel), markdownLinkCache(), root).map(({ url, reason }) => ({ url, reason })) } describe('Markdown source discovery', () => { @@ -37,7 +37,7 @@ describe('Markdown source discovery', () => { 'packages/README.zh.md': '[removed group](removed/README.zh.md)\n', }) const sources = markdownLinkSourcePaths(root) - const violations = sources.flatMap(file => findViolations(join(root, file), anchorCache(), root)) + const violations = sources.flatMap(file => findViolations(join(root, file), markdownLinkCache(), root)) expect(violations.map(({ file, reason }) => ({ file, reason })).sort((a, b) => a.file.localeCompare(b.file))) .toEqual([ { file: join('packages', 'README.md'), reason: 'target' }, @@ -53,7 +53,7 @@ describe('Markdown source discovery', () => { '.agents/notes/archived/process/frozen.md': '[historical](missing.md)\n', }) const sources = markdownLinkSourcePaths(root) - const violations = sources.flatMap(file => findViolations(join(root, file), anchorCache(), root)) + const violations = sources.flatMap(file => findViolations(join(root, file), markdownLinkCache(), root)) expect(violations.map(({ file }) => file).sort()).toEqual([ join('packages', 'AGENTS.md'), join('packages', 'core', 'README.md'), diff --git a/scripts/verify-md-links.ts b/scripts/verify-md-links.ts index 2b3cb37791..f06041b13e 100644 --- a/scripts/verify-md-links.ts +++ b/scripts/verify-md-links.ts @@ -120,12 +120,13 @@ export function githubSlug(heading: string): string { * `Repeat` yields `repeat`, `repeat-1`, `repeat-2`. Matching is exact — * element ids are case-sensitive. * @param source - the document's full Markdown text. + * @param tree - the GFM parse of exactly `source`, when the caller already has it. * @returns the set of valid fragments for links into this document. */ -export function documentAnchors(source: string): Set { +export function documentAnchors(source: string, tree: Nodes = parseMarkdown(source)): Set { const anchors = new Set() const occurrences = new Map() - for (const heading of markdownHeadingLines(source)) { + for (const heading of markdownHeadingLines(source, tree)) { const base = githubSlug(heading.text) let result = base let bump = occurrences.get(base) ?? 0 @@ -136,7 +137,7 @@ export function documentAnchors(source: string): Set { occurrences.set(base, bump) anchors.add(result) } - visitMarkdown(parseMarkdown(source), (node: Nodes): void => { + visitMarkdown(tree, (node: Nodes): void => { if (node.type !== 'html') return const html = node.value.replace(//g, '') for (const match of html.matchAll(/ { return anchors } +/** One Markdown file's link inputs and exposed anchors, collected from a single parse. */ +export interface MarkdownLinkDocument { + /** Link, image, and definition destinations with their 1-based start lines, in document order. */ + destinations: { url: string; line: number }[] + /** Fragments that links into this document may name. */ + anchors: Set +} + +/** Per-run reads, parses, and target probes shared by every scanned source and link target. */ +export interface MarkdownLinkCache { + /** Read and parse one Markdown file at most once per run. */ + document: (absPath: string) => MarkdownLinkDocument + /** Whether a resolved link target exists, probed at most once per path. */ + exists: (absPath: string) => boolean +} + +function markdownLinkDocument(source: string): MarkdownLinkDocument { + const tree = parseMarkdown(source) + const destinations: MarkdownLinkDocument['destinations'] = [] + visitMarkdown(tree, (node: Nodes): void => { + if ((node.type === 'link' || node.type === 'image' || node.type === 'definition') && 'url' in node) { + destinations.push({ url: node.url, line: node.position?.start.line ?? 0 }) + } + }) + return { destinations, anchors: documentAnchors(source, tree) } +} + /** - * Lazily collect and cache the anchor set of any existing Markdown file — - * shared across all scanned sources so a target parses once. - * @returns the memoized absolute-path → anchor-set lookup. + * Create the memoized file reads and existence probes for one checker run. + * A file that is both a scanned source and a fragment target parses once; the + * repository is assumed not to change while the run lasts. + * @returns the cache to pass to every {@link findViolations} call of the run. */ -export function anchorCache(): (absPath: string) => Set { - const cache = new Map>() - return (absPath) => { - const hit = cache.get(absPath) - if (hit) return hit - const anchors = documentAnchors(readFileSync(absPath, 'utf8')) - cache.set(absPath, anchors) - return anchors +export function markdownLinkCache(): MarkdownLinkCache { + const documents = new Map() + const existence = new Map() + return { + document: (absPath) => { + const hit = documents.get(absPath) + if (hit) return hit + const document = markdownLinkDocument(readFileSync(absPath, 'utf8')) + documents.set(absPath, document) + return document + }, + exists: (absPath) => { + const hit = existence.get(absPath) + if (hit !== undefined) return hit + const exists = existsSync(absPath) + existence.set(absPath, exists) + return exists + }, } } @@ -167,48 +206,39 @@ export function anchorCache(): (absPath: string) => Set { * `` there. Fragments onto non-Markdown targets (`file.ts#L10`) carry * renderer-owned semantics and are not judged. * @param absPath - absolute path of the Markdown source to scan. - * @param anchorsOf - anchor lookup shared across files for cross-link checks. + * @param cache - parses and target probes shared across files for cross-link checks. * @param scanRoot - repository root violations are reported relative to. * @returns one entry per broken link, in document order. */ export function findViolations( absPath: string, - anchorsOf: (abs: string) => Set, + cache: MarkdownLinkCache = markdownLinkCache(), scanRoot: string = root, ): Violation[] { const file = relative(scanRoot, absPath) const dir = dirname(absPath) - const source = readFileSync(absPath, 'utf8') - const tree = parseMarkdown(source) const out: Violation[] = [] - - const check = (url: string, node: Nodes): void => { - if (isExternal(url)) return + for (const { url, line } of cache.document(absPath).destinations) { + if (isExternal(url)) continue const target = pathPart(url) const resolved = target === '' ? absPath : resolve(dir, target) - if (!existsSync(resolved)) { - out.push({ file, line: node.position?.start.line ?? 0, url, reason: 'target' }) - return + if (!cache.exists(resolved)) { + out.push({ file, line, url, reason: 'target' }) + continue } const fragment = fragmentPart(url) - if (fragment === null || !resolved.endsWith('.md')) return - if (!anchorsOf(resolved).has(fragment)) { - out.push({ file, line: node.position?.start.line ?? 0, url, reason: 'anchor' }) + if (fragment === null || !resolved.endsWith('.md')) continue + if (!cache.document(resolved).anchors.has(fragment)) { + out.push({ file, line, url, reason: 'anchor' }) } } - - visitMarkdown(tree, (node: Nodes): void => { - if ((node.type === 'link' || node.type === 'image' || node.type === 'definition') && 'url' in node) { - check(node.url, node) - } - }) return out } if (process.argv[1] && import.meta.filename === resolve(process.argv[1])) { const files = markdownLinkSourcePaths(root) - const anchorsOf = anchorCache() - const all = files.flatMap(file => findViolations(resolve(root, file), anchorsOf)) + const cache = markdownLinkCache() + const all = files.flatMap(file => findViolations(resolve(root, file), cache)) const checked = files.length if (all.length === 0) { diff --git a/scripts/verify-package-readme-limitations.ts b/scripts/verify-package-readme-limitations.ts index 4f64082365..5330165b52 100644 --- a/scripts/verify-package-readme-limitations.ts +++ b/scripts/verify-package-readme-limitations.ts @@ -8,7 +8,7 @@ import { existsSync, globSync, readFileSync } from 'node:fs' import { resolve, sep } from 'node:path' -import { markdownHeadingLines, markdownProseLines } from './markdown.ts' +import { markdownHeadingLines, markdownProseLines, parseMarkdown } from './markdown.ts' const root = resolve(import.meta.dirname, '..') @@ -51,8 +51,9 @@ for (const pkg of scannedPackages) { continue } const source = readFileSync(resolve(root, readme), 'utf8') - const lines = markdownProseLines(source) - const headings = markdownHeadingLines(source) + const tree = parseMarkdown(source) + const lines = markdownProseLines(source, tree) + const headings = markdownHeadingLines(source, tree) const limitations = headings.filter(heading => isLimitationsLike(heading.text)) if (Object.hasOwn(NO_LIMITATIONS, pkg)) { diff --git a/scripts/verify-package-readme-model-experience.ts b/scripts/verify-package-readme-model-experience.ts index 8c58c5ed58..848ef35b89 100644 --- a/scripts/verify-package-readme-model-experience.ts +++ b/scripts/verify-package-readme-model-experience.ts @@ -7,7 +7,7 @@ import { existsSync, globSync, readFileSync } from 'node:fs' import { relative, resolve, sep } from 'node:path' -import { markdownHeadingLines, markdownProseLines, type MarkdownProseLine } from './markdown.ts' +import { markdownHeadingLines, markdownProseLines, parseMarkdown, type MarkdownProseLine } from './markdown.ts' const root = resolve(import.meta.dirname, '..') const HEADING = '## Model Experience' @@ -369,8 +369,9 @@ for (const packageJson of packageJsons) { const text = readFileSync(abs, 'utf8') const rawLines = text.split('\n') - const lines = markdownProseLines(text) - const headings = markdownHeadingLines(text) + const tree = parseMarkdown(text) + const lines = markdownProseLines(text, tree) + const headings = markdownHeadingLines(text, tree) const h2Headings = headings.filter(heading => heading.depth === 2) const modelExperienceHeadings = headings.filter(heading => heading.text .trim().replaceAll(/\s+/g, ' ').toLowerCase() === 'model experience') diff --git a/scripts/verify-translation-pairing.ts b/scripts/verify-translation-pairing.ts index 43489c6b3c..2d950ab8ec 100644 --- a/scripts/verify-translation-pairing.ts +++ b/scripts/verify-translation-pairing.ts @@ -115,11 +115,10 @@ if (request.scope === 'pairs') { if (!indexMode && !repositoryFileExists(anchor)) files.add(anchor) } } else { - for (const pattern of SCOPE_PATTERNS) { - for (const match of globSync(pattern, { cwd: root, exclude: TRANSLATION_SCOPE_GLOB_EXCLUDES })) { - const normalized = match.split(sep).join('/') - if (isTranslationScopeFile(normalized)) files.add(normalized) - } + // One call shares a single directory walk across every pattern. + for (const match of globSync(SCOPE_PATTERNS, { cwd: root, exclude: TRANSLATION_SCOPE_GLOB_EXCLUDES })) { + const normalized = match.split(sep).join('/') + if (isTranslationScopeFile(normalized)) files.add(normalized) } } const translations = [...files].filter(f => f.endsWith('.zh.md')).sort() diff --git a/scripts/verify-type-equiv.ts b/scripts/verify-type-equiv.ts index 96146cc260..fdb739ed8f 100644 --- a/scripts/verify-type-equiv.ts +++ b/scripts/verify-type-equiv.ts @@ -14,6 +14,7 @@ import ts from 'typescript' import { markdownFences } from './markdown.ts' import { partitionPairedMarkdownDerivatives } from './paired-markdown-derivatives.ts' import { isArchivedAgentNotePath } from './repo-files.ts' +import { mayContainEquivFence } from './type-equiv-prefilter.ts' const root = resolve(import.meta.dirname, '..') @@ -90,7 +91,9 @@ function blockSymbol(code: string): string | null { /** Extract every source-equivalence block from one Markdown file. */ function extractEquivBlocks(docRel: string): EquivBlock[] { const blocks: EquivBlock[] = [] - for (const fence of markdownFences(readFileSync(resolve(root, docRel), 'utf8'))) { + const source = readFileSync(resolve(root, docRel), 'utf8') + if (!mayContainEquivFence(source)) return blocks + for (const fence of markdownFences(source)) { if (fence.info === 'ts type-equiv public-api') { throw new Error(`verify-type-equiv: ${docRel}:${fence.line} — use the concise \`ts public-api\` fence`) } @@ -113,6 +116,26 @@ function extractEquivBlocks(docRel: string): EquivBlock[] { return blocks } +/** One manifest source file's text and its parse. */ +interface ParsedSource { + text: string + sourceFile: ts.SourceFile +} + +/** Parsed manifest sources by absolute path; declaration extraction only reads them. */ +const parsedSources = new Map() + +/** Read and parse one manifest source file at most once per run. */ +function parsedSource(sourceRel: string): ParsedSource { + const abs = resolve(root, sourceRel) + const cached = parsedSources.get(abs) + if (cached !== undefined) return cached + const text = readFileSync(abs, 'utf8') + const parsed = { text, sourceFile: ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, /* setParentNodes */ true) } + parsedSources.set(abs, parsed) + return parsed +} + /** * The declaration text of `symbol` in `sourceRel`, with `export` stripped, or * null when the symbol is not declared there. Uses the TS parser so it spans @@ -120,9 +143,7 @@ function extractEquivBlocks(docRel: string): EquivBlock[] { * uniformly while including declaration and member JSDoc. */ function sourceDeclaration(sourceRel: string, symbol: string): string | null { - const abs = resolve(root, sourceRel) - const text = readFileSync(abs, 'utf8') - const sf = ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, /* setParentNodes */ true) + const { text, sourceFile: sf } = parsedSource(sourceRel) for (const stmt of sf.statements) { const named = ts.isInterfaceDeclaration(stmt) || ts.isTypeAliasDeclaration(stmt) @@ -148,9 +169,7 @@ interface InterfacePart { /** Find one top-level interface declaration in a source file. */ function sourceInterface(sourceRel: string, symbol: string): InterfacePart | null { - const abs = resolve(root, sourceRel) - const text = readFileSync(abs, 'utf8') - const sourceFile = ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, /* setParentNodes */ true) + const { text, sourceFile } = parsedSource(sourceRel) const declaration = sourceFile.statements.find((statement): statement is ts.InterfaceDeclaration => ts.isInterfaceDeclaration(statement) && statement.name.text === symbol, ) @@ -159,9 +178,7 @@ function sourceInterface(sourceRel: string, symbol: string): InterfacePart | nul /** Find one interface declaration inside an explicit string-literal module augmentation. */ function augmentedInterface(sourceRel: string, moduleName: string, symbol: string): InterfacePart | null { - const abs = resolve(root, sourceRel) - const text = readFileSync(abs, 'utf8') - const sourceFile = ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, /* setParentNodes */ true) + const { text, sourceFile } = parsedSource(sourceRel) for (const statement of sourceFile.statements) { if (!ts.isModuleDeclaration(statement) || !ts.isStringLiteral(statement.name) || statement.name.text !== moduleName || !statement.body || !ts.isModuleBlock(statement.body)) continue @@ -241,9 +258,7 @@ function bodylessMember(text: string, sf: ts.SourceFile, member: ts.ClassElement * retained so the projection is the source-owned public contract. */ function sourcePublicApi(sourceRel: string, symbol: string): string | null { - const abs = resolve(root, sourceRel) - const text = readFileSync(abs, 'utf8') - const sf = ts.createSourceFile(abs, text, ts.ScriptTarget.Latest, /* setParentNodes */ true) + const { text, sourceFile: sf } = parsedSource(sourceRel) for (const stmt of sf.statements) { if (!ts.isClassDeclaration(stmt) || stmt.name?.text !== symbol) continue const classDoc = sourceJSDoc(text, stmt)