diff --git a/scripts/repo-files.spec.ts b/scripts/repo-files.spec.ts index 60875341d7..453328de31 100644 --- a/scripts/repo-files.spec.ts +++ b/scripts/repo-files.spec.ts @@ -1,9 +1,30 @@ import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join, relative } from 'node:path' -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { uniqueRepoFiles } from './repo-files.ts' +const filesystem = vi.hoisted(() => ({ listed: [] as string[], canonicalized: [] as string[] })) +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + readdirSync: ((...args: Parameters) => { + filesystem.listed.push(String(args[0])) + return actual.readdirSync(...args) + }) as typeof actual.readdirSync, + realpathSync: Object.assign((...args: Parameters) => { + filesystem.canonicalized.push(String(args[0])) + return actual.realpathSync(...args) + }, { native: actual.realpathSync.native }), + } +}) + +afterEach(() => { + filesystem.listed.length = 0 + filesystem.canonicalized.length = 0 +}) + interface Tree { root: string clean: () => void @@ -106,6 +127,67 @@ describe('uniqueRepoFiles', () => { } }) + it('lists each directory once per call when patterns and ** revisit it', () => { + const tree = makeTree() + try { + mkdirSync(join(tree.root, 'a', 'b')) + writeFileSync(join(tree.root, 'a', 'b', 'deep.md'), 'deep\n') + filesystem.listed.length = 0 + const files = uniqueRepoFiles(tree.root, ['a/*.md', '**/*.md', 'a/b/*.md', '**/deep.md']) + expect(files.map(file => relative(tree.root, file.abs))).toEqual([join('a', 'snap.md'), join('a', 'b', 'deep.md')]) + expect(filesystem.listed.length).toBeGreaterThan(0) + expect(new Set(filesystem.listed).size).toBe(filesystem.listed.length) + // A second call lists the tree again instead of reusing stale entries. + filesystem.listed.length = 0 + uniqueRepoFiles(tree.root, ['a/*.md']) + expect(filesystem.listed).toEqual([tree.root, join(tree.root, 'a')]) + } finally { + tree.clean() + } + }) + + it('canonicalizes symlinked matches through realpath and dedupes them onto regular files', () => { + const tree = makeTree() + try { + const rootReal = realpathSync(tree.root) + filesystem.canonicalized.length = 0 + const linkFirst = uniqueRepoFiles(tree.root, ['d/*.md', 'a/*.md']) + expect(linkFirst).toEqual([{ abs: join(tree.root, 'd', 'snap.md'), real: join(rootReal, 'a', 'snap.md') }]) + // The symlinked file and the root are canonicalized; the regular a/snap.md is not. + expect(filesystem.canonicalized).toEqual([join(tree.root, 'd', 'snap.md'), tree.root]) + expect(uniqueRepoFiles(tree.root, ['a/*.md', 'd/*.md'])).toEqual([{ abs: join(tree.root, 'a', 'snap.md'), real: join(rootReal, 'a', 'snap.md') }]) + // A regular file below a literal symlinked directory still resolves to its target. + const linked = uniqueRepoFiles(tree.root, ['linked-dir/*.md']) + expect(linked.map(file => file.real)).toEqual([realpathSync(join(tree.root, 'linked-dir', 'snap.md'))]) + expect(linked[0]!.real).not.toBe(join(rootReal, 'linked-dir', 'snap.md')) + // A symlinked root canonicalizes through its target. + const rootLink = join(tree.root, '..', 'root-link') + symlinkSync(tree.root, rootLink) + expect(uniqueRepoFiles(rootLink, ['a/*.md']).map(file => file.real)).toEqual([join(rootReal, 'a', 'snap.md')]) + } finally { + tree.clean() + } + }) + + it('never lists a directory the caller skips and returns the unskipped result', () => { + const tree = makeTree() + try { + mkdirSync(join(tree.root, 'p', 'lib', 'nested'), { recursive: true }) + writeFileSync(join(tree.root, 'p', 'lib', 'nested', 'built.ts'), 'built\n') + writeFileSync(join(tree.root, 'p', 'source.ts'), 'source\n') + const isExcluded = (path: string): boolean => path.includes('/lib/') + const unskipped = uniqueRepoFiles(tree.root, ['**/*.ts'], isExcluded) + filesystem.listed.length = 0 + const skipped = uniqueRepoFiles(tree.root, ['**/*.ts'], isExcluded, dir => dir.split('/').slice(1).includes('lib')) + expect(skipped).toEqual(unskipped) + expect(skipped.map(file => relative(tree.root, file.abs))).toEqual([join('p', 'source.ts')]) + expect(filesystem.listed).not.toContain(join(tree.root, 'p', 'lib')) + expect(filesystem.listed).not.toContain(join(tree.root, 'p', 'lib', 'nested')) + } finally { + tree.clean() + } + }) + it('fails loudly on a broken symlink instead of shrinking the corpus', () => { const tree = makeTree() try { @@ -125,6 +207,13 @@ describe('uniqueRepoFiles', () => { // under the node-glob implementation, instead of silently shrinking the // scanned corpus. expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow() + // The same holds under `**` and for a cyclic symlink. + expect(() => uniqueRepoFiles(root, ['**/*.md'])).toThrow() + rmSync(join(root, 'broken-link.md')) + symlinkSync(join(root, 'loop-b.md'), join(root, 'loop-a.md')) + symlinkSync(join(root, 'loop-a.md'), join(root, 'loop-b.md')) + expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow() + expect(() => uniqueRepoFiles(root, ['**/loop-a.md'])).toThrow() // A broken symlink under a literal non-final segment matches nothing, // again as node glob silently returns no match for it. symlinkSync(join(root, 'gone-dir'), join(root, 'broken-dir')) diff --git a/scripts/repo-files.ts b/scripts/repo-files.ts index c65a4d72a0..4b8796c177 100644 --- a/scripts/repo-files.ts +++ b/scripts/repo-files.ts @@ -1,6 +1,6 @@ /** Shared repository file discovery and line-oriented reference scanning. */ -import { readdirSync, readFileSync, realpathSync, statSync } from 'node:fs' +import { readdirSync, readFileSync, realpathSync, statSync, type Dirent } from 'node:fs' import { join, relative, resolve, sep } from 'node:path' /** One authored path plus its canonical target for symlink deduplication. */ @@ -11,6 +11,14 @@ export interface RepoFile { real: string } +/** One walker match before canonicalization. */ +interface GlobMatch { + /** Repository-relative slash path. */ + path: string + /** False only for a regular file reached through real directories alone. */ + viaSymlink: boolean +} + /** A rejected line-oriented repository reference. */ export interface ReferenceViolation { /** Repo-relative file containing the reference. */ @@ -89,10 +97,19 @@ function assertSupportedSegment(segment: string): void { * Pattern segments support `*`, `?`, and literals that contain none of * `[]{}()!+@\`; other node-glob syntax, including a trailing `**`, `..`, and * empty segments, is rejected loudly up front, while a `.` segment is folded - * away exactly as node glob normalizes it. Returns repository-relative slash - * paths, sorted. + * away exactly as node glob normalizes it. Directory listings come from + * `listDirectory`, which callers share across patterns so each directory is + * read once. A directory for which `skipDirectory` returns true is never + * entered. Returns repository-relative slash paths, sorted, each marked + * `viaSymlink` unless it names a regular file reached only through real + * directories. */ -function expandGlob(root: string, pattern: string): string[] { +function expandGlob( + root: string, + pattern: string, + listDirectory: (dirAbs: string) => readonly Dirent[], + skipDirectory: (relativeDirectory: string) => boolean, +): GlobMatch[] { const segments: string[] = [] for (const segment of pattern.split('/')) { if (segment === '.') { @@ -127,32 +144,34 @@ function expandGlob(root: string, pattern: string): string[] { if (segments[segments.length - 1] === '**') { throw new TypeError(`repo-files walker does not model a trailing ** segment in pattern: ${pattern}`) } - const out: string[] = [] + const out: GlobMatch[] = [] - const visit = (dirAbs: string, dirRel: string, index: number): void => { + const visit = (dirAbs: string, dirRel: string, index: number, viaSymlink: boolean): void => { if (index >= segments.length) return if (segments[index] !== '**') { - visitSegment(dirAbs, dirRel, index) + visitSegment(dirAbs, dirRel, index, viaSymlink) return } // `**` consumes zero directories here and one directory per recursion. - visitSegment(dirAbs, dirRel, index + 1) - for (const entry of readdirSync(dirAbs, { withFileTypes: true })) { + visitSegment(dirAbs, dirRel, index + 1, viaSymlink) + for (const entry of listDirectory(dirAbs)) { if (entry.name.startsWith('.') || !entry.isDirectory()) continue - visit(join(dirAbs, entry.name), dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`, index) + const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}` + if (skipDirectory(childRel)) continue + visit(join(dirAbs, entry.name), childRel, index, viaSymlink) } } - const visitSegment = (dirAbs: string, dirRel: string, index: number): void => { + const visitSegment = (dirAbs: string, dirRel: string, index: number, viaSymlink: boolean): void => { if (index >= segments.length) return const segment = segments[index] if (segment === undefined) return if (segment === '**') { - visit(dirAbs, dirRel, index) + visit(dirAbs, dirRel, index, viaSymlink) return } const last = index === segments.length - 1 - for (const entry of readdirSync(dirAbs, { withFileTypes: true })) { + for (const entry of listDirectory(dirAbs)) { if (!segmentMatches(segment, entry.name)) continue const childAbs = join(dirAbs, entry.name) const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}` @@ -160,11 +179,13 @@ function expandGlob(root: string, pattern: string): string[] { // Files and symlinks match; a broken or cyclic symlink then fails // loudly in the caller's realpathSync exactly as node glob did, // rather than silently shrinking the scanned corpus. - if (entry.isFile() || entry.isSymbolicLink()) out.push(childRel) + if (entry.isFile() || entry.isSymbolicLink()) { + out.push({ path: childRel, viaSymlink: viaSymlink || !entry.isFile() }) + } continue } if (entry.isDirectory()) { - visitSegment(childAbs, childRel, index + 1) + if (!skipDirectory(childRel)) visitSegment(childAbs, childRel, index + 1, viaSymlink) continue } // A literal non-final segment resolves through stat like node glob's, @@ -179,13 +200,13 @@ function expandGlob(root: string, pattern: string): string[] { // glob silently returns no match for it. continue } - if (!target.isDirectory()) continue - visitSegment(childAbs, childRel, index + 1) + if (!target.isDirectory() || skipDirectory(childRel)) continue + visitSegment(childAbs, childRel, index + 1, true) } } - visit(root, '.', 0) - return out.sort() + visit(root, '.', 0, false) + return out.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0) } /** Whether a pattern segment contains `*` or `?` and is therefore wildcard. */ @@ -195,23 +216,40 @@ function hasWildcard(segment: string): boolean { /** * Expand repository-relative globs and deduplicate symlinked files. + * Each directory is listed at most once per call, shared across patterns. + * A regular file reached only through real directories canonicalizes to + * `realpathSync(root)` joined with its relative path; every other match goes + * through `realpathSync`, so a broken or cyclic symlink still throws. * @param root - absolute repository root. * @param patterns - repository-relative glob patterns, processed in order. * @param isExcluded - optional predicate over each matched relative path. + * @param skipDirectory - optional predicate over each repository-relative directory before the walker + * lists it; return true only when `isExcluded` holds for every path below that directory. * @returns matched files in stable first-seen order. */ export function uniqueRepoFiles( root: string, patterns: readonly string[], isExcluded: (relativePath: string) => boolean = () => false, + skipDirectory: (relativeDirectory: string) => boolean = () => false, ): RepoFile[] { + const listings = new Map() + const listDirectory = (dirAbs: string): readonly Dirent[] => { + let entries = listings.get(dirAbs) + if (entries === undefined) { + entries = readdirSync(dirAbs, { withFileTypes: true }) + listings.set(dirAbs, entries) + } + return entries + } + let realRoot: string | undefined const seen = new Set() const files: RepoFile[] = [] for (const pattern of patterns) { - for (const repoPath of expandGlob(root, pattern)) { + for (const { path: repoPath, viaSymlink } of expandGlob(root, pattern, listDirectory, skipDirectory)) { if (isExcluded(repoPath)) continue const abs = resolve(root, repoPath) - const real = realpathSync(abs) + const real = viaSymlink ? realpathSync(abs) : join(realRoot ??= realpathSync(root), repoPath) if (seen.has(real)) continue seen.add(real) files.push({ abs, real }) diff --git a/scripts/verify-doc-refs.ts b/scripts/verify-doc-refs.ts index ebc63d8ceb..9bb086dc2d 100644 --- a/scripts/verify-doc-refs.ts +++ b/scripts/verify-doc-refs.ts @@ -18,6 +18,9 @@ const PATTERNS = ['packages/**/*.ts'] const isExcluded = (p: string): boolean => p.includes('/lib/') || p.endsWith('.d.ts') || p.startsWith('vendor/') +/** A non-first `lib` segment puts `/lib/` in every path below the directory, so the walker skips it. */ +const isExcludedDirectory = (dir: string): boolean => dir.split('/').slice(1).includes('lib') + /** Root-relative Markdown path token, excluding trailing prose. */ const DOC_REF = /(?:\bdocs|\.agents\/notes)\/[A-Za-z0-9._/-]+\.md/g @@ -26,7 +29,7 @@ function findViolations(absPath: string): Violation[] { return findReferenceViolations(root, absPath, DOC_REF, ref => ref, ref => !existsSync(resolve(root, ref))) } -const files = uniqueRepoFiles(root, PATTERNS, isExcluded) +const files = uniqueRepoFiles(root, PATTERNS, isExcluded, isExcludedDirectory) const all = files.flatMap(file => findViolations(file.abs)) const checked = files.length