perf(docs): list each repository directory once per file walk

uniqueRepoFiles read every directory once per pattern and ** level and
resolved every file with realpath. One walk now lists each directory
once across patterns, resolves only paths reached through symbolic
links, and lets doc refs skip non-leading lib directories, whose files
it always excluded. markdown wrap, doc refs, and package paths return
the same file lists with about half the directory reads.

Refs #5852

scripts/repo-files.spec.ts

@@ -1,9 +1,30 @@
 import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join, relative } from 'node:path'
-import { describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
 import { uniqueRepoFiles } from './repo-files.ts'
 
+const filesystem = vi.hoisted(() => ({ listed: [] as string[], canonicalized: [] as string[] }))
+vi.mock('node:fs', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('node:fs')>()
+  return {
+    ...actual,
+    readdirSync: ((...args: Parameters<typeof actual.readdirSync>) => {
+      filesystem.listed.push(String(args[0]))
+      return actual.readdirSync(...args)
+    }) as typeof actual.readdirSync,
+    realpathSync: Object.assign((...args: Parameters<typeof actual.realpathSync>) => {
+      filesystem.canonicalized.push(String(args[0]))
+      return actual.realpathSync(...args)
+    }, { native: actual.realpathSync.native }),
+  }
+})
+
+afterEach(() => {
+  filesystem.listed.length = 0
+  filesystem.canonicalized.length = 0
+})
+
 interface Tree {
   root: string
   clean: () => void
@@ -106,6 +127,67 @@ describe('uniqueRepoFiles', () => {
     }
   })
 
+  it('lists each directory once per call when patterns and ** revisit it', () => {
+    const tree = makeTree()
+    try {
+      mkdirSync(join(tree.root, 'a', 'b'))
+      writeFileSync(join(tree.root, 'a', 'b', 'deep.md'), 'deep\n')
+      filesystem.listed.length = 0
+      const files = uniqueRepoFiles(tree.root, ['a/*.md', '**/*.md', 'a/b/*.md', '**/deep.md'])
+      expect(files.map(file => relative(tree.root, file.abs))).toEqual([join('a', 'snap.md'), join('a', 'b', 'deep.md')])
+      expect(filesystem.listed.length).toBeGreaterThan(0)
+      expect(new Set(filesystem.listed).size).toBe(filesystem.listed.length)
+      // A second call lists the tree again instead of reusing stale entries.
+      filesystem.listed.length = 0
+      uniqueRepoFiles(tree.root, ['a/*.md'])
+      expect(filesystem.listed).toEqual([tree.root, join(tree.root, 'a')])
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('canonicalizes symlinked matches through realpath and dedupes them onto regular files', () => {
+    const tree = makeTree()
+    try {
+      const rootReal = realpathSync(tree.root)
+      filesystem.canonicalized.length = 0
+      const linkFirst = uniqueRepoFiles(tree.root, ['d/*.md', 'a/*.md'])
+      expect(linkFirst).toEqual([{ abs: join(tree.root, 'd', 'snap.md'), real: join(rootReal, 'a', 'snap.md') }])
+      // The symlinked file and the root are canonicalized; the regular a/snap.md is not.
+      expect(filesystem.canonicalized).toEqual([join(tree.root, 'd', 'snap.md'), tree.root])
+      expect(uniqueRepoFiles(tree.root, ['a/*.md', 'd/*.md'])).toEqual([{ abs: join(tree.root, 'a', 'snap.md'), real: join(rootReal, 'a', 'snap.md') }])
+      // A regular file below a literal symlinked directory still resolves to its target.
+      const linked = uniqueRepoFiles(tree.root, ['linked-dir/*.md'])
+      expect(linked.map(file => file.real)).toEqual([realpathSync(join(tree.root, 'linked-dir', 'snap.md'))])
+      expect(linked[0]!.real).not.toBe(join(rootReal, 'linked-dir', 'snap.md'))
+      // A symlinked root canonicalizes through its target.
+      const rootLink = join(tree.root, '..', 'root-link')
+      symlinkSync(tree.root, rootLink)
+      expect(uniqueRepoFiles(rootLink, ['a/*.md']).map(file => file.real)).toEqual([join(rootReal, 'a', 'snap.md')])
+    } finally {
+      tree.clean()
+    }
+  })
+
+  it('never lists a directory the caller skips and returns the unskipped result', () => {
+    const tree = makeTree()
+    try {
+      mkdirSync(join(tree.root, 'p', 'lib', 'nested'), { recursive: true })
+      writeFileSync(join(tree.root, 'p', 'lib', 'nested', 'built.ts'), 'built\n')
+      writeFileSync(join(tree.root, 'p', 'source.ts'), 'source\n')
+      const isExcluded = (path: string): boolean => path.includes('/lib/')
+      const unskipped = uniqueRepoFiles(tree.root, ['**/*.ts'], isExcluded)
+      filesystem.listed.length = 0
+      const skipped = uniqueRepoFiles(tree.root, ['**/*.ts'], isExcluded, dir => dir.split('/').slice(1).includes('lib'))
+      expect(skipped).toEqual(unskipped)
+      expect(skipped.map(file => relative(tree.root, file.abs))).toEqual([join('p', 'source.ts')])
+      expect(filesystem.listed).not.toContain(join(tree.root, 'p', 'lib'))
+      expect(filesystem.listed).not.toContain(join(tree.root, 'p', 'lib', 'nested'))
+    } finally {
+      tree.clean()
+    }
+  })
+
   it('fails loudly on a broken symlink instead of shrinking the corpus', () => {
     const tree = makeTree()
     try {
@@ -125,6 +207,13 @@ describe('uniqueRepoFiles', () => {
       // under the node-glob implementation, instead of silently shrinking the
       // scanned corpus.
       expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow()
+      // The same holds under `**` and for a cyclic symlink.
+      expect(() => uniqueRepoFiles(root, ['**/*.md'])).toThrow()
+      rmSync(join(root, 'broken-link.md'))
+      symlinkSync(join(root, 'loop-b.md'), join(root, 'loop-a.md'))
+      symlinkSync(join(root, 'loop-a.md'), join(root, 'loop-b.md'))
+      expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow()
+      expect(() => uniqueRepoFiles(root, ['**/loop-a.md'])).toThrow()
       // A broken symlink under a literal non-final segment matches nothing,
       // again as node glob silently returns no match for it.
       symlinkSync(join(root, 'gone-dir'), join(root, 'broken-dir'))

scripts/repo-files.ts

@@ -1,6 +1,6 @@
 /** Shared repository file discovery and line-oriented reference scanning. */
 
-import { readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
+import { readdirSync, readFileSync, realpathSync, statSync, type Dirent } from 'node:fs'
 import { join, relative, resolve, sep } from 'node:path'
 
 /** One authored path plus its canonical target for symlink deduplication. */
@@ -11,6 +11,14 @@ export interface RepoFile {
   real: string
 }
 
+/** One walker match before canonicalization. */
+interface GlobMatch {
+  /** Repository-relative slash path. */
+  path: string
+  /** False only for a regular file reached through real directories alone. */
+  viaSymlink: boolean
+}
+
 /** A rejected line-oriented repository reference. */
 export interface ReferenceViolation {
   /** Repo-relative file containing the reference. */
@@ -89,10 +97,19 @@ function assertSupportedSegment(segment: string): void {
  * Pattern segments support `*`, `?`, and literals that contain none of
  * `[]{}()!+@\`; other node-glob syntax, including a trailing `**`, `..`, and
  * empty segments, is rejected loudly up front, while a `.` segment is folded
- * away exactly as node glob normalizes it. Returns repository-relative slash
- * paths, sorted.
+ * away exactly as node glob normalizes it. Directory listings come from
+ * `listDirectory`, which callers share across patterns so each directory is
+ * read once. A directory for which `skipDirectory` returns true is never
+ * entered. Returns repository-relative slash paths, sorted, each marked
+ * `viaSymlink` unless it names a regular file reached only through real
+ * directories.
  */
-function expandGlob(root: string, pattern: string): string[] {
+function expandGlob(
+  root: string,
+  pattern: string,
+  listDirectory: (dirAbs: string) => readonly Dirent[],
+  skipDirectory: (relativeDirectory: string) => boolean,
+): GlobMatch[] {
   const segments: string[] = []
   for (const segment of pattern.split('/')) {
     if (segment === '.') {
@@ -127,32 +144,34 @@ function expandGlob(root: string, pattern: string): string[] {
   if (segments[segments.length - 1] === '**') {
     throw new TypeError(`repo-files walker does not model a trailing ** segment in pattern: ${pattern}`)
   }
-  const out: string[] = []
+  const out: GlobMatch[] = []
 
-  const visit = (dirAbs: string, dirRel: string, index: number): void => {
+  const visit = (dirAbs: string, dirRel: string, index: number, viaSymlink: boolean): void => {
     if (index >= segments.length) return
     if (segments[index] !== '**') {
-      visitSegment(dirAbs, dirRel, index)
+      visitSegment(dirAbs, dirRel, index, viaSymlink)
       return
     }
     // `**` consumes zero directories here and one directory per recursion.
-    visitSegment(dirAbs, dirRel, index + 1)
-    for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
+    visitSegment(dirAbs, dirRel, index + 1, viaSymlink)
+    for (const entry of listDirectory(dirAbs)) {
       if (entry.name.startsWith('.') || !entry.isDirectory()) continue
-      visit(join(dirAbs, entry.name), dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`, index)
+      const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`
+      if (skipDirectory(childRel)) continue
+      visit(join(dirAbs, entry.name), childRel, index, viaSymlink)
     }
   }
 
-  const visitSegment = (dirAbs: string, dirRel: string, index: number): void => {
+  const visitSegment = (dirAbs: string, dirRel: string, index: number, viaSymlink: boolean): void => {
     if (index >= segments.length) return
     const segment = segments[index]
     if (segment === undefined) return
     if (segment === '**') {
-      visit(dirAbs, dirRel, index)
+      visit(dirAbs, dirRel, index, viaSymlink)
       return
     }
     const last = index === segments.length - 1
-    for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
+    for (const entry of listDirectory(dirAbs)) {
       if (!segmentMatches(segment, entry.name)) continue
       const childAbs = join(dirAbs, entry.name)
       const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`
@@ -160,11 +179,13 @@ function expandGlob(root: string, pattern: string): string[] {
         // Files and symlinks match; a broken or cyclic symlink then fails
         // loudly in the caller's realpathSync exactly as node glob did,
         // rather than silently shrinking the scanned corpus.
-        if (entry.isFile() || entry.isSymbolicLink()) out.push(childRel)
+        if (entry.isFile() || entry.isSymbolicLink()) {
+          out.push({ path: childRel, viaSymlink: viaSymlink || !entry.isFile() })
+        }
         continue
       }
       if (entry.isDirectory()) {
-        visitSegment(childAbs, childRel, index + 1)
+        if (!skipDirectory(childRel)) visitSegment(childAbs, childRel, index + 1, viaSymlink)
         continue
       }
       // A literal non-final segment resolves through stat like node glob's,
@@ -179,13 +200,13 @@ function expandGlob(root: string, pattern: string): string[] {
         // glob silently returns no match for it.
         continue
       }
-      if (!target.isDirectory()) continue
-      visitSegment(childAbs, childRel, index + 1)
+      if (!target.isDirectory() || skipDirectory(childRel)) continue
+      visitSegment(childAbs, childRel, index + 1, true)
     }
   }
 
-  visit(root, '.', 0)
-  return out.sort()
+  visit(root, '.', 0, false)
+  return out.sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0)
 }
 
 /** Whether a pattern segment contains `*` or `?` and is therefore wildcard. */
@@ -195,23 +216,40 @@ function hasWildcard(segment: string): boolean {
 
 /**
  * Expand repository-relative globs and deduplicate symlinked files.
+ * Each directory is listed at most once per call, shared across patterns.
+ * A regular file reached only through real directories canonicalizes to
+ * `realpathSync(root)` joined with its relative path; every other match goes
+ * through `realpathSync`, so a broken or cyclic symlink still throws.
  * @param root - absolute repository root.
  * @param patterns - repository-relative glob patterns, processed in order.
  * @param isExcluded - optional predicate over each matched relative path.
+ * @param skipDirectory - optional predicate over each repository-relative directory before the walker
+ *   lists it; return true only when `isExcluded` holds for every path below that directory.
  * @returns matched files in stable first-seen order.
  */
 export function uniqueRepoFiles(
   root: string,
   patterns: readonly string[],
   isExcluded: (relativePath: string) => boolean = () => false,
+  skipDirectory: (relativeDirectory: string) => boolean = () => false,
 ): RepoFile[] {
+  const listings = new Map<string, readonly Dirent[]>()
+  const listDirectory = (dirAbs: string): readonly Dirent[] => {
+    let entries = listings.get(dirAbs)
+    if (entries === undefined) {
+      entries = readdirSync(dirAbs, { withFileTypes: true })
+      listings.set(dirAbs, entries)
+    }
+    return entries
+  }
+  let realRoot: string | undefined
   const seen = new Set<string>()
   const files: RepoFile[] = []
   for (const pattern of patterns) {
-    for (const repoPath of expandGlob(root, pattern)) {
+    for (const { path: repoPath, viaSymlink } of expandGlob(root, pattern, listDirectory, skipDirectory)) {
       if (isExcluded(repoPath)) continue
       const abs = resolve(root, repoPath)
-      const real = realpathSync(abs)
+      const real = viaSymlink ? realpathSync(abs) : join(realRoot ??= realpathSync(root), repoPath)
       if (seen.has(real)) continue
       seen.add(real)
       files.push({ abs, real })

scripts/verify-doc-refs.ts

@@ -18,6 +18,9 @@ const PATTERNS = ['packages/**/*.ts']
 const isExcluded = (p: string): boolean =>
   p.includes('/lib/') || p.endsWith('.d.ts') || p.startsWith('vendor/')
 
+/** A non-first `lib` segment puts `/lib/` in every path below the directory, so the walker skips it. */
+const isExcludedDirectory = (dir: string): boolean => dir.split('/').slice(1).includes('lib')
+
 /** Root-relative Markdown path token, excluding trailing prose. */
 const DOC_REF = /(?:\bdocs|\.agents\/notes)\/[A-Za-z0-9._/-]+\.md/g
 
@@ -26,7 +29,7 @@ function findViolations(absPath: string): Violation[] {
   return findReferenceViolations(root, absPath, DOC_REF, ref => ref, ref => !existsSync(resolve(root, ref)))
 }
 
-const files = uniqueRepoFiles(root, PATTERNS, isExcluded)
+const files = uniqueRepoFiles(root, PATTERNS, isExcluded, isExcludedDirectory)
 const all = files.flatMap(file => findViolations(file.abs))
 const checked = files.length