diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 39ebd5cd9c..d1766b9e80 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -74,4 +74,4 @@ Hunt these in any doc; [dsh-doc](../.agents/skills/dsh-doc/SKILL.md) runs this l ## Repository references -Use relative Markdown links for current files and tags or PR numbers for historical references. `verify-md-links` checks local targets. [Reference validation](../scripts/verify-repository-references.ts) rejects actual commit identifiers and disallowed organization URLs in maintained files. +Use relative Markdown links for current files and tags or PR numbers for historical references. `verify-md-links` checks local targets. [Reference validation](../scripts/verify-repository-references.ts) rejects commit identifiers containing a hexadecimal letter and disallowed organization URLs in maintained files. Decimal-only tokens are ignored to avoid collisions with numeric data, even when they identify commits. diff --git a/scripts/verify-repository-references.spec.ts b/scripts/verify-repository-references.spec.ts index 0cead459ca..2c53c1d526 100644 --- a/scripts/verify-repository-references.spec.ts +++ b/scripts/verify-repository-references.spec.ts @@ -1,10 +1,10 @@ import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { mkdirSync, mkdtempSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { describe, expect, it, type TestContext } from 'vitest' -import { findRepositoryReferences, scanRepositoryReferences, uniquePrefixMatch } from './verify-repository-references.ts' +import { findRepositoryReferences, repositoryCommits, scanRepositoryReferences, uniquePrefixMatch } from './verify-repository-references.ts' const organizationUrl = `https://${['github.com', ['deepseek', 'harness'].join('-')].join('/')}` @@ -51,6 +51,7 @@ function collidingCommitBodies(tree: string, length: number): [string, string] { for (let index = 0; ; index += 1) { const body = `tree ${tree}\nauthor T 0 +0000\ncommitter T 0 +0000\n\ncollision ${String(index)}\n` const prefix = createHash(algorithm).update(`commit ${String(Buffer.byteLength(body))}\0${body}`).digest('hex').slice(0, length) + if (!/[a-f]/.test(prefix)) continue const previous = seen.get(prefix) if (previous !== undefined) return [previous, body] seen.set(prefix, body) @@ -74,50 +75,73 @@ describe('unique object-name prefix search', () => { describe('maintained repository reference policy', () => { const numericCommit = String(6_344_383) - it.each(['uv.lock', 'python/sdk/uv.lock'])( - 'accepts artifact byte counts in %s that match a numeric commit prefix', (file) => { - const source = [ - `sdist = { url = "https://example.invalid/source.tar.gz", size = ${numericCommit} }`, - 'wheels = [', - ` { url = "https://example.invalid/package.whl", size = ${numericCommit}, hash = "sha256:${'a'.repeat(64)}" },`, - ']', - ].join('\n') - expect(findRepositoryReferences(file, source, new Set([numericCommit]))).toEqual([]) - }, - ) + it.each([ + ['uv.lock', `artifact = { size = ${numericCommit} }`], + ['python/sdk/uv.lock', `sdist.size = ${numericCommit}`], + ['fixture.json', `{"context_length": ${numericCommit}}`], + ['package.json', `{"version": "1.0.${numericCommit}"}`], + ['config.toml', `rev = "${numericCommit}"`], + ['notes.md', `commit ${numericCommit}\nhttps://example.invalid/commit/${numericCommit}`], + ['source.ts', `// commit ${numericCommit}`], + ])('accepts decimal-only tokens in %s even when they name a commit', (file, source) => { + expect(findRepositoryReferences(file, source, new Set([numericCommit]))).toEqual([]) + }) + + it.each([7, 12, 40])('accepts a decimal-only %i-character token beside a hexadecimal commit', (length) => { + const decimal = '1'.repeat(length) + const hexadecimal = `${'1'.repeat(length - 1)}a` + expect(findRepositoryReferences('source.md', `${decimal} ${hexadecimal}`, new Set([decimal, hexadecimal]))).toEqual([ + { file: 'source.md', line: 1, kind: 'commit-hash' }, + ]) + expect(findRepositoryReferences('source.md', decimal, new Set([decimal]))).toEqual([]) + }) it.each([ - `source = { git = "https://example.invalid/repo?rev=${numericCommit}" }`, - `size = "${numericCommit}"`, - `size = '${numericCommit}'`, - `description = "size = ${numericCommit}"`, - `description = 'size = ${numericCommit}'`, - `description = "escaped \\" quote, size = ${numericCommit}"`, - `description = """\nsize = ${numericCommit}\n"""`, - `description = '''\nsize = ${numericCommit}\n'''`, - `# size = ${numericCommit}`, - `size = ${numericCommit} # commit ${numericCommit}`, - `artifact = { size = ${numericCommit}, rev = "${numericCommit}" }`, - ])('rejects numeric commit references in uv.lock strings and comments: %s', (source) => { - const line = source.startsWith('description = """') || source.startsWith("description = '''") ? 2 : 1 - expect(findRepositoryReferences('python/sdk/uv.lock', source, new Set([numericCommit]))).toEqual([ - { file: 'python/sdk/uv.lock', line, kind: 'commit-hash' }, + ['python/sdk/uv.lock', 'size = "HASH"', 1], + ['fixture.json', '{"revision": "HASH"}', 1], + ['notes.md', 'release\ncommit HASH', 2], + ['source.ts', '// commit HASH', 1], + ] as const)('rejects commit tokens containing a hex letter in %s', (file, template, line) => { + const hash = `${numericCommit}a` + expect(findRepositoryReferences(file, template.replace('HASH', hash), new Set([hash]))).toEqual([ + { file, line, kind: 'commit-hash' }, ]) }) - it('keeps checking other uv.lock fields, organization URLs, and other file types', () => { - const commits = new Set([numericCommit]) - expect(findRepositoryReferences('uv.lock', `revision = ${numericCommit}`, commits)).toEqual([ - { file: 'uv.lock', line: 1, kind: 'commit-hash' }, - ]) - expect(findRepositoryReferences('notes.md', `size = ${numericCommit}`, commits)).toEqual([ - { file: 'notes.md', line: 1, kind: 'commit-hash' }, - ]) - expect(findRepositoryReferences('uv.lock', `artifact = { size = ${numericCommit}, url = "${organizationUrl}" }`, commits)).toEqual([ - { file: 'uv.lock', line: 1, kind: 'organization-url' }, + it('accepts byte counts in the real lockfile and still rejects references on added lines', () => { + const file = 'python/sdk/uv.lock' + const source = readFileSync(join(import.meta.dirname, '..', file), 'utf8').trimEnd() + const sizes = new Set([...source.matchAll(/\bsize\s*=\s*(\d+)/g)].map(match => match[1] ?? '')) + expect(sizes.size).toBeGreaterThan(0) + expect(sizes).not.toContain('') + expect(findRepositoryReferences(file, source, sizes)).toEqual([]) + const hash = `${numericCommit}a` + expect(findRepositoryReferences(file, `${source}\n# commit ${hash}\n# ${organizationUrl}`, new Set([...sizes, hash]))).toEqual([ + { file, line: source.split('\n').length + 1, kind: 'commit-hash' }, + { file, line: source.split('\n').length + 2, kind: 'organization-url' }, ]) }) + it('excludes a real numeric commit prefix from both candidate resolution and file findings', (test) => { + const fixture = repository(test) + const algorithm = fixture.tree.length === 64 ? 'sha256' : 'sha1' + for (let index = 0; ; index += 1) { + const body = `tree ${fixture.tree}\nauthor T 0 +0000\ncommitter T 0 +0000\n\nnumeric ${String(index)}\n` + const commit = createHash(algorithm).update(`commit ${String(Buffer.byteLength(body))}\0${body}`).digest('hex') + if (!/^\d{7}/.test(commit) || !/[a-f]/.test(commit.slice(0, 40))) continue + fixture.git(['hash-object', '-t', 'commit', '-w', '--stdin'], body) + const decimal = commit.slice(0, 7) + const hexadecimal = commit.slice(0, Math.max(8, commit.search(/[a-f]/) + 1)) + const source = `commit ${decimal}\ncommit ${hexadecimal}` + expect(repositoryCommits(fixture.root, [source])).toEqual(new Set([hexadecimal])) + fixture.write('references.md', source) + expect(scanRepositoryReferences(fixture.root)).toEqual([ + { file: 'references.md', line: 2, kind: 'commit-hash' }, + ]) + return + } + }) + it('permits only the independent kit repository and its source URLs', () => { for (const suffix of ['', '.git', '/tree/main/packages/entry']) { expect(findRepositoryReferences('package.json', `${organizationUrl}/libreoffice-kit${suffix}`, new Set())).toEqual([]) @@ -130,9 +154,10 @@ describe('maintained repository reference policy', () => { it('rejects complete and abbreviated commit identifiers in tracked, staged, and new files', (test) => { const fixture = repository(test) fixture.write('tracked.md', `release\n${fixture.commit}\n${fixture.commit.toUpperCase()}\n`) - fixture.write('staged.md', fixture.commit.slice(0, 7)) + const firstLetterEnd = fixture.commit.search(/[a-f]/) + 1 + fixture.write('staged.md', fixture.commit.slice(0, Math.max(7, firstLetterEnd))) fixture.git(['add', 'staged.md']) - fixture.write('new.md', `\0${fixture.commit.slice(0, 12)}`) + fixture.write('new.md', `\0${fixture.commit.slice(0, Math.max(12, firstLetterEnd))}`) expect(scanRepositoryReferences(fixture.root)).toEqual(expect.arrayContaining([ { file: 'tracked.md', line: 2, kind: 'commit-hash' }, diff --git a/scripts/verify-repository-references.ts b/scripts/verify-repository-references.ts index 45853b1b7c..2781c5e0a1 100644 --- a/scripts/verify-repository-references.ts +++ b/scripts/verify-repository-references.ts @@ -2,7 +2,7 @@ import { execFileSync } from 'node:child_process' import { lstatSync, readFileSync, readlinkSync } from 'node:fs' -import { basename, resolve } from 'node:path' +import { resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { canonicalReferenceText } from './verify-public-repository-links.ts' @@ -11,13 +11,8 @@ const organization = ['deepseek', 'harness'].join('-') const organizationUrl = new RegExp(`\\bgithub\\.com/${organization}(?![a-z0-9-])`) // The independent kit repository owns the engine source and documentation. const kitRepositoryUrl = new RegExp(`\\bgithub\\.com/${organization}/libreoffice-kit(?:\\.git)?(?=/|[^a-zA-Z0-9_.-]|$)`, 'g') -const commitCandidate = /(? - bytes === undefined ? token : token.slice(0, -bytes.length) + ' '.repeat(bytes.length)) - : source - for (const [index, line] of referenceText.split('\n').entries()) { + for (const [index, line] of source.split('\n').entries()) { if (organizationUrl.test(canonicalReferenceText(line).replace(kitRepositoryUrl, ''))) { references.push({ file, line: index + 1, kind: 'organization-url' }) } @@ -118,7 +109,7 @@ export function uniquePrefixMatch(objects: readonly string[], prefix: string): s * Resolve commit-identifier candidates in maintained sources against the local object database. * @param repoRoot - Working tree whose Git objects are inspected. * @param sources - File texts or symlink targets to extract candidates from. - * @returns Lowercase candidates that name exactly one available object, which is a commit. + * @returns Lowercase candidates containing a hexadecimal letter that name exactly one available object, which is a commit. */ export function repositoryCommits(repoRoot: string, sources: Iterable): Set { const candidates = [...new Set([...sources].flatMap(source => @@ -157,7 +148,7 @@ const invokedPath = process.argv[1] if (invokedPath !== undefined && import.meta.url === pathToFileURL(resolve(invokedPath)).href) { const references = scanRepositoryReferences(root) if (references.length === 0) { - console.log('verify-repository-references: maintained files contain no repository commit identifiers or disallowed organization URLs.') + console.log('verify-repository-references: maintained files contain no prohibited references (decimal-only tokens are ignored).') } else { console.error('verify-repository-references: use release tags or maintained repository links:') for (const { file, line, kind } of references) console.error(` ${file}:${String(line)} ${kind}`)