test(web): run fixture browser cases through Host

This commit is contained in:
imccyu
2026-09-13 21:30:02 +08:00
parent 9b091919e2
commit eb2cb668a5
4 changed files with 143 additions and 63 deletions
+123 -41
View File
@@ -1,22 +1,29 @@
/** /**
* Opt-in browser stress reproduction for reasoning-stream renderer stalls. * Opt-in browser stress reproduction for reasoning-stream renderer stalls.
* The fixture emits 100,000 individual chunks through the normal async * A test-owned model adapter emits 100,000 individual chunks through the real
* carrier; the test measures event-loop and scheduled-interaction delay while * Host, Gateway, and browser carriers; the test measures event-loop and
* the assembled React surface keeps a collapsed Think row live. * scheduled-interaction delay while the assembled React surface keeps a
* collapsed Think row live.
*/ */
import type { Browser, Page } from 'playwright' import type { Browser, Page } from 'playwright'
import { chromium } from 'playwright' import { chromium } from 'playwright'
import { expect, it, onTestFailed } from 'vitest' import { expect, it, onTestFailed } from 'vitest'
import { LlmAdapter, type GenerateOptions, type StreamChunk } from '@deepseek-ai/dsh-llm'
import type {} from '@deepseek-ai/dsh-agent-default-model'
import { launchWebScaffold, watchConsole, type WebScaffold } from '../tests/scaffold.ts' import { launchWebScaffold, watchConsole, type WebScaffold } from '../tests/scaffold.ts'
import { newEnglishPage, saveFailureShot } from '../tests/support.ts' import {
connectFreshWorkspace, newEnglishPage, saveFailureShot, writeComposerDraft,
} from '../tests/support.ts'
const CHUNK_COUNT = 100_000 const CHUNK_COUNT = 100_000
const CHUNKS_PER_INTERVAL = 128 const CHUNKS_PER_INTERVAL = 128
const CHUNK_INTERVAL_MS = 16 const CHUNK_INTERVAL_MS = 16
const MAIN_THREAD_DELAY_BUDGET_MS = 250 const MAIN_THREAD_DELAY_BUDGET_MS = 250
const PROVIDER = 'reasoning-stress-test'
const MODEL = 'reasoning-storm'
const MARKER = `REASONING_STRESS_COMPLETE:${String(CHUNK_COUNT)}`
interface ReasoningChunkStormState { interface ReasoningChunkStormState {
sessionId: string
chunkCount: number chunkCount: number
chunksPerInterval: number chunksPerInterval: number
intervalMs: number intervalMs: number
@@ -25,6 +32,86 @@ interface ReasoningChunkStormState {
emitting: boolean emitting: boolean
} }
/** Externally paced model stream whose counters are observed outside the browser. */
class ReasoningStressAdapter extends LlmAdapter {
private finishStream!: () => void
private resolveStarted!: () => void
private resolveCompleted!: () => void
private releaseBatch: (() => void) | undefined
private resolveBatch: (() => void) | undefined
private batchInFlight = false
private finishRequested = false
private readonly finishGate = new Promise<void>((resolve) => { this.finishStream = resolve })
readonly started = new Promise<void>((resolve) => { this.resolveStarted = resolve })
readonly completed = new Promise<void>((resolve) => { this.resolveCompleted = resolve })
private emitted = 0
private emitting = false
finish(): void {
this.finishRequested = true
this.releaseBatch?.()
this.finishStream()
}
async emitNextBatch(): Promise<void> {
if (!this.emitting || this.releaseBatch === undefined) {
throw new Error('reasoning stress adapter has no batch awaiting release')
}
if (this.batchInFlight) throw new Error('reasoning stress adapter accepts one batch release at a time')
this.batchInFlight = true
const delivered = new Promise<void>((resolve) => { this.resolveBatch = resolve })
const release = this.releaseBatch
this.releaseBatch = undefined
release()
await delivered
}
state(): ReasoningChunkStormState {
return {
chunkCount: CHUNK_COUNT,
chunksPerInterval: CHUNKS_PER_INTERVAL,
intervalMs: CHUNK_INTERVAL_MS,
emitted: this.emitted,
marker: MARKER,
emitting: this.emitting,
}
}
override async *stream(options: GenerateOptions): AsyncIterable<StreamChunk> {
if (this.emitting || this.emitted !== 0) throw new Error('reasoning stress adapter accepts one model call')
this.emitting = true
const parts: string[] = []
yield { type: 'block-start', index: 0, blockType: 'reasoning' }
while (this.emitted < CHUNK_COUNT) {
await new Promise<void>((resolve) => {
this.releaseBatch = resolve
if (this.emitted === 0) this.resolveStarted()
if (this.finishRequested) resolve()
})
if (this.finishRequested) break
options.signal?.throwIfAborted()
const end = Math.min(this.emitted + CHUNKS_PER_INTERVAL, CHUNK_COUNT)
while (this.emitted < end) {
const text = this.emitted === CHUNK_COUNT - 1
? `\n${MARKER}`
: this.emitted % 64 === 63 ? '推理\n' : '推理'
parts.push(text)
yield { type: 'reasoning-delta', index: 0, text }
this.emitted += 1
}
this.batchInFlight = false
this.resolveBatch?.()
this.resolveBatch = undefined
}
this.emitting = false
if (this.emitted === CHUNK_COUNT) this.resolveCompleted()
await this.finishGate
options.signal?.throwIfAborted()
yield { type: 'block-end', index: 0, block: { type: 'reasoning', text: parts.join('') } }
yield { type: 'finish', reason: { kind: 'stop' } }
}
}
interface StressProbe { interface StressProbe {
intervalId: number intervalId: number
intervalMs: number intervalMs: number
@@ -36,10 +123,6 @@ interface StressProbe {
} }
interface StressWindow extends Window { interface StressWindow extends Window {
__fxTiming?: {
startReasoningChunkStorm(id: string, chunkCount: number, chunksPerInterval: number, intervalMs: number): string
reasoningChunkStormState(): ReasoningChunkStormState | null
}
__reasoningStressProbe?: StressProbe __reasoningStressProbe?: StressProbe
} }
@@ -47,23 +130,22 @@ it('keeps the browser responsive while rendering 100,000 reasoning chunks', asyn
let scaffold: WebScaffold | undefined let scaffold: WebScaffold | undefined
let browser: Browser | undefined let browser: Browser | undefined
let page: Page | undefined let page: Page | undefined
const adapter = new ReasoningStressAdapter()
try { try {
scaffold = await launchWebScaffold() scaffold = await launchWebScaffold()
scaffold.ctx.effect(
() => scaffold!.ctx.llm.registerAdapter([PROVIDER], adapter),
'reasoning stress adapter',
)
await scaffold.ctx.agentDefaultModel.saveSelection({ provider: PROVIDER, model: MODEL })
browser = await chromium.launch({ headless: process.env.DSH_WEB_STRESS_HEADFUL !== '1' }) browser = await chromium.launch({ headless: process.env.DSH_WEB_STRESS_HEADFUL !== '1' })
page = await newEnglishPage(browser) page = await newEnglishPage(browser)
const activePage = page const activePage = page
await activePage.addInitScript(() => {
localStorage.setItem('dsh.sessions.current', JSON.stringify({ sessionId: 'fx-alpha' }))
})
const tripwire = watchConsole(activePage) const tripwire = watchConsole(activePage)
onTestFailed(() => saveFailureShot(activePage, 'web-stress-reasoning-chunks')) onTestFailed(() => saveFailureShot(activePage, 'web-stress-reasoning-chunks'))
await activePage.goto(`${scaffold.baseUrl}?fixture`, { waitUntil: 'load' }) await activePage.goto(scaffold.authenticatedUrl, { waitUntil: 'load' })
await activePage.waitForSelector('[class*="frame"]', { timeout: 30_000 }) await activePage.waitForSelector('[class*="frame"]', { timeout: 30_000 })
// Fixture settings deliberately reject writes, so its welcome notice await connectFreshWorkspace(activePage, scaffold.workspaceCwd)
// cannot acknowledge. Hide only that test overlay; the assembled chat
// tree beneath it remains mounted and exercises the production renderer.
await activePage.addStyleTag({ content: '[class*="onboardingOverlay"] { display: none !important; }' })
await activePage.locator('[data-sample="bash"]').first().waitFor({ timeout: 30_000 })
await activePage.evaluate(() => { await activePage.evaluate(() => {
const intervalMs = 50 const intervalMs = 50
@@ -92,45 +174,42 @@ it('keeps the browser responsive while rendering 100,000 reasoning chunks', asyn
;(window as StressWindow).__reasoningStressProbe = probe ;(window as StressWindow).__reasoningStressProbe = probe
}) })
const marker = await activePage.evaluate(({ chunkCount, chunksPerInterval, intervalMs }) => { const settled = scaffold.whenTurnSettled(540_000)
const hooks = (window as StressWindow).__fxTiming const input = activePage.locator('[data-composer-input]').first()
if (hooks === undefined) throw new Error('reasoning stress fixture hooks unavailable') await writeComposerDraft(activePage, input, `Render ${String(CHUNK_COUNT)} reasoning chunks.`)
return hooks.startReasoningChunkStorm('fx-alpha', chunkCount, chunksPerInterval, intervalMs) await input.press('Enter')
}, { await adapter.started
chunkCount: CHUNK_COUNT,
chunksPerInterval: CHUNKS_PER_INTERVAL,
intervalMs: CHUNK_INTERVAL_MS,
})
await adapter.emitNextBatch()
const liveThink = activePage.locator('[data-variant="think"][data-state="running"]').last() const liveThink = activePage.locator('[data-variant="think"][data-state="running"]').last()
await liveThink.waitFor({ timeout: 60_000 }) await liveThink.waitFor({ timeout: 60_000 })
await expect.poll(async () => await activePage.evaluate(() => { await activePage.evaluate(intervalMs => new Promise<void>((resolve) => {
const hooks = (window as StressWindow).__fxTiming window.setTimeout(resolve, intervalMs)
return hooks?.reasoningChunkStormState()?.emitted ?? 0 }), CHUNK_INTERVAL_MS)
}), { timeout: 540_000, interval: 100 }).toBe(CHUNK_COUNT) while (adapter.state().emitted < CHUNK_COUNT) {
await expect.poll(() => liveThink.textContent(), { timeout: 60_000, interval: 100 }).toContain(marker) await adapter.emitNextBatch()
await activePage.evaluate(intervalMs => new Promise<void>((resolve) => {
window.setTimeout(resolve, intervalMs)
}), CHUNK_INTERVAL_MS)
}
await adapter.completed
await expect.poll(() => liveThink.textContent(), { timeout: 60_000, interval: 100 }).toContain(MARKER)
const report = await activePage.evaluate(() => { const browserReport = await activePage.evaluate(() => {
const win = window as StressWindow const win = window as StressWindow
const probe = win.__reasoningStressProbe const probe = win.__reasoningStressProbe
const state = win.__fxTiming?.reasoningChunkStormState() if (probe === undefined) throw new Error('reasoning stress metrics unavailable')
if (probe === undefined || state === undefined || state === null) {
throw new Error('reasoning stress metrics unavailable')
}
window.clearInterval(probe.intervalId) window.clearInterval(probe.intervalId)
const interactionDelayMs = probe.interactionHandledAt === null const interactionDelayMs = probe.interactionHandledAt === null
? null ? null
: probe.interactionHandledAt - probe.interactionDueAt : probe.interactionHandledAt - probe.interactionDueAt
return { return {
chunkCount: state.chunkCount,
chunksPerInterval: state.chunksPerInterval,
intervalMs: state.intervalMs,
emitted: state.emitted,
maxMainThreadDelayMs: Math.max(0, probe.maxDelayMs), maxMainThreadDelayMs: Math.max(0, probe.maxDelayMs),
interactionDelayMs, interactionDelayMs,
heartbeatSamples: probe.samples, heartbeatSamples: probe.samples,
} }
}) })
const report = { ...adapter.state(), ...browserReport }
process.stdout.write(`reasoning-chunk stress report: ${JSON.stringify(report)}\n`) process.stdout.write(`reasoning-chunk stress report: ${JSON.stringify(report)}\n`)
expect(report).toMatchObject({ expect(report).toMatchObject({
@@ -146,7 +225,10 @@ it('keeps the browser responsive while rendering 100,000 reasoning chunks', asyn
expect(interactionDelayMs, JSON.stringify(report)).toBeLessThan(MAIN_THREAD_DELAY_BUDGET_MS) expect(interactionDelayMs, JSON.stringify(report)).toBeLessThan(MAIN_THREAD_DELAY_BUDGET_MS)
expect(tripwire.pageErrors).toEqual([]) expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([]) expect(tripwire.warnings).toEqual([])
adapter.finish()
await settled
} finally { } finally {
adapter.finish()
await browser?.close() await browser?.close()
await scaffold?.close() await scaffold?.close()
} }
+15 -16
View File
@@ -1,18 +1,19 @@
// Keyless assembled-browser coverage for the goal bar over the shipped Web // Keyless browser coverage for the goal bar over the shipped Web composition.
// bundles and the fixture Connection RPC. The command creates a real projected // The command creates a real projected goal in a real Host session. The
// goal in the fixture session; the golden pins the active strip, while the // goldens pin the active and disarmed strips, while the clear gesture proves
// clear gesture proves the acknowledged tombstone leaves neither stale chrome // the acknowledged tombstone leaves neither stale chrome nor a
// nor a duplicate-mutation error. // duplicate-mutation error.
import { fileURLToPath } from 'node:url' import { fileURLToPath } from 'node:url'
import { join } from 'node:path' import { join } from 'node:path'
import type { Browser, Page } from 'playwright' import type { Browser, Page } from 'playwright'
import { chromium } from 'playwright' import { chromium } from 'playwright'
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest' import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import type {} from '@deepseek-ai/dsh-goal'
import { import {
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, assertFixtureInventory, captureStableAria, compareOrRefreshGolden,
launchWebScaffold, watchConsole, webSnapshotMode, type WebScaffold, launchWebScaffold, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts' } from './scaffold.ts'
import { newEnglishPage, saveFailureShot } from './support.ts' import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts'
const SNAPSHOT_DIR = fileURLToPath(new URL('./expected/goal-bar', import.meta.url)) const SNAPSHOT_DIR = fileURLToPath(new URL('./expected/goal-bar', import.meta.url))
const ACTIVE_EXPECTED = join(SNAPSHOT_DIR, 'active.expected.md') const ACTIVE_EXPECTED = join(SNAPSHOT_DIR, 'active.expected.md')
@@ -27,14 +28,13 @@ describe('web e2e: goal bar clear convergence', () => {
let tripwire: ReturnType<typeof watchConsole> let tripwire: ReturnType<typeof watchConsole>
beforeAll(async () => { beforeAll(async () => {
scaffold = await launchWebScaffold({ extraOverlayPath: OVERLAY, welcomeNoticePending: true }) scaffold = await launchWebScaffold({ extraOverlayPath: OVERLAY })
browser = await chromium.launch() browser = await chromium.launch()
page = await newEnglishPage(browser) page = await newEnglishPage(browser)
tripwire = watchConsole(page) tripwire = watchConsole(page)
const login = await page.context().request.get(scaffold.authenticatedUrl, { maxRedirects: 0 }) await page.goto(scaffold.authenticatedUrl, { waitUntil: 'load' })
expect(login.status()).toBe(303)
await page.goto(`${scaffold.baseUrl}?fixture`, { waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
await connectFreshWorkspace(page, scaffold.workspaceCwd)
}, 120_000) }, 120_000)
afterAll(async () => { afterAll(async () => {
@@ -44,8 +44,6 @@ describe('web e2e: goal bar clear convergence', () => {
it('renders one active goal and clears it without exposing a stale error', async () => { it('renders one active goal and clears it without exposing a stale error', async () => {
onTestFailed(() => saveFailureShot(page, 'web-e2e-goal-bar-clear')) onTestFailed(() => saveFailureShot(page, 'web-e2e-goal-bar-clear'))
// Startup reuses the fixture workspace's blank session, keeping this
// command independent of alpha's running replay and pending question.
const input = page.locator('[data-composer-input][data-placeholder="Describe what you want to build, / commands, @ files or sessions"]') const input = page.locator('[data-composer-input][data-placeholder="Describe what you want to build, / commands, @ files or sessions"]')
await input.waitFor({ timeout: 10_000 }) await input.waitFor({ timeout: 10_000 })
await input.fill('/goal guard rapid clear clicks') await input.fill('/goal guard rapid clear clicks')
@@ -53,15 +51,16 @@ describe('web e2e: goal bar clear convergence', () => {
const bar = page.locator('[data-goal-bar]') const bar = page.locator('[data-goal-bar]')
await bar.waitFor({ timeout: 10_000 }) await bar.waitFor({ timeout: 10_000 })
await expect.poll(() => bar.getByRole('button', { name: 'Pause goal' }).count(), { const pause = bar.getByRole('button', { name: 'Pause goal' })
await expect.poll(() => pause.count(), {
timeout: 10_000, timeout: 10_000,
}).toBe(1) }).toBe(1)
const snapshot = await captureStableAria(page, '[data-goal-bar]', scaffold.workspaceCwd) const snapshot = await captureStableAria(page, '[data-goal-bar]', scaffold.workspaceCwd)
await compareOrRefreshGolden(ACTIVE_EXPECTED, snapshot, MODE) await compareOrRefreshGolden(ACTIVE_EXPECTED, snapshot, MODE)
await page.evaluate(() => { const agents = scaffold.ctx.agents.list()
(globalThis as unknown as { __fxTiming?: { disarmOnlyGoal(): void } }).__fxTiming?.disarmOnlyGoal() expect(agents).toHaveLength(1)
}) scaffold.ctx.goals.disarm(agents[0]!)
await expect.poll(() => bar.getByRole('button', { name: 'Resume goal' }).count(), { await expect.poll(() => bar.getByRole('button', { name: 'Resume goal' }).count(), {
timeout: 10_000, timeout: 10_000,
}).toBe(1) }).toBe(1)
+3 -4
View File
@@ -1,5 +1,4 @@
# The client-side FixtureApiClient intentionally rejects settings traffic, so # This scenario owns Goal bar actions, not autonomous Goal rounds. Keeping the
# this goal-only scenario omits the settings shell and the onboarding steps it # driver out prevents an active Goal from issuing an unrelated model request.
# would mount. Onboarding owns separate assembled-browser coverage. - id: goal-round-driver
- id: ui-settings-general
disabled: true disabled: true
+2 -2
View File
@@ -1,10 +1,10 @@
import tsconfigPaths from 'vite-tsconfig-paths' import tsconfigPaths from 'vite-tsconfig-paths'
import { defineConfig } from 'vitest/config' import { defineConfig } from 'vitest/config'
import { vitestExecArgv } from './vitest.shared.ts' import { standardDecoratorPlugin, vitestExecArgv } from './vitest.shared.ts'
/** Opt-in browser performance lane; no default Vitest config includes *.stress.ts. */ /** Opt-in browser performance lane; no default Vitest config includes *.stress.ts. */
export default defineConfig({ export default defineConfig({
plugins: [tsconfigPaths({ projects: ['./tsconfig.base.json'] })], plugins: [tsconfigPaths({ projects: ['./tsconfig.base.json'] }), standardDecoratorPlugin()],
test: { test: {
execArgv: vitestExecArgv, execArgv: vitestExecArgv,
include: ['apps/web/stress-tests/**/*.stress.ts'], include: ['apps/web/stress-tests/**/*.stress.ts'],