From 5a17181af25c80263ff42845b5983b1457cf71d4 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Sat, 31 Oct 2026 01:32:46 +0800 Subject: [PATCH] perf(llm): avoid copying unchanged request histories --- ...backend-continuation-performance.i18n.yaml | 3 + ...-09-06-backend-continuation-performance.md | 18 ++++++ ...-06-backend-continuation-performance.zh.md | 18 ++++++ packages/llm/llm/README.i18n.yaml | 4 +- packages/llm/llm/README.md | 2 + packages/llm/llm/README.zh.md | 2 + packages/llm/llm/src/content.ts | 11 +++- packages/llm/llm/src/index.ts | 30 ++++++---- packages/llm/llm/tests/content.spec.ts | 13 ++++- packages/llm/llm/tests/service.spec.ts | 57 ++++++++++--------- 10 files changed, 114 insertions(+), 44 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml index fe9f28b0f2..3ca97fe5cf 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.i18n.yaml @@ -17,6 +17,9 @@ /agent-note-performance-baselines-for-tool-heavy-backend-continuation/calibration-evidence/sdk-minimal-checkpoint-calibration: en: f4b52851e35cec56 zh: b419546dda8abf21 +/agent-note-performance-baselines-for-tool-heavy-backend-continuation/calibration-evidence/unchanged-request-projection: + en: 10711f81b5eda63f + zh: 210ce3ffaee986f5 /agent-note-performance-baselines-for-tool-heavy-backend-continuation/alternatives-considered: en: 3a61dff7c926eb22 zh: 8f2633b16be78233 diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md index e1de8eec76..fe768d4ea5 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.md @@ -70,6 +70,24 @@ The upper observed run median is 5,432.567 ms; the slowest single sample is 5,94 This calibration accepts the measured cost of the stronger durability policy; it is not a performance improvement. Local on/off diagnosis attributes most additional persistence time to frequent file synchronization. The browser input-overlap failures in attempts 1 and 4 are separate failures and receive no exemption from this budget. Deterministic controls use the measured-verdict assertion: the recorded upper-median sample group fails the old 4,250 ms ceiling and passes 7,000 ms, while a synthetic 7,800 ms median remains rejected. +### Unchanged request projection + +Developer-message filtering and foreign replay-state removal inspect every request, allocating output arrays only after a message changes. Unchanged histories retain their array identity, and retained messages keep their identities. This removes discarded full-history arrays and a final equality scan without caching mutable inputs or adapter ownership. + +On macOS arm64 with Node 24.22.2, the compiled continuation worker uses five fresh processes per variant, each copying the same seeded history into a private temporary root. Every sample completes 40 requests and 13,925 events over 800 historical turns. The table reports total elapsed milliseconds, rounded to three decimal places; fixture setup, library builds and a separate CPU profile are outside these measurements. + +| Implementation | Five request-history totals (ms) | Median (ms) | +|---|---|---:| +| Original | 55.750, 52.999, 54.812, 53.478, 55.106 | 54.812 | +| Developer filtering only | 49.431, 49.826, 51.421, 50.679, 49.494 | 49.826 | +| Replay-state stripping only | 54.444, 53.043, 52.007, 52.776, 53.101 | 53.043 | +| Combined | 47.814, 47.326, 49.925, 47.867, 48.722 | 47.867 | +| Original restored | 55.670, 55.497, 56.265, 54.353, 58.718 | 55.670 | +| Combined restored | 48.423, 48.717, 48.199, 48.534, 50.129 | 48.534 | +| Combined, for-of traversal | 48.280, 48.631, 51.909, 48.966, 54.301 | 48.966 | + +The final total median is 12.0% below the restored original. Median process CPU time falls from 79.751 to 72.009 ms, while post-GC retained heap remains about 1.35 MiB. Whole-process maximum RSS is 201.61/202.52 MiB for the restored original/final samples. Source restoration and rebuilding isolate the removed cost; both local implementations remain below the unchanged 297 ms CI budget, so this comparison is not a budget-rejection control or proof of hosted-runner stability. Functional tests cover unchanged-array identity, leading/trailing developer removal, mixed replay-state ownership, retained message identity and frozen projection output. + ## Alternatives considered **Repeat existing migration and first-open variants.** Rejected: those twelve cases already distinguish read-only preparation from writable publication. These cases use the current generation and begin or continue actual model work, or enumerate a corpus rather than open one Session. diff --git a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md index cc96d63e95..ac4c313762 100644 --- a/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md +++ b/.agents/notes/implemented/testing/2026-09-06-backend-continuation-performance.zh.md @@ -70,6 +70,24 @@ SDK-minimal profile 在 step、模型请求和顶层工具执行前等待检查 此校准接受更强持久化策略的实测成本，不表示性能已改善。本地开关对照将额外持久化时间的大部分归因于频繁的文件同步。第 1、4 次尝试中的浏览器输入重叠失败属于独立问题，不因本预算而豁免。确定性对照与实测判定使用同一断言：记录的最高中位数样本组被旧 4,250 ms 上限拒绝，在 7,000 ms 下通过；合成的 7,800 ms 中位数仍被拒绝。 +### 未改变的请求投影 + +Developer 消息过滤和跨适配器 replay-state 移除检查每次请求，只在消息改变后分配输出数组。未改变的历史保留数组身份，保留的消息维持对象身份。这移除了被丢弃的完整历史数组和最后的相等性扫描，不缓存可变输入或适配器归属。 + +在 macOS arm64、Node 24.22.2 上，编译后的续聊 worker 对每种变体运行五个全新进程，每次将相同的种子历史复制到独立临时根目录。每个样本基于 800 个历史轮次完成 40 次请求和 13,925 个事件。下表记录总耗时毫秒数，四舍五入至小数点后三位；夹具准备、library 构建和单独的 CPU profile 不计入这些测量。 + +| 实现 | 五次请求历史总耗时（ms） | 中位数（ms） | +|---|---|---:| +| 原实现 | 55.750, 52.999, 54.812, 53.478, 55.106 | 54.812 | +| 仅 developer 过滤 | 49.431, 49.826, 51.421, 50.679, 49.494 | 49.826 | +| 仅 replay-state 移除 | 54.444, 53.043, 52.007, 52.776, 53.101 | 53.043 | +| 组合 | 47.814, 47.326, 49.925, 47.867, 48.722 | 47.867 | +| 恢复原实现 | 55.670, 55.497, 56.265, 54.353, 58.718 | 55.670 | +| 恢复组合 | 48.423, 48.717, 48.199, 48.534, 50.129 | 48.534 | +| 组合（for-of 遍历） | 48.280, 48.631, 51.909, 48.966, 54.301 | 48.966 | + +最终总耗时中位数比恢复后的原实现低 12.0%。进程 CPU 时间中位数从 79.751 降至 72.009 ms，GC 后保留堆内存仍约为 1.35 MiB。恢复后的原实现／最终样本的全进程最大 RSS 为 201.61／202.52 MiB。恢复源码并重建隔离了被移除的成本；两个本地实现都低于未改变的 297 ms CI 预算，因此该比较不是预算拒绝对照，也不能证明托管运行器稳定性。功能测试覆盖未改变数组的身份、首尾 developer 消息移除、混合 replay-state 归属、保留消息身份和冻结投影输出。 + ## 考虑过的替代方案 **重复现有迁移和首次打开变体。** 拒绝：现有十二个用例已经区分只读准备与可写发布。这些用例使用当前代际并开始或继续实际模型工作，或者列举语料集合而不是打开单个 Session。 diff --git a/packages/llm/llm/README.i18n.yaml b/packages/llm/llm/README.i18n.yaml index dc90c35071..f7a69ffba7 100644 --- a/packages/llm/llm/README.i18n.yaml +++ b/packages/llm/llm/README.i18n.yaml @@ -39,8 +39,8 @@ en: ae1618577f4e8364 zh: a6e8ab1563c08dfb /deepseek-ai-dsh-llm/understand-the-implementation/main-flow: - en: c95d4947745e39bb - zh: ec37b969f27629e2 + en: d141f1b3c38c7352 + zh: 7f6f472490054e47 /deepseek-ai-dsh-llm/understand-the-implementation/invariants: en: 0163a050239248e5 zh: 885c95bedcbe2ff4 diff --git a/packages/llm/llm/README.md b/packages/llm/llm/README.md index e2c319f53d..bbe607cf85 100644 --- a/packages/llm/llm/README.md +++ b/packages/llm/llm/README.md @@ -130,6 +130,8 @@ A request is validated against its exact model's capability — context window, File detection reads current content, including tool-role result content, on every request without caching message identities or freeze state. The [archived file-scan decision](../../../.agents/notes/archived/simplification/2026-09-07-file-content-scan.md) records the measured traversal cost. +Developer-message filtering and replay-state stripping reuse unchanged request histories. A new history array is allocated only when a message is removed or replaced; retained messages keep their identities, and projected frozen requests remain frozen. Each dispatch reads the current messages and adapter ownership. + ### Invariants - **Model-visible ⟺ logged** — callers must keep every provider request's model-visible input reconstructable from the session log; loop-built requests arrive deep-frozen and cannot be rewritten. diff --git a/packages/llm/llm/README.zh.md b/packages/llm/llm/README.zh.md index 2fd7de3175..e07f0530c6 100644 --- a/packages/llm/llm/README.zh.md +++ b/packages/llm/llm/README.zh.md @@ -130,6 +130,8 @@ const configure = (maxTokens: number): ConfigureCall => 文件检测在每次请求时读取当前内容，包括 tool-role 结果内容，不缓存消息身份或冻结状态。[已归档的文件扫描决策](../../../.agents/notes/archived/simplification/2026-09-07-file-content-scan.md)记录了实测遍历成本。 +Developer 消息过滤和 replay-state 移除复用未改变的请求历史。只有删除或替换消息时才分配新的历史数组；保留的消息维持对象身份，已冻结请求的投影仍保持冻结。每次分发都读取当前消息和适配器归属。 + ### 不变式 - **模型可见 ⟺ 已记录**——调用方必须确保每个提供方请求中的模型可见输入都可以从会话日志重建；loop 构建的请求以深度冻结状态到达，不可改写。 diff --git a/packages/llm/llm/src/content.ts b/packages/llm/llm/src/content.ts index cecbfba938..17230bef0b 100644 --- a/packages/llm/llm/src/content.ts +++ b/packages/llm/llm/src/content.ts @@ -374,8 +374,14 @@ export interface ProjectedToolUpdates { } function withoutDeveloperMessages(messages: readonly RequestMessage[]): readonly RequestMessage[] { - const retained = messages.filter(message => message.role !== 'developer') - return retained.length === messages.length ? messages : retained + let retained: RequestMessage[] | undefined + let index = 0 + for (const message of messages) { + if (message.role === 'developer') retained ??= messages.slice(0, index) + else retained?.push(message) + index++ + } + return retained ?? messages } function toolDeclarations( @@ -415,6 +421,7 @@ function toolDeclarations( * Construct provider declarations from session-folded history without changing logged active tools. * Unsupported routes and incomplete history use current declarations without developer updates. * Explicitly deferred baseline tools become available only after their first retained addition. + * Histories without removed or replaced messages retain their original array identity. * @param messages - complete request inputs, or the prefix selected for an auxiliary call. * @param tools - currently active tool schemas. * @param toolUpdate - the resolved route's update mode. diff --git a/packages/llm/llm/src/index.ts b/packages/llm/llm/src/index.ts index 14be9c87f8..ffd23c41ad 100644 --- a/packages/llm/llm/src/index.ts +++ b/packages/llm/llm/src/index.ts @@ -1023,19 +1023,25 @@ export class LlmRuntime extends TypertRemoteService { return registration } - /** Remove replay state whose historical route is owned by another adapter. */ + /** Remove replay state owned by another adapter, retaining unchanged requests and message identities. */ private forAdapter(options: GenerateOptions, adapter: LlmAdapter): GenerateOptions { - const messages: RequestMessage[] = options.messages.map((message) => { - if (message.role !== 'assistant') return message - const source = message.source - if (source.replayState === undefined) return message - if (this.adapters.get(source.provider)?.adapter === adapter) return message - return freezeMessage({ - ...message, - source: { kind: 'model', provider: source.provider, model: source.model }, - }) - }) - if (messages.every((message, index) => message === options.messages[index])) return options + let messages: RequestMessage[] | undefined + let index = 0 + for (const message of options.messages) { + if (message.role === 'assistant' && message.source.replayState !== undefined + && this.adapters.get(message.source.provider)?.adapter !== adapter) { + const source = message.source + messages ??= options.messages.slice(0, index) + messages.push(freezeMessage({ + ...message, + source: { kind: 'model', provider: source.provider, model: source.model }, + })) + } else { + messages?.push(message) + } + index++ + } + if (messages === undefined) return options const filtered = { ...options, messages } return Object.isFrozen(options) ? deepFreeze(filtered) : filtered } diff --git a/packages/llm/llm/tests/content.spec.ts b/packages/llm/llm/tests/content.spec.ts index 19bb84e460..f226da9007 100644 --- a/packages/llm/llm/tests/content.spec.ts +++ b/packages/llm/llm/tests/content.spec.ts @@ -410,7 +410,7 @@ describe('projectToolUpdates', () => { const roles = (messages: readonly RequestMessage[]) => messages.filter(message => message.role === 'developer').map(message => message.content) it('returns the input history and tools by identity when nothing is projected', () => { - const messages = [prompt] + const messages = Object.freeze([prompt]) const tools = [search] const projected = projectToolUpdates(messages, tools, 'in-history', { tools, updates: [] }) expect(projected.messages).toBe(messages) @@ -418,6 +418,17 @@ describe('projectToolUpdates', () => { expect(projectToolUpdates(messages, tools, undefined).messages).toBe(messages) }) + it.each(['first', 'last', 'all'] as const)('filters developer messages at %s positions without replacing retained messages', (position) => { + const added = developer([{ type: 'tool-addition', toolName: 'fetch' }]) + const messages = Object.freeze(position === 'first' ? [added, prompt] + : position === 'last' ? [prompt, added] : [added, added]) + const projected = projectToolUpdates(messages, [search], undefined) + expect(projected.messages).toEqual(position === 'all' ? [] : [prompt]) + expect(projected.messages).not.toBe(messages) + if (position !== 'all') expect(projected.messages[0]).toBe(prompt) + expect(messages).toHaveLength(2) + }) + it('strips deferred loading and developer updates when the route declares no mode', () => { const added = developer([{ type: 'tool-addition', toolName: 'fetch' }]) const history: ToolHistory = { tools: [search], updates: [{ messageId: added.id, additions: [fetch] }] } diff --git a/packages/llm/llm/tests/service.spec.ts b/packages/llm/llm/tests/service.spec.ts index a5d8be4413..d8e6880499 100644 --- a/packages/llm/llm/tests/service.spec.ts +++ b/packages/llm/llm/tests/service.spec.ts @@ -345,6 +345,7 @@ describe('LlmRuntime', () => { expect(projected, fixture.name).toMatchObject({ type: 'text' }) if (projected?.type !== 'text') throw new Error(`expected projected text for ${fixture.name}`) expect(projected.text, fixture.name).toContain(fixture.expected) + expect(ctx.llm.fileRequestText(attachment), fixture.name).toBe(projected.text) if (fixture.fs !== undefined) { expect(projected.text, fixture.name).toContain('include this saved path in the delegation prompt') } @@ -1500,56 +1501,58 @@ describe('LlmRuntime', () => { expect(assistant.source.replayState).toEqual({ private: 'state' }) }) - it('keeps replay state when historical and target providers belong to the same adapter instance', async () => { + it('keeps history by identity when historical and target providers belong to the same adapter instance', async () => { const ctx = new Context() await ctx.plugin(LlmRuntime) const adapter = new RecordingAdapter(SCRIPT) ctx.llm.registerAdapter(['historical', 'target'], adapter) const replayState = { private: 'state' } + const messages = [createMessage({ + role: 'assistant', + content: [{ type: 'text', text: 'old response' }], + source: { kind: 'model', provider: 'historical', model: 'old-model', replayState }, + })] for await (const _chunk of ctx.llm.stream({ - provider: 'target', - model: 'new-model', - messages: [createMessage({ - role: 'assistant', - content: [{ type: 'text', text: 'old response' }], - source: { - kind: 'model', - ...{ provider: 'historical', model: 'old-model', replayState }, - }, - })], + provider: 'target', model: 'new-model', messages, })) { /* drain */ } + expect(adapter.lastOptions?.messages).toBe(messages) expect(adapter.lastOptions?.messages[0]?.source).toEqual({ kind: 'model', provider: 'historical', model: 'old-model', replayState, }) }) - it('strips replay state but preserves provider and model when the target uses a different adapter instance', async () => { + it('strips foreign replay state while retaining prefix, intervening and suffix message identities', async () => { const ctx = new Context() await ctx.plugin(LlmRuntime) ctx.llm.registerAdapter(['historical'], new RecordingAdapter(SCRIPT)) const target = new RecordingAdapter(SCRIPT) ctx.llm.registerAdapter(['target'], target) + const prompt = createUserMessage({ source: { kind: 'user' }, content: [{ type: 'text', text: 'continue' }] }) + const assistant = (provider: string) => createMessage({ + role: 'assistant', + content: [{ type: 'text', text: 'old response' }], + source: { kind: 'model', provider, model: 'old-model', replayState: { private: 'state' } }, + }) + const foreign = assistant('historical') + const owned = assistant('target') + const messages = [prompt, foreign, owned, foreign, prompt] + Object.freeze(messages) for await (const _chunk of ctx.llm.stream({ - provider: 'target', - model: 'new-model', - messages: [createMessage({ - role: 'assistant', - content: [{ type: 'text', text: 'old response' }], - source: { - kind: 'model', - ...{ provider: 'historical', model: 'old-model', replayState: { private: 'state' } }, - }, - })], + provider: 'target', model: 'new-model', messages, })) { /* drain */ } - expect(target.lastOptions?.messages[0]?.source).toEqual({ - kind: 'model', - provider: 'historical', - model: 'old-model', - }) + expect(target.lastOptions?.messages).not.toBe(messages) + expect(target.lastOptions?.messages).toHaveLength(5) + for (const index of [0, 2, 4]) expect(target.lastOptions?.messages[index]).toBe(messages[index]) + for (const index of [1, 3]) { + expect(target.lastOptions?.messages[index]).toEqual({ + ...foreign, source: { kind: 'model', provider: 'historical', model: 'old-model' }, + }) + } + expect(foreign.source.replayState).toEqual({ private: 'state' }) }) it('preserves immutability while stripping replay state from frozen requests', async () => {