From f789cef7508d25ae59b6cf1589429c17858e49b4 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 19:07:25 +0800 Subject: [PATCH 01/11] feat(runtime-node): pass through hallucinationGuard config for ablation runs --- packages/runtime-node/src/run.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/runtime-node/src/run.ts b/packages/runtime-node/src/run.ts index 5cda233..baf166a 100644 --- a/packages/runtime-node/src/run.ts +++ b/packages/runtime-node/src/run.ts @@ -8,6 +8,7 @@ import { type AgentSessionSnapshot, createAgent, type ExecutorStepTrace, + type HallucinationGuardConfig, type LLMBackend, } from '@frontagent/core'; import { createShellMCPClient } from '@frontagent/mcp-shell'; @@ -53,6 +54,8 @@ export interface RunFrontAgentTaskOptions extends RuntimeConfigInput { codeQualityIsolationMode?: 'process' | 'in_memory'; streamShellOutput?: boolean; llmBackend?: LLMBackend; + /** 评测/消融用：覆盖幻觉防控配置（缺省时保持核心默认行为） */ + hallucinationGuard?: HallucinationGuardConfig; signal?: AbortSignal; /** * 显式启用项目内 .frontagent/settings.json 的 hooks（默认关闭）。 @@ -167,6 +170,7 @@ export async function runFrontAgentTask( ...resolved.llm, backend: options.llmBackend, }, + hallucinationGuard: options.hallucinationGuard, execution: resolved.execution, rag: resolved.rag, filesense: resolved.filesense, @@ -436,6 +440,7 @@ export async function planFrontAgentTask( ...resolved.llm, backend: options.llmBackend, }, + hallucinationGuard: options.hallucinationGuard, execution: resolved.execution, rag: resolved.rag, filesense: resolved.filesense, From 936f1a1d06f6d6aaad25bca4b0393941c5d08937 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 19:10:19 +0800 Subject: [PATCH 02/11] feat(eval): frozen fixture project (typecheck-clean, five planted bug targets with failing tests) --- benchmarks/eval/fixture/.gitignore | 1 + benchmarks/eval/fixture/package.json | 13 + benchmarks/eval/fixture/pnpm-lock.yaml | 950 ++++++++++++++++++ benchmarks/eval/fixture/sdd.yaml | 65 ++ benchmarks/eval/fixture/src/App.tsx | 12 + .../eval/fixture/src/components/Button.tsx | 12 + .../eval/fixture/src/components/TodoList.tsx | 13 + .../eval/fixture/src/hooks/useToggle.ts | 7 + .../eval/fixture/src/utils/clamp.test.ts | 14 + benchmarks/eval/fixture/src/utils/clamp.ts | 5 + .../eval/fixture/src/utils/formatDate.test.ts | 8 + .../eval/fixture/src/utils/formatDate.ts | 6 + .../eval/fixture/src/utils/parseQuery.test.ts | 14 + .../eval/fixture/src/utils/parseQuery.ts | 10 + .../eval/fixture/src/utils/pluralize.test.ts | 11 + .../eval/fixture/src/utils/pluralize.ts | 3 + .../eval/fixture/src/utils/truncate.test.ts | 11 + benchmarks/eval/fixture/src/utils/truncate.ts | 4 + benchmarks/eval/fixture/tsconfig.json | 12 + 19 files changed, 1171 insertions(+) create mode 100644 benchmarks/eval/fixture/.gitignore create mode 100644 benchmarks/eval/fixture/package.json create mode 100644 benchmarks/eval/fixture/pnpm-lock.yaml create mode 100644 benchmarks/eval/fixture/sdd.yaml create mode 100644 benchmarks/eval/fixture/src/App.tsx create mode 100644 benchmarks/eval/fixture/src/components/Button.tsx create mode 100644 benchmarks/eval/fixture/src/components/TodoList.tsx create mode 100644 benchmarks/eval/fixture/src/hooks/useToggle.ts create mode 100644 benchmarks/eval/fixture/src/utils/clamp.test.ts create mode 100644 benchmarks/eval/fixture/src/utils/clamp.ts create mode 100644 benchmarks/eval/fixture/src/utils/formatDate.test.ts create mode 100644 benchmarks/eval/fixture/src/utils/formatDate.ts create mode 100644 benchmarks/eval/fixture/src/utils/parseQuery.test.ts create mode 100644 benchmarks/eval/fixture/src/utils/parseQuery.ts create mode 100644 benchmarks/eval/fixture/src/utils/pluralize.test.ts create mode 100644 benchmarks/eval/fixture/src/utils/pluralize.ts create mode 100644 benchmarks/eval/fixture/src/utils/truncate.test.ts create mode 100644 benchmarks/eval/fixture/src/utils/truncate.ts create mode 100644 benchmarks/eval/fixture/tsconfig.json diff --git a/benchmarks/eval/fixture/.gitignore b/benchmarks/eval/fixture/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/benchmarks/eval/fixture/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/benchmarks/eval/fixture/package.json b/benchmarks/eval/fixture/package.json new file mode 100644 index 0000000..ebde8ad --- /dev/null +++ b/benchmarks/eval/fixture/package.json @@ -0,0 +1,13 @@ +{ + "name": "eval-fixture", + "private": true, + "type": "module", + "scripts": { "typecheck": "tsc --noEmit", "test": "vitest run" }, + "devDependencies": { + "@types/react": "^18.3.0", + "react": "^18.3.0", + "react-dom": "^18.3.0", + "typescript": "^5.6.0", + "vitest": "^2.0.0" + } +} diff --git a/benchmarks/eval/fixture/pnpm-lock.yaml b/benchmarks/eval/fixture/pnpm-lock.yaml new file mode 100644 index 0000000..d756818 --- /dev/null +++ b/benchmarks/eval/fixture/pnpm-lock.yaml @@ -0,0 +1,950 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@types/react': + specifier: ^18.3.0 + version: 18.3.31 + react: + specifier: ^18.3.0 + version: 18.3.1 + react-dom: + specifier: ^18.3.0 + version: 18.3.1(react@18.3.1) + typescript: + specifier: ^5.6.0 + version: 5.9.3 + vitest: + specifier: ^2.0.0 + version: 2.1.9 + +packages: + + '@esbuild/aix-ppc64@0.21.5': + resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.21.5': + resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.21.5': + resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.21.5': + resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.21.5': + resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.21.5': + resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.21.5': + resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.21.5': + resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.21.5': + resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.21.5': + resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.21.5': + resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.21.5': + resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.21.5': + resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.21.5': + resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.21.5': + resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.21.5': + resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.21.5': + resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-x64@0.21.5': + resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-x64@0.21.5': + resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/sunos-x64@0.21.5': + resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.21.5': + resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.21.5': + resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.21.5': + resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@rollup/rollup-android-arm-eabi@4.62.2': + resolution: {integrity: sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.62.2': + resolution: {integrity: sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.62.2': + resolution: {integrity: sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.62.2': + resolution: {integrity: sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.62.2': + resolution: {integrity: sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.62.2': + resolution: {integrity: sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-arm64-musl@4.62.2': + resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-loong64-musl@4.62.2': + resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} + cpu: [s390x] + os: [linux] + + '@rollup/rollup-linux-x64-gnu@4.62.2': + resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-linux-x64-musl@4.62.2': + resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-openbsd-x64@4.62.2': + resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.62.2': + resolution: {integrity: sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + resolution: {integrity: sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + resolution: {integrity: sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.62.2': + resolution: {integrity: sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.62.2': + resolution: {integrity: sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==} + cpu: [x64] + os: [win32] + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/prop-types@15.7.15': + resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==} + + '@types/react@18.3.31': + resolution: {integrity: sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==} + + '@vitest/expect@2.1.9': + resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} + + '@vitest/mocker@2.1.9': + resolution: {integrity: sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@2.1.9': + resolution: {integrity: sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==} + + '@vitest/runner@2.1.9': + resolution: {integrity: sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==} + + '@vitest/snapshot@2.1.9': + resolution: {integrity: sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==} + + '@vitest/spy@2.1.9': + resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==} + + '@vitest/utils@2.1.9': + resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esbuild@0.21.5: + resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} + engines: {node: '>=12'} + hasBin: true + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.15: + resolution: {integrity: sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + pathe@1.1.2: + resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + postcss@8.5.16: + resolution: {integrity: sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==} + engines: {node: ^10 || ^12 || >=14} + + react-dom@18.3.1: + resolution: {integrity: sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==} + peerDependencies: + react: ^18.3.1 + + react@18.3.1: + resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==} + engines: {node: '>=0.10.0'} + + rollup@4.62.2: + resolution: {integrity: sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + scheduler@0.23.2: + resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==} + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@1.2.0: + resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} + engines: {node: '>=14.0.0'} + + tinyspy@3.0.2: + resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} + engines: {node: '>=14.0.0'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + vite-node@2.1.9: + resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + + vite@5.4.21: + resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || >=20.0.0 + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + + vitest@2.1.9: + resolution: {integrity: sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': 2.1.9 + '@vitest/ui': 2.1.9 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + +snapshots: + + '@esbuild/aix-ppc64@0.21.5': + optional: true + + '@esbuild/android-arm64@0.21.5': + optional: true + + '@esbuild/android-arm@0.21.5': + optional: true + + '@esbuild/android-x64@0.21.5': + optional: true + + '@esbuild/darwin-arm64@0.21.5': + optional: true + + '@esbuild/darwin-x64@0.21.5': + optional: true + + '@esbuild/freebsd-arm64@0.21.5': + optional: true + + '@esbuild/freebsd-x64@0.21.5': + optional: true + + '@esbuild/linux-arm64@0.21.5': + optional: true + + '@esbuild/linux-arm@0.21.5': + optional: true + + '@esbuild/linux-ia32@0.21.5': + optional: true + + '@esbuild/linux-loong64@0.21.5': + optional: true + + '@esbuild/linux-mips64el@0.21.5': + optional: true + + '@esbuild/linux-ppc64@0.21.5': + optional: true + + '@esbuild/linux-riscv64@0.21.5': + optional: true + + '@esbuild/linux-s390x@0.21.5': + optional: true + + '@esbuild/linux-x64@0.21.5': + optional: true + + '@esbuild/netbsd-x64@0.21.5': + optional: true + + '@esbuild/openbsd-x64@0.21.5': + optional: true + + '@esbuild/sunos-x64@0.21.5': + optional: true + + '@esbuild/win32-arm64@0.21.5': + optional: true + + '@esbuild/win32-ia32@0.21.5': + optional: true + + '@esbuild/win32-x64@0.21.5': + optional: true + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@rollup/rollup-android-arm-eabi@4.62.2': + optional: true + + '@rollup/rollup-android-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-x64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-arm64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-x64@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-musl@4.62.2': + optional: true + + '@rollup/rollup-openbsd-x64@4.62.2': + optional: true + + '@rollup/rollup-openharmony-arm64@4.62.2': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.62.2': + optional: true + + '@types/estree@1.0.9': {} + + '@types/prop-types@15.7.15': {} + + '@types/react@18.3.31': + dependencies: + '@types/prop-types': 15.7.15 + csstype: 3.2.3 + + '@vitest/expect@2.1.9': + dependencies: + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + tinyrainbow: 1.2.0 + + '@vitest/mocker@2.1.9(vite@5.4.21)': + dependencies: + '@vitest/spy': 2.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 5.4.21 + + '@vitest/pretty-format@2.1.9': + dependencies: + tinyrainbow: 1.2.0 + + '@vitest/runner@2.1.9': + dependencies: + '@vitest/utils': 2.1.9 + pathe: 1.1.2 + + '@vitest/snapshot@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + magic-string: 0.30.21 + pathe: 1.1.2 + + '@vitest/spy@2.1.9': + dependencies: + tinyspy: 3.0.2 + + '@vitest/utils@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + loupe: 3.2.1 + tinyrainbow: 1.2.0 + + assertion-error@2.0.1: {} + + cac@6.7.14: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + check-error@2.1.3: {} + + csstype@3.2.3: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-eql@5.0.2: {} + + es-module-lexer@1.7.0: {} + + esbuild@0.21.5: + optionalDependencies: + '@esbuild/aix-ppc64': 0.21.5 + '@esbuild/android-arm': 0.21.5 + '@esbuild/android-arm64': 0.21.5 + '@esbuild/android-x64': 0.21.5 + '@esbuild/darwin-arm64': 0.21.5 + '@esbuild/darwin-x64': 0.21.5 + '@esbuild/freebsd-arm64': 0.21.5 + '@esbuild/freebsd-x64': 0.21.5 + '@esbuild/linux-arm': 0.21.5 + '@esbuild/linux-arm64': 0.21.5 + '@esbuild/linux-ia32': 0.21.5 + '@esbuild/linux-loong64': 0.21.5 + '@esbuild/linux-mips64el': 0.21.5 + '@esbuild/linux-ppc64': 0.21.5 + '@esbuild/linux-riscv64': 0.21.5 + '@esbuild/linux-s390x': 0.21.5 + '@esbuild/linux-x64': 0.21.5 + '@esbuild/netbsd-x64': 0.21.5 + '@esbuild/openbsd-x64': 0.21.5 + '@esbuild/sunos-x64': 0.21.5 + '@esbuild/win32-arm64': 0.21.5 + '@esbuild/win32-ia32': 0.21.5 + '@esbuild/win32-x64': 0.21.5 + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.4.0: {} + + fsevents@2.3.3: + optional: true + + js-tokens@4.0.0: {} + + loose-envify@1.4.0: + dependencies: + js-tokens: 4.0.0 + + loupe@3.2.1: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + ms@2.1.3: {} + + nanoid@3.3.15: {} + + pathe@1.1.2: {} + + pathval@2.0.1: {} + + picocolors@1.1.1: {} + + postcss@8.5.16: + dependencies: + nanoid: 3.3.15 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + react-dom@18.3.1(react@18.3.1): + dependencies: + loose-envify: 1.4.0 + react: 18.3.1 + scheduler: 0.23.2 + + react@18.3.1: + dependencies: + loose-envify: 1.4.0 + + rollup@4.62.2: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.62.2 + '@rollup/rollup-android-arm64': 4.62.2 + '@rollup/rollup-darwin-arm64': 4.62.2 + '@rollup/rollup-darwin-x64': 4.62.2 + '@rollup/rollup-freebsd-arm64': 4.62.2 + '@rollup/rollup-freebsd-x64': 4.62.2 + '@rollup/rollup-linux-arm-gnueabihf': 4.62.2 + '@rollup/rollup-linux-arm-musleabihf': 4.62.2 + '@rollup/rollup-linux-arm64-gnu': 4.62.2 + '@rollup/rollup-linux-arm64-musl': 4.62.2 + '@rollup/rollup-linux-loong64-gnu': 4.62.2 + '@rollup/rollup-linux-loong64-musl': 4.62.2 + '@rollup/rollup-linux-ppc64-gnu': 4.62.2 + '@rollup/rollup-linux-ppc64-musl': 4.62.2 + '@rollup/rollup-linux-riscv64-gnu': 4.62.2 + '@rollup/rollup-linux-riscv64-musl': 4.62.2 + '@rollup/rollup-linux-s390x-gnu': 4.62.2 + '@rollup/rollup-linux-x64-gnu': 4.62.2 + '@rollup/rollup-linux-x64-musl': 4.62.2 + '@rollup/rollup-openbsd-x64': 4.62.2 + '@rollup/rollup-openharmony-arm64': 4.62.2 + '@rollup/rollup-win32-arm64-msvc': 4.62.2 + '@rollup/rollup-win32-ia32-msvc': 4.62.2 + '@rollup/rollup-win32-x64-gnu': 4.62.2 + '@rollup/rollup-win32-x64-msvc': 4.62.2 + fsevents: 2.3.3 + + scheduler@0.23.2: + dependencies: + loose-envify: 1.4.0 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinypool@1.1.1: {} + + tinyrainbow@1.2.0: {} + + tinyspy@3.0.2: {} + + typescript@5.9.3: {} + + vite-node@2.1.9: + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 1.1.2 + vite: 5.4.21 + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + + vite@5.4.21: + dependencies: + esbuild: 0.21.5 + postcss: 8.5.16 + rollup: 4.62.2 + optionalDependencies: + fsevents: 2.3.3 + + vitest@2.1.9: + dependencies: + '@vitest/expect': 2.1.9 + '@vitest/mocker': 2.1.9(vite@5.4.21) + '@vitest/pretty-format': 2.1.9 + '@vitest/runner': 2.1.9 + '@vitest/snapshot': 2.1.9 + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 1.1.2 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinypool: 1.1.1 + tinyrainbow: 1.2.0 + vite: 5.4.21 + vite-node: 2.1.9 + why-is-node-running: 2.3.0 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 diff --git a/benchmarks/eval/fixture/sdd.yaml b/benchmarks/eval/fixture/sdd.yaml new file mode 100644 index 0000000..9e8f5b4 --- /dev/null +++ b/benchmarks/eval/fixture/sdd.yaml @@ -0,0 +1,65 @@ +# FrontAgent SDD 配置文件（评测夹具冻结版） +# 规格驱动开发 - Agent 行为约束 + +version: "1.0" + +project: + name: "eval-fixture" + type: "react-spa" + description: "架构消融评测的冻结夹具工程" + +# 技术栈约束 +tech_stack: + framework: "react" + version: "^18.0.0" + language: "typescript" + styling: "css" + state_management: "none" + forbidden_packages: + - "jquery" + - "moment" + +# 目录结构约束 +directory_structure: + src/components: + pattern: "PascalCase" + max_lines: 300 + src/hooks: + pattern: "use*.ts" + src/utils: + pattern: "camelCase" + must_be_pure: true + +# 模块边界约束 +module_boundaries: + - from: "src/components/*" + can_import: + - "src/hooks/*" + - "src/utils/*" + cannot_import: + - "src/api/*" + +# 命名规范 +naming_conventions: + components: "PascalCase" + hooks: "camelCase with 'use' prefix" + utils: "camelCase" + constants: "SCREAMING_SNAKE_CASE" + +# 代码质量约束 +code_quality: + max_function_lines: 50 + max_file_lines: 300 + max_parameters: 4 + forbidden_patterns: + - "any" + - "// @ts-ignore" + - "console.log" + +# 修改安全边界 +modification_rules: + protected_files: + - "package.json" + - "tsconfig.json" + protected_directories: + - "node_modules" diff --git a/benchmarks/eval/fixture/src/App.tsx b/benchmarks/eval/fixture/src/App.tsx new file mode 100644 index 0000000..955fb8a --- /dev/null +++ b/benchmarks/eval/fixture/src/App.tsx @@ -0,0 +1,12 @@ +import { Button } from "./components/Button"; +import { TodoList } from "./components/TodoList"; + +export default function App() { + return ( +
+

Eval Fixture

+
+ ); +} diff --git a/benchmarks/eval/fixture/src/components/Button.tsx b/benchmarks/eval/fixture/src/components/Button.tsx new file mode 100644 index 0000000..ccfd627 --- /dev/null +++ b/benchmarks/eval/fixture/src/components/Button.tsx @@ -0,0 +1,12 @@ +interface ButtonProps { + label: string; + onClick: () => void; +} + +export function Button({ label, onClick }: ButtonProps) { + return ( + + ); +} diff --git a/benchmarks/eval/fixture/src/components/TodoList.tsx b/benchmarks/eval/fixture/src/components/TodoList.tsx new file mode 100644 index 0000000..413bffb --- /dev/null +++ b/benchmarks/eval/fixture/src/components/TodoList.tsx @@ -0,0 +1,13 @@ +interface TodoListProps { + items: string[]; +} + +export function TodoList({ items }: TodoListProps) { + return ( + + ); +} diff --git a/benchmarks/eval/fixture/src/hooks/useToggle.ts b/benchmarks/eval/fixture/src/hooks/useToggle.ts new file mode 100644 index 0000000..69996e5 --- /dev/null +++ b/benchmarks/eval/fixture/src/hooks/useToggle.ts @@ -0,0 +1,7 @@ +import { useState } from "react"; + +export function useToggle(initial = false): [boolean, () => void] { + const [on, setOn] = useState(initial); + const toggle = () => setOn((v) => !v); + return [on, toggle]; +} diff --git a/benchmarks/eval/fixture/src/utils/clamp.test.ts b/benchmarks/eval/fixture/src/utils/clamp.test.ts new file mode 100644 index 0000000..6de3d22 --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/clamp.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { clamp } from "./clamp"; + +describe("clamp", () => { + it("小于下界取下界", () => { + expect(clamp(-1, 0, 10)).toBe(0); + }); + it("大于上界取上界", () => { + expect(clamp(11, 0, 10)).toBe(10); + }); + it("区间内原样返回", () => { + expect(clamp(5, 0, 10)).toBe(5); + }); +}); diff --git a/benchmarks/eval/fixture/src/utils/clamp.ts b/benchmarks/eval/fixture/src/utils/clamp.ts new file mode 100644 index 0000000..2e041e3 --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/clamp.ts @@ -0,0 +1,5 @@ +export function clamp(value: number, min: number, max: number): number { + if (value < min) return min; + if (value > max) return min; + return value; +} diff --git a/benchmarks/eval/fixture/src/utils/formatDate.test.ts b/benchmarks/eval/fixture/src/utils/formatDate.test.ts new file mode 100644 index 0000000..d75a99c --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/formatDate.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, it } from "vitest"; +import { formatDate } from "./formatDate"; + +describe("formatDate", () => { + it("格式化为 YYYY-MM-DD", () => { + expect(formatDate(new Date(2026, 6, 11))).toBe("2026-07-11"); + }); +}); diff --git a/benchmarks/eval/fixture/src/utils/formatDate.ts b/benchmarks/eval/fixture/src/utils/formatDate.ts new file mode 100644 index 0000000..0c5aeae --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/formatDate.ts @@ -0,0 +1,6 @@ +export function formatDate(d: Date): string { + const y = d.getFullYear(); + const m = String(d.getMonth()).padStart(2, "0"); + const day = String(d.getDate()).padStart(2, "0"); + return `${y}-${m}-${day}`; +} diff --git a/benchmarks/eval/fixture/src/utils/parseQuery.test.ts b/benchmarks/eval/fixture/src/utils/parseQuery.test.ts new file mode 100644 index 0000000..2032794 --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/parseQuery.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { parseQuery } from "./parseQuery"; + +describe("parseQuery", () => { + it("解析问号前缀查询串", () => { + expect(parseQuery("?a=1&b=2")).toEqual({ a: "1", b: "2" }); + }); + it("URL 编码的值要解码", () => { + expect(parseQuery("q=hello%20world")).toEqual({ q: "hello world" }); + }); + it("空串返回空对象", () => { + expect(parseQuery("")).toEqual({}); + }); +}); diff --git a/benchmarks/eval/fixture/src/utils/parseQuery.ts b/benchmarks/eval/fixture/src/utils/parseQuery.ts new file mode 100644 index 0000000..56c25ef --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/parseQuery.ts @@ -0,0 +1,10 @@ +export function parseQuery(query: string): Record { + const result: Record = {}; + const trimmed = query.startsWith("?") ? query.slice(1) : query; + if (!trimmed) return result; + for (const pair of trimmed.split("&")) { + const [key, value] = pair.split("="); + if (key) result[key] = value ?? ""; + } + return result; +} diff --git a/benchmarks/eval/fixture/src/utils/pluralize.test.ts b/benchmarks/eval/fixture/src/utils/pluralize.test.ts new file mode 100644 index 0000000..3615c7d --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/pluralize.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; +import { pluralize } from "./pluralize"; + +describe("pluralize", () => { + it("单数不加 s", () => { + expect(pluralize("item", 1)).toBe("1 item"); + }); + it("复数加 s", () => { + expect(pluralize("item", 2)).toBe("2 items"); + }); +}); diff --git a/benchmarks/eval/fixture/src/utils/pluralize.ts b/benchmarks/eval/fixture/src/utils/pluralize.ts new file mode 100644 index 0000000..9a50803 --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/pluralize.ts @@ -0,0 +1,3 @@ +export function pluralize(word: string, count: number): string { + return `${count} ${word}s`; +} diff --git a/benchmarks/eval/fixture/src/utils/truncate.test.ts b/benchmarks/eval/fixture/src/utils/truncate.test.ts new file mode 100644 index 0000000..d789946 --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/truncate.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; +import { truncate } from "./truncate"; + +describe("truncate", () => { + it("不超长原样返回", () => { + expect(truncate("abc", 5)).toBe("abc"); + }); + it("超长截到 maxLength 并加省略号", () => { + expect(truncate("abcdefg", 3)).toBe("abc…"); + }); +}); diff --git a/benchmarks/eval/fixture/src/utils/truncate.ts b/benchmarks/eval/fixture/src/utils/truncate.ts new file mode 100644 index 0000000..fd40a93 --- /dev/null +++ b/benchmarks/eval/fixture/src/utils/truncate.ts @@ -0,0 +1,4 @@ +export function truncate(text: string, maxLength: number): string { + if (text.length <= maxLength) return text; + return `${text.slice(0, maxLength + 1)}…`; +} diff --git a/benchmarks/eval/fixture/tsconfig.json b/benchmarks/eval/fixture/tsconfig.json new file mode 100644 index 0000000..9714625 --- /dev/null +++ b/benchmarks/eval/fixture/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src"] +} From cf2d54b8670714950421e51a75bcfd8f50613609 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 19:12:08 +0800 Subject: [PATCH 03/11] feat(eval): freeze 30-task evaluation set (10 query / 10 create / 5 bugfix / 5 refactor) --- benchmarks/eval/tasks.json | 364 +++++++++++++++++++++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 benchmarks/eval/tasks.json diff --git a/benchmarks/eval/tasks.json b/benchmarks/eval/tasks.json new file mode 100644 index 0000000..d4b3b97 --- /dev/null +++ b/benchmarks/eval/tasks.json @@ -0,0 +1,364 @@ +[ + { + "id": "query-button-props", + "category": "query", + "smoke": true, + "task": "Button 组件接受哪些 props？逐个说明类型与用途。", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "label" }, + { "kind": "result_contains", "pattern": "onClick" } + ] + }, + { + "id": "query-structure", + "category": "query", + "smoke": false, + "task": "列出 src 下的目录结构，说明每个目录的职责。", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "components" }, + { "kind": "result_contains", "pattern": "hooks" }, + { "kind": "result_contains", "pattern": "utils" } + ] + }, + { + "id": "query-todolist-render", + "category": "query", + "smoke": false, + "task": "TodoList 组件如何渲染列表项？列表项的 key 用的是什么？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "key" }, + { "kind": "result_contains", "pattern": "item" } + ] + }, + { + "id": "query-usetoggle-behavior", + "category": "query", + "smoke": false, + "task": "useToggle hook 返回什么？内部状态是怎么切换的？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "toggle" }, + { "kind": "result_contains", "pattern": "useState" } + ] + }, + { + "id": "query-app-components", + "category": "query", + "smoke": false, + "task": "App 组件直接使用了哪些子组件？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "Button" }, + { "kind": "result_contains", "pattern": "TodoList" } + ] + }, + { + "id": "query-formatdate-io", + "category": "query", + "smoke": false, + "task": "formatDate 函数的输入和输出分别是什么类型和格式？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "Date" }, + { "kind": "result_contains", "pattern": "string|字符串" } + ] + }, + { + "id": "query-tsconfig-strict", + "category": "query", + "smoke": false, + "task": "这个项目的 tsconfig 开启了哪些编译选项？strict 开了吗？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "strict" } + ] + }, + { + "id": "query-clamp-purpose", + "category": "query", + "smoke": false, + "task": "clamp 函数是做什么的？三个参数分别是什么含义？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "min" }, + { "kind": "result_contains", "pattern": "max" } + ] + }, + { + "id": "query-test-files", + "category": "query", + "smoke": false, + "task": "项目里有哪些测试文件？分别测试什么？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "formatDate" }, + { "kind": "result_contains", "pattern": "clamp" } + ] + }, + { + "id": "query-button-export", + "category": "query", + "smoke": false, + "task": "Button 组件是命名导出还是默认导出？App 呢？", + "type": "query", + "checks": [ + { "kind": "result_contains", "pattern": "命名|named" }, + { "kind": "result_contains", "pattern": "默认|default" } + ] + }, + { + "id": "create-usedebounce", + "category": "create", + "smoke": true, + "task": "创建一个 useDebounce hook，接受 value 与 delayMs，返回防抖后的值，放在 src/hooks/useDebounce.ts。", + "type": "create", + "files": ["src/hooks/useDebounce.ts"], + "checks": [ + { "kind": "file_exists", "path": "src/hooks/useDebounce.ts" }, + { "kind": "file_contains", "path": "src/hooks/useDebounce.ts", "pattern": "export function useDebounce" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-card", + "category": "create", + "smoke": false, + "task": "创建一个 Card 组件（src/components/Card.tsx），接受 title 字符串与 children，渲染标题和内容区。", + "type": "create", + "files": ["src/components/Card.tsx"], + "checks": [ + { "kind": "file_exists", "path": "src/components/Card.tsx" }, + { "kind": "file_contains", "path": "src/components/Card.tsx", "pattern": "export function Card" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-capitalize", + "category": "create", + "smoke": false, + "task": "创建纯函数 capitalize（src/utils/capitalize.ts）：首字母大写、其余不变，空串原样返回。", + "type": "create", + "files": ["src/utils/capitalize.ts"], + "checks": [ + { "kind": "file_exists", "path": "src/utils/capitalize.ts" }, + { "kind": "file_contains", "path": "src/utils/capitalize.ts", "pattern": "export function capitalize" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-uselocalstorage", + "category": "create", + "smoke": false, + "task": "创建 useLocalStorage hook（src/hooks/useLocalStorage.ts）：读写 localStorage 并保持 state 同步，值用 JSON 序列化。", + "type": "create", + "files": ["src/hooks/useLocalStorage.ts"], + "checks": [ + { "kind": "file_exists", "path": "src/hooks/useLocalStorage.ts" }, + { "kind": "file_contains", "path": "src/hooks/useLocalStorage.ts", "pattern": "export function useLocalStorage" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-badge", + "category": "create", + "smoke": false, + "task": "创建 Badge 组件（src/components/Badge.tsx）：variant 属性为 'info' | 'success' | 'warning' 联合类型，渲染带对应 className 的 span。", + "type": "create", + "files": ["src/components/Badge.tsx"], + "checks": [ + { "kind": "file_exists", "path": "src/components/Badge.tsx" }, + { "kind": "file_contains", "path": "src/components/Badge.tsx", "pattern": "variant" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-formatcurrency", + "category": "create", + "smoke": false, + "task": "创建纯函数 formatCurrency（src/utils/formatCurrency.ts）：数字金额格式化为带千分位、两位小数的字符串，如 1234.5 → \"1,234.50\"。", + "type": "create", + "files": ["src/utils/formatCurrency.ts"], + "checks": [ + { "kind": "file_exists", "path": "src/utils/formatCurrency.ts" }, + { "kind": "file_contains", "path": "src/utils/formatCurrency.ts", "pattern": "export function formatCurrency" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-spinner", + "category": "create", + "smoke": false, + "task": "创建 Spinner 加载指示器组件（src/components/Spinner.tsx），可选 size 属性（'small' | 'large'，默认 'small'）。", + "type": "create", + "files": ["src/components/Spinner.tsx"], + "checks": [ + { "kind": "file_exists", "path": "src/components/Spinner.tsx" }, + { "kind": "file_contains", "path": "src/components/Spinner.tsx", "pattern": "export function Spinner" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-useinterval", + "category": "create", + "smoke": false, + "task": "创建 useInterval hook（src/hooks/useInterval.ts）：按 delayMs 周期执行回调，组件卸载时清理定时器。", + "type": "create", + "files": ["src/hooks/useInterval.ts"], + "checks": [ + { "kind": "file_exists", "path": "src/hooks/useInterval.ts" }, + { "kind": "file_contains", "path": "src/hooks/useInterval.ts", "pattern": "useEffect" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-slugify", + "category": "create", + "smoke": false, + "task": "创建纯函数 slugify（src/utils/slugify.ts）：字符串转 URL slug——小写、空白变连字符、去掉非字母数字连字符字符。", + "type": "create", + "files": ["src/utils/slugify.ts"], + "checks": [ + { "kind": "file_exists", "path": "src/utils/slugify.ts" }, + { "kind": "file_contains", "path": "src/utils/slugify.ts", "pattern": "export function slugify" }, + { "kind": "typecheck" } + ] + }, + { + "id": "create-checkbox", + "category": "create", + "smoke": true, + "task": "创建受控 Checkbox 组件（src/components/Checkbox.tsx）：checked 布尔与 onChange 回调，带 label 文本。", + "type": "create", + "files": ["src/components/Checkbox.tsx"], + "checks": [ + { "kind": "file_exists", "path": "src/components/Checkbox.tsx" }, + { "kind": "file_contains", "path": "src/components/Checkbox.tsx", "pattern": "onChange" }, + { "kind": "typecheck" } + ] + }, + { + "id": "bugfix-formatdate", + "category": "bugfix", + "smoke": true, + "task": "src/utils/formatDate.ts 输出的月份不对（2026-07-11 被格式化成 2026-06-11），修复它并保证现有测试通过。", + "type": "modify", + "files": ["src/utils/formatDate.ts"], + "checks": [ + { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/formatDate.test.ts"] }, + { "kind": "typecheck" } + ] + }, + { + "id": "bugfix-clamp", + "category": "bugfix", + "smoke": false, + "task": "src/utils/clamp.ts 在值大于上界时返回了错误的边界，修复并保证现有测试通过。", + "type": "modify", + "files": ["src/utils/clamp.ts"], + "checks": [ + { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/clamp.test.ts"] }, + { "kind": "typecheck" } + ] + }, + { + "id": "bugfix-pluralize", + "category": "bugfix", + "smoke": false, + "task": "src/utils/pluralize.ts 对单数（count 为 1）也加了 s，修复并保证现有测试通过。", + "type": "modify", + "files": ["src/utils/pluralize.ts"], + "checks": [ + { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/pluralize.test.ts"] }, + { "kind": "typecheck" } + ] + }, + { + "id": "bugfix-truncate", + "category": "bugfix", + "smoke": false, + "task": "src/utils/truncate.ts 截断长度差一位（maxLength=3 时留了 4 个字符），修复并保证现有测试通过。", + "type": "modify", + "files": ["src/utils/truncate.ts"], + "checks": [ + { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/truncate.test.ts"] }, + { "kind": "typecheck" } + ] + }, + { + "id": "bugfix-parsequery", + "category": "bugfix", + "smoke": false, + "task": "src/utils/parseQuery.ts 没有对 URL 编码的值解码（hello%20world 应变为 hello world），修复并保证现有测试通过。", + "type": "modify", + "files": ["src/utils/parseQuery.ts"], + "checks": [ + { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/parseQuery.test.ts"] }, + { "kind": "typecheck" } + ] + }, + { + "id": "refactor-todolist-empty", + "category": "refactor", + "smoke": true, + "task": "TodoList 在 items 为空时应渲染一行「暂无待办」提示而不是空 ul，重构组件实现。", + "type": "modify", + "files": ["src/components/TodoList.tsx"], + "checks": [ + { "kind": "file_contains", "path": "src/components/TodoList.tsx", "pattern": "暂无待办" }, + { "kind": "typecheck" } + ] + }, + { + "id": "refactor-button-disabled", + "category": "refactor", + "smoke": false, + "task": "给 Button 组件增加可选的 disabled 布尔属性并透传到 button 元素。", + "type": "modify", + "files": ["src/components/Button.tsx"], + "checks": [ + { "kind": "file_contains", "path": "src/components/Button.tsx", "pattern": "disabled" }, + { "kind": "typecheck" } + ] + }, + { + "id": "refactor-extract-listitem", + "category": "refactor", + "smoke": false, + "task": "把 TodoList 里的列表项抽成独立的 ListItem 组件（src/components/ListItem.tsx），TodoList 改为使用它。", + "type": "modify", + "files": ["src/components/TodoList.tsx", "src/components/ListItem.tsx"], + "checks": [ + { "kind": "file_exists", "path": "src/components/ListItem.tsx" }, + { "kind": "file_contains", "path": "src/components/TodoList.tsx", "pattern": "ListItem" }, + { "kind": "typecheck" } + ] + }, + { + "id": "refactor-app-title", + "category": "refactor", + "smoke": false, + "task": "App 组件的 h1 文案改为可配置的 title 属性，默认值保持 \"Eval Fixture\"。", + "type": "modify", + "files": ["src/App.tsx"], + "checks": [ + { "kind": "file_contains", "path": "src/App.tsx", "pattern": "title" }, + { "kind": "file_contains", "path": "src/App.tsx", "pattern": "Eval Fixture" }, + { "kind": "typecheck" } + ] + }, + { + "id": "refactor-button-memo", + "category": "refactor", + "smoke": false, + "task": "用 React.memo 包裹 Button 组件的导出，避免父组件重渲染时的无谓渲染。", + "type": "modify", + "files": ["src/components/Button.tsx"], + "checks": [ + { "kind": "file_contains", "path": "src/components/Button.tsx", "pattern": "memo" }, + { "kind": "typecheck" } + ] + } +] From 7d30898f2c80d3929023c5fe366ce6cd06a0ad87 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 19:12:57 +0800 Subject: [PATCH 04/11] feat(eval): acceptance check runner and claude CLI LLMBackend (subscription Haiku, no API key) --- benchmarks/eval/checks.mjs | 40 ++++++++++++ benchmarks/eval/claude-cli-backend.mjs | 87 ++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 benchmarks/eval/checks.mjs create mode 100644 benchmarks/eval/claude-cli-backend.mjs diff --git a/benchmarks/eval/checks.mjs b/benchmarks/eval/checks.mjs new file mode 100644 index 0000000..bddb823 --- /dev/null +++ b/benchmarks/eval/checks.mjs @@ -0,0 +1,40 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +/** 逐条执行任务 checks；返回 [{kind, ok, detail}]，全 ok 即任务 pass */ +export function runChecks(checks, { workspace, resultText }) { + return checks.map((check) => { + try { + switch (check.kind) { + case 'file_exists': { + const ok = existsSync(join(workspace, check.path)); + return { kind: check.kind, ok, detail: check.path }; + } + case 'file_contains': { + const p = join(workspace, check.path); + if (!existsSync(p)) return { kind: check.kind, ok: false, detail: `missing ${check.path}` }; + const ok = new RegExp(check.pattern).test(readFileSync(p, 'utf8')); + return { kind: check.kind, ok, detail: check.pattern }; + } + case 'result_contains': { + const ok = new RegExp(check.pattern, 'i').test(resultText ?? ''); + return { kind: check.kind, ok, detail: check.pattern }; + } + case 'typecheck': { + const r = spawnSync('npx', ['tsc', '--noEmit'], { cwd: workspace, timeout: 120000 }); + return { kind: check.kind, ok: r.status === 0, detail: r.status === 0 ? '' : String(r.stdout).slice(0, 300) }; + } + case 'cmd': { + const [bin, ...rest] = check.argv; + const r = spawnSync(bin, rest, { cwd: workspace, timeout: 180000 }); + return { kind: check.kind, ok: r.status === 0, detail: check.argv.join(' ') }; + } + default: + return { kind: check.kind, ok: false, detail: 'unknown check kind' }; + } + } catch (error) { + return { kind: check.kind, ok: false, detail: String(error).slice(0, 200) }; + } + }); +} diff --git a/benchmarks/eval/claude-cli-backend.mjs b/benchmarks/eval/claude-cli-backend.mjs new file mode 100644 index 0000000..3d4ca84 --- /dev/null +++ b/benchmarks/eval/claude-cli-backend.mjs @@ -0,0 +1,87 @@ +import { spawn } from 'node:child_process'; + +const MODEL = process.env.EVAL_MODEL ?? 'claude-haiku-4-5'; +const CALL_TIMEOUT_MS = Number(process.env.EVAL_LLM_TIMEOUT_MS ?? 180000); + +const tally = { calls: 0, inputTokens: 0, outputTokens: 0 }; +export function getUsageTally() { + return { ...tally }; +} + +function renderPrompt(messages, system) { + const parts = []; + if (system) parts.push(`[System]\n${system}`); + for (const m of messages ?? []) { + parts.push(`[${m.role}]\n${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`); + } + return parts.join('\n\n'); +} + +function callClaude(prompt) { + return new Promise((resolvePromise, reject) => { + const args = ['-p', '--model', MODEL, '--output-format', 'json', '--max-turns', '1']; + const child = spawn('claude', args, { stdio: ['pipe', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + const timer = setTimeout(() => { + child.kill('SIGKILL'); + reject(new Error('claude CLI timeout')); + }, CALL_TIMEOUT_MS); + child.stdout.on('data', (d) => { + stdout += d; + }); + child.stderr.on('data', (d) => { + stderr += d; + }); + child.on('error', (e) => { + clearTimeout(timer); + reject(e); + }); + child.on('close', (code) => { + clearTimeout(timer); + if (code !== 0) return reject(new Error(`claude exit ${code}: ${stderr.slice(0, 300)}`)); + try { + const parsed = JSON.parse(stdout); + tally.calls += 1; + tally.inputTokens += parsed.usage?.input_tokens ?? 0; + tally.outputTokens += parsed.usage?.output_tokens ?? 0; + if (parsed.is_error) return reject(new Error(`claude error result: ${String(parsed.result).slice(0, 300)}`)); + resolvePromise(String(parsed.result ?? '')); + } catch (e) { + reject(new Error(`bad CLI JSON: ${String(e)}; head=${stdout.slice(0, 200)}`)); + } + }); + child.stdin.end(prompt); + }); +} + +function extractJson(text) { + const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/); + const body = fenced ? fenced[1] : text; + const start = body.indexOf('{'); + const end = body.lastIndexOf('}'); + if (start === -1 || end <= start) throw new Error('no JSON object in output'); + return body.slice(start, end + 1); +} + +export function createClaudeCliBackend() { + return { + name: `claude-cli:${MODEL}`, + async generateText({ messages, system }) { + return callClaude(renderPrompt(messages, system)); + }, + async generateObject({ messages, system, schema, maxRetries = 2 }) { + let lastError = ''; + for (let attempt = 0; attempt <= maxRetries; attempt += 1) { + const suffix = `\n\n只输出一个 JSON 对象：不要 markdown 代码围栏、不要解释文字。${lastError ? `上次输出未通过 schema 校验（${lastError}），请修正。` : ''}`; + const text = await callClaude(renderPrompt(messages, system) + suffix); + try { + return schema.parse(JSON.parse(extractJson(text))); + } catch (error) { + lastError = String(error).slice(0, 400); + } + } + throw new Error(`generateObject retries exhausted: ${lastError}`); + }, + }; +} From dd03efc4ff3013fe349e0732b86de657de4b93b0 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 19:13:51 +0800 Subject: [PATCH 05/11] feat(eval): two-arm ablation orchestrator and markdown report generator --- benchmarks/eval/report.mjs | 57 ++++++++++++++++++ benchmarks/eval/run-eval.mjs | 109 +++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 benchmarks/eval/report.mjs create mode 100644 benchmarks/eval/run-eval.mjs diff --git a/benchmarks/eval/report.mjs b/benchmarks/eval/report.mjs new file mode 100644 index 0000000..3bb8931 --- /dev/null +++ b/benchmarks/eval/report.mjs @@ -0,0 +1,57 @@ +// 汇总双臂 JSONL 为 Markdown 报告。 +// 用法：node benchmarks/eval/report.mjs [outDir] > benchmarks/results/-architecture-ablation.md +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const dir = process.argv[2] ?? 'benchmarks/eval/out'; +const load = (arm) => + readFileSync(join(dir, `${arm}.jsonl`), 'utf8') + .trim() + .split('\n') + .map((l) => JSON.parse(l)); + +const arms = { full: load('full'), ablation: load('ablation') }; +const rate = (rows) => rows.filter((r) => r.pass).length / rows.length; +const avg = (rows, k) => Math.round(rows.reduce((s, r) => s + (r[k] ?? 0), 0) / rows.length); +const pct = (x) => `${(x * 100).toFixed(1)}%`; + +const categories = [...new Set(arms.full.map((r) => r.category))]; +const guardEvents = arms.full.reduce((s, r) => s + (r.events?.validation_failed ?? 0), 0); +const validations = arms.full.reduce((s, r) => s + (r.validationCount ?? 0), 0); + +console.log(`# FrontAgent 架构消融评测（guard + SDD on/off） + +- 任务集：${arms.full.length} 条冻结任务（benchmarks/eval/tasks.json） +- 模型：${process.env.EVAL_MODEL ?? 'claude-haiku-4-5'}（两臂同模型、同任务集、同夹具） +- 日期：${new Date().toISOString().slice(0, 10)} + +## 总表 + +| 指标 | 消融臂（关 guard+SDD） | 全开臂 | +|---|---|---| +| 一次通过率 | ${pct(rate(arms.ablation))} | ${pct(rate(arms.full))} | +| 平均 LLM 调用/任务 | ${avg(arms.ablation, 'llmCalls')} | ${avg(arms.full, 'llmCalls')} | +| 平均输出 token/任务 | ${avg(arms.ablation, 'outputTokens')} | ${avg(arms.full, 'outputTokens')} | +| 平均时延 ms/任务 | ${avg(arms.ablation, 'elapsedMs')} | ${avg(arms.full, 'elapsedMs')} | + +全开臂校验活动：validation_failed 事件 ${guardEvents} 次；结果内 validations 记录合计 ${validations} 条。 + +## 分类通过率 + +| 类别 | 消融臂 | 全开臂 | +|---|---|---|`); +for (const c of categories) { + const f = arms.full.filter((r) => r.category === c); + const a = arms.ablation.filter((r) => r.category === c); + console.log(`| ${c} | ${pct(rate(a))} | ${pct(rate(f))} |`); +} +console.log(` +## 失败清单（复盘素材） +`); +for (const [arm, rows] of Object.entries(arms)) { + for (const r of rows.filter((x) => !x.pass)) { + console.log( + `- [${arm}] ${r.taskId}: ${r.runError ?? r.checks.filter((c) => !c.ok).map((c) => `${c.kind}(${c.detail})`).join(', ')}`, + ); + } +} diff --git a/benchmarks/eval/run-eval.mjs b/benchmarks/eval/run-eval.mjs new file mode 100644 index 0000000..2d170cf --- /dev/null +++ b/benchmarks/eval/run-eval.mjs @@ -0,0 +1,109 @@ +// 架构消融评测编排器。 +// 用法：node benchmarks/eval/run-eval.mjs --arm full|ablation [--tasks smoke|all] [--out benchmarks/eval/out] +import { appendFileSync, cpSync, mkdirSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; +import { runFrontAgentTask } from '../../packages/runtime-node/dist/run.js'; +import { createClaudeCliBackend, getUsageTally } from './claude-cli-backend.mjs'; +import { runChecks } from './checks.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const FIXTURE = join(HERE, 'fixture'); +const args = Object.fromEntries( + process.argv + .slice(2) + .map((a, i, all) => (a.startsWith('--') ? [a.slice(2), all[i + 1]] : null)) + .filter(Boolean), +); +const ARM = args.arm; +if (ARM !== 'full' && ARM !== 'ablation') throw new Error('--arm full|ablation 必填'); +const SCOPE = args.tasks ?? 'all'; +const OUT_DIR = resolve(args.out ?? join(HERE, 'out')); +mkdirSync(OUT_DIR, { recursive: true }); +const OUT = join(OUT_DIR, `${ARM}.jsonl`); + +const ARM_OPTIONS = { + full: { sddPath: 'sdd.yaml' }, + ablation: { + // 指向不存在的文件 → run.ts existsSync 判定为无 SDD + sddPath: 'sdd.disabled.yaml', + hallucinationGuard: { + enabled: false, + checks: { fileExistence: false, importValidity: false, syntaxValidity: false, sddCompliance: false }, + }, + }, +}; + +function setupWorkspace(taskId) { + const ws = join('/tmp', 'frontagent-eval', ARM, taskId); + rmSync(ws, { recursive: true, force: true }); + mkdirSync(ws, { recursive: true }); + cpSync(FIXTURE, ws, { + recursive: true, + filter: (src) => !src.includes('node_modules'), + }); + symlinkSync(join(FIXTURE, 'node_modules'), join(ws, 'node_modules')); + return ws; +} + +const tasks = JSON.parse(readFileSync(join(HERE, 'tasks.json'), 'utf8')).filter((t) => + SCOPE === 'smoke' ? t.smoke : true, +); + +console.log(`arm=${ARM} scope=${SCOPE} tasks=${tasks.length} out=${OUT}`); +const backend = createClaudeCliBackend(); + +for (const task of tasks) { + const ws = setupWorkspace(task.id); + const events = {}; + const t0 = performance.now(); + const usageBefore = getUsageTally(); + let resultText = ''; + let agentSuccess = null; + let validationCount = null; + let runError = null; + try { + const result = await runFrontAgentTask({ + projectRoot: ws, + task: task.task, + type: task.type, + files: task.files, + llmBackend: backend, + runLog: false, + filterConsole: true, + ...ARM_OPTIONS[ARM], + onEvent: (e) => { + events[e.type] = (events[e.type] ?? 0) + 1; + }, + }); + resultText = result?.output ?? ''; + agentSuccess = result?.success ?? null; + validationCount = result?.validations?.length ?? null; + } catch (error) { + runError = String(error).slice(0, 400); + } + const usageAfter = getUsageTally(); + const checkResults = runError ? [] : runChecks(task.checks, { workspace: ws, resultText }); + const pass = !runError && checkResults.length > 0 && checkResults.every((c) => c.ok); + const record = { + taskId: task.id, + category: task.category, + arm: ARM, + pass, + agentSuccess, + validationCount, + runError, + checks: checkResults, + events, + elapsedMs: Math.round(performance.now() - t0), + llmCalls: usageAfter.calls - usageBefore.calls, + inputTokens: usageAfter.inputTokens - usageBefore.inputTokens, + outputTokens: usageAfter.outputTokens - usageBefore.outputTokens, + }; + appendFileSync(OUT, `${JSON.stringify(record)}\n`); + console.log( + `[${ARM}] ${task.id}: ${pass ? 'PASS' : 'FAIL'} (${record.elapsedMs}ms, ${record.llmCalls} calls, events=${JSON.stringify(events)})`, + ); +} +console.log('done. total usage:', JSON.stringify(getUsageTally())); From ad1db1a88f91655e1a07ea8d10900cb59d9958a4 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 19:35:21 +0800 Subject: [PATCH 06/11] fix(eval): surface backend call failures, auto-approve sandbox writes, fail fast on dead backend Smoke run exposed two silent-degradation paths: expired claude CLI auth produced zero-call 'completed' tasks, and balanced-mode overwrite approvals were denied without an approval handler. --- benchmarks/eval/claude-cli-backend.mjs | 28 +++++++++++++++++++++----- benchmarks/eval/run-eval.mjs | 15 ++++++++++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/benchmarks/eval/claude-cli-backend.mjs b/benchmarks/eval/claude-cli-backend.mjs index 3d4ca84..60c18b7 100644 --- a/benchmarks/eval/claude-cli-backend.mjs +++ b/benchmarks/eval/claude-cli-backend.mjs @@ -3,11 +3,16 @@ import { spawn } from 'node:child_process'; const MODEL = process.env.EVAL_MODEL ?? 'claude-haiku-4-5'; const CALL_TIMEOUT_MS = Number(process.env.EVAL_LLM_TIMEOUT_MS ?? 180000); -const tally = { calls: 0, inputTokens: 0, outputTokens: 0 }; +const tally = { calls: 0, failures: 0, inputTokens: 0, outputTokens: 0 }; export function getUsageTally() { return { ...tally }; } +function recordFailure(error) { + tally.failures += 1; + console.error(`[claude-cli-backend] call failed (#${tally.failures}): ${String(error?.message ?? error).slice(0, 200)}`); +} + function renderPrompt(messages, system) { const parts = []; if (system) parts.push(`[System]\n${system}`); @@ -25,7 +30,9 @@ function callClaude(prompt) { let stderr = ''; const timer = setTimeout(() => { child.kill('SIGKILL'); - reject(new Error('claude CLI timeout')); + const err = new Error('claude CLI timeout'); + recordFailure(err); + reject(err); }, CALL_TIMEOUT_MS); child.stdout.on('data', (d) => { stdout += d; @@ -35,20 +42,31 @@ function callClaude(prompt) { }); child.on('error', (e) => { clearTimeout(timer); + recordFailure(e); reject(e); }); child.on('close', (code) => { clearTimeout(timer); - if (code !== 0) return reject(new Error(`claude exit ${code}: ${stderr.slice(0, 300)}`)); + if (code !== 0) { + const err = new Error(`claude exit ${code}: ${stderr.slice(0, 300)}`); + recordFailure(err); + return reject(err); + } try { const parsed = JSON.parse(stdout); tally.calls += 1; tally.inputTokens += parsed.usage?.input_tokens ?? 0; tally.outputTokens += parsed.usage?.output_tokens ?? 0; - if (parsed.is_error) return reject(new Error(`claude error result: ${String(parsed.result).slice(0, 300)}`)); + if (parsed.is_error) { + const err = new Error(`claude error result: ${String(parsed.result).slice(0, 300)}`); + recordFailure(err); + return reject(err); + } resolvePromise(String(parsed.result ?? '')); } catch (e) { - reject(new Error(`bad CLI JSON: ${String(e)}; head=${stdout.slice(0, 200)}`)); + const err = new Error(`bad CLI JSON: ${String(e)}; head=${stdout.slice(0, 200)}`); + recordFailure(err); + reject(err); } }); child.stdin.end(prompt); diff --git a/benchmarks/eval/run-eval.mjs b/benchmarks/eval/run-eval.mjs index 2d170cf..c78ee08 100644 --- a/benchmarks/eval/run-eval.mjs +++ b/benchmarks/eval/run-eval.mjs @@ -72,6 +72,9 @@ for (const task of tasks) { llmBackend: backend, runLog: false, filterConsole: true, + // 评测工作区是 /tmp 下的隔离沙箱副本：覆写等 ask 级安全决策一律放行， + // 否则 balanced 模式下 bugfix/refactor 的文件覆写会被静默拒绝 + onApprovalRequest: async () => true, ...ARM_OPTIONS[ARM], onEvent: (e) => { events[e.type] = (events[e.type] ?? 0) + 1; @@ -86,6 +89,8 @@ for (const task of tasks) { const usageAfter = getUsageTally(); const checkResults = runError ? [] : runChecks(task.checks, { workspace: ws, resultText }); const pass = !runError && checkResults.length > 0 && checkResults.every((c) => c.ok); + const llmCalls = usageAfter.calls - usageBefore.calls; + const llmFailures = usageAfter.failures - usageBefore.failures; const record = { taskId: task.id, category: task.category, @@ -97,13 +102,19 @@ for (const task of tasks) { checks: checkResults, events, elapsedMs: Math.round(performance.now() - t0), - llmCalls: usageAfter.calls - usageBefore.calls, + llmCalls, + llmFailures, inputTokens: usageAfter.inputTokens - usageBefore.inputTokens, outputTokens: usageAfter.outputTokens - usageBefore.outputTokens, }; appendFileSync(OUT, `${JSON.stringify(record)}\n`); console.log( - `[${ARM}] ${task.id}: ${pass ? 'PASS' : 'FAIL'} (${record.elapsedMs}ms, ${record.llmCalls} calls, events=${JSON.stringify(events)})`, + `[${ARM}] ${task.id}: ${pass ? 'PASS' : 'FAIL'} (${record.elapsedMs}ms, ${llmCalls} calls, ${llmFailures} failures, events=${JSON.stringify(events)})`, ); + // 后端一次成功调用都没有 = 环境坏了（登录态过期等），继续跑只会产出降级噪音 + if (llmCalls === 0) { + console.error(`[${ARM}] ${task.id} 零成功 LLM 调用（失败 ${llmFailures} 次）——判定后端环境异常，中止本臂评测`); + process.exit(2); + } } console.log('done. total usage:', JSON.stringify(getUsageTally())); From e0e74f5f58976762a9f3201fdab6e8750f86fe1a Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 20:33:49 +0800 Subject: [PATCH 07/11] fix(eval): pre-run amendments from smoke diagnosis - disable external RAG for both arms (default repo pulls unrelated wiki content into query context; noise + token cost) - query tasks get --files hints mirroring real CLI usage (deterministic query planning read a nonexistent README instead of target sources); two navigation tasks intentionally keep no hints - fixture gains a minimal README (structure only, no interface docs) Task set amended before the measured run; both arms use the identical set. --- benchmarks/eval/fixture/README.md | 10 + benchmarks/eval/run-eval.mjs | 17 +- benchmarks/eval/tasks.json | 487 ++++++++++++++++++++++++------ 3 files changed, 419 insertions(+), 95 deletions(-) create mode 100644 benchmarks/eval/fixture/README.md diff --git a/benchmarks/eval/fixture/README.md b/benchmarks/eval/fixture/README.md new file mode 100644 index 0000000..b80fccc --- /dev/null +++ b/benchmarks/eval/fixture/README.md @@ -0,0 +1,10 @@ +# Eval Fixture + +架构消融评测的冻结夹具：一个最小 React + TypeScript 工程。 + +- `src/components/` —— UI 组件（PascalCase 命名导出） +- `src/hooks/` —— 自定义 hooks（`use*` 前缀） +- `src/utils/` —— 纯函数工具与对应测试 +- `npm run typecheck` / `npm run test` 为验收门禁 + +组件与函数的具体接口以源码为准，本 README 不重复文档。 diff --git a/benchmarks/eval/run-eval.mjs b/benchmarks/eval/run-eval.mjs index c78ee08..2d209fc 100644 --- a/benchmarks/eval/run-eval.mjs +++ b/benchmarks/eval/run-eval.mjs @@ -47,9 +47,13 @@ function setupWorkspace(taskId) { return ws; } -const tasks = JSON.parse(readFileSync(join(HERE, 'tasks.json'), 'utf8')).filter((t) => - SCOPE === 'smoke' ? t.smoke : true, -); +// --tasks smoke|all|（单任务用于诊断探针） +const tasks = JSON.parse(readFileSync(join(HERE, 'tasks.json'), 'utf8')).filter((t) => { + if (SCOPE === 'smoke') return t.smoke; + if (SCOPE === 'all') return true; + return t.id === SCOPE; +}); +if (tasks.length === 0) throw new Error(`--tasks ${SCOPE} 未匹配任何任务`); console.log(`arm=${ARM} scope=${SCOPE} tasks=${tasks.length} out=${OUT}`); const backend = createClaudeCliBackend(); @@ -61,6 +65,7 @@ for (const task of tasks) { const usageBefore = getUsageTally(); let resultText = ''; let agentSuccess = null; + let agentError = null; let validationCount = null; let runError = null; try { @@ -72,6 +77,9 @@ for (const task of tasks) { llmBackend: backend, runLog: false, filterConsole: true, + // 外部 RAG 仓库（默认拉 ceilf6/Lab）与夹具无关：检索噪音混入 query 上下文 + // 且消耗 token，评测两臂一律关闭 + disableRag: true, // 评测工作区是 /tmp 下的隔离沙箱副本：覆写等 ask 级安全决策一律放行， // 否则 balanced 模式下 bugfix/refactor 的文件覆写会被静默拒绝 onApprovalRequest: async () => true, @@ -82,6 +90,7 @@ for (const task of tasks) { }); resultText = result?.output ?? ''; agentSuccess = result?.success ?? null; + agentError = result?.error ? String(result.error).slice(0, 300) : null; validationCount = result?.validations?.length ?? null; } catch (error) { runError = String(error).slice(0, 400); @@ -97,8 +106,10 @@ for (const task of tasks) { arm: ARM, pass, agentSuccess, + agentError, validationCount, runError, + resultHead: resultText.slice(0, 200), checks: checkResults, events, elapsedMs: Math.round(performance.now() - t0), diff --git a/benchmarks/eval/tasks.json b/benchmarks/eval/tasks.json index d4b3b97..aa13439 100644 --- a/benchmarks/eval/tasks.json +++ b/benchmarks/eval/tasks.json @@ -6,8 +6,17 @@ "task": "Button 组件接受哪些 props？逐个说明类型与用途。", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "label" }, - { "kind": "result_contains", "pattern": "onClick" } + { + "kind": "result_contains", + "pattern": "label" + }, + { + "kind": "result_contains", + "pattern": "onClick" + } + ], + "files": [ + "src/components/Button.tsx" ] }, { @@ -17,9 +26,18 @@ "task": "列出 src 下的目录结构，说明每个目录的职责。", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "components" }, - { "kind": "result_contains", "pattern": "hooks" }, - { "kind": "result_contains", "pattern": "utils" } + { + "kind": "result_contains", + "pattern": "components" + }, + { + "kind": "result_contains", + "pattern": "hooks" + }, + { + "kind": "result_contains", + "pattern": "utils" + } ] }, { @@ -29,8 +47,17 @@ "task": "TodoList 组件如何渲染列表项？列表项的 key 用的是什么？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "key" }, - { "kind": "result_contains", "pattern": "item" } + { + "kind": "result_contains", + "pattern": "key" + }, + { + "kind": "result_contains", + "pattern": "item" + } + ], + "files": [ + "src/components/TodoList.tsx" ] }, { @@ -40,8 +67,17 @@ "task": "useToggle hook 返回什么？内部状态是怎么切换的？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "toggle" }, - { "kind": "result_contains", "pattern": "useState" } + { + "kind": "result_contains", + "pattern": "toggle" + }, + { + "kind": "result_contains", + "pattern": "useState" + } + ], + "files": [ + "src/hooks/useToggle.ts" ] }, { @@ -51,8 +87,17 @@ "task": "App 组件直接使用了哪些子组件？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "Button" }, - { "kind": "result_contains", "pattern": "TodoList" } + { + "kind": "result_contains", + "pattern": "Button" + }, + { + "kind": "result_contains", + "pattern": "TodoList" + } + ], + "files": [ + "src/App.tsx" ] }, { @@ -62,8 +107,17 @@ "task": "formatDate 函数的输入和输出分别是什么类型和格式？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "Date" }, - { "kind": "result_contains", "pattern": "string|字符串" } + { + "kind": "result_contains", + "pattern": "Date" + }, + { + "kind": "result_contains", + "pattern": "string|字符串" + } + ], + "files": [ + "src/utils/formatDate.ts" ] }, { @@ -73,7 +127,13 @@ "task": "这个项目的 tsconfig 开启了哪些编译选项？strict 开了吗？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "strict" } + { + "kind": "result_contains", + "pattern": "strict" + } + ], + "files": [ + "tsconfig.json" ] }, { @@ -83,8 +143,17 @@ "task": "clamp 函数是做什么的？三个参数分别是什么含义？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "min" }, - { "kind": "result_contains", "pattern": "max" } + { + "kind": "result_contains", + "pattern": "min" + }, + { + "kind": "result_contains", + "pattern": "max" + } + ], + "files": [ + "src/utils/clamp.ts" ] }, { @@ -94,8 +163,14 @@ "task": "项目里有哪些测试文件？分别测试什么？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "formatDate" }, - { "kind": "result_contains", "pattern": "clamp" } + { + "kind": "result_contains", + "pattern": "formatDate" + }, + { + "kind": "result_contains", + "pattern": "clamp" + } ] }, { @@ -105,8 +180,18 @@ "task": "Button 组件是命名导出还是默认导出？App 呢？", "type": "query", "checks": [ - { "kind": "result_contains", "pattern": "命名|named" }, - { "kind": "result_contains", "pattern": "默认|default" } + { + "kind": "result_contains", + "pattern": "命名|named" + }, + { + "kind": "result_contains", + "pattern": "默认|default" + } + ], + "files": [ + "src/components/Button.tsx", + "src/App.tsx" ] }, { @@ -115,11 +200,22 @@ "smoke": true, "task": "创建一个 useDebounce hook，接受 value 与 delayMs，返回防抖后的值，放在 src/hooks/useDebounce.ts。", "type": "create", - "files": ["src/hooks/useDebounce.ts"], + "files": [ + "src/hooks/useDebounce.ts" + ], "checks": [ - { "kind": "file_exists", "path": "src/hooks/useDebounce.ts" }, - { "kind": "file_contains", "path": "src/hooks/useDebounce.ts", "pattern": "export function useDebounce" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/hooks/useDebounce.ts" + }, + { + "kind": "file_contains", + "path": "src/hooks/useDebounce.ts", + "pattern": "export function useDebounce" + }, + { + "kind": "typecheck" + } ] }, { @@ -128,11 +224,22 @@ "smoke": false, "task": "创建一个 Card 组件（src/components/Card.tsx），接受 title 字符串与 children，渲染标题和内容区。", "type": "create", - "files": ["src/components/Card.tsx"], + "files": [ + "src/components/Card.tsx" + ], "checks": [ - { "kind": "file_exists", "path": "src/components/Card.tsx" }, - { "kind": "file_contains", "path": "src/components/Card.tsx", "pattern": "export function Card" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/components/Card.tsx" + }, + { + "kind": "file_contains", + "path": "src/components/Card.tsx", + "pattern": "export function Card" + }, + { + "kind": "typecheck" + } ] }, { @@ -141,11 +248,22 @@ "smoke": false, "task": "创建纯函数 capitalize（src/utils/capitalize.ts）：首字母大写、其余不变，空串原样返回。", "type": "create", - "files": ["src/utils/capitalize.ts"], + "files": [ + "src/utils/capitalize.ts" + ], "checks": [ - { "kind": "file_exists", "path": "src/utils/capitalize.ts" }, - { "kind": "file_contains", "path": "src/utils/capitalize.ts", "pattern": "export function capitalize" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/utils/capitalize.ts" + }, + { + "kind": "file_contains", + "path": "src/utils/capitalize.ts", + "pattern": "export function capitalize" + }, + { + "kind": "typecheck" + } ] }, { @@ -154,11 +272,22 @@ "smoke": false, "task": "创建 useLocalStorage hook（src/hooks/useLocalStorage.ts）：读写 localStorage 并保持 state 同步，值用 JSON 序列化。", "type": "create", - "files": ["src/hooks/useLocalStorage.ts"], + "files": [ + "src/hooks/useLocalStorage.ts" + ], "checks": [ - { "kind": "file_exists", "path": "src/hooks/useLocalStorage.ts" }, - { "kind": "file_contains", "path": "src/hooks/useLocalStorage.ts", "pattern": "export function useLocalStorage" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/hooks/useLocalStorage.ts" + }, + { + "kind": "file_contains", + "path": "src/hooks/useLocalStorage.ts", + "pattern": "export function useLocalStorage" + }, + { + "kind": "typecheck" + } ] }, { @@ -167,11 +296,22 @@ "smoke": false, "task": "创建 Badge 组件（src/components/Badge.tsx）：variant 属性为 'info' | 'success' | 'warning' 联合类型，渲染带对应 className 的 span。", "type": "create", - "files": ["src/components/Badge.tsx"], + "files": [ + "src/components/Badge.tsx" + ], "checks": [ - { "kind": "file_exists", "path": "src/components/Badge.tsx" }, - { "kind": "file_contains", "path": "src/components/Badge.tsx", "pattern": "variant" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/components/Badge.tsx" + }, + { + "kind": "file_contains", + "path": "src/components/Badge.tsx", + "pattern": "variant" + }, + { + "kind": "typecheck" + } ] }, { @@ -180,11 +320,22 @@ "smoke": false, "task": "创建纯函数 formatCurrency（src/utils/formatCurrency.ts）：数字金额格式化为带千分位、两位小数的字符串，如 1234.5 → \"1,234.50\"。", "type": "create", - "files": ["src/utils/formatCurrency.ts"], + "files": [ + "src/utils/formatCurrency.ts" + ], "checks": [ - { "kind": "file_exists", "path": "src/utils/formatCurrency.ts" }, - { "kind": "file_contains", "path": "src/utils/formatCurrency.ts", "pattern": "export function formatCurrency" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/utils/formatCurrency.ts" + }, + { + "kind": "file_contains", + "path": "src/utils/formatCurrency.ts", + "pattern": "export function formatCurrency" + }, + { + "kind": "typecheck" + } ] }, { @@ -193,11 +344,22 @@ "smoke": false, "task": "创建 Spinner 加载指示器组件（src/components/Spinner.tsx），可选 size 属性（'small' | 'large'，默认 'small'）。", "type": "create", - "files": ["src/components/Spinner.tsx"], + "files": [ + "src/components/Spinner.tsx" + ], "checks": [ - { "kind": "file_exists", "path": "src/components/Spinner.tsx" }, - { "kind": "file_contains", "path": "src/components/Spinner.tsx", "pattern": "export function Spinner" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/components/Spinner.tsx" + }, + { + "kind": "file_contains", + "path": "src/components/Spinner.tsx", + "pattern": "export function Spinner" + }, + { + "kind": "typecheck" + } ] }, { @@ -206,11 +368,22 @@ "smoke": false, "task": "创建 useInterval hook（src/hooks/useInterval.ts）：按 delayMs 周期执行回调，组件卸载时清理定时器。", "type": "create", - "files": ["src/hooks/useInterval.ts"], + "files": [ + "src/hooks/useInterval.ts" + ], "checks": [ - { "kind": "file_exists", "path": "src/hooks/useInterval.ts" }, - { "kind": "file_contains", "path": "src/hooks/useInterval.ts", "pattern": "useEffect" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/hooks/useInterval.ts" + }, + { + "kind": "file_contains", + "path": "src/hooks/useInterval.ts", + "pattern": "useEffect" + }, + { + "kind": "typecheck" + } ] }, { @@ -219,11 +392,22 @@ "smoke": false, "task": "创建纯函数 slugify（src/utils/slugify.ts）：字符串转 URL slug——小写、空白变连字符、去掉非字母数字连字符字符。", "type": "create", - "files": ["src/utils/slugify.ts"], + "files": [ + "src/utils/slugify.ts" + ], "checks": [ - { "kind": "file_exists", "path": "src/utils/slugify.ts" }, - { "kind": "file_contains", "path": "src/utils/slugify.ts", "pattern": "export function slugify" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/utils/slugify.ts" + }, + { + "kind": "file_contains", + "path": "src/utils/slugify.ts", + "pattern": "export function slugify" + }, + { + "kind": "typecheck" + } ] }, { @@ -232,11 +416,22 @@ "smoke": true, "task": "创建受控 Checkbox 组件（src/components/Checkbox.tsx）：checked 布尔与 onChange 回调，带 label 文本。", "type": "create", - "files": ["src/components/Checkbox.tsx"], + "files": [ + "src/components/Checkbox.tsx" + ], "checks": [ - { "kind": "file_exists", "path": "src/components/Checkbox.tsx" }, - { "kind": "file_contains", "path": "src/components/Checkbox.tsx", "pattern": "onChange" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/components/Checkbox.tsx" + }, + { + "kind": "file_contains", + "path": "src/components/Checkbox.tsx", + "pattern": "onChange" + }, + { + "kind": "typecheck" + } ] }, { @@ -245,10 +440,22 @@ "smoke": true, "task": "src/utils/formatDate.ts 输出的月份不对（2026-07-11 被格式化成 2026-06-11），修复它并保证现有测试通过。", "type": "modify", - "files": ["src/utils/formatDate.ts"], + "files": [ + "src/utils/formatDate.ts" + ], "checks": [ - { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/formatDate.test.ts"] }, - { "kind": "typecheck" } + { + "kind": "cmd", + "argv": [ + "npx", + "vitest", + "run", + "src/utils/formatDate.test.ts" + ] + }, + { + "kind": "typecheck" + } ] }, { @@ -257,10 +464,22 @@ "smoke": false, "task": "src/utils/clamp.ts 在值大于上界时返回了错误的边界，修复并保证现有测试通过。", "type": "modify", - "files": ["src/utils/clamp.ts"], + "files": [ + "src/utils/clamp.ts" + ], "checks": [ - { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/clamp.test.ts"] }, - { "kind": "typecheck" } + { + "kind": "cmd", + "argv": [ + "npx", + "vitest", + "run", + "src/utils/clamp.test.ts" + ] + }, + { + "kind": "typecheck" + } ] }, { @@ -269,10 +488,22 @@ "smoke": false, "task": "src/utils/pluralize.ts 对单数（count 为 1）也加了 s，修复并保证现有测试通过。", "type": "modify", - "files": ["src/utils/pluralize.ts"], + "files": [ + "src/utils/pluralize.ts" + ], "checks": [ - { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/pluralize.test.ts"] }, - { "kind": "typecheck" } + { + "kind": "cmd", + "argv": [ + "npx", + "vitest", + "run", + "src/utils/pluralize.test.ts" + ] + }, + { + "kind": "typecheck" + } ] }, { @@ -281,10 +512,22 @@ "smoke": false, "task": "src/utils/truncate.ts 截断长度差一位（maxLength=3 时留了 4 个字符），修复并保证现有测试通过。", "type": "modify", - "files": ["src/utils/truncate.ts"], + "files": [ + "src/utils/truncate.ts" + ], "checks": [ - { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/truncate.test.ts"] }, - { "kind": "typecheck" } + { + "kind": "cmd", + "argv": [ + "npx", + "vitest", + "run", + "src/utils/truncate.test.ts" + ] + }, + { + "kind": "typecheck" + } ] }, { @@ -293,10 +536,22 @@ "smoke": false, "task": "src/utils/parseQuery.ts 没有对 URL 编码的值解码（hello%20world 应变为 hello world），修复并保证现有测试通过。", "type": "modify", - "files": ["src/utils/parseQuery.ts"], + "files": [ + "src/utils/parseQuery.ts" + ], "checks": [ - { "kind": "cmd", "argv": ["npx", "vitest", "run", "src/utils/parseQuery.test.ts"] }, - { "kind": "typecheck" } + { + "kind": "cmd", + "argv": [ + "npx", + "vitest", + "run", + "src/utils/parseQuery.test.ts" + ] + }, + { + "kind": "typecheck" + } ] }, { @@ -305,10 +560,18 @@ "smoke": true, "task": "TodoList 在 items 为空时应渲染一行「暂无待办」提示而不是空 ul，重构组件实现。", "type": "modify", - "files": ["src/components/TodoList.tsx"], + "files": [ + "src/components/TodoList.tsx" + ], "checks": [ - { "kind": "file_contains", "path": "src/components/TodoList.tsx", "pattern": "暂无待办" }, - { "kind": "typecheck" } + { + "kind": "file_contains", + "path": "src/components/TodoList.tsx", + "pattern": "暂无待办" + }, + { + "kind": "typecheck" + } ] }, { @@ -317,10 +580,18 @@ "smoke": false, "task": "给 Button 组件增加可选的 disabled 布尔属性并透传到 button 元素。", "type": "modify", - "files": ["src/components/Button.tsx"], + "files": [ + "src/components/Button.tsx" + ], "checks": [ - { "kind": "file_contains", "path": "src/components/Button.tsx", "pattern": "disabled" }, - { "kind": "typecheck" } + { + "kind": "file_contains", + "path": "src/components/Button.tsx", + "pattern": "disabled" + }, + { + "kind": "typecheck" + } ] }, { @@ -329,11 +600,23 @@ "smoke": false, "task": "把 TodoList 里的列表项抽成独立的 ListItem 组件（src/components/ListItem.tsx），TodoList 改为使用它。", "type": "modify", - "files": ["src/components/TodoList.tsx", "src/components/ListItem.tsx"], + "files": [ + "src/components/TodoList.tsx", + "src/components/ListItem.tsx" + ], "checks": [ - { "kind": "file_exists", "path": "src/components/ListItem.tsx" }, - { "kind": "file_contains", "path": "src/components/TodoList.tsx", "pattern": "ListItem" }, - { "kind": "typecheck" } + { + "kind": "file_exists", + "path": "src/components/ListItem.tsx" + }, + { + "kind": "file_contains", + "path": "src/components/TodoList.tsx", + "pattern": "ListItem" + }, + { + "kind": "typecheck" + } ] }, { @@ -342,11 +625,23 @@ "smoke": false, "task": "App 组件的 h1 文案改为可配置的 title 属性，默认值保持 \"Eval Fixture\"。", "type": "modify", - "files": ["src/App.tsx"], + "files": [ + "src/App.tsx" + ], "checks": [ - { "kind": "file_contains", "path": "src/App.tsx", "pattern": "title" }, - { "kind": "file_contains", "path": "src/App.tsx", "pattern": "Eval Fixture" }, - { "kind": "typecheck" } + { + "kind": "file_contains", + "path": "src/App.tsx", + "pattern": "title" + }, + { + "kind": "file_contains", + "path": "src/App.tsx", + "pattern": "Eval Fixture" + }, + { + "kind": "typecheck" + } ] }, { @@ -355,10 +650,18 @@ "smoke": false, "task": "用 React.memo 包裹 Button 组件的导出，避免父组件重渲染时的无谓渲染。", "type": "modify", - "files": ["src/components/Button.tsx"], + "files": [ + "src/components/Button.tsx" + ], "checks": [ - { "kind": "file_contains", "path": "src/components/Button.tsx", "pattern": "memo" }, - { "kind": "typecheck" } + { + "kind": "file_contains", + "path": "src/components/Button.tsx", + "pattern": "memo" + }, + { + "kind": "typecheck" + } ] } ] From 90d9c64e3cc3c6b082de5f177fe8b7d9cf7f035c Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 20:41:21 +0800 Subject: [PATCH 08/11] =?UTF-8?q?feat(eval):=20resume=20support=20?= =?UTF-8?q?=E2=80=94=20skip=20tasks=20already=20recorded=20in=20the=20outp?= =?UTF-8?q?ut=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/eval/run-eval.mjs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/benchmarks/eval/run-eval.mjs b/benchmarks/eval/run-eval.mjs index 2d209fc..5ef8687 100644 --- a/benchmarks/eval/run-eval.mjs +++ b/benchmarks/eval/run-eval.mjs @@ -1,6 +1,6 @@ // 架构消融评测编排器。 // 用法：node benchmarks/eval/run-eval.mjs --arm full|ablation [--tasks smoke|all] [--out benchmarks/eval/out] -import { appendFileSync, cpSync, mkdirSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; +import { appendFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { performance } from 'node:perf_hooks'; @@ -48,12 +48,21 @@ function setupWorkspace(taskId) { } // --tasks smoke|all|（单任务用于诊断探针） -const tasks = JSON.parse(readFileSync(join(HERE, 'tasks.json'), 'utf8')).filter((t) => { +const scoped = JSON.parse(readFileSync(join(HERE, 'tasks.json'), 'utf8')).filter((t) => { if (SCOPE === 'smoke') return t.smoke; if (SCOPE === 'all') return true; return t.id === SCOPE; }); -if (tasks.length === 0) throw new Error(`--tasks ${SCOPE} 未匹配任何任务`); +if (scoped.length === 0) throw new Error(`--tasks ${SCOPE} 未匹配任何任务`); + +// 断点续跑：OUT 里已有记录的任务跳过（中断后重新执行同一命令即接续） +const done = new Set( + existsSync(OUT) + ? readFileSync(OUT, 'utf8').trim().split('\n').filter(Boolean).map((l) => JSON.parse(l).taskId) + : [], +); +const tasks = scoped.filter((t) => !done.has(t.id)); +if (done.size > 0) console.log(`resume: 跳过已完成 ${done.size} 条（${[...done].join(', ')}）`); console.log(`arm=${ARM} scope=${SCOPE} tasks=${tasks.length} out=${OUT}`); const backend = createClaudeCliBackend(); From 328b215ad2d549775830d0d6f945a1f6f75b6e44 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sat, 11 Jul 2026 20:46:16 +0800 Subject: [PATCH 09/11] fix(eval): make claude CLI backend a pure text generator - disallow all tools + override system prompt: directory-listing style prompts triggered tool_use and error_max_turns under --max-turns 1 - run CLI from an empty cwd: Claude Code was injecting the host repo's CLAUDE.md and project context into every eval call (contamination + tokens) - include stdout head and prompt size in failure diagnostics --- benchmarks/eval/claude-cli-backend.mjs | 27 +++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/benchmarks/eval/claude-cli-backend.mjs b/benchmarks/eval/claude-cli-backend.mjs index 60c18b7..8636dcf 100644 --- a/benchmarks/eval/claude-cli-backend.mjs +++ b/benchmarks/eval/claude-cli-backend.mjs @@ -1,4 +1,9 @@ import { spawn } from 'node:child_process'; +import { mkdirSync } from 'node:fs'; + +// 空目录作为 CLI 工作目录：阻断 Claude Code 对宿主仓库 CLAUDE.md/项目上下文的注入 +const CLI_CWD = '/tmp/frontagent-eval/claude-cli-cwd'; +mkdirSync(CLI_CWD, { recursive: true }); const MODEL = process.env.EVAL_MODEL ?? 'claude-haiku-4-5'; const CALL_TIMEOUT_MS = Number(process.env.EVAL_LLM_TIMEOUT_MS ?? 180000); @@ -24,8 +29,22 @@ function renderPrompt(messages, system) { function callClaude(prompt) { return new Promise((resolvePromise, reject) => { - const args = ['-p', '--model', MODEL, '--output-format', 'json', '--max-turns', '1']; - const child = spawn('claude', args, { stdio: ['pipe', 'pipe', 'pipe'] }); + // 纯文本生成后端：禁用全部工具，否则「列目录」类 prompt 会诱发 tool_use + // 并在 --max-turns 1 下报 error_max_turns + const args = [ + '-p', + '--model', + MODEL, + '--output-format', + 'json', + '--max-turns', + '1', + '--disallowedTools', + '*', + '--system-prompt', + '你是一次性文本生成后端：只依据提示词中给出的信息直接作答，没有任何工具可用，不要尝试读取文件或执行命令。', + ]; + const child = spawn('claude', args, { stdio: ['pipe', 'pipe', 'pipe'], cwd: CLI_CWD }); let stdout = ''; let stderr = ''; const timer = setTimeout(() => { @@ -48,7 +67,9 @@ function callClaude(prompt) { child.on('close', (code) => { clearTimeout(timer); if (code !== 0) { - const err = new Error(`claude exit ${code}: ${stderr.slice(0, 300)}`); + const err = new Error( + `claude exit ${code}: stderr=${stderr.slice(0, 300)} stdoutHead=${stdout.slice(0, 200)} promptChars=${prompt.length}`, + ); recordFailure(err); return reject(err); } From 51fd4bbaf69ea926cd670f6708dbbbc6b5b52327 Mon Sep 17 00:00:00 2001 From: WJH <3506456886@qq.com> Date: Sun, 12 Jul 2026 16:17:29 +0800 Subject: [PATCH 10/11] feat(eval): SDD ablation results and reproduction guide Frozen 30-task benchmark (27 runs per arm; 3 refactor tasks unrun in both arms due to quota, symmetrically excluded). Findings: - SDD on/off shows no measurable difference in first-pass rate (-3.7pp). - The hallucination guard recorded zero interceptions across both arms, yet syntactically invalid files (markdown fences in .tsx, TS1127) landed on disk. Three root causes located and documented in the report. --- README.md | 13 ++ benchmarks/eval/out-smoke/ablation.jsonl | 5 + benchmarks/eval/out-smoke/full.jsonl | 5 + benchmarks/eval/out/ablation.jsonl | 27 +++++ benchmarks/eval/out/full.jsonl | 27 +++++ benchmarks/eval/report.mjs | 111 ++++++++++++++---- benchmarks/results/2026-07-12-sdd-ablation.md | 86 ++++++++++++++ benchmarks/results/ablation.jsonl | 27 +++++ benchmarks/results/full.jsonl | 27 +++++ 9 files changed, 306 insertions(+), 22 deletions(-) create mode 100644 benchmarks/eval/out-smoke/ablation.jsonl create mode 100644 benchmarks/eval/out-smoke/full.jsonl create mode 100644 benchmarks/eval/out/ablation.jsonl create mode 100644 benchmarks/eval/out/full.jsonl create mode 100644 benchmarks/results/2026-07-12-sdd-ablation.md create mode 100644 benchmarks/results/ablation.jsonl create mode 100644 benchmarks/results/full.jsonl diff --git a/README.md b/README.md index faa6c2a..ce5992c 100644 --- a/README.md +++ b/README.md @@ -1297,6 +1297,19 @@ pnpm clean - [RedBox](https://github.com/Jamailar/RedBox) - Local AI creation workspace for Xiaohongshu creators. - [1flowbase](https://github.com/taichuy/1flowbase) - Virtual model gateway for publishing multi-model workflows as OpenAI/Claude-compatible endpoints, with trace, token, latency, and cost visibility. +## Ablation Benchmark (reproducible) + +A frozen 30-task benchmark measures whether SDD constraints and the hallucination guard actually improve first-pass success. It runs on an isolated fixture project with machine-checkable acceptance (`tsc --noEmit`, `vitest`, file assertions), two arms, and resume-on-interrupt. + +```bash +pnpm build +node benchmarks/eval/run-eval.mjs --arm full --tasks all # SDD on +node benchmarks/eval/run-eval.mjs --arm ablation --tasks all # SDD off +node benchmarks/eval/report.mjs benchmarks/eval/out +``` + +Latest results: [`benchmarks/results/`](benchmarks/results/). **The current findings are negative and actionable**: SDD showed no measurable effect on first-pass rate, and the hallucination guard recorded zero interceptions while syntactically invalid files still landed on disk. Three root causes are documented in the report and tracked as issues. + ## Contributing Welcome to contribute! Submit issues, bugs, or suggestions: diff --git a/benchmarks/eval/out-smoke/ablation.jsonl b/benchmarks/eval/out-smoke/ablation.jsonl new file mode 100644 index 0000000..d1f4209 --- /dev/null +++ b/benchmarks/eval/out-smoke/ablation.jsonl @@ -0,0 +1,5 @@ +{"taskId":"query-button-props","category":"query","arm":"ablation","pass":false,"agentSuccess":true,"validationCount":1,"runError":null,"checks":[{"kind":"result_contains","ok":false,"detail":"label"},{"kind":"result_contains","ok":true,"detail":"onClick"}],"events":{"task_started":1,"status_update":15,"security_decision":4,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":150729,"llmCalls":1,"llmFailures":0,"inputTokens":10,"outputTokens":952} +{"taskId":"create-usedebounce","category":"create","arm":"ablation","pass":false,"agentSuccess":false,"validationCount":3,"runError":null,"checks":[{"kind":"file_exists","ok":false,"detail":"src/hooks/useDebounce.ts"},{"kind":"file_contains","ok":false,"detail":"missing src/hooks/useDebounce.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":5,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":2,"phase_completed":2,"step_failed":1,"task_completed":1},"elapsedMs":465285,"llmCalls":12,"llmFailures":1,"inputTokens":120,"outputTokens":29639} +{"taskId":"create-checkbox","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"validationCount":3,"runError":null,"checks":[{"kind":"file_exists","ok":true,"detail":"src/components/Checkbox.tsx"},{"kind":"file_contains","ok":true,"detail":"onChange"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":7,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":323016,"llmCalls":7,"llmFailures":0,"inputTokens":70,"outputTokens":18788} +{"taskId":"bugfix-formatdate","category":"bugfix","arm":"ablation","pass":true,"agentSuccess":true,"validationCount":2,"runError":null,"checks":[{"kind":"cmd","ok":true,"detail":"npx vitest run src/utils/formatDate.test.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":8,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":253827,"llmCalls":7,"llmFailures":0,"inputTokens":70,"outputTokens":11270} +{"taskId":"refactor-todolist-empty","category":"refactor","arm":"ablation","pass":false,"agentSuccess":true,"validationCount":2,"runError":null,"checks":[{"kind":"file_contains","ok":false,"detail":"暂无待办"},{"kind":"typecheck","ok":false,"detail":"src/components/TodoList.tsx(1,1): error TS1434: Unexpected keyword or identifier.\nsrc/components/TodoList.tsx(1,16): error TS1005: ';' expected.\nsrc/components/TodoList.tsx(1,31): error TS1127: Invalid character.\nsrc/components/TodoList.tsx(1,42): error TS1127: Invalid character.\nsrc/components/Todo"}],"events":{"task_started":1,"status_update":21,"security_decision":8,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":322254,"llmCalls":7,"llmFailures":0,"inputTokens":70,"outputTokens":16738} diff --git a/benchmarks/eval/out-smoke/full.jsonl b/benchmarks/eval/out-smoke/full.jsonl new file mode 100644 index 0000000..a79e2c6 --- /dev/null +++ b/benchmarks/eval/out-smoke/full.jsonl @@ -0,0 +1,5 @@ +{"taskId":"query-button-props","category":"query","arm":"full","pass":false,"agentSuccess":true,"validationCount":1,"runError":null,"checks":[{"kind":"result_contains","ok":false,"detail":"label"},{"kind":"result_contains","ok":false,"detail":"onClick"}],"events":{"task_started":1,"status_update":15,"security_decision":4,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":140240,"llmCalls":1,"llmFailures":0,"inputTokens":10,"outputTokens":577} +{"taskId":"create-usedebounce","category":"create","arm":"full","pass":false,"agentSuccess":false,"validationCount":3,"runError":null,"checks":[{"kind":"file_exists","ok":false,"detail":"src/hooks/useDebounce.ts"},{"kind":"file_contains","ok":false,"detail":"missing src/hooks/useDebounce.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":5,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":2,"phase_completed":2,"step_failed":1,"task_completed":1},"elapsedMs":393303,"llmCalls":9,"llmFailures":1,"inputTokens":90,"outputTokens":23821} +{"taskId":"create-checkbox","category":"create","arm":"full","pass":false,"agentSuccess":false,"validationCount":3,"runError":null,"checks":[{"kind":"file_exists","ok":false,"detail":"src/components/Checkbox.tsx"},{"kind":"file_contains","ok":false,"detail":"missing src/components/Checkbox.tsx"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":5,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":2,"phase_completed":2,"step_failed":1,"task_completed":1},"elapsedMs":478316,"llmCalls":12,"llmFailures":1,"inputTokens":120,"outputTokens":33917} +{"taskId":"bugfix-formatdate","category":"bugfix","arm":"full","pass":true,"agentSuccess":true,"validationCount":2,"runError":null,"checks":[{"kind":"cmd","ok":true,"detail":"npx vitest run src/utils/formatDate.test.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":8,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":256759,"llmCalls":7,"llmFailures":0,"inputTokens":70,"outputTokens":10789} +{"taskId":"refactor-todolist-empty","category":"refactor","arm":"full","pass":false,"agentSuccess":true,"validationCount":2,"runError":null,"checks":[{"kind":"file_contains","ok":false,"detail":"暂无待办"},{"kind":"typecheck","ok":false,"detail":"src/components/TodoList.tsx(1,13): error TS1127: Invalid character.\nsrc/components/TodoList.tsx(1,14): error TS1434: Unexpected keyword or identifier.\nsrc/components/TodoList.tsx(1,57): error TS1127: Invalid character.\nsrc/components/TodoList.tsx(1,69): error TS1127: Invalid character.\nsrc/component"}],"events":{"task_started":1,"status_update":21,"security_decision":8,"rag_retrieved":1,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":275785,"llmCalls":7,"llmFailures":0,"inputTokens":70,"outputTokens":13672} diff --git a/benchmarks/eval/out/ablation.jsonl b/benchmarks/eval/out/ablation.jsonl new file mode 100644 index 0000000..b69addb --- /dev/null +++ b/benchmarks/eval/out/ablation.jsonl @@ -0,0 +1,27 @@ +{"taskId":"query-button-props","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# Button 组件 Props 说明\n\n根据当前工作区文件 `src/components/Button.tsx`，Button 组件接受以下 props：\n\n| Prop | 类型 | 用途 |\n|------|------|------|\n| **label** | `string` | 按钮显示的文本内容 |\n| **onClick** | `() => void` | 按钮点击时的回调","checks":[{"kind":"result_contains","ok":true,"detail":"label"},{"kind":"result_contains","ok":true,"detail":"onClick"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":7483,"llmCalls":1,"llmFailures":0,"inputTokens":991,"outputTokens":440} +{"taskId":"query-structure","category":"query","arm":"ablation","pass":false,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"我无法直接列出 src 下的目录结构及其职责。\n\n**原因是**：提供的证据中没有包含关于 src 目录结构的信息。既没有知识库条目描述项目结构，也没有当前工作区文件的内容。\n\n**建议**：\n- 如果你需要我分析项目结构，需要先提供 src 目录的实际内容或文件清单\n- 或者提供项目的架构文档、README 等说明性文件作为参考\n\n我是 FrontAgent，具备分析和解释前端项目结构的能力，一","checks":[{"kind":"result_contains","ok":false,"detail":"components"},{"kind":"result_contains","ok":false,"detail":"hooks"},{"kind":"result_contains","ok":false,"detail":"utils"}],"events":{"task_started":1,"status_update":16,"security_decision":4,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":2,"step_completed":2,"phase_completed":2,"filesense_navigated":1,"task_completed":1},"elapsedMs":7870,"llmCalls":1,"llmFailures":0,"inputTokens":907,"outputTokens":506} +{"taskId":"query-todolist-render","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"## 答案\n\nTodoList 组件使用 **`.map()` 遍历列表项数组**，并将每个项渲染为 `
  • ` 元素；**列表项的 key 使用的是 `item` 本身（字符串值）**。\n\n## 说明\n\n根据当前工作区文件 `src/components/TodoList.tsx` 的实现：\n\n```tsx\n{items.map((item) => (\n
  • {it","checks":[{"kind":"result_contains","ok":true,"detail":"key"},{"kind":"result_contains","ok":true,"detail":"item"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":8904,"llmCalls":1,"llmFailures":0,"inputTokens":1005,"outputTokens":625} +{"taskId":"query-usetoggle-behavior","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# useToggle Hook 返回值和状态切换说明\n\n## 返回值\n\n根据当前工作区文件 `src/hooks/useToggle.ts`，**useToggle 返回一个元组 `[boolean, () => void]`**：\n\n1. **第一项**：当前的布尔状态值（on/off）\n2. **第二项**：触发切换的函数\n\n```typescript\n[on, toggle]\n```\n\n#","checks":[{"kind":"result_contains","ok":true,"detail":"toggle"},{"kind":"result_contains","ok":true,"detail":"useState"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":8740,"llmCalls":1,"llmFailures":0,"inputTokens":994,"outputTokens":629} +{"taskId":"query-app-components","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# 回答\n\n根据当前工作区文件 `src/App.tsx`，**App 组件直接使用了 2 个子组件**：\n\n1. **Button** —— 来自 `./components/Button`，在 JSX 中调用时传入了 `label` 和 `onClick` 属性\n2. **TodoList** —— 来自 `./components/TodoList`，在 JSX 中调用时传入了 `items","checks":[{"kind":"result_contains","ok":true,"detail":"Button"},{"kind":"result_contains","ok":true,"detail":"TodoList"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":8061,"llmCalls":1,"llmFailures":0,"inputTokens":1009,"outputTokens":493} +{"taskId":"query-formatdate-io","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# 答案\n\n根据当前工作区文件 `src/utils/formatDate.ts` 的定义：\n\n**输入类型**：`Date` 对象\n\n**输出类型和格式**：`string` 类型，格式为 `YYYY-MM-DD`（例如 `2026-07-12`）\n\n## 原理说明\n\n该函数接收一个 JavaScript 的 `Date` 对象，通过以下步骤生成格式化字符串：\n- 提取年份（`getFullYe","checks":[{"kind":"result_contains","ok":true,"detail":"Date"},{"kind":"result_contains","ok":true,"detail":"string|字符串"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":9440,"llmCalls":1,"llmFailures":0,"inputTokens":1004,"outputTokens":681} +{"taskId":"query-tsconfig-strict","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# 答案\n\n**strict 已开启**。\n\n根据当前工作区文件 `tsconfig.json`，该项目开启了以下编译选项：\n\n| 选项 | 值 |\n|------|------|\n| `target` | ES2022 |\n| `module` | ESNext |\n| `moduleResolution` | bundler |\n| `jsx` | react-jsx |\n| **`stric","checks":[{"kind":"result_contains","ok":true,"detail":"strict"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":8116,"llmCalls":1,"llmFailures":0,"inputTokens":1013,"outputTokens":519} +{"taskId":"query-clamp-purpose","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# clamp 函数说明\n\n**结论：** `clamp` 函数用于将一个数值限制在指定的最小值和最大值范围内。\n\n## 三个参数含义\n\n根据当前工作区文件 `src/utils/clamp.ts` 的实现：\n\n| 参数 | 类型 | 含义 |\n|------|------|------|\n| `value` | number | 要被限制的原始值 |\n| `min` | number | 允许的","checks":[{"kind":"result_contains","ok":true,"detail":"min"},{"kind":"result_contains","ok":true,"detail":"max"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":10896,"llmCalls":1,"llmFailures":0,"inputTokens":975,"outputTokens":791} +{"taskId":"query-test-files","category":"query","arm":"ablation","pass":false,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"我无法直接回答这个问题。\n\n**原因**：当前的可用证据中，除了我的智能体身份定义外，**没有提供任何关于项目测试文件的知识库条目或当前工作区文件信息**。\n\n要回答\"项目里有哪些测试文件？分别测试什么？\"这样的问题，我需要：\n- 当前工作区的实际文件访问权限（特别是 `test/`、`__tests__/`、`.spec.js` 等测试文件）\n- 或相关的项目文档/知识库条目\n\n**建议**：\n","checks":[{"kind":"result_contains","ok":false,"detail":"formatDate"},{"kind":"result_contains","ok":false,"detail":"clamp"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":9798,"llmCalls":1,"llmFailures":0,"inputTokens":904,"outputTokens":588} +{"taskId":"query-button-export","category":"query","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"## 结论\n\n- **Button 组件**是**命名导出**\n- **App 组件**是**默认导出**\n\n## 证据\n\n来自当前工作区文件：\n\n1. **Button.tsx** 使用 `export function Button`：\n ```typescript\n export function Button({ label, onClick }: ButtonProps) { .","checks":[{"kind":"result_contains","ok":true,"detail":"命名|named"},{"kind":"result_contains","ok":true,"detail":"默认|default"}],"events":{"task_started":1,"status_update":15,"security_decision":4,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":8161,"llmCalls":1,"llmFailures":0,"inputTokens":1089,"outputTokens":504} +{"taskId":"create-usedebounce","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/hooks/useDebounce.ts 是否已存在\n✅ 创建文件 src/hooks/useDebounce.ts","checks":[{"kind":"file_exists","ok":true,"detail":"src/hooks/useDebounce.ts"},{"kind":"file_contains","ok":true,"detail":"export function useDebounce"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":322182,"llmCalls":9,"llmFailures":0,"inputTokens":18548,"outputTokens":37610} +{"taskId":"create-card","category":"create","arm":"ablation","pass":false,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/components/Card.tsx 是否已存在\n✅ 创建文件 src/components/Card.tsx","checks":[{"kind":"file_exists","ok":true,"detail":"src/components/Card.tsx"},{"kind":"file_contains","ok":false,"detail":"export function Card"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":374622,"llmCalls":9,"llmFailures":0,"inputTokens":18285,"outputTokens":40257} +{"taskId":"create-capitalize","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/utils/capitalize.ts 是否已存在\n✅ 创建文件 src/utils/capitalize.ts","checks":[{"kind":"file_exists","ok":true,"detail":"src/utils/capitalize.ts"},{"kind":"file_contains","ok":true,"detail":"export function capitalize"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":313715,"llmCalls":8,"llmFailures":0,"inputTokens":15648,"outputTokens":33029} +{"taskId":"create-uselocalstorage","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/hooks/useLocalStorage.ts 是否已存在\n✅ 创建文件 src/hooks/useLocalStorage.ts","checks":[{"kind":"file_exists","ok":true,"detail":"src/hooks/useLocalStorage.ts"},{"kind":"file_contains","ok":true,"detail":"export function useLocalStorage"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":349031,"llmCalls":9,"llmFailures":0,"inputTokens":18493,"outputTokens":37339} +{"taskId":"create-badge","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/components/Badge.tsx 是否已存在\n✅ 创建文件 src/components/Badge.tsx","checks":[{"kind":"file_exists","ok":true,"detail":"src/components/Badge.tsx"},{"kind":"file_contains","ok":true,"detail":"variant"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":301683,"llmCalls":4,"llmFailures":2,"inputTokens":7855,"outputTokens":14848} +{"taskId":"create-formatcurrency","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/utils/formatCurrency.ts 是否已存在\n✅ 创建文件 src/utils/formatCurrency.ts","checks":[{"kind":"file_exists","ok":true,"detail":"src/utils/formatCurrency.ts"},{"kind":"file_contains","ok":true,"detail":"export function formatCurrency"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":435109,"llmCalls":9,"llmFailures":0,"inputTokens":18113,"outputTokens":48064} +{"taskId":"create-spinner","category":"create","arm":"ablation","pass":false,"agentSuccess":false,"agentError":"Syntax errors found in src/components/Spinner.tsx","validationCount":3,"runError":null,"resultHead":"","checks":[{"kind":"file_exists","ok":true,"detail":"src/components/Spinner.tsx"},{"kind":"file_contains","ok":false,"detail":"export function Spinner"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":5,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":2,"phase_completed":2,"step_failed":1,"task_completed":1},"elapsedMs":630159,"llmCalls":12,"llmFailures":0,"inputTokens":23991,"outputTokens":61670} +{"taskId":"create-useinterval","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/hooks/useInterval.ts 是否已存在\n✅ 创建文件 src/hooks/useInterval.ts","checks":[{"kind":"file_exists","ok":true,"detail":"src/hooks/useInterval.ts"},{"kind":"file_contains","ok":true,"detail":"useEffect"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":457924,"llmCalls":9,"llmFailures":0,"inputTokens":18206,"outputTokens":40463} +{"taskId":"create-slugify","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/utils/slugify.ts 是否已存在\n✅ 创建文件 src/utils/slugify.ts","checks":[{"kind":"file_exists","ok":true,"detail":"src/utils/slugify.ts"},{"kind":"file_contains","ok":true,"detail":"export function slugify"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":314668,"llmCalls":8,"llmFailures":0,"inputTokens":15472,"outputTokens":30378} +{"taskId":"create-checkbox","category":"create","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":3,"runError":null,"resultHead":"执行完成 (3/3 步骤成功)\n\n✅ 按需构建目录导航上下文：create task benefits from nearby placement conventions\n✅ 检查目标路径 src/components/Checkbox.tsx 是否已存在\n✅ 创建文件 src/components/Checkbox.tsx","checks":[{"kind":"file_exists","ok":true,"detail":"src/components/Checkbox.tsx"},{"kind":"file_contains","ok":true,"detail":"onChange"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":20,"security_decision":6,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":3,"filesense_navigated":1,"step_completed":3,"phase_completed":2,"task_completed":1},"elapsedMs":391073,"llmCalls":10,"llmFailures":0,"inputTokens":21029,"outputTokens":42355} +{"taskId":"bugfix-formatdate","category":"bugfix","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"执行完成 (2/2 步骤成功)\n\n✅ 分析文件结构 src/utils/formatDate.ts\n✅ 修改文件 src/utils/formatDate.ts","checks":[{"kind":"cmd","ok":true,"detail":"npx vitest run src/utils/formatDate.test.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":7,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":298582,"llmCalls":8,"llmFailures":0,"inputTokens":14576,"outputTokens":28712} +{"taskId":"bugfix-clamp","category":"bugfix","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"执行完成 (2/2 步骤成功)\n\n✅ 分析文件结构 src/utils/clamp.ts\n✅ 修改文件 src/utils/clamp.ts","checks":[{"kind":"cmd","ok":true,"detail":"npx vitest run src/utils/clamp.test.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":7,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":331327,"llmCalls":9,"llmFailures":0,"inputTokens":16884,"outputTokens":32491} +{"taskId":"bugfix-pluralize","category":"bugfix","arm":"ablation","pass":false,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"执行完成 (2/2 步骤成功)\n\n✅ 分析文件结构 src/utils/pluralize.ts\n✅ 修改文件 src/utils/pluralize.ts","checks":[{"kind":"cmd","ok":false,"detail":"npx vitest run src/utils/pluralize.test.ts"},{"kind":"typecheck","ok":false,"detail":"src/utils/pluralize.ts(1,13): error TS1127: Invalid character.\nsrc/utils/pluralize.ts(3,1): error TS1434: Unexpected keyword or identifier.\nsrc/utils/pluralize.ts(3,11): error TS1005: ';' expected.\nsrc/utils/pluralize.ts(3,44): error TS1127: Invalid character.\nsrc/utils/pluralize.ts(3,59): error TS1"}],"events":{"task_started":1,"status_update":21,"security_decision":7,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":298556,"llmCalls":9,"llmFailures":0,"inputTokens":16921,"outputTokens":28980} +{"taskId":"bugfix-truncate","category":"bugfix","arm":"ablation","pass":false,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"执行完成 (2/2 步骤成功)\n\n✅ 分析文件结构 src/utils/truncate.ts\n✅ 修改文件 src/utils/truncate.ts","checks":[{"kind":"cmd","ok":false,"detail":"npx vitest run src/utils/truncate.test.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":7,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":338711,"llmCalls":9,"llmFailures":0,"inputTokens":16723,"outputTokens":28584} +{"taskId":"bugfix-parsequery","category":"bugfix","arm":"ablation","pass":true,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"执行完成 (2/2 步骤成功)\n\n✅ 分析文件结构 src/utils/parseQuery.ts\n✅ 修改文件 src/utils/parseQuery.ts","checks":[{"kind":"cmd","ok":true,"detail":"npx vitest run src/utils/parseQuery.test.ts"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":21,"security_decision":7,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":394377,"llmCalls":8,"llmFailures":0,"inputTokens":14583,"outputTokens":40862} +{"taskId":"refactor-todolist-empty","category":"refactor","arm":"ablation","pass":false,"agentSuccess":false,"agentError":"claude CLI timeout","validationCount":2,"runError":null,"resultHead":"","checks":[{"kind":"file_contains","ok":false,"detail":"暂无待办"},{"kind":"typecheck","ok":true,"detail":""}],"events":{"task_started":1,"status_update":22,"security_decision":5,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":1,"step_failed":1,"phase_completed":1,"task_completed":1},"elapsedMs":919701,"llmCalls":9,"llmFailures":3,"inputTokens":16383,"outputTokens":28668} +{"taskId":"refactor-button-disabled","category":"refactor","arm":"ablation","pass":false,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"执行完成 (2/2 步骤成功)\n\n✅ 分析文件结构 src/components/Button.tsx\n✅ 修改文件 src/components/Button.tsx","checks":[{"kind":"file_contains","ok":true,"detail":"disabled"},{"kind":"typecheck","ok":false,"detail":"src/components/Button.tsx(1,3): error TS1127: Invalid character.\nsrc/components/Button.tsx(1,29): error TS1127: Invalid character.\nsrc/components/Button.tsx(1,30): error TS1127: Invalid character.\nsrc/components/Button.tsx(1,43): error TS1443: Module declaration names may only use ' or \" quoted stri"}],"events":{"task_started":1,"status_update":21,"security_decision":7,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":2,"step_completed":2,"phase_completed":1,"task_completed":1},"elapsedMs":376167,"llmCalls":4,"llmFailures":3,"inputTokens":7293,"outputTokens":7589} diff --git a/benchmarks/eval/out/full.jsonl b/benchmarks/eval/out/full.jsonl new file mode 100644 index 0000000..f1d2562 --- /dev/null +++ b/benchmarks/eval/out/full.jsonl @@ -0,0 +1,27 @@ +{"taskId":"query-button-props","category":"query","arm":"full","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# Button 组件 Props 说明\n\n根据当前工作区文件 `src/components/Button.tsx` 的定义，Button 组件接受以下 props：\n\n| Prop | 类型 | 用途 |\n|------|------|------|\n| **label** | `string` | 按钮显示的文本标签 |\n| **onClick** | `() => void` | 点击按钮","checks":[{"kind":"result_contains","ok":true,"detail":"label"},{"kind":"result_contains","ok":true,"detail":"onClick"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":24151,"llmCalls":1,"llmFailures":0,"inputTokens":991,"outputTokens":550} +{"taskId":"query-structure","category":"query","arm":"full","pass":false,"agentSuccess":true,"agentError":null,"validationCount":2,"runError":null,"resultHead":"我无法列出 src 目录的结构。\n\n**原因：** 提供的证据中不包含当前工作区的任何文件内容。我没有看到关于 src 目录的本地文件信息，也没有相关的知识库条目。\n\n**我需要：**\n1. 访问当前工作区的文件系统（查看 src 目录的实际结构）\n2. 或者你直接提供 src 目录的结构信息\n\n如果你希望我分析和说明 src 目录各子目录的职责，请：\n- 让我读取当前工作区中 src 目录下的文","checks":[{"kind":"result_contains","ok":false,"detail":"components"},{"kind":"result_contains","ok":false,"detail":"hooks"},{"kind":"result_contains","ok":false,"detail":"utils"}],"events":{"task_started":1,"status_update":16,"security_decision":4,"planning_started":1,"planning_completed":1,"phase_started":2,"step_started":2,"step_completed":2,"phase_completed":2,"filesense_navigated":1,"task_completed":1},"elapsedMs":10077,"llmCalls":1,"llmFailures":0,"inputTokens":907,"outputTokens":567} +{"taskId":"query-todolist-render","category":"query","arm":"full","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"## 答案\n\n根据当前工作区文件 `src/components/TodoList.tsx` 的代码：\n\n**渲染方式：** TodoList 组件使用 `.map()` 方法遍历 `items` 数组，将每个 item 渲染为一个 `
  • ` 元素。\n\n**Key 值：** 列表项的 `key` 使用的是 **item 本身**（即字符串值），具体代码为：\n```tsx\nitems.map((","checks":[{"kind":"result_contains","ok":true,"detail":"key"},{"kind":"result_contains","ok":true,"detail":"item"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":9793,"llmCalls":1,"llmFailures":0,"inputTokens":1005,"outputTokens":605} +{"taskId":"query-usetoggle-behavior","category":"query","arm":"full","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# useToggle hook 返回和状态切换说明\n\n## 返回值\n根据当前工作区文件 `src/hooks/useToggle.ts`，**useToggle 返回一个二元组 `[boolean, () => void]`**：\n- **第一个元素**：当前的布尔状态值（初始值由参数决定，默认 `false`）\n- **第二个元素**：一个无参函数，调用它可以切换状态\n\n## 内部状态切换原理","checks":[{"kind":"result_contains","ok":true,"detail":"toggle"},{"kind":"result_contains","ok":true,"detail":"useState"}],"events":{"task_started":1,"status_update":15,"security_decision":3,"planning_started":1,"planning_completed":1,"phase_started":1,"step_started":1,"step_completed":1,"phase_completed":1,"task_completed":1},"elapsedMs":7652,"llmCalls":1,"llmFailures":0,"inputTokens":994,"outputTokens":608} +{"taskId":"query-app-components","category":"query","arm":"full","pass":true,"agentSuccess":true,"agentError":null,"validationCount":1,"runError":null,"resultHead":"# 回答\n\nApp 组件直接使用了 **2 个子组件**：\n\n1. **Button** - 来自 `./components/Button`\n2. **TodoList** - 来自 `./components/TodoList`\n\n---\n\n## 依据\n\n根据当前工作区文件 `src/App.tsx`，App 组件在其 JSX 中直接实例化了这两个组件：\n- `