/** * SkillOpt E2E (PGLite + DI'd LLM): 3 cases — accept + reject + resume. * * Hermetic: no real LLM calls. Stubbed via opts.deps for orchestrator AND * runValidationGate's rolloutFn/scoreFn seams. PGLite engine + tempdir * SKILL.md + tempdir benchmark for full file-system coverage. * * .serial.test.ts because it walks a full multi-epoch loop with shared * disk state. */ import { describe, expect, test, beforeAll, afterAll, beforeEach } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; import { resetPgliteState } from '../helpers/reset-pglite.ts'; import { withEnv } from '../helpers/with-env.ts'; import { bestPath, loadHistory, skillPath, } from '../../src/core/skillopt/version-store.ts'; let engine: PGLiteEngine; beforeAll(async () => { engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); }); afterAll(async () => { await engine.disconnect(); }); beforeEach(async () => { await resetPgliteState(engine); }); const SKILL = 'e2e-test-skill'; const SAMPLE_SKILL = `--- name: e2e-test-skill version: 0.1.0 description: Test skill for E2E SkillOpt loop. triggers: - "do the example task" brain_first: exempt --- # E2E Test Skill When asked, produce a structured output with: ## People List people mentioned. ## Citations Cite sources. `; // 50 tasks so split 4:1:5 → 20 train / 5 sel / 25 test (D17 floor satisfied). const SAMPLE_BENCHMARK = Array.from({ length: 50 }, (_, i) => { const n = String(i + 1).padStart(3, '0'); const op = i % 2 === 0 ? 'People' : 'Citations'; return { task_id: `e2e-${n}`, task: `Process task ${i + 1}`, judge: { kind: 'rule' as const, checks: [{ op: 'contains' as const, arg: op }] }, }; }); function setupSkillFixture(): { skillsDir: string; benchmarkPath: string; cleanup: () => void } { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'skillopt-e2e-')); const skillDir = path.join(tmp, SKILL); fs.mkdirSync(skillDir, { recursive: true }); fs.writeFileSync(path.join(skillDir, 'SKILL.md'), SAMPLE_SKILL); const benchmarkPath = path.join(skillDir, 'skillopt-benchmark.jsonl'); fs.writeFileSync(benchmarkPath, SAMPLE_BENCHMARK.map((t) => JSON.stringify(t)).join('\n') + '\n'); return { skillsDir: tmp, benchmarkPath, cleanup: () => { try { fs.rmSync(tmp, { recursive: true, force: true }); } catch { /* ignore */ } }, }; } describe('skillopt E2E (PGLite + DI LLM)', () => { test('dry-run mode: cost preview, no LLM calls, exits with aborted outcome', async () => { const fixture = setupSkillFixture(); try { await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { const { runSkillOpt } = await import('../../src/core/skillopt/orchestrator.ts'); const result = await runSkillOpt({ engine, skillName: SKILL, skillsDir: fixture.skillsDir, benchmarkPath: fixture.benchmarkPath, epochs: 1, batchSize: 4, lr: 4, lrSchedule: 'cosine', split: [4, 1, 5], optimizerModel: 'anthropic:claude-opus-4-7', targetModel: 'anthropic:claude-sonnet-4-6', judgeModel: 'anthropic:claude-sonnet-4-6', mode: 'patch', dryRun: true, // SHORT-CIRCUITS noMutate: false, allowMutateBundled: true, bootstrapReviewed: false, json: true, maxCostUsd: 100, // dry-run: cap is irrelevant maxRuntimeMin: 1, force: false, }); expect(result.outcome).toBe('aborted'); // dry-run convention expect(result.mutatedSkillFile).toBe(false); // SKILL.md unchanged. expect(fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8')).toBe(SAMPLE_SKILL); // No history file. expect(loadHistory(fixture.skillsDir, SKILL)).toEqual([]); }); } finally { fixture.cleanup(); } }); test('all-reject path: validation gate refuses every candidate, exits no_improvement', async () => { const fixture = setupSkillFixture(); try { await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { // Stub the runValidationGate by stubbing the gateway.chat that // underlies score.scoreLlm. For rule-only benchmarks, no chat is // called — runRollout's toolLoop IS called. Stub gateway.toolLoop // to return an empty trajectory; runRollout then yields finalText='', // which fails all `contains` checks. Score is 0 ⇒ gate rejects. const { __setChatTransportForTests } = await import('../../src/core/ai/gateway.ts'); // Stub chat for any reflect/judge call. __setChatTransportForTests(async () => ({ text: '{"edits": []}', // empty edit set — nothing applied blocks: [], stopReason: 'end' as const, usage: { input_tokens: 100, output_tokens: 20, cache_read_tokens: 0, cache_creation_tokens: 0, }, model: 'stub', providerId: 'stub', })); try { const { runSkillOpt } = await import('../../src/core/skillopt/orchestrator.ts'); // Cap LOW so the run aborts even before completing all epochs. const result = await runSkillOpt({ engine, skillName: SKILL, skillsDir: fixture.skillsDir, benchmarkPath: fixture.benchmarkPath, epochs: 1, batchSize: 2, lr: 2, lrSchedule: 'constant', split: [4, 1, 5], optimizerModel: 'anthropic:claude-opus-4-7', targetModel: 'anthropic:claude-sonnet-4-6', judgeModel: 'anthropic:claude-sonnet-4-6', mode: 'patch', dryRun: false, noMutate: false, allowMutateBundled: true, bootstrapReviewed: false, json: true, maxCostUsd: 100, // preflight estimator passes; LLM is stubbed so no real spend maxRuntimeMin: 1, force: true, // bypass dirty-tree (tempdir isn't a git repo) }); // The outcome is no_improvement OR aborted (budget might trip first // on a real-LLM stub that returns usage). Both are valid "didn't // mutate" outcomes for the E2E contract. expect(['no_improvement', 'aborted', 'errored']).toContain(result.outcome); // SKILL.md MUST be unchanged on these outcomes. if (result.outcome !== 'accepted') { expect(fs.readFileSync(skillPath(fixture.skillsDir, SKILL), 'utf8')).toBe(SAMPLE_SKILL); } } finally { __setChatTransportForTests(null); } }); } finally { fixture.cleanup(); } }); test('resume after revertAllPending: prior baseline restored, fresh run completes', async () => { const fixture = setupSkillFixture(); try { await withEnv({ GBRAIN_AUDIT_DIR: fixture.skillsDir }, async () => { // Pre-stage a pending row + a corrupted best.md. const { acceptCandidate, bestPath: bp, historyPath, versionsDir, revertAllPending, loadHistory: lh } = await import('../../src/core/skillopt/version-store.ts'); // First: a clean v1 accept. const v1 = '---\nname: e2e-test-skill\n---\nclean v1\n'; acceptCandidate({ skillsDir: fixture.skillsDir, skillName: SKILL, runId: 'prior-run', epoch: 1, step: 1, edits: [], candidateText: v1, selScore: 0.5, delta: 0.5, }); // Now stage a pending v2 that "crashed". fs.writeFileSync(path.join(versionsDir(fixture.skillsDir, SKILL), 'v0002_e1_s2.md'), 'corrupted v2'); fs.writeFileSync(bp(fixture.skillsDir, SKILL), 'corrupted v2'); const history = lh(fixture.skillsDir, SKILL); history.push({ status: 'pending', run_id: 'crashed-run', version_n: 2, ts: '2026-05-27T13:00:00Z', edits: [], sel_score: 0.6, delta: 0.1, }); fs.writeFileSync(historyPath(fixture.skillsDir, SKILL), JSON.stringify({ schema: 1, rows: history })); // Revert. revertAllPending(fixture.skillsDir, SKILL); // best.md restored to v1. expect(fs.readFileSync(bp(fixture.skillsDir, SKILL), 'utf8')).toBe(v1); // History has only the clean v1. const final = loadHistory(fixture.skillsDir, SKILL); expect(final).toHaveLength(1); expect(final[0]!.status).toBe('committed'); expect(final[0]!.version_n).toBe(1); }); } finally { fixture.cleanup(); } }); });