savant-code 0.0.19 → 0.0.20

package/evals/v2/tests/schema.test.ts

@@ -1,127 +0,0 @@
-import { describe, expect, it } from 'bun:test'
-
-import {
-  taskDefinitionSchema,
-  taskRegistrySchema,
-  type TaskDefinition,
-} from '../src/schema'
-
-function createValidTask(): TaskDefinition {
-  return {
-    schema_version: '2.0',
-    task_id: 'savant-v2-auth-jwt-001',
-    category: 'multi_agent_orchestration',
-    difficulty: 'medium',
-    environment: {
-      setup_script: 'bun install',
-      network_disabled: true,
-    },
-    inputs: {
-      prompt: 'Migrate auth.ts to use the Jose library.',
-    },
-    validation: {
-      timeout_seconds: 300,
-      deterministic_checks: [
-        {
-          command: 'bun run build',
-          expected_exit_code: 0,
-          retry_count: 0,
-          retry_condition: 'infra',
-        },
-        {
-          command: 'tsc --noEmit',
-          expected_exit_code: 0,
-          retry_count: 0,
-          retry_condition: 'infra',
-        },
-        {
-          command: 'bun test src/auth.test.ts',
-          expected_exit_code: 0,
-          retry_count: 0,
-          retry_condition: 'infra',
-        },
-      ],
-      fsm_assertions: {
-        strict_phase_order: true,
-        allow_write_in_red: false,
-        expected_phase_sequence: ['RED', 'GREEN', 'AUDIT', 'COMPLETE'],
-      },
-      custom_tool_checks: [
-        {
-          tool_name: 'generate_ast_graph',
-          expected_calls: '>=1',
-        },
-      ],
-    },
-  }
-}
-
-describe('taskDefinitionSchema', () => {
-  it('accepts a valid task definition', () => {
-    const task = createValidTask()
-    const result = taskDefinitionSchema.safeParse(task)
-    expect(result.success).toBe(true)
-  })
-
-  it('rejects missing schema_version', () => {
-    const task = createValidTask()
-    // @ts-expect-error testing invalid input
-    delete task.schema_version
-    const result = taskDefinitionSchema.safeParse(task)
-    expect(result.success).toBe(false)
-  })
-
-  it('rejects invalid task_id with uppercase characters', () => {
-    const task = { ...createValidTask(), task_id: 'Savant-V2-001' }
-    const result = taskDefinitionSchema.safeParse(task)
-    expect(result.success).toBe(false)
-  })
-
-  it('rejects unknown category', () => {
-    const task = { ...createValidTask(), category: 'unknown_category' }
-    const result = taskDefinitionSchema.safeParse(task)
-    expect(result.success).toBe(false)
-  })
-
-  it('rejects empty deterministic_checks array', () => {
-    const task = {
-      ...createValidTask(),
-      validation: { ...createValidTask().validation, deterministic_checks: [] },
-    }
-    const result = taskDefinitionSchema.safeParse(task)
-    expect(result.success).toBe(false)
-  })
-
-  it('rejects negative timeout_seconds', () => {
-    const task = {
-      ...createValidTask(),
-      validation: { ...createValidTask().validation, timeout_seconds: -1 },
-    }
-    const result = taskDefinitionSchema.safeParse(task)
-    expect(result.success).toBe(false)
-  })
-
-  it('applies default values for optional fields', () => {
-    const input = { ...createValidTask() }
-    const parsed = taskDefinitionSchema.parse(input)
-    expect(parsed.validation.deterministic_checks[0].expected_exit_code).toBe(0)
-    expect(parsed.validation.deterministic_checks[0].retry_count).toBe(0)
-    expect(parsed.environment.network_disabled).toBe(true)
-  })
-})
-
-describe('taskRegistrySchema', () => {
-  it('accepts a registry keyed by task_id', () => {
-    const task = createValidTask()
-    const registry = { [task.task_id]: task }
-    const result = taskRegistrySchema.safeParse(registry)
-    expect(result.success).toBe(true)
-  })
-
-  it('rejects a registry where a value is an invalid task definition', () => {
-    const task = createValidTask()
-    const registry = { [task.task_id]: { ...task, task_id: 'BAD ID' } }
-    const result = taskRegistrySchema.safeParse(registry)
-    expect(result.success).toBe(false)
-  })
-})

package/evals/v2/tests/tempdir-sandbox.test.ts

@@ -1,49 +0,0 @@
-import { existsSync } from 'node:fs'
-
-import { describe, it, expect, beforeEach, afterEach } from 'bun:test'
-
-import { TempDirSandbox } from '../src/sandboxes/tempdir'
-
-describe('TempDirSandbox', () => {
-  let sandbox: TempDirSandbox
-
-  beforeEach(async () => {
-    sandbox = new TempDirSandbox({ prefix: 'bench-test-' })
-    await sandbox.prepare()
-  })
-
-  afterEach(async () => {
-    await sandbox.teardown()
-  })
-
-  it('creates a working directory that exists', () => {
-    const dir = sandbox.getWorkingDir()
-    expect(dir).toBeTruthy()
-    expect(existsSync(dir)).toBe(true)
-  })
-
-  it('runs a simple command and captures stdout', async () => {
-    const isWindows = process.platform === 'win32'
-    const result = await sandbox.runCommand(
-      isWindows ? 'echo hello' : 'printf hello',
-      {
-        shell: true,
-      },
-    )
-    expect(result.exitCode).toBe(0)
-    expect(result.stdout).toContain('hello')
-  })
-
-  it('captures stderr on failed commands', async () => {
-    const result = await sandbox.runCommand('this_command_should_not_exist', {
-      shell: true,
-    })
-    expect(result.exitCode).not.toBe(0)
-  })
-
-  it('removes the directory on teardown', async () => {
-    const dir = sandbox.getWorkingDir()
-    await sandbox.teardown()
-    expect(existsSync(dir)).toBe(false)
-  })
-})

package/evals/v2/tests/trace.test.ts

@@ -1,76 +0,0 @@
-import { describe, it, expect } from 'bun:test'
-
-import { TraceCollector } from '../src/trace'
-
-import type { PrintModeEvent } from '@savant-code/common/types/print-mode'
-
-describe('TraceCollector', () => {
-  it('starts with an empty trace', () => {
-    const collector = new TraceCollector('task-001', 'run-001')
-    const trace = collector.finalize()
-    expect(trace.task_id).toBe('task-001')
-    expect(trace.run_id).toBe('run-001')
-    expect(trace.events).toEqual([])
-    expect(trace.current_phase).toBe('idle')
-    expect(trace.metadata.final_phase).toBe('idle')
-  })
-
-  it('records print events', () => {
-    const collector = new TraceCollector('task-001', 'run-001')
-    collector.recordPrintEvent({ type: 'text', text: 'hello' })
-    const trace = collector.finalize()
-    expect(trace.events.length).toBe(1)
-    expect(trace.events[0]).toEqual({
-      type: 'print',
-      raw: { type: 'text', text: 'hello' },
-    })
-    expect(trace.metadata.total_steps).toBe(1)
-  })
-
-  it('derives phase transitions from transition_phase tool calls', () => {
-    const collector = new TraceCollector('task-001', 'run-001')
-    collector.recordPrintEvent({
-      type: 'tool_call',
-      toolCallId: 'tc-1',
-      toolName: 'transition_phase',
-      input: { phase: 'red' },
-    } as PrintModeEvent)
-
-    const trace = collector.finalize()
-    expect(trace.events.length).toBe(2)
-    expect(trace.current_phase).toBe('red')
-    expect(trace.metadata.phase_transition_count).toBe(1)
-  })
-
-  it('records stream chunks', () => {
-    const collector = new TraceCollector('task-001', 'run-001')
-    collector.recordStreamChunk({
-      type: 'subagent_chunk',
-      agentId: 'agent-1',
-      agentType: 'forge',
-      chunk: 'working...',
-    })
-    collector.recordStreamChunk({
-      type: 'reasoning_chunk',
-      agentId: 'agent-2',
-      ancestorRunIds: [],
-      chunk: 'because...',
-    })
-
-    const trace = collector.finalize()
-    expect(trace.events.some((e) => e.type === 'subagent_chunk')).toBe(true)
-    expect(trace.events.some((e) => e.type === 'reasoning_chunk')).toBe(true)
-  })
-
-  it('records faults', () => {
-    const collector = new TraceCollector('task-001', 'run-001')
-    collector.recordFault({
-      type: 'tool_failure',
-      message: 'boom',
-      targetTool: 'write_file',
-    })
-
-    const trace = collector.finalize()
-    expect(trace.events.some((e) => e.type === 'fault_injected')).toBe(true)
-  })
-})

package/evals/v2/tests/verify.test.ts

@@ -1,224 +0,0 @@
-import { writeFile } from 'node:fs/promises'
-import path from 'node:path'
-
-import { describe, it, expect, beforeEach } from 'bun:test'
-
-import { TempDirSandbox } from '../src/sandboxes/tempdir'
-import { DeterministicVerifier } from '../src/verify'
-
-import type { Sandbox, CommandResult } from '../src/sandbox'
-import type { TaskDefinition } from '../src/schema'
-
-function makeTask(overrides: Partial<TaskDefinition> = {}): TaskDefinition {
-  return {
-    schema_version: '2.0',
-    task_id: 'verify-test-001',
-    category: 'pure_coding',
-    difficulty: 'easy',
-    environment: {
-      setup_script: 'echo setup',
-      network_disabled: true,
-    },
-    inputs: { prompt: 'fix it' },
-    validation: {
-      timeout_seconds: 60,
-      deterministic_checks: [
-        {
-          command: 'echo ok',
-          expected_exit_code: 0,
-          retry_count: 0,
-          retry_condition: 'infra',
-        },
-      ],
-    },
-    ...overrides,
-  }
-}
-
-class MockSandbox implements Sandbox {
-  public id = 'mock'
-  public calls: { command: string; options?: unknown }[] = []
-  private responses: CommandResult[] = []
-
-  setResponses(responses: CommandResult[]) {
-    this.responses = responses
-  }
-
-  async prepare(): Promise<void> {}
-  getWorkingDir(): string {
-    return '/tmp/mock'
-  }
-  async runCommand(command: string, options?: unknown): Promise<CommandResult> {
-    this.calls.push({ command, options })
-    if (this.responses.length === 0) {
-      throw new Error('No mock response configured')
-    }
-    return this.responses.shift()!
-  }
-  async teardown(): Promise<void> {}
-}
-
-describe('DeterministicVerifier', () => {
-  let sandbox: MockSandbox
-  let verifier: DeterministicVerifier
-
-  beforeEach(() => {
-    sandbox = new MockSandbox()
-    verifier = new DeterministicVerifier(sandbox)
-  })
-
-  it('passes when all checks pass', async () => {
-    sandbox.setResponses([{ exitCode: 0, stdout: 'ok', stderr: '' }])
-    const task = makeTask()
-    const result = await verifier.verify(task)
-    expect(result.passed).toBe(true)
-    expect(result.status).toBe('PASS')
-    expect(result.checks[0].status).toBe('PASS')
-    expect(result.checks[0].retries).toBe(0)
-  })
-
-  it('fails when a check fails', async () => {
-    sandbox.setResponses([{ exitCode: 1, stdout: '', stderr: 'bad' }])
-    const task = makeTask()
-    const result = await verifier.verify(task)
-    expect(result.passed).toBe(false)
-    expect(result.status).toBe('FAIL')
-    expect(result.checks[0].status).toBe('FAIL')
-  })
-
-  it('retries and marks flaky passes', async () => {
-    sandbox.setResponses([
-      { exitCode: 1, stdout: '', stderr: 'timeout' },
-      { exitCode: 0, stdout: 'ok', stderr: '' },
-    ])
-    const task = makeTask({
-      validation: {
-        timeout_seconds: 60,
-        deterministic_checks: [
-          {
-            command: 'echo ok',
-            expected_exit_code: 0,
-            retry_count: 2,
-            retry_condition: 'always',
-          },
-        ],
-      },
-    })
-    const result = await verifier.verify(task)
-    expect(result.passed).toBe(true)
-    expect(result.status).toBe('FLAKY')
-    expect(result.checks[0].status).toBe('FLAKY_PASS')
-    expect(result.checks[0].retries).toBe(1)
-    expect(sandbox.calls.length).toBe(2)
-  })
-
-  it('does not retry non-infra failures when retry_condition is infra', async () => {
-    sandbox.setResponses([{ exitCode: 1, stdout: '', stderr: 'syntax error' }])
-    const task = makeTask({
-      validation: {
-        timeout_seconds: 60,
-        deterministic_checks: [
-          {
-            command: 'echo ok',
-            expected_exit_code: 0,
-            retry_count: 2,
-            retry_condition: 'infra',
-          },
-        ],
-      },
-    })
-    const result = await verifier.verify(task)
-    expect(result.passed).toBe(false)
-    expect(result.checks[0].status).toBe('FAIL')
-    expect(sandbox.calls.length).toBe(1)
-  })
-
-  it('applies the golden patch before checks when requested', async () => {
-    const sandbox = new TempDirSandbox({ prefix: 'verify-golden-' })
-    await sandbox.prepare()
-
-    try {
-      const workingDir = sandbox.getWorkingDir()
-      const patchPath = path.join(workingDir, 'golden.patch')
-      const filePath = path.join(workingDir, 'data.txt')
-
-      await writeFile(filePath, 'old\n', 'utf-8')
-      await writeFile(
-        patchPath,
-        `--- data.txt
-+++ data.txt
-@@ -1 +1 @@
--old
-+new
-`,
-        'utf-8',
-      )
-
-      const task = makeTask({
-        golden_patch: patchPath,
-        validation: {
-          timeout_seconds: 60,
-          deterministic_checks: [
-            {
-              command:
-                process.platform === 'win32' ? 'type data.txt' : 'cat data.txt',
-              expected_exit_code: 0,
-              retry_count: 0,
-              retry_condition: 'infra',
-            },
-          ],
-        },
-      })
-
-      const verifier = new DeterministicVerifier(sandbox, {
-        applyGoldenPatch: true,
-      })
-      const result = await verifier.verify(task)
-      expect(result.passed).toBe(true)
-      expect(result.checks[0].status).toBe('PASS')
-      expect(result.checks[0].result.stdout).toContain('new')
-    } finally {
-      await sandbox.teardown()
-    }
-  })
-
-  it('marks a timed-out check as TIMEOUT', async () => {
-    // The actual command string is not executed because we mock the sandbox.
-    // The mock returns timedOut:true, which should be classified as TIMEOUT.
-    sandbox.setResponses([
-      { exitCode: 1, stdout: '', stderr: 'killed', timedOut: true },
-    ])
-    const task = makeTask({
-      validation: {
-        timeout_seconds: 60,
-        deterministic_checks: [
-          {
-            command: 'long-running command',
-            expected_exit_code: 0,
-            retry_count: 0,
-            retry_condition: 'infra',
-          },
-        ],
-      },
-    })
-    const result = await verifier.verify(task)
-    expect(result.passed).toBe(false)
-    expect(result.checks[0].status).toBe('TIMEOUT')
-  })
-
-  it('marks an exception as ERROR', async () => {
-    const throwingSandbox: Sandbox = {
-      id: 'throw',
-      prepare: async () => {},
-      getWorkingDir: () => '/tmp',
-      runCommand: async () => {
-        throw new Error('spawn failure')
-      },
-      teardown: async () => {},
-    }
-    const v = new DeterministicVerifier(throwingSandbox)
-    const result = await v.verify(makeTask())
-    expect(result.checks[0].status).toBe('ERROR')
-    expect(result.passed).toBe(false)
-  })
-})

package/license

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2026 Spencer Howell (Savant AI)
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.