journey-evals 0.1.0__tar.gz

journey_evals-0.1.0/.env.example

@@ -0,0 +1,21 @@
+# Use one Jev key name. journey-evals maps JEV_API_KEY in process; do not configure conflicting values.
+JEV_API_KEY=
+TYPESAFE_API_KEY=
+TYPESAFE_MODEL=jev-1.13.0
+# Agent evaluations use the same evaluator configuration by default. The key is removed from the
+# environment while the target graph is imported and executed.
+
+# Required for TYPE_TEXT. OpenAI-compatible helper; credentials stay server-side and are never
+# passed to browser child processes. Every *_API_KEY, *_KEY, *_TOKEN and *_SECRET value in the
+# environment is redacted from artifacts before they are written.
+TEXT_MODEL_API_KEY=
+TEXT_MODEL_BASE_URL=https://openrouter.ai/api/v1
+TEXT_MODEL=inception/mercury-2.5
+TEXT_MODEL_REASONING=none
+# Optional. Azure's v1 surface wants max_completion_tokens and reasoning_effort rather than
+# max_tokens and a reasoning object. Detected from the endpoint when unset; set it explicitly to
+# override. Accepts openai or azure.
+# TEXT_MODEL_DIALECT=azure
+
+# Journey-specific values to redact are declared in the journey's own "redact" list, not here,
+# so they travel with the journey rather than with the machine.

journey_evals-0.1.0/.github/workflows/qa.yml

@@ -0,0 +1,153 @@
+# Two trust levels, deliberately separated.
+#
+# `contracts` runs for every push and pull request, including forks. It has no secrets, makes no
+# model calls, and never reaches the network: it proves the contracts, evaluators, report shape,
+# and browser guards still hold. A fork PR can run it safely because there is nothing to steal.
+#
+# `live` runs the real model-backed journey. It is gated on the `live-qa` environment so a
+# maintainer approves each run before any credential is issued. It is deliberately *not* wired to
+# `pull_request_target` and does not consume artifacts from an untrusted job: doing either would
+# execute fork-authored code with repository credentials in scope.
+name: qa
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      live:
+        description: "Also run the live model-backed journey"
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: qa-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  contracts:
+    name: offline contracts and browser guards
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e . ruff pytest
+
+      - name: Lint
+        run: ruff check .
+
+      - name: Parse browser sources
+        run: |
+          node --check journey_evals/telemetry.js
+          node --check journey_evals/snapshot.js
+          node --check journey_evals/static/app.js
+          node --check journey_evals/static/console.js
+          node --check examples/flight.js
+
+      - name: Offline tests
+        run: pytest -q --junitxml=artifacts/ci/pytest.xml
+
+      # The guards drive a real browser against loopback fixtures and make no model calls, so they
+      # are safe on untrusted code and still catch the evidence-collection regressions that unit
+      # tests cannot see.
+      - name: Browser guards
+        run: |
+          python -m journey_evals.feasibility preflight || true
+          python scripts/check_guards.py
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: contracts-${{ github.run_id }}
+          path: artifacts/ci
+          retention-days: 14
+          if-no-files-found: ignore
+
+  live:
+    name: live journey (advisory)
+    needs: contracts
+    # Never runs automatically on a pull request. Fork code must not reach these credentials.
+    if: >-
+      github.event_name == 'workflow_dispatch' && inputs.live &&
+      github.event.repository.fork == false
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    environment: live-qa
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+
+      - name: Readiness
+        run: python -m journey_evals.feasibility preflight
+
+      # Advisory mode. Semantic evaluators are not calibrated on a corpus large enough to gate a
+      # merge, so a finding is published rather than enforced. A runner error is still a failure:
+      # "the tool could not run" and "the application is fine" are not the same statement.
+      - name: Flight journey
+        id: journey
+        env:
+          JEV_API_KEY: ${{ secrets.JEV_API_KEY }}
+          TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
+          TYPESAFE_MODEL: ${{ vars.TYPESAFE_MODEL }}
+          TEXT_MODEL: ${{ vars.TEXT_MODEL }}
+          TEXT_MODEL_BASE_URL: ${{ vars.TEXT_MODEL_BASE_URL }}
+          TEXT_MODEL_API_KEY: ${{ secrets.TEXT_MODEL_API_KEY }}
+        run: |
+          set +e
+          python -m journey_evals.cli run \
+            --journey examples/journeys/flight-booking.json \
+            --out artifacts/ci/live
+          echo "code=$?" >> "$GITHUB_OUTPUT"
+          set -e
+
+      - name: Publish result
+        if: always()
+        run: |
+          code="${{ steps.journey.outputs.code }}"
+          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
+          import json, pathlib
+          for report in sorted(pathlib.Path("artifacts/ci/live").glob("*/report.json")):
+              data = json.loads(report.read_text())
+              print(f"### {data.get('journey_id')} - {data['result']}")
+              print(f"- goal: `{data['goal_status']}`, execution: `{data.get('execution_status')}`")
+              for finding in data["findings"]:
+                  print(f"- **{finding['severity']}** {finding['title']} ({finding['confirmation']})")
+          PY
+          # 3 is a runner/execution error: the tool failed, which is never advisory.
+          if [ "$code" = "3" ]; then exit 1; fi
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: live-${{ github.run_id }}
+          path: artifacts/ci/live
+          retention-days: 14
+          if-no-files-found: ignore

journey_evals-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+.env
+.env.*
+!.env.example
+.venv/
+__pycache__/
+.pytest_cache/
+.ruff_cache/
+.DS_Store
+
+# Campaign evidence is bulky: browser screenshots, per-journey application copies and full run
+# folders stay local. The receipts that gates actually read are small, so they are tracked, which
+# is what lets another machine resume a checkpoint instead of re-spending the budget to reproduce it.
+artifacts/*
+!artifacts/feasibility/
+artifacts/feasibility/*
+!artifacts/feasibility/checkpoints/
+!artifacts/feasibility/screening/
+!artifacts/feasibility/overhead/
+!artifacts/feasibility/reliability/
+!artifacts/feasibility/budget.json
+!artifacts/feasibility/cap-raise-0.10-to-0.40.json
+!artifacts/feasibility/budget-before-cap-raise.json
+!artifacts/feasibility/cp4-source-revalidation.json
+!artifacts/feasibility/live/
+artifacts/feasibility/live/*
+!artifacts/feasibility/live/summary/
+artifacts/visuals/
+
+*.egg-info/
+
+dist/
+.tools/
+
+# The npm shim's private Python environment and packed tarballs
+npm/.venv/
+npm/*.tgz
+npm/LICENSE
+npm/vendor/*.whl

journey_evals-0.1.0/AGENTS.md

@@ -0,0 +1,16 @@
+# Jev Ultrafast
+
+Read README.md before editing. Keep the loop small: page -> indexed elements -> operation + target -> execution.
+
+- The input is one natural-language goal. Do not add site-specific plans or hardcoded field values.
+- TypeSafe chooses an operation and operation-specific target heads in one request. Consume only the selected operation's target.
+- Targets must map to observed elements and supported operations. Never let the model emit selectors or executable code.
+- TYPE_TEXT invokes the text LLM. Cache a stale retry's value only while its entire helper input is identical.
+- Never retry a browser mutation. Log execution before observing its result.
+- Screenshots are optional; the model does not consume them. Keep demonstration footage at its original speed.
+- Keep credentials server-side and .env ignored. Tests must not call paid APIs.
+- Verify actual final outcomes independently. A DONE choice is not proof of success.
+- Keep examples, README claims, raw evidence, and model-call counts consistent.
+- Do not commit or push unless the user requests it.
+
+Checks: uv run ruff check ., uv run pytest, node --check jev_ultrafast/static/app.js, node --check journey_evals/static/console.js, uv build.

journey_evals-0.1.0/LICENSE

@@ -0,0 +1,29 @@
+MIT License
+
+Copyright (c) 2026 Browser Use
+Copyright (c) 2026 Pratyush Garg
+
+This project began as a fork of browser-use/jev-ultrafast, which is MIT
+licensed. The `jev_ultrafast` package is that upstream work, kept under its own
+name; the `journey_evals` package and everything built around it are this
+project's own contribution. Both are covered by the terms below, and the
+upstream copyright notice is retained as that licence requires. See NOTICE for
+the full provenance.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

journey_evals-0.1.0/NOTICE

@@ -0,0 +1,37 @@
+Journey Evals
+Copyright (c) 2026 Pratyush Garg
+
+This product includes software developed by Browser Use.
+
+    jev_ultrafast/
+    Copyright (c) 2026 Browser Use
+    https://github.com/browser-use/jev-ultrafast
+    Licensed under the MIT License.
+
+The `jev_ultrafast` package is the browser agent this project was built around:
+the actor loop, the CDP adapter, the model transport and the prompts. It was
+imported from browser-use/jev-ultrafast at commit
+452c1ad2dd628008f1d5608f28158d76e49e6cc0 and is kept under its own name rather
+than absorbed, so that its provenance stays visible in the source tree.
+
+It has since been modified in this fork. The changes are limited and are
+recorded in the Git history; the most substantial is that `jev_ultrafast/model.py`
+now carries a provider's 4xx response body into the raised error, because that
+body is often the only available diagnosis of a rejected request.
+
+The `journey_evals` package -- the journey and agent evaluation runtime, the
+reporting and evidence model, the budgeting, the consoles and the command line
+interface -- is this project's own work.
+
+Both are distributed under the MIT License. See LICENSE.
+
+---
+
+Third-party runtime dependencies are installed from their own distributions and
+are not vendored here. They carry their own licences:
+
+    browser-harness    https://github.com/browser-use/browser-harness
+    httpx              https://github.com/encode/httpx
+
+The optional `langchain-demo` extra, used only by the example agents under
+`examples/`, additionally installs langchain, langgraph and langchain-openai.