mechlens 0.1.0__tar.gz

mechlens-0.1.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

mechlens-0.1.0/PKG-INFO

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: mechlens
+Version: 0.1.0
+Summary: Record and share mechanistic interpretability experiments
+Author: MechLens
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://mechlens-cloud-fgstc.ondigitalocean.app
+Keywords: interpretability,mechanistic-interpretability,sae,steering,experiments
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: python-dotenv<2,>=1
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs<2,>=1.6; extra == "docs"
+Requires-Dist: mkdocs-material<10,>=9.7; extra == "docs"
+Requires-Dist: mkdocstrings-python<3,>=2; extra == "docs"
+Requires-Dist: ruff; extra == "docs"
+Dynamic: license-file
+
+# MechLens Python SDK
+
+The lightweight client for recording an experiment in existing Python code and
+sending it to MechLens Cloud. Model execution and metric calculation stay in the
+researcher's code.
+
+This package is intentionally separate from the GPU application and has no
+Torch, TransformerLens, or SAE dependencies.
+
+## Try the complete upload flow
+
+The included demo sends a synthetic baseline and steered run with the same four
+prompt IDs. It exercises local saving, API-key authentication, validation, and
+Cloud ingestion without downloading a model.
+
+Create an API key from **MechLens → Account → API keys**. Put it in
+`examples/.env` (the demo loads this file automatically):
+
+```dotenv
+MECHLENS_API_KEY="ml_your_key_here"
+# Omit this line to upload to MechLens Cloud. For a local Cloud:
+MECHLENS_API_URL="http://localhost:5175"
+```
+
+Then run:
+
+```bash
+cd /path/to/mechlens-sdk
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e .
+python examples/steering_demo.py
+```
+
+Variables exported in your shell take priority over values in `examples/.env`.
+
+The script prints both saved-run URLs and keeps local JSON copies under
+`.mechlens/runs/`. Its outputs are clearly marked synthetic and are only for
+testing the integration.
+
+## Use it in an experiment
+
+```python
+import mechlens
+
+run = mechlens.start_run(
+    project="refusal-steering",
+    name="strength-2",
+    model={"id": "google/gemma-2-2b", "revision": "exact-revision"},
+    config={"feature_id": 1234, "steering_strength": 2.0, "seed": 42},
+    question="Does steering this feature reduce refusals?",
+    dataset={"id": "refusal-prompts", "revision": "v1"},
+)
+
+for example in dataset:
+    answer = generate_with_steering(example["prompt"], strength=2.0)
+    run.log_example(id=example["id"], input=example["prompt"], output=answer)
+
+run.log_metrics({"refusal_rate": measured_refusal_rate})
+receipt = run.finish()
+print(receipt.url)
+```
+
+Set `MECHLENS_API_KEY` to the account key. Runs upload to MechLens Cloud
+(`https://mechlens-cloud-fgstc.ondigitalocean.app`); set `MECHLENS_API_URL` for another deployment, such as
+`http://localhost:5175` during local development.
+Finished runs are written atomically to `.mechlens/runs/<run-id>.json` before
+upload. Set `MECHLENS_DIR` to choose another local directory.
+
+## Local development
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e '.[test]'
+pytest
+```
+
+## Documentation
+
+The user docs live in `docs/` and build with Material for MkDocs. The API
+reference is generated from the docstrings in `src/mechlens`, so update those
+when the public API changes.
+
+```bash
+pip install -e '.[docs]'
+mkdocs serve          # preview at http://127.0.0.1:8000
+mkdocs build          # static site in site/ (strict: broken links fail)
+```
+
+## Cloud contract
+
+The SDK sends `POST /api/sdk/runs` with an `Authorization: Bearer ...`
+header and an `Idempotency-Key` equal to the run ID. Cloud returns:
+
+```json
+{"run_id": "...", "url": "https://.../app/projects/.../runs/..."}
+```
+
+If an upload fails, `finish()` preserves the completed local run and raises
+`MechLensUploadError`. Call `run.retry_upload()` to retry during the same
+process.

mechlens-0.1.0/README.md

@@ -0,0 +1,102 @@
+# MechLens Python SDK
+
+The lightweight client for recording an experiment in existing Python code and
+sending it to MechLens Cloud. Model execution and metric calculation stay in the
+researcher's code.
+
+This package is intentionally separate from the GPU application and has no
+Torch, TransformerLens, or SAE dependencies.
+
+## Try the complete upload flow
+
+The included demo sends a synthetic baseline and steered run with the same four
+prompt IDs. It exercises local saving, API-key authentication, validation, and
+Cloud ingestion without downloading a model.
+
+Create an API key from **MechLens → Account → API keys**. Put it in
+`examples/.env` (the demo loads this file automatically):
+
+```dotenv
+MECHLENS_API_KEY="ml_your_key_here"
+# Omit this line to upload to MechLens Cloud. For a local Cloud:
+MECHLENS_API_URL="http://localhost:5175"
+```
+
+Then run:
+
+```bash
+cd /path/to/mechlens-sdk
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e .
+python examples/steering_demo.py
+```
+
+Variables exported in your shell take priority over values in `examples/.env`.
+
+The script prints both saved-run URLs and keeps local JSON copies under
+`.mechlens/runs/`. Its outputs are clearly marked synthetic and are only for
+testing the integration.
+
+## Use it in an experiment
+
+```python
+import mechlens
+
+run = mechlens.start_run(
+    project="refusal-steering",
+    name="strength-2",
+    model={"id": "google/gemma-2-2b", "revision": "exact-revision"},
+    config={"feature_id": 1234, "steering_strength": 2.0, "seed": 42},
+    question="Does steering this feature reduce refusals?",
+    dataset={"id": "refusal-prompts", "revision": "v1"},
+)
+
+for example in dataset:
+    answer = generate_with_steering(example["prompt"], strength=2.0)
+    run.log_example(id=example["id"], input=example["prompt"], output=answer)
+
+run.log_metrics({"refusal_rate": measured_refusal_rate})
+receipt = run.finish()
+print(receipt.url)
+```
+
+Set `MECHLENS_API_KEY` to the account key. Runs upload to MechLens Cloud
+(`https://mechlens-cloud-fgstc.ondigitalocean.app`); set `MECHLENS_API_URL` for another deployment, such as
+`http://localhost:5175` during local development.
+Finished runs are written atomically to `.mechlens/runs/<run-id>.json` before
+upload. Set `MECHLENS_DIR` to choose another local directory.
+
+## Local development
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e '.[test]'
+pytest
+```
+
+## Documentation
+
+The user docs live in `docs/` and build with Material for MkDocs. The API
+reference is generated from the docstrings in `src/mechlens`, so update those
+when the public API changes.
+
+```bash
+pip install -e '.[docs]'
+mkdocs serve          # preview at http://127.0.0.1:8000
+mkdocs build          # static site in site/ (strict: broken links fail)
+```
+
+## Cloud contract
+
+The SDK sends `POST /api/sdk/runs` with an `Authorization: Bearer ...`
+header and an `Idempotency-Key` equal to the run ID. Cloud returns:
+
+```json
+{"run_id": "...", "url": "https://.../app/projects/.../runs/..."}
+```
+
+If an upload fails, `finish()` preserves the completed local run and raises
+`MechLensUploadError`. Call `run.retry_upload()` to retry during the same
+process.

mechlens-0.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "mechlens"
+version = "0.1.0"
+description = "Record and share mechanistic interpretability experiments"
+readme = "README.md"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+authors = [{ name = "MechLens" }]
+requires-python = ">=3.10"
+keywords = ["interpretability", "mechanistic-interpretability", "sae", "steering", "experiments"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Typing :: Typed",
+]
+dependencies = ["httpx>=0.27,<1", "python-dotenv>=1,<2"]
+
+[project.urls]
+Homepage = "https://mechlens-cloud-fgstc.ondigitalocean.app"
+
+[project.optional-dependencies]
+test = ["pytest>=8,<9"]
+docs = ["mkdocs>=1.6,<2", "mkdocs-material>=9.7,<10", "mkdocstrings-python>=2,<3", "ruff"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+mechlens = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+

mechlens-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mechlens-0.1.0/src/mechlens/__init__.py

@@ -0,0 +1,18 @@
+"""Public interface for the MechLens experiment SDK."""
+
+from .client import CloudClient, UploadReceipt
+from .errors import MechLensConfigurationError, MechLensError, MechLensUploadError
+from .run import Run, start_run
+
+__all__ = [
+    "CloudClient",
+    "MechLensConfigurationError",
+    "MechLensError",
+    "MechLensUploadError",
+    "Run",
+    "UploadReceipt",
+    "start_run",
+]
+
+__version__ = "0.1.0"
+

mechlens-0.1.0/src/mechlens/client.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from .errors import MechLensUploadError
+
+# Statuses where sending the same run again cannot succeed.
+_NOT_RETRYABLE = {
+    400: "Cloud could not read the run.",
+    401: "Check MECHLENS_API_KEY: the key is missing, invalid, or revoked.",
+    403: "This key cannot upload to the workspace.",
+    409: "A different run with this ID already exists. Record the run again to get a new ID.",
+    413: "The run is larger than Cloud accepts. Log fewer or shorter examples.",
+    422: "Cloud rejected the run's contents. Fix the run and record it again.",
+}
+
+
+def _server_detail(response: httpx.Response) -> str:
+    """Cloud's explanation of a failed request, from FastAPI JSON or a plain-text body."""
+    try:
+        detail = response.json().get("detail")
+    except (ValueError, AttributeError):
+        return response.text.strip()[:500]
+    if isinstance(detail, list):
+        # Request validation errors: [{"loc": [...], "msg": "..."}, ...]
+        return "; ".join(
+            f"{'.'.join(str(part) for part in item.get('loc', []))}: {item.get('msg', '')}"
+            if isinstance(item, dict)
+            else str(item)
+            for item in detail
+        )
+    return str(detail) if detail else response.text.strip()[:500]
+
+
+@dataclass(frozen=True)
+class UploadReceipt:
+    """Confirmation that Cloud stored a run.
+
+    Attributes:
+        run_id: The run's ID, the same as the local record's.
+        url: Where to open the run in MechLens Cloud.
+    """
+
+    run_id: str
+    url: str
+
+
+class CloudClient:
+    """Small synchronous client for the MechLens ingestion API."""
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str,
+        timeout: float = 30.0,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self.api_key = api_key
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.transport = transport
+
+    def upload_run(self, data: Mapping[str, Any], *, local_path) -> UploadReceipt:
+        """Send one finished run to Cloud.
+
+        Raises:
+            MechLensUploadError: Cloud could not be reached or rejected the run. Its
+                `status_code` and `retryable` attributes say whether to retry.
+        """
+        run_id = str(data["id"])
+        try:
+            with httpx.Client(
+                base_url=self.base_url,
+                timeout=self.timeout,
+                transport=self.transport,
+                follow_redirects=False,
+                trust_env=False,
+            ) as client:
+                response = client.post(
+                    "/api/sdk/runs",
+                    json=data,
+                    headers={
+                        "Authorization": f"Bearer {self.api_key}",
+                        "Idempotency-Key": run_id,
+                        "User-Agent": "mechlens-python/0.1.0",
+                    },
+                )
+                response.raise_for_status()
+                payload = response.json()
+            returned_id = payload.get("run_id")
+            url = payload.get("url")
+            if returned_id != run_id or not isinstance(url, str) or not url:
+                raise ValueError("response is missing the expected run_id or url")
+            return UploadReceipt(run_id=returned_id, url=url)
+        except httpx.HTTPStatusError as exc:
+            status = exc.response.status_code
+            detail = _server_detail(exc.response)
+            hint = _NOT_RETRYABLE.get(status, "Call run.retry_upload() to try again.")
+            raise MechLensUploadError(
+                f"Cloud rejected the upload ({status}): {detail or exc.response.reason_phrase}. {hint}",
+                local_path=local_path,
+                status_code=status,
+                retryable=status not in _NOT_RETRYABLE,
+            ) from exc
+        except httpx.TransportError as exc:
+            raise MechLensUploadError(
+                f"Could not reach MechLens at {self.base_url}: {exc}. Check MECHLENS_API_URL and your "
+                "connection, then call run.retry_upload().",
+                local_path=local_path,
+            ) from exc
+        except (httpx.HTTPError, ValueError, TypeError) as exc:
+            raise MechLensUploadError(f"Could not upload run: {exc}.", local_path=local_path) from exc
+

mechlens-0.1.0/src/mechlens/errors.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+
+class MechLensError(Exception):
+    """Base exception raised by the SDK."""
+
+
+class MechLensConfigurationError(MechLensError):
+    """The SDK is missing configuration required to contact MechLens."""
+
+
+class MechLensUploadError(MechLensError):
+    """An upload failed after the run was safely stored on disk.
+
+    Attributes:
+        local_path: The saved copy of the run.
+        status_code: Cloud's HTTP status, or `None` if Cloud was never reached.
+        retryable: Whether retrying the same run could succeed. `False` for an
+            invalid key or a rejected run, which need fixing first.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        local_path: Path,
+        status_code: int | None = None,
+        retryable: bool = True,
+    ):
+        super().__init__(f"{message} Local copy: {local_path}")
+        self.local_path = local_path
+        self.status_code = status_code
+        self.retryable = retryable
+

mechlens-0.1.0/src/mechlens/run.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+import json
+import math
+import os
+import subprocess
+import tempfile
+import uuid
+from collections.abc import Mapping
+from copy import deepcopy
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from .client import CloudClient, UploadReceipt
+from .errors import MechLensConfigurationError
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+DEFAULT_API_URL = "https://mechlens-cloud-fgstc.ondigitalocean.app"
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _git_commit() -> str | None:
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            capture_output=True,
+            text=True,
+            timeout=2,
+            check=True,
+        )
+        return result.stdout.strip() or None
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _json_copy(value: Any, *, label: str) -> Any:
+    try:
+        encoded = json.dumps(value, allow_nan=False)
+        return json.loads(encoded)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"{label} must contain only finite JSON values") from exc
+
+
+class Run:
+    """One experiment run, recorded locally and uploaded when finished.
+
+    Create runs with [`start_run`][mechlens.start_run] rather than directly. A run can
+    also be used as a context manager: leaving the `with` block without an exception
+    calls [`finish`][mechlens.Run.finish].
+
+    Attributes:
+        data: The run record that will be saved and uploaded.
+        local_path: Where the finished run is written as JSON.
+    """
+
+    def __init__(self, data: dict[str, Any], *, client: CloudClient, local_dir: Path) -> None:
+        self.data = data
+        self.client = client
+        self.local_path = local_dir / f"{data['id']}.json"
+        self._example_ids: set[str] = set()
+        self._sealed = False
+        self._receipt: UploadReceipt | None = None
+
+    def log_example(self, *, id: str, input: str, output: str, **fields: Any) -> None:
+        """Record one input and the model's output for it.
+
+        Use the same `id` for the same prompt in every run you want to compare;
+        MechLens matches examples across runs by ID.
+
+        Args:
+            id: Stable identifier for this example, unique within the run.
+            input: The prompt or input given to the model.
+            output: What the model produced.
+            **fields: Per-example measurements such as `refused=True` or
+                `helpfulness=4`. Values must be JSON-serializable. Cloud reports an
+                example as "behavior changed" when these fields differ between runs.
+
+        Raises:
+            ValueError: The ID is empty or already logged, or `fields` is not
+                JSON-serializable.
+            TypeError: `input` or `output` is not a string.
+            RuntimeError: The run is already finished.
+        """
+        self._ensure_open()
+        if not isinstance(id, str) or not id:
+            raise ValueError("example id must be a non-empty string")
+        if id in self._example_ids:
+            raise ValueError(f"example id {id!r} was already logged")
+        if not isinstance(input, str) or not isinstance(output, str):
+            raise TypeError("example input and output must be strings")
+        extra = _json_copy(fields, label="example fields")
+        forbidden = {"id", "input", "output"}.intersection(extra)
+        if forbidden:
+            raise ValueError(f"example fields cannot replace {sorted(forbidden)}")
+        self.data["examples"].append({"id": id, "input": input, "output": output, **extra})
+        self._example_ids.add(id)
+
+    def log_metrics(self, metrics: Mapping[str, Any]) -> None:
+        """Record run-level metrics, merging with any logged earlier.
+
+        Args:
+            metrics: Metric names mapped to JSON-serializable values. Numbers must be
+                finite. Names containing `rate` are shown as percentages in Cloud.
+
+        Raises:
+            ValueError: A key is empty, a value is not JSON-serializable, or a number
+                is NaN or infinite.
+            RuntimeError: The run is already finished.
+        """
+        self._ensure_open()
+        if not isinstance(metrics, Mapping) or not all(isinstance(key, str) and key for key in metrics):
+            raise ValueError("metrics must be a mapping with non-empty string keys")
+        clean = _json_copy(dict(metrics), label="metrics")
+        for key, value in clean.items():
+            if isinstance(value, (int, float)) and not isinstance(value, bool) and not math.isfinite(value):
+                raise ValueError(f"metric {key!r} must be finite")
+        self.data["metrics"].update(clean)
+
+    def finish(self) -> UploadReceipt:
+        """Seal the run, save it to disk, then upload it.
+
+        The run is written to [`local_path`][mechlens.Run] before any network call,
+        so a failed upload never loses data. Calling `finish` again returns the same
+        receipt without uploading twice.
+
+        Returns:
+            The uploaded run's ID and its Cloud URL.
+
+        Raises:
+            MechLensUploadError: The upload failed. The local copy is kept. If the
+                error's `retryable` is true, call
+                [`retry_upload`][mechlens.Run.retry_upload]; otherwise its message
+                says what to fix.
+        """
+        if self._receipt is not None:
+            return self._receipt
+        if not self._sealed:
+            self.data["finished_at"] = _now()
+            self._sealed = True
+            self._save_local()
+        return self.retry_upload()
+
+    def retry_upload(self) -> UploadReceipt:
+        """Upload a finished run whose earlier upload failed.
+
+        Uploads use the run ID as an idempotency key, so retrying never creates a
+        duplicate run in Cloud.
+
+        Returns:
+            The uploaded run's ID and its Cloud URL.
+
+        Raises:
+            RuntimeError: The run has not been finished.
+            MechLensUploadError: The upload failed again.
+        """
+        if not self._sealed:
+            raise RuntimeError("finish the run before retrying its upload")
+        if self._receipt is None:
+            self._receipt = self.client.upload_run(self.data, local_path=self.local_path)
+        return self._receipt
+
+    def _save_local(self) -> None:
+        self.local_path.parent.mkdir(parents=True, exist_ok=True)
+        with tempfile.NamedTemporaryFile(
+            "w", encoding="utf-8", dir=self.local_path.parent, prefix=f".{self.local_path.name}.", delete=False
+        ) as handle:
+            temporary = Path(handle.name)
+            try:
+                json.dump(self.data, handle, indent=2, sort_keys=True, allow_nan=False)
+                handle.write("\n")
+                handle.flush()
+                os.fsync(handle.fileno())
+            except Exception:
+                temporary.unlink(missing_ok=True)
+                raise
+        os.replace(temporary, self.local_path)
+
+    def _ensure_open(self) -> None:
+        if self._sealed:
+            raise RuntimeError("this run is finished and cannot be changed")
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type, exc, traceback) -> bool:
+        if exc_type is None:
+            self.finish()
+        return False
+
+
+def start_run(
+    *,
+    project: str,
+    name: str,
+    model: Mapping[str, Any],
+    config: Mapping[str, Any],
+    question: str,
+    dataset: Mapping[str, Any] | None = None,
+    sae: Mapping[str, Any] | None = None,
+    intervention: Mapping[str, Any] | None = None,
+    generation: Mapping[str, Any] | None = None,
+    metric_definitions: Mapping[str, Any] | None = None,
+    code_commit: str | None = None,
+    api_key: str | None = None,
+    base_url: str | None = None,
+    local_dir: str | Path | None = None,
+    client: CloudClient | None = None,
+) -> Run:
+    """Create a local-first experiment run ready to receive examples and metrics.
+
+    Everything except `project`, `name`, `model`, `config` and `question` is optional,
+    but Cloud can only check what you record: a field missing from either run is
+    reported as unknown rather than assumed to match.
+
+    Args:
+        project: Groups runs that can be compared, such as a baseline and its
+            intervention.
+        name: This run's label within the project. A name containing `baseline` or
+            `control` is chosen as the default baseline in Cloud.
+        model: Must include `id`. Include `revision` so the comparison can confirm
+            both runs used the same weights.
+        config: Any other settings for this run, such as `seed` or
+            `steering_strength`.
+        question: The research question the run helps answer.
+        dataset: Where the examples came from, ideally with `id`, `revision` and
+            `split`.
+        sae: The sparse autoencoder used, if any.
+        intervention: What this run changes relative to the baseline. Leave it out of
+            the baseline run.
+        generation: Decoding settings such as `temperature` and `max_new_tokens`.
+        metric_definitions: Metric names mapped to how each is calculated, e.g.
+            `{"refusal_rate": {"description": "..."}}`.
+        code_commit: Commit of the code that produced the run. Detected with
+            `git rev-parse HEAD` when omitted.
+        api_key: Overrides the `MECHLENS_API_KEY` environment variable.
+        base_url: Overrides `MECHLENS_API_URL`. Defaults to MechLens Cloud,
+            `https://mechlens-cloud-fgstc.ondigitalocean.app`.
+        local_dir: Overrides `MECHLENS_DIR`, where finished runs are saved.
+            Defaults to `.mechlens/runs`.
+        client: A preconfigured [`CloudClient`][mechlens.CloudClient], mainly for
+            tests. When given, `api_key` and `base_url` are ignored.
+
+    Returns:
+        An open run. Log examples and metrics, then call
+        [`finish`][mechlens.Run.finish].
+
+    Raises:
+        ValueError: A required argument is empty or not JSON-serializable.
+        TypeError: `config` is not a mapping.
+        MechLensConfigurationError: No API key was passed or set in the environment.
+    """
+    for label, value in (("project", project), ("name", name), ("question", question)):
+        if not isinstance(value, str) or not value.strip():
+            raise ValueError(f"{label} must be a non-empty string")
+    if not isinstance(model, Mapping) or not isinstance(model.get("id"), str) or not model["id"]:
+        raise ValueError("model must include a non-empty string id")
+    if not isinstance(config, Mapping):
+        raise TypeError("config must be a mapping")
+
+    if client is None:
+        resolved_key = api_key or os.environ.get("MECHLENS_API_KEY")
+        if not resolved_key:
+            raise MechLensConfigurationError("Set MECHLENS_API_KEY or pass api_key to start_run")
+        resolved_url = base_url or os.environ.get("MECHLENS_API_URL", DEFAULT_API_URL)
+        client = CloudClient(api_key=resolved_key, base_url=resolved_url)
+
+    root = Path(local_dir or os.environ.get("MECHLENS_DIR", ".mechlens/runs")).expanduser()
+    data = {
+        "schema_version": 1,
+        "id": str(uuid.uuid4()),
+        "project": project.strip(),
+        "name": name.strip(),
+        "question": question.strip(),
+        "created_at": _now(),
+        "finished_at": None,
+        "model": _json_copy(dict(model), label="model"),
+        "code_commit": code_commit if code_commit is not None else _git_commit(),
+        "dataset": _json_copy(dict(dataset), label="dataset") if dataset is not None else None,
+        "sae": _json_copy(dict(sae), label="sae") if sae is not None else None,
+        "intervention": _json_copy(dict(intervention), label="intervention") if intervention is not None else None,
+        "generation": _json_copy(dict(generation), label="generation") if generation is not None else None,
+        "config": _json_copy(dict(config), label="config"),
+        "metric_definitions": (
+            _json_copy(dict(metric_definitions), label="metric_definitions")
+            if metric_definitions is not None
+            else None
+        ),
+        "metrics": {},
+        "examples": [],
+    }
+    return Run(deepcopy(data), client=client, local_dir=root)
+

mechlens-0.1.0/src/mechlens.egg-info/PKG-INFO

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: mechlens
+Version: 0.1.0
+Summary: Record and share mechanistic interpretability experiments
+Author: MechLens
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://mechlens-cloud-fgstc.ondigitalocean.app
+Keywords: interpretability,mechanistic-interpretability,sae,steering,experiments
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: python-dotenv<2,>=1
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs<2,>=1.6; extra == "docs"
+Requires-Dist: mkdocs-material<10,>=9.7; extra == "docs"
+Requires-Dist: mkdocstrings-python<3,>=2; extra == "docs"
+Requires-Dist: ruff; extra == "docs"
+Dynamic: license-file
+
+# MechLens Python SDK
+
+The lightweight client for recording an experiment in existing Python code and
+sending it to MechLens Cloud. Model execution and metric calculation stay in the
+researcher's code.
+
+This package is intentionally separate from the GPU application and has no
+Torch, TransformerLens, or SAE dependencies.
+
+## Try the complete upload flow
+
+The included demo sends a synthetic baseline and steered run with the same four
+prompt IDs. It exercises local saving, API-key authentication, validation, and
+Cloud ingestion without downloading a model.
+
+Create an API key from **MechLens → Account → API keys**. Put it in
+`examples/.env` (the demo loads this file automatically):
+
+```dotenv
+MECHLENS_API_KEY="ml_your_key_here"
+# Omit this line to upload to MechLens Cloud. For a local Cloud:
+MECHLENS_API_URL="http://localhost:5175"
+```
+
+Then run:
+
+```bash
+cd /path/to/mechlens-sdk
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e .
+python examples/steering_demo.py
+```
+
+Variables exported in your shell take priority over values in `examples/.env`.
+
+The script prints both saved-run URLs and keeps local JSON copies under
+`.mechlens/runs/`. Its outputs are clearly marked synthetic and are only for
+testing the integration.
+
+## Use it in an experiment
+
+```python
+import mechlens
+
+run = mechlens.start_run(
+    project="refusal-steering",
+    name="strength-2",
+    model={"id": "google/gemma-2-2b", "revision": "exact-revision"},
+    config={"feature_id": 1234, "steering_strength": 2.0, "seed": 42},
+    question="Does steering this feature reduce refusals?",
+    dataset={"id": "refusal-prompts", "revision": "v1"},
+)
+
+for example in dataset:
+    answer = generate_with_steering(example["prompt"], strength=2.0)
+    run.log_example(id=example["id"], input=example["prompt"], output=answer)
+
+run.log_metrics({"refusal_rate": measured_refusal_rate})
+receipt = run.finish()
+print(receipt.url)
+```
+
+Set `MECHLENS_API_KEY` to the account key. Runs upload to MechLens Cloud
+(`https://mechlens-cloud-fgstc.ondigitalocean.app`); set `MECHLENS_API_URL` for another deployment, such as
+`http://localhost:5175` during local development.
+Finished runs are written atomically to `.mechlens/runs/<run-id>.json` before
+upload. Set `MECHLENS_DIR` to choose another local directory.
+
+## Local development
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e '.[test]'
+pytest
+```
+
+## Documentation
+
+The user docs live in `docs/` and build with Material for MkDocs. The API
+reference is generated from the docstrings in `src/mechlens`, so update those
+when the public API changes.
+
+```bash
+pip install -e '.[docs]'
+mkdocs serve          # preview at http://127.0.0.1:8000
+mkdocs build          # static site in site/ (strict: broken links fail)
+```
+
+## Cloud contract
+
+The SDK sends `POST /api/sdk/runs` with an `Authorization: Bearer ...`
+header and an `Idempotency-Key` equal to the run ID. Cloud returns:
+
+```json
+{"run_id": "...", "url": "https://.../app/projects/.../runs/..."}
+```
+
+If an upload fails, `finish()` preserves the completed local run and raises
+`MechLensUploadError`. Call `run.retry_upload()` to retry during the same
+process.

mechlens-0.1.0/src/mechlens.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+src/mechlens/__init__.py
+src/mechlens/client.py
+src/mechlens/errors.py
+src/mechlens/py.typed
+src/mechlens/run.py
+src/mechlens.egg-info/PKG-INFO
+src/mechlens.egg-info/SOURCES.txt
+src/mechlens.egg-info/dependency_links.txt
+src/mechlens.egg-info/requires.txt
+src/mechlens.egg-info/top_level.txt
+tests/test_run.py

mechlens-0.1.0/src/mechlens.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

mechlens-0.1.0/src/mechlens.egg-info/requires.txt

@@ -0,0 +1,11 @@
+httpx<1,>=0.27
+python-dotenv<2,>=1
+
+[docs]
+mkdocs<2,>=1.6
+mkdocs-material<10,>=9.7
+mkdocstrings-python<3,>=2
+ruff
+
+[test]
+pytest<9,>=8

mechlens-0.1.0/src/mechlens.egg-info/top_level.txt

@@ -0,0 +1 @@
+mechlens

mechlens-0.1.0/tests/test_run.py

@@ -0,0 +1,179 @@
+import json
+
+import httpx
+import pytest
+
+import mechlens
+
+
+def client(handler):
+    return mechlens.CloudClient(
+        api_key="test-key",
+        base_url="https://cloud.example",
+        transport=httpx.MockTransport(handler),
+    )
+
+
+def test_finish_saves_before_upload_and_returns_receipt(tmp_path):
+    observed = {}
+
+    def accept(request):
+        observed["authorization"] = request.headers["authorization"]
+        observed["idempotency_key"] = request.headers["idempotency-key"]
+        observed["body"] = json.loads(request.content)
+        assert run.local_path.exists()
+        return httpx.Response(201, json={"run_id": observed["body"]["id"], "url": "https://cloud.example/run/1"})
+
+    run = mechlens.start_run(
+        project="refusal-steering",
+        name="strength-2",
+        model={"id": "google/gemma-2-2b", "revision": "abc"},
+        config={"steering_strength": 2.0},
+        question="Does steering reduce refusals?",
+        dataset={"id": "prompts", "revision": "v1"},
+        code_commit="deadbeef",
+        client=client(accept),
+        local_dir=tmp_path,
+    )
+    run.log_example(id="001", input="Hello", output="Hi")
+    run.log_metrics({"refusal_rate": 0.25})
+
+    receipt = run.finish()
+
+    assert receipt.url == "https://cloud.example/run/1"
+    assert observed["authorization"] == "Bearer test-key"
+    assert observed["idempotency_key"] == observed["body"]["id"]
+    saved = json.loads(run.local_path.read_text())
+    assert saved["examples"][0]["id"] == "001"
+    assert saved["metrics"] == {"refusal_rate": 0.25}
+    assert saved["sae"] is None
+    assert saved["intervention"] is None
+    assert saved["finished_at"].endswith("Z")
+
+
+def test_failed_upload_keeps_local_copy_and_can_be_retried(tmp_path):
+    attempts = 0
+
+    def flaky(request):
+        nonlocal attempts
+        attempts += 1
+        if attempts == 1:
+            return httpx.Response(503, text="unavailable")
+        body = json.loads(request.content)
+        return httpx.Response(200, json={"run_id": body["id"], "url": "https://cloud.example/run/recovered"})
+
+    run = mechlens.start_run(
+        project="p",
+        name="baseline",
+        model={"id": "model"},
+        config={},
+        question="q",
+        code_commit="abc",
+        client=client(flaky),
+        local_dir=tmp_path,
+    )
+
+    with pytest.raises(mechlens.MechLensUploadError) as caught:
+        run.finish()
+    assert caught.value.local_path == run.local_path
+    assert run.local_path.exists()
+
+    assert run.retry_upload().url.endswith("/recovered")
+    assert attempts == 2
+
+
+def test_duplicate_example_ids_and_changes_after_finish_are_rejected(tmp_path):
+    def accept(request):
+        body = json.loads(request.content)
+        return httpx.Response(200, json={"run_id": body["id"], "url": "https://cloud.example/run/1"})
+
+    run = mechlens.start_run(
+        project="p",
+        name="n",
+        model={"id": "m"},
+        config={},
+        question="q",
+        code_commit="abc",
+        client=client(accept),
+        local_dir=tmp_path,
+    )
+    run.log_example(id="same", input="one", output="one")
+    with pytest.raises(ValueError, match="already logged"):
+        run.log_example(id="same", input="two", output="two")
+
+    run.finish()
+    with pytest.raises(RuntimeError, match="finished"):
+        run.log_metrics({"late": 1})
+
+
+def test_missing_api_key_fails_at_start(tmp_path, monkeypatch):
+    monkeypatch.delenv("MECHLENS_API_KEY", raising=False)
+    with pytest.raises(mechlens.MechLensConfigurationError):
+        mechlens.start_run(
+            project="p",
+            name="n",
+            model={"id": "m"},
+            config={},
+            question="q",
+            code_commit="abc",
+            local_dir=tmp_path,
+        )
+
+
+def finished_upload_error(tmp_path, handler):
+    run = mechlens.start_run(
+        project="p",
+        name="n",
+        model={"id": "m"},
+        config={},
+        question="q",
+        code_commit="abc",
+        client=client(handler),
+        local_dir=tmp_path,
+    )
+    with pytest.raises(mechlens.MechLensUploadError) as caught:
+        run.finish()
+    return caught.value
+
+
+@pytest.mark.parametrize(
+    ("response", "status", "retryable", "expected"),
+    [
+        (
+            httpx.Response(401, json={"detail": "Invalid or revoked MechLens API key"}),
+            401,
+            False,
+            ["Invalid or revoked MechLens API key", "Check MECHLENS_API_KEY"],
+        ),
+        (
+            httpx.Response(422, json={"detail": [{"loc": ["body", "model", "id"], "msg": "Field required"}]}),
+            422,
+            False,
+            ["body.model.id: Field required", "record it again"],
+        ),
+        (
+            httpx.Response(413, text="Run exceeds the 32 MiB upload limit"),
+            413,
+            False,
+            ["32 MiB"],
+        ),
+        (httpx.Response(503, text="unavailable"), 503, True, ["unavailable", "retry_upload()"]),
+    ],
+)
+def test_rejected_upload_explains_cause_and_whether_to_retry(tmp_path, response, status, retryable, expected):
+    error = finished_upload_error(tmp_path, lambda request: response)
+    assert error.status_code == status
+    assert error.retryable is retryable
+    for text in expected:
+        assert text in str(error)
+
+
+def test_unreachable_cloud_names_the_url(tmp_path):
+    def refuse(request):
+        raise httpx.ConnectError("connection refused", request=request)
+
+    error = finished_upload_error(tmp_path, refuse)
+    assert error.status_code is None
+    assert error.retryable
+    assert "https://cloud.example" in str(error)
+    assert "MECHLENS_API_URL" in str(error)