weightslab 1.0.0__tar.gz

weightslab-1.0.0/.env.template

@@ -0,0 +1,3 @@
+OPENROUTER_API_KEY=
+GOOGLE_API_KEY=
+OPENAI_API_KEY=

weightslab-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,489 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, dev ]
+    tags: [ 'v*' ]
+  pull_request:
+    branches: [ main ]
+    types: [ opened, reopened, ready_for_review ]
+  workflow_dispatch:
+
+jobs:
+  code-quality:
+    if: ${{ github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Fetch all history for comparison
+
+      - name: Set up Python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Cache pip and Ruff
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/ruff
+          key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-lint-cache
+          restore-keys: |
+            ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-
+            ${{ runner.os }}-
+
+      - name: Get changed Python files
+        id: changed-files
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            BASE_SHA=${{ github.event.pull_request.base.sha }}
+            HEAD_SHA=${{ github.event.pull_request.head.sha }}
+          else
+            BASE_SHA=${{ github.event.before }}
+            HEAD_SHA=${{ github.sha }}
+          fi
+
+          # Get changed Python files in weightslab directory, exclude generated files
+          CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT $BASE_SHA $HEAD_SHA | \
+            grep '^weightslab/.*\.py$' | \
+            grep -v '_pb2\.py$' | \
+            grep -v '_pb2_grpc\.py$' | \
+            grep -v 'ui\.py$' | \
+            grep -v 'trainer.*\.py$' || true)
+
+          echo "changed_files<<EOF" >> $GITHUB_OUTPUT
+          echo "$CHANGED_FILES" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+          if [ -z "$CHANGED_FILES" ]; then
+            echo "has_files=false" >> $GITHUB_OUTPUT
+            echo "No Python files changed in weightslab directory"
+          else
+            echo "has_files=true" >> $GITHUB_OUTPUT
+            echo "Changed files:"
+            echo "$CHANGED_FILES"
+          fi
+
+      - name: Install linting tools
+        if: steps.changed-files.outputs.has_files == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install ruff pylint vulture
+
+      - name: Lint with Ruff (unused, style, bugs)
+        if: steps.changed-files.outputs.has_files == 'true'
+        run: |
+          FILES="${{ steps.changed-files.outputs.changed_files }}"
+          if [ -n "$FILES" ]; then
+            echo "$FILES" | xargs ruff check --select F401,F841,E9,F632,F722,F823 --ignore E501 --fix
+          fi
+
+      - name: Pylint (errors only)
+        if: steps.changed-files.outputs.has_files == 'true'
+        run: |
+          FILES="${{ steps.changed-files.outputs.changed_files }}"
+          if [ -n "$FILES" ]; then
+            echo "$FILES" | xargs pylint --disable=R0913,E0401,C0301,W0511,C0114,C0115,C0116 --fail-under=7.0
+          fi
+
+      - name: Vulture (unused code)
+        if: steps.changed-files.outputs.has_files == 'true'
+        run: |
+          FILES="${{ steps.changed-files.outputs.changed_files }}"
+          if [ -n "$FILES" ]; then
+            echo "$FILES" | xargs vulture --min-confidence 70 . || true
+          fi
+
+  install:
+    if: ${{ github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') }}
+    runs-on: ubuntu-latest
+    needs: code-quality
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+          restore-keys: |
+            ${{ runner.os }}-pip
+
+      - name: Install project dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .
+
+      - name: Verify installation
+        run: |
+          python -c "import weightslab; print(f'weightslab imported successfully')"
+
+  test:
+    if: ${{ github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') }}
+    runs-on: ubuntu-latest
+    needs: install
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+          restore-keys: |
+            ${{ runner.os }}-pip
+
+      # - name: Install project dependencies
+      #   run: |
+      #     python -m pip install --upgrade pip
+      #     python -m pip install .
+
+      # - name: Run unit tests - General
+      #   run: |
+      #     export WEIGHTSLAB_LOG_LEVEL="DEBUG"
+      #     python -m unittest discover -s ./weightslab/tests -p "test_.py" -t . -v
+
+      # - name: Run unit tests - Model
+      #   run: |
+      #     export WEIGHTSLAB_LOG_LEVEL="DEBUG"
+      #     python -m unittest discover -s ./weightslab/tests/model -v
+
+      # - name: Run unit tests - Data
+      #   run: |
+      #     export WEIGHTSLAB_LOG_LEVEL="DEBUG"
+      #     python -m unittest discover -s ./weightslab/tests/data -v
+
+      # - name: Run unit tests - Integrations
+      #   run: |
+      #     export WEIGHTSLAB_LOG_LEVEL="DEBUG"
+      #     python -m unittest discover -s ./weightslab/tests/integrations -v
+
+      - name: Trigger WeightsStudio CI
+        env:
+          WS_TOKEN: ${{ secrets.WEIGHTS_STUDIO_API_TOKEN }}
+        run: |
+          if [ -z "${WS_TOKEN}" ]; then
+            echo "WEIGHTS_STUDIO_API_TOKEN not set; skipping WeightsStudio trigger."
+            exit 0
+          fi
+
+          # Trigger the ws-ci workflow in the weights_studio repository on main.
+          curl -fSs -X POST "https://api.github.com/repos/GrayboxTech/weights_studio/actions/workflows/ws-ci.yml/dispatches" \
+            -H "Authorization: Bearer ${WS_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            -H "Content-Type: application/json" \
+            -d '{"ref":"main"}'
+
+          echo "WeightsStudio workflow dispatch requested successfully."
+
+  build-and-publish-dev:
+    name: Build & Publish Dev (TestPyPI)
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      published_version: ${{ steps.version.outputs.published_version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build tools
+        run: python -m pip install --upgrade pip build twine
+
+      - name: Compute publish version
+        id: version
+        run: |
+          TS="$(date -u +%Y%m%d%H%M%S)"
+          SHA_HEX="$(git rev-parse --short=8 HEAD)"
+          # TestPyPI/PyPI reject local versions (+sha), so encode sha as decimal in devN.
+          SHA_DEC="$((16#${SHA_HEX}))"
+          PRETEND="${TS}.dev${SHA_DEC}"
+          echo "published_version=${PRETEND}" >> $GITHUB_OUTPUT
+          echo "Publishing version: ${PRETEND} (from sha ${SHA_HEX})"
+
+      - name: Build distributions
+        env:
+          # Package-specific override is the most reliable way with setuptools_scm.
+          SETUPTOOLS_SCM_PRETEND_VERSION_FOR_WEIGHTSLAB: ${{ steps.version.outputs.published_version }}
+        run: |
+          echo "Building with version ${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_WEIGHTSLAB}"
+          python -m build
+
+      - name: Verify built artifact version
+        env:
+          WL_VERSION: ${{ steps.version.outputs.published_version }}
+        run: |
+          ls -la dist
+          if ls dist | grep -q "0.0.0"; then
+            echo "Build still produced 0.0.0 artifacts"
+            exit 1
+          fi
+          if ! ls dist | grep -q "${WL_VERSION}"; then
+            echo "Expected version ${WL_VERSION} not found in dist artifacts"
+            exit 1
+          fi
+
+      - name: Check distributions
+        run: python -m twine check dist/*
+
+      - name: Publish to TestPyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        run: |
+          if [ -z "${{ secrets.TEST_PYPI_API_TOKEN }}" ]; then
+            echo "TEST_PYPI_API_TOKEN not set; skipping TestPyPI upload."
+            exit 0
+          fi
+          python -m twine upload --repository-url https://test.pypi.org/legacy/ --verbose dist/*
+
+  test-install-from-pip-dev:
+    name: Test Install From Pip Dev
+    needs: build-and-publish-dev
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Create isolated virtual environment
+        run: |
+          python -m venv .venv-ci
+
+      - name: Install from TestPyPI (with retry)
+        env:
+          WL_VERSION: ${{ needs.build-and-publish-dev.outputs.published_version }}
+        run: |
+          . .venv-ci/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install .
+
+      - name: Build distributions
+        run: python -m build
+
+      - name: Capture built version
+        id: main-version
+        run: |
+          WHEEL_PATH="$(ls dist/weightslab-*.whl | head -n 1)"
+          if [ -z "${WHEEL_PATH}" ]; then
+            echo "No wheel found in dist/"
+            exit 1
+          fi
+          WHEEL_FILE="$(basename "${WHEEL_PATH}")"
+          WL_VERSION="$(echo "${WHEEL_FILE}" | sed -E 's/^weightslab-([^-]+)-.*/\1/')"
+          if [ -z "${WL_VERSION}" ]; then
+            echo "Could not parse version from ${WHEEL_FILE}"
+            exit 1
+          fi
+          echo "published_version=${WL_VERSION}" >> $GITHUB_OUTPUT
+          echo "Main published version: ${WL_VERSION}"
+
+      - name: Check distributions
+        run: python -m twine check dist/*
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          if [ -z "${{ secrets.PYPI_API_TOKEN }}" ]; then
+            echo "PYPI_API_TOKEN not set; skipping PyPI upload."
+            exit 0
+          fi
+          python -m twine upload dist/* --non-interactive
+
+  test-install-from-pip-main:
+    name: Test Install From Pip Main
+    needs: build-and-publish-main
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Create isolated virtual environment
+        run: |
+          python -m venv .venv-ci
+
+      - name: Install from TestPyPI (with retry)
+        env:
+          WL_VERSION: ${{ needs.build-and-publish-main.outputs.published_version }}
+        run: |
+          . .venv-ci/bin/activate
+          python -m pip install --upgrade pip
+          if [ -z "${WL_VERSION}" ]; then
+            echo "Missing published version from upstream job"
+            exit 1
+          fi
+          sleep 60 && python -m pip install --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu "weightslab==${WL_VERSION}" --no-cache-dir
+          echo "weightslab==${WL_VERSION} installed from PyPI"
+
+      - name: Verify package import
+        run: |
+          export WEIGHTSLAB_LOG_LEVEL="DEBUG"
+          python -m unittest discover -s ./weightslab/tests/integrations -v
+
+  weightsstudio-ci:
+    if: ${{ github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') }}
+    name: WeightsStudio CI
+    needs: test
+    uses: GrayboxTech/weights_studio/.github/workflows/ws-ci.yml@dev
+    with:
+      weightslab_branch: ${{ (github.head_ref || github.ref_name) == 'main' && 'main' || 'dev' }}
+
+  build-and-publish-dev:
+    if: ${{ github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') }}
+    name: Build & Publish Dev (TestPyPI)
+    needs: weightsstudio-ci
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      published_version: ${{ steps.version.outputs.published_version }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install build tools
+        run: python -m pip install --upgrade pip build twine
+
+      - name: Compute publish version
+        id: version
+        run: |
+          TS="$(date -u +%Y%m%d%H%M%S)"
+          SHA_HEX="$(git rev-parse --short=8 HEAD)"
+          # TestPyPI/PyPI reject local versions (+sha), so encode sha as decimal in devN.
+          SHA_DEC="$((16#${SHA_HEX}))"
+          PRETEND="${TS}.dev${SHA_DEC}"
+          echo "published_version=${PRETEND}" >> $GITHUB_OUTPUT
+          echo "Publishing version: ${PRETEND} (from sha ${SHA_HEX})"
+
+      - name: Build distributions
+        env:
+          # Package-specific override is the most reliable way with setuptools_scm.
+          SETUPTOOLS_SCM_PRETEND_VERSION_FOR_WEIGHTSLAB: ${{ steps.version.outputs.published_version }}
+        run: |
+          echo "Building with version ${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_WEIGHTSLAB}"
+          python -m build
+
+      - name: Verify built artifact version
+        env:
+          WL_VERSION: ${{ steps.version.outputs.published_version }}
+        run: |
+          ls -la dist
+          if ls dist | grep -q "0.0.0"; then
+            echo "Build still produced 0.0.0 artifacts"
+            exit 1
+          fi
+          if ! ls dist | grep -q "${WL_VERSION}"; then
+            echo "Expected version ${WL_VERSION} not found in dist artifacts"
+            exit 1
+          fi
+
+      - name: Check distributions
+        run: python -m twine check dist/*
+
+      - name: Publish to TestPyPI
+        env:
+          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        run: |
+          if [ -z "${{ secrets.TEST_PYPI_API_TOKEN }}" ]; then
+            echo "TEST_PYPI_API_TOKEN not set; skipping TestPyPI upload."
+            exit 0
+          fi
+          python -m twine upload --repository-url https://test.pypi.org/legacy/ --verbose dist/*
+
+  test-install-from-pip-dev:
+    if: ${{ github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/') }}
+    name: Test Install From Pip Dev
+    needs: build-and-publish-dev
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Create isolated virtual environment
+        run: |
+          python -m venv .venv-ci
+
+      - name: Install from TestPyPI (wait until indexed)
+        env:
+          WL_VERSION: ${{ needs.build-and-publish-dev.outputs.published_version }}
+        run: |
+          . .venv-ci/bin/activate
+          python -m pip install --upgrade pip
+          if [ -z "${WL_VERSION}" ]; then
+            echo "Missing published version from upstream job"
+            exit 1
+          fi
+
+          # TestPyPI can take a few minutes to expose newly uploaded versions.
+          for attempt in $(seq 1 60); do
+            echo "Checking pip visibility for weightslab==${WL_VERSION} (attempt ${attempt}/60)"
+            if python -m pip download \
+                --index-url https://test.pypi.org/simple/ \
+                --no-deps \
+                "weightslab==${WL_VERSION}" \
+                -d /tmp/wl-probe >/dev/null 2>&1; then
+              echo "Found weightslab==${WL_VERSION} in TestPyPI simple index"
+              break
+            fi
+
+            if [ "${attempt}" -eq 60 ]; then
+              echo "Timed out waiting for weightslab==${WL_VERSION} on TestPyPI simple index"
+              exit 1
+            fi
+            sleep 7
+          done
+
+          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple "weightslab==${WL_VERSION}" --no-cache-dir --retries 10 --timeout 60
+          echo "weightslab==${WL_VERSION} installed from TestPyPI"
+
+      - name: Verify package import
+        run: |
+          . .venv-ci/bin/activate
+          python -c "import weightslab; import importlib.metadata as m; print('weightslab import OK'); print('installed version:', m.version('weightslab'))"

weightslab-1.0.0/.github/workflows/docs-pages.yml

@@ -0,0 +1,123 @@
+name: Docs (GitHub Pages)
+
+on:
+  push:
+    branches: [docstring, dev, main]
+    tags:
+      - 'v*'
+  pull_request:
+    branches: [main]
+    types: [opened, reopened, ready_for_review]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: docs-pages-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install docs dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r docs/requirements.txt
+          python -m pip install numpy pandas tqdm PyYAML
+
+      - name: Build Sphinx multiversion docs
+        run: sphinx-multiversion docs docs/_build/html
+
+      - name: Prepare latest alias and versions manifest
+        shell: bash
+        run: |
+          python - <<'PY'
+          import json
+          import re
+          import shutil
+          from pathlib import Path
+
+          root = Path("docs/_build/html")
+          main_dir = root / "main"
+          latest_dir = root / "latest"
+
+          if main_dir.exists():
+              shutil.copytree(main_dir, latest_dir, dirs_exist_ok=True)
+
+          versions = []
+          semver = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
+
+          for d in root.iterdir():
+              if not d.is_dir() or not (d / "index.html").exists():
+                  continue
+              name = d.name
+              if name not in {"latest", "dev"} and not semver.match(name):
+                  continue
+
+              if name == "latest":
+                  rank = (0, 0, 0, 0)
+                  label = "latest"
+              elif name == "dev":
+                  rank = (1, 0, 0, 0)
+                  label = "dev"
+              else:
+                  m = semver.match(name)
+                  rank = (2, -int(m.group(1)), -int(m.group(2)), -int(m.group(3)))
+                  label = name
+
+              versions.append({"name": name, "label": label, "rank": rank})
+
+          versions.sort(key=lambda v: v["rank"])
+          payload = [{"name": v["name"], "label": v["label"]} for v in versions]
+
+          (root / "versions.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
+          for v in payload:
+              (root / v["name"] / "versions.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
+
+          if payload:
+              default_name = payload[0]["name"]
+              redirect = f"""<!doctype html>
+          <html><head>
+            <meta charset=\"utf-8\" />
+            <meta http-equiv=\"refresh\" content=\"0; url=./{default_name}/index.html\" />
+            <title>WeightsLab Docs</title>
+          </head><body>
+            Redirecting to <a href=\"./{default_name}/index.html\">{default_name}</a>.
+          </body></html>
+          """
+              (root / "index.html").write_text(redirect, encoding="utf-8")
+          PY
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/_build/html
+
+  deploy:
+    if: ${{ github.event_name != 'pull_request' }}
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4