pyfabric 0.1.0__tar.gz

pyfabric-0.1.0/.codespellrc

@@ -0,0 +1,12 @@
+[codespell]
+# Skip generated, vendored, and binary content. Notebook fixture files
+# contain raw cell-marker syntax and synthetic IDs that confuse codespell.
+skip = .git,*.pyc,*.pyo,.venv,build,dist,*.egg-info,.test-report.json,tests/fixtures/notebooks/**,src/pyfabric/_version.py
+# Allowlist project-specific words. Add sparingly — every entry here is
+# a future false positive someone has to read past.
+# - falsy: legitimate jargon (truthy/falsy)
+# - caf: matches the literal "café" (café) in test_logging.py's UTF-8
+#   round-trip test; codespell sees the ASCII prefix only.
+# - skool: the Skool community platform (https://www.skool.com), used
+#   as an example data source in claude_memory/open_mirror_landing_zone.md.
+ignore-words-list = falsy,caf,skool

pyfabric-0.1.0/.gitattributes

@@ -0,0 +1,18 @@
+# Normalize all text files to LF in the repository
+* text=auto eol=lf
+
+# Explicitly mark binary files
+*.png binary
+*.jpg binary
+*.whl binary
+
+# Ensure key file types use LF everywhere
+*.py text eol=lf
+*.toml text eol=lf
+*.yml text eol=lf
+*.yaml text eol=lf
+*.md text eol=lf
+*.txt text eol=lf
+*.cfg text eol=lf
+*.ini text eol=lf
+*.json text eol=lf

pyfabric-0.1.0/.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# All changes require review from an admin
+* @hughdecatte

pyfabric-0.1.0/.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+updates:
+  # Keep GitHub Actions current (no CVE tracking for Actions)
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"
+
+  # Python deps: security updates ONLY (CVE-driven)
+  # Version updates are handled by Dependabot security updates (repo setting),
+  # not dependabot.yml version updates. This entry ensures the dependency
+  # graph stays current for vulnerability scanning without auto-bumping
+  # versions that have no known CVEs.
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0  # Disable version update PRs

pyfabric-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,70 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint & Format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install -e ".[dev]"
+      - run: ruff check .
+      - run: ruff format --check .
+
+  docs-lint:
+    name: Docs Lint (codespell + markdownlint)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install codespell==2.3.0
+      - run: codespell --config=.codespellrc
+      - uses: DavidAnson/markdownlint-cli2-action@v17
+
+  type-check:
+    name: Type Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install -e ".[dev]"
+      - run: mypy src/
+
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e ".[dev]"
+      - run: pytest --cov=src/pyfabric --cov-branch --cov-report=term-missing --json-report --json-report-file=test-report.json
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: test-report-py${{ matrix.python-version }}
+          path: test-report.json

pyfabric-0.1.0/.github/workflows/dependency-review.yml

@@ -0,0 +1,17 @@
+name: Dependency Review
+
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    name: Dependency Review
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/dependency-review-action@v4
+        with:
+          fail-on-severity: high
+          deny-licenses: GPL-3.0, AGPL-3.0

pyfabric-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,59 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0  # Required for hatch-vcs to derive version from tags
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: python -m pip install --upgrade pip
+      - run: pip install build pip-audit
+      - run: pip-audit
+      - run: python -m build
+      - uses: anchore/sbom-action@v0
+        with:
+          path: dist/
+          format: spdx-json
+          output-file: sbom.spdx.json
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+      - uses: actions/upload-artifact@v7
+        with:
+          name: sbom
+          path: sbom.spdx.json
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pyfabric
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: actions/attest-build-provenance@v4
+        with:
+          subject-path: dist/*
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          attestations: true

pyfabric-0.1.0/.gitignore

@@ -0,0 +1,215 @@
+# hatch-vcs generated version file
+src/pyfabric/_version.py
+
+# Test artifacts
+.test-report.json
+.logs/
+tests/fixtures/sample.duckdb
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

pyfabric-0.1.0/.markdownlint-cli2.jsonc

@@ -0,0 +1,25 @@
+// markdownlint-cli2 configuration. Conservative ruleset — start strict,
+// loosen only when a rule produces more noise than value.
+{
+  "config": {
+    "default": true,
+    // MD013 (line length) is too noisy for technical writing with URLs,
+    // and we already line-wrap prose by hand.
+    "MD013": false,
+    // MD033 (no inline HTML) blocks legitimate uses like <details> and
+    // <kbd>. Enable case-by-case if a specific tag becomes a problem.
+    "MD033": false,
+    // MD041 (first line must be a top-level heading) breaks for files
+    // that lead with frontmatter (e.g. claude_memory/*.md).
+    "MD041": false
+  },
+  "globs": ["**/*.md"],
+  "ignores": [
+    ".venv/**",
+    "build/**",
+    "dist/**",
+    "*.egg-info/**",
+    "node_modules/**",
+    ".git/**"
+  ]
+}

pyfabric-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+# See https://pre-commit.com for more information
+# Run `pre-commit install` once after cloning to enable hooks.
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.10
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.20.0
+    hooks:
+      - id: mypy
+        args: [--config-file=pyproject.toml]
+        additional_dependencies:
+          - structlog>=24.0
+          - pytest>=8.0
+        pass_filenames: false
+        entry: mypy src/
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.3.0
+    hooks:
+      - id: codespell
+        additional_dependencies: [tomli]
+        args: [--config=.codespellrc]
+
+  - repo: https://github.com/DavidAnson/markdownlint-cli2
+    rev: v0.14.0
+    hooks:
+      - id: markdownlint-cli2
+
+  - repo: local
+    hooks:
+      - id: pytest
+        name: pytest
+        entry: python -m pytest --no-header -q
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [pre-push]

pyfabric-0.1.0/CLAUDE.md

@@ -0,0 +1,178 @@
+# CLAUDE.md
+
+Instructions for AI coding assistants (Claude, Copilot, etc.) working in
+this repository.
+
+## Project Overview
+
+pyfabric is a Python library for programmatically creating, validating,
+and locally testing Microsoft Fabric items that are compatible with Fabric
+git sync. Target users are AI coding assistants and developers who need to
+generate and test Fabric item definitions locally.
+
+## What pyfabric is — and is not
+
+pyfabric is a **dev-time / build-time** tool. It runs on the developer's
+machine (or in CI), not inside Fabric. Its three jobs are:
+
+1. **Create** Fabric artifacts (notebooks, semantic models, reports,
+   variable libraries, lakehouse DDL) as files committed to a
+   git-synced workspace repo.
+2. **Test locally** — DuckDB lakehouse mock, OneLake helpers, schema
+   validation, transform logic — without standing up Fabric.
+3. **Manage Fabric operationally** from outside the platform — trigger
+   notebook runs via the Jobs API, refresh semantic models, audit
+   workspaces.
+
+**pyfabric is NOT a notebook runtime dependency.** A Fabric Spark
+notebook should not `import pyfabric`. Several past sessions have
+wasted time trying to install pyfabric inside Fabric (in
+`Resources/builtin/`, in custom Environments, via `%pip install
+pyfabric` cells). The right pattern is:
+
+- Notebooks stay thin — orchestration only.
+- The project's transform / DDL / SQL helpers live in a project
+  wheel (e.g. `fabric_utils-*.whl`) shipped via
+  `Resources/builtin/` (see `claude_memory/notebook_wheel_resources_pattern.md`).
+- That project wheel depends only on Fabric Spark runtime
+  pre-installed packages (`azure-identity`,
+  `azure-storage-file-datalake`, `pyarrow`, `requests`), not on
+  pyfabric.
+- pyfabric stays on the dev machine, where it builds and validates
+  the notebook + the project wheel.
+
+When you find yourself reaching for pyfabric inside a notebook,
+that's a sign the logic should move into the project wheel where
+it's locally testable; the notebook then calls the wheel.
+
+## Technical Stack
+
+- **Python**: 3.11+ (use modern syntax: `match`, `X | Y` unions, `Self` from `typing`). Avoid 3.12-only syntax — Fabric Spark runtime 1.3 is 3.11. Specifically: do NOT use `type X = Y` (PEP 695, 3.12+) or `class Foo[T]` (PEP 695, 3.12+).
+- **Build**: hatchling with hatch-vcs (version from git tags)
+- **Layout**: src-layout (`src/pyfabric/`)
+- **Linting**: ruff (configured in pyproject.toml)
+- **Type checking**: mypy in strict mode
+- **Logging**: structlog (JSON output, token masking, context binding)
+- **Testing**: pytest (tests in `tests/`)
+- **CI**: GitHub Actions (lint, type-check, test, dependency review)
+
+## Sub-package Structure
+
+```text
+src/pyfabric/
+  client/       — Auth, REST API, HTTP client for Fabric service
+  items/        — Create, load, save, validate Fabric item definitions
+  data/         — OneLake DFS, SQL connections, lakehouse table I/O
+  workspace/    — Workspace-level operations
+  testing/      — pytest fixtures, DuckDB Spark mock for users
+  cli.py        — CLI entry point
+```
+
+## Key Conventions
+
+- All code must pass `ruff check`, `ruff format --check`, and `mypy --strict`
+- All public functions and classes must have type annotations
+- Use `X | None` not `typing.Optional[X]`; use `X | Y` not `typing.Union`
+- Use `list`, `dict`, `tuple` not `typing.List`, `typing.Dict`, `typing.Tuple`
+- `from __future__ import annotations` is NOT needed (Python 3.11+ supports `X | Y` unions natively via PEP 604)
+- Prefer `pathlib.Path` over `os.path`
+- Prefer dataclasses or named tuples over plain dicts for structured data
+- Tests use pytest fixtures; avoid unittest.TestCase
+- No mutable default arguments
+- Use `import structlog` and `log = structlog.get_logger()` (NOT stdlib `logging`)
+- Use `log.info("message", key=value)` for structured context (NOT f-strings in messages)
+
+## Running Checks
+
+```bash
+ruff check .             # Lint
+ruff format --check .    # Format check
+mypy src/                # Type check
+pytest                   # Run tests
+```
+
+## Version Management
+
+Version is derived from git tags via hatch-vcs. Do NOT manually edit
+version strings. The file `src/pyfabric/_version.py` is auto-generated
+and must not be committed.
+
+## Dependencies
+
+- Runtime dependencies go in `[project] dependencies` in pyproject.toml
+- Optional dependency groups: `[azure]`, `[data]`, `[testing]`, `[all]`, `[dev]`
+- Keep runtime dependencies minimal — heavy deps belong in optional groups
+
+## Fabric Item Structure (git sync format)
+
+All Fabric items follow this directory structure:
+
+```text
+{DisplayName}.{ItemType}/
+  .platform                 # Required: metadata + logicalId (UUID)
+  {definition_files...}     # Item-specific content
+```
+
+The `.platform` file uses schema version 2.0:
+
+```json
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json",
+  "metadata": {
+    "type": "{ItemType}",
+    "displayName": "{DisplayName}"
+  },
+  "config": {
+    "version": "2.0",
+    "logicalId": "{UUID}"
+  }
+}
+```
+
+Supported item types: Notebook, Lakehouse, Dataflow, Environment,
+VariableLibrary, SemanticModel, Report, DataPipeline, Warehouse,
+MirroredDatabase, Map.
+
+## Provisioning Workflow (git-first)
+
+pyfabric ships two complementary ways to create Fabric items:
+
+1. **Builder → git artifact** (`NotebookBuilder.save_to_disk()`,
+   `MirroredDatabaseBuilder.save_to_disk()`, etc.) writes a
+   `.platform` + definition files to a folder under your
+   git-synced workspace repo. **This is the primary path.**
+2. **REST helpers** (`create_item`, `create_mirrored_database`, …)
+   call the Fabric REST API directly. **Use only for scripted
+   automation in workspaces that are not git-synced.**
+
+Mixing the two on the same workspace causes a duplicate-item
+conflict: the REST-created item and the later git-synced item live
+under different IDs, and Fabric will not merge them. The
+git-synced item also receives a fresh ID, breaking any committed
+references (e.g. notebook parameters that hard-code the mirror's
+GUID).
+
+### Recommended order for a git-synced workspace
+
+1. Generate the artifact locally with the builder.
+2. Run local tests (producer logic, schema-compat, artifact shape,
+   `validate_workspace`).
+3. Commit + push to the workspace's git-synced branch.
+4. Have a human trigger the **manual git-sync** in the Fabric portal
+   (and bind any required OAuth credentials on first refresh — see
+   `claude_memory/first_refresh_cred_binding.md`).
+5. Only then perform data-plane mutations (upload landing-zone
+   files, `start_mirroring`, `update_definition`, …) — these
+   target the just-synced item by its real GUID.
+
+AI assistants using pyfabric in a git-synced workspace should
+**pause after step 3** and wait for the user to confirm the sync
+before issuing any REST mutation that would create or modify an
+item.
+
+## What NOT to Do
+
+- Do not add `__version__ = "..."` manually anywhere
+- Do not create setup.py or setup.cfg
+- Do not commit `src/pyfabric/_version.py`
+- Do not use `datetime.utcnow()` — use `datetime.now(timezone.utc)`