mquery-toolkit 0.1.0__py3-none-any.whl

mquery_toolkit/fabric.py

@@ -0,0 +1,213 @@
+"""Mockable, caller-authenticated Fabric Execute Query state machine."""
+
+from __future__ import annotations
+
+import io
+import re
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from time import monotonic, sleep
+from typing import Any
+from urllib.parse import quote, urljoin, urlsplit, urlunsplit
+
+from .core import MAX_BYTES, AdapterError
+
+Response = Mapping[str, Any]
+Transport = Callable[
+    [str, Mapping[str, str], Mapping[str, Any] | None, float], Response
+]
+ArrowValidator = Callable[[bytes], None]
+_OPERATION_ID = re.compile(r"^[A-Za-z0-9-]{1,128}$")
+
+
+def redact(value: str) -> str:
+    return "<redacted>" if value else ""
+
+
+def _same_origin(start: str, candidate: str) -> bool:
+    left, right = urlsplit(start), urlsplit(candidate)
+    return (left.scheme, left.netloc) == (right.scheme, right.netloc)
+
+
+def _validate_arrow(body: bytes) -> None:
+    if len(body) > MAX_BYTES:
+        raise AdapterError("Fabric Arrow stream exceeds 10 MiB")
+    try:
+        import pyarrow as pa  # type: ignore[import-untyped]
+    except ImportError as error:
+        raise AdapterError(
+            "Arrow validation requires the mquery-toolkit[fabric] extra"
+        ) from error
+    stream = io.BytesIO(body)
+    streams = 0
+    total_bytes = 0
+    try:
+        while stream.tell() < len(body):
+            start = stream.tell()
+            reader = pa.ipc.open_stream(stream)
+            metadata = {
+                key.decode("utf-8", "replace"): value.decode("utf-8", "replace")
+                for key, value in (reader.schema.metadata or {}).items()
+            }
+            if metadata.get("IsError", "").lower() == "true":
+                code = metadata.get("FaultCode", "unknown")
+                message = metadata.get("FaultString", "Fabric query failed")
+                raise AdapterError(f"Fabric Arrow error [{code}]: {message}")
+            while True:
+                try:
+                    batch = reader.read_next_batch()
+                except StopIteration:
+                    break
+                total_bytes += batch.nbytes
+                if total_bytes > MAX_BYTES:
+                    raise AdapterError("Fabric Arrow stream exceeds 10 MiB")
+            streams += 1
+            if stream.tell() <= start:
+                raise AdapterError("Fabric Arrow decoder made no progress")
+    except AdapterError:
+        raise
+    except Exception as error:
+        raise AdapterError("Fabric returned an invalid Arrow stream") from error
+    if streams == 0:
+        raise AdapterError("Fabric returned an empty Arrow stream")
+
+
+def _headers(response: Response) -> Mapping[str, Any]:
+    return {
+        str(key).lower(): value for key, value in response.get("headers", {}).items()
+    }
+
+
+@dataclass(frozen=True)
+class FabricClient:
+    transport: Transport
+    timeout_seconds: int = 90
+    sleeper: Callable[[float], None] = sleep
+    clock: Callable[[], float] = monotonic
+    arrow_validator: ArrowValidator = _validate_arrow
+
+    def _request(
+        self,
+        url: str,
+        headers: Mapping[str, str],
+        payload: Mapping[str, Any] | None,
+        deadline: float,
+    ) -> Response:
+        remaining = deadline - self.clock()
+        if remaining <= 0:
+            raise AdapterError("Fabric operation exceeded caller timeout")
+        try:
+            return self.transport(url, headers, payload, remaining)
+        except TimeoutError as error:
+            raise AdapterError("Fabric transport timed out") from error
+
+    @staticmethod
+    def _require_https_same_origin(start: str, candidate: str) -> None:
+        if urlsplit(candidate).scheme != "https" or not _same_origin(start, candidate):
+            raise AdapterError(
+                "Fabric operation Location must remain HTTPS and same-origin"
+            )
+
+    @staticmethod
+    def _poll_url(start: str, location: str, operation_id: str) -> str:
+        candidate = urljoin(start, location) if location else ""
+        if not candidate:
+            if not _OPERATION_ID.fullmatch(operation_id):
+                raise AdapterError("Fabric operation id is missing or invalid")
+            origin = urlsplit(start)
+            candidate = (
+                f"{origin.scheme}://{origin.netloc}/v1/operations/"
+                f"{quote(operation_id, safe='')}"
+            )
+        FabricClient._require_https_same_origin(start, candidate)
+        return candidate
+
+    def execute(self, url: str, token: str, payload: Mapping[str, Any]) -> Response:
+        if not token:
+            raise AdapterError("Fabric caller token is required")
+        if urlsplit(url).scheme != "https":
+            raise AdapterError("Fabric URL must use HTTPS")
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+        }
+        deadline = self.clock() + self.timeout_seconds
+        request_url: str = url
+        request_payload: Mapping[str, Any] | None = payload
+        operation_id: str | None = None
+        result_requested = False
+        while True:
+            if self.clock() > deadline:
+                raise AdapterError("Fabric operation exceeded caller timeout")
+            response = self._request(request_url, headers, request_payload, deadline)
+            status = response.get("status")
+            response_headers = _headers(response)
+            if status == 200:
+                content_type = str(response_headers.get("content-type", "")).lower()
+                body = response.get("body")
+                if "json" in content_type and isinstance(body, Mapping):
+                    state = str(body.get("status", ""))
+                    if state in {"Running", "NotStarted"}:
+                        request_payload = None
+                        self.sleeper(1)
+                        continue
+                    if state == "Failed":
+                        raise AdapterError("Fabric operation reported Failed")
+                    if state == "Succeeded":
+                        if request_url == url:
+                            raise AdapterError(
+                                "Fabric returned a JSON status instead of an "
+                                "Arrow stream"
+                            )
+                        if result_requested:
+                            raise AdapterError(
+                                "Fabric result endpoint returned JSON instead of Arrow"
+                            )
+                        parts = urlsplit(request_url)
+                        result_url = urlunsplit(
+                            parts._replace(path=parts.path.rstrip("/") + "/result")
+                        )
+                        self._require_https_same_origin(url, result_url)
+                        request_url = result_url
+                        request_payload = None
+                        result_requested = True
+                        continue
+                if (
+                    "arrow" not in content_type
+                    or not isinstance(body, bytes)
+                    or not body
+                ):
+                    raise AdapterError(
+                        "Fabric 200 response is not a non-empty Arrow stream"
+                    )
+                self.arrow_validator(body)
+                return response
+            retry_raw = response_headers.get("retry-after", 1)
+            try:
+                retry = max(1, int(str(retry_raw)))
+            except ValueError as error:
+                raise AdapterError(
+                    "Fabric Retry-After must be whole seconds"
+                ) from error
+            if status == 429:
+                self.sleeper(min(retry, max(0, deadline - self.clock())))
+                continue
+            if status != 202:
+                shown = status if status is not None else "unknown"
+                raise AdapterError(f"Fabric returned HTTP {shown}")
+            location = str(response_headers.get("location", ""))
+            next_operation = str(response_headers.get("x-ms-operation-id", ""))
+            if not location and not next_operation:
+                raise AdapterError(
+                    "Fabric 202 response requires Location or x-ms-operation-id"
+                )
+            if (
+                operation_id is not None
+                and next_operation
+                and operation_id != next_operation
+            ):
+                raise AdapterError("Fabric operation id changed during polling")
+            operation_id = next_operation or operation_id
+            request_url = self._poll_url(url, location, operation_id or "")
+            request_payload = None
+            self.sleeper(min(retry, max(0, deadline - self.clock())))

mquery_toolkit/pqtest.py

@@ -0,0 +1,93 @@
+"""Bounded wrapper for a user-installed PQTest executable."""
+
+from __future__ import annotations
+
+import platform
+import re
+import subprocess
+from collections.abc import Callable, Sequence
+from pathlib import Path
+
+from .core import (
+    MAX_BYTES,
+    AdapterError,
+    _ProcessOutputLimit,
+    _run_process_bounded,
+)
+
+PQTEST_VERSION = "2.155.2"
+PQTEST_TIMEOUT_SECONDS = 300
+Runner = Callable[..., subprocess.CompletedProcess[str]]
+
+
+def _run(
+    runner: Runner, command: Sequence[str], timeout: int = PQTEST_TIMEOUT_SECONDS
+) -> subprocess.CompletedProcess[str]:
+    if runner is subprocess.run:
+        return _run_bounded(command, timeout)
+    try:
+        result = runner(
+            command,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="strict",
+            check=False,
+            timeout=timeout,
+        )
+    except (OSError, subprocess.SubprocessError, UnicodeError) as error:
+        raise AdapterError("PQTest process failed") from error
+    if (
+        len(result.stdout.encode()) > MAX_BYTES
+        or len(result.stderr.encode()) > MAX_BYTES
+    ):
+        raise AdapterError("PQTest output exceeds 10 MiB")
+    if result.returncode:
+        raise AdapterError(f"PQTest exited {result.returncode}")
+    return result
+
+
+def _run_bounded(
+    command: Sequence[str], timeout: int = PQTEST_TIMEOUT_SECONDS
+) -> subprocess.CompletedProcess[str]:
+    try:
+        result = _run_process_bounded(list(command), None, timeout)
+    except subprocess.TimeoutExpired as error:
+        raise AdapterError(f"PQTest timed out after {timeout} seconds") from error
+    except _ProcessOutputLimit as error:
+        raise AdapterError("PQTest output exceeds 10 MiB") from error
+    except OSError as error:
+        raise AdapterError("PQTest process failed") from error
+    try:
+        output = result.stdout.decode("utf-8", "strict")
+        error_output = result.stderr.decode("utf-8", "strict")
+    except UnicodeDecodeError as error:
+        raise AdapterError("PQTest output is not valid UTF-8") from error
+    if result.returncode:
+        raise AdapterError(f"PQTest exited {result.returncode}")
+    return subprocess.CompletedProcess(command, result.returncode, output, error_output)
+
+
+def validate_pqtest(
+    path: Path,
+    runner: Runner = subprocess.run,
+    timeout: int = PQTEST_TIMEOUT_SECONDS,
+) -> Path:
+    if platform.system() != "Windows":
+        raise AdapterError("PQTest adapter is supported on Windows only")
+    if not path.is_file() or path.is_symlink() or path.suffix.lower() != ".exe":
+        raise AdapterError("PQTest path must name a user-installed regular .exe")
+    version = _run(runner, [str(path), "version"], timeout).stdout
+    if not re.search(rf"(?<![0-9.]){re.escape(PQTEST_VERSION)}(?![0-9.])", version):
+        raise AdapterError(f"PQTest {PQTEST_VERSION} is required")
+    return path
+
+
+def run_pqtest(
+    path: Path,
+    args: Sequence[str],
+    runner: Runner = subprocess.run,
+    timeout: int = PQTEST_TIMEOUT_SECONDS,
+) -> str:
+    validate_pqtest(path, runner, timeout)
+    return _run(runner, [str(path), *args], timeout).stdout

mquery_toolkit-0.1.0.dist-info/METADATA

@@ -0,0 +1,189 @@
+Metadata-Version: 2.4
+Name: mquery-toolkit
+Version: 0.1.0
+Summary: Unofficial offline tooling for Power Query M source (parse, format, check, rename) via Microsoft's official parser and formatter
+Author: Gopal Bagaswar
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/GopalGB/mquery-toolkit
+Project-URL: Repository, https://github.com/GopalGB/mquery-toolkit
+Project-URL: Issues, https://github.com/GopalGB/mquery-toolkit/issues
+Keywords: power-query,m-language,power-bi,fabric,linter,formatter
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Provides-Extra: fabric
+Requires-Dist: pyarrow>=14; extra == "fabric"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-cov>=5; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=6.1; extra == "dev"
+Requires-Dist: pip-audit>=2.7; extra == "dev"
+Dynamic: license-file
+
+# mquery-toolkit
+
+Offline command-line and Python tooling for Power Query M source: parse, format,
+lint (`check`), and safely rename a `let` binding.
+
+> **Unofficial.** Not affiliated with or endorsed by Microsoft. Not an M runtime -
+> it parses and formats M source text; it does not evaluate queries.
+
+## Install
+
+```bash
+pip install mquery-toolkit
+```
+
+Requires **Node.js 22 or newer** on `PATH`, or point `MQUERY_NODE` at a Node
+binary. The Microsoft parser and formatter packages are bundled inside the
+wheel (`_bridge.cjs`) - no `npm install` needed.
+
+## Quick start
+
+```bash
+# Parse to deterministic JSON (tokens, root kind, bindings/references)
+mquery parse query.pq
+
+# Format - dry run prints a unified diff, nothing is written
+mquery format query.pq
+
+# Format and write in place (atomic replace, preserves mode/newline/encoding)
+mquery format query.pq --write
+
+# Lint, machine-readable output; exit code 2 if any diagnostic is severity=error
+mquery check query.pq --json
+
+# Rename one top-level let binding - dry run first
+mquery rename query.pq --old OldName --new NewName
+```
+
+## Python API
+
+```python
+from mquery_toolkit import check, format_source, parse, rename, update_file
+
+parsed = parse(source_text)  # dict: tokens, rootKind, analysis
+formatted = format_source(source_text)  # formatted M source, same encoding
+diagnostics = check(source_text, "query.pq")  # list[Diagnostic]
+renamed = rename(source_text, "OldName", "NewName")
+
+# File-level edit with the same dry-run/--write safety model as the CLI
+diff = update_file(path, format_source)  # dry run: unified diff
+diff = update_file(path, format_source, write=True)  # atomic write
+```
+
+## Diagnostics
+
+| Code | Severity | Meaning |
+|---|---|---|
+| `M_PARSE_ERROR` | error | source does not parse |
+| `M001` | error | duplicate `let` binding name |
+| `M002` | warning | `Web.Contents` called with a non-literal (dynamic) URL |
+| `M003` | warning | credential-like literal (`password`/`token`/`secret` = `"..."`) |
+| `M004` | warning | `let` binding unreachable from the result |
+| `M005` | warning | unresolved unqualified reference |
+| `M006` | info | source-function inventory (`*.Contents` dependency) |
+
+`M002` and `M003` are conservative text-pattern checks on the raw source, not
+AST checks; expect occasional false positives inside comments or strings.
+Every matching occurrence is reported, one diagnostic per call site or
+literal.
+
+`check --json` emits stable objects; `check` without `--json` prints
+`file:line:column: severity code: message` per diagnostic. The CLI exits `2`
+when any diagnostic has severity `error`, `0` otherwise.
+
+## Safety model
+
+- **Dry-run by default.** Every edit command (`format`, `rename`,
+  `replace-source`) prints a unified diff and touches nothing unless `--write`
+  is passed.
+- **`--write` is an atomic replace**: the file is written to a sibling temp
+  file, `fsync`'d, `chmod`'d to match the original, then moved into place with
+  `os.replace`.
+- **Layout is preserved**: UTF-8 encoding, a leading BOM (present in every
+  Power Query SDK connector file), newline convention (`\n` vs `\r\n`),
+  final-newline state, and file mode all round-trip unchanged.
+- **Refuses symlinks and hardlinks** - writes require a regular, single-link
+  file.
+- **Detects concurrent change**: the source is snapshotted before the
+  transform and re-checked immediately before the atomic replace - this final
+  snapshot check, not the lock, is the guarantee against lost updates; a
+  change in that microsecond window raises `SafeWriteError`.
+- **Advisory lock while writing only** - a `--write` call takes a
+  cross-process advisory lock (`fcntl`/`msvcrt`) for the duration of the
+  write and removes the lock file afterward, best-effort. It only serialises
+  cooperating `mquery` processes and is not a correctness guarantee: because
+  the lock file is removed after use, a waiting process and a freshly
+  started one can end up locking different inodes. Dry-run calls take no
+  lock and create no lock file.
+- This is **not mandatory locking** - no OS provides a portable mandatory
+  lock, and the advisory lock is not itself the correctness guard. Use
+  source control or external exclusive ownership for concurrent editors.
+
+## Limits
+
+- Input and output are capped at **10 MiB**.
+- The Node subprocess is bounded to a **30 second** timeout.
+- Supported extensions: `.pq`, `.m`, `.pqm`, and any `*.query.pq` file.
+- `rename` scope: exactly **one unquoted top-level `let` binding**. It refuses
+  quoted identifiers (`#"..."`), record literals, lambda expressions, and
+  non-ASCII source.
+- `Retry-After` on the Fabric adapter must be whole seconds; HTTP-date values
+  are rejected.
+
+## Optional adapters
+
+- **`fabric` extra** (`pip install "mquery-toolkit[fabric]"`) - a Fabric
+  Execute Query client that takes a caller-provided bearer token and an
+  injected HTTP transport. It never manages credentials itself and is fully
+  mocked in tests (no network access in the test suite).
+- **`pqtest`** - a bounded wrapper around a user-installed Microsoft PQTest
+  executable, Windows-only, pinned to version `2.155.2`. It never downloads a
+  binary; it only validates and runs one already on disk.
+
+## What it is not
+
+- Not an M language runtime or evaluator.
+- Not a Power BI or Fabric client, and it does not manage credentials.
+- Not a general-purpose file editor - it only touches files with a supported
+  extension and only through the safety model above.
+- Not a replacement for Microsoft's own parser/formatter - it vendors and
+  calls them directly rather than reimplementing M syntax.
+
+## Development
+
+```bash
+git clone https://github.com/GopalGB/mquery-toolkit
+cd mquery-toolkit
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev,fabric]"
+npm ci --ignore-scripts
+
+pytest -q --cov=mquery_toolkit --cov-fail-under=80
+mypy src
+ruff check .
+ruff format --check .
+npm test
+python -m build
+```
+
+## License
+
+MIT - see `LICENSE`. Bundled Microsoft packages
+(`@microsoft/powerquery-parser`, `@microsoft/powerquery-formatter`) and their
+dependencies are also MIT; see `THIRD_PARTY_NOTICES.txt` and `NOTICE`.

mquery_toolkit-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+mquery_toolkit/THIRD_PARTY_NOTICES.txt,sha256=D77F5aSJndizys2F9xVBof9KLyJlXZ-dbJmUC7XKvSU,1438
+mquery_toolkit/__init__.py,sha256=kqj3em44u-DjdVMhTo0kszbKeeU6SBnUbcY7aIeqzrw,578
+mquery_toolkit/_bridge.cjs,sha256=obBJjhiN2LdYF_cTgyaV_O0gy9UcPiXzbZi0YkLXMXM,2601383
+mquery_toolkit/cli.py,sha256=tnK8-UMsmAlQvW46f97NKCRK1rUaD8hsgKsU-iqnVm4,3627
+mquery_toolkit/core.py,sha256=1w1ocGduLXjLwgRsMRs-wg5sRYc8a7OvdHQ5sdJhCPU,20363
+mquery_toolkit/fabric.py,sha256=-8HfRfVxEhrw-xuYzo0l-zXuftROkGkMQCfSqvv-v7c,8703
+mquery_toolkit/pqtest.py,sha256=5Fh5l6X8updrezAgkm-5pB4uBuzBPw1xK9mrfjVenIw,3152
+mquery_toolkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mquery_toolkit-0.1.0.dist-info/licenses/LICENSE,sha256=gszjyPUNaVUz78eaIo48Yi0qo0GvBv9lLMerAT-C5lk,1071
+mquery_toolkit-0.1.0.dist-info/licenses/NOTICE,sha256=XchoRITIzvbsT8KGQc3RLpb-Tbuz0a_76Skq3J_ZwCg,509
+mquery_toolkit-0.1.0.dist-info/METADATA,sha256=aehnhePuQYzibBaRzPMJp90lkwB5R8-WUexVxibPMjw,7645
+mquery_toolkit-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+mquery_toolkit-0.1.0.dist-info/entry_points.txt,sha256=uT91guSn55M3YHnhCxV6Vw6bo2VuqXcovQwrT9ATl_s,51
+mquery_toolkit-0.1.0.dist-info/top_level.txt,sha256=tlfo50yrjuNCMRVyi2HM_yIEWB24nhp7Px2hyKxJ3Rc,15
+mquery_toolkit-0.1.0.dist-info/RECORD,,

mquery_toolkit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

mquery_toolkit-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+mquery = mquery_toolkit.cli:main

mquery_toolkit-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Gopal Bagaswar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mquery_toolkit-0.1.0.dist-info/licenses/NOTICE

@@ -0,0 +1,9 @@
+Unofficial project, not affiliated with Microsoft. The bundled JavaScript
+components retain their copyright and MIT permission notices in
+`mquery_toolkit/THIRD_PARTY_NOTICES.txt`.
+The test corpus includes a short Power Query M let-expression example from the
+Microsoft M language specification:
+https://learn.microsoft.com/en-us/powerquery-m/m-spec-let
+
+The vendored DataConnectors samples remain Copyright (c) Microsoft Corporation
+and are redistributed under the MIT License included beside those fixtures.

mquery_toolkit-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+mquery_toolkit