sqlscope-rs 0.2.0__py3-none-any.whl

sqlscope/__init__.py

@@ -0,0 +1,303 @@
+"""Scope-aware SQL analysis and rewriting.
+
+=====================  ==========================================================
+Function               Purpose
+=====================  ==========================================================
+apply_row_filter       Filter every in-scope table by a predicate.
+inject_ctes            Prepend CTE definitions to a query's root ``WITH``.
+rewrite_tables         Replace table references with derived tables.
+column_origins         Source columns whose values reach the result (lineage).
+output_columns         Names of the columns a statement outputs.
+referenced_columns     Columns referenced anywhere, per table.
+column_usages          Column references with the clause they appear in.
+=====================  ==========================================================
+
+Every function accepts ``dialect`` (default ``"trino"``). Functions release the
+GIL while they run and are safe to call from several threads.
+
+The SQL engine is the sqlscope FFI shared library (``libsqlscope_ffi.so``,
+``libsqlscope_ffi.dylib`` or ``sqlscope_ffi.dll``) published with each sqlscope
+release. It is loaded on first use; see :func:`load` for how it is located.
+This package neither bundles nor downloads it.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
+
+from ._library import (
+    LIBRARY_PATH_ENV,
+    Error,
+    InternalError,
+    InvalidArgumentError,
+    LibraryNotFoundError,
+    ParseError,
+    UnsupportedError,
+    call,
+    library_file_name,
+    library_version,
+    load,
+)
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "Clause",
+    "ColumnUsage",
+    "CteDef",
+    "Error",
+    "InternalError",
+    "InvalidArgumentError",
+    "LIBRARY_PATH_ENV",
+    "LibraryNotFoundError",
+    "ParseError",
+    "Schema",
+    "TableRef",
+    "TableRewrite",
+    "UnionRewrite",
+    "UnsupportedError",
+    "apply_row_filter",
+    "column_origins",
+    "column_usages",
+    "inject_ctes",
+    "library_file_name",
+    "library_version",
+    "load",
+    "output_columns",
+    "referenced_columns",
+    "rewrite_tables",
+]
+
+Schema = Mapping[str, Sequence[str]]
+"""Table name (bare, ``schema.table`` or ``catalog.schema.table``) -> ordered column names."""
+
+
+@dataclass(frozen=True)
+class CteDef:
+    """A common table expression ``name AS (query)``.
+
+    ``name`` is an identifier value, not SQL text; it is quoted as needed.
+    """
+
+    name: str
+    query: str
+
+
+@dataclass(frozen=True)
+class TableRef:
+    """A table name, optionally schema- and catalog-qualified."""
+
+    table: str
+    schema: Optional[str] = None
+    catalog: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class UnionRewrite:
+    """A derived table backed by ``UNION DISTINCT`` over ``branches``."""
+
+    table_alias: str
+    columns: Sequence[str]
+    branches: Sequence[TableRef]
+
+
+@dataclass(frozen=True)
+class TableRewrite:
+    """Replaces references to ``match_key`` (``schema.table``).
+
+    Set exactly one of ``inline`` (``(SELECT * FROM inline)``) or ``union``.
+    """
+
+    match_key: str
+    inline: Optional[TableRef] = None
+    union: Optional[UnionRewrite] = None
+
+
+class Clause(str, Enum):
+    """The SQL clause that contains a column reference."""
+
+    SELECT = "SELECT"
+    FROM = "FROM"
+    JOIN_ON = "JOIN_ON"
+    JOIN_USING = "JOIN_USING"
+    WHERE = "WHERE"
+    GROUP_BY = "GROUP_BY"
+    HAVING = "HAVING"
+    QUALIFY = "QUALIFY"
+    WINDOW = "WINDOW"
+    ORDER_BY = "ORDER_BY"
+    SORT_BY = "SORT_BY"
+    DISTRIBUTE_BY = "DISTRIBUTE_BY"
+    CLUSTER_BY = "CLUSTER_BY"
+    CONNECT_BY = "CONNECT_BY"
+    LATERAL_VIEW = "LATERAL_VIEW"
+    UPDATE_SET_TARGET = "UPDATE_SET_TARGET"
+    UPDATE_SET_VALUE = "UPDATE_SET_VALUE"
+    MERGE_ON = "MERGE_ON"
+    MERGE_WHEN = "MERGE_WHEN"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+@dataclass(frozen=True, order=True)
+class ColumnUsage:
+    """One distinct use of a root table column in a clause."""
+
+    table: str
+    column: str
+    clause: Clause = field(compare=True)
+
+
+def _schema(schema: Optional[Schema]) -> Optional[Dict[str, List[str]]]:
+    if schema is None:
+        return None
+    return {str(table): [str(column) for column in columns] for table, columns in schema.items()}
+
+
+def _list(values: Optional[Iterable[str]]) -> Optional[List[str]]:
+    if values is None:
+        return None
+    if isinstance(values, str):
+        return [values]
+    return list(values)
+
+
+def _run(operation: str, sql: str, *, dialect: Optional[str], **fields: Any) -> Any:
+    """Calls ``operation``; ``fields`` holds request fields and options, ``None`` meaning unset."""
+    options = {"dialect": dialect}
+    for name in ("schema", "tableNames", "tablePatterns", "defaultDb", "stripCatalogs"):
+        options[name] = fields.pop(name, None)
+    request = {"sql": sql, **fields, "options": {k: v for k, v in options.items() if v is not None}}
+    return call(operation, request)
+
+
+def apply_row_filter(
+    sql: str,
+    predicate: str,
+    *,
+    dialect: Optional[str] = None,
+    table_names: Optional[Iterable[str]] = None,
+    table_patterns: Optional[Iterable[str]] = None,
+    default_db: Optional[str] = None,
+) -> str:
+    """Wrap every in-scope table of a query in a derived table filtered by ``predicate``.
+
+    ``SELECT * FROM a`` becomes ``SELECT * FROM (SELECT * FROM a WHERE <predicate>) AS a``.
+    By default every physical table is filtered; ``table_names`` (bare,
+    ``schema.table`` or ``catalog.schema.table``), ``table_patterns`` (regular
+    expressions) and ``default_db`` restrict the scope. CTE references, table
+    functions and ``DUAL`` are never wrapped. ``predicate`` is parsed as one
+    boolean expression; bind or escape its values before calling. Returns the
+    input unchanged when no table is in scope.
+    """
+    result: str = _run(
+        "apply_row_filter",
+        sql,
+        predicate=predicate,
+        dialect=dialect,
+        tableNames=_list(table_names),
+        tablePatterns=_list(table_patterns),
+        defaultDb=default_db,
+    )
+    return result
+
+
+def inject_ctes(
+    sql: str,
+    ctes: Iterable[Union[CteDef, Tuple[str, str]]],
+    *,
+    dialect: Optional[str] = None,
+) -> str:
+    """Add CTE definitions to the root ``WITH`` of a query, before existing CTEs.
+
+    Definitions keep their order, so each may use earlier ones. A name that
+    repeats another definition or an existing root CTE is rejected.
+    """
+    defs = [
+        {"name": cte.name, "query": cte.query} if isinstance(cte, CteDef) else {"name": cte[0], "query": cte[1]}
+        for cte in ctes
+    ]
+    result: str = _run("inject_ctes", sql, ctes=defs, dialect=dialect)
+    return result
+
+
+def _table(ref: TableRef) -> Dict[str, Optional[str]]:
+    return {"table": ref.table, "schema": ref.schema, "catalog": ref.catalog}
+
+
+def rewrite_tables(
+    sql: str,
+    rewrites: Iterable[TableRewrite],
+    *,
+    dialect: Optional[str] = None,
+    strip_catalogs: Optional[Iterable[str]] = None,
+) -> str:
+    """Replace physical table references according to ``rewrites``.
+
+    Matched references become derived tables that keep their alias (or bare
+    name); qualified column references are rebound onto it. Catalogs listed in
+    ``strip_catalogs`` are transparent while matching. Returns the input
+    unchanged when nothing matches.
+    """
+    plan = [
+        {
+            "matchKey": rewrite.match_key,
+            "inline": _table(rewrite.inline) if rewrite.inline is not None else None,
+            "union": {
+                "tableAlias": rewrite.union.table_alias,
+                "columns": list(rewrite.union.columns),
+                "branches": [_table(branch) for branch in rewrite.union.branches],
+            }
+            if rewrite.union is not None
+            else None,
+        }
+        for rewrite in rewrites
+    ]
+    result: str = _run("rewrite_tables", sql, rewrites=plan, dialect=dialect, stripCatalogs=_list(strip_catalogs))
+    return result
+
+
+def column_origins(
+    sql: str, *, dialect: Optional[str] = None, schema: Optional[Schema] = None
+) -> Dict[str, List[str]]:
+    """Source columns whose values flow into the result, keyed by root table.
+
+    Filter-only positions (WHERE, JOIN ON, GROUP BY, HAVING, ORDER BY) and the
+    right side of INTERSECT / EXCEPT are excluded. Every table read is present,
+    possibly with an empty list.
+    """
+    result: Dict[str, List[str]] = _run("column_origins", sql, dialect=dialect, schema=_schema(schema))
+    return result
+
+
+def output_columns(
+    sql: str, *, dialect: Optional[str] = None, schema: Optional[Schema] = None
+) -> Optional[List[str]]:
+    """Names of the columns a statement outputs, in order.
+
+    Unaliased expressions are named ``_col{i}``; ``*`` expands from ``schema``
+    or stays ``"*"``. Returns ``None`` for statements without columns.
+    """
+    result: Optional[List[str]] = _run("output_columns", sql, dialect=dialect, schema=_schema(schema))
+    return result
+
+
+def referenced_columns(
+    sql: str, *, dialect: Optional[str] = None, schema: Optional[Schema] = None
+) -> Dict[str, List[str]]:
+    """Columns referenced anywhere in a statement (including filters), keyed by root table."""
+    result: Dict[str, List[str]] = _run("referenced_columns", sql, dialect=dialect, schema=_schema(schema))
+    return result
+
+
+def column_usages(
+    sql: str, *, dialect: Optional[str] = None, schema: Optional[Schema] = None
+) -> List[ColumnUsage]:
+    """Every distinct ``(table, column, clause)`` use in a statement, sorted."""
+    return [
+        ColumnUsage(usage["table"], usage["column"], Clause(usage["clause"]))
+        for usage in _run("column_usages", sql, dialect=dialect, schema=_schema(schema))
+    ]

sqlscope/_library.py

@@ -0,0 +1,157 @@
+"""Loads the sqlscope FFI library and calls into it through ctypes."""
+
+from __future__ import annotations
+
+import ctypes
+import json
+import os
+import sys
+import threading
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+LIBRARY_PATH_ENV = "SQLSCOPE_LIBRARY_PATH"
+"""Environment variable that locates the library when :func:`load` gets no path."""
+
+_ABI_VERSION = 1
+
+
+class Error(Exception):
+    """Base class of every sqlscope error."""
+
+
+class InvalidArgumentError(Error):
+    """An option or argument is invalid."""
+
+
+class ParseError(Error):
+    """The SQL text could not be parsed."""
+
+
+class UnsupportedError(Error):
+    """The statement shape is not supported, or the input exceeded a safety limit."""
+
+
+class InternalError(Error):
+    """sqlscope produced an invalid result (a bug)."""
+
+
+class LibraryNotFoundError(Error, OSError):
+    """The sqlscope FFI library could not be loaded."""
+
+
+_ERRORS = {
+    "invalid_argument": InvalidArgumentError,
+    "parse": ParseError,
+    "unsupported": UnsupportedError,
+}
+
+
+def library_file_name() -> str:
+    """The platform's file name of the sqlscope shared library."""
+    if sys.platform == "win32":
+        return "sqlscope_ffi.dll"
+    if sys.platform == "darwin":
+        return "libsqlscope_ffi.dylib"
+    return "libsqlscope_ffi.so"
+
+
+class _Library:
+    def __init__(self, path: str) -> None:
+        try:
+            lib = ctypes.CDLL(path)
+        except OSError as error:
+            raise LibraryNotFoundError(f"cannot load {path}: {error}") from error
+        try:
+            lib.sqlscope_abi_version.restype = ctypes.c_uint32
+            lib.sqlscope_abi_version.argtypes = []
+            lib.sqlscope_version.restype = ctypes.c_char_p
+            lib.sqlscope_version.argtypes = []
+            lib.sqlscope_call.restype = ctypes.c_void_p
+            lib.sqlscope_call.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+            lib.sqlscope_free.restype = None
+            lib.sqlscope_free.argtypes = [ctypes.c_void_p]
+        except AttributeError as error:
+            raise LibraryNotFoundError(f"{path} is not a sqlscope library: {error}") from error
+        abi = lib.sqlscope_abi_version()
+        if abi != _ABI_VERSION:
+            raise LibraryNotFoundError(f"{path} implements ABI {abi}, expected {_ABI_VERSION}")
+        self.path = path
+        self.version: str = lib.sqlscope_version().decode()
+        self._lib = lib
+
+    def call(self, operation: str, request: bytes) -> bytes:
+        # ctypes releases the GIL for the duration of the call.
+        response = self._lib.sqlscope_call(operation.encode(), request)
+        try:
+            return ctypes.string_at(response)
+        finally:
+            self._lib.sqlscope_free(response)
+
+
+_loaded: Optional[_Library] = None
+_lock = threading.Lock()
+
+
+def _default_candidates() -> list[str]:
+    path = os.environ.get(LIBRARY_PATH_ENV)
+    if path:
+        return [path]
+    name = library_file_name()
+    return [str(Path(__file__).resolve().parent / name), name]
+
+
+def load(path: Optional[str] = None) -> None:
+    """Load the sqlscope FFI library, the shared library published with each sqlscope release.
+
+    Without ``path`` it tries, in order, the path in ``$SQLSCOPE_LIBRARY_PATH``,
+    :func:`library_file_name` inside the ``sqlscope`` package directory, and
+    :func:`library_file_name` on the system library search path. Nothing is
+    ever downloaded.
+
+    Calling ``load`` is optional: the first operation loads the library the
+    same way. Once a library is loaded it stays loaded; loading a different
+    path afterwards raises :class:`LibraryNotFoundError`.
+    """
+    global _loaded
+    with _lock:
+        if _loaded is not None:
+            if path is None or os.fspath(path) == _loaded.path:
+                return
+            raise LibraryNotFoundError(f"sqlscope library already loaded from {_loaded.path}")
+        if path is not None:
+            _loaded = _Library(os.fspath(path))
+            return
+        failures = []
+        for candidate in _default_candidates():
+            try:
+                _loaded = _Library(candidate)
+                return
+            except LibraryNotFoundError as error:
+                failures.append(str(error))
+        raise LibraryNotFoundError(
+            f"cannot load {library_file_name()}; download it from a sqlscope release and pass its "
+            f"path to sqlscope.load() or set {LIBRARY_PATH_ENV}: " + "; ".join(failures)
+        )
+
+
+def _library() -> _Library:
+    if _loaded is None:
+        load()
+    assert _loaded is not None
+    return _loaded
+
+
+def library_version() -> str:
+    """The version of the loaded sqlscope FFI library, loading it if needed."""
+    return _library().version
+
+
+def call(operation: str, request: Dict[str, Any]) -> Any:
+    """Run one operation and return its decoded result, raising on error."""
+    payload = json.dumps(request).encode()
+    response = json.loads(_library().call(operation, payload))
+    error = response.get("error")
+    if error is not None:
+        raise _ERRORS.get(error["kind"], InternalError)(error["message"])
+    return response["ok"]

sqlscope_rs-0.2.0.dist-info/METADATA

@@ -0,0 +1,45 @@
+Metadata-Version: 2.5
+Name: sqlscope-rs
+Version: 0.2.0
+Summary: Scope-aware SQL analysis and rewriting: row filters, CTE injection, table rewrites, column lineage.
+Project-URL: Repository, https://github.com/dcalsky/sqlscope
+License-Expression: MIT
+Keywords: lineage,parser,row-level-security,sql
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# sqlscope (Python)
+
+Scope-aware SQL analysis and rewriting, backed by the
+[sqlscope](https://github.com/dcalsky/sqlscope) FFI library.
+
+This package is pure Python. It loads the sqlscope shared library
+(`libsqlscope_ffi.so`, `libsqlscope_ffi.dylib` or `sqlscope_ffi.dll`) from a
+[sqlscope release](https://github.com/dcalsky/sqlscope/releases) through
+`ctypes`, and never bundles or downloads it. Point it at the library with
+`SQLSCOPE_LIBRARY_PATH` or `sqlscope.load(path)`; otherwise it looks in the
+package directory and then on the system library search path.
+
+```bash
+pip install sqlscope-rs
+```
+
+```python
+import sqlscope
+
+sqlscope.load("/opt/sqlscope/libsqlscope_ffi.so")  # optional with SQLSCOPE_LIBRARY_PATH
+
+sqlscope.apply_row_filter("SELECT id FROM orders", "tenant_id = 7", dialect="postgres")
+# 'SELECT id FROM (SELECT * FROM orders WHERE tenant_id = 7) AS orders'
+
+sqlscope.column_origins(
+    "SELECT o.id, p.amount FROM orders o JOIN payments p ON o.id = p.order_id WHERE o.status = 'PAID'"
+)
+# {'orders': ['id'], 'payments': ['amount']}
+```
+
+See the [project README](https://github.com/dcalsky/sqlscope#readme) for the
+full API.

sqlscope_rs-0.2.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+sqlscope/__init__.py,sha256=v4IUB84bPKElg7mUgBDAw4H2vz2dpgSoI6mxqn8yv6U,9757
+sqlscope/_library.py,sha256=4ILjg8eBSf6P6XueGKYZ6161ul_XAHOHuCXEL0tzg28,5124
+sqlscope/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sqlscope_rs-0.2.0.dist-info/METADATA,sha256=CyTEudoqegKpIt_7Fx_yuDAMd71Z1HKqokWj4E0R5q0,1615
+sqlscope_rs-0.2.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+sqlscope_rs-0.2.0.dist-info/RECORD,,

sqlscope_rs-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any