sqlscope-rs 0.1.0__tar.gz → 0.3.0__tar.gz

sqlscope_rs-0.3.0/.gitignore

@@ -0,0 +1,10 @@
+target/
+.venv/
+__pycache__/
+*.pyc
+*.so
+.pytest_cache/
+dist/
+*.egg-info/
+*.dylib
+*.dll

{sqlscope_rs-0.1.0 → sqlscope_rs-0.3.0}/PKG-INFO

@@ -1,21 +1,27 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: sqlscope-rs
-Version: 0.1.0
+Version: 0.3.0
+Summary: Scope-aware SQL analysis and rewriting: row filters, CTE injection, table rewrites, column lineage.
+Project-URL: Repository, https://github.com/dcalsky/sqlscope
+License-Expression: MIT
+Keywords: lineage,parser,row-level-security,sql
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Rust
 Classifier: Topic :: Database
 Classifier: Typing :: Typed
-Summary: Scope-aware SQL analysis and rewriting: row filters, CTE injection, table rewrites, column lineage.
-Keywords: sql,lineage,row-level-security,parser
-License-Expression: MIT
 Requires-Python: >=3.9
-Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
-Project-URL: Repository, https://github.com/dcalsky/sqlscope
+Description-Content-Type: text/markdown
 
 # sqlscope (Python)
 
-Scope-aware SQL analysis and rewriting, backed by the Rust
-[sqlscope](https://github.com/dcalsky/sqlscope) crate.
+Scope-aware SQL analysis and rewriting, backed by the
+[sqlscope](https://github.com/dcalsky/sqlscope) FFI library.
+
+This package is pure Python. It loads the sqlscope shared library
+(`libsqlscope_ffi.so`, `libsqlscope_ffi.dylib` or `sqlscope_ffi.dll`) from a
+[sqlscope release](https://github.com/dcalsky/sqlscope/releases) through
+`ctypes`, and never bundles or downloads it. Point it at the library with
+`SQLSCOPE_LIBRARY_PATH` or `sqlscope.load(path)`; otherwise it looks in the
+package directory and then on the system library search path.
 
 ```bash
 pip install sqlscope-rs
@@ -24,6 +30,8 @@ pip install sqlscope-rs
 ```python
 import sqlscope
 
+sqlscope.load("/opt/sqlscope/libsqlscope_ffi.so")  # optional with SQLSCOPE_LIBRARY_PATH
+
 sqlscope.apply_row_filter("SELECT id FROM orders", "tenant_id = 7", dialect="postgres")
 # 'SELECT id FROM (SELECT * FROM orders WHERE tenant_id = 7) AS orders'
 
@@ -35,4 +43,3 @@ sqlscope.column_origins(
 
 See the [project README](https://github.com/dcalsky/sqlscope#readme) for the
 full API.
-

sqlscope_rs-0.3.0/README.md

@@ -0,0 +1,32 @@
+# sqlscope (Python)
+
+Scope-aware SQL analysis and rewriting, backed by the
+[sqlscope](https://github.com/dcalsky/sqlscope) FFI library.
+
+This package is pure Python. It loads the sqlscope shared library
+(`libsqlscope_ffi.so`, `libsqlscope_ffi.dylib` or `sqlscope_ffi.dll`) from a
+[sqlscope release](https://github.com/dcalsky/sqlscope/releases) through
+`ctypes`, and never bundles or downloads it. Point it at the library with
+`SQLSCOPE_LIBRARY_PATH` or `sqlscope.load(path)`; otherwise it looks in the
+package directory and then on the system library search path.
+
+```bash
+pip install sqlscope-rs
+```
+
+```python
+import sqlscope
+
+sqlscope.load("/opt/sqlscope/libsqlscope_ffi.so")  # optional with SQLSCOPE_LIBRARY_PATH
+
+sqlscope.apply_row_filter("SELECT id FROM orders", "tenant_id = 7", dialect="postgres")
+# 'SELECT id FROM (SELECT * FROM orders WHERE tenant_id = 7) AS orders'
+
+sqlscope.column_origins(
+    "SELECT o.id, p.amount FROM orders o JOIN payments p ON o.id = p.order_id WHERE o.status = 'PAID'"
+)
+# {'orders': ['id'], 'payments': ['amount']}
+```
+
+See the [project README](https://github.com/dcalsky/sqlscope#readme) for the
+full API.

{sqlscope_rs-0.1.0 → sqlscope_rs-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
-requires = ["maturin>=1.8,<2"]
-build-backend = "maturin"
+requires = ["hatchling>=1.24"]
+build-backend = "hatchling.build"
 
 [project]
 name = "sqlscope-rs"
@@ -12,7 +12,6 @@ dynamic = ["version"]
 keywords = ["sql", "lineage", "row-level-security", "parser"]
 classifiers = [
     "Programming Language :: Python :: 3",
-    "Programming Language :: Rust",
     "Topic :: Database",
     "Typing :: Typed",
 ]
@@ -20,14 +19,15 @@ classifiers = [
 [project.urls]
 Repository = "https://github.com/dcalsky/sqlscope"
 
-[tool.maturin]
-module-name = "sqlscope._native"
-features = ["pyo3/extension-module"]
-manifest-path = "python/Cargo.toml"
-python-source = "python"
+[tool.hatch.version]
+path = "src/sqlscope/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/sqlscope"]
 
 [dependency-groups]
 dev = ["pytest>=8", "mypy>=1.10"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+pythonpath = ["src"]

{sqlscope_rs-0.1.0/python → sqlscope_rs-0.3.0/src}/sqlscope/__init__.py

@@ -14,24 +14,34 @@ column_usages          Column references with the clause they appear in.
 
 Every function accepts ``dialect`` (default ``"trino"``). Functions release the
 GIL while they run and are safe to call from several threads.
+
+The SQL engine is the sqlscope FFI shared library (``libsqlscope_ffi.so``,
+``libsqlscope_ffi.dylib`` or ``sqlscope_ffi.dll``) published with each sqlscope
+release. It is loaded on first use; see :func:`load` for how it is located.
+This package neither bundles nor downloads it.
 """
 
 from __future__ import annotations
 
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
 
-from . import _native
-from ._native import (
+from ._library import (
+    LIBRARY_PATH_ENV,
     Error,
     InternalError,
     InvalidArgumentError,
+    LibraryNotFoundError,
     ParseError,
     UnsupportedError,
+    call,
+    library_file_name,
+    library_version,
+    load,
 )
 
-__version__: str = _native.__version__
+__version__ = "0.3.0"
 
 __all__ = [
     "Clause",
@@ -40,6 +50,8 @@ __all__ = [
     "Error",
     "InternalError",
     "InvalidArgumentError",
+    "LIBRARY_PATH_ENV",
+    "LibraryNotFoundError",
     "ParseError",
     "Schema",
     "TableRef",
@@ -50,6 +62,9 @@ __all__ = [
     "column_origins",
     "column_usages",
     "inject_ctes",
+    "library_file_name",
+    "library_version",
+    "load",
     "output_columns",
     "referenced_columns",
     "rewrite_tables",
@@ -150,6 +165,15 @@ def _list(values: Optional[Iterable[str]]) -> Optional[List[str]]:
     return list(values)
 
 
+def _run(operation: str, sql: str, *, dialect: Optional[str], **fields: Any) -> Any:
+    """Calls ``operation``; ``fields`` holds request fields and options, ``None`` meaning unset."""
+    options = {"dialect": dialect}
+    for name in ("schema", "tableNames", "tablePatterns", "defaultDb", "stripCatalogs"):
+        options[name] = fields.pop(name, None)
+    request = {"sql": sql, **fields, "options": {k: v for k, v in options.items() if v is not None}}
+    return call(operation, request)
+
+
 def apply_row_filter(
     sql: str,
     predicate: str,
@@ -169,14 +193,16 @@ def apply_row_filter(
     boolean expression; bind or escape its values before calling. Returns the
     input unchanged when no table is in scope.
     """
-    return _native.apply_row_filter(
+    result: str = _run(
+        "apply_row_filter",
         sql,
-        predicate,
+        predicate=predicate,
         dialect=dialect,
-        table_names=_list(table_names),
-        table_patterns=_list(table_patterns),
-        default_db=default_db,
+        tableNames=_list(table_names),
+        tablePatterns=_list(table_patterns),
+        defaultDb=default_db,
     )
+    return result
 
 
 def inject_ctes(
@@ -190,12 +216,16 @@ def inject_ctes(
     Definitions keep their order, so each may use earlier ones. A name that
     repeats another definition or an existing root CTE is rejected.
     """
-    pairs = [(cte.name, cte.query) if isinstance(cte, CteDef) else (cte[0], cte[1]) for cte in ctes]
-    return _native.inject_ctes(sql, pairs, dialect=dialect)
+    defs = [
+        {"name": cte.name, "query": cte.query} if isinstance(cte, CteDef) else {"name": cte[0], "query": cte[1]}
+        for cte in ctes
+    ]
+    result: str = _run("inject_ctes", sql, ctes=defs, dialect=dialect)
+    return result
 
 
-def _table(ref: TableRef) -> Tuple[str, Optional[str], Optional[str]]:
-    return (ref.table, ref.schema, ref.catalog)
+def _table(ref: TableRef) -> Dict[str, Optional[str]]:
+    return {"table": ref.table, "schema": ref.schema, "catalog": ref.catalog}
 
 
 def rewrite_tables(
@@ -213,20 +243,21 @@ def rewrite_tables(
     unchanged when nothing matches.
     """
     plan = [
-        (
-            rewrite.match_key,
-            _table(rewrite.inline) if rewrite.inline is not None else None,
-            (
-                rewrite.union.table_alias,
-                list(rewrite.union.columns),
-                [_table(branch) for branch in rewrite.union.branches],
-            )
+        {
+            "matchKey": rewrite.match_key,
+            "inline": _table(rewrite.inline) if rewrite.inline is not None else None,
+            "union": {
+                "tableAlias": rewrite.union.table_alias,
+                "columns": list(rewrite.union.columns),
+                "branches": [_table(branch) for branch in rewrite.union.branches],
+            }
             if rewrite.union is not None
             else None,
-        )
+        }
         for rewrite in rewrites
     ]
-    return _native.rewrite_tables(sql, plan, dialect=dialect, strip_catalogs=_list(strip_catalogs))
+    result: str = _run("rewrite_tables", sql, rewrites=plan, dialect=dialect, stripCatalogs=_list(strip_catalogs))
+    return result
 
 
 def column_origins(
@@ -238,7 +269,8 @@ def column_origins(
     right side of INTERSECT / EXCEPT are excluded. Every table read is present,
     possibly with an empty list.
     """
-    return _native.column_origins(sql, dialect=dialect, schema=_schema(schema))
+    result: Dict[str, List[str]] = _run("column_origins", sql, dialect=dialect, schema=_schema(schema))
+    return result
 
 
 def output_columns(
@@ -249,14 +281,16 @@ def output_columns(
     Unaliased expressions are named ``_col{i}``; ``*`` expands from ``schema``
     or stays ``"*"``. Returns ``None`` for statements without columns.
     """
-    return _native.output_columns(sql, dialect=dialect, schema=_schema(schema))
+    result: Optional[List[str]] = _run("output_columns", sql, dialect=dialect, schema=_schema(schema))
+    return result
 
 
 def referenced_columns(
     sql: str, *, dialect: Optional[str] = None, schema: Optional[Schema] = None
 ) -> Dict[str, List[str]]:
     """Columns referenced anywhere in a statement (including filters), keyed by root table."""
-    return _native.referenced_columns(sql, dialect=dialect, schema=_schema(schema))
+    result: Dict[str, List[str]] = _run("referenced_columns", sql, dialect=dialect, schema=_schema(schema))
+    return result
 
 
 def column_usages(
@@ -264,6 +298,6 @@ def column_usages(
 ) -> List[ColumnUsage]:
     """Every distinct ``(table, column, clause)`` use in a statement, sorted."""
     return [
-        ColumnUsage(table, column, Clause(clause))
-        for table, column, clause in _native.column_usages(sql, dialect=dialect, schema=_schema(schema))
+        ColumnUsage(usage["table"], usage["column"], Clause(usage["clause"]))
+        for usage in _run("column_usages", sql, dialect=dialect, schema=_schema(schema))
     ]

sqlscope_rs-0.3.0/src/sqlscope/_library.py

@@ -0,0 +1,157 @@
+"""Loads the sqlscope FFI library and calls into it through ctypes."""
+
+from __future__ import annotations
+
+import ctypes
+import json
+import os
+import sys
+import threading
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+LIBRARY_PATH_ENV = "SQLSCOPE_LIBRARY_PATH"
+"""Environment variable that locates the library when :func:`load` gets no path."""
+
+_ABI_VERSION = 1
+
+
+class Error(Exception):
+    """Base class of every sqlscope error."""
+
+
+class InvalidArgumentError(Error):
+    """An option or argument is invalid."""
+
+
+class ParseError(Error):
+    """The SQL text could not be parsed."""
+
+
+class UnsupportedError(Error):
+    """The statement shape is not supported, or the input exceeded a safety limit."""
+
+
+class InternalError(Error):
+    """sqlscope produced an invalid result (a bug)."""
+
+
+class LibraryNotFoundError(Error, OSError):
+    """The sqlscope FFI library could not be loaded."""
+
+
+_ERRORS = {
+    "invalid_argument": InvalidArgumentError,
+    "parse": ParseError,
+    "unsupported": UnsupportedError,
+}
+
+
+def library_file_name() -> str:
+    """The platform's file name of the sqlscope shared library."""
+    if sys.platform == "win32":
+        return "sqlscope_ffi.dll"
+    if sys.platform == "darwin":
+        return "libsqlscope_ffi.dylib"
+    return "libsqlscope_ffi.so"
+
+
+class _Library:
+    def __init__(self, path: str) -> None:
+        try:
+            lib = ctypes.CDLL(path)
+        except OSError as error:
+            raise LibraryNotFoundError(f"cannot load {path}: {error}") from error
+        try:
+            lib.sqlscope_abi_version.restype = ctypes.c_uint32
+            lib.sqlscope_abi_version.argtypes = []
+            lib.sqlscope_version.restype = ctypes.c_char_p
+            lib.sqlscope_version.argtypes = []
+            lib.sqlscope_call.restype = ctypes.c_void_p
+            lib.sqlscope_call.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+            lib.sqlscope_free.restype = None
+            lib.sqlscope_free.argtypes = [ctypes.c_void_p]
+        except AttributeError as error:
+            raise LibraryNotFoundError(f"{path} is not a sqlscope library: {error}") from error
+        abi = lib.sqlscope_abi_version()
+        if abi != _ABI_VERSION:
+            raise LibraryNotFoundError(f"{path} implements ABI {abi}, expected {_ABI_VERSION}")
+        self.path = path
+        self.version: str = lib.sqlscope_version().decode()
+        self._lib = lib
+
+    def call(self, operation: str, request: bytes) -> bytes:
+        # ctypes releases the GIL for the duration of the call.
+        response = self._lib.sqlscope_call(operation.encode(), request)
+        try:
+            return ctypes.string_at(response)
+        finally:
+            self._lib.sqlscope_free(response)
+
+
+_loaded: Optional[_Library] = None
+_lock = threading.Lock()
+
+
+def _default_candidates() -> list[str]:
+    path = os.environ.get(LIBRARY_PATH_ENV)
+    if path:
+        return [path]
+    name = library_file_name()
+    return [str(Path(__file__).resolve().parent / name), name]
+
+
+def load(path: Optional[str] = None) -> None:
+    """Load the sqlscope FFI library, the shared library published with each sqlscope release.
+
+    Without ``path`` it tries, in order, the path in ``$SQLSCOPE_LIBRARY_PATH``,
+    :func:`library_file_name` inside the ``sqlscope`` package directory, and
+    :func:`library_file_name` on the system library search path. Nothing is
+    ever downloaded.
+
+    Calling ``load`` is optional: the first operation loads the library the
+    same way. Once a library is loaded it stays loaded; loading a different
+    path afterwards raises :class:`LibraryNotFoundError`.
+    """
+    global _loaded
+    with _lock:
+        if _loaded is not None:
+            if path is None or os.fspath(path) == _loaded.path:
+                return
+            raise LibraryNotFoundError(f"sqlscope library already loaded from {_loaded.path}")
+        if path is not None:
+            _loaded = _Library(os.fspath(path))
+            return
+        failures = []
+        for candidate in _default_candidates():
+            try:
+                _loaded = _Library(candidate)
+                return
+            except LibraryNotFoundError as error:
+                failures.append(str(error))
+        raise LibraryNotFoundError(
+            f"cannot load {library_file_name()}; download it from a sqlscope release and pass its "
+            f"path to sqlscope.load() or set {LIBRARY_PATH_ENV}: " + "; ".join(failures)
+        )
+
+
+def _library() -> _Library:
+    if _loaded is None:
+        load()
+    assert _loaded is not None
+    return _loaded
+
+
+def library_version() -> str:
+    """The version of the loaded sqlscope FFI library, loading it if needed."""
+    return _library().version
+
+
+def call(operation: str, request: Dict[str, Any]) -> Any:
+    """Run one operation and return its decoded result, raising on error."""
+    payload = json.dumps(request).encode()
+    response = json.loads(_library().call(operation, payload))
+    error = response.get("error")
+    if error is not None:
+        raise _ERRORS.get(error["kind"], InternalError)(error["message"])
+    return response["ok"]

{sqlscope_rs-0.1.0/python → sqlscope_rs-0.3.0}/tests/test_sqlscope.py

@@ -101,5 +101,20 @@ def test_threads():
     assert sorted(results) == sorted([[f"a{i}"] for i in range(8)])
 
 
-def test_version():
+def test_library():
     assert sqlscope.__version__
+    assert sqlscope.library_version()
+    sqlscope.load()  # already loaded: no-op
+    with pytest.raises(sqlscope.LibraryNotFoundError):
+        sqlscope.load("/nonexistent/" + sqlscope.library_file_name())
+
+
+def test_deep_nesting_in_threads():
+    sql = "SELECT a FROM t"
+    for i in range(120):
+        sql = f"SELECT a FROM ({sql}) x{i}"
+    results = []
+    thread = threading.Thread(target=lambda: results.append(sqlscope.column_origins(sql)))
+    thread.start()
+    thread.join()
+    assert results == [{"t": ["a"]}]