datalevin 0.10.16__py3-none-any.whl

datalevin/__init__.py

@@ -0,0 +1,37 @@
+"""Datalevin Python bindings over the JVM interop bridge."""
+
+from ._interop import api_info, connect, exec_json, new_client, open_kv
+from ._jvm import jvm_started, start_jvm
+from ._raw import interop
+from .client import Client
+from .connection import Connection
+from .errors import (
+    DatalevinConfigurationError,
+    DatalevinError,
+    DatalevinJavaError,
+    DatalevinJvmError,
+)
+from .kv import KV
+from .udf import UdfRegistry, create_udf_registry
+
+__all__ = [
+    "Client",
+    "Connection",
+    "DatalevinConfigurationError",
+    "DatalevinError",
+    "DatalevinJavaError",
+    "DatalevinJvmError",
+    "KV",
+    "UdfRegistry",
+    "api_info",
+    "connect",
+    "create_udf_registry",
+    "exec_json",
+    "interop",
+    "jvm_started",
+    "new_client",
+    "open_kv",
+    "start_jvm",
+]
+
+__version__ = "0.10.16"

datalevin/_convert.py

@@ -0,0 +1,192 @@
+"""Python <-> Java conversion helpers for Datalevin interop."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from decimal import Decimal
+import uuid as py_uuid
+
+import jpype
+from jpype.types import JByte, JLong
+
+from ._java import call_java, classes, is_java_object
+
+INT64_MIN = -(2**63)
+INT64_MAX = 2**63 - 1
+FORM_SYMBOL_STRINGS = {"*", "...", ".", "$", "%", "_"}
+
+
+def to_java(value):
+    """Recursively convert Python values into Java/JVM-friendly values."""
+
+    if value is None or isinstance(value, (bool, float, str)):
+        return value
+    if is_java_object(value):
+        return value
+
+    raw_handle = getattr(value, "raw_handle", None)
+    if callable(raw_handle):
+        return raw_handle()
+
+    if isinstance(value, int):
+        if INT64_MIN <= value <= INT64_MAX:
+            return JLong(value)
+        return classes().big_integer(str(value))
+
+    if isinstance(value, Decimal):
+        return classes().big_decimal(str(value))
+
+    if isinstance(value, py_uuid.UUID):
+        return classes().uuid.fromString(str(value))
+
+    if isinstance(value, datetime):
+        if value.tzinfo is None:
+            value = value.replace(tzinfo=timezone.utc)
+        iso = value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+        return classes().instant.parse(iso)
+
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return jpype.JArray(JByte)(bytes(value))
+
+    if isinstance(value, Mapping):
+        jmap = classes().linked_hash_map()
+        for key, item in value.items():
+            jmap.put(to_java(key), to_java(item))
+        return jmap
+
+    if isinstance(value, (set, frozenset)):
+        jset = classes().linked_hash_set()
+        for item in value:
+            jset.add(to_java(item))
+        return jset
+
+    if isinstance(value, (list, tuple)):
+        jlist = classes().array_list()
+        for item in value:
+            jlist.add(to_java(item))
+        return jlist
+
+    return value
+
+
+def to_python(value):
+    """Recursively convert Java results into Python-native values."""
+
+    if value is None or isinstance(value, (bool, int, float, str, bytes)):
+        return value
+
+    if not is_java_object(value):
+        return value
+
+    cls = classes()
+    byte_array_type = jpype.JArray(JByte)
+
+    if isinstance(value, byte_array_type):
+        return bytes(value)
+
+    if isinstance(value, cls.uuid):
+        return py_uuid.UUID(str(value))
+
+    if isinstance(value, cls.instant):
+        return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+
+    if isinstance(value, cls.date):
+        return datetime.fromtimestamp(value.getTime() / 1000.0, tz=timezone.utc)
+
+    if isinstance(value, cls.big_integer):
+        return int(str(value))
+
+    if isinstance(value, cls.big_decimal):
+        return Decimal(str(value))
+
+    if isinstance(value, (cls.keyword_type, cls.symbol_type)):
+        return str(value)
+
+    if isinstance(value, cls.map_type) or hasattr(value, "entrySet"):
+        result = {}
+        for entry in value.entrySet():
+            result[to_python(entry.getKey())] = to_python(entry.getValue())
+        return result
+
+    if isinstance(value, cls.set_type):
+        return {to_python(item) for item in value}
+
+    if isinstance(value, (cls.list_type, cls.collection_type)) or hasattr(value, "iterator"):
+        return [to_python(item) for item in value]
+
+    return value
+
+
+def to_edn_form(value):
+    """Convert Python form structures into EDN-friendly JVM values."""
+
+    if value is None or is_java_object(value):
+        return value
+
+    raw_handle = getattr(value, "raw_handle", None)
+    if callable(raw_handle):
+        return raw_handle()
+
+    if isinstance(value, str):
+        if value.startswith(":"):
+            return call_java(classes().interop.keyword, value)
+        if value.startswith("?") or value in FORM_SYMBOL_STRINGS:
+            return call_java(classes().interop.symbol, value)
+        return value
+
+    if isinstance(value, Mapping):
+        jmap = classes().linked_hash_map()
+        for key, item in value.items():
+            jmap.put(to_edn_form(key), to_edn_form(item))
+        return jmap
+
+    if isinstance(value, (set, frozenset)):
+        jset = classes().linked_hash_set()
+        for item in value:
+            jset.add(to_edn_form(item))
+        return jset
+
+    if isinstance(value, (list, tuple)):
+        jlist = classes().array_list()
+        for item in value:
+            jlist.add(to_edn_form(item))
+        return jlist
+
+    return to_java(value)
+
+
+def to_query_input(value):
+    """Convert Python query inputs while preserving ordinary string literals."""
+
+    if value is None or is_java_object(value):
+        return value
+
+    raw_handle = getattr(value, "raw_handle", None)
+    if callable(raw_handle):
+        return raw_handle()
+
+    if isinstance(value, str):
+        if value.startswith(":"):
+            return call_java(classes().interop.keyword, value)
+        return value
+
+    if isinstance(value, Mapping):
+        jmap = classes().linked_hash_map()
+        for key, item in value.items():
+            jmap.put(to_query_input(key), to_query_input(item))
+        return jmap
+
+    if isinstance(value, (set, frozenset)):
+        jset = classes().linked_hash_set()
+        for item in value:
+            jset.add(to_query_input(item))
+        return jset
+
+    if isinstance(value, (list, tuple)):
+        jlist = classes().array_list()
+        for item in value:
+            jlist.add(to_query_input(item))
+        return jlist
+
+    return to_java(value)

datalevin/_interop.py

@@ -0,0 +1,197 @@
+"""Internal interop bindings and public constructors."""
+
+from __future__ import annotations
+
+import json
+
+from ._convert import to_java, to_python
+from ._java import call_java, classes
+from ._jvm import jvm_started, start_jvm
+from .errors import DatalevinError
+
+
+class InteropBindings:
+    """Thin wrapper around the Datalevin JVM bridge."""
+
+    def api_info_raw(self):
+        return call_java(classes().datalevin.apiInfo)
+
+    def exec_json_raw(self, request_json: str) -> str:
+        json_api_exec = getattr(classes().json_api, "exec_", None)
+        if json_api_exec is None:
+            json_api_exec = getattr(classes().json_api, "exec")
+        return str(call_java(json_api_exec, request_json))
+
+    def core_invoke(self, function: str, args=None):
+        return call_java(classes().interop.coreInvoke, function, to_java(list(args or ())))
+
+    def client_invoke(self, function: str, args=None):
+        return call_java(classes().interop.clientInvoke, function, to_java(list(args or ())))
+
+    def create_connection(self, dir=None, schema=None, opts=None, *, shared: bool = False):
+        target = classes().interop.getConnection if shared else classes().interop.createConnection
+        return call_java(target, dir, to_java(schema), to_java(opts))
+
+    def close_connection(self, handle) -> None:
+        call_java(classes().interop.closeConnection, handle)
+
+    def connection_closed(self, handle) -> bool:
+        return bool(call_java(classes().interop.connectionClosed, handle))
+
+    def connection_db(self, handle):
+        return call_java(classes().interop.connectionDb, handle)
+
+    def open_key_value(self, dir, opts=None):
+        return call_java(classes().interop.openKeyValue, dir, to_java(opts))
+
+    def close_key_value(self, handle) -> None:
+        call_java(classes().interop.closeKeyValue, handle)
+
+    def key_value_closed(self, handle) -> bool:
+        return bool(call_java(classes().interop.keyValueClosed, handle))
+
+    def new_client(self, uri, opts=None):
+        return call_java(classes().interop.newClient, uri, to_java(opts))
+
+    def close_client(self, handle) -> None:
+        call_java(classes().interop.closeClient, handle)
+
+    def client_disconnected(self, handle) -> bool:
+        return bool(call_java(classes().interop.clientDisconnected, handle))
+
+    def read_edn(self, edn: str):
+        return call_java(classes().interop.readEdn, edn)
+
+    def keyword(self, value: str):
+        return call_java(classes().interop.keyword, value)
+
+    def symbol(self, value: str):
+        return call_java(classes().interop.symbol, value)
+
+    def schema(self, schema):
+        if schema is None:
+            return None
+        return call_java(classes().interop.schema, to_java(schema))
+
+    def options(self, opts):
+        if opts is None:
+            return None
+        return call_java(classes().interop.options, to_java(opts))
+
+    def udf_descriptor(self, descriptor):
+        if descriptor is None:
+            return None
+        return call_java(classes().interop.udfDescriptor, to_java(descriptor))
+
+    def create_udf_registry(self):
+        return call_java(classes().interop.createUdfRegistry)
+
+    def register_udf(self, registry, descriptor, fn):
+        return call_java(classes().interop.registerUdf, registry, to_java(descriptor), fn)
+
+    def unregister_udf(self, registry, descriptor):
+        return call_java(classes().interop.unregisterUdf, registry, to_java(descriptor))
+
+    def registered_udf(self, registry, descriptor) -> bool:
+        return bool(call_java(classes().interop.registeredUdf, registry, to_java(descriptor)))
+
+    def rename_map(self, rename_map):
+        if rename_map is None:
+            return None
+        return call_java(classes().interop.renameMap, to_java(rename_map))
+
+    def delete_attrs(self, attrs):
+        if attrs is None:
+            return None
+        return call_java(classes().interop.deleteAttrs, to_java(list(attrs or ())))
+
+    def lookup_ref(self, value):
+        if value is None:
+            return None
+        return call_java(classes().interop.lookupRef, to_java(value))
+
+    def tx_data(self, tx_data):
+        if tx_data is None:
+            return None
+        return call_java(classes().interop.txData, to_java(tx_data))
+
+    def kv_txs(self, txs):
+        if txs is None:
+            return None
+        return call_java(classes().interop.kvTxs, to_java(txs))
+
+    def kv_type(self, value):
+        if value is None:
+            return None
+        return call_java(classes().interop.kvType, to_java(value))
+
+    def database_type(self, value: str):
+        return call_java(classes().interop.databaseType, value)
+
+    def role(self, role: str):
+        return call_java(classes().interop.role, role)
+
+    def permission_keyword(self, value: str):
+        return call_java(classes().interop.permissionKeyword, value)
+
+    def permission_target(self, object_type: str, target):
+        return call_java(classes().interop.permissionTarget, object_type, to_java(target))
+
+
+_BINDINGS = InteropBindings()
+
+
+def api_info():
+    """Return Datalevin JSON/API metadata as a Python dictionary."""
+
+    return to_python(_BINDINGS.api_info_raw())
+
+
+def exec_json(op: str, args=None):
+    """Execute a raw JSON API operation."""
+
+    request = json.dumps({"op": op, "args": args or {}})
+    envelope = json.loads(_BINDINGS.exec_json_raw(request))
+    if envelope.get("ok"):
+        return envelope.get("result")
+    raise DatalevinError(
+        envelope.get("error") or "Datalevin JSON API request failed.",
+        type_name=envelope.get("type"),
+        data=envelope.get("data"),
+    )
+
+
+def connect(dir=None, schema=None, opts=None, *, shared: bool = False) -> Connection:
+    """Create or open a Datalevin Datalog connection."""
+
+    from .connection import Connection
+
+    return Connection(_BINDINGS.create_connection(dir, schema, opts, shared=shared))
+
+
+def open_kv(dir, opts=None) -> KV:
+    """Open a Datalevin KV store."""
+
+    from .kv import KV
+
+    return KV(_BINDINGS.open_key_value(dir, opts))
+
+
+def new_client(uri, opts=None) -> Client:
+    """Open a remote Datalevin admin client."""
+
+    from .client import Client
+
+    return Client(_BINDINGS.new_client(uri, opts))
+
+
+__all__ = [
+    "_BINDINGS",
+    "api_info",
+    "connect",
+    "exec_json",
+    "jvm_started",
+    "new_client",
+    "open_kv",
+    "start_jvm",
+]

datalevin/_java.py

@@ -0,0 +1,84 @@
+"""JPype class loading and Java call helpers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+
+import jpype
+
+from ._jvm import start_jvm
+from .errors import DatalevinJavaError
+
+
+@dataclass(frozen=True)
+class JavaClasses:
+    """Cached Java classes used by the Python wrapper."""
+
+    datalevin: object
+    interop: object
+    json_api: object
+    udf_function: object
+    linked_hash_map: object
+    linked_hash_set: object
+    array_list: object
+    uuid: object
+    instant: object
+    date: object
+    big_integer: object
+    big_decimal: object
+    map_type: object
+    list_type: object
+    set_type: object
+    collection_type: object
+    keyword_type: object
+    symbol_type: object
+
+
+@lru_cache(maxsize=1)
+def classes() -> JavaClasses:
+    """Load and cache the Java classes used by the binding."""
+
+    start_jvm()
+    return JavaClasses(
+        datalevin=jpype.JClass("datalevin.Datalevin"),
+        interop=jpype.JClass("datalevin.DatalevinInterop"),
+        json_api=jpype.JClass("datalevin.JSONApi"),
+        udf_function=jpype.JClass("datalevin.UdfFunction"),
+        linked_hash_map=jpype.JClass("java.util.LinkedHashMap"),
+        linked_hash_set=jpype.JClass("java.util.LinkedHashSet"),
+        array_list=jpype.JClass("java.util.ArrayList"),
+        uuid=jpype.JClass("java.util.UUID"),
+        instant=jpype.JClass("java.time.Instant"),
+        date=jpype.JClass("java.util.Date"),
+        big_integer=jpype.JClass("java.math.BigInteger"),
+        big_decimal=jpype.JClass("java.math.BigDecimal"),
+        map_type=jpype.JClass("java.util.Map"),
+        list_type=jpype.JClass("java.util.List"),
+        set_type=jpype.JClass("java.util.Set"),
+        collection_type=jpype.JClass("java.util.Collection"),
+        keyword_type=jpype.JClass("clojure.lang.Keyword"),
+        symbol_type=jpype.JClass("clojure.lang.Symbol"),
+    )
+
+
+def call_java(target, *args):
+    """Call a Java method and wrap JPype Java exceptions."""
+
+    try:
+        return target(*args)
+    except jpype.JException as exc:
+        raise DatalevinJavaError.from_java(exc) from exc
+
+
+def is_java_object(value) -> bool:
+    """Best-effort detection for live Java objects exposed through JPype."""
+
+    if value is None or not jpype.isJVMStarted():
+        return False
+    module = type(value).__module__
+    return (
+        module.startswith("jpype")
+        or module.startswith("_jpype")
+        or str(type(value)).startswith("<java ")
+    )

datalevin/_jvm.py

@@ -0,0 +1,143 @@
+"""JVM bootstrap and runtime jar resolution."""
+
+from __future__ import annotations
+
+import importlib.resources as resources
+import os
+import re
+import shlex
+from pathlib import Path
+
+import jpype
+
+from .errors import DatalevinConfigurationError, DatalevinJvmError
+
+DATALEVIN_JAR_ENV = "DATALEVIN_JAR"
+DATALEVIN_CLASSPATH_ENV = "DATALEVIN_CLASSPATH"
+DATALEVIN_JVM_ARGS_ENV = "DATALEVIN_JVM_ARGS"
+DATALEVIN_JAVACPP_CACHEDIR_ENV = "DATALEVIN_JAVACPP_CACHEDIR"
+PACKAGE_NAME = "datalevin"
+TARGET_JAR_PATTERNS = (
+    "datalevin-runtime-*.jar",
+    "datalevin-python-runtime-*.jar",
+    "datalevin-java-*.jar",
+)
+DEFAULT_JAVACPP_CACHEDIR = Path("/tmp/datalevin-javacpp-cache")
+
+
+def jvm_started() -> bool:
+    """Return whether the current Python process already has a JVM."""
+
+    return jpype.isJVMStarted()
+
+
+def resolve_classpath(classpath: list[str] | tuple[str, ...] | None = None) -> list[str]:
+    """Resolve the JVM classpath for the Datalevin runtime."""
+
+    if classpath:
+        return [str(Path(entry).expanduser()) for entry in classpath]
+
+    env_classpath = os.environ.get(DATALEVIN_CLASSPATH_ENV)
+    if env_classpath:
+        return [entry for entry in env_classpath.split(os.pathsep) if entry]
+
+    env_jar = os.environ.get(DATALEVIN_JAR_ENV)
+    if env_jar:
+        return [str(Path(env_jar).expanduser())]
+
+    vendored = _vendored_jars()
+    if vendored:
+        return vendored
+
+    repo_local = _repo_local_jars()
+    if repo_local:
+        return repo_local
+
+    raise DatalevinConfigurationError(
+        "Unable to resolve a Datalevin runtime jar. "
+        f"Set {DATALEVIN_JAR_ENV}, set {DATALEVIN_CLASSPATH_ENV}, "
+        "or vendor a jar under datalevin/jars/."
+    )
+
+
+def start_jvm(
+    *,
+    classpath: list[str] | tuple[str, ...] | None = None,
+    jvm_args: list[str] | tuple[str, ...] | None = None,
+    convert_strings: bool = False,
+) -> None:
+    """Start the JVM if it is not already running."""
+
+    if jpype.isJVMStarted():
+        return
+
+    resolved_classpath = resolve_classpath(classpath)
+    resolved_jvm_args = list(jvm_args or ())
+    if not resolved_jvm_args:
+        env_args = os.environ.get(DATALEVIN_JVM_ARGS_ENV)
+        if env_args:
+            resolved_jvm_args = shlex.split(env_args)
+    _ensure_javacpp_cachedir_arg(resolved_jvm_args)
+
+    try:
+        jpype.startJVM(
+            *resolved_jvm_args,
+            classpath=resolved_classpath,
+            convertStrings=convert_strings,
+        )
+    except OSError as exc:
+        raise DatalevinJvmError(
+            "Failed to start the JVM for Datalevin. "
+            "Check that Java 21+ is installed and that the runtime jar is valid.",
+            cause=exc,
+        ) from exc
+
+
+def _vendored_jars() -> list[str]:
+    try:
+        jar_root = resources.files(PACKAGE_NAME).joinpath("jars")
+    except (FileNotFoundError, ModuleNotFoundError):
+        return []
+
+    if not jar_root.is_dir():
+        return []
+
+    jar = _preferred_runtime_jar(Path(str(jar_root)))
+    return [] if jar is None else [str(jar)]
+
+
+def _repo_local_jars() -> list[str]:
+    repo_root = Path(__file__).resolve().parents[4]
+    target_dir = repo_root / "target"
+    if not target_dir.exists():
+        return []
+    jar = _preferred_runtime_jar(target_dir)
+    return [] if jar is None else [str(jar)]
+
+
+def _preferred_runtime_jar(dir_path: Path) -> Path | None:
+    for pattern in TARGET_JAR_PATTERNS:
+        matches = sorted(dir_path.glob(pattern), key=_jar_sort_key)
+        if matches:
+            return matches[-1]
+    return None
+
+
+def _jar_sort_key(path: Path):
+    stem = path.stem
+    version = stem.rsplit("-", 1)[-1]
+    parts = re.split(r"([0-9]+)", version)
+    return [int(part) if part.isdigit() else part for part in parts]
+
+
+def _ensure_javacpp_cachedir_arg(jvm_args: list[str]) -> None:
+    for arg in jvm_args:
+        if (
+            arg.startswith("-Dorg.bytedeco.javacpp.cachedir=")
+            or arg.startswith("-Dorg.bytedeco.javacpp.cacheDir=")
+        ):
+            return
+
+    cache_dir = Path(os.environ.get(DATALEVIN_JAVACPP_CACHEDIR_ENV, DEFAULT_JAVACPP_CACHEDIR))
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    jvm_args.append(f"-Dorg.bytedeco.javacpp.cachedir={cache_dir}")