arcadedb-embedded-native 26.10.2.dev1__py3-none-win_amd64.whl

arcadedb_native/_native/bulk.py

@@ -0,0 +1,96 @@
+"""Columnar bulk insert through one crossing (``arcadedb_n_insert_cols``): numpy int/float/bool columns and float32 vector
+columns (a 2-D array or a list of equal-length 1-D float arrays) go over as one buffer; anything else returns None and the
+caller falls back to the per-row path. The documents are built Java-side as ``DocumentBatcher.insertColumns`` builds them on JPype.
+"""
+
+import json
+import os
+
+from . import lib
+
+try:
+    import numpy as _np
+except ImportError:  # pragma: no cover
+    _np = None
+
+_ENABLED = os.environ.get("ARCADEDB_NATIVE_BULK", "1") != "0"
+
+
+def insert_columns_fast(db, type_name, columns, commit_every):
+    if (
+        not _ENABLED
+        or _np is None
+        or lib._lib is None
+        or not hasattr(lib._lib, "arcadedb_n_insert_cols")
+    ):
+        return None
+    cols, parts, n = [], [], None
+    for name, values in columns.items():
+        if hasattr(values, "to_numpy") and not isinstance(values, (list, tuple)):
+            return None
+        kind, dim = None, 0
+        if isinstance(values, _np.ndarray):
+            k = values.dtype.kind
+            if (
+                values.ndim == 1
+                and k in "iu"
+                and (
+                    k == "i"
+                    or not values.size
+                    or int(values.max()) <= _np.iinfo(_np.int64).max
+                )
+            ):
+                kind, arr = "q", _np.ascontiguousarray(values, dtype=_np.int64)
+            elif values.ndim == 1 and k == "f":
+                kind, arr = "d", _np.ascontiguousarray(values, dtype=_np.float64)
+            elif values.ndim == 1 and k == "b":
+                kind, arr = "b", _np.ascontiguousarray(values, dtype=_np.uint8)
+            elif values.ndim == 2 and k == "f" and values.shape[1] > 0:
+                kind, arr, dim = (
+                    "f",
+                    _np.ascontiguousarray(values, dtype=_np.float32),
+                    int(values.shape[1]),
+                )
+        elif (
+            isinstance(values, (list, tuple))
+            and values
+            and all(
+                isinstance(v, _np.ndarray)
+                and v.ndim == 1
+                and v.dtype.kind == "f"
+                and v.shape == values[0].shape
+                for v in values
+            )
+            and values[0].shape[0] > 0
+        ):
+            kind, arr, dim = (
+                "f",
+                _np.ascontiguousarray(_np.stack(values), dtype=_np.float32),
+                int(values[0].shape[0]),
+            )
+        if kind is None:
+            return None
+        rows = int(arr.shape[0])
+        if n is not None and rows != n:
+            return None  # the per-row path reports the ragged columns
+        n = rows
+        cols.append([name, kind, dim])
+        parts.append(arr.tobytes())
+    if not n:
+        return None
+    header = json.dumps(
+        {"n": n, "commit": min(int(commit_every), 2**31 - 1), "cols": cols}
+    )
+    buf = b"".join(parts)
+    from .database import _call
+
+    reply = _call(
+        "Failed to bulk-insert",
+        "arcadedb_n_insert_cols",
+        db._h,
+        lib.b(type_name),
+        lib.b(header),
+        buf,
+        len(buf),
+    )
+    return int(reply["n"])

arcadedb_native/_native/codec.py

@@ -0,0 +1,229 @@
+"""Python <-> native JSON: query parameters out, typed rows in."""
+
+import json
+import os
+from collections.abc import Mapping
+from datetime import date, datetime, timezone
+from decimal import Decimal
+
+try:  # numpy is optional
+    import numpy as _np
+except ImportError:  # pragma: no cover
+    _np = None
+
+TAG = "\u0001"
+_EPOCH_NAIVE = datetime(1970, 1, 1)
+
+
+def _default(value, _bin=None):
+    """json.dumps hook: the values JSON cannot carry, as tagged objects (see Json.java).
+
+    ``_bin`` (binary mode): ``[arrays, floats_so_far]``; a float ndarray then goes to the side buffer as float32 and the JSON carries
+    ``["B", offset, count]`` instead of the number list (no float formatting in Python, no text parsing in Java).
+    """
+    if isinstance(value, RID):
+        return {TAG: ["R", value.text]}
+    if isinstance(value, Decimal):
+        return {TAG: ["n", str(value)]}
+    if isinstance(value, datetime):
+        utc = (
+            value.astimezone(timezone.utc).replace(tzinfo=None)
+            if value.utcoffset() is not None
+            else value
+        )
+        return {TAG: ["T", utc.isoformat()]}
+    if isinstance(value, date):
+        return {TAG: ["D", value.isoformat()]}
+    if isinstance(value, (set, frozenset)):
+        return {TAG: ["s", list(value)]}
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        import base64
+
+        return {TAG: ["b", base64.b64encode(bytes(value)).decode("ascii")]}
+    if _np is not None:
+        if isinstance(value, _np.ndarray):
+            if value.dtype.kind == "f":
+                if _bin is not None:
+                    arr = _np.ascontiguousarray(value.ravel(), dtype=_np.float32)
+                    off = _bin[1]
+                    _bin[0].append(arr)
+                    _bin[1] = off + arr.size
+                    return {TAG: ["B", off, arr.size]}
+                return {TAG: ["F", value.tolist()]}
+            return value.tolist()
+        if isinstance(value, _np.bool_):
+            return bool(value)
+        if isinstance(value, _np.integer):
+            return int(value)
+        if isinstance(value, _np.floating):
+            return float(value)
+    if isinstance(value, Mapping):
+        return dict(value)
+    raise TypeError(
+        f"cannot pass {type(value).__name__} as a parameter on the native backend"
+    )
+
+
+_PLAIN = frozenset((str, int, float, bool, type(None)))
+_dumps = json.dumps
+_fast = os.environ.get("ARCADEDB_NATIVE_FASTENC", "1") != "0"
+
+
+def _prep(value):
+    """A value ready for json.dumps: a dict with a key that is not a str becomes a tagged ``[[k, v], ..]`` map (JSON would turn
+    the key into text; the JPype path keeps it), and containers are walked for such dicts. Plain scalars pass untouched.
+    """
+    t = type(value)
+    if t in _PLAIN:
+        return value
+    if isinstance(value, Mapping):
+        if all(type(k) is str for k in value):
+            return {k: _prep(v) for k, v in value.items()}
+        return {TAG: ["m", [[_prep(k), _prep(v)] for k, v in value.items()]]}
+    if t is list or t is tuple:
+        return [_prep(v) for v in value]
+    return value
+
+
+def encode_params(args, binary=False):
+    """The ``params`` text for ``arcadedb_n_exec``: '' for none, else ``{"named":{..}}`` or ``{"pos":[..]}``.
+
+    Same selection as the JPype path: a lone list/tuple is the positional list itself, a lone mapping (also alone in
+    a list) is the named map, anything else is positional.
+    """
+    if not args:
+        return ("", None, 0) if binary else ""
+    values = args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args
+    plain = _PLAIN
+    if _fast and set(map(type, values)) <= plain:
+        # all plain scalars (the per-row INSERT of a load): the cached default encoder, no default hook, no per-call encoder object
+        text = _dumps({"pos": values if type(values) is list else list(values)})
+        return (text, None, 0) if binary else text
+    if len(values) == 1 and isinstance(values[0], Mapping):
+        named = values[0]
+        if not all(type(v) in plain for v in named.values()):
+            named = {k: _prep(v) for k, v in named.items()}
+        return _dump({"named": dict(named)}, binary)
+    if not all(type(v) in plain for v in values):
+        values = [_prep(v) for v in values]
+    return _dump({"pos": list(values)}, binary)
+
+
+def _dump(obj, binary):
+    """JSON text; binary mode returns ``(text, float32 side buffer bytes or None, float count)``."""
+    if not binary:
+        return json.dumps(obj, default=_default)
+    state = [[], 0]
+    text = json.dumps(obj, default=lambda v: _default(v, state))
+    arrays = state[0]
+    if not arrays:
+        return text, None, 0
+    buf = arrays[0].tobytes() if len(arrays) == 1 else _np.concatenate(arrays).tobytes()
+    return text, buf, state[1]
+
+
+class RID:
+    """A record id as the engine returns it in a row; passed back as a parameter it is a link, not a string."""
+
+    __slots__ = ("text",)
+
+    def __init__(self, text):
+        self.text = text
+
+    def __str__(self):
+        return self.text
+
+    def __repr__(self):
+        return f"RID({self.text!r})"
+
+    def __eq__(self, other):
+        return self.text == (other.text if isinstance(other, RID) else other)
+
+    def __hash__(self):
+        return hash(self.text)
+
+
+def _restore_float(text):
+    return float(text)
+
+
+def _restore_map(pairs):
+    return {k: v for k, v in pairs}
+
+
+_SIMPLE = {
+    "n": Decimal,
+    "i": int,
+    "D": date.fromisoformat,
+    "T": datetime.fromisoformat,
+    "Z": datetime.fromisoformat,
+    "s": set,
+    "f": _restore_float,
+    "m": _restore_map,
+    "R": RID,
+}
+
+
+def _hook(obj):
+    tagged = obj.get(TAG)
+    # a column may be called like the tag: only a lone [kind, payload] pair of a known kind is a tagged value
+    if (
+        tagged is None
+        or len(obj) != 1
+        or type(tagged) is not list
+        or len(tagged) != 2
+        or tagged[0] not in _SIMPLE
+    ):
+        return obj
+    kind, payload = tagged
+    return _SIMPLE[kind](payload)
+
+
+def loads(text: str, floats=None):
+    """Decode a reply, restoring tagged values. ``floats``: the float32 tail of a binary reply (``["B", offset, count]`` values)."""
+    if floats is None:
+        return json.loads(text, object_hook=_hook)
+
+    def hook(obj):
+        tagged = obj.get(TAG)
+        if (
+            tagged is not None
+            and len(obj) == 1
+            and type(tagged) is list
+            and len(tagged) == 3
+            and tagged[0] == "B"
+        ):
+            return floats[
+                tagged[1] : tagged[1] + tagged[2]
+            ].tolist()  # float32 widens to the same double as the JPype path
+        return _hook(obj)
+
+    return json.loads(text, object_hook=hook)
+
+
+# --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float.
+
+
+def json_native(value):
+    if isinstance(value, datetime):
+        return (
+            int((value.replace(tzinfo=None) - _EPOCH_NAIVE).total_seconds() * 1000)
+            if value.tzinfo is None
+            else int(value.timestamp() * 1000)
+        )
+    if isinstance(value, date):
+        return int(
+            (
+                datetime.combine(value, datetime.min.time()) - _EPOCH_NAIVE
+            ).total_seconds()
+            * 1000
+        )
+    if isinstance(value, Decimal):
+        return float(value)
+    if isinstance(value, (set, frozenset)):
+        return [json_native(v) for v in value]
+    if isinstance(value, list):
+        return [json_native(v) for v in value]
+    if isinstance(value, dict):
+        return {k: json_native(v) for k, v in value.items()}
+    return value

arcadedb_native/_native/columns.py

@@ -0,0 +1,167 @@
+"""Rows to columns for the native backend (the columnar path is Python over the rows; the JPype path decodes a Java buffer)."""
+
+from datetime import date, datetime, timezone
+from decimal import Decimal
+
+
+def _names(rows, columns):
+    if columns is not None:
+        return list(columns)
+    seen = {}
+    for row in rows:
+        for k in row:
+            seen.setdefault(k, None)
+    return list(seen)
+
+
+def _kind(values):
+    """'b', 'i', 'f', 'dt', 'dec', 'obj' for the non-null values of a column."""
+    kinds = set()
+    for v in values:
+        if v is None:
+            continue
+        t = type(v)
+        if t is bool:
+            kinds.add("b")
+        elif t is int:
+            kinds.add("i")
+        elif t is float:
+            kinds.add("f")
+        elif t is datetime or t is date:
+            kinds.add("dt")
+        elif t is Decimal:
+            kinds.add("dec")
+        else:
+            kinds.add("obj")
+    if not kinds:
+        return "obj"
+    if len(kinds) == 1:
+        return kinds.pop()
+    if kinds == {"i", "f"}:
+        return "f"
+    return "obj"
+
+
+def _ms(v):
+    if isinstance(v, datetime):
+        if v.tzinfo is not None:
+            v = v.astimezone(timezone.utc).replace(tzinfo=None)
+        return v
+    return datetime(v.year, v.month, v.day)
+
+
+def _float_vectors(values):
+    """A column of equal-length float lists (None allowed) that float32 holds exactly, as a (count, dim) float32 array with
+    NaN rows for the nulls; None for any other column. Rows cannot say float[] from List<Double>, so only a column whose every
+    value survives float32 unchanged is read as a vector column (the JPype path decides on the Java element type).
+    """
+    import numpy as np
+
+    dim = 0
+    for v in values:
+        if v is None:
+            continue
+        if type(v) is not list or not v or not all(type(x) is float for x in v):
+            return None
+        if dim == 0:
+            dim = len(v)
+        elif len(v) != dim:
+            return None
+    if dim == 0:
+        return None
+    arr = np.full((len(values), dim), np.nan, dtype=np.float32)
+    for i, v in enumerate(values):
+        if v is not None:
+            arr[i] = v
+            if not np.array_equal(arr[i].astype(np.float64), np.array(v)):
+                return None
+    return arr
+
+
+def to_columns(rows, columns=None):
+    import numpy as np
+
+    out = {}
+    for name in _names(rows, columns):
+        values = [row.get(name) for row in rows]
+        if _kind(values) == "obj":
+            vectors = _float_vectors(values)
+            if vectors is not None:
+                out[name] = vectors
+                continue
+        has_null = any(v is None for v in values)
+        kind = _kind(values)
+        if kind == "i":
+            out[name] = (
+                np.array([np.nan if v is None else v for v in values], dtype=np.float64)
+                if has_null
+                else np.array(values, dtype=np.int64)
+            )
+        elif kind == "f":
+            out[name] = np.array(
+                [np.nan if v is None else v for v in values], dtype=np.float64
+            )
+        elif kind == "b":
+            out[name] = values if has_null else np.array(values, dtype=bool)
+        elif kind == "dt":
+            out[name] = np.array(
+                [
+                    (
+                        np.datetime64("NaT", "ms")
+                        if v is None
+                        else np.datetime64(_ms(v), "ms")
+                    )
+                    for v in values
+                ],
+                dtype="datetime64[ms]",
+            )
+        elif kind == "dec":
+            arr = np.empty(len(values), dtype=object)
+            arr[:] = values
+            out[name] = arr
+        else:
+            out[name] = values
+    return out
+
+
+def to_arrow(rows, columns=None):
+    import pyarrow as pa
+
+    arrays = {}
+    for name in _names(rows, columns):
+        values = [row.get(name) for row in rows]
+        kind = _kind(values)
+        if kind == "i":
+            arrays[name] = pa.array(values, type=pa.int64())
+        elif kind == "f":
+            try:
+                arrays[name] = pa.array(values, type=pa.float64())
+            except (
+                pa.ArrowInvalid
+            ):  # an int beyond +-2**53 next to floats: the column degrades to exact text
+                arrays[name] = pa.array(
+                    [None if v is None else str(v) for v in values], type=pa.string()
+                )
+        elif kind == "b":
+            arrays[name] = pa.array(values, type=pa.bool_())
+        elif kind == "dt":
+            arrays[name] = pa.array(
+                [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms")
+            )
+        elif kind == "dec":
+            try:
+                arrays[name] = pa.array(values)
+            except (
+                pa.ArrowInvalid
+            ):  # wider than decimal256 (76 digits): an exact string
+                arrays[name] = pa.array(
+                    [None if v is None else str(v) for v in values], type=pa.string()
+                )
+        else:
+            try:
+                arrays[name] = pa.array([None if v is None else v for v in values])
+            except (pa.ArrowInvalid, pa.ArrowTypeError):
+                arrays[name] = pa.array(
+                    [None if v is None else str(v) for v in values], type=pa.string()
+                )
+    return pa.table(arrays)