sfs-v2 0.0.1.dev1__py3-none-win_amd64.whl

sfs/__init__.py

@@ -0,0 +1,95 @@
+"""SqlFileSystem Python SDK.
+
+Only names listed in ``__all__`` are part of the stable Python-first contract.
+Backend experiments live in :mod:`sfs.experimental`.
+"""
+
+from .async_ import AsyncSfs, AsyncSfsScan
+from .errors import (
+    ErrorCode,
+    SfsAbiMismatchError,
+    SfsClosedError,
+    SfsCorruptDataError,
+    SfsDatabaseError,
+    SfsDatabaseLockedError,
+    SfsError,
+    SfsCompatibilityError,
+    SfsDictionaryError,
+    SfsInternalError,
+    SfsInvalidArgumentError,
+    SfsIoError,
+    SfsNotFoundError,
+    SfsPreconditionError,
+    SfsQueueClosedError,
+    SfsQueueError,
+    SfsQueuePanicError,
+    SfsReadonlyError,
+    SfsResourceExhaustedError,
+    SfsUnsupportedError,
+)
+from .models import (
+    AsyncOptions,
+    CursorPage,
+    DirectoryInfo,
+    FileInfo,
+    OffsetPage,
+    OpenOptions,
+    ReadResult,
+    RuntimeInfo,
+    ScanError,
+    ScanErrorMode,
+    ScanFilter,
+    ScanOptions,
+    ScanOrder,
+    ScanStats,
+    ScannedFile,
+    ShrinkReport,
+    WriteItem,
+)
+from .sync import Sfs, SfsScan
+
+__version__ = "0.0.1.dev1"
+
+__all__ = [
+    "AsyncOptions",
+    "AsyncSfs",
+    "AsyncSfsScan",
+    "CursorPage",
+    "DirectoryInfo",
+    "ErrorCode",
+    "FileInfo",
+    "OffsetPage",
+    "OpenOptions",
+    "ReadResult",
+    "RuntimeInfo",
+    "ScanError",
+    "ScanErrorMode",
+    "ScanFilter",
+    "ScanOptions",
+    "ScanOrder",
+    "ScanStats",
+    "ScannedFile",
+    "Sfs",
+    "SfsAbiMismatchError",
+    "SfsClosedError",
+    "SfsCorruptDataError",
+    "SfsDatabaseError",
+    "SfsCompatibilityError",
+    "SfsDatabaseLockedError",
+    "SfsDictionaryError",
+    "SfsError",
+    "SfsInternalError",
+    "SfsInvalidArgumentError",
+    "SfsIoError",
+    "SfsNotFoundError",
+    "SfsPreconditionError",
+    "SfsQueueClosedError",
+    "SfsQueueError",
+    "SfsQueuePanicError",
+    "SfsReadonlyError",
+    "SfsResourceExhaustedError",
+    "SfsScan",
+    "SfsUnsupportedError",
+    "ShrinkReport",
+    "WriteItem",
+]

sfs/_native.py

@@ -0,0 +1,417 @@
+"""Private ctypes bridge. Nothing in this module is a public SDK contract."""
+
+import ctypes
+import os
+import platform
+import sys
+import threading
+from ctypes import (
+    POINTER,
+    Structure,
+    c_char_p,
+    c_int32,
+    c_int64,
+    c_size_t,
+    c_uint32,
+    c_uint64,
+    c_uint8,
+    c_void_p,
+)
+from typing import Optional
+
+from .errors import ErrorCode, SfsAbiMismatchError, SfsError, error_from_native
+
+ABI_VERSION = 2
+
+
+class _SfsOpenOptionsC(Structure):
+    _fields_ = [
+        ("abi_version", c_uint32),
+        ("struct_size", c_uint32),
+        ("readonly", c_int32),
+        ("max_vol_size_mb", c_uint64),
+        ("write_flush_ms", c_uint64),
+        ("write_batch_max", c_size_t),
+        ("enable_write_queue", c_int32),
+        ("compression_level", c_int32),
+        ("enable_dict_training", c_int32),
+    ]
+
+
+class _SfsPageC(Structure):
+    _fields_ = [
+        ("limit", c_int64),
+        ("offset", c_int64),
+        ("start", c_char_p),
+    ]
+
+
+class _SfsFileInfoC(Structure):
+    _fields_ = [
+        ("full_path", c_char_p),
+        ("parent_path", c_char_p),
+        ("file_name", c_char_p),
+        ("size", c_int64),
+        ("storage_position", c_char_p),
+        ("file_uuid", c_char_p),
+        ("created_at", c_char_p),
+        ("updated_at", c_char_p),
+        ("is_file", c_int32),
+    ]
+
+
+class _SfsDirInfoC(Structure):
+    _fields_ = [
+        ("file_count", c_uint64),
+        ("directory_count", c_uint64),
+    ]
+
+
+class _SfsShrinkReportC(Structure):
+    """Legacy 24-byte report used by the required ``sfs_shrink`` symbol."""
+
+    _fields_ = [
+        ("volumes_scanned", c_uint64),
+        ("rows_deleted", c_uint64),
+        ("bytes_reclaimed", c_uint64),
+    ]
+
+
+class _SfsShrinkReportV2C(Structure):
+    _fields_ = [
+        ("abi_version", c_uint32),
+        ("struct_size", c_uint32),
+        ("volumes_scanned", c_uint64),
+        ("rows_deleted", c_uint64),
+        ("bytes_reclaimed", c_uint64),
+        ("vacuum_failed_vols", POINTER(c_uint32)),
+        ("vacuum_failed_count", c_size_t),
+    ]
+
+
+class _SfsBufferC(Structure):
+    _fields_ = [("data", POINTER(c_uint8)), ("len", c_size_t)]
+
+
+class _SfsWriteItemC(Structure):
+    _fields_ = [
+        ("path", c_char_p),
+        ("data", POINTER(c_uint8)),
+        ("len", c_size_t),
+    ]
+
+
+class _SfsErrorC(Structure):
+    _fields_ = [
+        ("code", c_int32),
+        ("message", c_char_p),
+        ("db_code", c_char_p),
+    ]
+
+
+class _SfsScanFilterC(Structure):
+    _fields_ = [
+        ("path_prefix", c_char_p),
+        ("extensions", POINTER(c_char_p)),
+        ("n_extensions", c_size_t),
+        ("name_like", c_char_p),
+        ("name_prefix", c_char_p),
+        ("name_suffix", c_char_p),
+        ("has_size_range", c_int32),
+        ("size_min", c_int64),
+        ("size_max", c_int64),
+        ("updated_after", c_char_p),
+        ("updated_before", c_char_p),
+    ]
+
+
+class _SfsScanOptionsC(Structure):
+    _fields_ = [
+        ("filter", _SfsScanFilterC),
+        ("order", c_int32),
+        ("batch_size", c_size_t),
+        ("error_mode", c_int32),
+    ]
+
+
+class _SfsScanErrorC(Structure):
+    _fields_ = [
+        ("code", c_int32),
+        ("path", c_char_p),
+        ("message", c_char_p),
+    ]
+
+
+class _SfsAbiInfoC(Structure):
+    _fields_ = [
+        ("abi_version", c_uint32),
+        ("struct_size", c_uint32),
+        ("open_options_size", c_size_t),
+        ("page_size", c_size_t),
+        ("file_info_size", c_size_t),
+        ("dir_info_size", c_size_t),
+        ("shrink_report_size", c_size_t),
+        ("buffer_size", c_size_t),
+        ("write_item_size", c_size_t),
+        ("error_size", c_size_t),
+        ("scan_filter_size", c_size_t),
+        ("scan_options_size", c_size_t),
+        ("scan_error_size", c_size_t),
+    ]
+
+
+def encode_text(value: str, field: str) -> bytes:
+    if not isinstance(value, str):
+        raise TypeError("%s must be str" % field)
+    if type(value) is not str:
+        value = str.__str__(value)
+    if "\x00" in value:
+        raise ValueError("%s cannot contain NUL" % field)
+    return value.encode("utf-8")
+
+
+def decode_text(value: Optional[bytes]) -> Optional[str]:
+    return value.decode("utf-8") if value else None
+
+
+def _platform_directory() -> str:
+    platform_name = {"win32": "win32", "linux": "linux", "darwin": "darwin"}.get(sys.platform)
+    machine = platform.machine().lower()
+    arch = {"x86_64": "x86_64", "amd64": "x86_64", "arm64": "aarch64", "aarch64": "aarch64"}.get(machine)
+    if not platform_name or not arch:
+        raise RuntimeError("Unsupported platform: %s-%s" % (sys.platform, machine))
+    return "%s-%s" % (platform_name, arch)
+
+
+def _library_name() -> str:
+    if sys.platform == "win32":
+        return "sfs_ffi.dll"
+    if sys.platform == "darwin":
+        return "libsfs_ffi.dylib"
+    return "libsfs_ffi.so"
+
+
+def _library_path() -> str:
+    configured = os.environ.get("SFS_NATIVE_PATH")
+    if configured:
+        return configured
+    package_native = os.path.join(os.path.dirname(__file__), "native", _platform_directory(), _library_name())
+    if os.path.exists(package_native):
+        return package_native
+    legacy_native = os.path.join(os.path.dirname(os.path.dirname(__file__)), "native", _platform_directory(), _library_name())
+    return legacy_native
+
+
+class _NativeApi:
+    def __init__(self) -> None:
+        try:
+            self.lib = ctypes.CDLL(_library_path())
+        except OSError as exc:
+            raise RuntimeError("Unable to load sfs native library: %s" % exc) from exc
+        self._bind()
+        self.abi_info = self._verify_abi()
+
+    def _bind(self) -> None:
+        lib = self.lib
+        required_symbols = (
+            "sfs_abi_version", "sfs_get_abi_info", "sfs_open",
+            "sfs_open_postgres", "sfs_open_mysql", "sfs_open_remote",
+            "sfs_close_checked", "sfs_stat", "sfs_exists_file",
+            "sfs_exists_dir", "sfs_list", "sfs_dir_info", "sfs_fs_info",
+            "sfs_read", "sfs_read_batch", "sfs_write", "sfs_write_batch",
+            "sfs_scan_open", "sfs_scan_next", "sfs_scan_take_errors",
+            "sfs_scan_counts", "sfs_scan_close", "sfs_mkdir", "sfs_delete",
+            "sfs_rename", "sfs_shrink", "sfs_free_buffer",
+            "sfs_free_buffers", "sfs_free_file_infos",
+            "sfs_free_scan_errors", "sfs_error_free",
+        )
+        missing = [name for name in required_symbols if not hasattr(lib, name)]
+        if missing:
+            raise SfsAbiMismatchError(
+                "native library is missing ABI v2 symbols: " + ", ".join(missing)
+            )
+        try:
+            lib.sfs_abi_version.restype = c_uint32
+            lib.sfs_abi_version.argtypes = []
+            lib.sfs_get_abi_info.restype = c_int32
+            lib.sfs_get_abi_info.argtypes = [POINTER(_SfsAbiInfoC), c_size_t]
+        except AttributeError as exc:
+            raise SfsAbiMismatchError("native library does not expose the SFS ABI v2 handshake") from exc
+
+        lib.sfs_open.restype = c_void_p
+        lib.sfs_open.argtypes = [c_char_p, POINTER(_SfsOpenOptionsC), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_open_postgres.restype = c_void_p
+        lib.sfs_open_postgres.argtypes = [c_char_p, POINTER(_SfsOpenOptionsC), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_open_mysql.restype = c_void_p
+        lib.sfs_open_mysql.argtypes = [c_char_p, POINTER(_SfsOpenOptionsC), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_open_remote.restype = c_void_p
+        lib.sfs_open_remote.argtypes = [c_char_p, POINTER(POINTER(_SfsErrorC))]
+
+        # These capabilities were added after the initial ABI v2 release. Keep
+        # them optional at load time so the stable local SQLite surface can
+        # still diagnose/use an early ABI v2 library. Call sites must handle
+        # absence explicitly: token auth reports UNSUPPORTED, while shrink v2
+        # deliberately falls back to the documented legacy 24-byte report.
+        open_remote_with_token = getattr(lib, "sfs_open_remote_with_token", None)
+        if open_remote_with_token is not None:
+            open_remote_with_token.restype = c_void_p
+            open_remote_with_token.argtypes = [
+                c_char_p,
+                c_char_p,
+                POINTER(POINTER(_SfsErrorC)),
+            ]
+        stream_flush = getattr(lib, "sfs_stream_flush", None)
+        if stream_flush is not None:
+            stream_flush.restype = c_int32
+            stream_flush.argtypes = [c_void_p, POINTER(POINTER(_SfsErrorC))]
+        shrink_report_v2_size = getattr(lib, "sfs_shrink_report_v2_size", None)
+        shrink_v2 = getattr(lib, "sfs_shrink_v2", None)
+        free_u32_array = getattr(lib, "sfs_free_u32_array", None)
+        shrink_capabilities = (shrink_report_v2_size, shrink_v2, free_u32_array)
+        if not all(capability is not None for capability in shrink_capabilities):
+            # Optional extensions are atomic capabilities, not base-ABI requirements. A mixed
+            # deployment may briefly expose only part of the trio; disable v2 and retain the
+            # required legacy sfs_shrink path rather than making the entire SDK unloadable.
+            shrink_report_v2_size = None
+            shrink_v2 = None
+            free_u32_array = None
+        if shrink_v2 is not None:
+            shrink_report_v2_size.restype = c_size_t
+            shrink_report_v2_size.argtypes = []
+            shrink_v2.restype = c_int32
+            shrink_v2.argtypes = [
+                c_void_p,
+                POINTER(_SfsShrinkReportV2C),
+                c_size_t,
+                POINTER(POINTER(_SfsErrorC)),
+            ]
+            free_u32_array.restype = None
+            free_u32_array.argtypes = [POINTER(c_uint32), c_size_t]
+
+        lib.sfs_close_checked.restype = c_int32
+        lib.sfs_close_checked.argtypes = [c_void_p, POINTER(POINTER(_SfsErrorC))]
+
+        lib.sfs_stat.restype = c_int32
+        lib.sfs_stat.argtypes = [c_void_p, c_char_p, POINTER(POINTER(_SfsFileInfoC)), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_exists_file.restype = c_int32
+        lib.sfs_exists_file.argtypes = [c_void_p, c_char_p, POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_exists_dir.restype = c_int32
+        lib.sfs_exists_dir.argtypes = [c_void_p, c_char_p, POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_list.restype = c_int32
+        lib.sfs_list.argtypes = [c_void_p, c_char_p, POINTER(_SfsPageC), c_char_p, c_int32, POINTER(POINTER(_SfsFileInfoC)), POINTER(c_size_t), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_dir_info.restype = c_int32
+        lib.sfs_dir_info.argtypes = [c_void_p, c_char_p, POINTER(_SfsDirInfoC), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_fs_info.restype = c_int32
+        lib.sfs_fs_info.argtypes = [c_void_p, POINTER(_SfsDirInfoC), POINTER(POINTER(_SfsErrorC))]
+
+        lib.sfs_read.restype = c_int32
+        lib.sfs_read.argtypes = [c_void_p, c_char_p, POINTER(POINTER(c_uint8)), POINTER(c_size_t), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_read_batch.restype = c_int32
+        lib.sfs_read_batch.argtypes = [c_void_p, POINTER(c_char_p), c_size_t, POINTER(POINTER(_SfsBufferC)), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_write.restype = c_int32
+        lib.sfs_write.argtypes = [c_void_p, c_char_p, POINTER(c_uint8), c_size_t, POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_write_batch.restype = c_int32
+        lib.sfs_write_batch.argtypes = [c_void_p, POINTER(_SfsWriteItemC), c_size_t, POINTER(POINTER(_SfsErrorC))]
+
+        lib.sfs_scan_open.restype = c_void_p
+        lib.sfs_scan_open.argtypes = [c_void_p, POINTER(_SfsScanOptionsC), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_scan_next.restype = c_int32
+        lib.sfs_scan_next.argtypes = [c_void_p, POINTER(POINTER(_SfsFileInfoC)), POINTER(POINTER(c_uint8)), POINTER(c_size_t), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_scan_take_errors.restype = c_int32
+        lib.sfs_scan_take_errors.argtypes = [c_void_p, POINTER(POINTER(_SfsScanErrorC)), POINTER(c_size_t), POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_scan_counts.restype = None
+        lib.sfs_scan_counts.argtypes = [c_void_p, POINTER(c_uint64), POINTER(c_uint64), POINTER(c_uint64)]
+        lib.sfs_scan_close.restype = None
+        lib.sfs_scan_close.argtypes = [c_void_p]
+
+        lib.sfs_mkdir.restype = c_int32
+        lib.sfs_mkdir.argtypes = [c_void_p, c_char_p, POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_delete.restype = c_int32
+        lib.sfs_delete.argtypes = [c_void_p, c_char_p, c_int32, POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_rename.restype = c_int32
+        lib.sfs_rename.argtypes = [c_void_p, c_char_p, c_char_p, POINTER(POINTER(_SfsErrorC))]
+        lib.sfs_shrink.restype = c_int32
+        lib.sfs_shrink.argtypes = [c_void_p, POINTER(_SfsShrinkReportC), POINTER(POINTER(_SfsErrorC))]
+
+        lib.sfs_free_buffer.restype = None
+        lib.sfs_free_buffer.argtypes = [POINTER(c_uint8), c_size_t]
+        lib.sfs_free_buffers.restype = None
+        lib.sfs_free_buffers.argtypes = [POINTER(_SfsBufferC), c_size_t]
+        lib.sfs_free_file_infos.restype = None
+        lib.sfs_free_file_infos.argtypes = [POINTER(_SfsFileInfoC), c_size_t]
+        lib.sfs_free_scan_errors.restype = None
+        lib.sfs_free_scan_errors.argtypes = [POINTER(_SfsScanErrorC), c_size_t]
+        lib.sfs_error_free.restype = None
+        lib.sfs_error_free.argtypes = [POINTER(_SfsErrorC)]
+
+    def _verify_abi(self) -> _SfsAbiInfoC:
+        version = int(self.lib.sfs_abi_version())
+        if version != ABI_VERSION:
+            raise SfsAbiMismatchError("native ABI version %d does not match Python SDK ABI %d" % (version, ABI_VERSION))
+        info = _SfsAbiInfoC()
+        code = int(self.lib.sfs_get_abi_info(ctypes.byref(info), ctypes.sizeof(info)))
+        if code != 0:
+            raise SfsAbiMismatchError("native library rejected ABI info request with code %d" % code)
+        if int(info.abi_version) != ABI_VERSION:
+            raise SfsAbiMismatchError(
+                "ABI info version %d does not match Python SDK ABI %d"
+                % (int(info.abi_version), ABI_VERSION)
+            )
+        if int(info.struct_size) < ctypes.sizeof(_SfsAbiInfoC):
+            raise SfsAbiMismatchError(
+                "ABI info struct is too small: native=%d python=%d"
+                % (int(info.struct_size), ctypes.sizeof(_SfsAbiInfoC))
+            )
+        expected = {
+            "open_options_size": ctypes.sizeof(_SfsOpenOptionsC),
+            "page_size": ctypes.sizeof(_SfsPageC),
+            "file_info_size": ctypes.sizeof(_SfsFileInfoC),
+            "dir_info_size": ctypes.sizeof(_SfsDirInfoC),
+            "shrink_report_size": ctypes.sizeof(_SfsShrinkReportC),
+            "buffer_size": ctypes.sizeof(_SfsBufferC),
+            "write_item_size": ctypes.sizeof(_SfsWriteItemC),
+            "error_size": ctypes.sizeof(_SfsErrorC),
+            "scan_filter_size": ctypes.sizeof(_SfsScanFilterC),
+            "scan_options_size": ctypes.sizeof(_SfsScanOptionsC),
+            "scan_error_size": ctypes.sizeof(_SfsScanErrorC),
+        }
+        mismatches = ["%s python=%d native=%d" % (name, size, int(getattr(info, name))) for name, size in expected.items() if int(getattr(info, name)) != size]
+        if mismatches:
+            raise SfsAbiMismatchError("native ABI layout mismatch: " + "; ".join(mismatches))
+        return info
+
+    def check_error(
+        self,
+        code: int,
+        err_ref: POINTER(_SfsErrorC),
+        *,
+        operation: Optional[str] = None,
+        path: Optional[str] = None,
+    ) -> None:
+        if code == 0:
+            return
+        raw_code = code
+        message = "Unknown native error (code=%d)" % code
+        database_code = None
+        if bool(err_ref):
+            try:
+                error = err_ref.contents
+                raw_code = int(error.code)
+                message = error.message.decode("utf-8") if error.message else message
+                database_code = error.db_code.decode("utf-8") if error.db_code else None
+            finally:
+                self.lib.sfs_error_free(err_ref)
+        raise error_from_native(raw_code, message, operation=operation, path=path, database_code=database_code)
+
+
+_native_lock = threading.Lock()
+_native_instance: Optional[_NativeApi] = None
+
+
+def get_native() -> _NativeApi:
+    global _native_instance
+    if _native_instance is None:
+        with _native_lock:
+            if _native_instance is None:
+                _native_instance = _NativeApi()
+    return _native_instance