notebookutils-py 0.1.0__py3-none-any.whl

notebookutils/__init__.py

@@ -0,0 +1,19 @@
+"""notebookutils: platform-agnostic notebook utilities.
+
+Provides :mod:`notebookutils.fs`, a Fabric/Synapse-compatible file system
+utility module backed by ``fsspec`` (memory-space operations) and FUSE
+binaries (POSIX mounting), usable on any local, VM, or Spark/K8s environment.
+"""
+
+import sys as _sys
+
+from notebookutils import fs
+
+__all__ = ["fs"]
+__version__ = "0.1.0"
+
+# Best-effort compatibility shim: after `import notebookutils`, legacy code
+# using `import mssparkutils` / `mssparkutils.fs` also works within the same
+# interpreter session.
+_sys.modules.setdefault("mssparkutils", _sys.modules[__name__])
+_sys.modules.setdefault("mssparkutils.fs", fs)

notebookutils/fs/__init__.py

@@ -0,0 +1,359 @@
+"""notebookutils.fs — Fabric/Synapse-compatible file system utilities.
+
+Memory-space operations are backed by ``fsspec`` (s3fs, adlfs, gcsfs, local)
+and POSIX mounting is backed by FUSE binaries (mount-s3, blobfuse2, gcsfuse).
+
+Run ``notebookutils.fs.help()`` for an overview of the available methods.
+"""
+
+from __future__ import annotations
+
+import logging
+import posixpath
+import shutil
+import subprocess
+from typing import Any, Dict, List, Optional
+
+from notebookutils.fs._client import (
+    get_canonical_protocol,
+    get_fs_and_path,
+    get_fs_client,
+    strip_protocol_for_local,
+)
+from notebookutils.fs._fileinfo import FileInfo, MountPointInfo
+from notebookutils.fs._mount import (
+    getMountPath,
+    mount,
+    mounts,
+    unmount,
+)
+from notebookutils.fs._registry import configure
+
+# Suppress underlying vendor telemetry noise
+logging.getLogger("azure.identity").setLevel(logging.ERROR)
+logging.getLogger("google.auth").setLevel(logging.ERROR)
+
+__all__ = [
+    "ls",
+    "mkdirs",
+    "cp",
+    "fastcp",
+    "mv",
+    "put",
+    "head",
+    "append",
+    "rm",
+    "exists",
+    "getProperties",
+    "help",
+    "mount",
+    "unmount",
+    "mounts",
+    "getMountPath",
+    "configure",
+    "FileInfo",
+    "MountPointInfo",
+]
+
+_COPY_CHUNK_SIZE = 8 * 1024 * 1024  # 8 MiB streaming buffer
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _to_fileinfo(entry: Dict[str, Any], protocol: str) -> FileInfo:
+    raw_path = entry.get("name", "")
+    is_dir = entry.get("type") == "directory"
+    size = int(entry.get("size") or 0)
+
+    name = posixpath.basename(raw_path.rstrip("/"))
+    if protocol != "file" and "://" not in raw_path:
+        display_path = f"{protocol}://{raw_path}"
+    else:
+        display_path = raw_path
+
+    mtime = entry.get("mtime") or entry.get("last_modified") or entry.get("LastModified")
+    modify_time: Optional[int] = None
+    if mtime is not None:
+        try:
+            modify_time = int(mtime.timestamp() * 1000)  # datetime-like
+        except AttributeError:
+            try:
+                modify_time = int(float(mtime) * 1000)
+            except (TypeError, ValueError):
+                modify_time = None
+
+    return FileInfo(
+        path=display_path,
+        name=name,
+        size=size,
+        isDir=is_dir,
+        modifyTime=modify_time,
+    )
+
+
+def _is_dir(fs: Any, path: str) -> bool:
+    try:
+        return fs.isdir(path)
+    except Exception:
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Memory-space public API (Fabric-compatible signatures)
+# ---------------------------------------------------------------------------
+
+def ls(path: str) -> List[FileInfo]:
+    """List the contents of a directory as :class:`FileInfo` objects."""
+    fs, native = get_fs_and_path(path)
+    protocol = get_canonical_protocol(path)
+    entries = fs.ls(native, detail=True)
+    return [_to_fileinfo(e, protocol) for e in entries]
+
+
+def mkdirs(path: str) -> bool:
+    """Create the given directory and any necessary parent directories."""
+    fs, native = get_fs_and_path(path)
+    fs.makedirs(native, exist_ok=True)
+    return True
+
+
+def exists(path: str) -> bool:
+    """Check whether a file or directory exists at the given path."""
+    fs, native = get_fs_and_path(path)
+    return bool(fs.exists(native))
+
+
+def rm(path: str, recurse: bool = False) -> bool:
+    """Remove a file or directory. Set ``recurse=True`` for directories."""
+    fs, native = get_fs_and_path(path)
+    fs.rm(native, recursive=recurse)
+    return True
+
+
+def put(file: str, content: str, overwrite: bool = False) -> bool:
+    """Write the given string out to a file, encoded in UTF-8."""
+    fs, native = get_fs_and_path(file)
+    if not overwrite and fs.exists(native):
+        raise FileExistsError(
+            f"File already exists: {file}. Pass overwrite=True to replace it."
+        )
+    parent = posixpath.dirname(native.rstrip("/"))
+    if parent:
+        try:
+            fs.makedirs(parent, exist_ok=True)
+        except Exception:
+            pass
+    with fs.open(native, "wb") as f:
+        f.write(content.encode("utf-8"))
+    return True
+
+
+def append(file: str, content: str, createFileIfNotExists: bool = False) -> bool:
+    """Append the content to a file, encoded in UTF-8.
+
+    .. note:: Not safe for concurrent writes to the same file. Object-store
+       backends lack a native append primitive, so this performs
+       read-modify-write there; local files use a true O_APPEND.
+    """
+    fs, native = get_fs_and_path(file)
+    file_exists = fs.exists(native)
+    if not file_exists and not createFileIfNotExists:
+        raise FileNotFoundError(
+            f"File not found: {file}. Pass createFileIfNotExists=True to create it."
+        )
+    protocol = get_canonical_protocol(file)
+    data = content.encode("utf-8")
+    if protocol == "file":
+        with fs.open(native, "ab") as f:
+            f.write(data)
+    else:
+        existing = fs.cat_file(native) if file_exists else b""
+        with fs.open(native, "wb") as f:
+            f.write(existing + data)
+    return True
+
+
+def head(file: str, max_bytes: int = 1024 * 100) -> str:
+    """Return up to the first ``max_bytes`` bytes of the file as a UTF-8 string."""
+    fs, native = get_fs_and_path(file)
+    with fs.open(native, "rb") as f:
+        data = f.read(max_bytes)
+    return data.decode("utf-8", errors="replace")
+
+
+def getProperties(path: str) -> Dict[str, Any]:
+    """Get the metadata properties of the given path as a dictionary."""
+    fs, native = get_fs_and_path(path)
+    info = dict(fs.info(native))
+    # Normalize non-JSON-friendly values to strings for portability
+    return {k: (v if isinstance(v, (str, int, float, bool, list, dict, type(None))) else str(v)) for k, v in info.items()}
+
+
+def cp(src: str, dest: str, recurse: bool = False) -> bool:
+    """Copy a file or directory, possibly across file systems / clouds.
+
+    Same-protocol copies use the backend's native copy. Cross-protocol copies
+    stream through memory in chunks — no intermediate files are written to
+    local disk.
+    """
+    src_proto = get_canonical_protocol(src)
+    dest_proto = get_canonical_protocol(dest)
+
+    if src_proto == dest_proto:
+        fs, native_src = get_fs_and_path(src)
+        _, native_dest = get_fs_and_path(dest)
+        if _is_dir(fs, native_src) and not recurse:
+            raise IsADirectoryError(
+                f"{src} is a directory. Pass recurse=True to copy directories."
+            )
+        fs.cp(native_src, native_dest, recursive=recurse)
+        return True
+
+    # Cross-provider streaming bridge
+    src_fs, native_src = get_fs_and_path(src)
+    dest_fs, native_dest = get_fs_and_path(dest)
+
+    if _is_dir(src_fs, native_src):
+        if not recurse:
+            raise IsADirectoryError(
+                f"{src} is a directory. Pass recurse=True to copy directories."
+            )
+        base = native_src.rstrip("/")
+        for src_file in src_fs.find(base):
+            rel = src_file[len(base) :].lstrip("/")
+            target = posixpath.join(native_dest.rstrip("/"), rel)
+            _stream_copy_file(src_fs, src_file, dest_fs, target)
+    else:
+        _stream_copy_file(src_fs, native_src, dest_fs, native_dest)
+    return True
+
+
+def _stream_copy_file(src_fs: Any, src_path: str, dest_fs: Any, dest_path: str) -> None:
+    parent = posixpath.dirname(dest_path.rstrip("/"))
+    if parent:
+        try:
+            dest_fs.makedirs(parent, exist_ok=True)
+        except Exception:
+            pass
+    with src_fs.open(src_path, "rb") as rf, dest_fs.open(dest_path, "wb") as wf:
+        shutil.copyfileobj(rf, wf, _COPY_CHUNK_SIZE)
+
+
+def fastcp(
+    src: str,
+    dest: str,
+    recurse: bool = True,
+    extraConfigs: Optional[Dict[str, Any]] = None,
+) -> bool:
+    """Copy via ``azcopy`` for better performance with large data volumes.
+
+    Falls back to :func:`cp` transparently when the ``azcopy`` binary is not
+    available or when neither endpoint is Azure/local.
+    """
+    azcopy = shutil.which("azcopy")
+    src_proto = get_canonical_protocol(src)
+    dest_proto = get_canonical_protocol(dest)
+    azcopy_protos = {"file", "abfs", "az", "https"}
+
+    if (
+        azcopy
+        and src_proto in azcopy_protos
+        and dest_proto in azcopy_protos
+        and "abfs" in (src_proto, dest_proto)
+    ):
+        cmd = [azcopy, "copy", _azcopy_url(src), _azcopy_url(dest)]
+        if recurse:
+            cmd.append("--recursive")
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        if result.returncode == 0:
+            return True
+        logging.getLogger(__name__).warning(
+            "azcopy failed (%s); falling back to cp()", result.stderr.strip()[:500]
+        )
+    return cp(src, dest, recurse=recurse)
+
+
+def _azcopy_url(path: str) -> str:
+    proto = get_canonical_protocol(path)
+    if proto == "file":
+        return strip_protocol_for_local(path)
+    if proto == "abfs":
+        clean = path.split("://", 1)[1]
+        if "@" in clean:
+            container, _, host_and_path = clean.partition("@")
+            host, _, sub = host_and_path.partition("/")
+            account = host.split(".", 1)[0]
+            url = f"https://{account}.blob.core.windows.net/{container}"
+            return f"{url}/{sub}" if sub else url
+    return path
+
+
+def mv(
+    src: str,
+    dest: str,
+    create_path: bool = False,
+    overwrite: bool = False,
+) -> bool:
+    """Move a file or directory, possibly across file systems."""
+    src_proto = get_canonical_protocol(src)
+    dest_proto = get_canonical_protocol(dest)
+
+    dest_fs, native_dest = get_fs_and_path(dest)
+    if dest_fs.exists(native_dest) and not overwrite:
+        raise FileExistsError(
+            f"Destination already exists: {dest}. Pass overwrite=True to replace it."
+        )
+    parent = posixpath.dirname(native_dest.rstrip("/"))
+    if parent and not dest_fs.exists(parent):
+        if create_path:
+            dest_fs.makedirs(parent, exist_ok=True)
+        elif dest_proto == "file":
+            # Object stores have virtual directories; only real local/POSIX
+            # filesystems require the parent to pre-exist.
+            raise FileNotFoundError(
+                f"Parent directory does not exist: {parent}. "
+                "Pass create_path=True to create it."
+            )
+
+    src_fs, native_src = get_fs_and_path(src)
+    recurse = _is_dir(src_fs, native_src)
+
+    if dest_fs.exists(native_dest) and overwrite:
+        dest_fs.rm(native_dest, recursive=True)
+
+    if src_proto == dest_proto:
+        src_fs.mv(native_src, native_dest, recursive=recurse)
+    else:
+        cp(src, dest, recurse=recurse)
+        src_fs.rm(native_src, recursive=recurse)
+    return True
+
+
+def help(method: Optional[str] = None) -> None:
+    """Print an overview of the available notebookutils.fs methods."""
+    table = {
+        "ls": "ls(path: str) -> List[FileInfo]: Lists the contents of a directory.",
+        "mkdirs": "mkdirs(path: str) -> bool: Creates the directory and necessary parents.",
+        "cp": "cp(src: str, dest: str, recurse: bool = False) -> bool: Copies a file or directory, possibly across file systems.",
+        "fastcp": "fastcp(src: str, dest: str, recurse: bool = True, extraConfigs: dict = None) -> bool: Copies via azcopy for better performance; falls back to cp().",
+        "mv": "mv(src: str, dest: str, create_path: bool = False, overwrite: bool = False) -> bool: Moves a file or directory, possibly across file systems.",
+        "put": "put(file: str, content: str, overwrite: bool = False) -> bool: Writes the given string out to a file, encoded in UTF-8.",
+        "head": "head(file: str, max_bytes: int = 102400) -> str: Returns up to the first max_bytes of the file as a UTF-8 string.",
+        "append": "append(file: str, content: str, createFileIfNotExists: bool = False) -> bool: Appends the content to a file.",
+        "rm": "rm(path: str, recurse: bool = False) -> bool: Removes a file or directory.",
+        "exists": "exists(path: str) -> bool: Checks if a file or directory exists.",
+        "getProperties": "getProperties(path: str) -> dict: Gets the metadata properties of the given path.",
+        "mount": "mount(source: str, mount_point: str, extra_configs: dict = None) -> bool: Mounts remote storage at a local POSIX path.",
+        "unmount": "unmount(mount_point: str, extra_configs: dict = None) -> bool: Unmounts and removes a mount point.",
+        "mounts": "mounts() -> List[MountPointInfo]: Lists all existing mount points.",
+        "getMountPath": "getMountPath(mount_point: str, scope: str = '') -> str: Gets the local path for a mount point.",
+        "configure": "configure(protocol: str, **kwargs) -> None: Overrides credential/config discovery for a protocol.",
+    }
+    if method:
+        print(table.get(method, f"Unknown method: {method}"))
+        return
+    print("notebookutils.fs provides file system utilities backed by fsspec and FUSE.\n")
+    for line in table.values():
+        print(f"  {line}")

notebookutils/fs/_client.py

@@ -0,0 +1,69 @@
+"""Protocol parsing and fsspec client resolution."""
+
+from __future__ import annotations
+
+from typing import Any, Tuple
+
+from notebookutils.fs import _registry
+
+# Maps canonical protocol -> (pip extra, backend package) for error messages.
+_BACKEND_HINTS = {
+    "s3": ("s3", "s3fs"),
+    "abfs": ("azure", "adlfs"),
+    "az": ("azure", "adlfs"),
+    "gcs": ("gcs", "gcsfs"),
+}
+
+
+def get_protocol(path: str) -> str:
+    """Extract the raw protocol scheme from a path (without normalizing)."""
+    if "://" in path:
+        return path.split("://", 1)[0]
+    if path.startswith("file:"):
+        return "file"
+    return "file"
+
+
+def get_canonical_protocol(path: str) -> str:
+    return _registry.normalize_protocol(get_protocol(path))
+
+
+def strip_protocol_for_local(path: str) -> str:
+    """For ``file:`` URLs return the plain OS path; otherwise return as-is."""
+    if path.startswith("file://"):
+        return path[len("file://") :] or "/"
+    if path.startswith("file:"):
+        return path[len("file:") :]
+    return path
+
+
+def get_fs_client(path: str) -> Any:
+    """Resolve and instantiate the proper fsspec storage client wrapper."""
+    import fsspec
+
+    protocol = get_canonical_protocol(path)
+    cached = _registry.get_cached_instance(protocol)
+    if cached is not None:
+        return cached
+
+    config = _registry.get_fsspec_options(protocol)
+    try:
+        instance = fsspec.filesystem(protocol, **config)
+    except ImportError as exc:
+        extra, pkg = _BACKEND_HINTS.get(protocol, (None, None))
+        if extra:
+            raise ImportError(
+                f"The '{protocol}' protocol requires the '{pkg}' package. "
+                f"Install it with: pip install notebookutils[{extra}]"
+            ) from exc
+        raise
+    _registry.set_cached_instance(protocol, instance)
+    return instance
+
+
+def get_fs_and_path(path: str) -> Tuple[Any, str]:
+    """Return ``(fsspec_filesystem, backend_native_path)`` for a user path."""
+    fs = get_fs_client(path)
+    if get_canonical_protocol(path) == "file":
+        return fs, strip_protocol_for_local(path)
+    return fs, path

notebookutils/fs/_fileinfo.py

@@ -0,0 +1,47 @@
+"""FileInfo and MountPointInfo data structures (Fabric-compatible)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+
+@dataclass(frozen=True)
+class FileInfo:
+    """Describes a file system entry, mirroring Fabric's notebookutils.
+
+    Attributes match the Fabric usage pattern::
+
+        for f in notebookutils.fs.ls(path):
+            print(f.name, f.isDir, f.isFile, f.path, f.size)
+    """
+
+    path: str
+    name: str
+    size: int
+    isDir: bool
+    modifyTime: Optional[int] = None
+
+    @property
+    def isFile(self) -> bool:
+        return not self.isDir
+
+    def __repr__(self) -> str:  # compact, notebook-friendly
+        kind = "dir" if self.isDir else "file"
+        return f"FileInfo(path={self.path!r}, name={self.name!r}, size={self.size}, {kind})"
+
+
+@dataclass(frozen=True)
+class MountPointInfo:
+    """Describes an active mount point, mirroring Fabric's ``mounts()`` output."""
+
+    mountPoint: str
+    source: str
+    localPath: str
+    extraConfigs: Dict[str, Any] = field(default_factory=dict)
+
+    def __repr__(self) -> str:
+        return (
+            f"MountPointInfo(mountPoint={self.mountPoint!r}, "
+            f"source={self.source!r}, localPath={self.localPath!r})"
+        )