fsspec-data 0.2.3__cp310-abi3-win_amd64.whl

fsspec_data/__init__.py

@@ -0,0 +1,36 @@
+from .filesystem import DataFileSystem
+from .interchange import (
+    DEFAULT_REGISTRY,
+    Codec,
+    CodecCapabilities,
+    CodecRegistry,
+    DataFormat,
+    DecodedBatches,
+    DecodedBatchStream,
+    FieldMapping,
+    InterchangePlan,
+    InterchangeRequest,
+    PlannedBatchStream,
+    SchemaPolicy,
+    plan_schema,
+)
+
+__version__ = "0.2.3"
+
+__all__ = [
+    "Codec",
+    "CodecCapabilities",
+    "CodecRegistry",
+    "DEFAULT_REGISTRY",
+    "DataFormat",
+    "DataFileSystem",
+    "DecodedBatchStream",
+    "DecodedBatches",
+    "FieldMapping",
+    "InterchangePlan",
+    "InterchangeRequest",
+    "PlannedBatchStream",
+    "SchemaPolicy",
+    "__version__",
+    "plan_schema",
+]

fsspec_data/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

fsspec_data/backends/__init__.py

@@ -0,0 +1,2 @@
+from .http import HTTPFilesystem
+from .sqlalchemy import SQLAlchemyFilesystem

fsspec_data/backends/base.py

@@ -0,0 +1,222 @@
+from fsspec import AbstractFileSystem
+from fsspec.spec import AbstractBufferedFile
+
+
+class BaseFileSystem(AbstractFileSystem):
+    """
+    Docs: https://filesystem-spec.readthedocs.io/en/latest/developer.html#implementing-a-backend
+    Minimal: https://github.com/fsspec/filesystem_spec/blob/45dcfa99e2dc320bf072c28bef767b71f8299a4a/fsspec/implementations/github.py#L10
+    """
+
+    # def start_transaction(self):
+    # def end_transaction(self):
+    # def invalidate_cache(self, path=None):
+
+    def mkdir(self, path, create_parents=True, **kwargs):
+        """Create directory entry at path
+
+        For systems that don't have true directories, may create an for
+        this instance only and not touch the real filesystem
+
+        Parameters
+        ----------
+        path: str
+            location
+        create_parents: bool
+            if True, this is equivalent to ``makedirs``
+        kwargs:
+            may be permissions, etc.
+        """
+        ...
+
+    def makedirs(self, path, exist_ok=False):
+        """Recursively make directories
+
+        Creates directory at path and any intervening required directories.
+        Raises exception if, for instance, the path already exists but is a
+        file.
+
+        Parameters
+        ----------
+        path: str
+            leaf directory name
+        exist_ok: bool (False)
+            If False, will error if the target already exists
+        """
+        ...
+
+    def rmdir(self, path):
+        """Remove a directory, if empty"""
+        ...
+
+    def ls(self, path, detail=True, **kwargs):
+        """List objects at path.
+
+        This should include subdirectories and files at that location. The
+        difference between a file and a directory must be clear when details
+        are requested.
+
+        The specific keys, or perhaps a FileInfo class, or similar, is TBD,
+        but must be consistent across implementations.
+        Must include:
+
+        - full path to the entry (without protocol)
+        - size of the entry, in bytes. If the value cannot be determined, will
+          be ``None``.
+        - type of entry, "file", "directory" or other
+
+        Additional information
+        may be present, aproriate to the file-system, e.g., generation,
+        checksum, etc.
+
+        May use refresh=True|False to allow use of self._ls_from_cache to
+        check for a saved listing and avoid calling the backend. This would be
+        common where listing may be expensive.
+
+        Parameters
+        ----------
+        path: str
+        detail: bool
+            if True, gives a list of dictionaries, where each is the same as
+            the result of ``info(path)``. If False, gives a list of paths
+            (str).
+        kwargs: may have additional backend-specific options, such as version
+            information
+
+        Returns
+        -------
+        List of strings if detail is False, or list of directory information
+        dicts if detail is True.
+        """
+        raise NotImplementedError
+
+    def cp_file(self, path1, path2, **kwargs):
+        raise NotImplementedError
+
+    def _rm(self, path):
+        """Delete one file"""
+        # this is the old name for the method, prefer rm_file
+        raise NotImplementedError
+
+    def created(self, path):
+        """Return the created timestamp of a file as a datetime.datetime"""
+        raise NotImplementedError
+
+    def modified(self, path):
+        """Return the modified timestamp of a file as a datetime.datetime"""
+        raise NotImplementedError
+
+    # def walk(self, path, maxdepth=None, **kwargs):
+    # def find(self, path, maxdepth=None, withdirs=False, **kwargs):
+    # def du(self, path, total=True, maxdepth=None, **kwargs):
+    # def glob(self, path, **kwargs):
+    # def exists(self, path, **kwargs):
+    # def lexists(self, path, **kwargs):
+    # def info(self, path, **kwargs):
+    # def checksum(self, path):
+    # def size(self, path):
+    # def sizes(self, paths):
+    # def isdir(self, path):
+    # def isfile(self, path):
+    # def cat_file(self, path, start=None, end=None, **kwargs):
+    # def pipe_file(self, path, value, **kwargs):
+    # def pipe(self, path, value=None, **kwargs):
+    # def cat_ranges(self, paths, starts, ends, max_gap=None, **kwargs):
+    # def cat(self, path, recursive=False, on_error="raise", **kwargs):
+    # def get_file(self, rpath, lpath, callback=_DEFAULT_CALLBACK, **kwargs):
+    # def get(self, rpath, lpath, recursive=False, callback=_DEFAULT_CALLBACK, **kwargs):
+    # def put_file(self, lpath, rpath, callback=_DEFAULT_CALLBACK, **kwargs):
+    # def put(self, lpath, rpath, recursive=False, callback=_DEFAULT_CALLBACK, **kwargs):
+    # def head(self, path, size=1024):
+    # def tail(self, path, size=1024):
+    # def copy(self, path1, path2, recursive=False, on_error=None, **kwargs):
+    # def expand_path(self, path, recursive=False, maxdepth=None):
+    # def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs):
+    # def rm_file(self, path):
+    # def rm(self, path, recursive=False, maxdepth=None):
+    # def _parent(cls, path):
+    # def _open(
+    # def open(
+    # def touch(self, path, truncate=True, **kwargs):
+    # def ukey(self, path):
+    # def read_block(self, fn, offset, length, delimiter=None):
+    # def to_json(self):
+    # def from_json(blob):
+    # def get_mapper(self, root="", check=False, create=False, missing_exceptions=None):
+    # ------------------------------------------------------------------------
+    # Aliases
+    # def makedir(self, path, create_parents=True, **kwargs):
+    # def mkdirs(self, path, exist_ok=False):
+    # def listdir(self, path, detail=True, **kwargs):
+    # def cp(self, path1, path2, **kwargs):
+    # def move(self, path1, path2, **kwargs):
+    # def stat(self, path, **kwargs):
+    # def disk_usage(self, path, total=True, maxdepth=None, **kwargs):
+    # def rename(self, path1, path2, **kwargs):
+    # def delete(self, path, recursive=False, maxdepth=None):
+    # def upload(self, lpath, rpath, recursive=False, **kwargs):
+    # def download(self, rpath, lpath, recursive=False, **kwargs):
+
+    def sign(self, path, expiration=100, **kwargs):
+        """Create a signed URL representing the given path
+
+        Some implementations allow temporary URLs to be generated, as a
+        way of delegating credentials.
+
+        Parameters
+        ----------
+        path : str
+             The path on the filesystem
+        expiration : int
+            Number of seconds to enable the URL for (if supported)
+
+        Returns
+        -------
+        URL : str
+            The signed URL
+
+        Raises
+        ------
+        NotImplementedError : if method is not implemented for a filesystem
+        """
+        raise NotImplementedError("Sign is not implemented for this filesystem")
+
+
+class BaseBufferedFile(AbstractBufferedFile):
+    # def details(self):
+    # def details(self, value):
+    # def full_name(self):
+    # def closed(self):
+    # def closed(self, c):
+
+    def commit(self):
+        """Move from temp to final destination"""
+
+    def discard(self):
+        """Throw away temporary file"""
+
+    # def info(self):
+    # def tell(self):
+    # def seek(self, loc, whence=0):
+    # def write(self, data):
+    # def flush(self, force=False):
+    # def _upload_chunk(self, final=False):
+
+    def _initiate_upload(self):
+        """Create remote file/upload"""
+        pass
+
+    def _fetch_range(self, start, end):
+        """Get the specified set of bytes from remote"""
+        raise NotImplementedError
+
+    # def read(self, length=-1):
+    # def readinto(self, b):
+    # def readuntil(self, char=b"\n", blocks=None):
+    # def readline(self):
+    # def readlines(self):
+    # def readinto1(self, b):
+    # def close(self):
+    # def readable(self):
+    # def seekable(self):
+    # def writable(self):

fsspec_data/backends/http.py

@@ -0,0 +1,4 @@
+from .base import BaseFileSystem
+
+
+class HTTPFilesystem(BaseFileSystem): ...

fsspec_data/backends/sqlalchemy.py

@@ -0,0 +1,4 @@
+from .base import BaseFileSystem
+
+
+class SQLAlchemyFilesystem(BaseFileSystem): ...

fsspec_data/filesystem.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import tempfile
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+import pyarrow as pa
+from fsspec import AbstractFileSystem, filesystem
+from fsspec.core import url_to_fs
+from fsspec.implementations.chained import ChainedFileSystem
+
+from .interchange import DEFAULT_REGISTRY, DataFormat, InterchangeRequest, SchemaPolicy
+
+
+class DataFileSystem(ChainedFileSystem):
+    """Read-only format and schema conversion layered over another filesystem."""
+
+    protocol = "fsspec-data"
+
+    def __init__(
+        self,
+        fo: str,
+        target_protocol: str | None = None,
+        target_options: dict[str, Any] | None = None,
+        fs: AbstractFileSystem | None = None,
+        provided_format: DataFormat | str | None = None,
+        requested_format: DataFormat | str | None = None,
+        provided_schema: pa.Schema | Mapping[str, Any] | Sequence[Mapping[str, Any]] | None = None,
+        requested_schema: pa.Schema | Mapping[str, Any] | Sequence[Mapping[str, Any]] | None = None,
+        schema_policy: SchemaPolicy | str = SchemaPolicy.EXACT,
+        batch_size: int = 1024,
+        row_limit: int | None = None,
+        byte_limit: int | None = None,
+        spool_max_size: int = 8 * 1024 * 1024,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+        if fs is not None and target_protocol is not None:
+            raise ValueError("provide either fs or target_protocol, not both")
+        if fs is None:
+            if target_protocol is None:
+                fs, fo = url_to_fs(fo, **(target_options or {}))
+            else:
+                fs = filesystem(target_protocol, **(target_options or {}))
+
+        self.fs = fs
+        self.fo = fs._strip_protocol(fo)
+        self.provided_format = DataFormat(provided_format) if provided_format is not None else None
+        self.requested_format = DataFormat(requested_format) if requested_format is not None else None
+        self.provided_schema = _schema_from_options(provided_schema)
+        self.requested_schema = _schema_from_options(requested_schema)
+        self.schema_policy = SchemaPolicy(schema_policy)
+        self.batch_size = batch_size
+        self.row_limit = row_limit
+        self.byte_limit = byte_limit
+        self.spool_max_size = spool_max_size
+        self._sizes: dict[str, int] = {}
+
+    def _open(
+        self,
+        path: str,
+        mode: str = "rb",
+        block_size: int | None = None,
+        autocommit: bool = True,
+        cache_options: dict[str, Any] | None = None,
+        **kwargs: Any,
+    ):
+        del block_size, autocommit, cache_options, kwargs
+        if mode != "rb":
+            raise ValueError("fsspec-data is a read-only filesystem")
+
+        path = self._strip_protocol(path)
+        file = tempfile.SpooledTemporaryFile(max_size=self.spool_max_size, mode="w+b")
+        try:
+            self._convert(path, file)
+            self._sizes[path] = file.tell()
+            file.seek(0)
+        except Exception:
+            file.close()
+            raise
+        return file
+
+    def info(self, path: str, **kwargs: Any) -> dict[str, Any]:
+        del kwargs
+        path = self._strip_protocol(path)
+        size = self._sizes.get(path)
+        if size is None:
+            with self._open(path) as file:
+                file.seek(0, 2)
+                size = file.tell()
+        return {"name": path, "size": size, "type": "file"}
+
+    def ls(self, path: str, detail: bool = True, **kwargs: Any):
+        path = self._strip_protocol(path)
+        if not path:
+            return []
+        entry = self.info(path, **kwargs)
+        return [entry] if detail else [entry["name"]]
+
+    def _convert(self, path: str, output) -> None:
+        provided_format = self.provided_format or _format_from_path(self.fo)
+        requested_format = self.requested_format or _format_from_path(path)
+        with self.fs.open(self.fo, "rb") as source:
+            decoded = DEFAULT_REGISTRY.get(provided_format).iter_batches(
+                source,
+                schema=self.provided_schema,
+                batch_size=self.batch_size,
+                row_limit=self.row_limit,
+                byte_limit=self.byte_limit,
+            )
+            if self.provided_schema is not None and not decoded.schema.equals(self.provided_schema, check_metadata=False):
+                raise ValueError("provided schema does not match the source schema")
+
+            provided_schema = self.provided_schema or decoded.schema
+            requested_schema = self.requested_schema or provided_schema
+            plan = InterchangeRequest(
+                provided_format,
+                requested_format,
+                provided_schema,
+                requested_schema,
+                self.schema_policy,
+            ).plan()
+            batches = (plan.apply_batch(batch) for batch in decoded)
+            DEFAULT_REGISTRY.get(requested_format).encode_batches_to(batches, output, schema=requested_schema)
+
+
+_SUFFIX_FORMATS = {
+    ".arrow": DataFormat.ARROW,
+    ".csv": DataFormat.CSV,
+    ".ipc": DataFormat.ARROW,
+    ".jsonl": DataFormat.JSONL,
+    ".ndjson": DataFormat.JSONL,
+    ".parquet": DataFormat.PARQUET,
+    ".pq": DataFormat.PARQUET,
+}
+
+
+def _format_from_path(path: str) -> DataFormat:
+    suffix = "." + path.rsplit(".", 1)[-1].lower() if "." in path.rsplit("/", 1)[-1] else ""
+    try:
+        return _SUFFIX_FORMATS[suffix]
+    except KeyError as error:
+        raise ValueError(f"cannot infer tabular format from path {path!r}") from error
+
+
+def _schema_from_options(
+    value: pa.Schema | Mapping[str, Any] | Sequence[Mapping[str, Any]] | None,
+) -> pa.Schema | None:
+    if value is None or isinstance(value, pa.Schema):
+        return value
+    fields = value.get("fields") if isinstance(value, Mapping) else value
+    if not isinstance(fields, Sequence) or isinstance(fields, (str, bytes)):
+        raise TypeError("schema options must contain a sequence of fields")
+    return pa.schema([_field_from_options(field) for field in fields])
+
+
+def _field_from_options(value: Mapping[str, Any]) -> pa.Field:
+    if not isinstance(value, Mapping):
+        raise TypeError("schema fields must be mappings")
+    try:
+        name = value["name"]
+        data_type = value["type"]
+    except KeyError as error:
+        raise ValueError(f"schema field is missing {error.args[0]!r}") from error
+    if not isinstance(name, str) or not isinstance(data_type, (str, pa.DataType)):
+        raise TypeError("schema field name and type must be strings or PyArrow data types")
+    if isinstance(data_type, str):
+        data_type = pa.type_for_alias(data_type)
+    nullable = value.get("nullable", True)
+    if not isinstance(nullable, bool):
+        raise TypeError("schema field nullable must be a boolean")
+    return pa.field(name, data_type, nullable=nullable)