fugue 0.8.7.dev6__py3-none-any.whl → 0.8.7.dev7__py3-none-any.whl

fugue/__init__.py

@@ -1,6 +1,5 @@
 # flake8: noqa
 from triad.collections import Schema
-from triad.collections.fs import FileSystem
 
 from fugue.api import out_transform, transform
 from fugue.bag.array_bag import ArrayBag

fugue/_utils/io.py

@@ -1,14 +1,14 @@
 import os
 import pathlib
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
-from urllib.parse import urlparse
 
-import fs as pfs
 import pandas as pd
+from fsspec import AbstractFileSystem
+from fsspec.implementations.local import LocalFileSystem
 from triad.collections.dict import ParamDict
-from triad.collections.fs import FileSystem
 from triad.collections.schema import Schema
 from triad.utils.assertion import assert_or_throw
+from triad.utils.io import join, url_to_fs
 from triad.utils.pandas_like import PD_UTILS
 
 from fugue.dataframe import LocalBoundedDataFrame, LocalDataFrame, PandasDataFrame
@@ -16,23 +16,14 @@ from fugue.dataframe import LocalBoundedDataFrame, LocalDataFrame, PandasDataFra
 
 class FileParser(object):
     def __init__(self, path: str, format_hint: Optional[str] = None):
-        last = len(path)
-        has_glob = False
         self._orig_format_hint = format_hint
-        for i in range(len(path)):
-            if path[i] in ["/", "\\"]:
-                last = i
-            if path[i] in ["*", "?"]:
-                has_glob = True
-                break
-        if not has_glob:
-            self._uri = urlparse(path)
-            self._glob_pattern = ""
-            self._path = self._uri.path
+        self._has_glob = "*" in path or "?" in path
+        self._raw_path = path
+        self._fs, self._fs_path = url_to_fs(path)
+        if not self.is_local:
+            self._path = self._fs.unstrip_protocol(self._fs_path)
         else:
-            self._uri = urlparse(path[:last])
-            self._glob_pattern = path[last + 1 :]
-            self._path = pfs.path.combine(self._uri.path, self._glob_pattern)
+            self._path = os.path.abspath(self._fs._strip_protocol(path))
 
         if format_hint is None or format_hint == "":
             for k, v in _FORMAT_MAP.items():
@@ -48,56 +39,64 @@ class FileParser(object):
             self._format = format_hint
 
     def assert_no_glob(self) -> "FileParser":
-        assert_or_throw(self.glob_pattern == "", f"{self.path} has glob pattern")
+        assert_or_throw(not self.has_glob, f"{self.raw_path} has glob pattern")
         return self
 
-    def with_glob(self, glob: str, format_hint: Optional[str] = None) -> "FileParser":
-        uri = self.uri
-        if glob != "":
-            uri = pfs.path.combine(uri, glob)
-        return FileParser(uri, format_hint or self._orig_format_hint)
-
     @property
-    def glob_pattern(self) -> str:
-        return self._glob_pattern
+    def has_glob(self):
+        return self._has_glob
 
     @property
-    def uri(self) -> str:
-        return self._uri.geturl()
+    def is_local(self):
+        return isinstance(self._fs, LocalFileSystem)
 
-    @property
-    def uri_with_glob(self) -> str:
-        if self.glob_pattern == "":
-            return self.uri
-        return pfs.path.combine(self.uri, self.glob_pattern)
+    def join(self, path: str, format_hint: Optional[str] = None) -> "FileParser":
+        if not self.has_glob:
+            _path = join(self.path, path)
+        else:
+            _path = join(self.parent, path)
+        return FileParser(_path, format_hint or self._orig_format_hint)
 
     @property
     def parent(self) -> str:
-        dn = os.path.dirname(self.uri)
-        return dn if dn != "" else "."
-
-    @property
-    def scheme(self) -> str:
-        return self._uri.scheme
+        return self._fs.unstrip_protocol(self._fs._parent(self._fs_path))
 
     @property
     def path(self) -> str:
         return self._path
 
+    @property
+    def raw_path(self) -> str:
+        return self._raw_path
+
     @property
     def suffix(self) -> str:
-        return "".join(pathlib.Path(self.path.lower()).suffixes)
+        return "".join(pathlib.Path(self.raw_path.lower()).suffixes)
 
     @property
     def file_format(self) -> str:
         return self._format
 
+    def make_parent_dirs(self) -> None:
+        self._fs.makedirs(self._fs._parent(self._fs_path), exist_ok=True)
+
+    def find_all(self) -> Iterable["FileParser"]:
+        if self.has_glob:
+            for x in self._fs.glob(self._fs_path):
+                yield FileParser(self._fs.unstrip_protocol(x))
+        else:
+            yield self
+
+    def open(self, *args: Any, **kwargs: Any) -> Any:
+        self.assert_no_glob()
+        return self._fs.open(self._fs_path, *args, **kwargs)
+
 
 def load_df(
     uri: Union[str, List[str]],
     format_hint: Optional[str] = None,
     columns: Any = None,
-    fs: Optional[FileSystem] = None,
+    fs: Optional[AbstractFileSystem] = None,
     **kwargs: Any,
 ) -> LocalBoundedDataFrame:
     if isinstance(uri, str):
@@ -117,7 +116,7 @@ def save_df(
     uri: str,
     format_hint: Optional[str] = None,
     mode: str = "overwrite",
-    fs: Optional[FileSystem] = None,
+    fs: Optional[AbstractFileSystem] = None,
     **kwargs: Any,
 ) -> None:
     assert_or_throw(
@@ -125,40 +124,28 @@ def save_df(
     )
     p = FileParser(uri, format_hint).assert_no_glob()
     if fs is None:
-        fs = FileSystem()
+        fs, _ = url_to_fs(uri)
     if fs.exists(uri):
         assert_or_throw(mode == "overwrite", FileExistsError(uri))
         try:
-            fs.remove(uri)
-        except Exception:
-            try:
-                fs.removetree(uri)
-            except Exception:  # pragma: no cover
-                pass
+            fs.rm(uri, recursive=True)
+        except Exception:  # pragma: no cover
+            pass
     _FORMAT_SAVE[p.file_format](df, p, **kwargs)
 
 
 def _get_single_files(
-    fp: Iterable[FileParser], fs: Optional[FileSystem]
+    fp: Iterable[FileParser], fs: Optional[AbstractFileSystem]
 ) -> Iterable[FileParser]:
-    if fs is None:
-        fs = FileSystem()
     for f in fp:
-        if f.glob_pattern != "":
-            files = [
-                FileParser(pfs.path.combine(f.uri, pfs.path.basename(x.path)))
-                for x in fs.opendir(f.uri).glob(f.glob_pattern)
-            ]
-            yield from _get_single_files(files, fs)
-        else:
-            yield f
+        yield from f.find_all()
 
 
 def _save_parquet(df: LocalDataFrame, p: FileParser, **kwargs: Any) -> None:
     PD_UTILS.to_parquet_friendly(
         df.as_pandas(), partition_cols=kwargs.get("partition_cols", [])
     ).to_parquet(
-        p.uri,
+        p.assert_no_glob().path,
         **{
             "engine": "pyarrow",
             "schema": df.schema.pa_schema,
@@ -171,34 +158,36 @@ def _load_parquet(
     p: FileParser, columns: Any = None, **kwargs: Any
 ) -> Tuple[pd.DataFrame, Any]:
     if columns is None:
-        pdf = pd.read_parquet(p.uri, **{"engine": "pyarrow", **kwargs})
+        pdf = pd.read_parquet(p.path, **{"engine": "pyarrow", **kwargs})
         return pdf, None
     if isinstance(columns, list):  # column names
-        pdf = pd.read_parquet(p.uri, columns=columns, **{"engine": "pyarrow", **kwargs})
+        pdf = pd.read_parquet(
+            p.path, columns=columns, **{"engine": "pyarrow", **kwargs}
+        )
         return pdf, None
     schema = Schema(columns)
     pdf = pd.read_parquet(
-        p.uri, columns=schema.names, **{"engine": "pyarrow", **kwargs}
+        p.path, columns=schema.names, **{"engine": "pyarrow", **kwargs}
     )
     return pdf, schema
 
 
 def _save_csv(df: LocalDataFrame, p: FileParser, **kwargs: Any) -> None:
-    df.as_pandas().to_csv(p.uri, **{"index": False, "header": False, **kwargs})
+    with p.open("w") as f:
+        df.as_pandas().to_csv(f, **{"index": False, "header": False, **kwargs})
 
 
-def _safe_load_csv(path: str, **kwargs: Any) -> pd.DataFrame:
+def _safe_load_csv(p: FileParser, **kwargs: Any) -> pd.DataFrame:
     def load_dir() -> pd.DataFrame:
-        fs = FileSystem()
-        return pd.concat(
-            [
-                pd.read_csv(pfs.path.combine(path, pfs.path.basename(x.path)), **kwargs)
-                for x in fs.opendir(path).glob("*.csv")
-            ]
-        )
+        dfs: List[pd.DataFrame] = []
+        for _p in p.join("*.csv").find_all():  # type: ignore
+            with _p.open("r") as f:
+                dfs.append(pd.read_csv(f, **kwargs))
+        return pd.concat(dfs)
 
     try:
-        return pd.read_csv(path, **kwargs)
+        with p.open("r") as f:
+            return pd.read_csv(f, **kwargs)
     except IsADirectoryError:
         return load_dir()
     except pd.errors.ParserError:  # pragma: no cover
@@ -224,7 +213,7 @@ def _load_csv(  # noqa: C901
         header = kw["header"]
         del kw["header"]
     if str(header) in ["True", "0"]:
-        pdf = _safe_load_csv(p.uri, **{"index_col": False, "header": 0, **kw})
+        pdf = _safe_load_csv(p, **{"index_col": False, "header": 0, **kw})
         if columns is None:
             return pdf, None
         if isinstance(columns, list):  # column names
@@ -236,12 +225,14 @@ def _load_csv(  # noqa: C901
             raise ValueError("columns must be set if without header")
         if isinstance(columns, list):  # column names
             pdf = _safe_load_csv(
-                p.uri, **{"index_col": False, "header": None, "names": columns, **kw}
+                p,
+                **{"index_col": False, "header": None, "names": columns, **kw},
             )
             return pdf, None
         schema = Schema(columns)
         pdf = _safe_load_csv(
-            p.uri, **{"index_col": False, "header": None, "names": schema.names, **kw}
+            p,
+            **{"index_col": False, "header": None, "names": schema.names, **kw},
         )
         return pdf, schema
     else:
@@ -249,27 +240,31 @@ def _load_csv(  # noqa: C901
 
 
 def _save_json(df: LocalDataFrame, p: FileParser, **kwargs: Any) -> None:
-    df.as_pandas().to_json(p.uri, **{"orient": "records", "lines": True, **kwargs})
+    with p.open("w") as f:
+        df.as_pandas().to_json(f, **{"orient": "records", "lines": True, **kwargs})
 
 
-def _safe_load_json(path: str, **kwargs: Any) -> pd.DataFrame:
+def _safe_load_json(p: FileParser, **kwargs: Any) -> pd.DataFrame:
     kw = {"orient": "records", "lines": True, **kwargs}
+
+    def load_dir() -> pd.DataFrame:
+        dfs: List[pd.DataFrame] = []
+        for _p in p.join("*.json").find_all():  # type: ignore
+            with _p.open("r") as f:
+                dfs.append(pd.read_json(f, **kw))
+        return pd.concat(dfs)
+
     try:
-        return pd.read_json(path, **kw)
+        with p.open("r") as f:
+            return pd.read_json(f, **kw)
     except (IsADirectoryError, PermissionError):
-        fs = FileSystem()
-        return pd.concat(
-            [
-                pd.read_json(pfs.path.combine(path, pfs.path.basename(x.path)), **kw)
-                for x in fs.opendir(path).glob("*.json")
-            ]
-        )
+        return load_dir()
 
 
 def _load_json(
     p: FileParser, columns: Any = None, **kwargs: Any
 ) -> Tuple[pd.DataFrame, Any]:
-    pdf = _safe_load_json(p.uri, **kwargs).reset_index(drop=True)
+    pdf = _safe_load_json(p, **kwargs).reset_index(drop=True)
     if columns is None:
         return pdf, None
     if isinstance(columns, list):  # column names

fugue/dataframe/utils.py

@@ -1,15 +1,15 @@
-import os
 import pickle
-from typing import Any, Iterable, Optional, Tuple, List, Dict
+from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 import pandas as pd
 import pyarrow as pa
-from fs import open_fs
-from triad import FileSystem, Schema, assert_or_throw
+from fsspec import AbstractFileSystem
+from triad import Schema, assert_or_throw
 from triad.collections.schema import SchemaError
 from triad.exceptions import InvalidOperationError
 from triad.utils.assertion import assert_arg_not_none
 from triad.utils.assertion import assert_or_throw as aot
+from triad.utils.io import url_to_fs
 from triad.utils.pyarrow import pa_batch_to_dicts
 
 from .api import as_fugue_df, get_column_names, normalize_column_names, rename
@@ -112,7 +112,6 @@ def serialize_df(
     df: Optional[DataFrame],
     threshold: int = -1,
     file_path: Optional[str] = None,
-    fs: Optional[FileSystem] = None,
 ) -> Optional[bytes]:
     """Serialize input dataframe to base64 string or to file
     if it's larger than threshold
@@ -121,15 +120,8 @@ def serialize_df(
     :param threshold: file byte size threshold, defaults to -1
     :param file_path: file path to store the data (used only if the serialized data
       is larger than ``threshold``), defaults to None
-    :param fs: :class:`~triad:triad.collections.fs.FileSystem`, defaults to None
     :raises InvalidOperationError: if file is large but ``file_path`` is not provided
     :return: a pickled blob either containing the data or the file path
-
-    .. note::
-
-        If fs is not provided but it needs to write to disk, then it will use
-        :meth:`~fs:fs.opener.registry.Registry.open_fs` to try to open the file to
-        write.
     """
     if df is None:
         return None
@@ -140,24 +132,20 @@ def serialize_df(
     else:
         if file_path is None:
             raise InvalidOperationError("file_path is not provided")
-        if fs is None:
-            with open_fs(
-                os.path.dirname(file_path), writeable=True, create=False
-            ) as _fs:
-                _fs.writebytes(os.path.basename(file_path), data)
-        else:
-            fs.writebytes(file_path, data)
+        fs, path = url_to_fs(file_path)
+        with fs.open(path, "wb") as f:
+            f.write(data)
         return pickle.dumps(file_path)
 
 
 def deserialize_df(
-    data: Optional[bytes], fs: Optional[FileSystem] = None
+    data: Optional[bytes], fs: Optional[AbstractFileSystem] = None
 ) -> Optional[LocalBoundedDataFrame]:
     """Deserialize json string to
     :class:`~fugue.dataframe.dataframe.LocalBoundedDataFrame`
 
     :param json_str: json string containing the base64 data or a file path
-    :param fs: :class:`~triad:triad.collections.fs.FileSystem`, defaults to None
+    :param fs: the file system to use, defaults to None
     :raises ValueError: if the json string is invalid, not generated from
       :func:`~.serialize_df`
     :return: :class:`~fugue.dataframe.dataframe.LocalBoundedDataFrame` if ``json_str``
@@ -169,10 +157,9 @@ def deserialize_df(
     if isinstance(obj, LocalBoundedDataFrame):
         return obj
     elif isinstance(obj, str):
-        if fs is None:
-            with open_fs(os.path.dirname(obj), create=False) as _fs:
-                return pickle.loads(_fs.readbytes(os.path.basename(obj)))
-        return pickle.loads(fs.readbytes(obj))
+        fs, path = url_to_fs(obj)
+        with fs.open(path, "rb") as f:
+            return pickle.load(f)
     raise ValueError("data is invalid")
 
 

fugue/execution/execution_engine.py

@@ -18,7 +18,6 @@ from typing import (
 from uuid import uuid4
 
 from triad import ParamDict, Schema, SerializableRLock, assert_or_throw, to_uuid
-from triad.collections.fs import FileSystem
 from triad.collections.function_wrapper import AnnotatedParam
 from triad.exceptions import InvalidOperationError
 from triad.utils.convert import to_size
@@ -471,12 +470,6 @@ class ExecutionEngine(FugueEngineBase):
         """
         self._sql_engine = engine
 
-    @property
-    @abstractmethod
-    def fs(self) -> FileSystem:  # pragma: no cover
-        """File system of this engine instance"""
-        raise NotImplementedError
-
     @abstractmethod
     def create_default_map_engine(self) -> MapEngine:  # pragma: no cover
         """Default MapEngine if user doesn't specify"""

fugue/execution/native_execution_engine.py

@@ -1,12 +1,13 @@
 import logging
 import os
 from typing import Any, Callable, Dict, List, Optional, Type, Union
+
 import numpy as np
 import pandas as pd
 from triad import Schema
 from triad.collections.dict import IndexedOrderedDict
-from triad.collections.fs import FileSystem
 from triad.utils.assertion import assert_or_throw
+from triad.utils.io import makedirs
 from triad.utils.pandas_like import PandasUtils
 
 from fugue._utils.io import load_df, save_df
@@ -179,7 +180,6 @@ class NativeExecutionEngine(ExecutionEngine):
 
     def __init__(self, conf: Any = None):
         super().__init__(conf)
-        self._fs = FileSystem()
         self._log = logging.getLogger()
 
     def __repr__(self) -> str:
@@ -189,10 +189,6 @@ class NativeExecutionEngine(ExecutionEngine):
     def log(self) -> logging.Logger:
         return self._log
 
-    @property
-    def fs(self) -> FileSystem:
-        return self._fs
-
     @property
     def is_distributed(self) -> bool:
         return False
@@ -395,9 +391,7 @@ class NativeExecutionEngine(ExecutionEngine):
         **kwargs: Any,
     ) -> LocalBoundedDataFrame:
         return self.to_df(
-            load_df(
-                path, format_hint=format_hint, columns=columns, fs=self.fs, **kwargs
-            )
+            load_df(path, format_hint=format_hint, columns=columns, **kwargs)
         )
 
     def save_df(
@@ -413,9 +407,9 @@ class NativeExecutionEngine(ExecutionEngine):
         partition_spec = partition_spec or PartitionSpec()
         if not force_single and not partition_spec.empty:
             kwargs["partition_cols"] = partition_spec.partition_by
-        self.fs.makedirs(os.path.dirname(path), recreate=True)
+        makedirs(os.path.dirname(path), exist_ok=True)
         df = self.to_df(df)
-        save_df(df, path, format_hint=format_hint, mode=mode, fs=self.fs, **kwargs)
+        save_df(df, path, format_hint=format_hint, mode=mode, **kwargs)
 
 
 @fugue_annotated_param(NativeExecutionEngine)

fugue/workflow/_checkpoint.py

@@ -1,14 +1,15 @@
 from typing import Any
 
-import fs as pfs
+from triad.utils.assertion import assert_or_throw
+from triad.utils.hash import to_uuid
+from triad.utils.io import exists, join, makedirs, rm
+
 from fugue.collections.partition import PartitionSpec
 from fugue.collections.yielded import PhysicalYielded
 from fugue.constants import FUGUE_CONF_WORKFLOW_CHECKPOINT_PATH
 from fugue.dataframe import DataFrame
 from fugue.exceptions import FugueWorkflowCompileError, FugueWorkflowRuntimeError
 from fugue.execution.execution_engine import ExecutionEngine
-from triad.utils.assertion import assert_or_throw
-from triad.utils.hash import to_uuid
 
 
 class Checkpoint(object):
@@ -130,7 +131,6 @@ class WeakCheckpoint(Checkpoint):
 class CheckpointPath(object):
     def __init__(self, engine: ExecutionEngine):
         self._engine = engine
-        self._fs = engine.fs
         self._log = engine.log
         self._path = engine.conf.get(FUGUE_CONF_WORKFLOW_CHECKPOINT_PATH, "").strip()
         self._temp_path = ""
@@ -143,14 +143,14 @@ class CheckpointPath(object):
         if self._path == "":
             self._temp_path = ""
             return ""
-        self._temp_path = pfs.path.combine(self._path, execution_id)
-        self._fs.makedirs(self._temp_path, recreate=True)
+        self._temp_path = join(self._path, execution_id)
+        makedirs(self._temp_path, exist_ok=True)
         return self._temp_path
 
     def remove_temp_path(self):
         if self._temp_path != "":
             try:
-                self._fs.removetree(self._temp_path)
+                rm(self._temp_path, recursive=True)
             except Exception as e:  # pragma: no cover
                 self._log.info("Unable to remove " + self._temp_path, e)
 
@@ -162,7 +162,7 @@ class CheckpointPath(object):
                 f"{FUGUE_CONF_WORKFLOW_CHECKPOINT_PATH} is not set"
             ),
         )
-        return pfs.path.combine(path, obj_id + ".parquet")
+        return join(path, obj_id + ".parquet")
 
     def get_table_name(self, obj_id: str, permanent: bool) -> str:
         path = self._path if permanent else self._temp_path
@@ -170,6 +170,6 @@ class CheckpointPath(object):
 
     def temp_file_exists(self, path: str) -> bool:
         try:
-            return self._fs.exists(path)
+            return exists(path)
         except Exception:  # pragma: no cover
             return False

{fugue-0.8.7.dev6.dist-info → fugue-0.8.7.dev7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fugue
-Version: 0.8.7.dev6
+Version: 0.8.7.dev7
 Summary: An abstraction layer for distributed computation
 Home-page: http://github.com/fugue-project/fugue
 Author: The Fugue Development Team
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: triad ==0.9.2.dev5
+Requires-Dist: triad ==0.9.2.dev8
 Requires-Dist: adagio >=0.2.4
 Requires-Dist: qpd >=0.4.4
 Requires-Dist: fugue-sql-antlr >=0.1.6