fugue 0.8.7.dev6__py3-none-any.whl → 0.8.7.dev7__py3-none-any.whl

{fugue-0.8.7.dev6.dist-info → fugue-0.8.7.dev7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fugue/__init__.py,sha256=xT5zuNZfRkjbA8a-uTT5oLK6hLGuezGZLWYBl6eS5J4,2749
+fugue/__init__.py,sha256=LKkBEPEAMLG-Yuzqt0IgoIDEfNf9a1zUffNKb83D_l8,2705
 fugue/api.py,sha256=dLUrigFhDMB5x7cvlWSK8EyaY2o0AmhgPr0VRtfzSz0,1254
 fugue/constants.py,sha256=crd0VqX8WtBcjSUNwZDi2LDIEkhUMWOlSn73H8JI9ds,3385
 fugue/dev.py,sha256=GQCkezBBl4V0lVDWhGtUQKqomiCxgR9dMhfqj9C8cS8,1369
@@ -10,7 +10,7 @@ fugue/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/_utils/display.py,sha256=JV8oDA7efHm1wceZulCBOY5dMvjbWHvIm6ASisKfoWY,3164
 fugue/_utils/exception.py,sha256=SFIjwjV4CIEovp3P9k7ePNOFB12A5D8hDdhtfFUeM5Y,2247
 fugue/_utils/interfaceless.py,sha256=wI0H6L4W_1uQjh9tpjgT9HzN-fbrrtXXHC1x6Q_rrPg,2203
-fugue/_utils/io.py,sha256=qDwqgY389GhCHV-7EvuiysJVbHxhquuEva9IlOYsmDw,9271
+fugue/_utils/io.py,sha256=adrtj6Dq0ti426DNlkliApbTkp8b3bfBysAiE5MVQVc,9265
 fugue/_utils/misc.py,sha256=_huy0eylmRTEFoReGR2M4rbAI8m79hFcfY5bDceVEXU,887
 fugue/_utils/registry.py,sha256=lrbzTdUEVnW6paBGDj-Yb-aTIbP5mjCqrXuRU9_N6os,316
 fugue/bag/__init__.py,sha256=0Q0_rnrEThrTx2U-1xGNyAg95idp_xcnywymIcW4Xck,46
@@ -34,15 +34,15 @@ fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs
 fugue/dataframe/function_wrapper.py,sha256=V1eQMOn27UroEYT7_YiwoEF0RjZYIM0zkD3vfaMAQFs,14813
 fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
 fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
-fugue/dataframe/utils.py,sha256=VS1qLCr-9NEcEjaK-219rADJadDf6EfzYZCGRUpn1fY,11405
+fugue/dataframe/utils.py,sha256=shN1eHYTnPhb38BHEpLlCdLSzX_qpoQ3-fsDgu1hCzQ,10840
 fugue/dataset/__init__.py,sha256=5f2CAJ4xst6Z2o9Q2e2twfDOGUw8ZJoE2ild4JEU2pg,112
 fugue/dataset/api.py,sha256=DacI4L2w5NJ-eZ6nFxNMqmReEnb0WUXswbjVp7BeErk,2794
 fugue/dataset/dataset.py,sha256=jWXZqy3msMPFFkhas2PYJEX55ZAI3gk3Txq5f4-Qya4,4759
 fugue/execution/__init__.py,sha256=iZGxAznZz9piM3k4gp0tln97MDIBxdliLyNbD-0Zc48,427
 fugue/execution/api.py,sha256=KsFOLGdWQMlXmlQ5JRgRsbUeB64qzTVHxSEaunjiojo,39818
-fugue/execution/execution_engine.py,sha256=G_SsTmcuDcy6_azi_88lGzsOodiizu0JdWxebxgbqRg,47721
+fugue/execution/execution_engine.py,sha256=5lIlebgPK7q-Gf4bWt1t_Anq3MjPaJBpGWN9bbry1B4,47506
 fugue/execution/factory.py,sha256=5ICzfNh2QqqABuVyYLijY5-7LZgfRqczlaZN32p78bE,21003
-fugue/execution/native_execution_engine.py,sha256=Mm9BVC3dEMS3IWRZe4YvGKp6_mmW7dLmoLMK5HgAPcs,14408
+fugue/execution/native_execution_engine.py,sha256=lbKd3uGh00cSTkIM8l-u8jmsMxFzV2PSUeJgudayxKs,14236
 fugue/extensions/__init__.py,sha256=y-uLKd6mZ8sZ_8-OdW6ELoBO_9IfC0gDmEbE_rMCvOA,599
 fugue/extensions/_utils.py,sha256=Bi3pYKy2Z6fG6_5BpwIWldxetassXpB4Zp8QamWB-wg,5173
 fugue/extensions/context.py,sha256=c_y2UttzzIFoQTOCV42VCdj2nqah33xYuBjbKNIOpx8,4262
@@ -72,7 +72,7 @@ fugue/sql/_visitors.py,sha256=2pc0J-AHJAiIexsKgNjcgrCGOyhC3_7rzonSgtjy--k,33844
 fugue/sql/api.py,sha256=l2I9CAy_W2oFFTct9fDPLyXF0LiDxQhMx5O8jBHTAxU,10050
 fugue/sql/workflow.py,sha256=S1pOhp0b0t6johFAJWmj6xUB7Ti5LQgNABpAzmLGjrQ,3010
 fugue/workflow/__init__.py,sha256=tXM_KYO8Q358W6qAVlwhIQIaYNRDgZtTubrIEX4QMgM,229
-fugue/workflow/_checkpoint.py,sha256=MTMyNCdWHf5UK8bRepfR2u8y3cEhO1RYIYq558ZlXzA,5715
+fugue/workflow/_checkpoint.py,sha256=tt5Iv7c5ZStC0MD1inItksQ0GuK0ViniA3nvrgym-5c,5681
 fugue/workflow/_tasks.py,sha256=Zq_jXJO_VaF8DrWUuBiwO2Y3OVuhsiOQdzP4VBsp7Fo,11826
 fugue/workflow/_workflow_context.py,sha256=Wmp6n0lSrh2Gpslb5EaSX6BQNniKsvKn6SlhVkQ6ui0,2504
 fugue/workflow/api.py,sha256=uQoxPSCZ91-ST4vwuPWG7qioRGW4eo-Sgi3DdwtSL4k,12495
@@ -86,25 +86,25 @@ fugue_contrib/viz/__init__.py,sha256=osgZx63Br-yMZImyEfYf9MVzJNM2Cqqke_-WsuDmG5M
 fugue_contrib/viz/_ext.py,sha256=Lu_DlS5DcmrFz27fHcKTCkhKyknVWcfS5kzZVVuO9xM,1345
 fugue_dask/__init__.py,sha256=2CcJ0AsN-k_f7dZ-yAyYpaICfUMPfH3l0FvUJSBzTr0,161
 fugue_dask/_constants.py,sha256=35UmTVITk21GhRyRlbJOwPPdQsytM_p_2NytOXEay18,510
-fugue_dask/_io.py,sha256=9G516yM6zQvSC5_JA6qHb3LwBDmhWcxK5sjFHrQ81zo,6012
+fugue_dask/_io.py,sha256=HmL3Q2lRSptX1-GwiB3MN2VpjTRfmVKD8TDZkhS4x5c,5818
 fugue_dask/_utils.py,sha256=n70N3wPPMz13Jh0GWJM3Je-TCYpU36yGP_YCwIHqUrc,8908
 fugue_dask/dataframe.py,sha256=MuG9TqCND7qI66lPvxzuomfE7yA4sW7DjrvbyvE6XEU,13471
-fugue_dask/execution_engine.py,sha256=XJp6wrdkaNh5pOpwt-Hjoa2sxgCOgusFRWrcqoCcaNM,21153
+fugue_dask/execution_engine.py,sha256=PAClUP9lCdn2Aajt2AsoFOsgO-95WcdRDKkjNSbVbzA,20980
 fugue_dask/ibis_engine.py,sha256=kQdaG_KlZZ2AjtYETNCdTJOgtwI_eH0aGzLaAiIBbRI,2120
 fugue_dask/registry.py,sha256=7UTg_eie7zKlHYKMCyOo0TNn5y2TiIjE8kiS2PruHFc,2200
 fugue_duckdb/__init__.py,sha256=nSNv-fxBAKD6W23EbMeV4dVRIaSTqr9DzQUWuVOES8s,379
-fugue_duckdb/_io.py,sha256=Sq228unVnroYTq4GX-Wnv22SLHC9Ji-aWgiqrfdu81w,8880
+fugue_duckdb/_io.py,sha256=E35_GoD1uGuuAMOY4H8E2j-UazdAgTmLp4lLWqJrNsE,8437
 fugue_duckdb/_utils.py,sha256=ElKbHUyn5fWSPGXsK57iqMzcqKtCf0c8pBVBYGe5Ql4,5020
 fugue_duckdb/dask.py,sha256=agoLzeB7Swxj2kVWfmXFbWD1NS2lbbTlnrjSkR8kKWY,5014
 fugue_duckdb/dataframe.py,sha256=LRfTv7Y46wMM_IDYSP1R-5OXuHuBg8GHjPGFFt8u7l0,8444
-fugue_duckdb/execution_engine.py,sha256=fkkQb4Eh0m7SwKrTplVk2oQalLkNoj3CW0R12g01ofk,20536
+fugue_duckdb/execution_engine.py,sha256=IZDmSAtOMJGvulTStxjTmsqJyI5QRNyxBgSMlFMSrBI,20389
 fugue_duckdb/ibis_engine.py,sha256=MrypeABozqwetKOpqtrmWvCJX2QPfBXhbSEhvK9vqmI,1990
 fugue_duckdb/registry.py,sha256=Dj0Tng1cXVT6Q7t-KxOky2k1dD9xSBjYGQmI26UgZPo,3095
 fugue_ibis/__init__.py,sha256=PcUt66KlLyGGicad7asq5j2U567_fhR0HzvWQBhV1VM,362
 fugue_ibis/_compat.py,sha256=zKdTaTfuC02eUIzZPkcd7oObnVBi_X5mQjQf7SDme3Y,246
 fugue_ibis/_utils.py,sha256=BUL5swA5FE4eQu0t5Z17hZVu9a2MFfxlFH6Ymy9xifg,6607
 fugue_ibis/dataframe.py,sha256=0Fb1vJjwEeffgoUCDfDGIMuSFaPgUJqcB-JqJOAALfs,7789
-fugue_ibis/execution_engine.py,sha256=p5zy0IBXiJgLi67RBHCRcHgZsaJMANdNSpUxz0k_6C0,18453
+fugue_ibis/execution_engine.py,sha256=0GIjjMmitCKhjasAKFiFUCCUBNdxAiU0b61RsmFyhIk,18355
 fugue_ibis/extensions.py,sha256=H8l-SPfoqLuUoILtOuL2nccOpoL83zHeSoIhoqjtWQM,6905
 fugue_ibis/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ibis/execution/ibis_engine.py,sha256=-HdPnIFWD83n5WITdzJiu4attH7GOcO041wkT5Y5ChA,1499
@@ -127,37 +127,39 @@ fugue_ray/registry.py,sha256=xJRAhbwNrg695EwghQDnVtTKi4YkqZ0_61BD4OAblSA,1685
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
 fugue_ray/_utils/dataframe.py,sha256=_EadzS4rPom1A_cF0pqoPlwrNYZTfTwcyyu86_fFsqU,4400
-fugue_ray/_utils/io.py,sha256=SFTU4qXubGEmO5IGZA5yHy8Hu4b9aFZ9-eTU4Qs-NsQ,8757
+fugue_ray/_utils/io.py,sha256=4FfPS2DMeIHvbzGoJ_iPvwwVr7lZHXRoJZxceNZ4EHQ,8647
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
-fugue_spark/execution_engine.py,sha256=rqgY9U1bpjh0GFNyNkuPcI7iV0xeipadURhNIir4w08,33147
+fugue_spark/execution_engine.py,sha256=KPmBtH4zioXdWsvnPow4fOPQh8Yj0cn6yCJyKdbT544,33023
 fugue_spark/ibis_engine.py,sha256=Yl5xxwROo1idcD2hFaylaI1IpmBUgbvOZRWtcrE0Zjo,1697
 fugue_spark/registry.py,sha256=kyIMk6dAiKRSKCHawQKyXu9DhZ24T6j3gL57TiOAZ8c,4162
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_spark/_utils/convert.py,sha256=eRWkDYA4UO-FQu-2y4O80WEdawx7X_rIrWg55AlOiRc,10007
-fugue_spark/_utils/io.py,sha256=0ndQ70YlirPwGKjh5IDN6IdJxD26BnPpMonRob4dxII,5668
+fugue_spark/_utils/io.py,sha256=OdUezKpB29Lx9aUS2k9x0xUAGZrmgMZyQYGPEeHk7rQ,5574
 fugue_spark/_utils/misc.py,sha256=o8dZmXOHnA7D_ps37vgGXTPTiSEG9LQzPKq7l-MG-qM,860
 fugue_spark/_utils/partition.py,sha256=iaesyO5f4uXhj1W-p91cD5ecPiGlu0bzh8gl2ce2Uvg,3618
 fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=o8aMZTKa74nKBmcUTTBbliTJMtNbsXE9SPKZopS504o,78400
+fugue_test/builtin_suite.py,sha256=3uSY484Jl2985UoJravD4C-SlKBH0WwTWFobp4Pqgzg,78399
 fugue_test/dataframe_suite.py,sha256=LgB931CkASbGOrRQ9j92DGk9wPb__FoNusOk-HeqU9E,19165
-fugue_test/execution_suite.py,sha256=FI6UmwBvdoT1jkJRBqJT_Q0IDehFryvv00UL6jjxyAk,47689
+fugue_test/execution_suite.py,sha256=ClZUYt2R560LN4DZM_OP9cA5jaHmz3-u_BC3A0C24fQ,47472
 fugue_test/ibis_suite.py,sha256=Dk4AHVD00RcFsNm9VvJ4_4LOyFdGX30OnAtpO2SPruE,3529
 fugue_test/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_test/plugins/dask/__init__.py,sha256=LQVyNgGRvZrKUrNNV1z1X1GyIL3qJoxvNjFfpFzNVCc,55
 fugue_test/plugins/dask/fixtures.py,sha256=uf5gkO30L5-LvxpEpBjG4_bNUrpkemHvyVPxDHgMSGM,354
 fugue_test/plugins/duckdb/__init__.py,sha256=WXtNYQpbO0JScPpIA3QREv8cwOZP2GDOgGOtJKgpTVM,61
 fugue_test/plugins/duckdb/fixtures.py,sha256=UxQbIMRbSrTZ3pgCmKZgd5wd1YvnVrqLSUPaO8UYrSE,165
+fugue_test/plugins/misc/__init__.py,sha256=0SZvyo0xlw5NDbJly4yjaNDqL9M4D2Jsg33HCWE40q8,49
+fugue_test/plugins/misc/fixtures.py,sha256=GrD9WTTtcIDCWLHn-ToVv8pUiUCGCSczgs9bodWKo7c,353
 fugue_test/plugins/ray/__init__.py,sha256=nyKGW6xgTXtMhSs7yjgFNKO7mVboCNg63Bvdf39fO_I,55
 fugue_test/plugins/ray/fixtures.py,sha256=hZkvuo0AcD63XJl5JUroc9tm2LWHUPszg2zzY6FCSao,141
 fugue_version/__init__.py,sha256=vTwvdJOZi8jZb9U-Em7-d50qNDNPS2z51IXqRoojeNM,22
-fugue-0.8.7.dev6.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.8.7.dev6.dist-info/METADATA,sha256=0i4ibczIy_wEMtZ6vFvaCw40x5KmuQa6OsuBVWUTQyk,17860
-fugue-0.8.7.dev6.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-fugue-0.8.7.dev6.dist-info/entry_points.txt,sha256=N_BIIy3lSvF6Z32QE0yXTucgdHrPbUrOwH1zj7bZ0ow,536
-fugue-0.8.7.dev6.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.8.7.dev6.dist-info/RECORD,,
+fugue-0.8.7.dev7.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+fugue-0.8.7.dev7.dist-info/METADATA,sha256=nSp1i8apniEEe6U09_5RA8K89P40c7M5Gn9l6ofLTHQ,17860
+fugue-0.8.7.dev7.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+fugue-0.8.7.dev7.dist-info/entry_points.txt,sha256=Xrl3ISyVKAFIPn1klqeGsL9DinzoYqfqBkOT4qAVBNA,578
+fugue-0.8.7.dev7.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.8.7.dev7.dist-info/RECORD,,

{fugue-0.8.7.dev6.dist-info → fugue-0.8.7.dev7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.41.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{fugue-0.8.7.dev6.dist-info → fugue-0.8.7.dev7.dist-info}/entry_points.txt

@@ -2,7 +2,7 @@
 dask = fugue_dask.registry [dask]
 dask_ibis = fugue_dask.ibis_engine [dask,ibis]
 duckdb = fugue_duckdb.registry [duckdb]
-duckdb_ibis = fugue_duckdb.ibis_engine [duckdb,ibis]
+duckdb_ibis = fugue_duckdb.ibis_engine [ibis,duckdb]
 ibis = fugue_ibis [ibis]
 polars = fugue_polars.registry [polars]
 ray = fugue_ray.registry [ray]
@@ -12,5 +12,6 @@ spark_ibis = fugue_spark.ibis_engine [spark,ibis]
 [pytest11]
 fugue_test_dask = fugue_test.plugins.dask [dask]
 fugue_test_duckdb = fugue_test.plugins.duckdb [duckdb]
+fugue_test_misc = fugue_test.plugins.misc
 fugue_test_ray = fugue_test.plugins.ray [ray]
 

fugue_dask/_io.py

@@ -1,13 +1,12 @@
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
-import fsspec
-import fs as pfs
 import pandas as pd
 from dask import dataframe as dd
+from fsspec import AbstractFileSystem
 from triad.collections.dict import ParamDict
-from triad.collections.fs import FileSystem
 from triad.collections.schema import Schema
 from triad.utils.assertion import assert_or_throw
+from triad.utils.io import join, makedirs, url_to_fs
 
 from fugue._utils.io import FileParser, _get_single_files
 from fugue_dask.dataframe import DaskDataFrame
@@ -19,7 +18,7 @@ def load_df(
     uri: Union[str, List[str]],
     format_hint: Optional[str] = None,
     columns: Any = None,
-    fs: Optional[FileSystem] = None,
+    fs: Optional[AbstractFileSystem] = None,
     **kwargs: Any,
 ) -> DaskDataFrame:
     if isinstance(uri, str):
@@ -39,7 +38,7 @@ def save_df(
     uri: str,
     format_hint: Optional[str] = None,
     mode: str = "overwrite",
-    fs: Optional[FileSystem] = None,
+    fs: Optional[AbstractFileSystem] = None,
     **kwargs: Any,
 ) -> None:
     assert_or_throw(
@@ -48,16 +47,13 @@ def save_df(
     )
     p = FileParser(uri, format_hint).assert_no_glob()
     if fs is None:
-        fs = FileSystem()
+        fs, _ = url_to_fs(uri)
     if fs.exists(uri):
         assert_or_throw(mode == "overwrite", FileExistsError(uri))
         try:
-            fs.remove(uri)
-        except Exception:
-            try:
-                fs.removetree(uri)
-            except Exception:  # pragma: no cover
-                pass
+            fs.rm(uri, recursive=True)
+        except Exception:  # pragma: no cover
+            pass
     _FORMAT_SAVE[p.file_format](df, p, **kwargs)
 
 
@@ -67,7 +63,7 @@ def _save_parquet(df: DaskDataFrame, p: FileParser, **kwargs: Any) -> None:
         "write_index": False,
         **kwargs,
     }
-    DASK_UTILS.to_parquet_friendly(df.native).to_parquet(p.uri, **params)
+    DASK_UTILS.to_parquet_friendly(df.native).to_parquet(p.path, **params)
 
 
 def _load_parquet(
@@ -80,27 +76,26 @@ def _load_parquet(
     if pd.__version__ >= "1.5":
         dtype_backend = "pyarrow"
     if columns is None:
-        pdf = dd.read_parquet(p.uri, dtype_backend=dtype_backend, **params)
+        pdf = dd.read_parquet(p.path, dtype_backend=dtype_backend, **params)
         schema = Schema(pdf.head(1))
         return pdf, schema
     if isinstance(columns, list):  # column names
         pdf = dd.read_parquet(
-            p.uri, columns=columns, dtype_backend=dtype_backend, **params
+            p.path, columns=columns, dtype_backend=dtype_backend, **params
         )
         schema = Schema(pdf.head(1))
         return pdf, schema
     schema = Schema(columns)
     pdf = dd.read_parquet(
-        p.uri, columns=schema.names, dtype_backend=dtype_backend, **params
+        p.path, columns=schema.names, dtype_backend=dtype_backend, **params
     )
     return pdf, schema
 
 
 def _save_csv(df: DaskDataFrame, p: FileParser, **kwargs: Any) -> None:
-    fs, path = fsspec.core.url_to_fs(p.uri)
-    fs.makedirs(path, exist_ok=True)
+    makedirs(p.path, exist_ok=True)
     df.native.to_csv(
-        pfs.path.combine(p.uri, "*.csv"), **{"index": False, "header": False, **kwargs}
+        p.join("*.csv").path, **{"index": False, "header": False, **kwargs}
     )
 
 
@@ -108,7 +103,7 @@ def _safe_load_csv(path: str, **kwargs: Any) -> dd.DataFrame:
     try:
         return dd.read_csv(path, **kwargs)
     except (IsADirectoryError, PermissionError):
-        return dd.read_csv(pfs.path.combine(path, "*.csv"), **kwargs)
+        return dd.read_csv(join(path, "*.csv"), **kwargs)
 
 
 def _load_csv(  # noqa: C901
@@ -127,7 +122,7 @@ def _load_csv(  # noqa: C901
         header = kw["header"]
         del kw["header"]
     if str(header) in ["True", "0"]:
-        pdf = _safe_load_csv(p.uri, **{"header": 0, **kw})
+        pdf = _safe_load_csv(p.path, **{"header": 0, **kw})
         if columns is None:
             return pdf, None
         if isinstance(columns, list):  # column names
@@ -138,34 +133,32 @@ def _load_csv(  # noqa: C901
         if columns is None:
             raise ValueError("columns must be set if without header")
         if isinstance(columns, list):  # column names
-            pdf = _safe_load_csv(p.uri, **{"header": None, "names": columns, **kw})
+            pdf = _safe_load_csv(p.path, **{"header": None, "names": columns, **kw})
             return pdf, None
         schema = Schema(columns)
-        pdf = _safe_load_csv(p.uri, **{"header": None, "names": schema.names, **kw})
+        pdf = _safe_load_csv(p.path, **{"header": None, "names": schema.names, **kw})
         return pdf, schema
     else:
         raise NotImplementedError(f"{header} is not supported")
 
 
 def _save_json(df: DaskDataFrame, p: FileParser, **kwargs: Any) -> None:
-    fs, path = fsspec.core.url_to_fs(p.uri)
-    fs.makedirs(path, exist_ok=True)
-    df.native.to_json(pfs.path.combine(p.uri, "*.json"), **kwargs)
+    makedirs(p.path, exist_ok=True)
+    df.native.to_json(p.join("*.json").path, **kwargs)
 
 
 def _safe_load_json(path: str, **kwargs: Any) -> dd.DataFrame:
     try:
         return dd.read_json(path, **kwargs)
     except (IsADirectoryError, PermissionError):
-        x = dd.read_json(pfs.path.combine(path, "*.json"), **kwargs)
-        print(x.compute())
+        x = dd.read_json(join(path, "*.json"), **kwargs)
         return x
 
 
 def _load_json(
     p: FileParser, columns: Any = None, **kwargs: Any
 ) -> Tuple[dd.DataFrame, Any]:
-    pdf = _safe_load_json(p.uri, **kwargs).reset_index(drop=True)
+    pdf = _safe_load_json(p.path, **kwargs).reset_index(drop=True)
     if columns is None:
         return pdf, None
     if isinstance(columns, list):  # column names

fugue_dask/execution_engine.py

@@ -7,18 +7,17 @@ import pandas as pd
 from distributed import Client
 from triad.collections import Schema
 from triad.collections.dict import IndexedOrderedDict, ParamDict
-from triad.collections.fs import FileSystem
 from triad.utils.assertion import assert_or_throw
 from triad.utils.hash import to_uuid
 from triad.utils.pandas_like import PandasUtils
 from triad.utils.threading import RunOnce
+from triad.utils.io import makedirs
 from fugue import StructuredRawSQL
 from fugue.collections.partition import (
     PartitionCursor,
     PartitionSpec,
     parse_presort_exp,
 )
-from fugue.exceptions import FugueBug
 from fugue.constants import KEYWORD_PARALLELISM, KEYWORD_ROWCOUNT
 from fugue.dataframe import (
     AnyDataFrame,
@@ -28,6 +27,7 @@ from fugue.dataframe import (
     PandasDataFrame,
 )
 from fugue.dataframe.utils import get_join_schemas
+from fugue.exceptions import FugueBug
 from fugue.execution.execution_engine import ExecutionEngine, MapEngine, SQLEngine
 from fugue.execution.native_execution_engine import NativeExecutionEngine
 from fugue_dask._constants import FUGUE_DASK_DEFAULT_CONF
@@ -206,7 +206,6 @@ class DaskExecutionEngine(ExecutionEngine):
         p = ParamDict(FUGUE_DASK_DEFAULT_CONF)
         p.update(ParamDict(conf))
         super().__init__(p)
-        self._fs = FileSystem()
         self._log = logging.getLogger()
         self._client = DASK_UTILS.get_or_create_client(dask_client)
         self._native = NativeExecutionEngine(conf=conf)
@@ -227,10 +226,6 @@ class DaskExecutionEngine(ExecutionEngine):
     def log(self) -> logging.Logger:
         return self._log
 
-    @property
-    def fs(self) -> FileSystem:
-        return self._fs
-
     def create_default_sql_engine(self) -> SQLEngine:
         return DaskSQLEngine(self)
 
@@ -527,9 +522,7 @@ class DaskExecutionEngine(ExecutionEngine):
         **kwargs: Any,
     ) -> DaskDataFrame:
         return self.to_df(
-            load_df(
-                path, format_hint=format_hint, columns=columns, fs=self.fs, **kwargs
-            )
+            load_df(path, format_hint=format_hint, columns=columns, **kwargs)
         )
 
     def save_df(
@@ -556,9 +549,9 @@ class DaskExecutionEngine(ExecutionEngine):
         else:
             if not partition_spec.empty:
                 kwargs["partition_on"] = partition_spec.partition_by
-            self.fs.makedirs(os.path.dirname(path), recreate=True)
+            makedirs(os.path.dirname(path), exist_ok=True)
             df = self.to_df(df)
-            save_df(df, path, format_hint=format_hint, mode=mode, fs=self.fs, **kwargs)
+            save_df(df, path, format_hint=format_hint, mode=mode, **kwargs)
 
 
 def to_dask_engine_df(df: Any, schema: Any = None) -> DaskDataFrame:

fugue_duckdb/_io.py

@@ -3,9 +3,9 @@ from typing import Any, Iterable, List, Optional, Union
 
 from duckdb import DuckDBPyConnection
 from triad import ParamDict, Schema
-from triad.collections.fs import FileSystem
-from triad.utils.assertion import assert_or_throw
 
+from triad.utils.assertion import assert_or_throw
+from triad.utils.io import isdir, makedirs, rm, exists
 from fugue._utils.io import FileParser, load_df, save_df
 from fugue.collections.sql import TempTableName
 from fugue.dataframe import ArrowDataFrame, LocalBoundedDataFrame
@@ -18,26 +18,17 @@ from fugue_duckdb._utils import (
 from fugue_duckdb.dataframe import DuckDataFrame
 
 
-def _get_single_files(
-    fp: Iterable[FileParser], fs: FileSystem, fmt: str
-) -> Iterable[FileParser]:
-    def _isdir(d: str) -> bool:
-        try:
-            return fs.isdir(d)
-        except Exception:  # pragma: no cover
-            return False
-
+def _get_files(fp: Iterable[FileParser], fmt: str) -> Iterable[FileParser]:
     for f in fp:
-        if f.glob_pattern == "" and _isdir(f.uri):
-            yield f.with_glob("*." + fmt, fmt)
+        if not f.has_glob and isdir(f.path):
+            yield from f.join("*." + fmt, fmt).find_all()
         else:
             yield f
 
 
 class DuckDBIO:
-    def __init__(self, fs: FileSystem, con: DuckDBPyConnection) -> None:
+    def __init__(self, con: DuckDBPyConnection) -> None:
         self._con = con
-        self._fs = fs
         self._format_load = {"csv": self._load_csv, "parquet": self._load_parquet}
         self._format_save = {"csv": self._save_csv, "parquet": self._save_parquet}
 
@@ -55,11 +46,9 @@ class DuckDBIO:
         else:
             fp = [FileParser(u, format_hint) for u in uri]
         if fp[0].file_format not in self._format_load:
-            return load_df(
-                uri, format_hint=format_hint, columns=columns, fs=self._fs, **kwargs
-            )
+            return load_df(uri, format_hint=format_hint, columns=columns, **kwargs)
         dfs: List[DuckDataFrame] = []
-        for f in _get_single_files(fp, self._fs, fp[0].file_format):
+        for f in _get_files(fp, fp[0].file_format):
             df = self._format_load[f.file_format](f, columns, **kwargs)
             dfs.append(df)
         rel = dfs[0].native
@@ -83,26 +72,20 @@ class DuckDBIO:
         )
         p = FileParser(uri, format_hint).assert_no_glob()
         if (p.file_format not in self._format_save) or ("partition_cols" in kwargs):
-            self._fs.makedirs(os.path.dirname(uri), recreate=True)
+            makedirs(os.path.dirname(uri), exist_ok=True)
             ldf = ArrowDataFrame(df.as_arrow())
-            return save_df(
-                ldf, uri=uri, format_hint=format_hint, mode=mode, fs=self._fs, **kwargs
-            )
-        fs = self._fs
-        if fs.exists(uri):
+            return save_df(ldf, uri=uri, format_hint=format_hint, mode=mode, **kwargs)
+        if exists(uri):
             assert_or_throw(mode == "overwrite", FileExistsError(uri))
             try:
-                fs.remove(uri)
-            except Exception:
-                try:
-                    fs.removetree(uri)
-                except Exception:  # pragma: no cover
-                    pass
-        if not fs.exists(p.parent):
-            fs.makedirs(p.parent, recreate=True)
+                rm(uri, recursive=True)
+            except Exception:  # pragma: no cover
+                pass
+        p.make_parent_dirs()
         self._format_save[p.file_format](df, p, **kwargs)
 
     def _save_csv(self, df: DuckDataFrame, p: FileParser, **kwargs: Any):
+        p.assert_no_glob()
         dn = TempTableName()
         df.native.create_view(dn.key)
         kw = ParamDict({k.lower(): v for k, v in kwargs.items()})
@@ -111,7 +94,7 @@ class DuckDBIO:
         for k, v in kw.items():
             params.append(f"{k.upper()} " + encode_value_to_expr(v))
         pm = ", ".join(params)
-        query = f"COPY {dn.key} TO {encode_value_to_expr(p.uri)} WITH ({pm})"
+        query = f"COPY {dn.key} TO {encode_value_to_expr(p.path)} WITH ({pm})"
         self._con.execute(query)
 
     def _load_csv(  # noqa: C901
@@ -125,7 +108,7 @@ class DuckDBIO:
             ValueError("when csv has no header, columns must be specified"),
         )
         kw.pop("auto_detect", None)
-        params: List[str] = [encode_value_to_expr(p.uri_with_glob)]
+        params: List[str] = [encode_value_to_expr(p.path)]
         kw["header"] = 1 if header else 0
         kw["auto_detect"] = 1 if infer_schema else 0
         if infer_schema:
@@ -188,6 +171,7 @@ class DuckDBIO:
                 return DuckDataFrame(self._con.from_query(query))
 
     def _save_parquet(self, df: DuckDataFrame, p: FileParser, **kwargs: Any):
+        p.assert_no_glob()
         dn = TempTableName()
         df.native.create_view(dn.key)
         kw = ParamDict({k.lower(): v for k, v in kwargs.items()})
@@ -196,7 +180,7 @@ class DuckDBIO:
         for k, v in kw.items():
             params.append(f"{k.upper()} " + encode_value_to_expr(v))
         pm = ", ".join(params)
-        query = f"COPY {dn.key} TO {encode_value_to_expr(p.uri)}"
+        query = f"COPY {dn.key} TO {encode_value_to_expr(p.path)}"
         if len(params) > 0:
             query += f" WITH ({pm})"
         self._con.execute(query)
@@ -205,7 +189,7 @@ class DuckDBIO:
         self, p: FileParser, columns: Any = None, **kwargs: Any
     ) -> DuckDataFrame:
         kw = ParamDict({k.lower(): v for k, v in kwargs.items()})
-        params: List[str] = [encode_value_to_expr(p.uri_with_glob)]
+        params: List[str] = [encode_value_to_expr(p.path)]
         if isinstance(columns, list):
             cols = ", ".join(encode_column_names(columns))
         else:

fugue_duckdb/execution_engine.py

@@ -4,7 +4,6 @@ from typing import Any, Dict, Iterable, List, Optional, Union
 import duckdb
 from duckdb import DuckDBPyConnection, DuckDBPyRelation
 from triad import SerializableRLock
-from triad.collections.fs import FileSystem
 from triad.utils.assertion import assert_or_throw
 from triad.utils.schema import quote_name
 
@@ -195,10 +194,6 @@ class DuckExecutionEngine(ExecutionEngine):
     def log(self) -> logging.Logger:
         return self._native_engine.log
 
-    @property
-    def fs(self) -> FileSystem:
-        return self._native_engine.fs
-
     def create_default_sql_engine(self) -> SQLEngine:
         return DuckDBEngine(self)
 
@@ -488,7 +483,7 @@ class DuckExecutionEngine(ExecutionEngine):
         columns: Any = None,
         **kwargs: Any,
     ) -> LocalBoundedDataFrame:
-        dio = DuckDBIO(self.fs, self.connection)
+        dio = DuckDBIO(self.connection)
         return dio.load_df(path, format_hint, columns, **kwargs)
 
     def save_df(
@@ -504,7 +499,7 @@ class DuckExecutionEngine(ExecutionEngine):
         partition_spec = partition_spec or PartitionSpec()
         if not partition_spec.empty and not force_single:
             kwargs["partition_cols"] = partition_spec.partition_by
-        dio = DuckDBIO(self.fs, self.connection)
+        dio = DuckDBIO(self.connection)
         dio.save_df(_to_duck_df(self, df), path, format_hint, mode, **kwargs)
 
     def convert_yield_dataframe(self, df: DataFrame, as_local: bool) -> DataFrame:

fugue_ibis/execution_engine.py

@@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Optional, Type
 
 import ibis
 from ibis import BaseBackend
-from triad import FileSystem, assert_or_throw
+from triad import assert_or_throw
 
 from fugue import StructuredRawSQL
 from fugue.bag import Bag, LocalBag
@@ -375,10 +375,6 @@ class IbisExecutionEngine(ExecutionEngine):
     def log(self) -> logging.Logger:
         return self.non_ibis_engine.log
 
-    @property
-    def fs(self) -> FileSystem:
-        return self.non_ibis_engine.fs
-
     def get_current_parallelism(self) -> int:
         return self.non_ibis_engine.get_current_parallelism()
 

fugue_ray/_utils/io.py

@@ -4,23 +4,24 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Union
 
 import pyarrow as pa
 import ray.data as rd
-from fugue import ExecutionEngine
-from fugue._utils.io import FileParser, save_df
-from fugue.collections.partition import PartitionSpec
-from fugue.dataframe import DataFrame
-from fugue_ray.dataframe import RayDataFrame
 from pyarrow import csv as pacsv
 from pyarrow import json as pajson
 from ray.data.datasource import FileExtensionFilter
 from triad.collections import Schema
 from triad.collections.dict import ParamDict
 from triad.utils.assertion import assert_or_throw
+from triad.utils.io import exists, makedirs, rm
+
+from fugue import ExecutionEngine
+from fugue._utils.io import FileParser, save_df
+from fugue.collections.partition import PartitionSpec
+from fugue.dataframe import DataFrame
+from fugue_ray.dataframe import RayDataFrame
 
 
 class RayIO(object):
     def __init__(self, engine: ExecutionEngine):
         self._engine = engine
-        self._fs = engine.fs
         self._logger = engine.log
         self._loads: Dict[str, Callable[..., DataFrame]] = {
             "csv": self._load_csv,
@@ -49,7 +50,7 @@ class RayIO(object):
             len(fmts) == 1, NotImplementedError("can't support multiple formats")
         )
         fmt = fmts[0]
-        files = [f.uri for f in fp]
+        files = [f.path for f in fp]
         return self._loads[fmt](files, columns, **kwargs)
 
     def save_df(
@@ -63,24 +64,21 @@ class RayIO(object):
         **kwargs: Any,
     ) -> None:
         partition_spec = partition_spec or PartitionSpec()
-        if self._fs.exists(uri):
+        if exists(uri):
             assert_or_throw(mode == "overwrite", FileExistsError(uri))
             try:
-                self._fs.remove(uri)
-            except Exception:
-                try:
-                    self._fs.removetree(uri)
-                except Exception:  # pragma: no cover
-                    pass
+                rm(uri, recursive=True)
+            except Exception:  # pragma: no cover
+                pass
         p = FileParser(uri, format_hint)
         if not force_single:
             df = self._prepartition(df, partition_spec=partition_spec)
 
-            self._saves[p.file_format](df=df, uri=p.uri, **kwargs)
+            self._saves[p.file_format](df=df, uri=p.path, **kwargs)
         else:
             ldf = df.as_local()
-            self._fs.makedirs(os.path.dirname(uri), recreate=True)
-            save_df(ldf, uri, format_hint=format_hint, mode=mode, fs=self._fs, **kwargs)
+            makedirs(os.path.dirname(uri), exist_ok=True)
+            save_df(ldf, uri, format_hint=format_hint, mode=mode, **kwargs)
 
     def _save_parquet(
         self,