fugue 0.8.7.dev4__py3-none-any.whl → 0.8.7.dev6__py3-none-any.whl

{fugue-0.8.7.dev4.dist-info → fugue-0.8.7.dev6.dist-info}/RECORD

@@ -1,9 +1,9 @@
 fugue/__init__.py,sha256=xT5zuNZfRkjbA8a-uTT5oLK6hLGuezGZLWYBl6eS5J4,2749
-fugue/api.py,sha256=6_d3vYwJGAX7tW7NMhHB_NAX4aPsfzK2L06Zr2V78Ks,1240
+fugue/api.py,sha256=dLUrigFhDMB5x7cvlWSK8EyaY2o0AmhgPr0VRtfzSz0,1254
 fugue/constants.py,sha256=crd0VqX8WtBcjSUNwZDi2LDIEkhUMWOlSn73H8JI9ds,3385
 fugue/dev.py,sha256=GQCkezBBl4V0lVDWhGtUQKqomiCxgR9dMhfqj9C8cS8,1369
 fugue/exceptions.py,sha256=ylP8gkZL8ao_ZLinNYKv16FPyO_n7c29dN-4QChUxi0,1544
-fugue/plugins.py,sha256=SJ-jqs04StHIHJ65lgdGP0IDopVIGBDpmzHHllNK8p0,998
+fugue/plugins.py,sha256=kao-H5z-cRbujBKW1QC9IHUOBKxXMhpVQ6saIE7cXm8,1012
 fugue/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/registry.py,sha256=SNULGv08f37fRO-cIxFDmnVcod7ref2fNLSK6G7nVnI,868
 fugue/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -25,24 +25,24 @@ fugue/column/expressions.py,sha256=fdGX9oPCqJBuROFZqrOYVcwkjghdXT9ngaSTG5tW_i8,2
 fugue/column/functions.py,sha256=ygLyn2gp5lTdGbYqJXeGeMmRNhbm4-vfJvAY_Zt0pb0,9774
 fugue/column/sql.py,sha256=s_qTtHgnvRFqjhCWr7s595PTrHM-Pr9zHUQfU5xcTVA,17391
 fugue/dataframe/__init__.py,sha256=zm7TbsaJLIvfm7zymWm2LGcuJd3nxfGsFnQiyrSnenM,678
-fugue/dataframe/api.py,sha256=c5Err3c-ayl-k28IUi6kV_ClDWX30NpVNkv97hQKDac,9862
+fugue/dataframe/api.py,sha256=aWBvMaiSUxOvdQMfe79zHShWuPfLcgiWggC9HvVxvSE,11017
 fugue/dataframe/array_dataframe.py,sha256=4scWnmQ6sjy1A6o7IYdRc0VVutBEfcJrA1f9wkph4Kg,4440
-fugue/dataframe/arrow_dataframe.py,sha256=mJzrYBGs9mEMsHgxmnhDdiLUiOkcOs3YBAzHs75KFsI,10202
-fugue/dataframe/dataframe.py,sha256=a7jhYUaovN7w8vcJ-OU2AMfkfqxpvFF06cYWFqIJWqM,16418
+fugue/dataframe/arrow_dataframe.py,sha256=r5zcZBX_N6XO5dmixBkTCPgLcMmgDF022piZvrwRp_c,11485
+fugue/dataframe/dataframe.py,sha256=xmyG85i14A6LDRkNmPt29oYq7PJsq668s1QvFHK8PV4,16964
 fugue/dataframe/dataframe_iterable_dataframe.py,sha256=lx71KfaI4lsVKI-79buc-idaeT20JEMBOq21SQcAiY8,7259
 fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs,4160
-fugue/dataframe/function_wrapper.py,sha256=r6H1SQWaag2eSbJ50327t_bt7MZunbOMOl9OcOcQW2E,14827
-fugue/dataframe/iterable_dataframe.py,sha256=9g2BAF9A6QPbo63Si-trFq_9nPVqAD9vSePRCV71AfY,4620
-fugue/dataframe/pandas_dataframe.py,sha256=JNkr24h5gir1Msttx3lNfzFjwMqjHbjDswNynpCiizo,9158
-fugue/dataframe/utils.py,sha256=4l2Ag3iA9dh8zIbtyihe82X5WNB-6hbbRVvSlmJbSuY,9086
+fugue/dataframe/function_wrapper.py,sha256=V1eQMOn27UroEYT7_YiwoEF0RjZYIM0zkD3vfaMAQFs,14813
+fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
+fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
+fugue/dataframe/utils.py,sha256=VS1qLCr-9NEcEjaK-219rADJadDf6EfzYZCGRUpn1fY,11405
 fugue/dataset/__init__.py,sha256=5f2CAJ4xst6Z2o9Q2e2twfDOGUw8ZJoE2ild4JEU2pg,112
 fugue/dataset/api.py,sha256=DacI4L2w5NJ-eZ6nFxNMqmReEnb0WUXswbjVp7BeErk,2794
 fugue/dataset/dataset.py,sha256=jWXZqy3msMPFFkhas2PYJEX55ZAI3gk3Txq5f4-Qya4,4759
 fugue/execution/__init__.py,sha256=iZGxAznZz9piM3k4gp0tln97MDIBxdliLyNbD-0Zc48,427
 fugue/execution/api.py,sha256=KsFOLGdWQMlXmlQ5JRgRsbUeB64qzTVHxSEaunjiojo,39818
-fugue/execution/execution_engine.py,sha256=n-mw0k0QtK8FQgP4w4_NrWJbg0XvrR4sFn70tSaOi0I,47735
+fugue/execution/execution_engine.py,sha256=G_SsTmcuDcy6_azi_88lGzsOodiizu0JdWxebxgbqRg,47721
 fugue/execution/factory.py,sha256=5ICzfNh2QqqABuVyYLijY5-7LZgfRqczlaZN32p78bE,21003
-fugue/execution/native_execution_engine.py,sha256=_cXg7PTmDL4QvkcOnGhLFlVEZVZvGu9-wHPTBM0e-vI,14388
+fugue/execution/native_execution_engine.py,sha256=Mm9BVC3dEMS3IWRZe4YvGKp6_mmW7dLmoLMK5HgAPcs,14408
 fugue/extensions/__init__.py,sha256=y-uLKd6mZ8sZ_8-OdW6ELoBO_9IfC0gDmEbE_rMCvOA,599
 fugue/extensions/_utils.py,sha256=Bi3pYKy2Z6fG6_5BpwIWldxetassXpB4Zp8QamWB-wg,5173
 fugue/extensions/context.py,sha256=c_y2UttzzIFoQTOCV42VCdj2nqah33xYuBjbKNIOpx8,4262
@@ -86,9 +86,9 @@ fugue_contrib/viz/__init__.py,sha256=osgZx63Br-yMZImyEfYf9MVzJNM2Cqqke_-WsuDmG5M
 fugue_contrib/viz/_ext.py,sha256=Lu_DlS5DcmrFz27fHcKTCkhKyknVWcfS5kzZVVuO9xM,1345
 fugue_dask/__init__.py,sha256=2CcJ0AsN-k_f7dZ-yAyYpaICfUMPfH3l0FvUJSBzTr0,161
 fugue_dask/_constants.py,sha256=35UmTVITk21GhRyRlbJOwPPdQsytM_p_2NytOXEay18,510
-fugue_dask/_io.py,sha256=V-S6mA7VhDVeKfE46c6icZggwA6kobBNVsCUJsJTmTk,5836
-fugue_dask/_utils.py,sha256=uFoJAL95rmnBgieU2hPyqxFZGvR6ZJgPRMq5TAJqIBI,8520
-fugue_dask/dataframe.py,sha256=TdKjxhoQpsU5CvBTgO2c5Zo_4LfyelR0IK8bPgjAxcg,10218
+fugue_dask/_io.py,sha256=9G516yM6zQvSC5_JA6qHb3LwBDmhWcxK5sjFHrQ81zo,6012
+fugue_dask/_utils.py,sha256=n70N3wPPMz13Jh0GWJM3Je-TCYpU36yGP_YCwIHqUrc,8908
+fugue_dask/dataframe.py,sha256=MuG9TqCND7qI66lPvxzuomfE7yA4sW7DjrvbyvE6XEU,13471
 fugue_dask/execution_engine.py,sha256=XJp6wrdkaNh5pOpwt-Hjoa2sxgCOgusFRWrcqoCcaNM,21153
 fugue_dask/ibis_engine.py,sha256=kQdaG_KlZZ2AjtYETNCdTJOgtwI_eH0aGzLaAiIBbRI,2120
 fugue_dask/registry.py,sha256=7UTg_eie7zKlHYKMCyOo0TNn5y2TiIjE8kiS2PruHFc,2200
@@ -96,14 +96,14 @@ fugue_duckdb/__init__.py,sha256=nSNv-fxBAKD6W23EbMeV4dVRIaSTqr9DzQUWuVOES8s,379
 fugue_duckdb/_io.py,sha256=Sq228unVnroYTq4GX-Wnv22SLHC9Ji-aWgiqrfdu81w,8880
 fugue_duckdb/_utils.py,sha256=ElKbHUyn5fWSPGXsK57iqMzcqKtCf0c8pBVBYGe5Ql4,5020
 fugue_duckdb/dask.py,sha256=agoLzeB7Swxj2kVWfmXFbWD1NS2lbbTlnrjSkR8kKWY,5014
-fugue_duckdb/dataframe.py,sha256=vNZF2BC1sJpW3P5TVFTpU6C1Ddam81jPC_4i8kBuEpo,6512
+fugue_duckdb/dataframe.py,sha256=LRfTv7Y46wMM_IDYSP1R-5OXuHuBg8GHjPGFFt8u7l0,8444
 fugue_duckdb/execution_engine.py,sha256=fkkQb4Eh0m7SwKrTplVk2oQalLkNoj3CW0R12g01ofk,20536
 fugue_duckdb/ibis_engine.py,sha256=MrypeABozqwetKOpqtrmWvCJX2QPfBXhbSEhvK9vqmI,1990
 fugue_duckdb/registry.py,sha256=Dj0Tng1cXVT6Q7t-KxOky2k1dD9xSBjYGQmI26UgZPo,3095
 fugue_ibis/__init__.py,sha256=PcUt66KlLyGGicad7asq5j2U567_fhR0HzvWQBhV1VM,362
 fugue_ibis/_compat.py,sha256=zKdTaTfuC02eUIzZPkcd7oObnVBi_X5mQjQf7SDme3Y,246
 fugue_ibis/_utils.py,sha256=BUL5swA5FE4eQu0t5Z17hZVu9a2MFfxlFH6Ymy9xifg,6607
-fugue_ibis/dataframe.py,sha256=olGfVYY9n5wwPOZojS30Fs3XEOMlenCzX8fuR2WPaq4,7295
+fugue_ibis/dataframe.py,sha256=0Fb1vJjwEeffgoUCDfDGIMuSFaPgUJqcB-JqJOAALfs,7789
 fugue_ibis/execution_engine.py,sha256=p5zy0IBXiJgLi67RBHCRcHgZsaJMANdNSpUxz0k_6C0,18453
 fugue_ibis/extensions.py,sha256=H8l-SPfoqLuUoILtOuL2nccOpoL83zHeSoIhoqjtWQM,6905
 fugue_ibis/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -117,11 +117,11 @@ fugue_notebook/nbextension/description.yaml,sha256=CsXgx9CSLbAlO4Z1kvX9ejYA_TImP
 fugue_notebook/nbextension/main.js,sha256=Px2tQuBCNGEZOEBKsnfVruFEg-AxK7Tj0dY84ktub_U,3709
 fugue_polars/__init__.py,sha256=NDkjlbLhHPTjUaCAw6mAwIqeK3HSeh-z88s9dqmwheQ,61
 fugue_polars/_utils.py,sha256=7rGGWgB1-VqFwh4PcBLYk_5VNjd8FNOS4TDFyDVz2sg,159
-fugue_polars/polars_dataframe.py,sha256=Ll4ZUuRhAETWtmSf87KsdUCqZPiexFqy4FiPkvWQkN0,7348
+fugue_polars/polars_dataframe.py,sha256=8LQ0IB-JFFdjW2ltDzq8DfIbUC_jjjDr1YM29usJag0,8831
 fugue_polars/registry.py,sha256=gd6qQ-OxYtTAQFyvYbLDPXmSvCR-LW6n5K5ylgMY_7A,2950
 fugue_ray/__init__.py,sha256=HzEHfG2mpc0ugf3nf1Pdy15Bhg35K6maZpYejn1aoyI,119
 fugue_ray/_constants.py,sha256=vu5l1w-Wi-2V_nm0HLXKOYhh5HdWRCc5yQktO2XzhOg,569
-fugue_ray/dataframe.py,sha256=vyVShPnNtMef_KBsVP3iTHcssA_fm33-Y077c7S3J-A,10612
+fugue_ray/dataframe.py,sha256=7asw2qf9vm6vLBSzqghm9pUcNAppJOz5CkT7XyR0S5g,12514
 fugue_ray/execution_engine.py,sha256=NT_mnacijp1zskFbtganUwA3JNRPU-FNNvJswA6U_Yg,12607
 fugue_ray/registry.py,sha256=xJRAhbwNrg695EwghQDnVtTKi4YkqZ0_61BD4OAblSA,1685
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -130,21 +130,21 @@ fugue_ray/_utils/dataframe.py,sha256=_EadzS4rPom1A_cF0pqoPlwrNYZTfTwcyyu86_fFsqU
 fugue_ray/_utils/io.py,sha256=SFTU4qXubGEmO5IGZA5yHy8Hu4b9aFZ9-eTU4Qs-NsQ,8757
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
-fugue_spark/dataframe.py,sha256=-3kGdkuYSoM_l2xyXiEUtTjSfBfkLcg10hhWTQiORdI,9503
-fugue_spark/execution_engine.py,sha256=_0ldgIi4408LTCtMheelbnmx3p1_VHyc_zAMTLMj4eU,32928
+fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
+fugue_spark/execution_engine.py,sha256=rqgY9U1bpjh0GFNyNkuPcI7iV0xeipadURhNIir4w08,33147
 fugue_spark/ibis_engine.py,sha256=Yl5xxwROo1idcD2hFaylaI1IpmBUgbvOZRWtcrE0Zjo,1697
 fugue_spark/registry.py,sha256=kyIMk6dAiKRSKCHawQKyXu9DhZ24T6j3gL57TiOAZ8c,4162
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fugue_spark/_utils/convert.py,sha256=UlLGqwveT6H-O3EgC0r-2sLP5t5fUfBxdgbDqXUxdvE,8980
-fugue_spark/_utils/io.py,sha256=q-hQ2tlNa2AkWlGjJmoCLqran9jZLoyf2KZ5GP_3BIM,5511
+fugue_spark/_utils/convert.py,sha256=eRWkDYA4UO-FQu-2y4O80WEdawx7X_rIrWg55AlOiRc,10007
+fugue_spark/_utils/io.py,sha256=0ndQ70YlirPwGKjh5IDN6IdJxD26BnPpMonRob4dxII,5668
 fugue_spark/_utils/misc.py,sha256=o8dZmXOHnA7D_ps37vgGXTPTiSEG9LQzPKq7l-MG-qM,860
 fugue_spark/_utils/partition.py,sha256=iaesyO5f4uXhj1W-p91cD5ecPiGlu0bzh8gl2ce2Uvg,3618
 fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
 fugue_test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=eJ8jiBRbg61IkjUJa1r5PPHff_k2qHNg_ZQw1D4MIKY,78384
-fugue_test/dataframe_suite.py,sha256=mOr_x94H-Ylp0lJ-KBwHXJu-Q-qesqY3PzJxR9LI_Ko,18323
+fugue_test/builtin_suite.py,sha256=o8aMZTKa74nKBmcUTTBbliTJMtNbsXE9SPKZopS504o,78400
+fugue_test/dataframe_suite.py,sha256=LgB931CkASbGOrRQ9j92DGk9wPb__FoNusOk-HeqU9E,19165
 fugue_test/execution_suite.py,sha256=FI6UmwBvdoT1jkJRBqJT_Q0IDehFryvv00UL6jjxyAk,47689
 fugue_test/ibis_suite.py,sha256=Dk4AHVD00RcFsNm9VvJ4_4LOyFdGX30OnAtpO2SPruE,3529
 fugue_test/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -155,9 +155,9 @@ fugue_test/plugins/duckdb/fixtures.py,sha256=UxQbIMRbSrTZ3pgCmKZgd5wd1YvnVrqLSUP
 fugue_test/plugins/ray/__init__.py,sha256=nyKGW6xgTXtMhSs7yjgFNKO7mVboCNg63Bvdf39fO_I,55
 fugue_test/plugins/ray/fixtures.py,sha256=hZkvuo0AcD63XJl5JUroc9tm2LWHUPszg2zzY6FCSao,141
 fugue_version/__init__.py,sha256=vTwvdJOZi8jZb9U-Em7-d50qNDNPS2z51IXqRoojeNM,22
-fugue-0.8.7.dev4.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.8.7.dev4.dist-info/METADATA,sha256=q5KpH6iM_cZS-qZO357pbHw1saF1Dl03vustptvy9YY,17673
-fugue-0.8.7.dev4.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
-fugue-0.8.7.dev4.dist-info/entry_points.txt,sha256=US6kfp0GXKzBD0cACD9Senb90evV9xSeKEfkriLaU6M,536
-fugue-0.8.7.dev4.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.8.7.dev4.dist-info/RECORD,,
+fugue-0.8.7.dev6.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+fugue-0.8.7.dev6.dist-info/METADATA,sha256=0i4ibczIy_wEMtZ6vFvaCw40x5KmuQa6OsuBVWUTQyk,17860
+fugue-0.8.7.dev6.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+fugue-0.8.7.dev6.dist-info/entry_points.txt,sha256=N_BIIy3lSvF6Z32QE0yXTucgdHrPbUrOwH1zj7bZ0ow,536
+fugue-0.8.7.dev6.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.8.7.dev6.dist-info/RECORD,,

{fugue-0.8.7.dev4.dist-info → fugue-0.8.7.dev6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.1)
+Generator: bdist_wheel (0.41.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{fugue-0.8.7.dev4.dist-info → fugue-0.8.7.dev6.dist-info}/entry_points.txt

@@ -2,7 +2,7 @@
 dask = fugue_dask.registry [dask]
 dask_ibis = fugue_dask.ibis_engine [dask,ibis]
 duckdb = fugue_duckdb.registry [duckdb]
-duckdb_ibis = fugue_duckdb.ibis_engine [ibis,duckdb]
+duckdb_ibis = fugue_duckdb.ibis_engine [duckdb,ibis]
 ibis = fugue_ibis [ibis]
 polars = fugue_polars.registry [polars]
 ray = fugue_ray.registry [ray]

fugue_dask/_io.py

@@ -1,5 +1,6 @@
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
+import fsspec
 import fs as pfs
 import pandas as pd
 from dask import dataframe as dd
@@ -96,6 +97,8 @@ def _load_parquet(
 
 
 def _save_csv(df: DaskDataFrame, p: FileParser, **kwargs: Any) -> None:
+    fs, path = fsspec.core.url_to_fs(p.uri)
+    fs.makedirs(path, exist_ok=True)
     df.native.to_csv(
         pfs.path.combine(p.uri, "*.csv"), **{"index": False, "header": False, **kwargs}
     )
@@ -145,6 +148,8 @@ def _load_csv(  # noqa: C901
 
 
 def _save_json(df: DaskDataFrame, p: FileParser, **kwargs: Any) -> None:
+    fs, path = fsspec.core.url_to_fs(p.uri)
+    fs.makedirs(path, exist_ok=True)
     df.native.to_json(pfs.path.combine(p.uri, "*.json"), **kwargs)
 
 

fugue_dask/_utils.py

@@ -1,13 +1,14 @@
 import math
-from typing import Any, List, Optional, Tuple
+from typing import Any, Callable, List, Optional, Tuple, TypeVar
 
 import dask.dataframe as dd
 import numpy as np
 import pandas as pd
 import pyarrow as pa
 from dask.dataframe.core import DataFrame
+from dask.delayed import delayed
 from dask.distributed import Client, get_client
-from triad.utils.pandas_like import PandasLikeUtils, PD_UTILS
+from triad.utils.pandas_like import PD_UTILS, PandasLikeUtils
 from triad.utils.pyarrow import to_pandas_dtype
 
 import fugue.api as fa
@@ -16,6 +17,7 @@ from fugue.constants import FUGUE_CONF_DEFAULT_PARTITIONS
 from ._constants import FUGUE_DASK_CONF_DEFAULT_PARTITIONS
 
 _FUGUE_DASK_TEMP_IDX_COLUMN = "_fugue_dask_temp_index"
+T = TypeVar("T")
 
 
 def get_default_partitions() -> int:
@@ -28,6 +30,17 @@ def get_default_partitions() -> int:
     return n if n > 0 else fa.get_current_parallelism() * 2
 
 
+def collect(df: dd.DataFrame, func: Callable[[pd.DataFrame], T]) -> Tuple[T]:
+    """Compute each partition in parallel and collect the results
+
+    :param df: dask dataframe
+    :return: the collected result
+    """
+    dfs = df.to_delayed()
+    objs = [delayed(func)(df) for df in dfs]
+    return dd.compute(*objs)
+
+
 def hash_repartition(df: dd.DataFrame, num: int, cols: List[Any]) -> dd.DataFrame:
     """Repartition the dataframe by hashing the given columns
 

fugue_dask/dataframe.py

@@ -3,20 +3,21 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
 import dask.dataframe as dd
 import pandas as pd
 import pyarrow as pa
+from triad import assert_or_throw
 from triad.collections.schema import Schema
 from triad.utils.assertion import assert_arg_not_none
 from triad.utils.pandas_like import PD_UTILS
 from triad.utils.pyarrow import cast_pa_table
 
-from fugue.dataframe import (
-    ArrowDataFrame,
-    DataFrame,
-    LocalBoundedDataFrame,
-    PandasDataFrame,
-)
+from fugue.dataframe import DataFrame, LocalBoundedDataFrame, PandasDataFrame
 from fugue.dataframe.dataframe import _input_schema
+from fugue.dataframe.pandas_dataframe import _pd_as_dicts
 from fugue.exceptions import FugueDataFrameOperationError
 from fugue.plugins import (
+    as_array,
+    as_array_iterable,
+    as_dict_iterable,
+    as_dicts,
     as_local_bounded,
     count,
     drop_columns,
@@ -32,7 +33,7 @@ from fugue.plugins import (
 )
 
 from ._constants import FUGUE_DASK_USE_ARROW
-from ._utils import DASK_UTILS, get_default_partitions
+from ._utils import DASK_UTILS, collect, get_default_partitions
 
 
 class DaskDataFrame(DataFrame):
@@ -150,8 +151,16 @@ class DaskDataFrame(DataFrame):
         )
 
     def as_arrow(self, type_safe: bool = False) -> pa.Table:
-        adf = pa.Table.from_pandas(self.native.compute().reset_index(drop=True))
-        return cast_pa_table(adf, self.schema.pa_schema)
+        schema = self.schema.pa_schema
+        return pa.concat_tables(
+            collect(
+                self.native,
+                lambda df: cast_pa_table(
+                    pa.Table.from_pandas(df.reset_index(drop=True), schema=schema),
+                    schema=schema,
+                ),
+            )
+        )
 
     def rename(self, columns: Dict[str, str]) -> DataFrame:
         try:
@@ -170,17 +179,28 @@ class DaskDataFrame(DataFrame):
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
-        df: DataFrame = self
-        if columns is not None:
-            df = df[columns]
-        return ArrowDataFrame(df.as_pandas(), schema=df.schema).as_array(
-            type_safe=type_safe
-        )
+        chunks = _to_array_chunks(self.native, columns, type_safe, schema=self.schema)
+        res: List[List[Any]] = []
+        for x in chunks:
+            res += x
+        return res
 
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:
-        yield from self.as_array(columns=columns, type_safe=type_safe)
+        chunks = _to_array_chunks(self.native, columns, type_safe, schema=self.schema)
+        for x in chunks:
+            yield from x
+
+    def as_dicts(
+        self, columns: Optional[List[str]] = None, type_safe: bool = False
+    ) -> List[Dict[str, Any]]:
+        return _dd_as_dicts(self.native, columns)
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None, type_safe: bool = False
+    ) -> Iterable[Dict[str, Any]]:
+        yield from _dd_as_dict_iterable(self.native, columns)
 
     def head(
         self, n: int, columns: Optional[List[str]] = None
@@ -197,8 +217,11 @@ class DaskDataFrame(DataFrame):
             assert_arg_not_none(schema, "schema")
             return pdf, schema
         DASK_UTILS.ensure_compatible(pdf)
-        pschema = Schema(DASK_UTILS.to_schema(pdf))
-        if schema is None or pschema == schema:
+        # when pdf contains bytes, or any object types, and schema contains str
+        # there is no way to get the real schema of the pdf, (pschema will contain
+        # strs instead of the real types) so we have to force cast it to the schema
+        if schema is None:
+            pschema = Schema(DASK_UTILS.to_schema(pdf))
             return pdf, pschema.assert_not_empty()
         pdf = pdf[schema.assert_not_empty().names]
         return (
@@ -295,6 +318,48 @@ def _dd_head(
     return PandasDataFrame(res) if as_fugue else res
 
 
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_array(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Any]:
+    chunks = _to_array_chunks(df, columns, type_safe)
+    res: List[List[Any]] = []
+    for x in chunks:
+        res += x
+    return res
+
+
+@as_array_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_array_iterable(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Any]:
+    chunks = _to_array_chunks(df, columns, type_safe)
+    for x in chunks:
+        yield from x
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_dicts(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Dict[str, Any]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df[columns]
+    res: List[Dict[str, Any]] = []
+    for x in collect(_df, lambda df: _pd_as_dicts(df, columns)):
+        res += x
+    return res
+
+
+@as_dict_iterable.candidate(lambda df, *args, **kwargs: isinstance(df, dd.DataFrame))
+def _dd_as_dict_iterable(
+    df: dd.DataFrame, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Dict[str, Any]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df[columns]
+    for x in collect(_df, lambda df: _pd_as_dicts(df, columns)):
+        yield from x
+
+
 def _assert_no_missing(df: dd.DataFrame, columns: Iterable[Any]) -> None:
     missing = set(columns) - set(df.columns)
     if len(missing) > 0:
@@ -303,3 +368,25 @@ def _assert_no_missing(df: dd.DataFrame, columns: Iterable[Any]) -> None:
 
 def _adjust_df(res: dd.DataFrame, as_fugue: bool):
     return res if not as_fugue else DaskDataFrame(res)
+
+
+def _to_array_chunks(
+    df: dd.DataFrame,
+    columns: Optional[List[str]] = None,
+    type_safe: bool = False,
+    schema: Optional[Schema] = None,
+) -> Tuple[List[Any]]:
+    assert_or_throw(columns is None or len(columns) > 0, ValueError("empty columns"))
+    _df = df if columns is None or len(columns) == 0 else df[columns]
+
+    def _to_list(pdf: pd.DataFrame) -> List[Any]:
+        return list(
+            PD_UTILS.as_array_iterable(
+                pdf,
+                schema=None if schema is None else schema.pa_schema,
+                columns=columns,
+                type_safe=type_safe,
+            )
+        )
+
+    return collect(_df, _to_list)

fugue_duckdb/dataframe.py

@@ -3,21 +3,33 @@ from typing import Any, Dict, Iterable, List, Optional
 import pandas as pd
 import pyarrow as pa
 from duckdb import DuckDBPyRelation
-from triad import Schema
+from triad import Schema, assert_or_throw
 from triad.utils.pyarrow import LARGE_TYPES_REPLACEMENT, replace_types_in_table
 
-from fugue import ArrayDataFrame, ArrowDataFrame, DataFrame, LocalBoundedDataFrame
+from fugue import ArrowDataFrame, DataFrame, LocalBoundedDataFrame
 from fugue.dataframe.arrow_dataframe import _pa_table_as_pandas
+from fugue.dataframe.utils import (
+    pa_table_as_array,
+    pa_table_as_array_iterable,
+    pa_table_as_dict_iterable,
+    pa_table_as_dicts,
+)
 from fugue.exceptions import FugueDataFrameOperationError, FugueDatasetEmptyError
 from fugue.plugins import (
+    as_array,
+    as_array_iterable,
     as_arrow,
+    as_dict_iterable,
+    as_dicts,
     as_fugue_dataset,
     as_local_bounded,
     as_pandas,
+    drop_columns,
     get_column_names,
     get_num_partitions,
     get_schema,
     is_df,
+    select_columns,
 )
 
 from ._utils import encode_column_name, to_duck_type, to_pa_type
@@ -59,13 +71,10 @@ class DuckDataFrame(LocalBoundedDataFrame):
         return len(self._rel)
 
     def _drop_cols(self, cols: List[str]) -> DataFrame:
-        cols = [col for col in self._rel.columns if col not in cols]
-        rel = self._rel.project(",".join(encode_column_name(n) for n in cols))
-        return DuckDataFrame(rel)
+        return DuckDataFrame(_drop_duckdb_columns(self._rel, cols))
 
     def _select_cols(self, keys: List[Any]) -> DataFrame:
-        rel = self._rel.project(",".join(encode_column_name(n) for n in keys))
-        return DuckDataFrame(rel)
+        return DuckDataFrame(_select_duckdb_columns(self._rel, keys))
 
     def rename(self, columns: Dict[str, str]) -> DataFrame:
         _assert_no_missing(self._rel, columns.keys())
@@ -109,38 +118,29 @@ class DuckDataFrame(LocalBoundedDataFrame):
     def as_array(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> List[Any]:
-        if columns is not None:
-            return self[columns].as_array(type_safe=type_safe)
-        return self._fetchall(self._rel)
+        return _duck_as_array(self._rel, columns=columns, type_safe=type_safe)
 
     def as_array_iterable(
         self, columns: Optional[List[str]] = None, type_safe: bool = False
     ) -> Iterable[Any]:
-        if columns is not None:
-            yield from self[columns].as_array_iterable(type_safe=type_safe)
-        else:
-            yield from self._fetchall(self._rel)
+        yield from _duck_as_array_iterable(
+            self._rel, columns=columns, type_safe=type_safe
+        )
+
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        return _duck_as_dicts(self._rel, columns=columns)
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None
+    ) -> Iterable[Dict[str, Any]]:
+        yield from _duck_as_dict_iterable(self._rel, columns=columns)
 
     def head(
         self, n: int, columns: Optional[List[str]] = None
     ) -> LocalBoundedDataFrame:
         if columns is not None:
             return self[columns].head(n)
-        return ArrayDataFrame(self._fetchall(self._rel.limit(n)), schema=self.schema)
-
-    def _fetchall(self, rel: DuckDBPyRelation) -> List[List[Any]]:
-        map_pos = [i for i, t in enumerate(self.schema.types) if pa.types.is_map(t)]
-        if len(map_pos) == 0:
-            return [list(x) for x in rel.fetchall()]
-        else:
-
-            def to_list(row: Any) -> List[Any]:
-                res = list(row)
-                for p in map_pos:
-                    res[p] = list(zip(row[p]["key"], row[p]["value"]))
-                return res
-
-            return [to_list(x) for x in rel.fetchall()]
+        return ArrowDataFrame(_duck_as_arrow(self._rel.limit(n)))
 
 
 @as_fugue_dataset.candidate(lambda df, **kwargs: isinstance(df, DuckDBPyRelation))
@@ -186,6 +186,64 @@ def _get_duckdb_columns(df: DuckDBPyRelation) -> List[Any]:
     return list(df.columns)
 
 
+@select_columns.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _select_duckdb_columns(
+    df: DuckDBPyRelation, columns: List[Any]
+) -> DuckDBPyRelation:
+    if len(columns) == 0:
+        raise FugueDataFrameOperationError("must select at least one column")
+    _assert_no_missing(df, columns)
+    return df.project(",".join(encode_column_name(n) for n in columns))
+
+
+@drop_columns.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _drop_duckdb_columns(df: DuckDBPyRelation, columns: List[str]) -> DuckDBPyRelation:
+    # if len(columns) == 0:
+    #   return df
+    _columns = {c: 1 for c in columns}
+    cols = [col for col in df.columns if _columns.pop(col, None) is None]
+    assert_or_throw(
+        len(cols) > 0, FugueDataFrameOperationError("must keep at least one column")
+    )
+    assert_or_throw(
+        len(_columns) == 0,
+        FugueDataFrameOperationError("found nonexistent columns {_columns}"),
+    )
+    return df.project(",".join(encode_column_name(n) for n in cols))
+
+
+@as_array.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _duck_as_array(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> List[Any]:
+    return pa_table_as_array(df.arrow(), columns=columns)
+
+
+@as_array_iterable.candidate(
+    lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation)
+)
+def _duck_as_array_iterable(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None, type_safe: bool = False
+) -> Iterable[Any]:
+    yield from pa_table_as_array_iterable(df.arrow(), columns=columns)
+
+
+@as_dicts.candidate(lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation))
+def _duck_as_dicts(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None
+) -> List[Dict[str, Any]]:
+    return pa_table_as_dicts(df.arrow(), columns=columns)
+
+
+@as_dict_iterable.candidate(
+    lambda df, *args, **kwargs: isinstance(df, DuckDBPyRelation)
+)
+def _duck_as_dict_iterable(
+    df: DuckDBPyRelation, columns: Optional[List[str]] = None
+) -> Iterable[Dict[str, Any]]:
+    yield from pa_table_as_dict_iterable(df.arrow(), columns=columns)
+
+
 def _assert_no_missing(df: DuckDBPyRelation, columns: Iterable[Any]) -> None:
     missing = set(columns) - set(df.columns)
     if len(missing) > 0:

fugue_ibis/dataframe.py

@@ -143,6 +143,19 @@ class IbisDataFrame(DataFrame):
                 type_safe=type_safe
             )
 
+    def as_dicts(self, columns: Optional[List[str]] = None) -> List[Dict[str, Any]]:
+        if columns is not None:
+            return self[columns].as_dicts()
+        return self.as_local().as_dicts()
+
+    def as_dict_iterable(
+        self, columns: Optional[List[str]] = None
+    ) -> Iterable[Dict[str, Any]]:
+        if columns is not None:
+            yield from self[columns].as_dict_iterable()
+        else:
+            yield from self._to_iterable_df(self._table).as_dict_iterable()
+
     def head(
         self, n: int, columns: Optional[List[str]] = None
     ) -> LocalBoundedDataFrame: