fugue 0.8.7.dev7__py3-none-any.whl → 0.9.0__py3-none-any.whl

{fugue-0.8.7.dev7.dist-info → fugue-0.9.0.dist-info}/RECORD

@@ -18,7 +18,7 @@ fugue/bag/array_bag.py,sha256=b0UdDPmZpEAI3R0SBbZVOLVLAwMQnBCFeYDEpFWen14,1111
 fugue/bag/bag.py,sha256=sNBAzPmEh5fEm8ME8NEEOOre6l58ri6oouVBWwafqTc,3018
 fugue/collections/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue/collections/partition.py,sha256=rPkU-3y6E598Q7wvE-jTSbSwWh3fzIVxdwPpbQvWS-M,17257
-fugue/collections/sql.py,sha256=mXxgOr_BAwPWb5DAd0PZuNtCWpMWxIFp8uulVikvlZ8,4947
+fugue/collections/sql.py,sha256=3MjnuQMPuUMq55n-EypikkRqcpOCZtOjp7S2fs7ujAA,4955
 fugue/collections/yielded.py,sha256=KAvCXAZpeuErGww7Y217_F7M2zv9G5hfdl2AWiO7wEM,2040
 fugue/column/__init__.py,sha256=aoZwwzyJtNL-duLxzU2sNGoaKikWd-yesbigE_Wj29s,208
 fugue/column/expressions.py,sha256=fdGX9oPCqJBuROFZqrOYVcwkjghdXT9ngaSTG5tW_i8,26544
@@ -34,7 +34,7 @@ fugue/dataframe/dataframes.py,sha256=tBSpHsENgbcdOJ0Jgst6PTKbjG7_uoFJch96oTlaQIs
 fugue/dataframe/function_wrapper.py,sha256=V1eQMOn27UroEYT7_YiwoEF0RjZYIM0zkD3vfaMAQFs,14813
 fugue/dataframe/iterable_dataframe.py,sha256=TcOoNKa4jNbHbvAZ0XAhtMmGcioygIHPxI9budDtenQ,4758
 fugue/dataframe/pandas_dataframe.py,sha256=0L0wYCGhD2BpQbruoT07Ox9iQM5YLHLNrcgzudc-yKs,11633
-fugue/dataframe/utils.py,sha256=shN1eHYTnPhb38BHEpLlCdLSzX_qpoQ3-fsDgu1hCzQ,10840
+fugue/dataframe/utils.py,sha256=bA_otOJt9oju1yq5gtn21L_GDT_pUgNc6luYuBIhbUQ,10488
 fugue/dataset/__init__.py,sha256=5f2CAJ4xst6Z2o9Q2e2twfDOGUw8ZJoE2ild4JEU2pg,112
 fugue/dataset/api.py,sha256=DacI4L2w5NJ-eZ6nFxNMqmReEnb0WUXswbjVp7BeErk,2794
 fugue/dataset/dataset.py,sha256=jWXZqy3msMPFFkhas2PYJEX55ZAI3gk3Txq5f4-Qya4,4759
@@ -71,6 +71,9 @@ fugue/sql/_utils.py,sha256=khpjGeFCVlaqf2JIYvS4TVTJO3fe5-8bEsvy6AIP_5Q,2083
 fugue/sql/_visitors.py,sha256=2pc0J-AHJAiIexsKgNjcgrCGOyhC3_7rzonSgtjy--k,33844
 fugue/sql/api.py,sha256=l2I9CAy_W2oFFTct9fDPLyXF0LiDxQhMx5O8jBHTAxU,10050
 fugue/sql/workflow.py,sha256=S1pOhp0b0t6johFAJWmj6xUB7Ti5LQgNABpAzmLGjrQ,3010
+fugue/test/__init__.py,sha256=hvVrNbJYkWI_6otpILneyTjUafxURaA4obK6AoDyCUw,250
+fugue/test/pandas_tester.py,sha256=_w6rFqlzZKjBtmFf-08a4C97W5xtqGw5XorLhj6Zyoo,622
+fugue/test/plugins.py,sha256=GLZia5GCmy0eBVGNbIqTbX7Ou3euf2SY4litKgdigwY,12318
 fugue/workflow/__init__.py,sha256=tXM_KYO8Q358W6qAVlwhIQIaYNRDgZtTubrIEX4QMgM,229
 fugue/workflow/_checkpoint.py,sha256=tt5Iv7c5ZStC0MD1inItksQ0GuK0ViniA3nvrgym-5c,5681
 fugue/workflow/_tasks.py,sha256=Zq_jXJO_VaF8DrWUuBiwO2Y3OVuhsiOQdzP4VBsp7Fo,11826
@@ -86,29 +89,25 @@ fugue_contrib/viz/__init__.py,sha256=osgZx63Br-yMZImyEfYf9MVzJNM2Cqqke_-WsuDmG5M
 fugue_contrib/viz/_ext.py,sha256=Lu_DlS5DcmrFz27fHcKTCkhKyknVWcfS5kzZVVuO9xM,1345
 fugue_dask/__init__.py,sha256=2CcJ0AsN-k_f7dZ-yAyYpaICfUMPfH3l0FvUJSBzTr0,161
 fugue_dask/_constants.py,sha256=35UmTVITk21GhRyRlbJOwPPdQsytM_p_2NytOXEay18,510
-fugue_dask/_io.py,sha256=HmL3Q2lRSptX1-GwiB3MN2VpjTRfmVKD8TDZkhS4x5c,5818
-fugue_dask/_utils.py,sha256=n70N3wPPMz13Jh0GWJM3Je-TCYpU36yGP_YCwIHqUrc,8908
+fugue_dask/_io.py,sha256=pl4F7mbVgP7Rwh1FFG7xfOz2TBZRUj1l3lLvDY4jOf4,6020
+fugue_dask/_utils.py,sha256=1uplEqvpCDZDp2YdwJxa6cuGScpgG9VvN3057J02bys,8956
 fugue_dask/dataframe.py,sha256=MuG9TqCND7qI66lPvxzuomfE7yA4sW7DjrvbyvE6XEU,13471
-fugue_dask/execution_engine.py,sha256=PAClUP9lCdn2Aajt2AsoFOsgO-95WcdRDKkjNSbVbzA,20980
-fugue_dask/ibis_engine.py,sha256=kQdaG_KlZZ2AjtYETNCdTJOgtwI_eH0aGzLaAiIBbRI,2120
-fugue_dask/registry.py,sha256=7UTg_eie7zKlHYKMCyOo0TNn5y2TiIjE8kiS2PruHFc,2200
-fugue_duckdb/__init__.py,sha256=nSNv-fxBAKD6W23EbMeV4dVRIaSTqr9DzQUWuVOES8s,379
-fugue_duckdb/_io.py,sha256=E35_GoD1uGuuAMOY4H8E2j-UazdAgTmLp4lLWqJrNsE,8437
+fugue_dask/execution_engine.py,sha256=60IiwYRBVhN-pX3v6i9BZ8Pa4bcSh5UoklvCScM_XAM,21361
+fugue_dask/registry.py,sha256=jepWKH55VWNIWV3pOF5vpCl2OpO0rI1IULx5GM2Gk6w,2274
+fugue_dask/tester.py,sha256=E7BZjgFpJgrHsLMKzvSO5im5OwocYcratjzulJSQZl0,718
+fugue_duckdb/__init__.py,sha256=ZzhmAWbROR1YL9Kmlt7OlwkgPZzFhsSdwLV2pFmAqGI,268
+fugue_duckdb/_io.py,sha256=vnd8m8C6XeMCBJBbAdA5h695NMfsduQrvONyS0HcEFA,8475
 fugue_duckdb/_utils.py,sha256=ElKbHUyn5fWSPGXsK57iqMzcqKtCf0c8pBVBYGe5Ql4,5020
 fugue_duckdb/dask.py,sha256=agoLzeB7Swxj2kVWfmXFbWD1NS2lbbTlnrjSkR8kKWY,5014
 fugue_duckdb/dataframe.py,sha256=LRfTv7Y46wMM_IDYSP1R-5OXuHuBg8GHjPGFFt8u7l0,8444
 fugue_duckdb/execution_engine.py,sha256=IZDmSAtOMJGvulTStxjTmsqJyI5QRNyxBgSMlFMSrBI,20389
-fugue_duckdb/ibis_engine.py,sha256=MrypeABozqwetKOpqtrmWvCJX2QPfBXhbSEhvK9vqmI,1990
-fugue_duckdb/registry.py,sha256=Dj0Tng1cXVT6Q7t-KxOky2k1dD9xSBjYGQmI26UgZPo,3095
-fugue_ibis/__init__.py,sha256=PcUt66KlLyGGicad7asq5j2U567_fhR0HzvWQBhV1VM,362
+fugue_duckdb/registry.py,sha256=9_41KO42kXqcjF4yParQ5JGyg5TckcbhH-Q2IlGpSho,3987
+fugue_duckdb/tester.py,sha256=MzTkv3sdOwOjI59LRrSGGl4w59Njv3OArTU5kSRL-P0,1526
+fugue_ibis/__init__.py,sha256=z7TkK7M2_0p9XO6jQATNDgT0aHXn5k69Ttz2ga-eQG8,190
 fugue_ibis/_compat.py,sha256=zKdTaTfuC02eUIzZPkcd7oObnVBi_X5mQjQf7SDme3Y,246
 fugue_ibis/_utils.py,sha256=BUL5swA5FE4eQu0t5Z17hZVu9a2MFfxlFH6Ymy9xifg,6607
-fugue_ibis/dataframe.py,sha256=0Fb1vJjwEeffgoUCDfDGIMuSFaPgUJqcB-JqJOAALfs,7789
-fugue_ibis/execution_engine.py,sha256=0GIjjMmitCKhjasAKFiFUCCUBNdxAiU0b61RsmFyhIk,18355
-fugue_ibis/extensions.py,sha256=H8l-SPfoqLuUoILtOuL2nccOpoL83zHeSoIhoqjtWQM,6905
-fugue_ibis/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fugue_ibis/execution/ibis_engine.py,sha256=-HdPnIFWD83n5WITdzJiu4attH7GOcO041wkT5Y5ChA,1499
-fugue_ibis/execution/pandas_backend.py,sha256=r6Pfs06FXrWukSbK2uz5gsCDG6a4wrk4sWIJ9acSuNU,1615
+fugue_ibis/dataframe.py,sha256=k4Q6qBLBIADF5YhbvaDplXO7OkMZSHuf_Wg5o-AusEI,7796
+fugue_ibis/execution_engine.py,sha256=5I-ou5xPdomVu-srdvidvP8f7wDYbGrCV_lGffZa_ac,18679
 fugue_notebook/__init__.py,sha256=9r_-2uxu1lBeZ8GgpYCKom_OZy2soIOYZajg7JDO-HY,4326
 fugue_notebook/env.py,sha256=TYiTxYPFi-BVJJY49jDsvw9mddhK8WrifeRxBke30I8,4773
 fugue_notebook/nbextension/README.md,sha256=QLnr957YeGfwzy2r4c4qbZPaXyCbyGrKPvcqSBQYSnU,123
@@ -120,20 +119,21 @@ fugue_polars/_utils.py,sha256=7rGGWgB1-VqFwh4PcBLYk_5VNjd8FNOS4TDFyDVz2sg,159
 fugue_polars/polars_dataframe.py,sha256=8LQ0IB-JFFdjW2ltDzq8DfIbUC_jjjDr1YM29usJag0,8831
 fugue_polars/registry.py,sha256=gd6qQ-OxYtTAQFyvYbLDPXmSvCR-LW6n5K5ylgMY_7A,2950
 fugue_ray/__init__.py,sha256=HzEHfG2mpc0ugf3nf1Pdy15Bhg35K6maZpYejn1aoyI,119
-fugue_ray/_constants.py,sha256=vu5l1w-Wi-2V_nm0HLXKOYhh5HdWRCc5yQktO2XzhOg,569
+fugue_ray/_constants.py,sha256=RHlaVKyjQnwdbo5mFO_GBtQZcz5GvWcCbkOkLfVTQ1A,565
 fugue_ray/dataframe.py,sha256=7asw2qf9vm6vLBSzqghm9pUcNAppJOz5CkT7XyR0S5g,12514
-fugue_ray/execution_engine.py,sha256=NT_mnacijp1zskFbtganUwA3JNRPU-FNNvJswA6U_Yg,12607
-fugue_ray/registry.py,sha256=xJRAhbwNrg695EwghQDnVtTKi4YkqZ0_61BD4OAblSA,1685
+fugue_ray/execution_engine.py,sha256=PZlWbmdCwTPfZJhN2I-44JW7so8NVCFFumaKIhJLfoI,12566
+fugue_ray/registry.py,sha256=TS-HWy2IUozp6_A0vqc8_ZdVUT_Z9yVjG6e1gbbgy2A,1757
+fugue_ray/tester.py,sha256=oTA_xOzvQhJU3ohc4hsVpZc0zv4bwJn1c8a9u8kcuIs,537
 fugue_ray/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_ray/_utils/cluster.py,sha256=3T3Gyra6lAHlzktta-Ro35j6YZQfH6fNrj2hC5ATF9k,621
-fugue_ray/_utils/dataframe.py,sha256=_EadzS4rPom1A_cF0pqoPlwrNYZTfTwcyyu86_fFsqU,4400
-fugue_ray/_utils/io.py,sha256=4FfPS2DMeIHvbzGoJ_iPvwwVr7lZHXRoJZxceNZ4EHQ,8647
+fugue_ray/_utils/dataframe.py,sha256=5c4duGV--mdLkKrbJRgjDWvVcp9BegA3yX16pmYDYLE,3954
+fugue_ray/_utils/io.py,sha256=3hFNDeBuh4bfCud40ZsGrGZLSvCSuxL_1VlqCTnn6RA,9794
 fugue_spark/__init__.py,sha256=rvrMpFs9socMgyH_58gLbnAqmirBf5oidXoO4cekW6U,165
 fugue_spark/_constants.py,sha256=K2uLQfjvMxXk75K-7_Wn47Alpwq5rW57BtECAUrOeqA,177
 fugue_spark/dataframe.py,sha256=lYa8FizM3p_lsKYFR49FazkVZMJKyi2LABKTpP5YBLo,12006
-fugue_spark/execution_engine.py,sha256=KPmBtH4zioXdWsvnPow4fOPQh8Yj0cn6yCJyKdbT544,33023
-fugue_spark/ibis_engine.py,sha256=Yl5xxwROo1idcD2hFaylaI1IpmBUgbvOZRWtcrE0Zjo,1697
-fugue_spark/registry.py,sha256=kyIMk6dAiKRSKCHawQKyXu9DhZ24T6j3gL57TiOAZ8c,4162
+fugue_spark/execution_engine.py,sha256=YBMtNxCvpy77xICFSg9PHMa6feNoYhWEZe8MmxznX4U,33048
+fugue_spark/registry.py,sha256=_NmiV2cOooYK0YmqATEnNkPEMT9suUMtuecw2NNbIIk,4530
+fugue_spark/tester.py,sha256=VX003yGNlBukaZTQSN-w7XvgSk4rqxrWQIzno0dWrXg,2481
 fugue_spark/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fugue_spark/_utils/convert.py,sha256=eRWkDYA4UO-FQu-2y4O80WEdawx7X_rIrWg55AlOiRc,10007
 fugue_spark/_utils/io.py,sha256=OdUezKpB29Lx9aUS2k9x0xUAGZrmgMZyQYGPEeHk7rQ,5574
@@ -141,25 +141,16 @@ fugue_spark/_utils/misc.py,sha256=o8dZmXOHnA7D_ps37vgGXTPTiSEG9LQzPKq7l-MG-qM,86
 fugue_spark/_utils/partition.py,sha256=iaesyO5f4uXhj1W-p91cD5ecPiGlu0bzh8gl2ce2Uvg,3618
 fugue_sql/__init__.py,sha256=Cmr7w0Efr7PzoXdQzdJfc4Dgqd69qKqcHZZodENq7EU,287
 fugue_sql/exceptions.py,sha256=ltS0MC8gMnVVrJbQiOZ0kRUWvVQ2LTx33dCW3ugqtb0,260
-fugue_test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fugue_test/__init__.py,sha256=xoQuVobhU64uyODRdnzf6MSWe9lw5khkhpJ2atvADoc,2315
 fugue_test/bag_suite.py,sha256=WbDCFjuAHYoJh4GXSPiSJxOoOwE1VMtYpJ3lQrsUK-Y,2483
-fugue_test/builtin_suite.py,sha256=3uSY484Jl2985UoJravD4C-SlKBH0WwTWFobp4Pqgzg,78399
-fugue_test/dataframe_suite.py,sha256=LgB931CkASbGOrRQ9j92DGk9wPb__FoNusOk-HeqU9E,19165
-fugue_test/execution_suite.py,sha256=ClZUYt2R560LN4DZM_OP9cA5jaHmz3-u_BC3A0C24fQ,47472
-fugue_test/ibis_suite.py,sha256=Dk4AHVD00RcFsNm9VvJ4_4LOyFdGX30OnAtpO2SPruE,3529
-fugue_test/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fugue_test/plugins/dask/__init__.py,sha256=LQVyNgGRvZrKUrNNV1z1X1GyIL3qJoxvNjFfpFzNVCc,55
-fugue_test/plugins/dask/fixtures.py,sha256=uf5gkO30L5-LvxpEpBjG4_bNUrpkemHvyVPxDHgMSGM,354
-fugue_test/plugins/duckdb/__init__.py,sha256=WXtNYQpbO0JScPpIA3QREv8cwOZP2GDOgGOtJKgpTVM,61
-fugue_test/plugins/duckdb/fixtures.py,sha256=UxQbIMRbSrTZ3pgCmKZgd5wd1YvnVrqLSUPaO8UYrSE,165
-fugue_test/plugins/misc/__init__.py,sha256=0SZvyo0xlw5NDbJly4yjaNDqL9M4D2Jsg33HCWE40q8,49
-fugue_test/plugins/misc/fixtures.py,sha256=GrD9WTTtcIDCWLHn-ToVv8pUiUCGCSczgs9bodWKo7c,353
-fugue_test/plugins/ray/__init__.py,sha256=nyKGW6xgTXtMhSs7yjgFNKO7mVboCNg63Bvdf39fO_I,55
-fugue_test/plugins/ray/fixtures.py,sha256=hZkvuo0AcD63XJl5JUroc9tm2LWHUPszg2zzY6FCSao,141
-fugue_version/__init__.py,sha256=vTwvdJOZi8jZb9U-Em7-d50qNDNPS2z51IXqRoojeNM,22
-fugue-0.8.7.dev7.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-fugue-0.8.7.dev7.dist-info/METADATA,sha256=nSp1i8apniEEe6U09_5RA8K89P40c7M5Gn9l6ofLTHQ,17860
-fugue-0.8.7.dev7.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-fugue-0.8.7.dev7.dist-info/entry_points.txt,sha256=Xrl3ISyVKAFIPn1klqeGsL9DinzoYqfqBkOT4qAVBNA,578
-fugue-0.8.7.dev7.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
-fugue-0.8.7.dev7.dist-info/RECORD,,
+fugue_test/builtin_suite.py,sha256=cOkZG6w1RHhWWxtjQhZClZQaGT6haNd576BoUmNC_cA,77960
+fugue_test/dataframe_suite.py,sha256=7ym4sshDUly6004cq1UlppqDVtbwxD6CKxR4Lu70i0s,18994
+fugue_test/execution_suite.py,sha256=jcSSoKqTGbeWzTxkyYU-8i2zJAjzuXn7BqE8ul-JjIc,48646
+fugue_test/fixtures.py,sha256=8Pev-mxRZOWwTFlsGjcSZ0iIs78zyWbp5tq4KG1wyvk,1432
+fugue_version/__init__.py,sha256=H9NWRZb7NbeRRPLP_V1fARmLNXranorVM-OOY-8_2ug,22
+fugue-0.9.0.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+fugue-0.9.0.dist-info/METADATA,sha256=Ce9dp1NghY5AbNc4MuMD1VeGtRsrisvAhwgVajZY-ZM,18380
+fugue-0.9.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+fugue-0.9.0.dist-info/entry_points.txt,sha256=kiRuUkKOnnHFvlWpYSfVUZiXJW3hOez6gjYoOhGht3Q,302
+fugue-0.9.0.dist-info/top_level.txt,sha256=y1eCfzGdQ1_RkgcShcfbvXs-bopD3DwJcIOxP9EFXno,140
+fugue-0.9.0.dist-info/RECORD,,

{fugue-0.8.7.dev7.dist-info → fugue-0.9.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fugue-0.9.0.dist-info/entry_points.txt

@@ -0,0 +1,12 @@
+[fugue.plugins]
+dask = fugue_dask.registry [dask]
+duckdb = fugue_duckdb.registry [duckdb]
+ibis = fugue_ibis [ibis]
+polars = fugue_polars.registry [polars]
+ray = fugue_ray.registry [ray]
+spark = fugue_spark.registry [spark]
+
+[pytest11]
+fugue_test = fugue_test
+fugue_test_fixtures = fugue_test.fixtures
+

fugue_dask/_io.py

@@ -6,7 +6,7 @@ from fsspec import AbstractFileSystem
 from triad.collections.dict import ParamDict
 from triad.collections.schema import Schema
 from triad.utils.assertion import assert_or_throw
-from triad.utils.io import join, makedirs, url_to_fs
+from triad.utils.io import isfile, join, makedirs, url_to_fs
 
 from fugue._utils.io import FileParser, _get_single_files
 from fugue_dask.dataframe import DaskDataFrame
@@ -100,9 +100,11 @@ def _save_csv(df: DaskDataFrame, p: FileParser, **kwargs: Any) -> None:
 
 
 def _safe_load_csv(path: str, **kwargs: Any) -> dd.DataFrame:
+    if not isfile(path):
+        return dd.read_csv(join(path, "*.csv"), **kwargs)
     try:
         return dd.read_csv(path, **kwargs)
-    except (IsADirectoryError, PermissionError):
+    except (IsADirectoryError, PermissionError):  # pragma: no cover
         return dd.read_csv(join(path, "*.csv"), **kwargs)
 
 
@@ -148,11 +150,12 @@ def _save_json(df: DaskDataFrame, p: FileParser, **kwargs: Any) -> None:
 
 
 def _safe_load_json(path: str, **kwargs: Any) -> dd.DataFrame:
+    if not isfile(path):
+        return dd.read_json(join(path, "*.json"), **kwargs)
     try:
         return dd.read_json(path, **kwargs)
-    except (IsADirectoryError, PermissionError):
-        x = dd.read_json(join(path, "*.json"), **kwargs)
-        return x
+    except (IsADirectoryError, PermissionError):  # pragma: no cover
+        return dd.read_json(join(path, "*.json"), **kwargs)
 
 
 def _load_json(

fugue_dask/_utils.py

@@ -53,7 +53,7 @@ def hash_repartition(df: dd.DataFrame, num: int, cols: List[Any]) -> dd.DataFram
     if num < 1:
         return df
     if num == 1:
-        return df.repartition(1)
+        return df.repartition(npartitions=1)
     df = df.reset_index(drop=True).clear_divisions()
     idf, ct = _add_hash_index(df, num, cols)
     return _postprocess(idf, ct, num)
@@ -76,7 +76,7 @@ def even_repartition(df: dd.DataFrame, num: int, cols: List[Any]) -> dd.DataFram
         the number of partitions will be the number of groups.
     """
     if num == 1:
-        return df.repartition(1)
+        return df.repartition(npartitions=1)
     if len(cols) == 0 and num <= 0:
         return df
     df = df.reset_index(drop=True).clear_divisions()
@@ -111,7 +111,7 @@ def rand_repartition(
     if num < 1:
         return df
     if num == 1:
-        return df.repartition(1)
+        return df.repartition(npartitions=1)
     df = df.reset_index(drop=True).clear_divisions()
     if len(cols) == 0:
         idf, ct = _add_random_index(df, num=num, seed=seed)
@@ -124,7 +124,7 @@ def rand_repartition(
 def _postprocess(idf: dd.DataFrame, ct: int, num: int) -> dd.DataFrame:
     parts = min(ct, num)
     if parts <= 1:
-        return idf.repartition(1)
+        return idf.repartition(npartitions=1)
     divisions = list(np.arange(ct, step=math.ceil(ct / parts)))
     divisions.append(ct - 1)
     return idf.repartition(divisions=divisions, force=True)

fugue_dask/execution_engine.py

@@ -506,6 +506,17 @@ class DaskExecutionEngine(ExecutionEngine):
                 ).head(n)
 
         else:
+            if len(_presort.keys()) == 0 and n == 1:
+                return DaskDataFrame(
+                    d.drop_duplicates(
+                        subset=partition_spec.partition_by,
+                        ignore_index=True,
+                        keep="first",
+                    ),
+                    df.schema,
+                    type_safe=False,
+                )
+
             d = (
                 d.groupby(partition_spec.partition_by, dropna=False)
                 .apply(_partition_take, n=n, presort=_presort, meta=meta)

fugue_dask/registry.py

@@ -19,6 +19,8 @@ from fugue_dask._utils import DASK_UTILS
 from fugue_dask.dataframe import DaskDataFrame
 from fugue_dask.execution_engine import DaskExecutionEngine
 
+from .tester import DaskTestBackend  # noqa: F401  # pylint: disable-all
+
 
 @infer_execution_engine.candidate(
     lambda objs: is_pandas_or(objs, (dd.DataFrame, DaskDataFrame))

fugue_dask/tester.py

@@ -0,0 +1,24 @@
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator
+
+import dask
+from dask.distributed import Client
+
+import fugue.test as ft
+
+
+@ft.fugue_test_backend
+class DaskTestBackend(ft.FugueTestBackend):
+    name = "dask"
+
+    @classmethod
+    def transform_session_conf(cls, conf: Dict[str, Any]) -> Dict[str, Any]:
+        return ft.extract_conf(conf, "dask.", remove_prefix=True)
+
+    @classmethod
+    @contextmanager
+    def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+        with Client(**session_conf) as client:
+            dask.config.set({"dataframe.shuffle.method": "tasks"})
+            dask.config.set({"dataframe.convert-string": False})
+            yield client

fugue_duckdb/__init__.py

@@ -7,8 +7,3 @@ try:
     from fugue_duckdb.dask import DuckDaskExecutionEngine
 except Exception:  # pragma: no cover
     pass
-
-try:
-    from fugue_duckdb.ibis_engine import DuckDBIbisEngine
-except Exception:  # pragma: no cover
-    pass

fugue_duckdb/_io.py

@@ -140,6 +140,7 @@ class DuckDBIO:
         else:
             if header:
                 kw["ALL_VARCHAR"] = 1
+                kw["auto_detect"] = 1
                 if columns is None:
                     cols = "*"
                 elif isinstance(columns, list):

fugue_duckdb/registry.py

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Tuple
 
 from duckdb import DuckDBPyConnection, DuckDBPyRelation
 from triad import run_at_def
@@ -15,10 +15,17 @@ from fugue.dev import (
     fugue_annotated_param,
     is_pandas_or,
 )
-from fugue.plugins import infer_execution_engine
+from fugue.plugins import infer_execution_engine, parse_execution_engine
 from fugue_duckdb.dataframe import DuckDataFrame
 from fugue_duckdb.execution_engine import DuckDBEngine, DuckExecutionEngine
 
+from .tester import DuckDBTestBackend  # noqa: F401  # pylint: disable-all
+
+try:
+    from .tester import DuckDaskTestBackend  # noqa: F401  # pylint: disable-all
+except ImportError:  # pragma: no cover
+    pass
+
 
 @infer_execution_engine.candidate(
     lambda objs: is_pandas_or(objs, (DuckDBPyRelation, DuckDataFrame))
@@ -67,6 +74,27 @@ def _register_engines() -> None:
     register_sql_engine("duckdb", lambda engine: DuckDBEngine(engine))
 
 
+try:
+    from fugue_duckdb.dask import DuckDaskExecutionEngine
+    from dask.distributed import Client
+
+    @parse_execution_engine.candidate(
+        lambda engine, conf, **kwargs: isinstance(engine, list)
+        and len(engine) == 2
+        and isinstance(engine[0], DuckDBPyConnection)
+        and isinstance(engine[1], Client),
+    )
+    def _parse_duck_dask_client(
+        engine: Tuple[DuckDBPyConnection, Client], conf: Any, **kwargs: Any
+    ) -> DuckDaskExecutionEngine:
+        return DuckDaskExecutionEngine(
+            connection=engine[0], dask_client=engine[1], conf=conf
+        )
+
+except Exception:  # pragma: no cover
+    pass
+
+
 @fugue_annotated_param(DuckExecutionEngine)
 class _DuckExecutionEngineParam(ExecutionEngineParam):
     pass

fugue_duckdb/tester.py

@@ -0,0 +1,49 @@
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator
+
+import duckdb
+
+import fugue.test as ft
+
+try:
+    import dask.distributed as dd
+    import dask
+
+    _HAS_DASK = True
+except ImportError:  # pragma: no cover
+    _HAS_DASK = False
+
+
+@ft.fugue_test_backend
+class DuckDBTestBackend(ft.FugueTestBackend):
+    name = "duckdb"
+
+    @classmethod
+    @contextmanager
+    def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+        with duckdb.connect(config=session_conf) as conn:
+            yield conn
+
+
+if _HAS_DASK:
+
+    @ft.fugue_test_backend
+    class DuckDaskTestBackend(ft.FugueTestBackend):
+        name = "duckdask"
+
+        @classmethod
+        def transform_session_conf(cls, conf: Dict[str, Any]) -> Dict[str, Any]:
+            res = ft.extract_conf(conf, "duck.", remove_prefix=False)
+            res.update(ft.extract_conf(conf, "dask.", remove_prefix=False))
+            return res
+
+        @classmethod
+        @contextmanager
+        def session_context(cls, session_conf: Dict[str, Any]) -> Iterator[Any]:
+            duck_conf = ft.extract_conf(session_conf, "duck.", remove_prefix=True)
+            dask_conf = ft.extract_conf(session_conf, "dask.", remove_prefix=True)
+            with dd.Client(**dask_conf) as client:
+                dask.config.set({"dataframe.shuffle.method": "tasks"})
+                dask.config.set({"dataframe.convert-string": False})
+                with duckdb.connect(config=duck_conf) as conn:
+                    yield [conn, client]

fugue_ibis/__init__.py

@@ -3,7 +3,4 @@ from triad import run_at_def
 
 from ._compat import IbisSchema, IbisTable
 from .dataframe import IbisDataFrame
-from .execution.ibis_engine import IbisEngine, parse_ibis_engine
-from .execution.pandas_backend import PandasIbisEngine
 from .execution_engine import IbisExecutionEngine, IbisSQLEngine
-from .extensions import as_fugue, as_ibis, run_ibis

fugue_ibis/dataframe.py

@@ -78,7 +78,7 @@ class IbisDataFrame(DataFrame):
 
     @property
     def columns(self) -> List[str]:
-        return self._table.columns
+        return _get_ibis_columns(self._table)
 
     def peek_array(self) -> List[Any]:
         res = self._to_local_df(self._table.head(1)).as_array()
@@ -202,7 +202,7 @@ def _drop_ibis_columns(df: IbisTable, columns: List[str]) -> IbisTable:
 
 
 @rename.candidate(lambda df, *args, **kwargs: isinstance(df, IbisTable))
-def _rename_dask_dataframe(df: IbisTable, columns: Dict[str, Any]) -> IbisTable:
+def _rename_ibis_table(df: IbisTable, columns: Dict[str, Any]) -> IbisTable:
     _assert_no_missing(df, columns.keys())
     old_names = df.columns
     new_names = [columns.get(name, name) for name in old_names]

fugue_ibis/execution_engine.py

@@ -23,8 +23,8 @@ from ._compat import IbisTable
 from ._utils import to_ibis_schema
 from .dataframe import IbisDataFrame
 
-_JOIN_RIGHT_SUFFIX = "_ibis_y__"
-_GEN_TABLE_NAMES = (f"_fugue_temp_table_{i:d}" for i in itertools.count())
+_JOIN_RIGHT_SUFFIX = "_ibis_y__".upper()
+_GEN_TABLE_NAMES = (f"_fugue_temp_table_{i:d}".upper() for i in itertools.count())
 
 
 class IbisSQLEngine(SQLEngine):
@@ -92,11 +92,11 @@ class IbisSQLEngine(SQLEngine):
         _df2 = self.to_df(df2)
         key_schema, end_schema = get_join_schemas(_df1, _df2, how=how, on=on)
         on_fields = [_df1.native[k] == _df2.native[k] for k in key_schema]
-        if ibis.__version__ < "6":
+        if ibis.__version__ < "6":  # pragma: no cover
             suffixes: Dict[str, Any] = dict(suffixes=("", _JOIN_RIGHT_SUFFIX))
-        else:  # pragma: no cover
+        else:
             # breaking change in ibis 6.0
-            suffixes = dict(lname="", rname=_JOIN_RIGHT_SUFFIX)
+            suffixes = dict(lname="", rname="{name}" + _JOIN_RIGHT_SUFFIX)
         if how.lower() == "cross":
             tb = _df1.native.cross_join(_df2.native, **suffixes)
         elif how.lower() == "right_outer":
@@ -224,7 +224,7 @@ class IbisSQLEngine(SQLEngine):
             _presort = parse_presort_exp(presort)
         else:
             _presort = partition_spec.presort
-        tbn = "_temp"
+        tbn = "_TEMP"
         idf = self.to_df(df)
 
         if len(_presort) == 0:
@@ -233,9 +233,10 @@ class IbisSQLEngine(SQLEngine):
             pcols = ", ".join(
                 self.encode_column_name(x) for x in partition_spec.partition_by
             )
+            dummy_order_by = self._dummy_window_order_by()
             sql = (
                 f"SELECT * FROM ("
-                f"SELECT *, ROW_NUMBER() OVER (PARTITION BY {pcols}) "
+                f"SELECT *, ROW_NUMBER() OVER (PARTITION BY {pcols} {dummy_order_by}) "
                 f"AS __fugue_take_param FROM {tbn}"
                 f") WHERE __fugue_take_param<={n}"
             )
@@ -290,6 +291,12 @@ class IbisSQLEngine(SQLEngine):
     def load_table(self, table: str, **kwargs: Any) -> DataFrame:
         return self.to_df(self.backend.table(table))
 
+    def _dummy_window_order_by(self) -> str:
+        """Return a dummy window order by clause, this is required for
+        some SQL backends when there is no real order by clause in window
+        """
+        return ""
+
 
 class IbisMapEngine(MapEngine):
     """IbisExecutionEngine's MapEngine, it is a wrapper of the map engine

fugue_ray/_constants.py

@@ -1,6 +1,7 @@
 from typing import Any, Dict
 
 import ray
+from packaging import version
 
 FUGUE_RAY_CONF_SHUFFLE_PARTITIONS = "fugue.ray.shuffle.partitions"
 FUGUE_RAY_DEFAULT_PARTITIONS = "fugue.ray.default.partitions"
@@ -12,8 +13,6 @@ FUGUE_RAY_DEFAULT_CONF: Dict[str, Any] = {
     FUGUE_RAY_DEFAULT_PARTITIONS: 0,
     FUGUE_RAY_ZERO_COPY: True,
 }
+RAY_VERSION = version.parse(ray.__version__)
 
-if ray.__version__ >= "2.3":
-    _ZERO_COPY: Dict[str, Any] = {"zero_copy_batch": True}
-else:  # pragma: no cover
-    _ZERO_COPY = {}
+_ZERO_COPY: Dict[str, Any] = {"zero_copy_batch": True}

fugue_ray/_utils/dataframe.py

@@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optional, Tuple
 
 import pandas as pd
 import pyarrow as pa
-import ray
 import ray.data as rd
 from triad import Schema
 
@@ -31,31 +30,21 @@ def get_dataset_format(df: rd.Dataset) -> Tuple[Optional[str], rd.Dataset]:
     df = materialize(df)
     if df.count() == 0:
         return None, df
-    if ray.__version__ < "2.5.0":  # pragma: no cover
-        if hasattr(df, "_dataset_format"):  # pragma: no cover
-            return df._dataset_format(), df  # ray<2.2
-        ctx = rd.context.DatasetContext.get_current()
-        ctx.use_streaming_executor = False
-        return df.dataset_format(), df  # ray>=2.2
-    else:
-        schema = df.schema(fetch_if_missing=True)
-        if schema is None:  # pragma: no cover
-            return None, df
-        if isinstance(schema.base_schema, pa.Schema):
-            return "arrow", df
-        return "pandas", df
+    schema = df.schema(fetch_if_missing=True)
+    if schema is None:  # pragma: no cover
+        return None, df
+    if isinstance(schema.base_schema, pa.Schema):
+        return "arrow", df
+    return "pandas", df
 
 
 def to_schema(schema: Any) -> Schema:  # pragma: no cover
     if isinstance(schema, pa.Schema):
         return Schema(schema)
-    if ray.__version__ >= "2.5.0":
-        if isinstance(schema, rd.Schema):
-            if hasattr(schema, "base_schema") and isinstance(
-                schema.base_schema, pa.Schema
-            ):
-                return Schema(schema.base_schema)
-            return Schema(list(zip(schema.names, schema.types)))
+    if isinstance(schema, rd.Schema):
+        if hasattr(schema, "base_schema") and isinstance(schema.base_schema, pa.Schema):
+            return Schema(schema.base_schema)
+        return Schema(list(zip(schema.names, schema.types)))
     raise ValueError(f"{schema} is not supported")