pyspiral 0.6.8__cp312-abi3-manylinux_2_28_aarch64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. pyspiral-0.6.8.dist-info/METADATA +51 -0
  2. pyspiral-0.6.8.dist-info/RECORD +102 -0
  3. pyspiral-0.6.8.dist-info/WHEEL +4 -0
  4. pyspiral-0.6.8.dist-info/entry_points.txt +2 -0
  5. spiral/__init__.py +35 -0
  6. spiral/_lib.abi3.so +0 -0
  7. spiral/adbc.py +411 -0
  8. spiral/api/__init__.py +78 -0
  9. spiral/api/admin.py +15 -0
  10. spiral/api/client.py +164 -0
  11. spiral/api/filesystems.py +134 -0
  12. spiral/api/key_space_indexes.py +23 -0
  13. spiral/api/organizations.py +77 -0
  14. spiral/api/projects.py +219 -0
  15. spiral/api/telemetry.py +19 -0
  16. spiral/api/text_indexes.py +56 -0
  17. spiral/api/types.py +22 -0
  18. spiral/api/workers.py +40 -0
  19. spiral/api/workloads.py +52 -0
  20. spiral/arrow_.py +216 -0
  21. spiral/cli/__init__.py +88 -0
  22. spiral/cli/__main__.py +4 -0
  23. spiral/cli/admin.py +14 -0
  24. spiral/cli/app.py +104 -0
  25. spiral/cli/console.py +95 -0
  26. spiral/cli/fs.py +76 -0
  27. spiral/cli/iceberg.py +97 -0
  28. spiral/cli/key_spaces.py +89 -0
  29. spiral/cli/login.py +24 -0
  30. spiral/cli/orgs.py +89 -0
  31. spiral/cli/printer.py +53 -0
  32. spiral/cli/projects.py +147 -0
  33. spiral/cli/state.py +5 -0
  34. spiral/cli/tables.py +174 -0
  35. spiral/cli/telemetry.py +17 -0
  36. spiral/cli/text.py +115 -0
  37. spiral/cli/types.py +50 -0
  38. spiral/cli/workloads.py +58 -0
  39. spiral/client.py +178 -0
  40. spiral/core/__init__.pyi +0 -0
  41. spiral/core/_tools/__init__.pyi +5 -0
  42. spiral/core/authn/__init__.pyi +27 -0
  43. spiral/core/client/__init__.pyi +237 -0
  44. spiral/core/table/__init__.pyi +101 -0
  45. spiral/core/table/manifests/__init__.pyi +35 -0
  46. spiral/core/table/metastore/__init__.pyi +58 -0
  47. spiral/core/table/spec/__init__.pyi +213 -0
  48. spiral/dataloader.py +285 -0
  49. spiral/dataset.py +255 -0
  50. spiral/datetime_.py +27 -0
  51. spiral/debug/__init__.py +0 -0
  52. spiral/debug/manifests.py +87 -0
  53. spiral/debug/metrics.py +56 -0
  54. spiral/debug/scan.py +266 -0
  55. spiral/expressions/__init__.py +276 -0
  56. spiral/expressions/base.py +157 -0
  57. spiral/expressions/http.py +86 -0
  58. spiral/expressions/io.py +100 -0
  59. spiral/expressions/list_.py +68 -0
  60. spiral/expressions/mp4.py +62 -0
  61. spiral/expressions/png.py +18 -0
  62. spiral/expressions/qoi.py +18 -0
  63. spiral/expressions/refs.py +58 -0
  64. spiral/expressions/str_.py +39 -0
  65. spiral/expressions/struct.py +59 -0
  66. spiral/expressions/text.py +62 -0
  67. spiral/expressions/tiff.py +223 -0
  68. spiral/expressions/udf.py +46 -0
  69. spiral/grpc_.py +32 -0
  70. spiral/iceberg.py +31 -0
  71. spiral/iterable_dataset.py +106 -0
  72. spiral/key_space_index.py +44 -0
  73. spiral/project.py +199 -0
  74. spiral/protogen/_/__init__.py +0 -0
  75. spiral/protogen/_/arrow/__init__.py +0 -0
  76. spiral/protogen/_/arrow/flight/__init__.py +0 -0
  77. spiral/protogen/_/arrow/flight/protocol/__init__.py +0 -0
  78. spiral/protogen/_/arrow/flight/protocol/sql/__init__.py +2548 -0
  79. spiral/protogen/_/google/__init__.py +0 -0
  80. spiral/protogen/_/google/protobuf/__init__.py +2310 -0
  81. spiral/protogen/_/message_pool.py +3 -0
  82. spiral/protogen/_/py.typed +0 -0
  83. spiral/protogen/_/scandal/__init__.py +190 -0
  84. spiral/protogen/_/spfs/__init__.py +72 -0
  85. spiral/protogen/_/spql/__init__.py +61 -0
  86. spiral/protogen/_/substrait/__init__.py +6196 -0
  87. spiral/protogen/_/substrait/extensions/__init__.py +169 -0
  88. spiral/protogen/__init__.py +0 -0
  89. spiral/protogen/util.py +41 -0
  90. spiral/py.typed +0 -0
  91. spiral/scan.py +285 -0
  92. spiral/server.py +17 -0
  93. spiral/settings.py +114 -0
  94. spiral/snapshot.py +56 -0
  95. spiral/streaming_/__init__.py +3 -0
  96. spiral/streaming_/reader.py +133 -0
  97. spiral/streaming_/stream.py +157 -0
  98. spiral/substrait_.py +274 -0
  99. spiral/table.py +293 -0
  100. spiral/text_index.py +17 -0
  101. spiral/transaction.py +58 -0
  102. spiral/types_.py +6 -0
@@ -0,0 +1,51 @@
1
+ Metadata-Version: 2.4
2
+ Name: pyspiral
3
+ Version: 0.6.8
4
+ Classifier: Intended Audience :: Science/Research
5
+ Classifier: Operating System :: OS Independent
6
+ Classifier: Programming Language :: Python
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: Programming Language :: Python :: 3 :: Only
9
+ Classifier: Programming Language :: Python :: 3.10
10
+ Classifier: Programming Language :: Python :: 3.11
11
+ Classifier: Programming Language :: Python :: 3.12
12
+ Classifier: Programming Language :: Python :: 3.13
13
+ Classifier: Programming Language :: Rust
14
+ Classifier: License :: Other/Proprietary License
15
+ Requires-Dist: betterproto2>=0.9.0
16
+ Requires-Dist: google-re2>=1.1.20240702
17
+ Requires-Dist: grpclib>=0.4.7
18
+ Requires-Dist: hishel>=0.0.30
19
+ Requires-Dist: httpx>=0.27.0
20
+ Requires-Dist: nanoid>=2.0.0
21
+ Requires-Dist: numpy>=2
22
+ Requires-Dist: pyarrow>=21.0.0
23
+ Requires-Dist: pydantic-settings>=2.3.4
24
+ Requires-Dist: pydantic[email]>=2.5.3,<2.12
25
+ Requires-Dist: pyjwt[crypto]>=2.9.0
26
+ Requires-Dist: pyperclip>=1.9.0
27
+ Requires-Dist: questionary>=2.0.1
28
+ Requires-Dist: sqlglot[rs]>=25.25.1
29
+ Requires-Dist: tqdm>=4.66.5
30
+ Requires-Dist: typer>=0.16
31
+ Requires-Dist: xxhash>=3.4.1
32
+ Requires-Dist: polars>=1.31.0 ; extra == 'polars'
33
+ Requires-Dist: duckdb>=1.3.2 ; extra == 'duckdb'
34
+ Requires-Dist: datasets>=4.0.0 ; extra == 'datasets'
35
+ Requires-Dist: pyiceberg>=0.9.1 ; extra == 'pyiceberg'
36
+ Requires-Dist: mosaicml-streaming>=0.13.0 ; extra == 'streaming'
37
+ Requires-Dist: vortex-data>=0.52.1 ; extra == 'streaming'
38
+ Provides-Extra: polars
39
+ Provides-Extra: duckdb
40
+ Provides-Extra: datasets
41
+ Provides-Extra: pyiceberg
42
+ Provides-Extra: streaming
43
+ Summary: Python client for Spiral.
44
+ Home-Page: https://spiraldb.com
45
+ Author-email: SpiralDB <hello@spiraldb.com>
46
+ License: Proprietary License
47
+ Requires-Python: >=3.10
48
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
49
+
50
+ # PySpiral
51
+
@@ -0,0 +1,102 @@
1
+ pyspiral-0.6.8.dist-info/METADATA,sha256=aUluVWQasIRe0CgAZ3-zPTwJ2hmhWYfvgGy3m_xkoQA,1842
2
+ pyspiral-0.6.8.dist-info/WHEEL,sha256=I5JYpyYzeAl2SOerY_wvkm-HJti0rDQc6zMeJs35MpM,108
3
+ pyspiral-0.6.8.dist-info/entry_points.txt,sha256=uft7u-a6g40NLt4Q6BleWbK4NY0M8nZuYPpP8DV0EOk,45
4
+ spiral/__init__.py,sha256=n4JNLrO3wyw_k_U_JKyNiGON0wEpfvqxDhDdB2P6dhM,1007
5
+ spiral/_lib.abi3.so,sha256=0iMVGBDY-hzNusKBh9uflJBfVAAlB6PjoP_7ozo9KvU,58057048
6
+ spiral/adbc.py,sha256=7IxfWIeQN-fh0W5OdN_PP2x3pzQYg6ZUOLsHg3jktqw,14842
7
+ spiral/api/__init__.py,sha256=ULBlVq3PnfNOO6T5naE_ULmmii-83--qTuN2PpAUQN0,2241
8
+ spiral/api/admin.py,sha256=A1iVR1XYJSObZivPAD5UzmPuMgupXc9kaHNYYa_kwfs,585
9
+ spiral/api/client.py,sha256=wjxhjM2Li3UQZiWkihRKh5HHbWN4uURXXYaX_S1Bkew,4641
10
+ spiral/api/filesystems.py,sha256=yEHgHfo7t1_becm0UFedc3nd49_G77hHjYwtYQ6P9XU,4240
11
+ spiral/api/key_space_indexes.py,sha256=-38rZXTdkL4mLhp9h3CtqyIyutzzq88tV6bhK05MqYE,640
12
+ spiral/api/organizations.py,sha256=B-8zZ7lFJANGK7dUNbo_aU-cgI959JBP9VcWb6wdgi0,1895
13
+ spiral/api/projects.py,sha256=1JC7VjqZJfwR6zfhBZr3OCwaf6zb-MXMOBTE_NztmcE,6356
14
+ spiral/api/telemetry.py,sha256=tfdA3E_EWJwFVxkQfkm8tiYGRubnx2LuE5nbfsk1oG4,474
15
+ spiral/api/text_indexes.py,sha256=_zVlGBytl-9-Unbu2POfZgLh40H1YRcagFtplgIG428,1828
16
+ spiral/api/types.py,sha256=lGdiKViRgIEJXD2ubwnyEIEwHkfRumlZjVEaHMV3Tm8,682
17
+ spiral/api/workers.py,sha256=0wZNUHMioDT53P1OBJfpjyDfIodHwwT6858z2IlRIM4,636
18
+ spiral/api/workloads.py,sha256=XAyXV7vgZcoyyoPoGvOT4jTpyFKFMvrrAfhL6d1h1kE,1748
19
+ spiral/arrow_.py,sha256=fdSIfIs7UjDxXZlppvOW0zz86W_70Pa5pagJilH2kOE,7583
20
+ spiral/cli/__init__.py,sha256=LutjpWZu5Rvmba8C8bPa5vOCv74JuAoE1kvz0nd48dE,2476
21
+ spiral/cli/__main__.py,sha256=kNaKM2xgJo7GRogf83nYldLM-RGUR6vymdGwZxywQu0,71
22
+ spiral/cli/admin.py,sha256=-ubYqs8nKjnQStbQ68jpWx_9xh0TsaxI0wM1Hfko8_U,319
23
+ spiral/cli/app.py,sha256=smzGj5a2RwhM9RQChmlEeKZLN4Fk60-bP7Lm5_Is1Rw,2760
24
+ spiral/cli/console.py,sha256=6JHbAQV6MFWz3P-VzqPOjhHpkIQagsCdzTMvmuDKMkU,2580
25
+ spiral/cli/fs.py,sha256=vaPcSc2YghhHeipxNitIdsHaBhFwlwkvPFqYsFSN9P0,2927
26
+ spiral/cli/iceberg.py,sha256=Q14tcGcn1LixbFCYP0GhfYwFFXTmmi8tqBPYwalJEyE,3248
27
+ spiral/cli/key_spaces.py,sha256=x3IFRP5d47pKiAHeWExYMOBaT2TwxbWjVM01SUqKrwI,2943
28
+ spiral/cli/login.py,sha256=2tw6uN5rEpiMMAmjQSB3-JUPf3C0Wc1eTGCDxhYtJps,731
29
+ spiral/cli/orgs.py,sha256=fmOuLxpeIFfKqePRi292Gv9k-EF5pPn_tbKd2BLl2Ig,2869
30
+ spiral/cli/printer.py,sha256=aosc763hDFgoXJGkiANmNyO3kAsecAS1JWgjEhn8GCM,1784
31
+ spiral/cli/projects.py,sha256=1M1nGrBT-t0aY9RV5Cnmzy7YrhIvmHwdkpa3y9j8rG8,5756
32
+ spiral/cli/state.py,sha256=10wTIVQ0SJkY67Z6-KQ1LFlt3aVIPmZhoHFdTwp4kNA,130
33
+ spiral/cli/tables.py,sha256=fFte_wMNcB0V-fmfSXfSbtV4UlAi-Xw5nYDJ0b62CGk,6360
34
+ spiral/cli/telemetry.py,sha256=Uxo1Q1FkKJ6n6QNGOUmL3j_pRRWRx0qWIhoP-U9BuR0,589
35
+ spiral/cli/text.py,sha256=DlWGe4JrkdERAiqyITNpk91Wqb63Re99rNYlIFsIamc,4031
36
+ spiral/cli/types.py,sha256=XYzo1GgX7dBBItoBSrHI4vO5C2lLmS2sktb-2GnGH3E,1362
37
+ spiral/cli/workloads.py,sha256=2_SLfQTFN6y73R9H0i9dk8VIOVagKxSxOpHXC56yptY,2015
38
+ spiral/client.py,sha256=N4sQLxtQ6GYCnj00hm4VX1vUVUqzQdHhl_KfQwp-1LQ,6345
39
+ spiral/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
40
+ spiral/core/_tools/__init__.pyi,sha256=b2KLfTOQ67pjfbYt07o0IGiTu5o2bZw69lllV8v0Dps,143
41
+ spiral/core/authn/__init__.pyi,sha256=z_GWyIS62fuiYQrYO8hzw4W8oGaiciqS1u5qtAt54VY,769
42
+ spiral/core/client/__init__.pyi,sha256=1HK3SOMT1QKmD5Hai58ZFjiEZK0QzyYtP84hse8SBEI,6666
43
+ spiral/core/table/__init__.pyi,sha256=QqG_pMlPhMtXG-56dXyQjOWVKMugPP0nnYnvYaY0Q10,3288
44
+ spiral/core/table/manifests/__init__.pyi,sha256=eVfDpmhYSjafIvvALqAkZe5baN3Y1HpKpxYEbjwd4gQ,1043
45
+ spiral/core/table/metastore/__init__.pyi,sha256=rc3u9MwEKRvL2kxOc8lBorddFRnM8o_o1frqtae86a4,1697
46
+ spiral/core/table/spec/__init__.pyi,sha256=OFYJXPXix7gskYJIMog7IniZslEPJ0xvL-sUSFDPbXs,5643
47
+ spiral/dataloader.py,sha256=FFZhIflQPEygXe-xBLifQnnxANi4CFooaHRm4i-EGHo,10335
48
+ spiral/dataset.py,sha256=PMLoXnXuEUciP6-NXqTmQLXu0UIH7OcC4-iZtY_iuO8,7973
49
+ spiral/datetime_.py,sha256=elXaUWtZuuLVcu9E0aXnvYRPB9XWqZbLDToozQYQYjU,950
50
+ spiral/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
+ spiral/debug/manifests.py,sha256=7f1O3ba9mrA5nXpOF9cEIQuUAteP5wiBkFy_diQJ7No,3216
52
+ spiral/debug/metrics.py,sha256=XdRDcjggtsLNGCAjam6IxG9072pz_d2C8iLApNRFUtk,2044
53
+ spiral/debug/scan.py,sha256=UEm_aRnql5pwDPTpZgakMLNjlzkKL4RurBFFqH_BLAQ,9526
54
+ spiral/expressions/__init__.py,sha256=KhwFjVKoFgx1S6hkVcE8aZjoHY_1N-BgQ2rGEZfPQvM,7957
55
+ spiral/expressions/base.py,sha256=915gpvZZCTRCO5q93pwwmhf-R6C23LQsyDt4Q2dHk9s,5290
56
+ spiral/expressions/http.py,sha256=begUydWoFHEqjeLkATvI_v66Ez6_rR-OQBWO5cHbb9c,2742
57
+ spiral/expressions/io.py,sha256=gJ2a0FKMmdxarWKENulPRwH7KDvSJTIh_OUxX306xAM,3045
58
+ spiral/expressions/list_.py,sha256=MMt5lf5H1M3O-x6N_PvqOLGq9NOk6Ukv0fPWwPC_uy4,1809
59
+ spiral/expressions/mp4.py,sha256=_xGVnkygddzxP9a8OACJ8_KXnejuVbYCVKBCXBQ798Y,2151
60
+ spiral/expressions/png.py,sha256=KO8X0OmMzUFwpg2I_j0JTyldPzVXDWIMzjWMWDV9vIY,506
61
+ spiral/expressions/qoi.py,sha256=gvIbb6fXb_Bb080sn9wkpbGGrPs2UEcTXCfuv4-kcYQ,506
62
+ spiral/expressions/refs.py,sha256=omeHBQ5o6N4xgZ3x5Xz7IRrWwYBBtQY8DYK0NNAxeGo,2109
63
+ spiral/expressions/str_.py,sha256=tY8RXW3JWvr1-bEfCZtk5FAf11wKJnXPuA9EoeJ9tA4,1265
64
+ spiral/expressions/struct.py,sha256=pGAnCDh6AK0BK1XfZ1qG4ce4ranIQEE1HQsgmzBcfwQ,2038
65
+ spiral/expressions/text.py,sha256=-02gBWYoyNQ3qQ1--9HTa8IryUDojYQVIp8C7rgnOWQ,1893
66
+ spiral/expressions/tiff.py,sha256=fQwIn0kLFBM2Y3YYIHmTgb_EIRHKT2fNc77nioDQQw4,8044
67
+ spiral/expressions/udf.py,sha256=yb9MIcrFftpNDxgBF228cvdv6TY-hEFikYz2fq_nzWo,1353
68
+ spiral/grpc_.py,sha256=f3czdP1Mxme42Y5--a5ogYq1TTiWn-J_MlGjwJ2mWwM,1015
69
+ spiral/iceberg.py,sha256=JGq62Qnf296r9_hRAoH85GQq45-uSBjwXWw_CvPi6G4,930
70
+ spiral/iterable_dataset.py,sha256=Eekg9ad8tcwXcloHWReBbvCSr5ZappRHn2ldKTvwqS0,4622
71
+ spiral/key_space_index.py,sha256=NAB_nONEjpMYbse8suz42w7Qb5OPHuKN9h9CT2NJe08,1460
72
+ spiral/project.py,sha256=CO_Pn6vPqaonNvRdCNRFcBWr4TqO2AsAUTH5xawIeCE,7283
73
+ spiral/protogen/_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
74
+ spiral/protogen/_/arrow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
75
+ spiral/protogen/_/arrow/flight/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
76
+ spiral/protogen/_/arrow/flight/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
77
+ spiral/protogen/_/arrow/flight/protocol/sql/__init__.py,sha256=ooZZsDCRFpktUCH11OdxMRa_GLQYnY9w-1fBr5a7vBk,90023
78
+ spiral/protogen/_/google/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
79
+ spiral/protogen/_/google/protobuf/__init__.py,sha256=H0FVEXusqww2j5dl7Ee05tR6qMG_hQioUp1qFfDgnco,80036
80
+ spiral/protogen/_/message_pool.py,sha256=4-cRhhiM6bmfpUJZ8qxc8LEyqHBHpLCcotjbyZxl7JM,71
81
+ spiral/protogen/_/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
82
+ spiral/protogen/_/scandal/__init__.py,sha256=liUQAICLd2sPccCmqo0_c1duSbNj_m8p_IgmdnHsB3E,4965
83
+ spiral/protogen/_/spfs/__init__.py,sha256=zMMEDIfPXQNBkisLI-iMWbJABye-vK42Gf2BUQQYR_c,2028
84
+ spiral/protogen/_/spql/__init__.py,sha256=PEC4bI-PHdJ4Zd8Jb1k6Xk2iFYoYqIUbTGlL2JVGnT0,1548
85
+ spiral/protogen/_/substrait/__init__.py,sha256=-ngqHcYfio6s1B4M1_e1VsDymUcFK9qdM17ECA31qLw,209837
86
+ spiral/protogen/_/substrait/extensions/__init__.py,sha256=nhnEnho70GAT8WPj2xtwJUzk5GJ6X2e-HTvyk7emGsk,5326
87
+ spiral/protogen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
88
+ spiral/protogen/util.py,sha256=smnvVo6nYH3FfDm9jqhNLaXz4bbTBaQezHQDCTvZyiQ,1486
89
+ spiral/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
90
+ spiral/scan.py,sha256=4PUlI_DHbO1WTttLia6DinhGtOWsCiqek4ZljoEiRZc,10523
91
+ spiral/server.py,sha256=ztBmB5lBnUz-smQxR_tC8AI5SOhz17wH0MI3GuzDUdM,600
92
+ spiral/settings.py,sha256=JRQSwjJyNaCqQdQLxiqB_O_LZRQXMLyshJBrI2LZHwM,3113
93
+ spiral/snapshot.py,sha256=cTobi5jtiANxalGA-isokQHblNmXGtuUvgUGGNVybsI,1555
94
+ spiral/streaming_/__init__.py,sha256=s7MlW2ERsuZmZGExLFL6RcZon2e0tNBocBg5ANgki7k,61
95
+ spiral/streaming_/reader.py,sha256=tl_lC9xgh1-QFhsZn4xQT7It3PVTzHCEUT2BG2dWBRQ,4166
96
+ spiral/streaming_/stream.py,sha256=nXnygiuCxi1D3PhaxV8Ujif4J9ly_OczA7CZ3W4WN2w,5913
97
+ spiral/substrait_.py,sha256=AKeOD4KIXvz2J4TYxnIneOiHddtBIyOhuNxVO_uH0eg,12592
98
+ spiral/table.py,sha256=G05b6M0uVmT5ew5GxuzsVB4rQzg25W3zGMTftL07pJU,11026
99
+ spiral/text_index.py,sha256=FQ9rgIEGLSJryS9lFdMhKtPFey18BXoWbPXyvZPJJ04,442
100
+ spiral/transaction.py,sha256=h6YdAwOYX6qq-tXYV4i9yhy1Nq1tIfRphY_fk7Q_yLQ,1854
101
+ spiral/types_.py,sha256=W_jyO7F6rpPiH69jhgSgV7OxQZbOlb1Ho3InpKUP6Eo,155
102
+ pyspiral-0.6.8.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: maturin (1.9.6)
3
+ Root-Is-Purelib: false
4
+ Tag: cp312-abi3-manylinux_2_28_aarch64
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ spiral=spiral.cli.app:main
spiral/__init__.py ADDED
@@ -0,0 +1,35 @@
1
+ """Python client for Spiral"""
2
+
3
+ # This is here to make sure we load the native extension first
4
+ from spiral import _lib
5
+
6
+ # Eagerly import the Spiral library
7
+ assert _lib, "Spiral library"
8
+
9
+ from spiral.client import Spiral # noqa: E402
10
+ from spiral.core.client import Shard, ShuffleConfig # noqa: E402
11
+ from spiral.dataloader import SpiralDataLoader, World # noqa: E402
12
+ from spiral.iceberg import Iceberg # noqa: E402
13
+ from spiral.key_space_index import KeySpaceIndex # noqa: E402
14
+ from spiral.project import Project # noqa: E402
15
+ from spiral.scan import Scan # noqa: E402
16
+ from spiral.snapshot import Snapshot # noqa: E402
17
+ from spiral.table import Table # noqa: E402
18
+ from spiral.text_index import TextIndex # noqa: E402
19
+ from spiral.transaction import Transaction # noqa: E402
20
+
21
+ __all__ = [
22
+ "Spiral",
23
+ "Project",
24
+ "Table",
25
+ "Snapshot",
26
+ "Transaction",
27
+ "Scan",
28
+ "Shard",
29
+ "ShuffleConfig",
30
+ "TextIndex",
31
+ "KeySpaceIndex",
32
+ "SpiralDataLoader",
33
+ "World",
34
+ "Iceberg",
35
+ ]
spiral/_lib.abi3.so ADDED
Binary file
spiral/adbc.py ADDED
@@ -0,0 +1,411 @@
1
+ import abc
2
+ import functools
3
+ import logging
4
+ from concurrent.futures import ThreadPoolExecutor
5
+ from urllib.parse import urlparse
6
+
7
+ import pyarrow as pa
8
+ import pyarrow.compute as pc
9
+ import sqlglot
10
+ import sqlglot.expressions as exp
11
+ from pyarrow.flight import (
12
+ Action,
13
+ FlightDescriptor,
14
+ FlightEndpoint,
15
+ FlightError,
16
+ FlightInfo,
17
+ FlightMetadataWriter,
18
+ FlightServerBase,
19
+ MetadataRecordBatchReader,
20
+ RecordBatchStream,
21
+ ServerCallContext,
22
+ Ticket,
23
+ )
24
+
25
+ from spiral import Spiral
26
+ from spiral.api.projects import TableResource
27
+ from spiral.protogen._.arrow.flight.protocol import sql as rpc
28
+ from spiral.protogen._.arrow.flight.protocol.sql import (
29
+ CommandGetCatalogs,
30
+ CommandGetDbSchemas,
31
+ CommandGetSqlInfo,
32
+ CommandGetTables,
33
+ CommandStatementQuery,
34
+ SqlInfo,
35
+ SqlSupportedTransaction,
36
+ )
37
+ from spiral.protogen._.google.protobuf import Any
38
+ from spiral.snapshot import Snapshot
39
+
40
+ log = logging.getLogger(__name__)
41
+ logging.getLogger("sqlx").setLevel(logging.WARNING)
42
+
43
+
44
+ def debuggable(func):
45
+ """A decorator to enable GUI (i.e. PyCharm) debugging in the
46
+ decorated Arrow Flight RPC Server function.
47
+
48
+ See: https://github.com/apache/arrow/issues/36844
49
+ for more details...
50
+ """
51
+
52
+ @functools.wraps(func)
53
+ def wrapper_decorator(*args, **kwargs):
54
+ try:
55
+ import pydevd
56
+
57
+ pydevd.connected = True
58
+ pydevd.settrace(suspend=False)
59
+ except ImportError:
60
+ # Not running in debugger
61
+ pass
62
+ value = func(*args, **kwargs)
63
+ return value
64
+
65
+ return wrapper_decorator
66
+
67
+
68
+ class ADBCServerBase:
69
+ def get_sql_info(self, _req: CommandGetSqlInfo) -> pa.RecordBatchReader:
70
+ """Default implementation that reports no support for any complex features."""
71
+ info = {
72
+ SqlInfo.FLIGHT_SQL_SERVER_NAME: "Spiral ADBC Server",
73
+ SqlInfo.FLIGHT_SQL_SERVER_VERSION: "0.0.1",
74
+ SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION: pa.__version__,
75
+ SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY: True,
76
+ SqlInfo.FLIGHT_SQL_SERVER_TRANSACTION: SqlSupportedTransaction.NONE.value,
77
+ }
78
+
79
+ # See https://github.com/apache/arrow-adbc/blob/38c21c2311a59803559cb0091b3f34180c28b25f/rust/core/src/schemas.rs#L35
80
+ union_fields = [
81
+ pa.field("string_value", pa.string()),
82
+ pa.field("bool_value", pa.bool_()),
83
+ pa.field("int64_value", pa.int64()),
84
+ pa.field("int32_bitmask", pa.int32()),
85
+ pa.field("string_list", pa.list_(pa.string())),
86
+ pa.field(
87
+ "int32_to_int32_list_map",
88
+ pa.map_(pa.int32(), pa.list_(pa.int32()), keys_sorted=False),
89
+ ),
90
+ ]
91
+ schema = pa.schema(
92
+ [
93
+ pa.field("info_name", pa.uint32(), nullable=False),
94
+ pa.field("info_value", pa.dense_union(union_fields), nullable=False),
95
+ ]
96
+ )
97
+
98
+ # PyArrow doesn't support creating a dense union for us :(
99
+ types = []
100
+ offsets = []
101
+ ints = []
102
+ bools = []
103
+ strs = []
104
+ for value in info.values():
105
+ if isinstance(value, str):
106
+ types.append(0)
107
+ offsets.append(len(strs))
108
+ strs.append(value)
109
+ elif isinstance(value, bool):
110
+ types.append(1)
111
+ offsets.append(len(bools))
112
+ bools.append(value)
113
+ else:
114
+ types.append(1)
115
+ offsets.append(len(ints))
116
+ ints.append(value)
117
+
118
+ values = pa.UnionArray.from_dense(
119
+ pa.array(types, type=pa.int8()),
120
+ pa.array(offsets, type=pa.int32()),
121
+ [pa.array(data, type=f.type) for data, f in zip([strs, bools, ints, [], [], []], union_fields)],
122
+ [f.name for f in union_fields],
123
+ )
124
+
125
+ return pa.table(data=[pa.array(list(info.keys()), type=pa.uint32()), values], schema=schema).to_reader()
126
+
127
+ @abc.abstractmethod
128
+ def get_catalogs(self, req: CommandGetCatalogs) -> pa.RecordBatchReader: ...
129
+
130
+ @abc.abstractmethod
131
+ def get_db_schemas(self, req: CommandGetDbSchemas) -> pa.RecordBatchReader: ...
132
+
133
+ @abc.abstractmethod
134
+ def get_tables(self, req: CommandGetTables) -> pa.RecordBatchReader: ...
135
+
136
+ @abc.abstractmethod
137
+ def statement_query(self, req: CommandStatementQuery, limit: int | None = None) -> pa.RecordBatchReader: ...
138
+
139
+
140
+ class SpiralADBCServer(ADBCServerBase):
141
+ def __init__(self, spiral: Spiral):
142
+ self.sp = spiral
143
+
144
+ self.pool = ThreadPoolExecutor()
145
+
146
+ def open_snapshot(self, tbl) -> Snapshot:
147
+ """Open a table in the Spiral project and return it as a PyArrow Dataset."""
148
+ if tbl.catalog is None or tbl.catalog == "":
149
+ raise FlightError("Project (Data Catalog) must be specified to open a table.")
150
+
151
+ project = tbl.catalog
152
+ dataset = tbl.db or "default"
153
+ table = tbl.name
154
+
155
+ return self.sp.project(project).table(f"{dataset}.{table}").snapshot()
156
+
157
+ def get_catalogs(self, req: CommandGetCatalogs) -> pa.RecordBatchReader:
158
+ schema = pa.schema([pa.field("catalog_name", pa.string(), nullable=False)])
159
+
160
+ @debuggable
161
+ def batches():
162
+ yield pa.RecordBatch.from_arrays(
163
+ [[p.id for p in self.sp.list_projects()]],
164
+ schema=schema,
165
+ )
166
+
167
+ return pa.RecordBatchReader.from_batches(schema, batches())
168
+
169
+ def get_db_schemas(self, req: CommandGetDbSchemas) -> pa.RecordBatchReader:
170
+ """Get the schemas from the database."""
171
+
172
+ schema = pa.schema(
173
+ [
174
+ pa.field("catalog_name", pa.string()),
175
+ pa.field("db_schema_name", pa.string(), nullable=False),
176
+ ]
177
+ )
178
+
179
+ @debuggable
180
+ def batches():
181
+ if req.catalog == "":
182
+ # Empty string means databases _without_ a catalog, which we don't support
183
+ return
184
+ catalog = req.catalog
185
+
186
+ # Otherwise, catalog is either the project ID, or None.
187
+ if catalog is None:
188
+ projects = self.sp.list_projects()
189
+ else:
190
+ projects = [self.sp.project(req.catalog)]
191
+
192
+ for project in projects:
193
+ datasets = {tbl.dataset for tbl in project.list_tables()}
194
+
195
+ batch = pa.RecordBatch.from_arrays(
196
+ [
197
+ [project.id] * len(datasets),
198
+ list(datasets),
199
+ ],
200
+ schema=schema,
201
+ )
202
+
203
+ if req.db_schema_filter_pattern:
204
+ mask = pc.match_like(batch["db_schema_name"], req.db_schema_filter_pattern)
205
+ batch = batch.filter(mask)
206
+
207
+ yield batch
208
+
209
+ return pa.RecordBatchReader.from_batches(schema, batches())
210
+
211
+ def get_tables(self, req: CommandGetTables) -> pa.RecordBatchReader:
212
+ schema = pa.schema(
213
+ [
214
+ pa.field("catalog_name", pa.string()),
215
+ pa.field("db_schema_name", pa.string()),
216
+ pa.field("table_name", pa.string(), nullable=False),
217
+ pa.field("table_type", pa.string(), nullable=False),
218
+ ]
219
+ + [pa.field("table_schema", pa.binary(), nullable=False)]
220
+ if req.include_schema
221
+ else []
222
+ )
223
+
224
+ @debuggable
225
+ def batches():
226
+ if req.catalog == "":
227
+ # Empty string means databases _without_ a catalog, which we don't support
228
+ return
229
+
230
+ if req.catalog is None:
231
+ projects = list(self.sp.list_projects())
232
+ else:
233
+ projects = [self.sp.project(req.catalog)]
234
+ projects = sorted(projects, key=lambda p: p.id)
235
+
236
+ def _process_project(project):
237
+ tables: list[TableResource] = project.list_tables()
238
+
239
+ rows = []
240
+ for table in tables:
241
+ row = {
242
+ "catalog_name": project.id,
243
+ "db_schema_name": table.dataset,
244
+ "table_name": table.table,
245
+ "table_type": "TABLE",
246
+ }
247
+
248
+ if req.include_schema:
249
+ open_table = project.table(f"{table.dataset}.{table.table}")
250
+ row["table_schema"] = open_table.snapshot().to_dataset().schema.serialize().to_pybytes()
251
+
252
+ rows.append(row)
253
+
254
+ return pa.RecordBatch.from_pylist(rows, schema=schema)
255
+
256
+ yield from self.pool.map(_process_project, projects)
257
+
258
+ return pa.RecordBatchReader.from_batches(schema, batches())
259
+
260
+ @debuggable
261
+ def statement_query(self, req: CommandStatementQuery, limit: int | None = None) -> pa.RecordBatchReader:
262
+ # Extract the tables from the query, and bring them into the Python locals scope.
263
+ expr = sqlglot.parse_one(req.query, dialect="duckdb")
264
+ datasets = {}
265
+ for tbl in expr.find_all(exp.Table):
266
+ # We swap the three-part identifier out for a single identifier
267
+ # This lets us register a PyArrow Dataset with DuckDB for the query.
268
+ snapshot = self.open_snapshot(tbl)
269
+ name = snapshot.table.table_id
270
+ datasets[name] = snapshot.to_dataset()
271
+ tbl.replace(exp.table_(table=name))
272
+
273
+ try:
274
+ import duckdb
275
+ except ImportError:
276
+ raise FlightError("DuckDB is required for SQL queries.")
277
+
278
+ try:
279
+ # Create a DuckDB connection and register the datasets
280
+ conn = duckdb.connect()
281
+ for name, dataset in datasets.items():
282
+ conn.register(name, dataset)
283
+ sql = conn.sql(expr.sql(dialect="duckdb"))
284
+ except Exception as e:
285
+ raise FlightError(str(e))
286
+
287
+ if limit is not None:
288
+ sql = sql.limit(limit)
289
+
290
+ return sql.fetch_arrow_reader(batch_size=1_000)
291
+
292
+
293
+ class ADBCFlightServer(FlightServerBase):
294
+ """An implementation of a FlightSQL ADBC server."""
295
+
296
+ def __init__(self, abdc: ADBCServerBase, *, location=None, **kwargs):
297
+ super().__init__(location=location, **kwargs)
298
+ self.location = location
299
+ self.adbc = abdc
300
+
301
+ self.host = "localhost"
302
+ self.tls = False
303
+ if location:
304
+ parts = urlparse(location)
305
+ self.host = parts.hostname
306
+ self.tls = parts.scheme.endswith("s")
307
+
308
+ @debuggable
309
+ def do_action(self, context: ServerCallContext, action: Action):
310
+ log.info("DoAction %s: %s", context.peer(), action)
311
+ super().do_action(context, action)
312
+
313
+ @debuggable
314
+ def do_exchange(self, context: ServerCallContext, descriptor: FlightDescriptor, reader, writer):
315
+ log.info("DoExchange %s: %s", context.peer(), descriptor)
316
+ super().do_exchange(context, descriptor, reader, writer)
317
+
318
+ @debuggable
319
+ def do_get(self, context: ServerCallContext, ticket: Ticket):
320
+ log.info("DoGet %s: %s", context.peer(), ticket)
321
+ req = self.parse_command(ticket.ticket)
322
+ match req:
323
+ case CommandGetSqlInfo():
324
+ return RecordBatchStream(self.adbc.get_sql_info(req))
325
+ case CommandGetCatalogs():
326
+ return RecordBatchStream(self.adbc.get_catalogs(req))
327
+ case CommandGetDbSchemas():
328
+ return RecordBatchStream(self.adbc.get_db_schemas(req))
329
+ case CommandGetTables():
330
+ return RecordBatchStream(self.adbc.get_tables(req))
331
+ case CommandStatementQuery():
332
+ return RecordBatchStream(self.adbc.statement_query(req))
333
+ case _:
334
+ raise NotImplementedError(f"Unsupported do_Get: {req}")
335
+
336
+ @debuggable
337
+ def do_put(
338
+ self,
339
+ context: ServerCallContext,
340
+ descriptor: FlightDescriptor,
341
+ reader: MetadataRecordBatchReader,
342
+ writer: FlightMetadataWriter,
343
+ ):
344
+ log.info("DoPut %s: %s", context.peer(), descriptor)
345
+ super().do_put(context, descriptor, reader, writer)
346
+
347
+ @debuggable
348
+ def get_flight_info(self, context: ServerCallContext, descriptor: FlightDescriptor) -> FlightInfo:
349
+ log.info("GetFlightInfo %s: %s", context.peer(), descriptor)
350
+ req = self.parse_command(descriptor.command)
351
+ match req:
352
+ case CommandGetSqlInfo():
353
+ # Each metadata type contributes to the schema.
354
+ schema = self.adbc.get_sql_info(req).schema
355
+ case CommandGetCatalogs():
356
+ schema = self.adbc.get_catalogs(req).schema
357
+ case CommandGetDbSchemas():
358
+ schema = self.adbc.get_db_schemas(req).schema
359
+ case CommandGetTables():
360
+ schema = self.adbc.get_tables(req).schema
361
+ case CommandStatementQuery():
362
+ schema = self.adbc.statement_query(req, limit=0).schema
363
+ case _:
364
+ raise NotImplementedError(f"Unsupported command: {req}")
365
+
366
+ return self._make_flight_info(self.descriptor_to_key(descriptor), descriptor, schema)
367
+
368
+ @staticmethod
369
+ def parse_command(command: bytes):
370
+ command = Any().parse(command)
371
+
372
+ if not command.type_url.startswith("type.googleapis.com/arrow.flight.protocol.sql."):
373
+ raise NotImplementedError(f"Unsupported command: {command.type_url}")
374
+
375
+ proto_cls_name = command.type_url[len("type.googleapis.com/arrow.flight.protocol.sql.") :]
376
+ proto_cls = getattr(rpc, proto_cls_name)
377
+ return proto_cls().parse(command.value)
378
+
379
+ @staticmethod
380
+ def descriptor_to_key(descriptor):
381
+ return descriptor.command
382
+
383
+ @debuggable
384
+ def get_schema(self, context: ServerCallContext, descriptor: FlightDescriptor):
385
+ log.info("GetSchema %s: %s", context.peer(), descriptor)
386
+ return super().get_schema(context, descriptor)
387
+
388
+ @debuggable
389
+ def list_actions(self, context: ServerCallContext):
390
+ log.info("ListActions %s", context.peer())
391
+ super().list_actions(context)
392
+
393
+ @debuggable
394
+ def list_flights(self, context: ServerCallContext, criteria):
395
+ log.info("ListFlights %s: %s", context.peer(), criteria)
396
+ super().list_flights(context, criteria)
397
+
398
+ def _make_flight_info(self, key, descriptor, schema: pa.Schema):
399
+ # If we pass zero locations, the FlightSQL client should attempt to use the original connection.
400
+ endpoints = [FlightEndpoint(key, [])]
401
+ return FlightInfo(schema, descriptor, endpoints, -1, -1)
402
+
403
+
404
+ if __name__ == "__main__":
405
+ import logging
406
+
407
+ logging.basicConfig()
408
+ logging.getLogger("spiral").setLevel(logging.DEBUG)
409
+
410
+ server = ADBCFlightServer(SpiralADBCServer(Spiral()), location="grpc://localhost:5005")
411
+ server.serve()