pyspiral 0.3.1__cp310-abi3-macosx_11_0_arm64.whl → 0.4.1__cp310-abi3-macosx_11_0_arm64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {pyspiral-0.3.1.dist-info → pyspiral-0.4.1.dist-info}/METADATA +9 -13
  2. pyspiral-0.4.1.dist-info/RECORD +98 -0
  3. {pyspiral-0.3.1.dist-info → pyspiral-0.4.1.dist-info}/WHEEL +1 -1
  4. spiral/__init__.py +6 -9
  5. spiral/_lib.abi3.so +0 -0
  6. spiral/adbc.py +21 -14
  7. spiral/api/__init__.py +14 -175
  8. spiral/api/admin.py +12 -26
  9. spiral/api/client.py +160 -0
  10. spiral/api/filesystems.py +100 -72
  11. spiral/api/organizations.py +45 -58
  12. spiral/api/projects.py +171 -134
  13. spiral/api/telemetry.py +19 -0
  14. spiral/api/types.py +20 -0
  15. spiral/api/workloads.py +32 -25
  16. spiral/{arrow.py → arrow_.py} +12 -0
  17. spiral/cli/__init__.py +2 -5
  18. spiral/cli/admin.py +7 -12
  19. spiral/cli/app.py +23 -6
  20. spiral/cli/console.py +1 -1
  21. spiral/cli/fs.py +82 -17
  22. spiral/cli/iceberg/__init__.py +7 -0
  23. spiral/cli/iceberg/namespaces.py +47 -0
  24. spiral/cli/iceberg/tables.py +60 -0
  25. spiral/cli/indexes/__init__.py +19 -0
  26. spiral/cli/login.py +14 -5
  27. spiral/cli/orgs.py +90 -0
  28. spiral/cli/printer.py +9 -1
  29. spiral/cli/projects.py +136 -0
  30. spiral/cli/state.py +2 -0
  31. spiral/cli/tables/__init__.py +121 -0
  32. spiral/cli/telemetry.py +18 -0
  33. spiral/cli/types.py +8 -10
  34. spiral/cli/{workload.py → workloads.py} +11 -11
  35. spiral/{catalog.py → client.py} +23 -37
  36. spiral/core/client/__init__.pyi +117 -0
  37. spiral/core/index/__init__.pyi +15 -0
  38. spiral/core/{core → table}/__init__.pyi +44 -17
  39. spiral/core/{manifests → table/manifests}/__init__.pyi +5 -23
  40. spiral/core/table/metastore/__init__.pyi +62 -0
  41. spiral/core/{spec → table/spec}/__init__.pyi +41 -66
  42. spiral/datetime_.py +27 -0
  43. spiral/expressions/__init__.py +26 -18
  44. spiral/expressions/base.py +5 -5
  45. spiral/expressions/list_.py +1 -1
  46. spiral/expressions/mp4.py +2 -9
  47. spiral/expressions/png.py +1 -1
  48. spiral/expressions/qoi.py +1 -1
  49. spiral/expressions/refs.py +3 -9
  50. spiral/expressions/struct.py +7 -5
  51. spiral/expressions/text.py +62 -0
  52. spiral/expressions/udf.py +3 -3
  53. spiral/iceberg/__init__.py +3 -0
  54. spiral/iceberg/client.py +33 -0
  55. spiral/indexes/__init__.py +5 -0
  56. spiral/indexes/client.py +137 -0
  57. spiral/indexes/index.py +34 -0
  58. spiral/indexes/scan.py +22 -0
  59. spiral/project.py +19 -110
  60. spiral/{proto → protogen}/_/scandal/__init__.py +23 -135
  61. spiral/protogen/_/spiral/table/__init__.py +22 -0
  62. spiral/protogen/substrait/__init__.py +3399 -0
  63. spiral/protogen/substrait/extensions/__init__.py +115 -0
  64. spiral/server.py +17 -0
  65. spiral/settings.py +29 -91
  66. spiral/substrait_.py +9 -5
  67. spiral/tables/__init__.py +12 -0
  68. spiral/tables/client.py +130 -0
  69. spiral/{dataset.py → tables/dataset.py} +9 -199
  70. spiral/tables/debug/manifests.py +70 -0
  71. spiral/tables/debug/metrics.py +56 -0
  72. spiral/{debug.py → tables/debug/scan.py} +6 -9
  73. spiral/{maintenance.py → tables/maintenance.py} +1 -1
  74. spiral/{scan_.py → tables/scan.py} +63 -89
  75. spiral/tables/snapshot.py +78 -0
  76. spiral/{table.py → tables/table.py} +59 -73
  77. spiral/{txn.py → tables/transaction.py} +7 -3
  78. pyspiral-0.3.1.dist-info/RECORD +0 -85
  79. spiral/api/tables.py +0 -91
  80. spiral/api/tokens.py +0 -56
  81. spiral/authn/authn.py +0 -89
  82. spiral/authn/device.py +0 -206
  83. spiral/authn/github_.py +0 -33
  84. spiral/authn/modal_.py +0 -18
  85. spiral/cli/org.py +0 -90
  86. spiral/cli/project.py +0 -109
  87. spiral/cli/table.py +0 -20
  88. spiral/cli/token.py +0 -27
  89. spiral/core/metastore/__init__.pyi +0 -91
  90. spiral/proto/_/spfs/__init__.py +0 -36
  91. spiral/proto/_/spiral/table/__init__.py +0 -276
  92. spiral/proto/_/spiraldb/metastore/__init__.py +0 -499
  93. spiral/proto/__init__.py +0 -0
  94. spiral/proto/scandal/__init__.py +0 -45
  95. spiral/proto/spiral/__init__.py +0 -0
  96. spiral/proto/spiral/table/__init__.py +0 -96
  97. {pyspiral-0.3.1.dist-info → pyspiral-0.4.1.dist-info}/entry_points.txt +0 -0
  98. /spiral/{authn/__init__.py → core/__init__.pyi} +0 -0
  99. /spiral/{core → protogen/_}/__init__.py +0 -0
  100. /spiral/{proto/_ → protogen/_/arrow}/__init__.py +0 -0
  101. /spiral/{proto/_/arrow → protogen/_/arrow/flight}/__init__.py +0 -0
  102. /spiral/{proto/_/arrow/flight → protogen/_/arrow/flight/protocol}/__init__.py +0 -0
  103. /spiral/{proto → protogen}/_/arrow/flight/protocol/sql/__init__.py +0 -0
  104. /spiral/{proto/_/arrow/flight/protocol → protogen/_/spiral}/__init__.py +0 -0
  105. /spiral/{proto → protogen/_}/substrait/__init__.py +0 -0
  106. /spiral/{proto → protogen/_}/substrait/extensions/__init__.py +0 -0
  107. /spiral/{proto/_/spiral → protogen}/__init__.py +0 -0
  108. /spiral/{proto → protogen}/util.py +0 -0
  109. /spiral/{proto/_/spiraldb → tables/debug}/__init__.py +0 -0
@@ -1,276 +0,0 @@
1
- # Generated by the protocol buffer compiler. DO NOT EDIT!
2
- # sources: spiral/table/common.proto, spiral/table/metadata.proto, spiral/table/statistics.proto, spiral/table/wal.proto
3
- # plugin: python-betterproto
4
- # This file has been @generated
5
-
6
- from dataclasses import dataclass
7
- from typing import (
8
- List,
9
- Optional,
10
- )
11
-
12
- import betterproto
13
-
14
-
15
- class FileFormat(betterproto.Enum):
16
- UNSPECIFIED = 0
17
- PARQUET = 1
18
- PROTOBUF = 2
19
- BINARY_ARRAY = 3
20
- VORTEX = 4
21
-
22
-
23
- class Level(betterproto.Enum):
24
- UNSPECIFIED = 0
25
- L0 = 1
26
- L1 = 2
27
-
28
-
29
- @dataclass(eq=False, repr=False)
30
- class ApproximateSetMembership(betterproto.Message):
31
- bloom_filter: "BloomFilter" = betterproto.message_field(
32
- 2, group="membership_strategy"
33
- )
34
-
35
-
36
- @dataclass(eq=False, repr=False)
37
- class BloomFilter(betterproto.Message):
38
- bit_vec: bytes = betterproto.bytes_field(1)
39
- bitmap_bits: int = betterproto.uint64_field(2)
40
- k_num: int = betterproto.uint32_field(3)
41
-
42
-
43
- @dataclass(eq=False, repr=False)
44
- class Schema(betterproto.Message):
45
- arrow: bytes = betterproto.bytes_field(1)
46
-
47
-
48
- @dataclass(eq=False, repr=False)
49
- class ColumnGroup(betterproto.Message):
50
- parts: List[str] = betterproto.string_field(1)
51
-
52
-
53
- @dataclass(eq=False, repr=False)
54
- class KeySpan(betterproto.Message):
55
- """/ Span of keys defined as indices into a key space (exclusive end)."""
56
-
57
- begin: int = betterproto.uint64_field(1)
58
- end: int = betterproto.uint64_field(2)
59
-
60
-
61
- @dataclass(eq=False, repr=False)
62
- class KeyExtent(betterproto.Message):
63
- """/ Extent of keys in terms of min and max (an inclusive range)."""
64
-
65
- min: bytes = betterproto.bytes_field(1)
66
- max: bytes = betterproto.bytes_field(2)
67
-
68
-
69
- @dataclass(eq=False, repr=False)
70
- class KeyMap(betterproto.Message):
71
- """/ Displacement map."""
72
-
73
- bitmap: bytes = betterproto.bytes_field(1)
74
-
75
-
76
- @dataclass(eq=False, repr=False)
77
- class ManifestHandle(betterproto.Message):
78
- """
79
- / Handle to a manifest file, with additional metadata to accelerate reads.
80
- """
81
-
82
- id: str = betterproto.string_field(1)
83
- format: "FileFormat" = betterproto.enum_field(2)
84
- file_size: int = betterproto.uint64_field(3)
85
- spfs_format_metadata: Optional[bytes] = betterproto.bytes_field(4, optional=True)
86
-
87
-
88
- @dataclass(eq=False, repr=False)
89
- class VersionedSchema(betterproto.Message):
90
- """/ Timestamped schema."""
91
-
92
- ts: int = betterproto.uint64_field(1)
93
- schema: "Schema" = betterproto.message_field(2)
94
- column_ids: List[str] = betterproto.string_field(3)
95
- """
96
- List of column IDs, in the same order as columns in the schema.
97
- Must have the same length as the number of columns in the schema.
98
- """
99
-
100
-
101
- @dataclass(eq=False, repr=False)
102
- class WriteAheadLog(betterproto.Message):
103
- """
104
- / Table's Write Ahead Log stores sequence of operations and table metadata.
105
- """
106
-
107
- entries: List["LogEntry"] = betterproto.message_field(1)
108
- truncated_up_to: int = betterproto.uint64_field(3)
109
- """
110
- Timestamp of the latest entry that has been removed from the log.
111
- Entries with ts <= truncated_up_to are NOT present in the log.
112
- """
113
-
114
- ks_manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
115
- 2, optional=True
116
- )
117
- """An absent manifest means that one has not yet been written."""
118
-
119
-
120
- @dataclass(eq=False, repr=False)
121
- class LogEntry(betterproto.Message):
122
- ts: int = betterproto.uint64_field(1)
123
- key_space_write: "KeySpaceWriteOp" = betterproto.message_field(2, group="operation")
124
- fragment_set_write: "FragmentSetWriteOp" = betterproto.message_field(
125
- 3, group="operation"
126
- )
127
- configuration: "ConfigurationOp" = betterproto.message_field(4, group="operation")
128
- schema_evolution: "SchemaEvolutionOp" = betterproto.message_field(
129
- 5, group="operation"
130
- )
131
- schema_break: "SchemaBreakOp" = betterproto.message_field(6, group="operation")
132
- compact_key_space: "CompactKeySpaceOp" = betterproto.message_field(
133
- 7, group="operation"
134
- )
135
- compact_column_group: "CompactColumnGroupOp" = betterproto.message_field(
136
- 8, group="operation"
137
- )
138
-
139
-
140
- @dataclass(eq=False, repr=False)
141
- class KeySpaceWriteOp(betterproto.Message):
142
- ks_id: str = betterproto.string_field(1)
143
- manifest_handle: "ManifestHandle" = betterproto.message_field(2)
144
-
145
-
146
- @dataclass(eq=False, repr=False)
147
- class FragmentSetWriteOp(betterproto.Message):
148
- column_group: "ColumnGroup" = betterproto.message_field(1)
149
- fs_id: str = betterproto.string_field(2)
150
- fs_level: "Level" = betterproto.enum_field(3)
151
- manifest_handle: "ManifestHandle" = betterproto.message_field(4)
152
- key_span: "KeySpan" = betterproto.message_field(5)
153
- key_extent: "KeyExtent" = betterproto.message_field(6)
154
- column_ids: List[str] = betterproto.string_field(7)
155
- data_ts: Optional[int] = betterproto.uint64_field(8, optional=True)
156
- """
157
- Timestamp of the data in the fragments.
158
- Used as committed_ts for files in the manifest.
159
- If not present, timestamp of the operation is used.
160
- """
161
-
162
-
163
- @dataclass(eq=False, repr=False)
164
- class ConfigurationOp(betterproto.Message):
165
- column_group: "ColumnGroup" = betterproto.message_field(1)
166
- immutable_schema: Optional[bool] = betterproto.bool_field(2, optional=True)
167
- """All column group configuration is stored in column group metadata."""
168
-
169
-
170
- @dataclass(eq=False, repr=False)
171
- class SchemaEvolutionOp(betterproto.Message):
172
- column_group: "ColumnGroup" = betterproto.message_field(1)
173
- new_schema: "VersionedSchema" = betterproto.message_field(2)
174
-
175
-
176
- @dataclass(eq=False, repr=False)
177
- class SchemaBreakOp(betterproto.Message):
178
- column_group: "ColumnGroup" = betterproto.message_field(1)
179
- removed_column_names: List[str] = betterproto.string_field(2)
180
-
181
-
182
- @dataclass(eq=False, repr=False)
183
- class CompactKeySpaceOp(betterproto.Message):
184
- results: List["CompactKeySpaceResult"] = betterproto.message_field(1)
185
-
186
-
187
- @dataclass(eq=False, repr=False)
188
- class CompactKeySpaceResult(betterproto.Message):
189
- """
190
- TODO(marko): Do we really need to know all of this? UpdateKeySpaceOp?
191
- """
192
-
193
- ks_id: str = betterproto.string_field(1)
194
- compacted: "CompactKeySpaceResultCompacted" = betterproto.message_field(
195
- 2, group="action"
196
- )
197
- """Key space has been compacted."""
198
-
199
- created: "CompactKeySpaceResultCreated" = betterproto.message_field(
200
- 3, group="action"
201
- )
202
- """New output key space has been created."""
203
-
204
- moved: "CompactKeySpaceResultMoved" = betterproto.message_field(4, group="action")
205
- """Key space has been promoted to L1."""
206
-
207
- extended: "CompactKeySpaceResultExtended" = betterproto.message_field(
208
- 5, group="action"
209
- )
210
- """Key space has been extended with new key files."""
211
-
212
-
213
- @dataclass(eq=False, repr=False)
214
- class CompactKeySpaceResultCompacted(betterproto.Message):
215
- pass
216
-
217
-
218
- @dataclass(eq=False, repr=False)
219
- class CompactKeySpaceResultCreated(betterproto.Message):
220
- pass
221
-
222
-
223
- @dataclass(eq=False, repr=False)
224
- class CompactKeySpaceResultMoved(betterproto.Message):
225
- pass
226
-
227
-
228
- @dataclass(eq=False, repr=False)
229
- class CompactKeySpaceResultExtended(betterproto.Message):
230
- pass
231
-
232
-
233
- @dataclass(eq=False, repr=False)
234
- class CompactColumnGroupOp(betterproto.Message):
235
- column_group: "ColumnGroup" = betterproto.message_field(1)
236
- from_fragment_ids: List[str] = betterproto.string_field(2)
237
- """
238
- NOTE: While key space compaction always compacts a full key space,
239
- column group compaction can compact only a subset of the fragment set,
240
- therefore the event specifies the fragment ids.
241
- """
242
-
243
- into_fs_ids: List[str] = betterproto.string_field(3)
244
-
245
-
246
- @dataclass(eq=False, repr=False)
247
- class ColumnGroupMetadata(betterproto.Message):
248
- column_group: "ColumnGroup" = betterproto.message_field(1)
249
- manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
250
- 2, optional=True
251
- )
252
- """An absent manifest means that one has not yet been written."""
253
-
254
- last_modified_at: int = betterproto.uint64_field(3)
255
- """Timestamp of the last WAL entry that modified this metadata."""
256
-
257
- schema_versions: List["VersionedSchema"] = betterproto.message_field(4)
258
- """
259
- TODO(marko): Add config that truncates this list and breaks time travel.
260
- Versions of the schema. Higher index is more recent, last element is latest
261
- schema. Stored to support time travel through the schema, e.g. reading
262
- deleted column.
263
- """
264
-
265
- immutable_schema: bool = betterproto.bool_field(5)
266
- """
267
- True if schema can NOT be evolved on write.
268
- An explicit schema evolution is required.
269
- """
270
-
271
- schema_salt: int = betterproto.int32_field(6)
272
- """
273
- Schema salt is used to compute column IDs. It is modified on breaking
274
- change, e.g. column deletion. This ensures that if the previously existing
275
- column is added again, it will have a different ID.
276
- """