pyspiral 0.3.1__cp310-abi3-macosx_11_0_arm64.whl → 0.4.1__cp310-abi3-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyspiral-0.3.1.dist-info → pyspiral-0.4.1.dist-info}/METADATA +9 -13
- pyspiral-0.4.1.dist-info/RECORD +98 -0
- {pyspiral-0.3.1.dist-info → pyspiral-0.4.1.dist-info}/WHEEL +1 -1
- spiral/__init__.py +6 -9
- spiral/_lib.abi3.so +0 -0
- spiral/adbc.py +21 -14
- spiral/api/__init__.py +14 -175
- spiral/api/admin.py +12 -26
- spiral/api/client.py +160 -0
- spiral/api/filesystems.py +100 -72
- spiral/api/organizations.py +45 -58
- spiral/api/projects.py +171 -134
- spiral/api/telemetry.py +19 -0
- spiral/api/types.py +20 -0
- spiral/api/workloads.py +32 -25
- spiral/{arrow.py → arrow_.py} +12 -0
- spiral/cli/__init__.py +2 -5
- spiral/cli/admin.py +7 -12
- spiral/cli/app.py +23 -6
- spiral/cli/console.py +1 -1
- spiral/cli/fs.py +82 -17
- spiral/cli/iceberg/__init__.py +7 -0
- spiral/cli/iceberg/namespaces.py +47 -0
- spiral/cli/iceberg/tables.py +60 -0
- spiral/cli/indexes/__init__.py +19 -0
- spiral/cli/login.py +14 -5
- spiral/cli/orgs.py +90 -0
- spiral/cli/printer.py +9 -1
- spiral/cli/projects.py +136 -0
- spiral/cli/state.py +2 -0
- spiral/cli/tables/__init__.py +121 -0
- spiral/cli/telemetry.py +18 -0
- spiral/cli/types.py +8 -10
- spiral/cli/{workload.py → workloads.py} +11 -11
- spiral/{catalog.py → client.py} +23 -37
- spiral/core/client/__init__.pyi +117 -0
- spiral/core/index/__init__.pyi +15 -0
- spiral/core/{core → table}/__init__.pyi +44 -17
- spiral/core/{manifests → table/manifests}/__init__.pyi +5 -23
- spiral/core/table/metastore/__init__.pyi +62 -0
- spiral/core/{spec → table/spec}/__init__.pyi +41 -66
- spiral/datetime_.py +27 -0
- spiral/expressions/__init__.py +26 -18
- spiral/expressions/base.py +5 -5
- spiral/expressions/list_.py +1 -1
- spiral/expressions/mp4.py +2 -9
- spiral/expressions/png.py +1 -1
- spiral/expressions/qoi.py +1 -1
- spiral/expressions/refs.py +3 -9
- spiral/expressions/struct.py +7 -5
- spiral/expressions/text.py +62 -0
- spiral/expressions/udf.py +3 -3
- spiral/iceberg/__init__.py +3 -0
- spiral/iceberg/client.py +33 -0
- spiral/indexes/__init__.py +5 -0
- spiral/indexes/client.py +137 -0
- spiral/indexes/index.py +34 -0
- spiral/indexes/scan.py +22 -0
- spiral/project.py +19 -110
- spiral/{proto → protogen}/_/scandal/__init__.py +23 -135
- spiral/protogen/_/spiral/table/__init__.py +22 -0
- spiral/protogen/substrait/__init__.py +3399 -0
- spiral/protogen/substrait/extensions/__init__.py +115 -0
- spiral/server.py +17 -0
- spiral/settings.py +29 -91
- spiral/substrait_.py +9 -5
- spiral/tables/__init__.py +12 -0
- spiral/tables/client.py +130 -0
- spiral/{dataset.py → tables/dataset.py} +9 -199
- spiral/tables/debug/manifests.py +70 -0
- spiral/tables/debug/metrics.py +56 -0
- spiral/{debug.py → tables/debug/scan.py} +6 -9
- spiral/{maintenance.py → tables/maintenance.py} +1 -1
- spiral/{scan_.py → tables/scan.py} +63 -89
- spiral/tables/snapshot.py +78 -0
- spiral/{table.py → tables/table.py} +59 -73
- spiral/{txn.py → tables/transaction.py} +7 -3
- pyspiral-0.3.1.dist-info/RECORD +0 -85
- spiral/api/tables.py +0 -91
- spiral/api/tokens.py +0 -56
- spiral/authn/authn.py +0 -89
- spiral/authn/device.py +0 -206
- spiral/authn/github_.py +0 -33
- spiral/authn/modal_.py +0 -18
- spiral/cli/org.py +0 -90
- spiral/cli/project.py +0 -109
- spiral/cli/table.py +0 -20
- spiral/cli/token.py +0 -27
- spiral/core/metastore/__init__.pyi +0 -91
- spiral/proto/_/spfs/__init__.py +0 -36
- spiral/proto/_/spiral/table/__init__.py +0 -276
- spiral/proto/_/spiraldb/metastore/__init__.py +0 -499
- spiral/proto/__init__.py +0 -0
- spiral/proto/scandal/__init__.py +0 -45
- spiral/proto/spiral/__init__.py +0 -0
- spiral/proto/spiral/table/__init__.py +0 -96
- {pyspiral-0.3.1.dist-info → pyspiral-0.4.1.dist-info}/entry_points.txt +0 -0
- /spiral/{authn/__init__.py → core/__init__.pyi} +0 -0
- /spiral/{core → protogen/_}/__init__.py +0 -0
- /spiral/{proto/_ → protogen/_/arrow}/__init__.py +0 -0
- /spiral/{proto/_/arrow → protogen/_/arrow/flight}/__init__.py +0 -0
- /spiral/{proto/_/arrow/flight → protogen/_/arrow/flight/protocol}/__init__.py +0 -0
- /spiral/{proto → protogen}/_/arrow/flight/protocol/sql/__init__.py +0 -0
- /spiral/{proto/_/arrow/flight/protocol → protogen/_/spiral}/__init__.py +0 -0
- /spiral/{proto → protogen/_}/substrait/__init__.py +0 -0
- /spiral/{proto → protogen/_}/substrait/extensions/__init__.py +0 -0
- /spiral/{proto/_/spiral → protogen}/__init__.py +0 -0
- /spiral/{proto → protogen}/util.py +0 -0
- /spiral/{proto/_/spiraldb → tables/debug}/__init__.py +0 -0
@@ -1,276 +0,0 @@
|
|
1
|
-
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
2
|
-
# sources: spiral/table/common.proto, spiral/table/metadata.proto, spiral/table/statistics.proto, spiral/table/wal.proto
|
3
|
-
# plugin: python-betterproto
|
4
|
-
# This file has been @generated
|
5
|
-
|
6
|
-
from dataclasses import dataclass
|
7
|
-
from typing import (
|
8
|
-
List,
|
9
|
-
Optional,
|
10
|
-
)
|
11
|
-
|
12
|
-
import betterproto
|
13
|
-
|
14
|
-
|
15
|
-
class FileFormat(betterproto.Enum):
|
16
|
-
UNSPECIFIED = 0
|
17
|
-
PARQUET = 1
|
18
|
-
PROTOBUF = 2
|
19
|
-
BINARY_ARRAY = 3
|
20
|
-
VORTEX = 4
|
21
|
-
|
22
|
-
|
23
|
-
class Level(betterproto.Enum):
|
24
|
-
UNSPECIFIED = 0
|
25
|
-
L0 = 1
|
26
|
-
L1 = 2
|
27
|
-
|
28
|
-
|
29
|
-
@dataclass(eq=False, repr=False)
|
30
|
-
class ApproximateSetMembership(betterproto.Message):
|
31
|
-
bloom_filter: "BloomFilter" = betterproto.message_field(
|
32
|
-
2, group="membership_strategy"
|
33
|
-
)
|
34
|
-
|
35
|
-
|
36
|
-
@dataclass(eq=False, repr=False)
|
37
|
-
class BloomFilter(betterproto.Message):
|
38
|
-
bit_vec: bytes = betterproto.bytes_field(1)
|
39
|
-
bitmap_bits: int = betterproto.uint64_field(2)
|
40
|
-
k_num: int = betterproto.uint32_field(3)
|
41
|
-
|
42
|
-
|
43
|
-
@dataclass(eq=False, repr=False)
|
44
|
-
class Schema(betterproto.Message):
|
45
|
-
arrow: bytes = betterproto.bytes_field(1)
|
46
|
-
|
47
|
-
|
48
|
-
@dataclass(eq=False, repr=False)
|
49
|
-
class ColumnGroup(betterproto.Message):
|
50
|
-
parts: List[str] = betterproto.string_field(1)
|
51
|
-
|
52
|
-
|
53
|
-
@dataclass(eq=False, repr=False)
|
54
|
-
class KeySpan(betterproto.Message):
|
55
|
-
"""/ Span of keys defined as indices into a key space (exclusive end)."""
|
56
|
-
|
57
|
-
begin: int = betterproto.uint64_field(1)
|
58
|
-
end: int = betterproto.uint64_field(2)
|
59
|
-
|
60
|
-
|
61
|
-
@dataclass(eq=False, repr=False)
|
62
|
-
class KeyExtent(betterproto.Message):
|
63
|
-
"""/ Extent of keys in terms of min and max (an inclusive range)."""
|
64
|
-
|
65
|
-
min: bytes = betterproto.bytes_field(1)
|
66
|
-
max: bytes = betterproto.bytes_field(2)
|
67
|
-
|
68
|
-
|
69
|
-
@dataclass(eq=False, repr=False)
|
70
|
-
class KeyMap(betterproto.Message):
|
71
|
-
"""/ Displacement map."""
|
72
|
-
|
73
|
-
bitmap: bytes = betterproto.bytes_field(1)
|
74
|
-
|
75
|
-
|
76
|
-
@dataclass(eq=False, repr=False)
|
77
|
-
class ManifestHandle(betterproto.Message):
|
78
|
-
"""
|
79
|
-
/ Handle to a manifest file, with additional metadata to accelerate reads.
|
80
|
-
"""
|
81
|
-
|
82
|
-
id: str = betterproto.string_field(1)
|
83
|
-
format: "FileFormat" = betterproto.enum_field(2)
|
84
|
-
file_size: int = betterproto.uint64_field(3)
|
85
|
-
spfs_format_metadata: Optional[bytes] = betterproto.bytes_field(4, optional=True)
|
86
|
-
|
87
|
-
|
88
|
-
@dataclass(eq=False, repr=False)
|
89
|
-
class VersionedSchema(betterproto.Message):
|
90
|
-
"""/ Timestamped schema."""
|
91
|
-
|
92
|
-
ts: int = betterproto.uint64_field(1)
|
93
|
-
schema: "Schema" = betterproto.message_field(2)
|
94
|
-
column_ids: List[str] = betterproto.string_field(3)
|
95
|
-
"""
|
96
|
-
List of column IDs, in the same order as columns in the schema.
|
97
|
-
Must have the same length as the number of columns in the schema.
|
98
|
-
"""
|
99
|
-
|
100
|
-
|
101
|
-
@dataclass(eq=False, repr=False)
|
102
|
-
class WriteAheadLog(betterproto.Message):
|
103
|
-
"""
|
104
|
-
/ Table's Write Ahead Log stores sequence of operations and table metadata.
|
105
|
-
"""
|
106
|
-
|
107
|
-
entries: List["LogEntry"] = betterproto.message_field(1)
|
108
|
-
truncated_up_to: int = betterproto.uint64_field(3)
|
109
|
-
"""
|
110
|
-
Timestamp of the latest entry that has been removed from the log.
|
111
|
-
Entries with ts <= truncated_up_to are NOT present in the log.
|
112
|
-
"""
|
113
|
-
|
114
|
-
ks_manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
|
115
|
-
2, optional=True
|
116
|
-
)
|
117
|
-
"""An absent manifest means that one has not yet been written."""
|
118
|
-
|
119
|
-
|
120
|
-
@dataclass(eq=False, repr=False)
|
121
|
-
class LogEntry(betterproto.Message):
|
122
|
-
ts: int = betterproto.uint64_field(1)
|
123
|
-
key_space_write: "KeySpaceWriteOp" = betterproto.message_field(2, group="operation")
|
124
|
-
fragment_set_write: "FragmentSetWriteOp" = betterproto.message_field(
|
125
|
-
3, group="operation"
|
126
|
-
)
|
127
|
-
configuration: "ConfigurationOp" = betterproto.message_field(4, group="operation")
|
128
|
-
schema_evolution: "SchemaEvolutionOp" = betterproto.message_field(
|
129
|
-
5, group="operation"
|
130
|
-
)
|
131
|
-
schema_break: "SchemaBreakOp" = betterproto.message_field(6, group="operation")
|
132
|
-
compact_key_space: "CompactKeySpaceOp" = betterproto.message_field(
|
133
|
-
7, group="operation"
|
134
|
-
)
|
135
|
-
compact_column_group: "CompactColumnGroupOp" = betterproto.message_field(
|
136
|
-
8, group="operation"
|
137
|
-
)
|
138
|
-
|
139
|
-
|
140
|
-
@dataclass(eq=False, repr=False)
|
141
|
-
class KeySpaceWriteOp(betterproto.Message):
|
142
|
-
ks_id: str = betterproto.string_field(1)
|
143
|
-
manifest_handle: "ManifestHandle" = betterproto.message_field(2)
|
144
|
-
|
145
|
-
|
146
|
-
@dataclass(eq=False, repr=False)
|
147
|
-
class FragmentSetWriteOp(betterproto.Message):
|
148
|
-
column_group: "ColumnGroup" = betterproto.message_field(1)
|
149
|
-
fs_id: str = betterproto.string_field(2)
|
150
|
-
fs_level: "Level" = betterproto.enum_field(3)
|
151
|
-
manifest_handle: "ManifestHandle" = betterproto.message_field(4)
|
152
|
-
key_span: "KeySpan" = betterproto.message_field(5)
|
153
|
-
key_extent: "KeyExtent" = betterproto.message_field(6)
|
154
|
-
column_ids: List[str] = betterproto.string_field(7)
|
155
|
-
data_ts: Optional[int] = betterproto.uint64_field(8, optional=True)
|
156
|
-
"""
|
157
|
-
Timestamp of the data in the fragments.
|
158
|
-
Used as committed_ts for files in the manifest.
|
159
|
-
If not present, timestamp of the operation is used.
|
160
|
-
"""
|
161
|
-
|
162
|
-
|
163
|
-
@dataclass(eq=False, repr=False)
|
164
|
-
class ConfigurationOp(betterproto.Message):
|
165
|
-
column_group: "ColumnGroup" = betterproto.message_field(1)
|
166
|
-
immutable_schema: Optional[bool] = betterproto.bool_field(2, optional=True)
|
167
|
-
"""All column group configuration is stored in column group metadata."""
|
168
|
-
|
169
|
-
|
170
|
-
@dataclass(eq=False, repr=False)
|
171
|
-
class SchemaEvolutionOp(betterproto.Message):
|
172
|
-
column_group: "ColumnGroup" = betterproto.message_field(1)
|
173
|
-
new_schema: "VersionedSchema" = betterproto.message_field(2)
|
174
|
-
|
175
|
-
|
176
|
-
@dataclass(eq=False, repr=False)
|
177
|
-
class SchemaBreakOp(betterproto.Message):
|
178
|
-
column_group: "ColumnGroup" = betterproto.message_field(1)
|
179
|
-
removed_column_names: List[str] = betterproto.string_field(2)
|
180
|
-
|
181
|
-
|
182
|
-
@dataclass(eq=False, repr=False)
|
183
|
-
class CompactKeySpaceOp(betterproto.Message):
|
184
|
-
results: List["CompactKeySpaceResult"] = betterproto.message_field(1)
|
185
|
-
|
186
|
-
|
187
|
-
@dataclass(eq=False, repr=False)
|
188
|
-
class CompactKeySpaceResult(betterproto.Message):
|
189
|
-
"""
|
190
|
-
TODO(marko): Do we really need to know all of this? UpdateKeySpaceOp?
|
191
|
-
"""
|
192
|
-
|
193
|
-
ks_id: str = betterproto.string_field(1)
|
194
|
-
compacted: "CompactKeySpaceResultCompacted" = betterproto.message_field(
|
195
|
-
2, group="action"
|
196
|
-
)
|
197
|
-
"""Key space has been compacted."""
|
198
|
-
|
199
|
-
created: "CompactKeySpaceResultCreated" = betterproto.message_field(
|
200
|
-
3, group="action"
|
201
|
-
)
|
202
|
-
"""New output key space has been created."""
|
203
|
-
|
204
|
-
moved: "CompactKeySpaceResultMoved" = betterproto.message_field(4, group="action")
|
205
|
-
"""Key space has been promoted to L1."""
|
206
|
-
|
207
|
-
extended: "CompactKeySpaceResultExtended" = betterproto.message_field(
|
208
|
-
5, group="action"
|
209
|
-
)
|
210
|
-
"""Key space has been extended with new key files."""
|
211
|
-
|
212
|
-
|
213
|
-
@dataclass(eq=False, repr=False)
|
214
|
-
class CompactKeySpaceResultCompacted(betterproto.Message):
|
215
|
-
pass
|
216
|
-
|
217
|
-
|
218
|
-
@dataclass(eq=False, repr=False)
|
219
|
-
class CompactKeySpaceResultCreated(betterproto.Message):
|
220
|
-
pass
|
221
|
-
|
222
|
-
|
223
|
-
@dataclass(eq=False, repr=False)
|
224
|
-
class CompactKeySpaceResultMoved(betterproto.Message):
|
225
|
-
pass
|
226
|
-
|
227
|
-
|
228
|
-
@dataclass(eq=False, repr=False)
|
229
|
-
class CompactKeySpaceResultExtended(betterproto.Message):
|
230
|
-
pass
|
231
|
-
|
232
|
-
|
233
|
-
@dataclass(eq=False, repr=False)
|
234
|
-
class CompactColumnGroupOp(betterproto.Message):
|
235
|
-
column_group: "ColumnGroup" = betterproto.message_field(1)
|
236
|
-
from_fragment_ids: List[str] = betterproto.string_field(2)
|
237
|
-
"""
|
238
|
-
NOTE: While key space compaction always compacts a full key space,
|
239
|
-
column group compaction can compact only a subset of the fragment set,
|
240
|
-
therefore the event specifies the fragment ids.
|
241
|
-
"""
|
242
|
-
|
243
|
-
into_fs_ids: List[str] = betterproto.string_field(3)
|
244
|
-
|
245
|
-
|
246
|
-
@dataclass(eq=False, repr=False)
|
247
|
-
class ColumnGroupMetadata(betterproto.Message):
|
248
|
-
column_group: "ColumnGroup" = betterproto.message_field(1)
|
249
|
-
manifest_handle: Optional["ManifestHandle"] = betterproto.message_field(
|
250
|
-
2, optional=True
|
251
|
-
)
|
252
|
-
"""An absent manifest means that one has not yet been written."""
|
253
|
-
|
254
|
-
last_modified_at: int = betterproto.uint64_field(3)
|
255
|
-
"""Timestamp of the last WAL entry that modified this metadata."""
|
256
|
-
|
257
|
-
schema_versions: List["VersionedSchema"] = betterproto.message_field(4)
|
258
|
-
"""
|
259
|
-
TODO(marko): Add config that truncates this list and breaks time travel.
|
260
|
-
Versions of the schema. Higher index is more recent, last element is latest
|
261
|
-
schema. Stored to support time travel through the schema, e.g. reading
|
262
|
-
deleted column.
|
263
|
-
"""
|
264
|
-
|
265
|
-
immutable_schema: bool = betterproto.bool_field(5)
|
266
|
-
"""
|
267
|
-
True if schema can NOT be evolved on write.
|
268
|
-
An explicit schema evolution is required.
|
269
|
-
"""
|
270
|
-
|
271
|
-
schema_salt: int = betterproto.int32_field(6)
|
272
|
-
"""
|
273
|
-
Schema salt is used to compute column IDs. It is modified on breaking
|
274
|
-
change, e.g. column deletion. This ensures that if the previously existing
|
275
|
-
column is added again, it will have a different ID.
|
276
|
-
"""
|