lance-namespace 0.0.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,104 @@
1
+ """
2
+ Lance Namespace Python Client
3
+
4
+ A Python client for the Lance Namespace API that provides a unified interface
5
+ for managing namespaces and tables across different backend implementations.
6
+ """
7
+
8
+ from .namespace import LanceNamespace, connect
9
+
10
+ # Re-export all models from the urllib3 client
11
+ from lance_namespace_urllib3_client.models import (
12
+ ListNamespacesRequest,
13
+ ListNamespacesResponse,
14
+ DescribeNamespaceRequest,
15
+ DescribeNamespaceResponse,
16
+ CreateNamespaceRequest,
17
+ CreateNamespaceResponse,
18
+ DropNamespaceRequest,
19
+ DropNamespaceResponse,
20
+ NamespaceExistsRequest,
21
+ ListTablesRequest,
22
+ ListTablesResponse,
23
+ DescribeTableRequest,
24
+ DescribeTableResponse,
25
+ RegisterTableRequest,
26
+ RegisterTableResponse,
27
+ TableExistsRequest,
28
+ DropTableRequest,
29
+ DropTableResponse,
30
+ DeregisterTableRequest,
31
+ DeregisterTableResponse,
32
+ CountTableRowsRequest,
33
+ CreateTableRequest,
34
+ CreateTableResponse,
35
+ InsertIntoTableRequest,
36
+ InsertIntoTableResponse,
37
+ MergeInsertIntoTableRequest,
38
+ MergeInsertIntoTableResponse,
39
+ UpdateTableRequest,
40
+ UpdateTableResponse,
41
+ DeleteFromTableRequest,
42
+ DeleteFromTableResponse,
43
+ QueryTableRequest,
44
+ CreateTableIndexRequest,
45
+ CreateTableIndexResponse,
46
+ ListTableIndicesRequest,
47
+ ListTableIndicesResponse,
48
+ DescribeTableIndexStatsRequest,
49
+ DescribeTableIndexStatsResponse,
50
+ DescribeTransactionRequest,
51
+ DescribeTransactionResponse,
52
+ AlterTransactionRequest,
53
+ AlterTransactionResponse,
54
+ )
55
+
56
+ __all__ = [
57
+ # Main interface and connect function
58
+ "LanceNamespace",
59
+ "connect",
60
+
61
+ # Request/Response models
62
+ "ListNamespacesRequest",
63
+ "ListNamespacesResponse",
64
+ "DescribeNamespaceRequest",
65
+ "DescribeNamespaceResponse",
66
+ "CreateNamespaceRequest",
67
+ "CreateNamespaceResponse",
68
+ "DropNamespaceRequest",
69
+ "DropNamespaceResponse",
70
+ "NamespaceExistsRequest",
71
+ "ListTablesRequest",
72
+ "ListTablesResponse",
73
+ "DescribeTableRequest",
74
+ "DescribeTableResponse",
75
+ "RegisterTableRequest",
76
+ "RegisterTableResponse",
77
+ "TableExistsRequest",
78
+ "DropTableRequest",
79
+ "DropTableResponse",
80
+ "DeregisterTableRequest",
81
+ "DeregisterTableResponse",
82
+ "CountTableRowsRequest",
83
+ "CreateTableRequest",
84
+ "CreateTableResponse",
85
+ "InsertIntoTableRequest",
86
+ "InsertIntoTableResponse",
87
+ "MergeInsertIntoTableRequest",
88
+ "MergeInsertIntoTableResponse",
89
+ "UpdateTableRequest",
90
+ "UpdateTableResponse",
91
+ "DeleteFromTableRequest",
92
+ "DeleteFromTableResponse",
93
+ "QueryTableRequest",
94
+ "CreateTableIndexRequest",
95
+ "CreateTableIndexResponse",
96
+ "ListTableIndicesRequest",
97
+ "ListTableIndicesResponse",
98
+ "DescribeTableIndexStatsRequest",
99
+ "DescribeTableIndexStatsResponse",
100
+ "DescribeTransactionRequest",
101
+ "DescribeTransactionResponse",
102
+ "AlterTransactionRequest",
103
+ "AlterTransactionResponse",
104
+ ]
lance_namespace/dir.py ADDED
@@ -0,0 +1,348 @@
1
+ """
2
+ Lance Directory Namespace implementation using OpenDAL.
3
+ """
4
+ from typing import Dict, List, Optional
5
+ from urllib.parse import urlparse
6
+ import os
7
+
8
+ import opendal
9
+
10
+ import lance
11
+ import pyarrow as pa
12
+
13
+ from lance_namespace.namespace import LanceNamespace
14
+ from lance_namespace_urllib3_client.models import (
15
+ ListNamespacesRequest,
16
+ ListNamespacesResponse,
17
+ DescribeNamespaceRequest,
18
+ DescribeNamespaceResponse,
19
+ CreateNamespaceRequest,
20
+ CreateNamespaceResponse,
21
+ DropNamespaceRequest,
22
+ DropNamespaceResponse,
23
+ NamespaceExistsRequest,
24
+ ListTablesRequest,
25
+ ListTablesResponse,
26
+ CreateTableRequest,
27
+ CreateTableResponse,
28
+ DropTableRequest,
29
+ DropTableResponse,
30
+ DescribeTableRequest,
31
+ DescribeTableResponse,
32
+ JsonArrowSchema,
33
+ JsonArrowField,
34
+ JsonArrowDataType,
35
+ )
36
+
37
+
38
+ class DirectoryNamespace(LanceNamespace):
39
+ """Lance Directory Namespace implementation using OpenDAL."""
40
+
41
+ def __init__(self, **properties):
42
+ """Initialize the directory namespace.
43
+
44
+ Args:
45
+ root: The root directory of the namespace (optional, defaults to current directory)
46
+ **properties: Additional configuration properties for specific storage backends
47
+ """
48
+
49
+ self.config = DirectoryNamespaceConfig(properties)
50
+ root = self.config.root
51
+
52
+ # Use current directory if root is not specified
53
+ if not root:
54
+ root = os.getcwd()
55
+
56
+ self.namespace_path = self._parse_path(root)
57
+ self.operator = self._initialize_operator(root)
58
+
59
+ def create_namespace(self, request: CreateNamespaceRequest) -> CreateNamespaceResponse:
60
+ """Create a namespace - not supported for directory namespace."""
61
+ raise NotImplementedError(
62
+ "Directory namespace only contains a flat list of tables and does not support creating namespaces"
63
+ )
64
+
65
+ def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesResponse:
66
+ """List namespaces - not supported for directory namespace."""
67
+ raise NotImplementedError(
68
+ "Directory namespace only contains a flat list of tables and does not support listing namespaces"
69
+ )
70
+
71
+ def describe_namespace(self, request: DescribeNamespaceRequest) -> DescribeNamespaceResponse:
72
+ """Describe namespace - not supported for directory namespace."""
73
+ raise NotImplementedError(
74
+ "Directory namespace only contains a flat list of tables and does not support describing namespaces"
75
+ )
76
+
77
+ def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse:
78
+ """Drop namespace - not supported for directory namespace."""
79
+ raise NotImplementedError(
80
+ "Directory namespace only contains a flat list of tables and does not support dropping namespaces"
81
+ )
82
+
83
+ def namespace_exists(self, request: NamespaceExistsRequest) -> None:
84
+ """Check namespace exists - not supported for directory namespace."""
85
+ raise NotImplementedError(
86
+ "Directory namespace only contains a flat list of tables and does not support namespace existence checks"
87
+ )
88
+
89
+ def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
90
+ """List all tables in the namespace."""
91
+ self._validate_root_namespace_id(request.id)
92
+
93
+ try:
94
+ tables = []
95
+ entries = self.operator.list("", recursive=False)
96
+
97
+ for entry in entries:
98
+ path = entry.path.rstrip('/')
99
+
100
+ # Only process paths that contain ".lance"
101
+ if ".lance" not in path:
102
+ continue
103
+
104
+ # Strip .lance suffix to get clean table name
105
+ table_name = path[:-6] # Remove '.lance' (6 characters)
106
+
107
+ # Check if it's a valid Lance dataset
108
+ try:
109
+ versions_path = f"{table_name}.lance/_versions/"
110
+ version_entries = list(self.operator.list(versions_path, limit=1))
111
+ if version_entries:
112
+ tables.append(table_name) # Add clean name without .lance
113
+ except:
114
+ # If _versions doesn't exist, it's not a Lance dataset
115
+ pass
116
+
117
+ response = ListTablesResponse(tables=tables)
118
+ return response
119
+ except Exception as e:
120
+ raise RuntimeError(f"Failed to list tables: {e}")
121
+
122
+
123
+ def create_table(self, request: CreateTableRequest, request_data: bytes) -> CreateTableResponse:
124
+ """Create a table using Lance dataset."""
125
+ if not request.id:
126
+ raise ValueError("table ID cannot be empty")
127
+
128
+ if not request.var_schema:
129
+ raise ValueError("Schema is required in CreateTableRequest")
130
+
131
+ table_name = self._normalize_table_id(request.id)
132
+ table_path = self._get_table_path(table_name)
133
+
134
+ if request.location and request.location != table_path:
135
+ raise ValueError(f"Cannot create table {table_name} at location {request.location}, must be at location {table_path}")
136
+
137
+ # Convert JsonArrowSchema to PyArrow Schema
138
+ schema = self._convert_json_arrow_schema_to_pyarrow(request.var_schema)
139
+
140
+ # Create empty table with schema
141
+ arrays = []
142
+ for field in schema:
143
+ # Create empty array for each field
144
+ empty_array = pa.array([], type=field.type)
145
+ arrays.append(empty_array)
146
+
147
+ empty_table = pa.Table.from_arrays(arrays, schema=schema)
148
+
149
+ # Create Lance dataset
150
+ lance.write_dataset(empty_table, table_path, storage_options=self.config.storage_options)
151
+
152
+ response = CreateTableResponse(location=table_path, version=1)
153
+ return response
154
+
155
+ def drop_table(self, request: DropTableRequest) -> DropTableResponse:
156
+ """Drop a table by removing its Lance dataset."""
157
+ if not request.id:
158
+ raise ValueError("table ID cannot be empty")
159
+
160
+ table_name = self._normalize_table_id(request.id)
161
+ table_path = self._get_table_path(table_name)
162
+
163
+ try:
164
+ # Remove the entire table directory
165
+ self.operator.remove_all(f"{table_name}.lance/")
166
+ response = DropTableResponse()
167
+ return response
168
+ except Exception as e:
169
+ raise RuntimeError(f"Failed to drop table {table_name}: {e}")
170
+
171
+ def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
172
+ """Describe a table by checking its existence and returning location."""
173
+ if not request.id:
174
+ raise ValueError("table ID cannot be empty")
175
+
176
+ table_name = self._normalize_table_id(request.id)
177
+ table_path = self._get_table_path(table_name)
178
+
179
+ try:
180
+ # Check if it's a Lance dataset by looking for objects with _versions/ prefix
181
+ versions_path = f"{table_name}.lance/_versions/"
182
+ version_entries = list(self.operator.list(versions_path, limit=1))
183
+ if not version_entries:
184
+ raise RuntimeError(f"Table does not exist: {table_name}")
185
+ except Exception as e:
186
+ raise RuntimeError(f"Table does not exist: {table_name}: {e}")
187
+
188
+ response = DescribeTableResponse(location=table_path)
189
+ return response
190
+
191
+ def _normalize_table_id(self, id: List[str]) -> str:
192
+ """Normalize table ID - only single-level IDs are supported."""
193
+ if not id:
194
+ raise ValueError("Directory namespace table ID cannot be empty")
195
+
196
+ if len(id) != 1:
197
+ raise ValueError(
198
+ f"Directory namespace only supports single-level table IDs, but got: {id}"
199
+ )
200
+
201
+ return id[0]
202
+
203
+ def _validate_root_namespace_id(self, id: Optional[List[str]]) -> None:
204
+ """Validate that the namespace ID represents a root namespace."""
205
+ if id:
206
+ raise ValueError(
207
+ f"Directory namespace only supports root namespace operations, "
208
+ f"but got namespace ID: {id}. Expected empty ID."
209
+ )
210
+
211
+ def _get_table_path(self, table_name: str) -> str:
212
+ """Get the full path for a table."""
213
+ root = self.config.root if self.config.root else os.getcwd()
214
+ return f"{root}/{table_name}.lance"
215
+
216
+ def _convert_json_arrow_schema_to_pyarrow(self, json_schema: JsonArrowSchema) -> pa.Schema:
217
+ """Convert JsonArrowSchema to PyArrow Schema."""
218
+ fields = []
219
+ for json_field in json_schema.fields:
220
+ arrow_type = self._convert_json_arrow_type_to_pyarrow(json_field.type)
221
+ field = pa.field(json_field.name, arrow_type, nullable=json_field.nullable)
222
+ fields.append(field)
223
+
224
+ return pa.schema(fields, metadata=json_schema.metadata)
225
+
226
+ def _convert_json_arrow_type_to_pyarrow(self, json_type: JsonArrowDataType) -> pa.DataType:
227
+ """Convert JsonArrowDataType to PyArrow DataType."""
228
+ type_name = json_type.type.lower()
229
+
230
+ if type_name == "null":
231
+ return pa.null()
232
+ elif type_name in ["bool", "boolean"]:
233
+ return pa.bool_()
234
+ elif type_name == "int8":
235
+ return pa.int8()
236
+ elif type_name == "uint8":
237
+ return pa.uint8()
238
+ elif type_name == "int16":
239
+ return pa.int16()
240
+ elif type_name == "uint16":
241
+ return pa.uint16()
242
+ elif type_name == "int32":
243
+ return pa.int32()
244
+ elif type_name == "uint32":
245
+ return pa.uint32()
246
+ elif type_name == "int64":
247
+ return pa.int64()
248
+ elif type_name == "uint64":
249
+ return pa.uint64()
250
+ elif type_name == "float32":
251
+ return pa.float32()
252
+ elif type_name == "float64":
253
+ return pa.float64()
254
+ elif type_name == "utf8":
255
+ return pa.utf8()
256
+ elif type_name == "binary":
257
+ return pa.binary()
258
+ else:
259
+ raise ValueError(f"Unsupported Arrow type: {type_name}")
260
+
261
+ def _parse_path(self, path: str) -> str:
262
+ """Parse the path and convert to a proper URI if needed."""
263
+ parsed = urlparse(path)
264
+ if parsed.scheme:
265
+ return path
266
+
267
+ # Handle absolute and relative POSIX paths
268
+ if path.startswith('/'):
269
+ return f"file://{path}"
270
+ else:
271
+ current_dir = os.getcwd()
272
+ absolute_path = os.path.abspath(os.path.join(current_dir, path))
273
+ return f"file://{absolute_path}"
274
+
275
+ def _normalize_scheme(self, scheme: Optional[str]) -> str:
276
+ """Normalize scheme with aliases."""
277
+ if scheme is None:
278
+ return 'fs'
279
+
280
+ # Handle scheme aliases
281
+ scheme_lower = scheme.lower()
282
+ if scheme_lower in ['s3a', 's3n']:
283
+ return 's3'
284
+ elif scheme_lower == 'abfs':
285
+ return 'azblob'
286
+ elif scheme_lower == 'file':
287
+ return 'fs'
288
+ else:
289
+ return scheme_lower
290
+
291
+ def _initialize_operator(self, root: str) -> opendal.Operator:
292
+ """Initialize the OpenDAL operator based on the root path."""
293
+ scheme_split = root.split("://", 1)
294
+
295
+ # Local file system path
296
+ if len(scheme_split) < 2:
297
+ return opendal.Operator("fs", root=root)
298
+
299
+ scheme = self._normalize_scheme(scheme_split[0])
300
+ authority_split = scheme_split[1].split("/", 1)
301
+ authority = authority_split[0]
302
+ path = authority_split[1] if len(authority_split) > 1 else ""
303
+
304
+ if scheme in ["s3", "gcs"]:
305
+ return opendal.Operator(scheme, root=path, bucket=authority)
306
+ elif scheme == "azblob":
307
+ return opendal.Operator(scheme, root=path, container=authority)
308
+ else:
309
+ return opendal.Operator(scheme, root=scheme_split[1])
310
+
311
+
312
+
313
+ class DirectoryNamespaceConfig:
314
+ """Configuration for DirectoryNamespace."""
315
+
316
+ ROOT = "root"
317
+ STORAGE_OPTIONS_PREFIX = "storage."
318
+
319
+ def __init__(self, properties: Optional[Dict[str, str]] = None):
320
+ """Initialize configuration from properties.
321
+
322
+ Args:
323
+ properties: Dictionary of configuration properties
324
+ """
325
+ if properties is None:
326
+ properties = {}
327
+
328
+ self._root = properties.get(self.ROOT)
329
+ self._storage_options = self._extract_storage_options(properties)
330
+
331
+ def _extract_storage_options(self, properties: Dict[str, str]) -> Dict[str, str]:
332
+ """Extract storage configuration properties by removing the prefix."""
333
+ storage_options = {}
334
+ for key, value in properties.items():
335
+ if key.startswith(self.STORAGE_OPTIONS_PREFIX):
336
+ storage_key = key[len(self.STORAGE_OPTIONS_PREFIX):]
337
+ storage_options[storage_key] = value
338
+ return storage_options
339
+
340
+ @property
341
+ def root(self) -> Optional[str]:
342
+ """Get the namespace root directory."""
343
+ return self._root
344
+
345
+ @property
346
+ def storage_options(self) -> Dict[str, str]:
347
+ """Get the storage configuration properties."""
348
+ return self._storage_options.copy()
@@ -0,0 +1,183 @@
1
+ """
2
+ Lance Namespace base interface and implementations.
3
+ """
4
+ from abc import ABC, abstractmethod
5
+ from typing import Dict, Any, Optional, List, Union
6
+ import importlib
7
+ import inspect
8
+
9
+ from lance_namespace_urllib3_client.models import (
10
+ ListNamespacesRequest,
11
+ ListNamespacesResponse,
12
+ DescribeNamespaceRequest,
13
+ DescribeNamespaceResponse,
14
+ CreateNamespaceRequest,
15
+ CreateNamespaceResponse,
16
+ DropNamespaceRequest,
17
+ DropNamespaceResponse,
18
+ NamespaceExistsRequest,
19
+ ListTablesRequest,
20
+ ListTablesResponse,
21
+ DescribeTableRequest,
22
+ DescribeTableResponse,
23
+ RegisterTableRequest,
24
+ RegisterTableResponse,
25
+ TableExistsRequest,
26
+ DropTableRequest,
27
+ DropTableResponse,
28
+ DeregisterTableRequest,
29
+ DeregisterTableResponse,
30
+ CountTableRowsRequest,
31
+ CreateTableRequest,
32
+ CreateTableResponse,
33
+ InsertIntoTableRequest,
34
+ InsertIntoTableResponse,
35
+ MergeInsertIntoTableRequest,
36
+ MergeInsertIntoTableResponse,
37
+ UpdateTableRequest,
38
+ UpdateTableResponse,
39
+ DeleteFromTableRequest,
40
+ DeleteFromTableResponse,
41
+ QueryTableRequest,
42
+ CreateTableIndexRequest,
43
+ CreateTableIndexResponse,
44
+ ListTableIndicesRequest,
45
+ ListTableIndicesResponse,
46
+ DescribeTableIndexStatsRequest,
47
+ DescribeTableIndexStatsResponse,
48
+ DescribeTransactionRequest,
49
+ DescribeTransactionResponse,
50
+ AlterTransactionRequest,
51
+ AlterTransactionResponse,
52
+ )
53
+
54
+
55
+ class LanceNamespace(ABC):
56
+ """Base interface for Lance Namespace implementations."""
57
+
58
+ def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesResponse:
59
+ """List namespaces."""
60
+ raise NotImplementedError("Not supported: list_namespaces")
61
+
62
+ def describe_namespace(self, request: DescribeNamespaceRequest) -> DescribeNamespaceResponse:
63
+ """Describe a namespace."""
64
+ raise NotImplementedError("Not supported: describe_namespace")
65
+
66
+ def create_namespace(self, request: CreateNamespaceRequest) -> CreateNamespaceResponse:
67
+ """Create a new namespace."""
68
+ raise NotImplementedError("Not supported: create_namespace")
69
+
70
+ def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse:
71
+ """Drop a namespace."""
72
+ raise NotImplementedError("Not supported: drop_namespace")
73
+
74
+ def namespace_exists(self, request: NamespaceExistsRequest) -> None:
75
+ """Check if a namespace exists."""
76
+ raise NotImplementedError("Not supported: namespace_exists")
77
+
78
+ def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
79
+ """List tables in a namespace."""
80
+ raise NotImplementedError("Not supported: list_tables")
81
+
82
+ def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
83
+ """Describe a table."""
84
+ raise NotImplementedError("Not supported: describe_table")
85
+
86
+ def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse:
87
+ """Register a table."""
88
+ raise NotImplementedError("Not supported: register_table")
89
+
90
+ def table_exists(self, request: TableExistsRequest) -> None:
91
+ """Check if a table exists."""
92
+ raise NotImplementedError("Not supported: table_exists")
93
+
94
+ def drop_table(self, request: DropTableRequest) -> DropTableResponse:
95
+ """Drop a table."""
96
+ raise NotImplementedError("Not supported: drop_table")
97
+
98
+ def deregister_table(self, request: DeregisterTableRequest) -> DeregisterTableResponse:
99
+ """Deregister a table."""
100
+ raise NotImplementedError("Not supported: deregister_table")
101
+
102
+ def count_table_rows(self, request: CountTableRowsRequest) -> int:
103
+ """Count rows in a table."""
104
+ raise NotImplementedError("Not supported: count_table_rows")
105
+
106
+ def create_table(self, request: CreateTableRequest, request_data: bytes) -> CreateTableResponse:
107
+ """Create a new table."""
108
+ raise NotImplementedError("Not supported: create_table")
109
+
110
+ def insert_into_table(self, request: InsertIntoTableRequest, request_data: bytes) -> InsertIntoTableResponse:
111
+ """Insert data into a table."""
112
+ raise NotImplementedError("Not supported: insert_into_table")
113
+
114
+ def merge_insert_into_table(
115
+ self, request: MergeInsertIntoTableRequest, request_data: bytes
116
+ ) -> MergeInsertIntoTableResponse:
117
+ """Merge insert data into a table."""
118
+ raise NotImplementedError("Not supported: merge_insert_into_table")
119
+
120
+ def update_table(self, request: UpdateTableRequest) -> UpdateTableResponse:
121
+ """Update a table."""
122
+ raise NotImplementedError("Not supported: update_table")
123
+
124
+ def delete_from_table(self, request: DeleteFromTableRequest) -> DeleteFromTableResponse:
125
+ """Delete from a table."""
126
+ raise NotImplementedError("Not supported: delete_from_table")
127
+
128
+ def query_table(self, request: QueryTableRequest) -> bytes:
129
+ """Query a table."""
130
+ raise NotImplementedError("Not supported: query_table")
131
+
132
+ def create_table_index(self, request: CreateTableIndexRequest) -> CreateTableIndexResponse:
133
+ """Create a table index."""
134
+ raise NotImplementedError("Not supported: create_table_index")
135
+
136
+ def list_table_indices(self, request: ListTableIndicesRequest) -> ListTableIndicesResponse:
137
+ """List table indices."""
138
+ raise NotImplementedError("Not supported: list_table_indices")
139
+
140
+ def describe_table_index_stats(
141
+ self, request: DescribeTableIndexStatsRequest
142
+ ) -> DescribeTableIndexStatsResponse:
143
+ """Describe table index statistics."""
144
+ raise NotImplementedError("Not supported: describe_table_index_stats")
145
+
146
+ def describe_transaction(self, request: DescribeTransactionRequest) -> DescribeTransactionResponse:
147
+ """Describe a transaction."""
148
+ raise NotImplementedError("Not supported: describe_transaction")
149
+
150
+ def alter_transaction(self, request: AlterTransactionRequest) -> AlterTransactionResponse:
151
+ """Alter a transaction."""
152
+ raise NotImplementedError("Not supported: alter_transaction")
153
+
154
+
155
+ NATIVE_IMPLS = {
156
+ "rest": "lance_namespace.rest.LanceRestNamespace",
157
+ "dir": "lance_namespace.dir.DirectoryNamespace",
158
+ }
159
+
160
+ def connect(impl: str, properties: Dict[str, str]) -> LanceNamespace:
161
+ """
162
+ Connect to a Lance namespace implementation.
163
+
164
+ Args:
165
+ impl: Implementation alias or full class path
166
+ properties: Configuration properties
167
+ conf: Optional configuration object
168
+
169
+ Returns:
170
+ LanceNamespace instance
171
+ """
172
+ impl_class = NATIVE_IMPLS.get(impl, impl)
173
+ try:
174
+ module_name, class_name = impl_class.rsplit(".", 1)
175
+ module = importlib.import_module(module_name)
176
+ namespace_class = getattr(module, class_name)
177
+
178
+ if not issubclass(namespace_class, LanceNamespace):
179
+ raise ValueError(f"Class {impl_class} does not implement LanceNamespace interface")
180
+
181
+ return namespace_class(**properties)
182
+ except Exception as e:
183
+ raise ValueError(f"Failed to construct namespace impl {impl_class}: {e}")
File without changes
@@ -0,0 +1,322 @@
1
+ """
2
+ Lance REST Namespace implementation.
3
+ """
4
+ from typing import Dict, Any, Optional
5
+ import json
6
+
7
+ from lance_namespace_urllib3_client import (
8
+ ApiClient,
9
+ Configuration,
10
+ NamespaceApi,
11
+ TableApi,
12
+ TransactionApi,
13
+ )
14
+ from lance_namespace_urllib3_client.models import (
15
+ ListNamespacesRequest,
16
+ ListNamespacesResponse,
17
+ DescribeNamespaceRequest,
18
+ DescribeNamespaceResponse,
19
+ CreateNamespaceRequest,
20
+ CreateNamespaceResponse,
21
+ DropNamespaceRequest,
22
+ DropNamespaceResponse,
23
+ NamespaceExistsRequest,
24
+ ListTablesRequest,
25
+ ListTablesResponse,
26
+ DescribeTableRequest,
27
+ DescribeTableResponse,
28
+ RegisterTableRequest,
29
+ RegisterTableResponse,
30
+ TableExistsRequest,
31
+ DropTableRequest,
32
+ DropTableResponse,
33
+ DeregisterTableRequest,
34
+ DeregisterTableResponse,
35
+ CountTableRowsRequest,
36
+ CreateTableRequest,
37
+ CreateTableResponse,
38
+ InsertIntoTableRequest,
39
+ InsertIntoTableResponse,
40
+ MergeInsertIntoTableRequest,
41
+ MergeInsertIntoTableResponse,
42
+ UpdateTableRequest,
43
+ UpdateTableResponse,
44
+ DeleteFromTableRequest,
45
+ DeleteFromTableResponse,
46
+ QueryTableRequest,
47
+ CreateTableIndexRequest,
48
+ CreateTableIndexResponse,
49
+ ListTableIndicesRequest,
50
+ ListTableIndicesResponse,
51
+ DescribeTableIndexStatsRequest,
52
+ DescribeTableIndexStatsResponse,
53
+ DescribeTransactionRequest,
54
+ DescribeTransactionResponse,
55
+ AlterTransactionRequest,
56
+ AlterTransactionResponse,
57
+ )
58
+
59
+ from .namespace import LanceNamespace
60
+
61
+
62
+ class RestNamespaceConfig:
63
+ """Configuration for REST namespace."""
64
+
65
+ DELIMITER = "delimiter"
66
+ HEADER_PREFIX = "header."
67
+
68
+ def __init__(self, properties: Dict[str, str]):
69
+ self._delimiter = properties.get(self.DELIMITER, ".")
70
+ self._headers = {}
71
+
72
+ for key, value in properties.items():
73
+ if key.startswith(self.HEADER_PREFIX):
74
+ header_name = key[len(self.HEADER_PREFIX):]
75
+ self._headers[header_name] = value
76
+
77
+ def delimiter(self) -> str:
78
+ return self._delimiter
79
+
80
+ def additional_headers(self) -> Dict[str, str]:
81
+ return self._headers
82
+
83
+
84
+ def object_id_str(id_list: list[str], delimiter: str = ".", obj: Any = None) -> str:
85
+ """Convert a list of strings to a string identifier using the specified delimiter.
86
+ If the list is empty, returns the delimiter itself.
87
+
88
+ Args:
89
+ id_list: List of strings representing the identifier
90
+ delimiter: Delimiter to join the strings
91
+ obj: The object containing the id (for error messages)
92
+
93
+ Returns:
94
+ String identifier
95
+
96
+ Raises:
97
+ ValueError: If id_list is None
98
+ """
99
+ if id_list is None:
100
+ object_type = type(obj).__name__ if obj is not None else "Unknown"
101
+ raise ValueError(f"Object of type '{object_type}' must have an 'id' field")
102
+ if len(id_list) == 0:
103
+ return delimiter
104
+ return delimiter.join(id_list)
105
+
106
+
107
+ class LanceRestNamespace(LanceNamespace):
108
+ """REST implementation of Lance Namespace."""
109
+
110
+ def __init__(self, **kwargs):
111
+ configuration = Configuration()
112
+ if "uri" in kwargs:
113
+ configuration.host = kwargs["uri"]
114
+
115
+ self.api_client = ApiClient(configuration)
116
+ self.namespace_api = NamespaceApi(self.api_client)
117
+ self.table_api = TableApi(self.api_client)
118
+ self.transaction_api = TransactionApi(self.api_client)
119
+ self.config = RestNamespaceConfig(kwargs)
120
+
121
+ def list_namespaces(self, request: ListNamespacesRequest) -> ListNamespacesResponse:
122
+ return self.namespace_api.list_namespaces(
123
+ id=object_id_str(request.id, self.config.delimiter(), request),
124
+ delimiter=self.config.delimiter(),
125
+ page_token=request.page_token,
126
+ limit=request.limit,
127
+ _headers=self.config.additional_headers(),
128
+ )
129
+
130
+ def describe_namespace(self, request: DescribeNamespaceRequest) -> DescribeNamespaceResponse:
131
+ return self.namespace_api.describe_namespace(
132
+ id=object_id_str(request.id, self.config.delimiter(), request),
133
+ describe_namespace_request=request,
134
+ delimiter=self.config.delimiter(),
135
+ _headers=self.config.additional_headers(),
136
+ )
137
+
138
+ def create_namespace(self, request: CreateNamespaceRequest) -> CreateNamespaceResponse:
139
+ return self.namespace_api.create_namespace(
140
+ id=object_id_str(request.id, self.config.delimiter(), request),
141
+ create_namespace_request=request,
142
+ delimiter=self.config.delimiter(),
143
+ _headers=self.config.additional_headers(),
144
+ )
145
+
146
+ def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse:
147
+ return self.namespace_api.drop_namespace(
148
+ id=object_id_str(request.id, self.config.delimiter(), request),
149
+ drop_namespace_request=request,
150
+ delimiter=self.config.delimiter(),
151
+ _headers=self.config.additional_headers(),
152
+ )
153
+
154
+ def namespace_exists(self, request: NamespaceExistsRequest) -> None:
155
+ self.namespace_api.namespace_exists(
156
+ id=object_id_str(request.id, self.config.delimiter(), request),
157
+ namespace_exists_request=request,
158
+ delimiter=self.config.delimiter(),
159
+ _headers=self.config.additional_headers(),
160
+ )
161
+
162
+ def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
163
+ return self.table_api.list_tables(
164
+ id=object_id_str(request.id, self.config.delimiter(), request),
165
+ delimiter=self.config.delimiter(),
166
+ page_token=request.page_token,
167
+ limit=request.limit,
168
+ _headers=self.config.additional_headers(),
169
+ )
170
+
171
+ def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
172
+ return self.table_api.describe_table(
173
+ id=object_id_str(request.id, self.config.delimiter(), request),
174
+ describe_table_request=request,
175
+ delimiter=self.config.delimiter(),
176
+ _headers=self.config.additional_headers(),
177
+ )
178
+
179
+ def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse:
180
+ return self.table_api.register_table(
181
+ id=object_id_str(request.id, self.config.delimiter(), request),
182
+ register_table_request=request,
183
+ delimiter=self.config.delimiter(),
184
+ _headers=self.config.additional_headers(),
185
+ )
186
+
187
+ def table_exists(self, request: TableExistsRequest) -> None:
188
+ self.table_api.table_exists(
189
+ id=object_id_str(request.id, self.config.delimiter(), request),
190
+ table_exists_request=request,
191
+ delimiter=self.config.delimiter(),
192
+ _headers=self.config.additional_headers(),
193
+ )
194
+
195
+ def drop_table(self, request: DropTableRequest) -> DropTableResponse:
196
+ return self.table_api.drop_table(
197
+ id=object_id_str(request.id, self.config.delimiter(), request),
198
+ drop_table_request=request,
199
+ delimiter=self.config.delimiter(),
200
+ _headers=self.config.additional_headers(),
201
+ )
202
+
203
+ def deregister_table(self, request: DeregisterTableRequest) -> DeregisterTableResponse:
204
+ return self.table_api.deregister_table(
205
+ id=object_id_str(request.id, self.config.delimiter(), request),
206
+ deregister_table_request=request,
207
+ delimiter=self.config.delimiter(),
208
+ _headers=self.config.additional_headers(),
209
+ )
210
+
211
+ def count_table_rows(self, request: CountTableRowsRequest) -> int:
212
+ return self.table_api.count_table_rows(
213
+ id=object_id_str(request.id, self.config.delimiter(), request),
214
+ count_table_rows_request=request,
215
+ delimiter=self.config.delimiter(),
216
+ _headers=self.config.additional_headers(),
217
+ )
218
+
219
+ def create_table(self, request: CreateTableRequest, request_data: bytes) -> CreateTableResponse:
220
+ table_properties = json.dumps(request.properties) if request.properties else None
221
+ return self.table_api.create_table(
222
+ id=object_id_str(request.id, self.config.delimiter(), request),
223
+ x_lance_table_location=request.location,
224
+ body=request_data,
225
+ delimiter=self.config.delimiter(),
226
+ x_lance_table_properties=table_properties,
227
+ _headers=self.config.additional_headers(),
228
+ )
229
+
230
+ def insert_into_table(self, request: InsertIntoTableRequest, request_data: bytes) -> InsertIntoTableResponse:
231
+ mode = request.mode if request.mode else None
232
+
233
+ return self.table_api.insert_into_table(
234
+ id=object_id_str(request.id, self.config.delimiter(), request),
235
+ body=request_data,
236
+ delimiter=self.config.delimiter(),
237
+ mode=mode,
238
+ _headers=self.config.additional_headers(),
239
+ )
240
+
241
+ def merge_insert_into_table(
242
+ self, request: MergeInsertIntoTableRequest, request_data: bytes
243
+ ) -> MergeInsertIntoTableResponse:
244
+ return self.table_api.merge_insert_into_table(
245
+ id=object_id_str(request.id, self.config.delimiter(), request),
246
+ on=request.on,
247
+ body=request_data,
248
+ delimiter=self.config.delimiter(),
249
+ when_matched_update_all=request.when_matched_update_all,
250
+ when_matched_update_all_filt=request.when_matched_update_all_filt,
251
+ when_not_matched_insert_all=request.when_not_matched_insert_all,
252
+ when_not_matched_by_source_delete=request.when_not_matched_by_source_delete,
253
+ when_not_matched_by_source_delete_filt=request.when_not_matched_by_source_delete_filt,
254
+ _headers=self.config.additional_headers(),
255
+ )
256
+
257
+ def update_table(self, request: UpdateTableRequest) -> UpdateTableResponse:
258
+ return self.table_api.update_table(
259
+ id=object_id_str(request.id, self.config.delimiter(), request),
260
+ update_table_request=request,
261
+ delimiter=self.config.delimiter(),
262
+ _headers=self.config.additional_headers(),
263
+ )
264
+
265
+ def delete_from_table(self, request: DeleteFromTableRequest) -> DeleteFromTableResponse:
266
+ return self.table_api.delete_from_table(
267
+ id=object_id_str(request.id, self.config.delimiter(), request),
268
+ delete_from_table_request=request,
269
+ delimiter=self.config.delimiter(),
270
+ _headers=self.config.additional_headers(),
271
+ )
272
+
273
+ def query_table(self, request: QueryTableRequest) -> bytes:
274
+ return self.table_api.query_table(
275
+ id=object_id_str(request.id, self.config.delimiter(), request),
276
+ query_table_request=request,
277
+ delimiter=self.config.delimiter(),
278
+ _headers=self.config.additional_headers(),
279
+ )
280
+
281
+ def create_table_index(self, request: CreateTableIndexRequest) -> CreateTableIndexResponse:
282
+ return self.table_api.create_table_index(
283
+ id=object_id_str(request.id, self.config.delimiter(), request),
284
+ create_table_index_request=request,
285
+ delimiter=self.config.delimiter(),
286
+ _headers=self.config.additional_headers(),
287
+ )
288
+
289
+ def list_table_indices(self, request: ListTableIndicesRequest) -> ListTableIndicesResponse:
290
+ return self.table_api.list_table_indices(
291
+ id=object_id_str(request.id, self.config.delimiter(), request),
292
+ list_table_indices_request=request,
293
+ delimiter=self.config.delimiter(),
294
+ _headers=self.config.additional_headers(),
295
+ )
296
+
297
+ def describe_table_index_stats(
298
+ self, request: DescribeTableIndexStatsRequest, index_name: str
299
+ ) -> DescribeTableIndexStatsResponse:
300
+ return self.table_api.describe_table_index_stats(
301
+ id=object_id_str(request.id, self.config.delimiter(), request),
302
+ index_name=index_name,
303
+ describe_table_index_stats_request=request,
304
+ delimiter=self.config.delimiter(),
305
+ _headers=self.config.additional_headers(),
306
+ )
307
+
308
+ def describe_transaction(self, request: DescribeTransactionRequest) -> DescribeTransactionResponse:
309
+ return self.transaction_api.describe_transaction(
310
+ id=object_id_str(request.id, self.config.delimiter(), request),
311
+ describe_transaction_request=request,
312
+ delimiter=self.config.delimiter(),
313
+ _headers=self.config.additional_headers(),
314
+ )
315
+
316
+ def alter_transaction(self, request: AlterTransactionRequest) -> AlterTransactionResponse:
317
+ return self.transaction_api.alter_transaction(
318
+ id=object_id_str(request.id, self.config.delimiter(), request),
319
+ alter_transaction_request=request,
320
+ delimiter=self.config.delimiter(),
321
+ _headers=self.config.additional_headers(),
322
+ )
@@ -0,0 +1,14 @@
1
+ Metadata-Version: 2.4
2
+ Name: lance-namespace
3
+ Version: 0.0.5
4
+ Summary: Python client for Lance Namespace API
5
+ Author-email: Jack Ye <yezhaoqin@gmail.com>
6
+ Requires-Python: >=3.10
7
+ Requires-Dist: lance-namespace-urllib3-client
8
+ Requires-Dist: opendal>=0.46.0
9
+ Requires-Dist: pyarrow>=14.0.0
10
+ Requires-Dist: pylance>=0.18.0
11
+ Requires-Dist: typing-extensions>=4.0.0
12
+ Provides-Extra: test
13
+ Requires-Dist: pytest-cov>=4.0.0; extra == 'test'
14
+ Requires-Dist: pytest>=7.0.0; extra == 'test'
@@ -0,0 +1,8 @@
1
+ lance_namespace/__init__.py,sha256=_ciyt9wISTrNx_mMht6mqTJYv3D9UW2pZo40dVaggFI,2911
2
+ lance_namespace/dir.py,sha256=GGOeDYM6Mv1SoLP6ejSXKgUF-W1NlN_MuuytP94siL8,13434
3
+ lance_namespace/namespace.py,sha256=ChAt5oG6fejdzCboPU7g4sGB884CXzkrDhDYuZcqNWE,7143
4
+ lance_namespace/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ lance_namespace/rest.py,sha256=_6N7DN6cYgsTwxOrb0R1Tr76KZ4Q-paPaI_BHANfcmQ,12972
6
+ lance_namespace-0.0.5.dist-info/METADATA,sha256=ZbeD6IkWIJ9Q2vpark4-g7xUjNmieEEzZcWU3RYnTLA,470
7
+ lance_namespace-0.0.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
8
+ lance_namespace-0.0.5.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.27.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any