lazynwb 1.0.0.dev1__tar.gz → 1.0.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lazynwb-1.0.0.dev1/src/lazynwb.egg-info → lazynwb-1.0.0.dev2}/PKG-INFO +22 -3
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/README.md +50 -31
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/pyproject.toml +1 -1
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/__init__.py +6 -5
- lazynwb-1.0.0.dev2/src/lazynwb/_config.py +82 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/base.py +331 -32
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/conversion.py +129 -62
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/file_io.py +20 -33
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/tables.py +61 -5
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2/src/lazynwb.egg-info}/PKG-INFO +22 -3
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb.egg-info/SOURCES.txt +1 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_conversion.py +69 -23
- lazynwb-1.0.0.dev2/tests/test_core.py +134 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_file_io.py +27 -9
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_tables.py +25 -9
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_utils.py +33 -9
- lazynwb-1.0.0.dev1/tests/test_core.py +0 -69
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/LICENSE +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/setup.cfg +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cache/__init__.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cache/sqlite.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_catalog/__init__.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_catalog/_schema.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_catalog/accessor.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_catalog/backend.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_catalog/models.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_catalog/polars.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/__init__.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_config.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_context.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_errors.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_formatting.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_main.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_preview.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_query.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_schema.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_sources.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_cli/_tables.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_hdf5/__init__.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_hdf5/parser.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_hdf5/range_reader.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_hdf5/reader.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_storage_options.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_zarr/__init__.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/_zarr/reader.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/attrs.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/dandi.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/exceptions.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/lazyframe.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/py.typed +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/table_metadata.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/timeseries.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/types_.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb/utils.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb.egg-info/dependency_links.txt +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb.egg-info/entry_points.txt +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb.egg-info/requires.txt +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/src/lazynwb.egg-info/top_level.txt +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_attrs.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_cache.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_catalog_backend.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_catalog_schema.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_cli.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_dandi.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_dandi_metadata.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_dandi_tables.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_dandi_timeseries.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_dandi_unit.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_hdf5_backend.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_lazyframe.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_range_reader.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_remote_schema_budget.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_speed.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_table_metadata.py +0 -0
- {lazynwb-1.0.0.dev1 → lazynwb-1.0.0.dev2}/tests/test_timeseries.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: lazynwb
|
|
3
|
-
Version: 1.0.0.
|
|
3
|
+
Version: 1.0.0.dev2
|
|
4
4
|
Summary: An attempt to speed-up access to large NWB (Neurodata Without Borders) files stored in the cloud.
|
|
5
5
|
Author-email: Ben Hardcastle <ben.hardcastle@alleninstitue.org>
|
|
6
6
|
License: MIT
|
|
@@ -204,6 +204,17 @@ lazynwb.get_internal_paths('my_file.nwb')
|
|
|
204
204
|
# ['/intervals/trials', '/processing/behavior/running_speed', '/units', ...]
|
|
205
205
|
```
|
|
206
206
|
|
|
207
|
+
Use `get` when you want lazynwb to choose the return type from the NWB container:
|
|
208
|
+
```python
|
|
209
|
+
df = lazynwb.get('my_file.nwb', '/units') # pandas DataFrame by default
|
|
210
|
+
ts = lazynwb.get('my_file.nwb', '/processing/behavior/running_speed', exact_path=True)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Force DataFrame materialization, even for a TimeSeries container:
|
|
214
|
+
```python
|
|
215
|
+
df = lazynwb.get('my_file.nwb', '/processing/behavior/running_speed', exact_path=True, as_df=True)
|
|
216
|
+
```
|
|
217
|
+
|
|
207
218
|
---
|
|
208
219
|
|
|
209
220
|
## Reading tables
|
|
@@ -342,6 +353,13 @@ Get a table as polars:
|
|
|
342
353
|
df = nwb.get_df('/units', as_polars=True)
|
|
343
354
|
```
|
|
344
355
|
|
|
356
|
+
Use the general accessor when you want tables as DataFrames and TimeSeries as
|
|
357
|
+
`TimeSeries` objects:
|
|
358
|
+
```python
|
|
359
|
+
result = nwb.get('/processing/behavior/running_speed', exact_path=True)
|
|
360
|
+
df = nwb.get('/processing/behavior/running_speed', exact_path=True, as_df=True)
|
|
361
|
+
```
|
|
362
|
+
|
|
345
363
|
Get a summary of everything in the file:
|
|
346
364
|
```python
|
|
347
365
|
nwb.describe()
|
|
@@ -499,10 +517,11 @@ df = lazynwb.get_df('az://my-container/data/file.nwb', '/units')
|
|
|
499
517
|
df = lazynwb.get_df('https://example.com/data/file.nwb', '/units')
|
|
500
518
|
```
|
|
501
519
|
|
|
502
|
-
Configure
|
|
520
|
+
Configure global defaults via `lazynwb.config`:
|
|
503
521
|
```python
|
|
504
|
-
from lazynwb
|
|
522
|
+
from lazynwb import config
|
|
505
523
|
|
|
524
|
+
config.use_polars = True # return Polars by default from get_df/get_metadata_df
|
|
506
525
|
config.use_obstore = True # use obstore for S3/GCS/Azure (default: False)
|
|
507
526
|
config.use_remfile = False # use remfile for HTTP byte-range requests (default: True)
|
|
508
527
|
config.anon = True # anonymous access across backends
|
|
@@ -156,17 +156,28 @@ import lazynwb
|
|
|
156
156
|
df = lazynwb.get_df('my_file.nwb', '/intervals/trials')
|
|
157
157
|
```
|
|
158
158
|
|
|
159
|
-
Use `get_internal_paths` to find available paths if you're not sure what's in a file:
|
|
160
|
-
```python
|
|
161
|
-
lazynwb.get_internal_paths('my_file.nwb')
|
|
162
|
-
# ['/intervals/trials', '/processing/behavior/running_speed', '/units', ...]
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
159
|
+
Use `get_internal_paths` to find available paths if you're not sure what's in a file:
|
|
160
|
+
```python
|
|
161
|
+
lazynwb.get_internal_paths('my_file.nwb')
|
|
162
|
+
# ['/intervals/trials', '/processing/behavior/running_speed', '/units', ...]
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Use `get` when you want lazynwb to choose the return type from the NWB container:
|
|
166
|
+
```python
|
|
167
|
+
df = lazynwb.get('my_file.nwb', '/units') # pandas DataFrame by default
|
|
168
|
+
ts = lazynwb.get('my_file.nwb', '/processing/behavior/running_speed', exact_path=True)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Force DataFrame materialization, even for a TimeSeries container:
|
|
172
|
+
```python
|
|
173
|
+
df = lazynwb.get('my_file.nwb', '/processing/behavior/running_speed', exact_path=True, as_df=True)
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## Reading tables
|
|
179
|
+
|
|
180
|
+
### As a pandas or polars DataFrame (`get_df`)
|
|
170
181
|
|
|
171
182
|
Returns a pandas DataFrame by default:
|
|
172
183
|
```python
|
|
@@ -295,16 +306,23 @@ nwb.subject.strain
|
|
|
295
306
|
nwb.subject.date_of_birth
|
|
296
307
|
```
|
|
297
308
|
|
|
298
|
-
Get a table as polars:
|
|
299
|
-
```python
|
|
300
|
-
df = nwb.get_df('/units', as_polars=True)
|
|
301
|
-
```
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
309
|
+
Get a table as polars:
|
|
310
|
+
```python
|
|
311
|
+
df = nwb.get_df('/units', as_polars=True)
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
Use the general accessor when you want tables as DataFrames and TimeSeries as
|
|
315
|
+
`TimeSeries` objects:
|
|
316
|
+
```python
|
|
317
|
+
result = nwb.get('/processing/behavior/running_speed', exact_path=True)
|
|
318
|
+
df = nwb.get('/processing/behavior/running_speed', exact_path=True, as_df=True)
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
Get a summary of everything in the file:
|
|
322
|
+
```python
|
|
323
|
+
nwb.describe()
|
|
324
|
+
# {'identifier': '...', 'session_id': '...', ..., 'paths': ['/acquisition/...', '/units', ...]}
|
|
325
|
+
```
|
|
308
326
|
|
|
309
327
|
---
|
|
310
328
|
|
|
@@ -457,16 +475,17 @@ df = lazynwb.get_df('az://my-container/data/file.nwb', '/units')
|
|
|
457
475
|
df = lazynwb.get_df('https://example.com/data/file.nwb', '/units')
|
|
458
476
|
```
|
|
459
477
|
|
|
460
|
-
Configure
|
|
461
|
-
```python
|
|
462
|
-
from lazynwb
|
|
463
|
-
|
|
464
|
-
config.
|
|
465
|
-
config.
|
|
466
|
-
config.
|
|
467
|
-
config.
|
|
468
|
-
config.
|
|
469
|
-
|
|
478
|
+
Configure global defaults via `lazynwb.config`:
|
|
479
|
+
```python
|
|
480
|
+
from lazynwb import config
|
|
481
|
+
|
|
482
|
+
config.use_polars = True # return Polars by default from get_df/get_metadata_df
|
|
483
|
+
config.use_obstore = True # use obstore for S3/GCS/Azure (default: False)
|
|
484
|
+
config.use_remfile = False # use remfile for HTTP byte-range requests (default: True)
|
|
485
|
+
config.anon = True # anonymous access across backends
|
|
486
|
+
config.fsspec_storage_options = {"request_payer": True} # backend-specific extras if needed
|
|
487
|
+
config.disable_cache = False # disable FileAccessor caching (default: False)
|
|
488
|
+
```
|
|
470
489
|
|
|
471
490
|
For normal AWS S3 buckets, the region belongs to the bucket, not the caller's
|
|
472
491
|
current AWS session. The fast HDF5 range reader discovers and caches bucket
|
|
@@ -24,7 +24,7 @@ dependencies = [
|
|
|
24
24
|
"aiosqlite>=0.20.0",
|
|
25
25
|
"tomli>=2.0.0; python_version < '3.11'",
|
|
26
26
|
]
|
|
27
|
-
version = "1.0.0.
|
|
27
|
+
version = "1.0.0.dev2"
|
|
28
28
|
classifiers = [
|
|
29
29
|
"Development Status :: 3 - Alpha", # https://pypi.org/classifiers/
|
|
30
30
|
"Programming Language :: Python :: 3",
|
|
@@ -6,11 +6,12 @@ import doctest
|
|
|
6
6
|
import importlib.metadata
|
|
7
7
|
import logging
|
|
8
8
|
|
|
9
|
-
import dotenv
|
|
10
|
-
|
|
11
|
-
from lazynwb.
|
|
12
|
-
from lazynwb.
|
|
13
|
-
from lazynwb.
|
|
9
|
+
import dotenv
|
|
10
|
+
|
|
11
|
+
from lazynwb._config import *
|
|
12
|
+
from lazynwb.attrs import *
|
|
13
|
+
from lazynwb.base import *
|
|
14
|
+
from lazynwb.conversion import *
|
|
14
15
|
from lazynwb.dandi import *
|
|
15
16
|
from lazynwb.file_io import *
|
|
16
17
|
from lazynwb.lazyframe import *
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import pydantic
|
|
7
|
+
import pydantic_settings
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Config(pydantic_settings.BaseSettings):
|
|
13
|
+
"""
|
|
14
|
+
Global configuration for lazynwb behavior.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
model_config = pydantic_settings.SettingsConfigDict(
|
|
18
|
+
env_prefix="",
|
|
19
|
+
populate_by_name=True,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
use_polars: bool = pydantic.Field(
|
|
23
|
+
default=False,
|
|
24
|
+
validation_alias=pydantic.AliasChoices(
|
|
25
|
+
"LAZYNWB_USE_POLARS",
|
|
26
|
+
"LAZYNWB_FILE_IO_USE_POLARS",
|
|
27
|
+
),
|
|
28
|
+
)
|
|
29
|
+
use_remfile: bool = pydantic.Field(
|
|
30
|
+
default=True,
|
|
31
|
+
validation_alias=pydantic.AliasChoices(
|
|
32
|
+
"LAZYNWB_USE_REMFILE",
|
|
33
|
+
"LAZYNWB_FILE_IO_USE_REMFILE",
|
|
34
|
+
),
|
|
35
|
+
)
|
|
36
|
+
use_obstore: bool = pydantic.Field(
|
|
37
|
+
default=False,
|
|
38
|
+
validation_alias=pydantic.AliasChoices(
|
|
39
|
+
"LAZYNWB_USE_OBSTORE",
|
|
40
|
+
"LAZYNWB_FILE_IO_USE_OBSTORE",
|
|
41
|
+
),
|
|
42
|
+
)
|
|
43
|
+
anon: bool | None = pydantic.Field(
|
|
44
|
+
default=None,
|
|
45
|
+
validation_alias=pydantic.AliasChoices(
|
|
46
|
+
"LAZYNWB_ANON",
|
|
47
|
+
"LAZYNWB_FILE_IO_ANON",
|
|
48
|
+
),
|
|
49
|
+
)
|
|
50
|
+
fsspec_storage_options: dict[str, Any] = pydantic.Field(
|
|
51
|
+
default_factory=lambda: {"anon": False},
|
|
52
|
+
validation_alias=pydantic.AliasChoices(
|
|
53
|
+
"LAZYNWB_FSSPEC_STORAGE_OPTIONS",
|
|
54
|
+
"LAZYNWB_FILE_IO_FSSPEC_STORAGE_OPTIONS",
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
disable_cache: bool = pydantic.Field(
|
|
58
|
+
default=False,
|
|
59
|
+
validation_alias=pydantic.AliasChoices(
|
|
60
|
+
"LAZYNWB_DISABLE_CACHE",
|
|
61
|
+
"LAZYNWB_FILE_IO_DISABLE_CACHE",
|
|
62
|
+
),
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
config = Config()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _resolve_as_polars(as_polars: bool | None) -> bool:
|
|
70
|
+
if as_polars is not None:
|
|
71
|
+
logger.debug(
|
|
72
|
+
"using explicit DataFrame backend setting: as_polars=%s", as_polars
|
|
73
|
+
)
|
|
74
|
+
return as_polars
|
|
75
|
+
logger.debug(
|
|
76
|
+
"using configured DataFrame backend setting: use_polars=%s",
|
|
77
|
+
config.use_polars,
|
|
78
|
+
)
|
|
79
|
+
return config.use_polars
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
__all__ = ["Config", "config"]
|