datajoint 2.2.0__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datajoint-2.2.0 → datajoint-2.2.2}/CONTRIBUTING.md +38 -2
- {datajoint-2.2.0 → datajoint-2.2.2}/PKG-INFO +2 -2
- {datajoint-2.2.0 → datajoint-2.2.2}/README.md +1 -1
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/__init__.py +7 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/mysql.py +3 -7
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/connection.py +57 -21
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/declare.py +8 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/gc.py +14 -8
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/heading.py +10 -7
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/migrate.py +10 -6
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/schemas.py +7 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/settings.py +34 -14
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/storage.py +14 -5
- datajoint-2.2.2/src/datajoint/storage_adapter.py +109 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/version.py +1 -1
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_gc.py +118 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_json.py +4 -0
- datajoint-2.2.2/tests/unit/test_connection_warning.py +42 -0
- datajoint-2.2.2/tests/unit/test_declare_hidden_attribute.py +51 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_settings.py +65 -0
- datajoint-2.2.2/tests/unit/test_storage_adapter.py +313 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.coveragerc +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.devcontainer/Dockerfile +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.devcontainer/devcontainer.json +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.devcontainer/docker-compose.yml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.dockerignore +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.gitattributes +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/dependabot.yml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/release_drafter.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/draft_release.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/lint.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/post_draft_release_published.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/test.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.gitignore +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.pre-commit-config.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/.vscode/launch.json +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/CHANGELOG-archive.md +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/Dockerfile +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/LICENSE +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/RELEASE_MEMO.md +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/activate.sh +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/docker-compose.yaml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/images/pipeline.drawio +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/images/pipeline.png +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/pixi.lock +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/pyproject.toml +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/__init__.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/base.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/postgres.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/autopopulate.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/blob.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/__init__.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/attach.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/blob.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/filepath.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/hash.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/npy.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/object.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/schema.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/cli.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/codecs.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/condition.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/dependencies.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/diagram.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/errors.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/expression.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/hash_registry.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/instance.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/jobs.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/lineage.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/logging.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/objectref.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/preview.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/staged_insert.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/table.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/types.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/user_tables.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/utils.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/__init__.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/conftest.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/__init__.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Course.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/CurrentTerm.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Department.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Enroll.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Grade.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Section.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Student.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/StudentMajor.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Term.csv +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_aggr_regressions.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_alter.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_attach.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_autopopulate.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_blob.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_blob_matlab.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_cascade_delete.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_cascading_delete.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_cli.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_codec_chaining.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_codecs.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_connection.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_declare.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_dependencies.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_erd.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_fetch.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_fetch_same.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_foreign_keys.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_groupby.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_hash_storage.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_hidden_job_metadata.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_insert.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_jobs.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_multi_backend.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_nan.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_npy_codec.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_object.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_privileges.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_reconnection.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_relation.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_relation_u.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_relational_operand.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_schema.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_schema_keywords.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_semantic_matching.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_skip_duplicates.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_tls.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_type_aliases.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_university.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_update1.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_utils.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_uuid.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_virtual_module.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_advanced.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_aggr_regress.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_alter.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_codecs.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_external.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_object.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_privileges.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_simple.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_type_aliases.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_university.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_uuid.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/test_package.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/__init__.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_adapters.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_codecs.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_condition.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_fetch_compat.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_lazy_imports.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_pk_rules.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_storage_urls.py +0 -0
- {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_thread_safe.py +0 -0
|
@@ -36,13 +36,17 @@ pytest tests/
|
|
|
36
36
|
|
|
37
37
|
## Running Tests
|
|
38
38
|
|
|
39
|
-
Tests use [testcontainers](https://testcontainers.com/) to automatically manage MySQL and MinIO containers. No manual `docker-compose up` required.
|
|
39
|
+
Tests use [testcontainers](https://testcontainers.com/) to automatically manage MySQL, PostgreSQL, and MinIO containers. No manual `docker-compose up` required.
|
|
40
|
+
|
|
41
|
+
Integration tests are **backend-parameterized** — tests using the `backend` fixture run automatically against both MySQL and PostgreSQL.
|
|
40
42
|
|
|
41
43
|
```bash
|
|
42
|
-
pixi run test # All tests
|
|
44
|
+
pixi run test # All tests (both backends)
|
|
43
45
|
pixi run test-cov # With coverage
|
|
44
46
|
pixi run -e test pytest tests/unit/ # Unit tests only
|
|
45
47
|
pixi run -e test pytest tests/integration/test_blob.py -v # Specific file
|
|
48
|
+
pixi run -e test pytest -m mysql # MySQL tests only
|
|
49
|
+
pixi run -e test pytest -m postgresql # PostgreSQL tests only
|
|
46
50
|
```
|
|
47
51
|
|
|
48
52
|
**macOS Docker Desktop users:** If tests fail to connect:
|
|
@@ -50,12 +54,28 @@ pixi run -e test pytest tests/integration/test_blob.py -v # Specific file
|
|
|
50
54
|
export DOCKER_HOST=unix://$HOME/.docker/run/docker.sock
|
|
51
55
|
```
|
|
52
56
|
|
|
57
|
+
### PostgreSQL Backend
|
|
58
|
+
|
|
59
|
+
DataJoint supports MySQL 8.0.13+ and PostgreSQL 15+ as production database backends. To install the PostgreSQL driver:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
pip install -e ".[postgres]" # Installs psycopg2-binary
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Tests automatically spin up both MySQL and PostgreSQL containers via testcontainers. Backend-parameterized tests (those using the `backend` fixture in `tests/conftest.py`) run against both backends to ensure feature parity.
|
|
66
|
+
|
|
53
67
|
### External Containers (for debugging)
|
|
54
68
|
|
|
55
69
|
```bash
|
|
70
|
+
# MySQL + MinIO
|
|
56
71
|
docker compose up -d db minio
|
|
57
72
|
DJ_USE_EXTERNAL_CONTAINERS=1 pixi run test
|
|
58
73
|
docker compose down
|
|
74
|
+
|
|
75
|
+
# MySQL + PostgreSQL + MinIO
|
|
76
|
+
docker compose up -d db postgres minio
|
|
77
|
+
DJ_USE_EXTERNAL_CONTAINERS=1 pixi run test
|
|
78
|
+
docker compose down
|
|
59
79
|
```
|
|
60
80
|
|
|
61
81
|
### Full Docker
|
|
@@ -91,12 +111,28 @@ Hooks include: **ruff** (lint/format), **codespell**, YAML/JSON/TOML validation.
|
|
|
91
111
|
|
|
92
112
|
For `DJ_USE_EXTERNAL_CONTAINERS=1`:
|
|
93
113
|
|
|
114
|
+
### MySQL
|
|
115
|
+
|
|
94
116
|
| Variable | Default | Description |
|
|
95
117
|
|----------|---------|-------------|
|
|
96
118
|
| `DJ_HOST` | `localhost` | MySQL hostname |
|
|
97
119
|
| `DJ_PORT` | `3306` | MySQL port |
|
|
98
120
|
| `DJ_USER` | `root` | MySQL username |
|
|
99
121
|
| `DJ_PASS` | `password` | MySQL password |
|
|
122
|
+
|
|
123
|
+
### PostgreSQL
|
|
124
|
+
|
|
125
|
+
| Variable | Default | Description |
|
|
126
|
+
|----------|---------|-------------|
|
|
127
|
+
| `DJ_PG_HOST` | `localhost` | PostgreSQL hostname |
|
|
128
|
+
| `DJ_PG_PORT` | `5432` | PostgreSQL port |
|
|
129
|
+
| `DJ_PG_USER` | `postgres` | PostgreSQL username |
|
|
130
|
+
| `DJ_PG_PASS` | `password` | PostgreSQL password |
|
|
131
|
+
|
|
132
|
+
### Object Storage
|
|
133
|
+
|
|
134
|
+
| Variable | Default | Description |
|
|
135
|
+
|----------|---------|-------------|
|
|
100
136
|
| `S3_ENDPOINT` | `localhost:9000` | MinIO endpoint |
|
|
101
137
|
|
|
102
138
|
---
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datajoint
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
|
|
5
5
|
Project-URL: Homepage, https://docs.datajoint.com/
|
|
6
6
|
Project-URL: Documentation, https://docs.datajoint.com/
|
|
@@ -346,4 +346,4 @@ conda install -c conda-forge datajoint
|
|
|
346
346
|
|
|
347
347
|
## Contributing
|
|
348
348
|
|
|
349
|
-
See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines.
|
|
349
|
+
See [CONTRIBUTING.md](https://github.com/datajoint/datajoint-python/blob/master/CONTRIBUTING.md) for development setup and guidelines.
|
|
@@ -85,4 +85,4 @@ conda install -c conda-forge datajoint
|
|
|
85
85
|
|
|
86
86
|
## Contributing
|
|
87
87
|
|
|
88
|
-
See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines.
|
|
88
|
+
See [CONTRIBUTING.md](https://github.com/datajoint/datajoint-python/blob/master/CONTRIBUTING.md) for development setup and guidelines.
|
|
@@ -30,6 +30,8 @@ __all__ = [
|
|
|
30
30
|
"list_schemas",
|
|
31
31
|
"Table",
|
|
32
32
|
"FreeTable",
|
|
33
|
+
"AutoPopulate",
|
|
34
|
+
"Job",
|
|
33
35
|
"Manual",
|
|
34
36
|
"Lookup",
|
|
35
37
|
"Imported",
|
|
@@ -81,6 +83,8 @@ from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global
|
|
|
81
83
|
from .logging import logger
|
|
82
84
|
from .objectref import ObjectRef
|
|
83
85
|
from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
|
|
86
|
+
from .autopopulate import AutoPopulate
|
|
87
|
+
from .jobs import Job
|
|
84
88
|
from .table import FreeTable as _FreeTable, Table, ValidationResult
|
|
85
89
|
from .user_tables import Computed, Imported, Lookup, Manual, Part
|
|
86
90
|
from .version import __version__
|
|
@@ -271,6 +275,9 @@ _lazy_modules = {
|
|
|
271
275
|
"diagram": (".diagram", None), # Return the module itself
|
|
272
276
|
# cli imports click
|
|
273
277
|
"cli": (".cli", "cli"),
|
|
278
|
+
# gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
|
|
279
|
+
# and in the user docs (how-to/garbage-collection.md).
|
|
280
|
+
"gc": (".gc", None), # Return the module itself
|
|
274
281
|
}
|
|
275
282
|
|
|
276
283
|
|
|
@@ -423,7 +423,7 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
423
423
|
fk_cols = ", ".join(self.quote_identifier(col) for col in fk["columns"])
|
|
424
424
|
ref_cols = ", ".join(self.quote_identifier(col) for col in fk["ref_columns"])
|
|
425
425
|
lines.append(
|
|
426
|
-
f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols})
|
|
426
|
+
f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols}) ON UPDATE CASCADE ON DELETE RESTRICT"
|
|
427
427
|
)
|
|
428
428
|
|
|
429
429
|
# Indexes
|
|
@@ -735,14 +735,10 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
735
735
|
return result
|
|
736
736
|
|
|
737
737
|
def get_indexes_sql(self, schema_name: str, table_name: str) -> str:
|
|
738
|
-
"""Query to get index definitions.
|
|
739
|
-
|
|
740
|
-
Note: For MySQL 8.0+, EXPRESSION column contains the expression for
|
|
741
|
-
functional indexes. COLUMN_NAME is NULL for such indexes.
|
|
742
|
-
"""
|
|
738
|
+
"""Query to get index definitions. Functional indexes (NULL COLUMN_NAME) are skipped downstream."""
|
|
743
739
|
return (
|
|
744
740
|
f"SELECT INDEX_NAME as index_name, "
|
|
745
|
-
f"
|
|
741
|
+
f"COLUMN_NAME as column_name, "
|
|
746
742
|
f"NON_UNIQUE as non_unique, SEQ_IN_INDEX as seq_in_index "
|
|
747
743
|
f"FROM information_schema.statistics "
|
|
748
744
|
f"WHERE table_schema = {self.quote_string(schema_name)} "
|
|
@@ -108,6 +108,18 @@ def conn(
|
|
|
108
108
|
return conn.connection
|
|
109
109
|
|
|
110
110
|
|
|
111
|
+
def _warn_if_mariadb(version_str: str) -> None:
|
|
112
|
+
"""Emit a UserWarning if `version_str` looks like MariaDB. No-op for MySQL."""
|
|
113
|
+
if "MariaDB" in version_str:
|
|
114
|
+
warnings.warn(
|
|
115
|
+
f"MariaDB is not officially supported by DataJoint 2.x "
|
|
116
|
+
f"(server reports {version_str}). Compatibility is best-effort "
|
|
117
|
+
f"and may break in future releases.",
|
|
118
|
+
UserWarning,
|
|
119
|
+
stacklevel=3,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
111
123
|
class EmulatedCursor:
|
|
112
124
|
"""acts like a cursor"""
|
|
113
125
|
|
|
@@ -168,6 +180,7 @@ class Connection:
|
|
|
168
180
|
port: int | None = None,
|
|
169
181
|
use_tls: bool | dict | None = None,
|
|
170
182
|
*,
|
|
183
|
+
database_name: str | None = None,
|
|
171
184
|
backend: str | None = None,
|
|
172
185
|
config_override: "Config | None" = None,
|
|
173
186
|
) -> None:
|
|
@@ -180,7 +193,9 @@ class Connection:
|
|
|
180
193
|
port = int(port)
|
|
181
194
|
elif port is None:
|
|
182
195
|
port = self._config["database.port"]
|
|
183
|
-
|
|
196
|
+
if database_name is None:
|
|
197
|
+
database_name = self._config.get("database.name")
|
|
198
|
+
self.conn_info = dict(host=host, port=port, user=user, passwd=password, database_name=database_name)
|
|
184
199
|
if use_tls is not False:
|
|
185
200
|
# use_tls can be: None (auto-detect), True (enable), False (disable), or dict (custom config)
|
|
186
201
|
if isinstance(use_tls, dict):
|
|
@@ -201,12 +216,29 @@ class Connection:
|
|
|
201
216
|
backend = self._config["database.backend"]
|
|
202
217
|
self.adapter = get_adapter(backend)
|
|
203
218
|
|
|
219
|
+
if database_name and self.adapter.backend == "mysql":
|
|
220
|
+
warnings.warn(
|
|
221
|
+
"database.name is set but the MySQL backend does not support database selection. "
|
|
222
|
+
"This setting only applies to PostgreSQL connections.",
|
|
223
|
+
UserWarning,
|
|
224
|
+
stacklevel=2,
|
|
225
|
+
)
|
|
226
|
+
|
|
204
227
|
self.connect()
|
|
205
228
|
if self.is_connected:
|
|
206
|
-
|
|
229
|
+
db = self.conn_info.get("database_name")
|
|
230
|
+
db_str = f"/{db}" if db else ""
|
|
231
|
+
logger.info(
|
|
232
|
+
f"DataJoint {__version__} connected to "
|
|
233
|
+
f"{self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}{db_str}"
|
|
234
|
+
)
|
|
207
235
|
self.connection_id = self.adapter.get_connection_id(self._conn)
|
|
236
|
+
if self.adapter.backend == "mysql":
|
|
237
|
+
_warn_if_mariadb(self.query("SELECT @@version").fetchone()[0])
|
|
208
238
|
else:
|
|
209
|
-
raise errors.LostConnectionError(
|
|
239
|
+
raise errors.LostConnectionError(
|
|
240
|
+
f"Connection failed {self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}"
|
|
241
|
+
)
|
|
210
242
|
self._in_transaction = False
|
|
211
243
|
self.schemas = dict()
|
|
212
244
|
self.dependencies = Dependencies(self)
|
|
@@ -216,22 +248,33 @@ class Connection:
|
|
|
216
248
|
|
|
217
249
|
def __repr__(self):
|
|
218
250
|
connected = "connected" if self.is_connected else "disconnected"
|
|
219
|
-
|
|
251
|
+
user = self.conn_info["user"]
|
|
252
|
+
host = self.conn_info["host"]
|
|
253
|
+
port = self.conn_info["port"]
|
|
254
|
+
db = self.conn_info.get("database_name")
|
|
255
|
+
db_str = f"/{db}" if db else ""
|
|
256
|
+
return f"DataJoint connection ({connected}) {user}@{host}:{port}{db_str}"
|
|
257
|
+
|
|
258
|
+
def _build_connect_kwargs(self, use_tls=None):
|
|
259
|
+
"""Build kwargs dict for adapter.connect()."""
|
|
260
|
+
kwargs = dict(
|
|
261
|
+
host=self.conn_info["host"],
|
|
262
|
+
port=self.conn_info["port"],
|
|
263
|
+
user=self.conn_info["user"],
|
|
264
|
+
password=self.conn_info["passwd"],
|
|
265
|
+
charset=self._config["connection.charset"],
|
|
266
|
+
use_tls=use_tls if use_tls is not None else self.conn_info.get("ssl"),
|
|
267
|
+
)
|
|
268
|
+
if self.conn_info.get("database_name"):
|
|
269
|
+
kwargs["dbname"] = self.conn_info["database_name"]
|
|
270
|
+
return kwargs
|
|
220
271
|
|
|
221
272
|
def connect(self) -> None:
|
|
222
273
|
"""Establish or re-establish connection to the database server."""
|
|
223
274
|
with warnings.catch_warnings():
|
|
224
275
|
warnings.filterwarnings("ignore", ".*deprecated.*")
|
|
225
276
|
try:
|
|
226
|
-
|
|
227
|
-
self._conn = self.adapter.connect(
|
|
228
|
-
host=self.conn_info["host"],
|
|
229
|
-
port=self.conn_info["port"],
|
|
230
|
-
user=self.conn_info["user"],
|
|
231
|
-
password=self.conn_info["passwd"],
|
|
232
|
-
charset=self._config["connection.charset"],
|
|
233
|
-
use_tls=self.conn_info.get("ssl"),
|
|
234
|
-
)
|
|
277
|
+
self._conn = self.adapter.connect(**self._build_connect_kwargs())
|
|
235
278
|
except Exception as ssl_error:
|
|
236
279
|
# If SSL fails, retry without SSL (if it was auto-detected)
|
|
237
280
|
if self.conn_info.get("ssl_input") is None:
|
|
@@ -240,14 +283,7 @@ class Connection:
|
|
|
240
283
|
"To require SSL, set use_tls=True explicitly.",
|
|
241
284
|
ssl_error,
|
|
242
285
|
)
|
|
243
|
-
self._conn = self.adapter.connect(
|
|
244
|
-
host=self.conn_info["host"],
|
|
245
|
-
port=self.conn_info["port"],
|
|
246
|
-
user=self.conn_info["user"],
|
|
247
|
-
password=self.conn_info["passwd"],
|
|
248
|
-
charset=self._config["connection.charset"],
|
|
249
|
-
use_tls=False, # Explicitly disable SSL for fallback
|
|
250
|
-
)
|
|
286
|
+
self._conn = self.adapter.connect(**self._build_connect_kwargs(use_tls=False))
|
|
251
287
|
else:
|
|
252
288
|
raise
|
|
253
289
|
self._is_closed = False # Mark as connected after successful connection
|
|
@@ -855,6 +855,14 @@ def compile_attribute(
|
|
|
855
855
|
DataJointError
|
|
856
856
|
If syntax is invalid, primary key is nullable, or blob has invalid default.
|
|
857
857
|
"""
|
|
858
|
+
if line.lstrip().startswith("_"):
|
|
859
|
+
raise DataJointError(
|
|
860
|
+
f'Attribute name in line "{line}" starts with an underscore. '
|
|
861
|
+
"Names with leading underscore are reserved for platform-managed "
|
|
862
|
+
"columns (e.g. _job_start_time, _singleton). Use a regular "
|
|
863
|
+
"attribute name; if you need to control visibility at the call "
|
|
864
|
+
"site, use proj()."
|
|
865
|
+
)
|
|
858
866
|
try:
|
|
859
867
|
match = attribute_parser.parse_string(line + "#", parse_all=True)
|
|
860
868
|
except pp.ParseException as err:
|
|
@@ -229,11 +229,14 @@ def scan_hash_references(
|
|
|
229
229
|
if verbose:
|
|
230
230
|
logger.info(f" Scanning {table_name}.{attr_name}")
|
|
231
231
|
|
|
232
|
-
#
|
|
232
|
+
# Read raw JSON metadata via cursor — bypasses decode_attribute
|
|
233
|
+
# so we get the stored dict (PostgreSQL/JSONB) or JSON string
|
|
234
|
+
# (MySQL), not the decoded codec output. _extract_hash_refs
|
|
235
|
+
# handles both shapes.
|
|
233
236
|
try:
|
|
234
|
-
|
|
235
|
-
for
|
|
236
|
-
for path, ref_store in _extract_hash_refs(
|
|
237
|
+
cursor = table.proj(attr_name).cursor(as_dict=True)
|
|
238
|
+
for row in cursor:
|
|
239
|
+
for path, ref_store in _extract_hash_refs(row[attr_name]):
|
|
237
240
|
# Filter by store if specified
|
|
238
241
|
if store_name is None or ref_store == store_name:
|
|
239
242
|
referenced.add(path)
|
|
@@ -291,11 +294,14 @@ def scan_schema_references(
|
|
|
291
294
|
if verbose:
|
|
292
295
|
logger.info(f" Scanning {table_name}.{attr_name}")
|
|
293
296
|
|
|
294
|
-
#
|
|
297
|
+
# Read raw JSON metadata via cursor — bypasses decode_attribute
|
|
298
|
+
# so we get the stored dict (PostgreSQL/JSONB) or JSON string
|
|
299
|
+
# (MySQL), not the decoded codec output. _extract_schema_refs
|
|
300
|
+
# handles both shapes.
|
|
295
301
|
try:
|
|
296
|
-
|
|
297
|
-
for
|
|
298
|
-
for path, ref_store in _extract_schema_refs(
|
|
302
|
+
cursor = table.proj(attr_name).cursor(as_dict=True)
|
|
303
|
+
for row in cursor:
|
|
304
|
+
for path, ref_store in _extract_schema_refs(row[attr_name]):
|
|
299
305
|
# Filter by store if specified
|
|
300
306
|
if store_name is None or ref_store == store_name:
|
|
301
307
|
referenced.add(path)
|
|
@@ -508,8 +508,16 @@ class Heading:
|
|
|
508
508
|
if category == "UUID":
|
|
509
509
|
attr["uuid"] = True
|
|
510
510
|
elif category in CORE_TYPE_NAMES:
|
|
511
|
-
# Core type alias - already resolved in DB
|
|
512
|
-
|
|
511
|
+
# Core type alias - already resolved in DB.
|
|
512
|
+
# MariaDB-specific recovery: MariaDB stores `json` columns
|
|
513
|
+
# as `longtext` and reports them back that way through
|
|
514
|
+
# information_schema, so the DB-type-based detection above
|
|
515
|
+
# leaves attr["json"] False. The :json: comment marker
|
|
516
|
+
# survives this aliasing, so we recover the json flag here
|
|
517
|
+
# from the original declared type. No-op on MySQL/PostgreSQL
|
|
518
|
+
# (attr["json"] is already True from the regex match above).
|
|
519
|
+
if category == "JSON":
|
|
520
|
+
attr["json"] = True
|
|
513
521
|
|
|
514
522
|
# Check primary key constraints
|
|
515
523
|
if attr["in_key"] and (attr["is_blob"] or attr["json"]):
|
|
@@ -555,14 +563,9 @@ class Heading:
|
|
|
555
563
|
adapter.get_indexes_sql(database, table_name),
|
|
556
564
|
as_dict=True,
|
|
557
565
|
):
|
|
558
|
-
# Note: adapter.get_indexes_sql() already filters out PRIMARY key
|
|
559
|
-
# MySQL/PostgreSQL adapters return: index_name, column_name, non_unique
|
|
560
566
|
index_name = item.get("index_name") or item.get("Key_name")
|
|
561
567
|
seq = item.get("seq_in_index") or item.get("Seq_in_index") or len(keys[index_name]) + 1
|
|
562
568
|
column = item.get("column_name") or item.get("Column_name")
|
|
563
|
-
# MySQL EXPRESSION column stores escaped single quotes - unescape them
|
|
564
|
-
if column:
|
|
565
|
-
column = column.replace("\\'", "'")
|
|
566
569
|
non_unique = item.get("non_unique") or item.get("Non_unique")
|
|
567
570
|
nullable = item.get("nullable") or (item.get("Null", "NO").lower() == "yes")
|
|
568
571
|
|
|
@@ -142,14 +142,14 @@ def analyze_columns(schema: Schema) -> dict:
|
|
|
142
142
|
for (table_name,) in tables:
|
|
143
143
|
# Get all columns for this table
|
|
144
144
|
columns_query = """
|
|
145
|
-
SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT
|
|
145
|
+
SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT, IS_NULLABLE
|
|
146
146
|
FROM information_schema.COLUMNS
|
|
147
147
|
WHERE TABLE_SCHEMA = %s
|
|
148
148
|
AND TABLE_NAME = %s
|
|
149
149
|
"""
|
|
150
150
|
columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
|
|
151
151
|
|
|
152
|
-
for column_name, column_type, data_type, comment in columns:
|
|
152
|
+
for column_name, column_type, data_type, comment, is_nullable in columns:
|
|
153
153
|
comment = comment or ""
|
|
154
154
|
|
|
155
155
|
# Check if column already has a type label (starts with :type:)
|
|
@@ -167,6 +167,7 @@ def analyze_columns(schema: Schema) -> dict:
|
|
|
167
167
|
"column": column_name,
|
|
168
168
|
"native_type": column_type,
|
|
169
169
|
"comment": comment,
|
|
170
|
+
"is_nullable": is_nullable == "YES",
|
|
170
171
|
}
|
|
171
172
|
|
|
172
173
|
if is_external:
|
|
@@ -270,9 +271,10 @@ def migrate_columns(
|
|
|
270
271
|
new_comment_escaped = new_comment.replace("\\", "\\\\").replace("'", "\\'")
|
|
271
272
|
|
|
272
273
|
# Generate ALTER TABLE statement
|
|
274
|
+
not_null = "" if col["is_nullable"] else " NOT NULL"
|
|
273
275
|
sql = (
|
|
274
276
|
f"ALTER TABLE `{db_name}`.`{table_name}` "
|
|
275
|
-
f"MODIFY COLUMN `{col['column']}` {col['native_type']} "
|
|
277
|
+
f"MODIFY COLUMN `{col['column']}` {col['native_type']}{not_null} "
|
|
276
278
|
f"COMMENT '{new_comment_escaped}'"
|
|
277
279
|
)
|
|
278
280
|
result["sql_statements"].append(sql)
|
|
@@ -365,7 +367,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
|
|
|
365
367
|
for (table_name,) in tables:
|
|
366
368
|
# Get column information for each table
|
|
367
369
|
columns_query = """
|
|
368
|
-
SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT
|
|
370
|
+
SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT, IS_NULLABLE
|
|
369
371
|
FROM information_schema.COLUMNS
|
|
370
372
|
WHERE TABLE_SCHEMA = %s
|
|
371
373
|
AND TABLE_NAME = %s
|
|
@@ -374,7 +376,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
|
|
|
374
376
|
|
|
375
377
|
columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
|
|
376
378
|
|
|
377
|
-
for column_name, column_type, comment in columns:
|
|
379
|
+
for column_name, column_type, comment, is_nullable in columns:
|
|
378
380
|
# Check if comment already has a codec type (starts with :type:)
|
|
379
381
|
has_codec = comment and comment.startswith(":")
|
|
380
382
|
|
|
@@ -385,6 +387,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
|
|
|
385
387
|
"column_type": column_type,
|
|
386
388
|
"current_comment": comment or "",
|
|
387
389
|
"needs_migration": not has_codec,
|
|
390
|
+
"is_nullable": is_nullable == "YES",
|
|
388
391
|
}
|
|
389
392
|
)
|
|
390
393
|
|
|
@@ -447,9 +450,10 @@ def generate_migration_sql(
|
|
|
447
450
|
db_name, table_name = col["table_name"].split(".")
|
|
448
451
|
|
|
449
452
|
# Generate ALTER TABLE statement
|
|
453
|
+
not_null = "" if col.get("is_nullable", True) else " NOT NULL"
|
|
450
454
|
sql = (
|
|
451
455
|
f"ALTER TABLE `{db_name}`.`{table_name}` "
|
|
452
|
-
f"MODIFY COLUMN `{col['column_name']}` {col['column_type']} "
|
|
456
|
+
f"MODIFY COLUMN `{col['column_name']}` {col['column_type']}{not_null} "
|
|
453
457
|
f"COMMENT '{new_comment_escaped}'"
|
|
454
458
|
)
|
|
455
459
|
sql_statements.append(sql)
|
|
@@ -172,6 +172,13 @@ class _Schema:
|
|
|
172
172
|
self.connection = connection
|
|
173
173
|
if self.connection is None:
|
|
174
174
|
self.connection = _get_singleton_connection()
|
|
175
|
+
if self.connection._config.get("database.database_prefix"):
|
|
176
|
+
warnings.warn(
|
|
177
|
+
"database_prefix is deprecated and will be removed in DataJoint 2.3. "
|
|
178
|
+
"Use database.name to select a PostgreSQL database instead.",
|
|
179
|
+
DeprecationWarning,
|
|
180
|
+
stacklevel=2,
|
|
181
|
+
)
|
|
175
182
|
self.database = schema_name
|
|
176
183
|
if create_schema is not None:
|
|
177
184
|
self.create_schema = create_schema
|
|
@@ -65,6 +65,7 @@ ENV_VAR_MAPPING = {
|
|
|
65
65
|
"database.password": "DJ_PASS",
|
|
66
66
|
"database.backend": "DJ_BACKEND",
|
|
67
67
|
"database.port": "DJ_PORT",
|
|
68
|
+
"database.name": "DJ_DATABASE_NAME",
|
|
68
69
|
"database.database_prefix": "DJ_DATABASE_PREFIX",
|
|
69
70
|
"database.create_tables": "DJ_CREATE_TABLES",
|
|
70
71
|
"loglevel": "DJ_LOG_LEVEL",
|
|
@@ -196,13 +197,17 @@ class DatabaseSettings(BaseSettings):
|
|
|
196
197
|
description="Database backend: 'mysql' or 'postgresql'",
|
|
197
198
|
)
|
|
198
199
|
port: int | None = Field(default=None, validation_alias="DJ_PORT")
|
|
200
|
+
name: str | None = Field(
|
|
201
|
+
default=None,
|
|
202
|
+
validation_alias="DJ_DATABASE_NAME",
|
|
203
|
+
description="Database name for PostgreSQL connections. Defaults to 'postgres' if not set.",
|
|
204
|
+
)
|
|
199
205
|
reconnect: bool = True
|
|
200
206
|
use_tls: bool | None = Field(default=None, validation_alias="DJ_USE_TLS")
|
|
201
207
|
database_prefix: str = Field(
|
|
202
208
|
default="",
|
|
203
209
|
validation_alias="DJ_DATABASE_PREFIX",
|
|
204
|
-
description="
|
|
205
|
-
"Not automatically applied; use dj.config.database.database_prefix when creating schemas.",
|
|
210
|
+
description="Deprecated. Use database.name instead.",
|
|
206
211
|
)
|
|
207
212
|
create_tables: bool = Field(
|
|
208
213
|
default=True,
|
|
@@ -415,24 +420,29 @@ class Config(BaseSettings):
|
|
|
415
420
|
|
|
416
421
|
spec = dict(self.stores[store])
|
|
417
422
|
|
|
418
|
-
|
|
419
|
-
spec.setdefault("subfolding", None) # No subfolding by default
|
|
420
|
-
spec.setdefault("partition_pattern", None) # No partitioning by default
|
|
421
|
-
spec.setdefault("token_length", 8) # Default token length
|
|
422
|
-
|
|
423
|
-
# Set defaults for storage section prefixes
|
|
424
|
-
spec.setdefault("hash_prefix", "_hash") # Hash-addressed storage section
|
|
425
|
-
spec.setdefault("schema_prefix", "_schema") # Schema-addressed storage section
|
|
426
|
-
spec.setdefault("filepath_prefix", None) # Filepath storage (unrestricted by default)
|
|
423
|
+
self._apply_common_store_defaults(spec)
|
|
427
424
|
|
|
428
425
|
# Validate protocol
|
|
429
426
|
protocol = spec.get("protocol", "").lower()
|
|
430
427
|
supported_protocols = ("file", "s3", "gcs", "azure")
|
|
431
428
|
if protocol not in supported_protocols:
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
429
|
+
from .storage_adapter import get_storage_adapter
|
|
430
|
+
|
|
431
|
+
adapter = get_storage_adapter(protocol)
|
|
432
|
+
if adapter is None:
|
|
433
|
+
raise DataJointError(
|
|
434
|
+
f'Unknown protocol "{protocol}" in config.stores["{store}"]. '
|
|
435
|
+
f"Built-in: {', '.join(supported_protocols)}. "
|
|
436
|
+
f"Install a plugin package for additional protocols."
|
|
437
|
+
)
|
|
438
|
+
adapter.validate_spec(spec)
|
|
439
|
+
self._validate_prefix_separation(
|
|
440
|
+
store_name=store,
|
|
441
|
+
hash_prefix=spec.get("hash_prefix"),
|
|
442
|
+
schema_prefix=spec.get("schema_prefix"),
|
|
443
|
+
filepath_prefix=spec.get("filepath_prefix"),
|
|
435
444
|
)
|
|
445
|
+
return spec
|
|
436
446
|
|
|
437
447
|
# Set protocol-specific defaults
|
|
438
448
|
if protocol == "s3":
|
|
@@ -577,6 +587,16 @@ class Config(BaseSettings):
|
|
|
577
587
|
f"Storage section prefixes must be mutually exclusive."
|
|
578
588
|
)
|
|
579
589
|
|
|
590
|
+
@staticmethod
|
|
591
|
+
def _apply_common_store_defaults(spec: dict[str, Any]) -> None:
|
|
592
|
+
"""Apply defaults shared by every store protocol (built-in and plugin)."""
|
|
593
|
+
spec.setdefault("subfolding", None)
|
|
594
|
+
spec.setdefault("partition_pattern", None)
|
|
595
|
+
spec.setdefault("token_length", 8)
|
|
596
|
+
spec.setdefault("hash_prefix", "_hash")
|
|
597
|
+
spec.setdefault("schema_prefix", "_schema")
|
|
598
|
+
spec.setdefault("filepath_prefix", None)
|
|
599
|
+
|
|
580
600
|
def load(self, filename: str | Path) -> None:
|
|
581
601
|
"""
|
|
582
602
|
Load settings from a JSON file.
|
|
@@ -330,6 +330,15 @@ class StorageBackend:
|
|
|
330
330
|
self._fs = self._create_filesystem()
|
|
331
331
|
return self._fs
|
|
332
332
|
|
|
333
|
+
def _require_adapter(self):
|
|
334
|
+
"""Look up a registered storage adapter, raising if none is registered."""
|
|
335
|
+
from .storage_adapter import get_storage_adapter
|
|
336
|
+
|
|
337
|
+
adapter = get_storage_adapter(self.protocol)
|
|
338
|
+
if adapter is None:
|
|
339
|
+
raise errors.DataJointError(f"Unsupported storage protocol: {self.protocol}")
|
|
340
|
+
return adapter
|
|
341
|
+
|
|
333
342
|
def _create_filesystem(self) -> fsspec.AbstractFileSystem:
|
|
334
343
|
"""Create fsspec filesystem based on protocol."""
|
|
335
344
|
if self.protocol == "file":
|
|
@@ -368,7 +377,7 @@ class StorageBackend:
|
|
|
368
377
|
)
|
|
369
378
|
|
|
370
379
|
else:
|
|
371
|
-
|
|
380
|
+
return self._require_adapter().create_filesystem(self.spec)
|
|
372
381
|
|
|
373
382
|
def _full_path(self, path: str | PurePosixPath) -> str:
|
|
374
383
|
"""
|
|
@@ -397,12 +406,13 @@ class StorageBackend:
|
|
|
397
406
|
if location:
|
|
398
407
|
return f"{bucket}/{location}/{path}"
|
|
399
408
|
return f"{bucket}/{path}"
|
|
400
|
-
|
|
401
|
-
# Local filesystem - prepend location if specified
|
|
409
|
+
elif self.protocol == "file":
|
|
402
410
|
location = self.spec.get("location", "")
|
|
403
411
|
if location:
|
|
404
412
|
return str(Path(location) / path)
|
|
405
413
|
return path
|
|
414
|
+
else:
|
|
415
|
+
return self._require_adapter().full_path(self.spec, path)
|
|
406
416
|
|
|
407
417
|
def get_url(self, path: str | PurePosixPath) -> str:
|
|
408
418
|
"""
|
|
@@ -448,8 +458,7 @@ class StorageBackend:
|
|
|
448
458
|
elif self.protocol == "azure":
|
|
449
459
|
return f"az://{full_path}"
|
|
450
460
|
else:
|
|
451
|
-
|
|
452
|
-
return f"{self.protocol}://{full_path}"
|
|
461
|
+
return self._require_adapter().get_url(self.spec, full_path)
|
|
453
462
|
|
|
454
463
|
def put_file(self, local_path: str | Path, remote_path: str | PurePosixPath, metadata: dict | None = None) -> None:
|
|
455
464
|
"""
|