datajoint 2.2.4__tar.gz → 2.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/release_drafter.yaml +1 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/test.yaml +16 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/.pre-commit-config.yaml +3 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/PKG-INFO +8 -3
- {datajoint-2.2.4 → datajoint-2.3.1}/README.md +1 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/pyproject.toml +19 -2
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/__init__.py +6 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/base.py +23 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/mysql.py +10 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/postgres.py +39 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/autopopulate.py +77 -6
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/hash.py +1 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/npy.py +1 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/object.py +1 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/schema.py +3 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/codecs.py +40 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/condition.py +14 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/dependencies.py +42 -0
- datajoint-2.3.1/src/datajoint/deploy.py +185 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/diagram.py +482 -19
- datajoint-2.3.1/src/datajoint/gc.py +397 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/hash_registry.py +26 -9
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/heading.py +47 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/schemas.py +17 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/settings.py +13 -0
- datajoint-2.3.1/src/datajoint/spark.py +92 -0
- datajoint-2.3.1/src/datajoint/staged_insert.py +275 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/storage.py +10 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/table.py +45 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/version.py +1 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/conftest.py +4 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_autopopulate.py +243 -0
- datajoint-2.3.1/tests/integration/test_cascade_delete.py +581 -0
- datajoint-2.3.1/tests/integration/test_cascade_integrity.py +157 -0
- datajoint-2.3.1/tests/integration/test_gc.py +764 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_hash_storage.py +30 -17
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_object.py +104 -1
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_semantic_matching.py +150 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_tls.py +15 -3
- datajoint-2.3.1/tests/integration/test_trace.py +449 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_adapters.py +17 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_codecs.py +38 -0
- datajoint-2.3.1/tests/unit/test_deploy.py +172 -0
- datajoint-2.3.1/tests/unit/test_spark.py +105 -0
- datajoint-2.2.4/src/datajoint/gc.py +0 -710
- datajoint-2.2.4/src/datajoint/staged_insert.py +0 -310
- datajoint-2.2.4/tests/integration/test_cascade_delete.py +0 -294
- datajoint-2.2.4/tests/integration/test_gc.py +0 -467
- {datajoint-2.2.4 → datajoint-2.3.1}/.coveragerc +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.dockerignore +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.gitattributes +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/dependabot.yml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/draft_release.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/lint.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/post_draft_release_published.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/.gitignore +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/CHANGELOG-archive.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/CONTRIBUTING.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/Dockerfile +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/LICENSE +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/RELEASE_MEMO.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/activate.sh +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/docker-compose.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/images/pipeline.png +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/images/pipeline.svg +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/pixi.lock +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/blob.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/attach.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/blob.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/filepath.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/cli.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/connection.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/declare.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/errors.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/expression.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/instance.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/jobs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/lineage.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/logging.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/migrate.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/objectref.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/preview.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/storage_adapter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/types.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/user_tables.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/utils.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Course.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/CurrentTerm.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Department.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Enroll.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Grade.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Section.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Student.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/StudentMajor.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Term.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_aggr_regressions.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_alter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_attach.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_blob.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_blob_matlab.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_cascading_delete.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_cli.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_codec_chaining.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_codecs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_connection.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_declare.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_dependencies.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_erd.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_fetch.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_fetch_same.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_foreign_keys.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_groupby.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_hidden_job_metadata.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_insert.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_jobs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_json.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_multi_backend.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_nan.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_npy_codec.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_privileges.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_reconnection.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_relation.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_relation_u.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_relational_operand.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_schema.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_schema_keywords.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_skip_duplicates.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_type_aliases.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_university.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_update1.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_utils.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_uuid.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_virtual_module.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_advanced.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_aggr_regress.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_alter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_codecs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_external.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_object.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_privileges.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_simple.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_type_aliases.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_university.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_uuid.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/test_package.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_condition.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_connection_warning.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_declare_hidden_attribute.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_fetch_compat.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_lazy_imports.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_pk_rules.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_settings.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_storage_adapter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_storage_urls.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_thread_safe.py +0 -0
|
@@ -28,7 +28,7 @@ categories:
|
|
|
28
28
|
- title: '📝 Documentation'
|
|
29
29
|
label: 'documentation'
|
|
30
30
|
change-template: '- $TITLE(#$NUMBER)@$AUTHOR'
|
|
31
|
-
change-title-escapes: '\<*_
|
|
31
|
+
change-title-escapes: '\<*_&@#' # escape @ and # so decorator/issue refs in PR titles (e.g. `@schema`) don't render as spurious mentions/links in release notes.
|
|
32
32
|
template: |
|
|
33
33
|
$CHANGES
|
|
34
34
|
|
|
@@ -27,6 +27,18 @@ on:
|
|
|
27
27
|
jobs:
|
|
28
28
|
test:
|
|
29
29
|
runs-on: ubuntu-latest
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
matrix:
|
|
33
|
+
# Python: exercise both ends of the supported range (requires-python
|
|
34
|
+
# >=3.10,<3.15); version-pinned pixi environments live in pyproject.toml
|
|
35
|
+
# under [tool.pixi.environments].
|
|
36
|
+
environment: [test-py310, test-py314]
|
|
37
|
+
# MySQL: exercise both LTS lines we support / that AWS RDS runs. 8.0 uses
|
|
38
|
+
# the datajoint image (SSL configured); 8.4 uses the official image (TLS
|
|
39
|
+
# tests already skip outside external containers). See #1497.
|
|
40
|
+
mysql-image: ["datajoint/mysql:8.0", "mysql:8.4"]
|
|
41
|
+
name: test (${{ matrix.environment }}, mysql=${{ matrix.mysql-image }})
|
|
30
42
|
steps:
|
|
31
43
|
- uses: actions/checkout@v4
|
|
32
44
|
|
|
@@ -35,9 +47,12 @@ jobs:
|
|
|
35
47
|
with:
|
|
36
48
|
cache: true
|
|
37
49
|
locked: false
|
|
50
|
+
environments: ${{ matrix.environment }}
|
|
38
51
|
|
|
39
52
|
- name: Run tests
|
|
40
|
-
|
|
53
|
+
env:
|
|
54
|
+
DJ_TEST_MYSQL_IMAGE: ${{ matrix.mysql-image }}
|
|
55
|
+
run: pixi run -e ${{ matrix.environment }} test-cov
|
|
41
56
|
|
|
42
57
|
# Unit tests run without containers (faster feedback)
|
|
43
58
|
unit-tests:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datajoint
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.1
|
|
4
4
|
Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
|
|
5
5
|
Project-URL: Homepage, https://docs.datajoint.com/
|
|
6
6
|
Project-URL: Documentation, https://docs.datajoint.com/
|
|
@@ -207,11 +207,16 @@ Classifier: Intended Audience :: Healthcare Industry
|
|
|
207
207
|
Classifier: Intended Audience :: Science/Research
|
|
208
208
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
209
209
|
Classifier: Programming Language :: Python
|
|
210
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
211
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
212
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
213
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
214
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
210
215
|
Classifier: Topic :: Scientific/Engineering
|
|
211
216
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
212
217
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
213
218
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
214
|
-
Requires-Python: <3.
|
|
219
|
+
Requires-Python: <3.15,>=3.10
|
|
215
220
|
Requires-Dist: deepdiff
|
|
216
221
|
Requires-Dist: fsspec>=2023.1.0
|
|
217
222
|
Requires-Dist: networkx
|
|
@@ -267,7 +272,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
|
|
|
267
272
|
- **Tables represent workflow steps** — Each table is a step in your pipeline
|
|
268
273
|
- **Foreign keys encode dependencies** — Parent tables must be populated before child tables
|
|
269
274
|
- **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
|
|
270
|
-
- **Results are immutable** — Full
|
|
275
|
+
- **Results are immutable** — Full lineage and reproducibility
|
|
271
276
|
|
|
272
277
|
**Documentation:** https://docs.datajoint.com
|
|
273
278
|
|
|
@@ -5,7 +5,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
|
|
|
5
5
|
- **Tables represent workflow steps** — Each table is a step in your pipeline
|
|
6
6
|
- **Foreign keys encode dependencies** — Parent tables must be populated before child tables
|
|
7
7
|
- **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
|
|
8
|
-
- **Results are immutable** — Full
|
|
8
|
+
- **Results are immutable** — Full lineage and reproducibility
|
|
9
9
|
|
|
10
10
|
**Documentation:** https://docs.datajoint.com
|
|
11
11
|
|
|
@@ -19,7 +19,7 @@ dependencies = [
|
|
|
19
19
|
"packaging",
|
|
20
20
|
]
|
|
21
21
|
|
|
22
|
-
requires-python = ">=3.10,<3.
|
|
22
|
+
requires-python = ">=3.10,<3.15"
|
|
23
23
|
authors = [
|
|
24
24
|
{name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"},
|
|
25
25
|
{name = "Thinh Nguyen", email = "thinh@datajoint.com"},
|
|
@@ -57,6 +57,11 @@ keywords = [
|
|
|
57
57
|
# https://pypi.org/classifiers/
|
|
58
58
|
classifiers = [
|
|
59
59
|
"Programming Language :: Python",
|
|
60
|
+
"Programming Language :: Python :: 3.10",
|
|
61
|
+
"Programming Language :: Python :: 3.11",
|
|
62
|
+
"Programming Language :: Python :: 3.12",
|
|
63
|
+
"Programming Language :: Python :: 3.13",
|
|
64
|
+
"Programming Language :: Python :: 3.14",
|
|
60
65
|
"Development Status :: 5 - Production/Stable",
|
|
61
66
|
"Intended Audience :: Science/Research",
|
|
62
67
|
"Intended Audience :: Healthcare Industry",
|
|
@@ -250,10 +255,22 @@ datajoint = { path = ".", editable = true, extras = ["test"] }
|
|
|
250
255
|
[tool.pixi.feature.dev.pypi-dependencies]
|
|
251
256
|
datajoint = { path = ".", editable = true, extras = ["dev", "test"] }
|
|
252
257
|
|
|
258
|
+
# Pin the interpreter for the min/max supported versions so CI can exercise
|
|
259
|
+
# both ends of the requires-python range (see .github/workflows/test.yaml).
|
|
260
|
+
[tool.pixi.feature.py310.dependencies]
|
|
261
|
+
python = "3.10.*"
|
|
262
|
+
|
|
263
|
+
[tool.pixi.feature.py314.dependencies]
|
|
264
|
+
python = "3.14.*"
|
|
265
|
+
|
|
253
266
|
[tool.pixi.environments]
|
|
254
267
|
default = { solve-group = "default" }
|
|
255
268
|
dev = { features = ["dev"], solve-group = "default" }
|
|
256
269
|
test = { features = ["test"], solve-group = "default" }
|
|
270
|
+
# Version-pinned test environments (each solves independently — a solve-group
|
|
271
|
+
# cannot span different Python versions).
|
|
272
|
+
test-py310 = { features = ["test", "py310"] }
|
|
273
|
+
test-py314 = { features = ["test", "py314"] }
|
|
257
274
|
|
|
258
275
|
[tool.pixi.tasks]
|
|
259
276
|
# Tests use testcontainers - no manual setup required
|
|
@@ -265,7 +282,7 @@ services-down = "docker compose down"
|
|
|
265
282
|
test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] }
|
|
266
283
|
|
|
267
284
|
[tool.pixi.dependencies]
|
|
268
|
-
python = ">=3.10,<3.
|
|
285
|
+
python = ">=3.10,<3.15"
|
|
269
286
|
graphviz = ">=13.1.2,<14"
|
|
270
287
|
|
|
271
288
|
[tool.pixi.activation]
|
|
@@ -51,12 +51,15 @@ __all__ = [
|
|
|
51
51
|
"get_codec",
|
|
52
52
|
"ObjectRef",
|
|
53
53
|
"NpyRef",
|
|
54
|
+
# SparkAdapter Codec Protocol
|
|
55
|
+
"SparkAdapter",
|
|
54
56
|
# Storage Adapter API
|
|
55
57
|
"StorageAdapter",
|
|
56
58
|
"get_storage_adapter",
|
|
57
59
|
# Other
|
|
58
60
|
"errors",
|
|
59
61
|
"migrate",
|
|
62
|
+
"deploy",
|
|
60
63
|
"DataJointError",
|
|
61
64
|
"ThreadSafetyError",
|
|
62
65
|
"logger",
|
|
@@ -69,6 +72,7 @@ __all__ = [
|
|
|
69
72
|
# =============================================================================
|
|
70
73
|
from . import errors
|
|
71
74
|
from . import migrate
|
|
75
|
+
from . import deploy
|
|
72
76
|
from .codecs import (
|
|
73
77
|
Codec,
|
|
74
78
|
get_codec,
|
|
@@ -85,6 +89,7 @@ from .expression import AndList, Not, Top, U
|
|
|
85
89
|
from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global_config, _check_thread_safe
|
|
86
90
|
from .logging import logger
|
|
87
91
|
from .objectref import ObjectRef
|
|
92
|
+
from .spark import SparkAdapter
|
|
88
93
|
from .storage_adapter import StorageAdapter, get_storage_adapter
|
|
89
94
|
from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
|
|
90
95
|
from .autopopulate import AutoPopulate
|
|
@@ -279,7 +284,7 @@ _lazy_modules = {
|
|
|
279
284
|
"diagram": (".diagram", None), # Return the module itself
|
|
280
285
|
# cli imports click
|
|
281
286
|
"cli": (".cli", "cli"),
|
|
282
|
-
# gc — exposed lazily so `dj.gc.
|
|
287
|
+
# gc — exposed lazily so `dj.gc.GarbageCollector(...)` works as documented in gc.py
|
|
283
288
|
# and in the user docs (how-to/garbage-collection.md).
|
|
284
289
|
"gc": (".gc", None), # Return the module itself
|
|
285
290
|
}
|
|
@@ -850,6 +850,29 @@ class DatabaseAdapter(ABC):
|
|
|
850
850
|
raise NotImplementedError
|
|
851
851
|
...
|
|
852
852
|
|
|
853
|
+
def find_upstream_schemas_sql(self, schemas_list: str) -> str:
|
|
854
|
+
"""
|
|
855
|
+
Generate query to find schemas that the given schemas reference via FK.
|
|
856
|
+
|
|
857
|
+
Used to discover unloaded schemas that the loaded ones depend on
|
|
858
|
+
(the upstream / ancestor direction). Symmetric to
|
|
859
|
+
:meth:`find_downstream_schemas_sql`.
|
|
860
|
+
|
|
861
|
+
Parameters
|
|
862
|
+
----------
|
|
863
|
+
schemas_list : str
|
|
864
|
+
Comma-separated, quoted schema names for an IN clause.
|
|
865
|
+
|
|
866
|
+
Returns
|
|
867
|
+
-------
|
|
868
|
+
str
|
|
869
|
+
SQL query returning rows with a single column ``schema_name``
|
|
870
|
+
containing distinct schema names that are referenced by the
|
|
871
|
+
given schemas.
|
|
872
|
+
"""
|
|
873
|
+
raise NotImplementedError
|
|
874
|
+
...
|
|
875
|
+
|
|
853
876
|
@abstractmethod
|
|
854
877
|
def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
|
|
855
878
|
"""
|
|
@@ -696,6 +696,16 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
696
696
|
f"AND table_schema NOT IN ({schemas_list})"
|
|
697
697
|
)
|
|
698
698
|
|
|
699
|
+
def find_upstream_schemas_sql(self, schemas_list: str) -> str:
|
|
700
|
+
"""Find schemas that the given schemas reference via FK."""
|
|
701
|
+
return (
|
|
702
|
+
f"SELECT DISTINCT referenced_table_schema as schema_name "
|
|
703
|
+
f"FROM information_schema.key_column_usage "
|
|
704
|
+
f"WHERE table_schema IN ({schemas_list}) "
|
|
705
|
+
f"AND referenced_table_schema IS NOT NULL "
|
|
706
|
+
f"AND referenced_table_schema NOT IN ({schemas_list})"
|
|
707
|
+
)
|
|
708
|
+
|
|
699
709
|
def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
|
|
700
710
|
"""Query to get FK constraint details from information_schema."""
|
|
701
711
|
return (
|
|
@@ -861,6 +861,20 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
861
861
|
f"AND ns1.nspname NOT IN ({schemas_list})"
|
|
862
862
|
)
|
|
863
863
|
|
|
864
|
+
def find_upstream_schemas_sql(self, schemas_list: str) -> str:
|
|
865
|
+
"""Find schemas that the given schemas reference via FK."""
|
|
866
|
+
return (
|
|
867
|
+
f"SELECT DISTINCT ns2.nspname as schema_name "
|
|
868
|
+
f"FROM pg_constraint c "
|
|
869
|
+
f"JOIN pg_class cl1 ON c.conrelid = cl1.oid "
|
|
870
|
+
f"JOIN pg_namespace ns1 ON cl1.relnamespace = ns1.oid "
|
|
871
|
+
f"JOIN pg_class cl2 ON c.confrelid = cl2.oid "
|
|
872
|
+
f"JOIN pg_namespace ns2 ON cl2.relnamespace = ns2.oid "
|
|
873
|
+
f"WHERE c.contype = 'f' "
|
|
874
|
+
f"AND ns1.nspname IN ({schemas_list}) "
|
|
875
|
+
f"AND ns2.nspname NOT IN ({schemas_list})"
|
|
876
|
+
)
|
|
877
|
+
|
|
864
878
|
def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
|
|
865
879
|
"""
|
|
866
880
|
Query to get FK constraint details from information_schema.
|
|
@@ -1266,6 +1280,31 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
1266
1280
|
quoted_values = ", ".join(f"'{v}'" for v in values)
|
|
1267
1281
|
return f"CREATE TYPE {self.quote_identifier(type_name)} AS ENUM ({quoted_values})"
|
|
1268
1282
|
|
|
1283
|
+
def replica_identity_ddl(self, full_table_name: str, mode: str) -> str:
|
|
1284
|
+
"""
|
|
1285
|
+
Generate ALTER TABLE ... REPLICA IDENTITY statement.
|
|
1286
|
+
|
|
1287
|
+
Controls how much of the old row PostgreSQL writes to WAL on UPDATE/DELETE.
|
|
1288
|
+
``"default"`` logs only primary-key columns; ``"full"`` logs the entire row.
|
|
1289
|
+
Required by some CDC tools (e.g. Databricks Lakehouse Sync) that need the
|
|
1290
|
+
full pre-image to drive Slowly-Changing-Dimension history.
|
|
1291
|
+
|
|
1292
|
+
The ALTER is metadata-only, instant, and idempotent — re-applying the same
|
|
1293
|
+
mode is a no-op at the storage layer.
|
|
1294
|
+
|
|
1295
|
+
Examples
|
|
1296
|
+
--------
|
|
1297
|
+
>>> adapter.replica_identity_ddl('"schema"."table"', 'full')
|
|
1298
|
+
'ALTER TABLE "schema"."table" REPLICA IDENTITY FULL'
|
|
1299
|
+
>>> adapter.replica_identity_ddl('"schema"."table"', 'default')
|
|
1300
|
+
'ALTER TABLE "schema"."table" REPLICA IDENTITY DEFAULT'
|
|
1301
|
+
"""
|
|
1302
|
+
if mode not in ("default", "full"):
|
|
1303
|
+
from ..errors import DataJointError
|
|
1304
|
+
|
|
1305
|
+
raise DataJointError(f"Unsupported replica_identity mode: {mode!r}. Expected 'default' or 'full'.")
|
|
1306
|
+
return f"ALTER TABLE {full_table_name} REPLICA IDENTITY {mode.upper()}"
|
|
1307
|
+
|
|
1269
1308
|
def get_pending_enum_ddl(self, schema_name: str) -> list[str]:
|
|
1270
1309
|
"""
|
|
1271
1310
|
Get DDL statements for pending enum types and clear the pending list.
|
|
@@ -22,6 +22,14 @@ if TYPE_CHECKING:
|
|
|
22
22
|
|
|
23
23
|
logger = logging.getLogger(__name__.split(".")[0])
|
|
24
24
|
|
|
25
|
+
# Parallel populate hands the live table and its open connection to workers by
|
|
26
|
+
# process inheritance, so it requires the `fork` start method. Python 3.14
|
|
27
|
+
# changed the default from `fork` to `forkserver`, which pickles the payload
|
|
28
|
+
# instead — that neither carries a live DB connection nor the table's internal
|
|
29
|
+
# state, and it deadlocks. Pin to `fork` where available (all POSIX platforms);
|
|
30
|
+
# fall back to the platform default elsewhere.
|
|
31
|
+
_MP_START_METHOD = "fork" if "fork" in mp.get_all_start_methods() else None
|
|
32
|
+
|
|
25
33
|
|
|
26
34
|
# --- helper functions for multiprocessing --
|
|
27
35
|
|
|
@@ -30,7 +38,8 @@ def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[s
|
|
|
30
38
|
"""
|
|
31
39
|
Initialize a worker process for multiprocessing.
|
|
32
40
|
|
|
33
|
-
|
|
41
|
+
Stores the inherited table on the worker process and reconnects to database
|
|
42
|
+
(the parent closes its connection before forking; each worker reopens one).
|
|
34
43
|
|
|
35
44
|
Parameters
|
|
36
45
|
----------
|
|
@@ -89,6 +98,55 @@ class AutoPopulate:
|
|
|
89
98
|
_key_source = None
|
|
90
99
|
_allow_insert = False
|
|
91
100
|
_jobs = None
|
|
101
|
+
_upstream = None # memoized upstream trace for the current make(); built lazily on first use
|
|
102
|
+
_upstream_key = None # current make() key; not None iff executing inside make()
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def upstream(self):
|
|
106
|
+
"""
|
|
107
|
+
Pre-restricted ancestor view for the current ``make(self, key)`` call.
|
|
108
|
+
|
|
109
|
+
Inside ``make()``, ``self.upstream`` is a ``Diagram`` constructed via
|
|
110
|
+
:meth:`Diagram.trace(self & key) <datajoint.Diagram.trace>`. Use
|
|
111
|
+
``self.upstream[T]`` to obtain a pre-restricted ``QueryExpression``
|
|
112
|
+
(or ``FreeTable``, when indexed by a string) for any ancestor of
|
|
113
|
+
``self``.
|
|
114
|
+
|
|
115
|
+
Reading via ``self.upstream`` is the recommended pattern for the
|
|
116
|
+
make() reproducibility contract: the framework guarantees the
|
|
117
|
+
restriction matches the current ``key``, and indexing a non-ancestor
|
|
118
|
+
table raises ``DataJointError``. See
|
|
119
|
+
:doc:`reference/specs/autopopulate` for the contract.
|
|
120
|
+
|
|
121
|
+
Raises
|
|
122
|
+
------
|
|
123
|
+
DataJointError
|
|
124
|
+
If accessed outside ``make()`` execution. To construct a trace
|
|
125
|
+
explicitly, use ``dj.Diagram.trace(self & key)``.
|
|
126
|
+
|
|
127
|
+
Examples
|
|
128
|
+
--------
|
|
129
|
+
::
|
|
130
|
+
|
|
131
|
+
def make(self, key):
|
|
132
|
+
date = self.upstream[Session].fetch1("session_date")
|
|
133
|
+
traces = self.upstream[ExtractTraces].to_arrays("trace")
|
|
134
|
+
self.insert1({**key, "summary": compute(traces, date)})
|
|
135
|
+
"""
|
|
136
|
+
if self._upstream_key is None:
|
|
137
|
+
raise DataJointError(
|
|
138
|
+
"self.upstream is only available inside make(). "
|
|
139
|
+
"Outside make(), construct a trace explicitly: "
|
|
140
|
+
"dj.Diagram.trace(self & key)."
|
|
141
|
+
)
|
|
142
|
+
if self._upstream is None:
|
|
143
|
+
# Build the trace lazily on first access and memoize it for this
|
|
144
|
+
# make() call, so make() calls that never read self.upstream pay
|
|
145
|
+
# nothing. See #1493.
|
|
146
|
+
from .diagram import Diagram
|
|
147
|
+
|
|
148
|
+
self._upstream = Diagram.trace(self & self._upstream_key)
|
|
149
|
+
return self._upstream
|
|
92
150
|
|
|
93
151
|
class _JobsDescriptor:
|
|
94
152
|
"""Descriptor allowing jobs access on both class and instance."""
|
|
@@ -436,7 +494,9 @@ class AutoPopulate:
|
|
|
436
494
|
if hasattr(self.connection._conn, "ctx"):
|
|
437
495
|
del self.connection._conn.ctx
|
|
438
496
|
with (
|
|
439
|
-
mp.Pool(
|
|
497
|
+
mp.get_context(_MP_START_METHOD).Pool(
|
|
498
|
+
processes, _initialize_populate, (self, None, populate_kwargs)
|
|
499
|
+
) as pool,
|
|
440
500
|
tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar,
|
|
441
501
|
):
|
|
442
502
|
for status in pool.imap(_call_populate1, keys, chunksize=1):
|
|
@@ -532,7 +592,9 @@ class AutoPopulate:
|
|
|
532
592
|
if hasattr(self.connection._conn, "ctx"):
|
|
533
593
|
del self.connection._conn.ctx # SSLContext is not pickleable
|
|
534
594
|
with (
|
|
535
|
-
mp.Pool(
|
|
595
|
+
mp.get_context(_MP_START_METHOD).Pool(
|
|
596
|
+
processes, _initialize_populate, (self, self.jobs, populate_kwargs)
|
|
597
|
+
) as pool,
|
|
536
598
|
tqdm(desc="Processes: ", total=nkeys)
|
|
537
599
|
if display_progress
|
|
538
600
|
else contextlib.nullcontext() as progress_bar,
|
|
@@ -611,6 +673,12 @@ class AutoPopulate:
|
|
|
611
673
|
logger.jobs(f"Making {key} -> {self.full_table_name}")
|
|
612
674
|
self.__class__._allow_insert = True
|
|
613
675
|
|
|
676
|
+
# Record the key for a lazily-constructed upstream view. The trace is
|
|
677
|
+
# built on first `self.upstream` access (see the `upstream` property),
|
|
678
|
+
# so make() calls that never read upstream pay nothing. See #1493.
|
|
679
|
+
self._upstream_key = dict(key)
|
|
680
|
+
self._upstream = None
|
|
681
|
+
|
|
614
682
|
try:
|
|
615
683
|
if not is_generator:
|
|
616
684
|
make(dict(key), **(make_kwargs or {}))
|
|
@@ -668,6 +736,11 @@ class AutoPopulate:
|
|
|
668
736
|
return True
|
|
669
737
|
finally:
|
|
670
738
|
self.__class__._allow_insert = False
|
|
739
|
+
# Clear the per-make() upstream view so subsequent attribute
|
|
740
|
+
# access raises a clear error rather than silently using a
|
|
741
|
+
# stale trace from the previous make() call.
|
|
742
|
+
self._upstream = None
|
|
743
|
+
self._upstream_key = None
|
|
671
744
|
|
|
672
745
|
def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]:
|
|
673
746
|
"""
|
|
@@ -782,8 +855,6 @@ class AutoPopulate:
|
|
|
782
855
|
|
|
783
856
|
pk_condition = make_condition(self, key, set())
|
|
784
857
|
self.connection.query(
|
|
785
|
-
f"UPDATE {self.full_table_name} SET "
|
|
786
|
-
"_job_start_time=%s, _job_duration=%s, _job_version=%s "
|
|
787
|
-
f"WHERE {pk_condition}",
|
|
858
|
+
f"UPDATE {self.full_table_name} SET _job_start_time=%s, _job_duration=%s, _job_version=%s WHERE {pk_condition}",
|
|
788
859
|
args=(start_time, duration, version[:64] if version else ""),
|
|
789
860
|
)
|
|
@@ -21,7 +21,7 @@ class HashCodec(Codec):
|
|
|
21
21
|
The database column stores JSON metadata: ``{hash, store, size}``.
|
|
22
22
|
Duplicate content is automatically deduplicated across all tables.
|
|
23
23
|
|
|
24
|
-
Deletion: Requires garbage collection via ``dj.gc.
|
|
24
|
+
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
|
|
25
25
|
|
|
26
26
|
External only - requires @ modifier.
|
|
27
27
|
|
|
@@ -272,7 +272,7 @@ class NpyCodec(SchemaCodec):
|
|
|
272
272
|
- Path: ``{schema}/{table}/{pk}/{attribute}.npy``
|
|
273
273
|
- Database column: JSON with ``{path, store, dtype, shape}``
|
|
274
274
|
|
|
275
|
-
Deletion: Requires garbage collection via ``dj.gc.
|
|
275
|
+
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
|
|
276
276
|
|
|
277
277
|
See Also
|
|
278
278
|
--------
|
|
@@ -53,7 +53,7 @@ class ObjectCodec(SchemaCodec):
|
|
|
53
53
|
|
|
54
54
|
{store_root}/{schema}/{table}/{pk}/{field}/
|
|
55
55
|
|
|
56
|
-
Deletion: Requires garbage collection via ``dj.gc.
|
|
56
|
+
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
|
|
57
57
|
|
|
58
58
|
Comparison with hash-addressed::
|
|
59
59
|
|
|
@@ -114,7 +114,7 @@ class SchemaCodec(Codec, register=False):
|
|
|
114
114
|
Build schema-addressed storage path.
|
|
115
115
|
|
|
116
116
|
Constructs a path that mirrors the database schema structure:
|
|
117
|
-
``{schema}/{table}/{pk_values}/{field}{ext}``
|
|
117
|
+
``{schema_prefix}/{schema}/{table}/{pk_values}/{field}{ext}``
|
|
118
118
|
|
|
119
119
|
Supports partitioning if configured in the store.
|
|
120
120
|
|
|
@@ -150,6 +150,7 @@ class SchemaCodec(Codec, register=False):
|
|
|
150
150
|
spec = config.get_store_spec(store_name)
|
|
151
151
|
partition_pattern = spec.get("partition_pattern")
|
|
152
152
|
token_length = spec.get("token_length", 8)
|
|
153
|
+
schema_prefix = spec["schema_prefix"] # always present: settings applies the default
|
|
153
154
|
|
|
154
155
|
return build_object_path(
|
|
155
156
|
schema=schema,
|
|
@@ -159,6 +160,7 @@ class SchemaCodec(Codec, register=False):
|
|
|
159
160
|
ext=ext,
|
|
160
161
|
partition_pattern=partition_pattern,
|
|
161
162
|
token_length=token_length,
|
|
163
|
+
schema_prefix=schema_prefix,
|
|
162
164
|
)
|
|
163
165
|
|
|
164
166
|
def _get_backend(self, store_name: str | None = None, config=None):
|
|
@@ -38,6 +38,7 @@ class MyTable(dj.Manual):
|
|
|
38
38
|
|
|
39
39
|
from __future__ import annotations
|
|
40
40
|
|
|
41
|
+
import json
|
|
41
42
|
import logging
|
|
42
43
|
from abc import ABC, abstractmethod
|
|
43
44
|
from typing import Any
|
|
@@ -219,6 +220,45 @@ class Codec(ABC):
|
|
|
219
220
|
"""
|
|
220
221
|
pass
|
|
221
222
|
|
|
223
|
+
def referenced_paths(self, stored: Any) -> list[tuple[str, str | None]]:
|
|
224
|
+
"""
|
|
225
|
+
Return the external store paths this stored value references.
|
|
226
|
+
|
|
227
|
+
Garbage collection and delete-time cleanup call this on the *stored*
|
|
228
|
+
(encoded) representation of a column value — the JSON/dict already in
|
|
229
|
+
the database — so discovery stays metadata-only (no download or decode).
|
|
230
|
+
|
|
231
|
+
The default recognizes DataJoint's standard external-storage metadata:
|
|
232
|
+
a dict (or JSON string) carrying a ``path`` and optional ``store``. This
|
|
233
|
+
covers hash-addressed (``<hash@>``/``<blob@>``/``<attach@>``) and
|
|
234
|
+
schema-addressed (``<object@>``/``<npy@>`` and any :class:`SchemaCodec`
|
|
235
|
+
subclass) storage with no extra work, and returns an empty list for
|
|
236
|
+
values that are not externally stored (e.g. in-table ``<blob>``).
|
|
237
|
+
|
|
238
|
+
Override only if your codec stores external references in a non-standard
|
|
239
|
+
shape. Returning the paths here is what lets garbage collection see a
|
|
240
|
+
custom codec's files as *referenced* rather than orphaned (see #1469).
|
|
241
|
+
|
|
242
|
+
Parameters
|
|
243
|
+
----------
|
|
244
|
+
stored : any
|
|
245
|
+
The stored (encoded) value as read from the database column.
|
|
246
|
+
|
|
247
|
+
Returns
|
|
248
|
+
-------
|
|
249
|
+
list[tuple[str, str | None]]
|
|
250
|
+
``(path, store_name)`` for each external artifact referenced.
|
|
251
|
+
"""
|
|
252
|
+
value = stored
|
|
253
|
+
if isinstance(value, str):
|
|
254
|
+
try:
|
|
255
|
+
value = json.loads(value)
|
|
256
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
257
|
+
return []
|
|
258
|
+
if isinstance(value, dict) and "path" in value:
|
|
259
|
+
return [(value["path"], value.get("store"))]
|
|
260
|
+
return []
|
|
261
|
+
|
|
222
262
|
def __repr__(self) -> str:
|
|
223
263
|
return f"<{self.__class__.__name__}(name={self.name!r})>"
|
|
224
264
|
|
|
@@ -268,6 +268,20 @@ def assert_join_compatibility(
|
|
|
268
268
|
lineage2 = expr2.heading[name].lineage
|
|
269
269
|
# Semantic match requires both lineages to be non-None and equal
|
|
270
270
|
if lineage1 is None or lineage2 is None or lineage1 != lineage2:
|
|
271
|
+
if lineage1 is None or lineage2 is None:
|
|
272
|
+
# Missing lineage usually means stale ~lineage rows that survived
|
|
273
|
+
# an upgrade or a partial declare. Decoration in 2.3+ refreshes
|
|
274
|
+
# lineage automatically, so this typically indicates a schema
|
|
275
|
+
# that has not been re-decorated since the upgrade.
|
|
276
|
+
raise DataJointError(
|
|
277
|
+
f"Cannot join on attribute `{name}`: lineage missing on "
|
|
278
|
+
f"one side ({lineage1} vs {lineage2}). This usually "
|
|
279
|
+
f"indicates a stale `~lineage` entry from an older "
|
|
280
|
+
f"DataJoint version or an incomplete declare. Run "
|
|
281
|
+
f"`schema.rebuild_lineage()` to recompute lineage from "
|
|
282
|
+
f"current FK definitions. If the lineages are genuinely "
|
|
283
|
+
f"different, use `.proj()` to rename one of the attributes."
|
|
284
|
+
)
|
|
271
285
|
raise DataJointError(
|
|
272
286
|
f"Cannot join on attribute `{name}`: "
|
|
273
287
|
f"different lineages ({lineage1} vs {lineage2}). "
|
|
@@ -132,11 +132,13 @@ class Dependencies(nx.DiGraph):
|
|
|
132
132
|
self._conn = connection
|
|
133
133
|
self._node_alias_count = itertools.count()
|
|
134
134
|
self._loaded = False
|
|
135
|
+
self._loaded_schemas = set() # schema names currently represented in the graph
|
|
135
136
|
super().__init__(self)
|
|
136
137
|
|
|
137
138
|
def clear(self) -> None:
|
|
138
139
|
"""Clear the graph and reset loaded state."""
|
|
139
140
|
self._loaded = False
|
|
141
|
+
self._loaded_schemas = set()
|
|
140
142
|
self._node_alias_count = itertools.count() # reset alias IDs for consistency
|
|
141
143
|
super().clear()
|
|
142
144
|
|
|
@@ -165,6 +167,7 @@ class Dependencies(nx.DiGraph):
|
|
|
165
167
|
|
|
166
168
|
# Build schema list for IN clause
|
|
167
169
|
names = schema_names if schema_names is not None else set(self._conn.schemas)
|
|
170
|
+
self._loaded_schemas = set(names)
|
|
168
171
|
if not names:
|
|
169
172
|
self._loaded = True
|
|
170
173
|
return
|
|
@@ -257,6 +260,45 @@ class Dependencies(nx.DiGraph):
|
|
|
257
260
|
break
|
|
258
261
|
known_schemas |= new_schemas
|
|
259
262
|
|
|
263
|
+
# Skip the expensive rebuild when the graph already contains every needed
|
|
264
|
+
# schema, so repeated calls within one operation don't reload the whole
|
|
265
|
+
# dependency tree. See #1493.
|
|
266
|
+
if self._loaded and known_schemas <= self._loaded_schemas:
|
|
267
|
+
return
|
|
268
|
+
self.load(force=True, schema_names=known_schemas)
|
|
269
|
+
|
|
270
|
+
def load_all_upstream(self) -> None:
|
|
271
|
+
"""
|
|
272
|
+
Load dependencies including all upstream schemas referenced via FK chains.
|
|
273
|
+
|
|
274
|
+
Iteratively discovers schemas that the currently loaded schemas
|
|
275
|
+
reference, expanding the dependency graph until no new schemas
|
|
276
|
+
are found. This ensures that upstream restriction propagation
|
|
277
|
+
(``Diagram.trace()``) reaches all ancestor tables, including
|
|
278
|
+
those in schemas the user has not explicitly activated.
|
|
279
|
+
|
|
280
|
+
Called automatically by ``Diagram.trace()``. Symmetric to
|
|
281
|
+
:meth:`load_all_downstream`.
|
|
282
|
+
"""
|
|
283
|
+
adapter = self._conn.adapter
|
|
284
|
+
known_schemas = set(self._conn.schemas)
|
|
285
|
+
if not known_schemas:
|
|
286
|
+
self.load()
|
|
287
|
+
return
|
|
288
|
+
|
|
289
|
+
while True:
|
|
290
|
+
schemas_list = ", ".join(adapter.quote_string(s) for s in known_schemas)
|
|
291
|
+
result = self._conn.query(adapter.find_upstream_schemas_sql(schemas_list))
|
|
292
|
+
new_schemas = {row[0] for row in result} - known_schemas
|
|
293
|
+
if not new_schemas:
|
|
294
|
+
break
|
|
295
|
+
known_schemas |= new_schemas
|
|
296
|
+
|
|
297
|
+
# Skip the expensive rebuild when the graph already contains every needed
|
|
298
|
+
# schema, so repeated Diagram.trace() calls within one populate() don't
|
|
299
|
+
# reload the whole dependency tree per key. See #1493.
|
|
300
|
+
if self._loaded and known_schemas <= self._loaded_schemas:
|
|
301
|
+
return
|
|
260
302
|
self.load(force=True, schema_names=known_schemas)
|
|
261
303
|
|
|
262
304
|
def topo_sort(self) -> list[str]:
|