datajoint 2.3.0__tar.gz → 2.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/release_drafter.yaml +1 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/test.yaml +16 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/.pre-commit-config.yaml +3 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/PKG-INFO +8 -3
- {datajoint-2.3.0 → datajoint-2.3.1}/README.md +1 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/pyproject.toml +19 -2
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/__init__.py +1 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/autopopulate.py +38 -51
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/hash.py +1 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/npy.py +1 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/object.py +1 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/schema.py +3 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/codecs.py +40 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/dependencies.py +13 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/diagram.py +45 -12
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/expression.py +0 -6
- datajoint-2.3.1/src/datajoint/gc.py +397 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/hash_registry.py +26 -9
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/heading.py +47 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/settings.py +13 -11
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/staged_insert.py +1 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/storage.py +10 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/table.py +8 -24
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/version.py +1 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/conftest.py +4 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_autopopulate.py +14 -10
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_cascade_delete.py +100 -0
- datajoint-2.3.1/tests/integration/test_cascade_integrity.py +157 -0
- datajoint-2.3.1/tests/integration/test_gc.py +764 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_hash_storage.py +30 -17
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_object.py +6 -1
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_tls.py +15 -3
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_trace.py +58 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_codecs.py +38 -0
- datajoint-2.3.0/src/datajoint/gc.py +0 -710
- datajoint-2.3.0/src/datajoint/provenance.py +0 -206
- datajoint-2.3.0/tests/integration/test_gc.py +0 -467
- datajoint-2.3.0/tests/integration/test_strict_provenance.py +0 -322
- {datajoint-2.3.0 → datajoint-2.3.1}/.coveragerc +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.dockerignore +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.gitattributes +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/dependabot.yml +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/draft_release.yaml +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/lint.yaml +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/post_draft_release_published.yaml +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/.gitignore +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/CHANGELOG-archive.md +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/CONTRIBUTING.md +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/Dockerfile +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/LICENSE +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/RELEASE_MEMO.md +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/activate.sh +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/docker-compose.yaml +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/images/pipeline.png +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/images/pipeline.svg +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/pixi.lock +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/__init__.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/base.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/mysql.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/postgres.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/blob.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/__init__.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/attach.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/blob.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/filepath.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/cli.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/condition.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/connection.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/declare.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/deploy.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/errors.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/instance.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/jobs.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/lineage.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/logging.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/migrate.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/objectref.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/preview.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/schemas.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/spark.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/storage_adapter.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/types.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/user_tables.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/utils.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/__init__.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/__init__.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Course.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/CurrentTerm.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Department.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Enroll.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Grade.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Section.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Student.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/StudentMajor.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Term.csv +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_aggr_regressions.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_alter.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_attach.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_blob.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_blob_matlab.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_cascading_delete.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_cli.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_codec_chaining.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_codecs.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_connection.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_declare.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_dependencies.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_erd.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_fetch.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_fetch_same.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_foreign_keys.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_groupby.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_hidden_job_metadata.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_insert.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_jobs.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_json.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_multi_backend.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_nan.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_npy_codec.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_privileges.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_reconnection.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_relation.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_relation_u.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_relational_operand.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_schema.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_schema_keywords.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_semantic_matching.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_skip_duplicates.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_type_aliases.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_university.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_update1.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_utils.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_uuid.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_virtual_module.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_advanced.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_aggr_regress.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_alter.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_codecs.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_external.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_object.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_privileges.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_simple.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_type_aliases.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_university.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_uuid.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/test_package.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/__init__.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_adapters.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_condition.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_connection_warning.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_declare_hidden_attribute.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_deploy.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_fetch_compat.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_lazy_imports.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_pk_rules.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_settings.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_spark.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_storage_adapter.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_storage_urls.py +0 -0
- {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_thread_safe.py +0 -0
|
@@ -28,7 +28,7 @@ categories:
|
|
|
28
28
|
- title: '📝 Documentation'
|
|
29
29
|
label: 'documentation'
|
|
30
30
|
change-template: '- $TITLE(#$NUMBER)@$AUTHOR'
|
|
31
|
-
change-title-escapes: '\<*_
|
|
31
|
+
change-title-escapes: '\<*_&@#' # escape @ and # so decorator/issue refs in PR titles (e.g. `@schema`) don't render as spurious mentions/links in release notes.
|
|
32
32
|
template: |
|
|
33
33
|
$CHANGES
|
|
34
34
|
|
|
@@ -27,6 +27,18 @@ on:
|
|
|
27
27
|
jobs:
|
|
28
28
|
test:
|
|
29
29
|
runs-on: ubuntu-latest
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
matrix:
|
|
33
|
+
# Python: exercise both ends of the supported range (requires-python
|
|
34
|
+
# >=3.10,<3.15); version-pinned pixi environments live in pyproject.toml
|
|
35
|
+
# under [tool.pixi.environments].
|
|
36
|
+
environment: [test-py310, test-py314]
|
|
37
|
+
# MySQL: exercise both LTS lines we support / that AWS RDS runs. 8.0 uses
|
|
38
|
+
# the datajoint image (SSL configured); 8.4 uses the official image (TLS
|
|
39
|
+
# tests already skip outside external containers). See #1497.
|
|
40
|
+
mysql-image: ["datajoint/mysql:8.0", "mysql:8.4"]
|
|
41
|
+
name: test (${{ matrix.environment }}, mysql=${{ matrix.mysql-image }})
|
|
30
42
|
steps:
|
|
31
43
|
- uses: actions/checkout@v4
|
|
32
44
|
|
|
@@ -35,9 +47,12 @@ jobs:
|
|
|
35
47
|
with:
|
|
36
48
|
cache: true
|
|
37
49
|
locked: false
|
|
50
|
+
environments: ${{ matrix.environment }}
|
|
38
51
|
|
|
39
52
|
- name: Run tests
|
|
40
|
-
|
|
53
|
+
env:
|
|
54
|
+
DJ_TEST_MYSQL_IMAGE: ${{ matrix.mysql-image }}
|
|
55
|
+
run: pixi run -e ${{ matrix.environment }} test-cov
|
|
41
56
|
|
|
42
57
|
# Unit tests run without containers (faster feedback)
|
|
43
58
|
unit-tests:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datajoint
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.1
|
|
4
4
|
Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
|
|
5
5
|
Project-URL: Homepage, https://docs.datajoint.com/
|
|
6
6
|
Project-URL: Documentation, https://docs.datajoint.com/
|
|
@@ -207,11 +207,16 @@ Classifier: Intended Audience :: Healthcare Industry
|
|
|
207
207
|
Classifier: Intended Audience :: Science/Research
|
|
208
208
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
209
209
|
Classifier: Programming Language :: Python
|
|
210
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
211
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
212
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
213
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
214
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
210
215
|
Classifier: Topic :: Scientific/Engineering
|
|
211
216
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
212
217
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
213
218
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
214
|
-
Requires-Python: <3.
|
|
219
|
+
Requires-Python: <3.15,>=3.10
|
|
215
220
|
Requires-Dist: deepdiff
|
|
216
221
|
Requires-Dist: fsspec>=2023.1.0
|
|
217
222
|
Requires-Dist: networkx
|
|
@@ -267,7 +272,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
|
|
|
267
272
|
- **Tables represent workflow steps** — Each table is a step in your pipeline
|
|
268
273
|
- **Foreign keys encode dependencies** — Parent tables must be populated before child tables
|
|
269
274
|
- **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
|
|
270
|
-
- **Results are immutable** — Full
|
|
275
|
+
- **Results are immutable** — Full lineage and reproducibility
|
|
271
276
|
|
|
272
277
|
**Documentation:** https://docs.datajoint.com
|
|
273
278
|
|
|
@@ -5,7 +5,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
|
|
|
5
5
|
- **Tables represent workflow steps** — Each table is a step in your pipeline
|
|
6
6
|
- **Foreign keys encode dependencies** — Parent tables must be populated before child tables
|
|
7
7
|
- **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
|
|
8
|
-
- **Results are immutable** — Full
|
|
8
|
+
- **Results are immutable** — Full lineage and reproducibility
|
|
9
9
|
|
|
10
10
|
**Documentation:** https://docs.datajoint.com
|
|
11
11
|
|
|
@@ -19,7 +19,7 @@ dependencies = [
|
|
|
19
19
|
"packaging",
|
|
20
20
|
]
|
|
21
21
|
|
|
22
|
-
requires-python = ">=3.10,<3.
|
|
22
|
+
requires-python = ">=3.10,<3.15"
|
|
23
23
|
authors = [
|
|
24
24
|
{name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"},
|
|
25
25
|
{name = "Thinh Nguyen", email = "thinh@datajoint.com"},
|
|
@@ -57,6 +57,11 @@ keywords = [
|
|
|
57
57
|
# https://pypi.org/classifiers/
|
|
58
58
|
classifiers = [
|
|
59
59
|
"Programming Language :: Python",
|
|
60
|
+
"Programming Language :: Python :: 3.10",
|
|
61
|
+
"Programming Language :: Python :: 3.11",
|
|
62
|
+
"Programming Language :: Python :: 3.12",
|
|
63
|
+
"Programming Language :: Python :: 3.13",
|
|
64
|
+
"Programming Language :: Python :: 3.14",
|
|
60
65
|
"Development Status :: 5 - Production/Stable",
|
|
61
66
|
"Intended Audience :: Science/Research",
|
|
62
67
|
"Intended Audience :: Healthcare Industry",
|
|
@@ -250,10 +255,22 @@ datajoint = { path = ".", editable = true, extras = ["test"] }
|
|
|
250
255
|
[tool.pixi.feature.dev.pypi-dependencies]
|
|
251
256
|
datajoint = { path = ".", editable = true, extras = ["dev", "test"] }
|
|
252
257
|
|
|
258
|
+
# Pin the interpreter for the min/max supported versions so CI can exercise
|
|
259
|
+
# both ends of the requires-python range (see .github/workflows/test.yaml).
|
|
260
|
+
[tool.pixi.feature.py310.dependencies]
|
|
261
|
+
python = "3.10.*"
|
|
262
|
+
|
|
263
|
+
[tool.pixi.feature.py314.dependencies]
|
|
264
|
+
python = "3.14.*"
|
|
265
|
+
|
|
253
266
|
[tool.pixi.environments]
|
|
254
267
|
default = { solve-group = "default" }
|
|
255
268
|
dev = { features = ["dev"], solve-group = "default" }
|
|
256
269
|
test = { features = ["test"], solve-group = "default" }
|
|
270
|
+
# Version-pinned test environments (each solves independently — a solve-group
|
|
271
|
+
# cannot span different Python versions).
|
|
272
|
+
test-py310 = { features = ["test", "py310"] }
|
|
273
|
+
test-py314 = { features = ["test", "py314"] }
|
|
257
274
|
|
|
258
275
|
[tool.pixi.tasks]
|
|
259
276
|
# Tests use testcontainers - no manual setup required
|
|
@@ -265,7 +282,7 @@ services-down = "docker compose down"
|
|
|
265
282
|
test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] }
|
|
266
283
|
|
|
267
284
|
[tool.pixi.dependencies]
|
|
268
|
-
python = ">=3.10,<3.
|
|
285
|
+
python = ">=3.10,<3.15"
|
|
269
286
|
graphviz = ">=13.1.2,<14"
|
|
270
287
|
|
|
271
288
|
[tool.pixi.activation]
|
|
@@ -284,7 +284,7 @@ _lazy_modules = {
|
|
|
284
284
|
"diagram": (".diagram", None), # Return the module itself
|
|
285
285
|
# cli imports click
|
|
286
286
|
"cli": (".cli", "cli"),
|
|
287
|
-
# gc — exposed lazily so `dj.gc.
|
|
287
|
+
# gc — exposed lazily so `dj.gc.GarbageCollector(...)` works as documented in gc.py
|
|
288
288
|
# and in the user docs (how-to/garbage-collection.md).
|
|
289
289
|
"gc": (".gc", None), # Return the module itself
|
|
290
290
|
}
|
|
@@ -22,6 +22,14 @@ if TYPE_CHECKING:
|
|
|
22
22
|
|
|
23
23
|
logger = logging.getLogger(__name__.split(".")[0])
|
|
24
24
|
|
|
25
|
+
# Parallel populate hands the live table and its open connection to workers by
|
|
26
|
+
# process inheritance, so it requires the `fork` start method. Python 3.14
|
|
27
|
+
# changed the default from `fork` to `forkserver`, which pickles the payload
|
|
28
|
+
# instead — that neither carries a live DB connection nor the table's internal
|
|
29
|
+
# state, and it deadlocks. Pin to `fork` where available (all POSIX platforms);
|
|
30
|
+
# fall back to the platform default elsewhere.
|
|
31
|
+
_MP_START_METHOD = "fork" if "fork" in mp.get_all_start_methods() else None
|
|
32
|
+
|
|
25
33
|
|
|
26
34
|
# --- helper functions for multiprocessing --
|
|
27
35
|
|
|
@@ -30,7 +38,8 @@ def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[s
|
|
|
30
38
|
"""
|
|
31
39
|
Initialize a worker process for multiprocessing.
|
|
32
40
|
|
|
33
|
-
|
|
41
|
+
Stores the inherited table on the worker process and reconnects to database
|
|
42
|
+
(the parent closes its connection before forking; each worker reopens one).
|
|
34
43
|
|
|
35
44
|
Parameters
|
|
36
45
|
----------
|
|
@@ -89,7 +98,8 @@ class AutoPopulate:
|
|
|
89
98
|
_key_source = None
|
|
90
99
|
_allow_insert = False
|
|
91
100
|
_jobs = None
|
|
92
|
-
_upstream = None #
|
|
101
|
+
_upstream = None # memoized upstream trace for the current make(); built lazily on first use
|
|
102
|
+
_upstream_key = None # current make() key; not None iff executing inside make()
|
|
93
103
|
|
|
94
104
|
@property
|
|
95
105
|
def upstream(self):
|
|
@@ -102,10 +112,11 @@ class AutoPopulate:
|
|
|
102
112
|
(or ``FreeTable``, when indexed by a string) for any ancestor of
|
|
103
113
|
``self``.
|
|
104
114
|
|
|
105
|
-
Reading via ``self.upstream`` is the
|
|
106
|
-
|
|
107
|
-
and indexing a non-ancestor
|
|
108
|
-
|
|
115
|
+
Reading via ``self.upstream`` is the recommended pattern for the
|
|
116
|
+
make() reproducibility contract: the framework guarantees the
|
|
117
|
+
restriction matches the current ``key``, and indexing a non-ancestor
|
|
118
|
+
table raises ``DataJointError``. See
|
|
119
|
+
:doc:`reference/specs/autopopulate` for the contract.
|
|
109
120
|
|
|
110
121
|
Raises
|
|
111
122
|
------
|
|
@@ -122,12 +133,19 @@ class AutoPopulate:
|
|
|
122
133
|
traces = self.upstream[ExtractTraces].to_arrays("trace")
|
|
123
134
|
self.insert1({**key, "summary": compute(traces, date)})
|
|
124
135
|
"""
|
|
125
|
-
if self.
|
|
136
|
+
if self._upstream_key is None:
|
|
126
137
|
raise DataJointError(
|
|
127
138
|
"self.upstream is only available inside make(). "
|
|
128
139
|
"Outside make(), construct a trace explicitly: "
|
|
129
140
|
"dj.Diagram.trace(self & key)."
|
|
130
141
|
)
|
|
142
|
+
if self._upstream is None:
|
|
143
|
+
# Build the trace lazily on first access and memoize it for this
|
|
144
|
+
# make() call, so make() calls that never read self.upstream pay
|
|
145
|
+
# nothing. See #1493.
|
|
146
|
+
from .diagram import Diagram
|
|
147
|
+
|
|
148
|
+
self._upstream = Diagram.trace(self & self._upstream_key)
|
|
131
149
|
return self._upstream
|
|
132
150
|
|
|
133
151
|
class _JobsDescriptor:
|
|
@@ -476,7 +494,9 @@ class AutoPopulate:
|
|
|
476
494
|
if hasattr(self.connection._conn, "ctx"):
|
|
477
495
|
del self.connection._conn.ctx
|
|
478
496
|
with (
|
|
479
|
-
mp.Pool(
|
|
497
|
+
mp.get_context(_MP_START_METHOD).Pool(
|
|
498
|
+
processes, _initialize_populate, (self, None, populate_kwargs)
|
|
499
|
+
) as pool,
|
|
480
500
|
tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar,
|
|
481
501
|
):
|
|
482
502
|
for status in pool.imap(_call_populate1, keys, chunksize=1):
|
|
@@ -572,7 +592,9 @@ class AutoPopulate:
|
|
|
572
592
|
if hasattr(self.connection._conn, "ctx"):
|
|
573
593
|
del self.connection._conn.ctx # SSLContext is not pickleable
|
|
574
594
|
with (
|
|
575
|
-
mp.Pool(
|
|
595
|
+
mp.get_context(_MP_START_METHOD).Pool(
|
|
596
|
+
processes, _initialize_populate, (self, self.jobs, populate_kwargs)
|
|
597
|
+
) as pool,
|
|
576
598
|
tqdm(desc="Processes: ", total=nkeys)
|
|
577
599
|
if display_progress
|
|
578
600
|
else contextlib.nullcontext() as progress_bar,
|
|
@@ -651,40 +673,11 @@ class AutoPopulate:
|
|
|
651
673
|
logger.jobs(f"Making {key} -> {self.full_table_name}")
|
|
652
674
|
self.__class__._allow_insert = True
|
|
653
675
|
|
|
654
|
-
#
|
|
655
|
-
# `
|
|
656
|
-
#
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
self._upstream = Diagram.trace(self & dict(key))
|
|
660
|
-
|
|
661
|
-
# If strict_provenance is on, push the active-make context so the
|
|
662
|
-
# runtime gates in expression.cursor / table.insert can check this
|
|
663
|
-
# make()'s reads and writes. The context is popped in the finally
|
|
664
|
-
# block below.
|
|
665
|
-
strict_token = None
|
|
666
|
-
if self.connection._config.get("strict_provenance", False):
|
|
667
|
-
from .provenance import push_strict_make_context
|
|
668
|
-
from .user_tables import Part
|
|
669
|
-
|
|
670
|
-
allowed_tables = set(self._upstream._cascade_restrictions.keys()) | {self.full_table_name}
|
|
671
|
-
# Add Part tables of self to the allowed set. Use class __dict__
|
|
672
|
-
# (not dir/getattr) to avoid triggering descriptors like the
|
|
673
|
-
# _JobsDescriptor that lazy-declares the ~~ job table.
|
|
674
|
-
for cls in type(self).__mro__:
|
|
675
|
-
for attr_name, attr in cls.__dict__.items():
|
|
676
|
-
if attr_name.startswith("_"):
|
|
677
|
-
continue
|
|
678
|
-
if isinstance(attr, type) and issubclass(attr, Part):
|
|
679
|
-
# Instantiate to get full_table_name resolved against
|
|
680
|
-
# this schema. The Part class is already attached via
|
|
681
|
-
# @schema decoration of the master.
|
|
682
|
-
try:
|
|
683
|
-
part_ftn = attr().full_table_name
|
|
684
|
-
allowed_tables.add(part_ftn)
|
|
685
|
-
except Exception:
|
|
686
|
-
pass
|
|
687
|
-
strict_token = push_strict_make_context(self, frozenset(allowed_tables), dict(key))
|
|
676
|
+
# Record the key for a lazily-constructed upstream view. The trace is
|
|
677
|
+
# built on first `self.upstream` access (see the `upstream` property),
|
|
678
|
+
# so make() calls that never read upstream pay nothing. See #1493.
|
|
679
|
+
self._upstream_key = dict(key)
|
|
680
|
+
self._upstream = None
|
|
688
681
|
|
|
689
682
|
try:
|
|
690
683
|
if not is_generator:
|
|
@@ -747,11 +740,7 @@ class AutoPopulate:
|
|
|
747
740
|
# access raises a clear error rather than silently using a
|
|
748
741
|
# stale trace from the previous make() call.
|
|
749
742
|
self._upstream = None
|
|
750
|
-
|
|
751
|
-
if strict_token is not None:
|
|
752
|
-
from .provenance import pop_strict_make_context
|
|
753
|
-
|
|
754
|
-
pop_strict_make_context(strict_token)
|
|
743
|
+
self._upstream_key = None
|
|
755
744
|
|
|
756
745
|
def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]:
|
|
757
746
|
"""
|
|
@@ -866,8 +855,6 @@ class AutoPopulate:
|
|
|
866
855
|
|
|
867
856
|
pk_condition = make_condition(self, key, set())
|
|
868
857
|
self.connection.query(
|
|
869
|
-
f"UPDATE {self.full_table_name} SET "
|
|
870
|
-
"_job_start_time=%s, _job_duration=%s, _job_version=%s "
|
|
871
|
-
f"WHERE {pk_condition}",
|
|
858
|
+
f"UPDATE {self.full_table_name} SET _job_start_time=%s, _job_duration=%s, _job_version=%s WHERE {pk_condition}",
|
|
872
859
|
args=(start_time, duration, version[:64] if version else ""),
|
|
873
860
|
)
|
|
@@ -21,7 +21,7 @@ class HashCodec(Codec):
|
|
|
21
21
|
The database column stores JSON metadata: ``{hash, store, size}``.
|
|
22
22
|
Duplicate content is automatically deduplicated across all tables.
|
|
23
23
|
|
|
24
|
-
Deletion: Requires garbage collection via ``dj.gc.
|
|
24
|
+
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
|
|
25
25
|
|
|
26
26
|
External only - requires @ modifier.
|
|
27
27
|
|
|
@@ -272,7 +272,7 @@ class NpyCodec(SchemaCodec):
|
|
|
272
272
|
- Path: ``{schema}/{table}/{pk}/{attribute}.npy``
|
|
273
273
|
- Database column: JSON with ``{path, store, dtype, shape}``
|
|
274
274
|
|
|
275
|
-
Deletion: Requires garbage collection via ``dj.gc.
|
|
275
|
+
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
|
|
276
276
|
|
|
277
277
|
See Also
|
|
278
278
|
--------
|
|
@@ -53,7 +53,7 @@ class ObjectCodec(SchemaCodec):
|
|
|
53
53
|
|
|
54
54
|
{store_root}/{schema}/{table}/{pk}/{field}/
|
|
55
55
|
|
|
56
|
-
Deletion: Requires garbage collection via ``dj.gc.
|
|
56
|
+
Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
|
|
57
57
|
|
|
58
58
|
Comparison with hash-addressed::
|
|
59
59
|
|
|
@@ -114,7 +114,7 @@ class SchemaCodec(Codec, register=False):
|
|
|
114
114
|
Build schema-addressed storage path.
|
|
115
115
|
|
|
116
116
|
Constructs a path that mirrors the database schema structure:
|
|
117
|
-
``{schema}/{table}/{pk_values}/{field}{ext}``
|
|
117
|
+
``{schema_prefix}/{schema}/{table}/{pk_values}/{field}{ext}``
|
|
118
118
|
|
|
119
119
|
Supports partitioning if configured in the store.
|
|
120
120
|
|
|
@@ -150,6 +150,7 @@ class SchemaCodec(Codec, register=False):
|
|
|
150
150
|
spec = config.get_store_spec(store_name)
|
|
151
151
|
partition_pattern = spec.get("partition_pattern")
|
|
152
152
|
token_length = spec.get("token_length", 8)
|
|
153
|
+
schema_prefix = spec["schema_prefix"] # always present: settings applies the default
|
|
153
154
|
|
|
154
155
|
return build_object_path(
|
|
155
156
|
schema=schema,
|
|
@@ -159,6 +160,7 @@ class SchemaCodec(Codec, register=False):
|
|
|
159
160
|
ext=ext,
|
|
160
161
|
partition_pattern=partition_pattern,
|
|
161
162
|
token_length=token_length,
|
|
163
|
+
schema_prefix=schema_prefix,
|
|
162
164
|
)
|
|
163
165
|
|
|
164
166
|
def _get_backend(self, store_name: str | None = None, config=None):
|
|
@@ -38,6 +38,7 @@ class MyTable(dj.Manual):
|
|
|
38
38
|
|
|
39
39
|
from __future__ import annotations
|
|
40
40
|
|
|
41
|
+
import json
|
|
41
42
|
import logging
|
|
42
43
|
from abc import ABC, abstractmethod
|
|
43
44
|
from typing import Any
|
|
@@ -219,6 +220,45 @@ class Codec(ABC):
|
|
|
219
220
|
"""
|
|
220
221
|
pass
|
|
221
222
|
|
|
223
|
+
def referenced_paths(self, stored: Any) -> list[tuple[str, str | None]]:
|
|
224
|
+
"""
|
|
225
|
+
Return the external store paths this stored value references.
|
|
226
|
+
|
|
227
|
+
Garbage collection and delete-time cleanup call this on the *stored*
|
|
228
|
+
(encoded) representation of a column value — the JSON/dict already in
|
|
229
|
+
the database — so discovery stays metadata-only (no download or decode).
|
|
230
|
+
|
|
231
|
+
The default recognizes DataJoint's standard external-storage metadata:
|
|
232
|
+
a dict (or JSON string) carrying a ``path`` and optional ``store``. This
|
|
233
|
+
covers hash-addressed (``<hash@>``/``<blob@>``/``<attach@>``) and
|
|
234
|
+
schema-addressed (``<object@>``/``<npy@>`` and any :class:`SchemaCodec`
|
|
235
|
+
subclass) storage with no extra work, and returns an empty list for
|
|
236
|
+
values that are not externally stored (e.g. in-table ``<blob>``).
|
|
237
|
+
|
|
238
|
+
Override only if your codec stores external references in a non-standard
|
|
239
|
+
shape. Returning the paths here is what lets garbage collection see a
|
|
240
|
+
custom codec's files as *referenced* rather than orphaned (see #1469).
|
|
241
|
+
|
|
242
|
+
Parameters
|
|
243
|
+
----------
|
|
244
|
+
stored : any
|
|
245
|
+
The stored (encoded) value as read from the database column.
|
|
246
|
+
|
|
247
|
+
Returns
|
|
248
|
+
-------
|
|
249
|
+
list[tuple[str, str | None]]
|
|
250
|
+
``(path, store_name)`` for each external artifact referenced.
|
|
251
|
+
"""
|
|
252
|
+
value = stored
|
|
253
|
+
if isinstance(value, str):
|
|
254
|
+
try:
|
|
255
|
+
value = json.loads(value)
|
|
256
|
+
except (json.JSONDecodeError, TypeError, ValueError):
|
|
257
|
+
return []
|
|
258
|
+
if isinstance(value, dict) and "path" in value:
|
|
259
|
+
return [(value["path"], value.get("store"))]
|
|
260
|
+
return []
|
|
261
|
+
|
|
222
262
|
def __repr__(self) -> str:
|
|
223
263
|
return f"<{self.__class__.__name__}(name={self.name!r})>"
|
|
224
264
|
|
|
@@ -132,11 +132,13 @@ class Dependencies(nx.DiGraph):
|
|
|
132
132
|
self._conn = connection
|
|
133
133
|
self._node_alias_count = itertools.count()
|
|
134
134
|
self._loaded = False
|
|
135
|
+
self._loaded_schemas = set() # schema names currently represented in the graph
|
|
135
136
|
super().__init__(self)
|
|
136
137
|
|
|
137
138
|
def clear(self) -> None:
|
|
138
139
|
"""Clear the graph and reset loaded state."""
|
|
139
140
|
self._loaded = False
|
|
141
|
+
self._loaded_schemas = set()
|
|
140
142
|
self._node_alias_count = itertools.count() # reset alias IDs for consistency
|
|
141
143
|
super().clear()
|
|
142
144
|
|
|
@@ -165,6 +167,7 @@ class Dependencies(nx.DiGraph):
|
|
|
165
167
|
|
|
166
168
|
# Build schema list for IN clause
|
|
167
169
|
names = schema_names if schema_names is not None else set(self._conn.schemas)
|
|
170
|
+
self._loaded_schemas = set(names)
|
|
168
171
|
if not names:
|
|
169
172
|
self._loaded = True
|
|
170
173
|
return
|
|
@@ -257,6 +260,11 @@ class Dependencies(nx.DiGraph):
|
|
|
257
260
|
break
|
|
258
261
|
known_schemas |= new_schemas
|
|
259
262
|
|
|
263
|
+
# Skip the expensive rebuild when the graph already contains every needed
|
|
264
|
+
# schema, so repeated calls within one operation don't reload the whole
|
|
265
|
+
# dependency tree. See #1493.
|
|
266
|
+
if self._loaded and known_schemas <= self._loaded_schemas:
|
|
267
|
+
return
|
|
260
268
|
self.load(force=True, schema_names=known_schemas)
|
|
261
269
|
|
|
262
270
|
def load_all_upstream(self) -> None:
|
|
@@ -286,6 +294,11 @@ class Dependencies(nx.DiGraph):
|
|
|
286
294
|
break
|
|
287
295
|
known_schemas |= new_schemas
|
|
288
296
|
|
|
297
|
+
# Skip the expensive rebuild when the graph already contains every needed
|
|
298
|
+
# schema, so repeated Diagram.trace() calls within one populate() don't
|
|
299
|
+
# reload the whole dependency tree per key. See #1493.
|
|
300
|
+
if self._loaded and known_schemas <= self._loaded_schemas:
|
|
301
|
+
return
|
|
289
302
|
self.load(force=True, schema_names=known_schemas)
|
|
290
303
|
|
|
291
304
|
def topo_sort(self) -> list[str]:
|
|
@@ -340,6 +340,10 @@ class Diagram(nx.DiGraph): # noqa: C901
|
|
|
340
340
|
>>> # Inspect the cascade subgraph
|
|
341
341
|
>>> dj.Diagram.cascade(Session & 'subject_id=1')
|
|
342
342
|
"""
|
|
343
|
+
# Validate eagerly (mirrors Table.delete) — a typo like "casade" would
|
|
344
|
+
# otherwise silently degrade to enforce/ignore behavior.
|
|
345
|
+
if part_integrity not in ("enforce", "ignore", "cascade"):
|
|
346
|
+
raise ValueError(f"part_integrity must be 'enforce', 'ignore', or 'cascade', got {part_integrity!r}")
|
|
343
347
|
conn = table_expr.connection
|
|
344
348
|
conn.dependencies.load_all_downstream()
|
|
345
349
|
node = table_expr.full_table_name
|
|
@@ -692,13 +696,26 @@ class Diagram(nx.DiGraph): # noqa: C901
|
|
|
692
696
|
Walks the dependency graph in topological order, applying
|
|
693
697
|
propagation rules at each edge. Only processes descendants of
|
|
694
698
|
start_node to avoid duplicate propagation when chaining.
|
|
699
|
+
|
|
700
|
+
Invariant (``part_integrity="cascade"``): each restricted Part fires
|
|
701
|
+
its own upward walk to its Master exactly once — deduplicated per
|
|
702
|
+
``(part, master)`` pair (``visited_part_master_pairs``), not per
|
|
703
|
+
master — so a Master reached through several restricted Parts is
|
|
704
|
+
restricted from every one of them.
|
|
695
705
|
"""
|
|
696
706
|
|
|
697
707
|
sorted_nodes = topo_sort(self)
|
|
698
708
|
# Only propagate through descendants of start_node
|
|
699
709
|
allowed_nodes = {start_node} | set(nx.descendants(self, start_node))
|
|
700
710
|
propagated_edges = set()
|
|
701
|
-
|
|
711
|
+
# Dedup upward part→master walks by (part, master) PAIR, not by master
|
|
712
|
+
# alone: distinct Parts of the same master can be restricted via
|
|
713
|
+
# different FK paths from the seed, and each must contribute its own
|
|
714
|
+
# master rows. Keying by master alone under-restricted the master when
|
|
715
|
+
# a second Part was reached later (silent integrity violation in
|
|
716
|
+
# part_integrity="cascade" mode). The pair set is finite and global
|
|
717
|
+
# across passes, so the multi-pass loop still terminates.
|
|
718
|
+
visited_part_master_pairs = set()
|
|
702
719
|
|
|
703
720
|
restrictions = self._cascade_restrictions if mode == "cascade" else self._restrict_conditions
|
|
704
721
|
|
|
@@ -708,8 +725,8 @@ class Diagram(nx.DiGraph): # noqa: C901
|
|
|
708
725
|
# Trigger the upward propagation explicitly for the seed. See #1429.
|
|
709
726
|
if part_integrity == "cascade" and mode == "cascade":
|
|
710
727
|
seed_master = extract_master(start_node)
|
|
711
|
-
if seed_master and seed_master in self.nodes() and seed_master not in
|
|
712
|
-
|
|
728
|
+
if seed_master and seed_master in self.nodes() and (start_node, seed_master) not in visited_part_master_pairs:
|
|
729
|
+
visited_part_master_pairs.add((start_node, seed_master))
|
|
713
730
|
if self._propagate_part_to_master(start_node, seed_master, mode, restrictions):
|
|
714
731
|
allowed_nodes.add(seed_master)
|
|
715
732
|
allowed_nodes.update(nx.descendants(self, seed_master))
|
|
@@ -777,10 +794,16 @@ class Diagram(nx.DiGraph): # noqa: C901
|
|
|
777
794
|
# rules at each edge. Handles Part-of-Part chains and
|
|
778
795
|
# renamed FKs (via .proj()), unlike the prior implementation
|
|
779
796
|
# which assumed shared PK attribute names. See #1429.
|
|
797
|
+
# Deduped per (part, master) pair — every restricted Part
|
|
798
|
+
# of a master contributes its own master rows.
|
|
780
799
|
if part_integrity == "cascade" and mode == "cascade":
|
|
781
800
|
master_name = extract_master(target)
|
|
782
|
-
if
|
|
783
|
-
|
|
801
|
+
if (
|
|
802
|
+
master_name
|
|
803
|
+
and master_name in self.nodes()
|
|
804
|
+
and (target, master_name) not in visited_part_master_pairs
|
|
805
|
+
):
|
|
806
|
+
visited_part_master_pairs.add((target, master_name))
|
|
784
807
|
propagated = self._propagate_part_to_master(target, master_name, mode, restrictions)
|
|
785
808
|
if propagated:
|
|
786
809
|
allowed_nodes.add(master_name)
|
|
@@ -918,13 +941,23 @@ class Diagram(nx.DiGraph): # noqa: C901
|
|
|
918
941
|
so we read props from one and skip the alias node when walking.
|
|
919
942
|
|
|
920
943
|
After the walk, the master's restriction is **materialized** to a
|
|
921
|
-
literal value tuple via ``to_arrays()``.
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
944
|
+
literal value tuple via ``to_arrays()``. This is required for
|
|
945
|
+
correctness on **every backend**, not merely to avoid MySQL error
|
|
946
|
+
1093. ``Table.delete`` executes per-table deletes in reverse
|
|
947
|
+
topological order (leaves first), so the Part is deleted *before* the
|
|
948
|
+
Master. The master's restriction is derived from that Part, so if it
|
|
949
|
+
were left as a live subquery it would evaluate to empty once the Part
|
|
950
|
+
rows are gone — and the Master would silently escape deletion,
|
|
951
|
+
reintroducing the very compositional-integrity violation this walk
|
|
952
|
+
exists to prevent. Materializing the master's primary keys to a static
|
|
953
|
+
value set at plan time (before any rows are deleted) captures them
|
|
954
|
+
while the Part still exists, so the master delete becomes
|
|
955
|
+
``WHERE ... IN (literal-list)``.
|
|
956
|
+
|
|
957
|
+
(Secondary benefit: the literal set also sidesteps the self-referential
|
|
958
|
+
``WHERE ... IN (SELECT ... FROM <part>)`` subquery that MySQL rejects
|
|
959
|
+
with error 1093 — but note the delete-order reason above applies to
|
|
960
|
+
PostgreSQL too, so this must not be made a MySQL-only optimization.)
|
|
928
961
|
|
|
929
962
|
Limitations
|
|
930
963
|
-----------
|
|
@@ -1242,12 +1242,6 @@ class QueryExpression:
|
|
|
1242
1242
|
cursor
|
|
1243
1243
|
Database query cursor.
|
|
1244
1244
|
"""
|
|
1245
|
-
# Strict-provenance read gate. No-op outside make() or when the
|
|
1246
|
-
# config flag is off. See src/datajoint/provenance.py.
|
|
1247
|
-
from .provenance import assert_read_allowed
|
|
1248
|
-
|
|
1249
|
-
assert_read_allowed(self)
|
|
1250
|
-
|
|
1251
1245
|
sql = self.make_sql()
|
|
1252
1246
|
logger.debug(sql)
|
|
1253
1247
|
return self.connection.query(sql, as_dict=as_dict)
|