datajoint 2.3.0__tar.gz → 2.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. {datajoint-2.3.0 → datajoint-2.3.1}/.github/release_drafter.yaml +1 -1
  2. {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/test.yaml +16 -1
  3. {datajoint-2.3.0 → datajoint-2.3.1}/.pre-commit-config.yaml +3 -1
  4. {datajoint-2.3.0 → datajoint-2.3.1}/PKG-INFO +8 -3
  5. {datajoint-2.3.0 → datajoint-2.3.1}/README.md +1 -1
  6. {datajoint-2.3.0 → datajoint-2.3.1}/pyproject.toml +19 -2
  7. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/__init__.py +1 -1
  8. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/autopopulate.py +38 -51
  9. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/hash.py +1 -1
  10. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/npy.py +1 -1
  11. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/object.py +1 -1
  12. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/schema.py +3 -1
  13. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/codecs.py +40 -0
  14. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/dependencies.py +13 -0
  15. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/diagram.py +45 -12
  16. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/expression.py +0 -6
  17. datajoint-2.3.1/src/datajoint/gc.py +397 -0
  18. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/hash_registry.py +26 -9
  19. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/heading.py +47 -0
  20. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/settings.py +13 -11
  21. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/staged_insert.py +1 -0
  22. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/storage.py +10 -1
  23. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/table.py +8 -24
  24. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/version.py +1 -1
  25. {datajoint-2.3.0 → datajoint-2.3.1}/tests/conftest.py +4 -1
  26. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_autopopulate.py +14 -10
  27. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_cascade_delete.py +100 -0
  28. datajoint-2.3.1/tests/integration/test_cascade_integrity.py +157 -0
  29. datajoint-2.3.1/tests/integration/test_gc.py +764 -0
  30. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_hash_storage.py +30 -17
  31. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_object.py +6 -1
  32. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_tls.py +15 -3
  33. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_trace.py +58 -0
  34. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_codecs.py +38 -0
  35. datajoint-2.3.0/src/datajoint/gc.py +0 -710
  36. datajoint-2.3.0/src/datajoint/provenance.py +0 -206
  37. datajoint-2.3.0/tests/integration/test_gc.py +0 -467
  38. datajoint-2.3.0/tests/integration/test_strict_provenance.py +0 -322
  39. {datajoint-2.3.0 → datajoint-2.3.1}/.coveragerc +0 -0
  40. {datajoint-2.3.0 → datajoint-2.3.1}/.dockerignore +0 -0
  41. {datajoint-2.3.0 → datajoint-2.3.1}/.gitattributes +0 -0
  42. {datajoint-2.3.0 → datajoint-2.3.1}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
  43. {datajoint-2.3.0 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  44. {datajoint-2.3.0 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  45. {datajoint-2.3.0 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
  46. {datajoint-2.3.0 → datajoint-2.3.1}/.github/dependabot.yml +0 -0
  47. {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
  48. {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/draft_release.yaml +0 -0
  49. {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/lint.yaml +0 -0
  50. {datajoint-2.3.0 → datajoint-2.3.1}/.github/workflows/post_draft_release_published.yaml +0 -0
  51. {datajoint-2.3.0 → datajoint-2.3.1}/.gitignore +0 -0
  52. {datajoint-2.3.0 → datajoint-2.3.1}/CHANGELOG-archive.md +0 -0
  53. {datajoint-2.3.0 → datajoint-2.3.1}/CONTRIBUTING.md +0 -0
  54. {datajoint-2.3.0 → datajoint-2.3.1}/Dockerfile +0 -0
  55. {datajoint-2.3.0 → datajoint-2.3.1}/LICENSE +0 -0
  56. {datajoint-2.3.0 → datajoint-2.3.1}/RELEASE_MEMO.md +0 -0
  57. {datajoint-2.3.0 → datajoint-2.3.1}/activate.sh +0 -0
  58. {datajoint-2.3.0 → datajoint-2.3.1}/docker-compose.yaml +0 -0
  59. {datajoint-2.3.0 → datajoint-2.3.1}/images/pipeline.png +0 -0
  60. {datajoint-2.3.0 → datajoint-2.3.1}/images/pipeline.svg +0 -0
  61. {datajoint-2.3.0 → datajoint-2.3.1}/pixi.lock +0 -0
  62. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/__init__.py +0 -0
  63. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/base.py +0 -0
  64. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/mysql.py +0 -0
  65. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/adapters/postgres.py +0 -0
  66. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/blob.py +0 -0
  67. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/__init__.py +0 -0
  68. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/attach.py +0 -0
  69. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/blob.py +0 -0
  70. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/builtin_codecs/filepath.py +0 -0
  71. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/cli.py +0 -0
  72. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/condition.py +0 -0
  73. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/connection.py +0 -0
  74. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/declare.py +0 -0
  75. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/deploy.py +0 -0
  76. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/errors.py +0 -0
  77. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/instance.py +0 -0
  78. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/jobs.py +0 -0
  79. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/lineage.py +0 -0
  80. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/logging.py +0 -0
  81. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/migrate.py +0 -0
  82. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/objectref.py +0 -0
  83. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/preview.py +0 -0
  84. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/schemas.py +0 -0
  85. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/spark.py +0 -0
  86. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/storage_adapter.py +0 -0
  87. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/types.py +0 -0
  88. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/user_tables.py +0 -0
  89. {datajoint-2.3.0 → datajoint-2.3.1}/src/datajoint/utils.py +0 -0
  90. {datajoint-2.3.0 → datajoint-2.3.1}/tests/__init__.py +0 -0
  91. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/__init__.py +0 -0
  92. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Course.csv +0 -0
  93. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/CurrentTerm.csv +0 -0
  94. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Department.csv +0 -0
  95. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Enroll.csv +0 -0
  96. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Grade.csv +0 -0
  97. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Section.csv +0 -0
  98. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Student.csv +0 -0
  99. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/StudentMajor.csv +0 -0
  100. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/data/Term.csv +0 -0
  101. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_aggr_regressions.py +0 -0
  102. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_alter.py +0 -0
  103. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_attach.py +0 -0
  104. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_blob.py +0 -0
  105. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_blob_matlab.py +0 -0
  106. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_cascading_delete.py +0 -0
  107. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_cli.py +0 -0
  108. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_codec_chaining.py +0 -0
  109. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_codecs.py +0 -0
  110. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_connection.py +0 -0
  111. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_declare.py +0 -0
  112. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_dependencies.py +0 -0
  113. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_erd.py +0 -0
  114. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_fetch.py +0 -0
  115. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_fetch_same.py +0 -0
  116. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_foreign_keys.py +0 -0
  117. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_groupby.py +0 -0
  118. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_hidden_job_metadata.py +0 -0
  119. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_insert.py +0 -0
  120. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_jobs.py +0 -0
  121. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_json.py +0 -0
  122. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_multi_backend.py +0 -0
  123. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_nan.py +0 -0
  124. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_npy_codec.py +0 -0
  125. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_privileges.py +0 -0
  126. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_reconnection.py +0 -0
  127. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_relation.py +0 -0
  128. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_relation_u.py +0 -0
  129. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_relational_operand.py +0 -0
  130. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_schema.py +0 -0
  131. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_schema_keywords.py +0 -0
  132. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_semantic_matching.py +0 -0
  133. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_skip_duplicates.py +0 -0
  134. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_type_aliases.py +0 -0
  135. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_university.py +0 -0
  136. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_update1.py +0 -0
  137. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_utils.py +0 -0
  138. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_uuid.py +0 -0
  139. {datajoint-2.3.0 → datajoint-2.3.1}/tests/integration/test_virtual_module.py +0 -0
  140. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema.py +0 -0
  141. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_advanced.py +0 -0
  142. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_aggr_regress.py +0 -0
  143. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_alter.py +0 -0
  144. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_codecs.py +0 -0
  145. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_external.py +0 -0
  146. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_object.py +0 -0
  147. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_privileges.py +0 -0
  148. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_simple.py +0 -0
  149. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_type_aliases.py +0 -0
  150. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_university.py +0 -0
  151. {datajoint-2.3.0 → datajoint-2.3.1}/tests/schema_uuid.py +0 -0
  152. {datajoint-2.3.0 → datajoint-2.3.1}/tests/test_package.py +0 -0
  153. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/__init__.py +0 -0
  154. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_adapters.py +0 -0
  155. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_condition.py +0 -0
  156. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_connection_warning.py +0 -0
  157. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_declare_hidden_attribute.py +0 -0
  158. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_deploy.py +0 -0
  159. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_fetch_compat.py +0 -0
  160. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_lazy_imports.py +0 -0
  161. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_pk_rules.py +0 -0
  162. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_settings.py +0 -0
  163. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_spark.py +0 -0
  164. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_storage_adapter.py +0 -0
  165. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_storage_urls.py +0 -0
  166. {datajoint-2.3.0 → datajoint-2.3.1}/tests/unit/test_thread_safe.py +0 -0
@@ -28,7 +28,7 @@ categories:
28
28
  - title: '📝 Documentation'
29
29
  label: 'documentation'
30
30
  change-template: '- $TITLE(#$NUMBER)@$AUTHOR'
31
- change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
31
+ change-title-escapes: '\<*_&@#' # escape @ and # so decorator/issue refs in PR titles (e.g. `@schema`) don't render as spurious mentions/links in release notes.
32
32
  template: |
33
33
  $CHANGES
34
34
 
@@ -27,6 +27,18 @@ on:
27
27
  jobs:
28
28
  test:
29
29
  runs-on: ubuntu-latest
30
+ strategy:
31
+ fail-fast: false
32
+ matrix:
33
+ # Python: exercise both ends of the supported range (requires-python
34
+ # >=3.10,<3.15); version-pinned pixi environments live in pyproject.toml
35
+ # under [tool.pixi.environments].
36
+ environment: [test-py310, test-py314]
37
+ # MySQL: exercise both LTS lines we support / that AWS RDS runs. 8.0 uses
38
+ # the datajoint image (SSL configured); 8.4 uses the official image (TLS
39
+ # tests already skip outside external containers). See #1497.
40
+ mysql-image: ["datajoint/mysql:8.0", "mysql:8.4"]
41
+ name: test (${{ matrix.environment }}, mysql=${{ matrix.mysql-image }})
30
42
  steps:
31
43
  - uses: actions/checkout@v4
32
44
 
@@ -35,9 +47,12 @@ jobs:
35
47
  with:
36
48
  cache: true
37
49
  locked: false
50
+ environments: ${{ matrix.environment }}
38
51
 
39
52
  - name: Run tests
40
- run: pixi run -e test test-cov
53
+ env:
54
+ DJ_TEST_MYSQL_IMAGE: ${{ matrix.mysql-image }}
55
+ run: pixi run -e ${{ matrix.environment }} test-cov
41
56
 
42
57
  # Unit tests run without containers (faster feedback)
43
58
  unit-tests:
@@ -45,4 +45,6 @@ repos:
45
45
  - types-PyMySQL
46
46
  - types-tqdm
47
47
  - pandas-stubs
48
- - numpy
48
+ # numpy>=2.2 stubs use PEP 695 `type` statements, which mypy rejects
49
+ # when type-checking against python_version 3.10 (the project floor).
50
+ - numpy<2.2
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datajoint
3
- Version: 2.3.0
3
+ Version: 2.3.1
4
4
  Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
5
5
  Project-URL: Homepage, https://docs.datajoint.com/
6
6
  Project-URL: Documentation, https://docs.datajoint.com/
@@ -207,11 +207,16 @@ Classifier: Intended Audience :: Healthcare Industry
207
207
  Classifier: Intended Audience :: Science/Research
208
208
  Classifier: License :: OSI Approved :: Apache Software License
209
209
  Classifier: Programming Language :: Python
210
+ Classifier: Programming Language :: Python :: 3.10
211
+ Classifier: Programming Language :: Python :: 3.11
212
+ Classifier: Programming Language :: Python :: 3.12
213
+ Classifier: Programming Language :: Python :: 3.13
214
+ Classifier: Programming Language :: Python :: 3.14
210
215
  Classifier: Topic :: Scientific/Engineering
211
216
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
212
217
  Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
213
218
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
214
- Requires-Python: <3.14,>=3.10
219
+ Requires-Python: <3.15,>=3.10
215
220
  Requires-Dist: deepdiff
216
221
  Requires-Dist: fsspec>=2023.1.0
217
222
  Requires-Dist: networkx
@@ -267,7 +272,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
267
272
  - **Tables represent workflow steps** — Each table is a step in your pipeline
268
273
  - **Foreign keys encode dependencies** — Parent tables must be populated before child tables
269
274
  - **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
270
- - **Results are immutable** — Full provenance and reproducibility
275
+ - **Results are immutable** — Full lineage and reproducibility
271
276
 
272
277
  **Documentation:** https://docs.datajoint.com
273
278
 
@@ -5,7 +5,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
5
5
  - **Tables represent workflow steps** — Each table is a step in your pipeline
6
6
  - **Foreign keys encode dependencies** — Parent tables must be populated before child tables
7
7
  - **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
8
- - **Results are immutable** — Full provenance and reproducibility
8
+ - **Results are immutable** — Full lineage and reproducibility
9
9
 
10
10
  **Documentation:** https://docs.datajoint.com
11
11
 
@@ -19,7 +19,7 @@ dependencies = [
19
19
  "packaging",
20
20
  ]
21
21
 
22
- requires-python = ">=3.10,<3.14"
22
+ requires-python = ">=3.10,<3.15"
23
23
  authors = [
24
24
  {name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"},
25
25
  {name = "Thinh Nguyen", email = "thinh@datajoint.com"},
@@ -57,6 +57,11 @@ keywords = [
57
57
  # https://pypi.org/classifiers/
58
58
  classifiers = [
59
59
  "Programming Language :: Python",
60
+ "Programming Language :: Python :: 3.10",
61
+ "Programming Language :: Python :: 3.11",
62
+ "Programming Language :: Python :: 3.12",
63
+ "Programming Language :: Python :: 3.13",
64
+ "Programming Language :: Python :: 3.14",
60
65
  "Development Status :: 5 - Production/Stable",
61
66
  "Intended Audience :: Science/Research",
62
67
  "Intended Audience :: Healthcare Industry",
@@ -250,10 +255,22 @@ datajoint = { path = ".", editable = true, extras = ["test"] }
250
255
  [tool.pixi.feature.dev.pypi-dependencies]
251
256
  datajoint = { path = ".", editable = true, extras = ["dev", "test"] }
252
257
 
258
+ # Pin the interpreter for the min/max supported versions so CI can exercise
259
+ # both ends of the requires-python range (see .github/workflows/test.yaml).
260
+ [tool.pixi.feature.py310.dependencies]
261
+ python = "3.10.*"
262
+
263
+ [tool.pixi.feature.py314.dependencies]
264
+ python = "3.14.*"
265
+
253
266
  [tool.pixi.environments]
254
267
  default = { solve-group = "default" }
255
268
  dev = { features = ["dev"], solve-group = "default" }
256
269
  test = { features = ["test"], solve-group = "default" }
270
+ # Version-pinned test environments (each solves independently — a solve-group
271
+ # cannot span different Python versions).
272
+ test-py310 = { features = ["test", "py310"] }
273
+ test-py314 = { features = ["test", "py314"] }
257
274
 
258
275
  [tool.pixi.tasks]
259
276
  # Tests use testcontainers - no manual setup required
@@ -265,7 +282,7 @@ services-down = "docker compose down"
265
282
  test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] }
266
283
 
267
284
  [tool.pixi.dependencies]
268
- python = ">=3.10,<3.14"
285
+ python = ">=3.10,<3.15"
269
286
  graphviz = ">=13.1.2,<14"
270
287
 
271
288
  [tool.pixi.activation]
@@ -284,7 +284,7 @@ _lazy_modules = {
284
284
  "diagram": (".diagram", None), # Return the module itself
285
285
  # cli imports click
286
286
  "cli": (".cli", "cli"),
287
- # gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
287
+ # gc — exposed lazily so `dj.gc.GarbageCollector(...)` works as documented in gc.py
288
288
  # and in the user docs (how-to/garbage-collection.md).
289
289
  "gc": (".gc", None), # Return the module itself
290
290
  }
@@ -22,6 +22,14 @@ if TYPE_CHECKING:
22
22
 
23
23
  logger = logging.getLogger(__name__.split(".")[0])
24
24
 
25
+ # Parallel populate hands the live table and its open connection to workers by
26
+ # process inheritance, so it requires the `fork` start method. Python 3.14
27
+ # changed the default from `fork` to `forkserver`, which pickles the payload
28
+ # instead — that neither carries a live DB connection nor the table's internal
29
+ # state, and it deadlocks. Pin to `fork` where available (all POSIX platforms);
30
+ # fall back to the platform default elsewhere.
31
+ _MP_START_METHOD = "fork" if "fork" in mp.get_all_start_methods() else None
32
+
25
33
 
26
34
  # --- helper functions for multiprocessing --
27
35
 
@@ -30,7 +38,8 @@ def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[s
30
38
  """
31
39
  Initialize a worker process for multiprocessing.
32
40
 
33
- Saves the unpickled table to the current process and reconnects to database.
41
+ Stores the inherited table on the worker process and reconnects to database
42
+ (the parent closes its connection before forking; each worker reopens one).
34
43
 
35
44
  Parameters
36
45
  ----------
@@ -89,7 +98,8 @@ class AutoPopulate:
89
98
  _key_source = None
90
99
  _allow_insert = False
91
100
  _jobs = None
92
- _upstream = None # set per-make() by _populate_one; see `upstream` property below
101
+ _upstream = None # memoized upstream trace for the current make(); built lazily on first use
102
+ _upstream_key = None # current make() key; not None iff executing inside make()
93
103
 
94
104
  @property
95
105
  def upstream(self):
@@ -102,10 +112,11 @@ class AutoPopulate:
102
112
  (or ``FreeTable``, when indexed by a string) for any ancestor of
103
113
  ``self``.
104
114
 
105
- Reading via ``self.upstream`` is the provenance-safe pattern: the
106
- framework guarantees the restriction matches the current ``key``,
107
- and indexing a non-ancestor table raises ``DataJointError``. See
108
- :doc:`reference/specs/provenance` for the contract.
115
+ Reading via ``self.upstream`` is the recommended pattern for the
116
+ make() reproducibility contract: the framework guarantees the
117
+ restriction matches the current ``key``, and indexing a non-ancestor
118
+ table raises ``DataJointError``. See
119
+ :doc:`reference/specs/autopopulate` for the contract.
109
120
 
110
121
  Raises
111
122
  ------
@@ -122,12 +133,19 @@ class AutoPopulate:
122
133
  traces = self.upstream[ExtractTraces].to_arrays("trace")
123
134
  self.insert1({**key, "summary": compute(traces, date)})
124
135
  """
125
- if self._upstream is None:
136
+ if self._upstream_key is None:
126
137
  raise DataJointError(
127
138
  "self.upstream is only available inside make(). "
128
139
  "Outside make(), construct a trace explicitly: "
129
140
  "dj.Diagram.trace(self & key)."
130
141
  )
142
+ if self._upstream is None:
143
+ # Build the trace lazily on first access and memoize it for this
144
+ # make() call, so make() calls that never read self.upstream pay
145
+ # nothing. See #1493.
146
+ from .diagram import Diagram
147
+
148
+ self._upstream = Diagram.trace(self & self._upstream_key)
131
149
  return self._upstream
132
150
 
133
151
  class _JobsDescriptor:
@@ -476,7 +494,9 @@ class AutoPopulate:
476
494
  if hasattr(self.connection._conn, "ctx"):
477
495
  del self.connection._conn.ctx
478
496
  with (
479
- mp.Pool(processes, _initialize_populate, (self, None, populate_kwargs)) as pool,
497
+ mp.get_context(_MP_START_METHOD).Pool(
498
+ processes, _initialize_populate, (self, None, populate_kwargs)
499
+ ) as pool,
480
500
  tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar,
481
501
  ):
482
502
  for status in pool.imap(_call_populate1, keys, chunksize=1):
@@ -572,7 +592,9 @@ class AutoPopulate:
572
592
  if hasattr(self.connection._conn, "ctx"):
573
593
  del self.connection._conn.ctx # SSLContext is not pickleable
574
594
  with (
575
- mp.Pool(processes, _initialize_populate, (self, self.jobs, populate_kwargs)) as pool,
595
+ mp.get_context(_MP_START_METHOD).Pool(
596
+ processes, _initialize_populate, (self, self.jobs, populate_kwargs)
597
+ ) as pool,
576
598
  tqdm(desc="Processes: ", total=nkeys)
577
599
  if display_progress
578
600
  else contextlib.nullcontext() as progress_bar,
@@ -651,40 +673,11 @@ class AutoPopulate:
651
673
  logger.jobs(f"Making {key} -> {self.full_table_name}")
652
674
  self.__class__._allow_insert = True
653
675
 
654
- # Pre-construct the upstream view for this make() call. Lazy — only
655
- # `dj.Diagram.trace(self & key)` runs here (graph copy); the
656
- # expensive SQL fetch fires when the user accesses self.upstream[T].
657
- from .diagram import Diagram
658
-
659
- self._upstream = Diagram.trace(self & dict(key))
660
-
661
- # If strict_provenance is on, push the active-make context so the
662
- # runtime gates in expression.cursor / table.insert can check this
663
- # make()'s reads and writes. The context is popped in the finally
664
- # block below.
665
- strict_token = None
666
- if self.connection._config.get("strict_provenance", False):
667
- from .provenance import push_strict_make_context
668
- from .user_tables import Part
669
-
670
- allowed_tables = set(self._upstream._cascade_restrictions.keys()) | {self.full_table_name}
671
- # Add Part tables of self to the allowed set. Use class __dict__
672
- # (not dir/getattr) to avoid triggering descriptors like the
673
- # _JobsDescriptor that lazy-declares the ~~ job table.
674
- for cls in type(self).__mro__:
675
- for attr_name, attr in cls.__dict__.items():
676
- if attr_name.startswith("_"):
677
- continue
678
- if isinstance(attr, type) and issubclass(attr, Part):
679
- # Instantiate to get full_table_name resolved against
680
- # this schema. The Part class is already attached via
681
- # @schema decoration of the master.
682
- try:
683
- part_ftn = attr().full_table_name
684
- allowed_tables.add(part_ftn)
685
- except Exception:
686
- pass
687
- strict_token = push_strict_make_context(self, frozenset(allowed_tables), dict(key))
676
+ # Record the key for a lazily-constructed upstream view. The trace is
677
+ # built on first `self.upstream` access (see the `upstream` property),
678
+ # so make() calls that never read upstream pay nothing. See #1493.
679
+ self._upstream_key = dict(key)
680
+ self._upstream = None
688
681
 
689
682
  try:
690
683
  if not is_generator:
@@ -747,11 +740,7 @@ class AutoPopulate:
747
740
  # access raises a clear error rather than silently using a
748
741
  # stale trace from the previous make() call.
749
742
  self._upstream = None
750
- # Pop the strict-make context, if any.
751
- if strict_token is not None:
752
- from .provenance import pop_strict_make_context
753
-
754
- pop_strict_make_context(strict_token)
743
+ self._upstream_key = None
755
744
 
756
745
  def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]:
757
746
  """
@@ -866,8 +855,6 @@ class AutoPopulate:
866
855
 
867
856
  pk_condition = make_condition(self, key, set())
868
857
  self.connection.query(
869
- f"UPDATE {self.full_table_name} SET "
870
- "_job_start_time=%s, _job_duration=%s, _job_version=%s "
871
- f"WHERE {pk_condition}",
858
+ f"UPDATE {self.full_table_name} SET _job_start_time=%s, _job_duration=%s, _job_version=%s WHERE {pk_condition}",
872
859
  args=(start_time, duration, version[:64] if version else ""),
873
860
  )
@@ -21,7 +21,7 @@ class HashCodec(Codec):
21
21
  The database column stores JSON metadata: ``{hash, store, size}``.
22
22
  Duplicate content is automatically deduplicated across all tables.
23
23
 
24
- Deletion: Requires garbage collection via ``dj.gc.collect()``.
24
+ Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
25
25
 
26
26
  External only - requires @ modifier.
27
27
 
@@ -272,7 +272,7 @@ class NpyCodec(SchemaCodec):
272
272
  - Path: ``{schema}/{table}/{pk}/{attribute}.npy``
273
273
  - Database column: JSON with ``{path, store, dtype, shape}``
274
274
 
275
- Deletion: Requires garbage collection via ``dj.gc.collect()``.
275
+ Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
276
276
 
277
277
  See Also
278
278
  --------
@@ -53,7 +53,7 @@ class ObjectCodec(SchemaCodec):
53
53
 
54
54
  {store_root}/{schema}/{table}/{pk}/{field}/
55
55
 
56
- Deletion: Requires garbage collection via ``dj.gc.collect()``.
56
+ Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
57
57
 
58
58
  Comparison with hash-addressed::
59
59
 
@@ -114,7 +114,7 @@ class SchemaCodec(Codec, register=False):
114
114
  Build schema-addressed storage path.
115
115
 
116
116
  Constructs a path that mirrors the database schema structure:
117
- ``{schema}/{table}/{pk_values}/{field}{ext}``
117
+ ``{schema_prefix}/{schema}/{table}/{pk_values}/{field}{ext}``
118
118
 
119
119
  Supports partitioning if configured in the store.
120
120
 
@@ -150,6 +150,7 @@ class SchemaCodec(Codec, register=False):
150
150
  spec = config.get_store_spec(store_name)
151
151
  partition_pattern = spec.get("partition_pattern")
152
152
  token_length = spec.get("token_length", 8)
153
+ schema_prefix = spec["schema_prefix"] # always present: settings applies the default
153
154
 
154
155
  return build_object_path(
155
156
  schema=schema,
@@ -159,6 +160,7 @@ class SchemaCodec(Codec, register=False):
159
160
  ext=ext,
160
161
  partition_pattern=partition_pattern,
161
162
  token_length=token_length,
163
+ schema_prefix=schema_prefix,
162
164
  )
163
165
 
164
166
  def _get_backend(self, store_name: str | None = None, config=None):
@@ -38,6 +38,7 @@ class MyTable(dj.Manual):
38
38
 
39
39
  from __future__ import annotations
40
40
 
41
+ import json
41
42
  import logging
42
43
  from abc import ABC, abstractmethod
43
44
  from typing import Any
@@ -219,6 +220,45 @@ class Codec(ABC):
219
220
  """
220
221
  pass
221
222
 
223
+ def referenced_paths(self, stored: Any) -> list[tuple[str, str | None]]:
224
+ """
225
+ Return the external store paths this stored value references.
226
+
227
+ Garbage collection and delete-time cleanup call this on the *stored*
228
+ (encoded) representation of a column value — the JSON/dict already in
229
+ the database — so discovery stays metadata-only (no download or decode).
230
+
231
+ The default recognizes DataJoint's standard external-storage metadata:
232
+ a dict (or JSON string) carrying a ``path`` and optional ``store``. This
233
+ covers hash-addressed (``<hash@>``/``<blob@>``/``<attach@>``) and
234
+ schema-addressed (``<object@>``/``<npy@>`` and any :class:`SchemaCodec`
235
+ subclass) storage with no extra work, and returns an empty list for
236
+ values that are not externally stored (e.g. in-table ``<blob>``).
237
+
238
+ Override only if your codec stores external references in a non-standard
239
+ shape. Returning the paths here is what lets garbage collection see a
240
+ custom codec's files as *referenced* rather than orphaned (see #1469).
241
+
242
+ Parameters
243
+ ----------
244
+ stored : any
245
+ The stored (encoded) value as read from the database column.
246
+
247
+ Returns
248
+ -------
249
+ list[tuple[str, str | None]]
250
+ ``(path, store_name)`` for each external artifact referenced.
251
+ """
252
+ value = stored
253
+ if isinstance(value, str):
254
+ try:
255
+ value = json.loads(value)
256
+ except (json.JSONDecodeError, TypeError, ValueError):
257
+ return []
258
+ if isinstance(value, dict) and "path" in value:
259
+ return [(value["path"], value.get("store"))]
260
+ return []
261
+
222
262
  def __repr__(self) -> str:
223
263
  return f"<{self.__class__.__name__}(name={self.name!r})>"
224
264
 
@@ -132,11 +132,13 @@ class Dependencies(nx.DiGraph):
132
132
  self._conn = connection
133
133
  self._node_alias_count = itertools.count()
134
134
  self._loaded = False
135
+ self._loaded_schemas = set() # schema names currently represented in the graph
135
136
  super().__init__(self)
136
137
 
137
138
  def clear(self) -> None:
138
139
  """Clear the graph and reset loaded state."""
139
140
  self._loaded = False
141
+ self._loaded_schemas = set()
140
142
  self._node_alias_count = itertools.count() # reset alias IDs for consistency
141
143
  super().clear()
142
144
 
@@ -165,6 +167,7 @@ class Dependencies(nx.DiGraph):
165
167
 
166
168
  # Build schema list for IN clause
167
169
  names = schema_names if schema_names is not None else set(self._conn.schemas)
170
+ self._loaded_schemas = set(names)
168
171
  if not names:
169
172
  self._loaded = True
170
173
  return
@@ -257,6 +260,11 @@ class Dependencies(nx.DiGraph):
257
260
  break
258
261
  known_schemas |= new_schemas
259
262
 
263
+ # Skip the expensive rebuild when the graph already contains every needed
264
+ # schema, so repeated calls within one operation don't reload the whole
265
+ # dependency tree. See #1493.
266
+ if self._loaded and known_schemas <= self._loaded_schemas:
267
+ return
260
268
  self.load(force=True, schema_names=known_schemas)
261
269
 
262
270
  def load_all_upstream(self) -> None:
@@ -286,6 +294,11 @@ class Dependencies(nx.DiGraph):
286
294
  break
287
295
  known_schemas |= new_schemas
288
296
 
297
+ # Skip the expensive rebuild when the graph already contains every needed
298
+ # schema, so repeated Diagram.trace() calls within one populate() don't
299
+ # reload the whole dependency tree per key. See #1493.
300
+ if self._loaded and known_schemas <= self._loaded_schemas:
301
+ return
289
302
  self.load(force=True, schema_names=known_schemas)
290
303
 
291
304
  def topo_sort(self) -> list[str]:
@@ -340,6 +340,10 @@ class Diagram(nx.DiGraph): # noqa: C901
340
340
  >>> # Inspect the cascade subgraph
341
341
  >>> dj.Diagram.cascade(Session & 'subject_id=1')
342
342
  """
343
+ # Validate eagerly (mirrors Table.delete) — a typo like "casade" would
344
+ # otherwise silently degrade to enforce/ignore behavior.
345
+ if part_integrity not in ("enforce", "ignore", "cascade"):
346
+ raise ValueError(f"part_integrity must be 'enforce', 'ignore', or 'cascade', got {part_integrity!r}")
343
347
  conn = table_expr.connection
344
348
  conn.dependencies.load_all_downstream()
345
349
  node = table_expr.full_table_name
@@ -692,13 +696,26 @@ class Diagram(nx.DiGraph): # noqa: C901
692
696
  Walks the dependency graph in topological order, applying
693
697
  propagation rules at each edge. Only processes descendants of
694
698
  start_node to avoid duplicate propagation when chaining.
699
+
700
+ Invariant (``part_integrity="cascade"``): each restricted Part fires
701
+ its own upward walk to its Master exactly once — deduplicated per
702
+ ``(part, master)`` pair (``visited_part_master_pairs``), not per
703
+ master — so a Master reached through several restricted Parts is
704
+ restricted from every one of them.
695
705
  """
696
706
 
697
707
  sorted_nodes = topo_sort(self)
698
708
  # Only propagate through descendants of start_node
699
709
  allowed_nodes = {start_node} | set(nx.descendants(self, start_node))
700
710
  propagated_edges = set()
701
- visited_masters = set()
711
+ # Dedup upward part→master walks by (part, master) PAIR, not by master
712
+ # alone: distinct Parts of the same master can be restricted via
713
+ # different FK paths from the seed, and each must contribute its own
714
+ # master rows. Keying by master alone under-restricted the master when
715
+ # a second Part was reached later (silent integrity violation in
716
+ # part_integrity="cascade" mode). The pair set is finite and global
717
+ # across passes, so the multi-pass loop still terminates.
718
+ visited_part_master_pairs = set()
702
719
 
703
720
  restrictions = self._cascade_restrictions if mode == "cascade" else self._restrict_conditions
704
721
 
@@ -708,8 +725,8 @@ class Diagram(nx.DiGraph): # noqa: C901
708
725
  # Trigger the upward propagation explicitly for the seed. See #1429.
709
726
  if part_integrity == "cascade" and mode == "cascade":
710
727
  seed_master = extract_master(start_node)
711
- if seed_master and seed_master in self.nodes() and seed_master not in visited_masters:
712
- visited_masters.add(seed_master)
728
+ if seed_master and seed_master in self.nodes() and (start_node, seed_master) not in visited_part_master_pairs:
729
+ visited_part_master_pairs.add((start_node, seed_master))
713
730
  if self._propagate_part_to_master(start_node, seed_master, mode, restrictions):
714
731
  allowed_nodes.add(seed_master)
715
732
  allowed_nodes.update(nx.descendants(self, seed_master))
@@ -777,10 +794,16 @@ class Diagram(nx.DiGraph): # noqa: C901
777
794
  # rules at each edge. Handles Part-of-Part chains and
778
795
  # renamed FKs (via .proj()), unlike the prior implementation
779
796
  # which assumed shared PK attribute names. See #1429.
797
+ # Deduped per (part, master) pair — every restricted Part
798
+ # of a master contributes its own master rows.
780
799
  if part_integrity == "cascade" and mode == "cascade":
781
800
  master_name = extract_master(target)
782
- if master_name and master_name in self.nodes() and master_name not in visited_masters:
783
- visited_masters.add(master_name)
801
+ if (
802
+ master_name
803
+ and master_name in self.nodes()
804
+ and (target, master_name) not in visited_part_master_pairs
805
+ ):
806
+ visited_part_master_pairs.add((target, master_name))
784
807
  propagated = self._propagate_part_to_master(target, master_name, mode, restrictions)
785
808
  if propagated:
786
809
  allowed_nodes.add(master_name)
@@ -918,13 +941,23 @@ class Diagram(nx.DiGraph): # noqa: C901
918
941
  so we read props from one and skip the alias node when walking.
919
942
 
920
943
  After the walk, the master's restriction is **materialized** to a
921
- literal value tuple via ``to_arrays()``. Without materialization, a
922
- subsequent forward cascade from the master back down to its parts
923
- would produce a self-referential subquery (MySQL error 1093, since
924
- the master's restriction depends on the same Part being deleted).
925
- Materializing converts the restriction into a static value set, so
926
- the forward cascade generates ``WHERE ... IN (literal-list)`` rather
927
- than ``WHERE ... IN (SELECT ... FROM <part>)``.
944
+ literal value tuple via ``to_arrays()``. This is required for
945
+ correctness on **every backend**, not merely to avoid MySQL error
946
+ 1093. ``Table.delete`` executes per-table deletes in reverse
947
+ topological order (leaves first), so the Part is deleted *before* the
948
+ Master. The master's restriction is derived from that Part, so if it
949
+ were left as a live subquery it would evaluate to empty once the Part
950
+ rows are gone — and the Master would silently escape deletion,
951
+ reintroducing the very compositional-integrity violation this walk
952
+ exists to prevent. Materializing the master's primary keys to a static
953
+ value set at plan time (before any rows are deleted) captures them
954
+ while the Part still exists, so the master delete becomes
955
+ ``WHERE ... IN (literal-list)``.
956
+
957
+ (Secondary benefit: the literal set also sidesteps the self-referential
958
+ ``WHERE ... IN (SELECT ... FROM <part>)`` subquery that MySQL rejects
959
+ with error 1093 — but note the delete-order reason above applies to
960
+ PostgreSQL too, so this must not be made a MySQL-only optimization.)
928
961
 
929
962
  Limitations
930
963
  -----------
@@ -1242,12 +1242,6 @@ class QueryExpression:
1242
1242
  cursor
1243
1243
  Database query cursor.
1244
1244
  """
1245
- # Strict-provenance read gate. No-op outside make() or when the
1246
- # config flag is off. See src/datajoint/provenance.py.
1247
- from .provenance import assert_read_allowed
1248
-
1249
- assert_read_allowed(self)
1250
-
1251
1245
  sql = self.make_sql()
1252
1246
  logger.debug(sql)
1253
1247
  return self.connection.query(sql, as_dict=as_dict)