datajoint 2.2.4__tar.gz → 2.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. {datajoint-2.2.4 → datajoint-2.3.1}/.github/release_drafter.yaml +1 -1
  2. {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/test.yaml +16 -1
  3. {datajoint-2.2.4 → datajoint-2.3.1}/.pre-commit-config.yaml +3 -1
  4. {datajoint-2.2.4 → datajoint-2.3.1}/PKG-INFO +8 -3
  5. {datajoint-2.2.4 → datajoint-2.3.1}/README.md +1 -1
  6. {datajoint-2.2.4 → datajoint-2.3.1}/pyproject.toml +19 -2
  7. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/__init__.py +6 -1
  8. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/base.py +23 -0
  9. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/mysql.py +10 -0
  10. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/postgres.py +39 -0
  11. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/autopopulate.py +77 -6
  12. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/hash.py +1 -1
  13. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/npy.py +1 -1
  14. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/object.py +1 -1
  15. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/schema.py +3 -1
  16. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/codecs.py +40 -0
  17. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/condition.py +14 -0
  18. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/dependencies.py +42 -0
  19. datajoint-2.3.1/src/datajoint/deploy.py +185 -0
  20. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/diagram.py +482 -19
  21. datajoint-2.3.1/src/datajoint/gc.py +397 -0
  22. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/hash_registry.py +26 -9
  23. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/heading.py +47 -0
  24. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/schemas.py +17 -0
  25. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/settings.py +13 -0
  26. datajoint-2.3.1/src/datajoint/spark.py +92 -0
  27. datajoint-2.3.1/src/datajoint/staged_insert.py +275 -0
  28. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/storage.py +10 -1
  29. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/table.py +45 -1
  30. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/version.py +1 -1
  31. {datajoint-2.2.4 → datajoint-2.3.1}/tests/conftest.py +4 -1
  32. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_autopopulate.py +243 -0
  33. datajoint-2.3.1/tests/integration/test_cascade_delete.py +581 -0
  34. datajoint-2.3.1/tests/integration/test_cascade_integrity.py +157 -0
  35. datajoint-2.3.1/tests/integration/test_gc.py +764 -0
  36. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_hash_storage.py +30 -17
  37. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_object.py +104 -1
  38. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_semantic_matching.py +150 -0
  39. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_tls.py +15 -3
  40. datajoint-2.3.1/tests/integration/test_trace.py +449 -0
  41. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_adapters.py +17 -0
  42. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_codecs.py +38 -0
  43. datajoint-2.3.1/tests/unit/test_deploy.py +172 -0
  44. datajoint-2.3.1/tests/unit/test_spark.py +105 -0
  45. datajoint-2.2.4/src/datajoint/gc.py +0 -710
  46. datajoint-2.2.4/src/datajoint/staged_insert.py +0 -310
  47. datajoint-2.2.4/tests/integration/test_cascade_delete.py +0 -294
  48. datajoint-2.2.4/tests/integration/test_gc.py +0 -467
  49. {datajoint-2.2.4 → datajoint-2.3.1}/.coveragerc +0 -0
  50. {datajoint-2.2.4 → datajoint-2.3.1}/.dockerignore +0 -0
  51. {datajoint-2.2.4 → datajoint-2.3.1}/.gitattributes +0 -0
  52. {datajoint-2.2.4 → datajoint-2.3.1}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
  53. {datajoint-2.2.4 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  54. {datajoint-2.2.4 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  55. {datajoint-2.2.4 → datajoint-2.3.1}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
  56. {datajoint-2.2.4 → datajoint-2.3.1}/.github/dependabot.yml +0 -0
  57. {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
  58. {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/draft_release.yaml +0 -0
  59. {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/lint.yaml +0 -0
  60. {datajoint-2.2.4 → datajoint-2.3.1}/.github/workflows/post_draft_release_published.yaml +0 -0
  61. {datajoint-2.2.4 → datajoint-2.3.1}/.gitignore +0 -0
  62. {datajoint-2.2.4 → datajoint-2.3.1}/CHANGELOG-archive.md +0 -0
  63. {datajoint-2.2.4 → datajoint-2.3.1}/CONTRIBUTING.md +0 -0
  64. {datajoint-2.2.4 → datajoint-2.3.1}/Dockerfile +0 -0
  65. {datajoint-2.2.4 → datajoint-2.3.1}/LICENSE +0 -0
  66. {datajoint-2.2.4 → datajoint-2.3.1}/RELEASE_MEMO.md +0 -0
  67. {datajoint-2.2.4 → datajoint-2.3.1}/activate.sh +0 -0
  68. {datajoint-2.2.4 → datajoint-2.3.1}/docker-compose.yaml +0 -0
  69. {datajoint-2.2.4 → datajoint-2.3.1}/images/pipeline.png +0 -0
  70. {datajoint-2.2.4 → datajoint-2.3.1}/images/pipeline.svg +0 -0
  71. {datajoint-2.2.4 → datajoint-2.3.1}/pixi.lock +0 -0
  72. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/adapters/__init__.py +0 -0
  73. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/blob.py +0 -0
  74. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/__init__.py +0 -0
  75. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/attach.py +0 -0
  76. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/blob.py +0 -0
  77. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/builtin_codecs/filepath.py +0 -0
  78. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/cli.py +0 -0
  79. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/connection.py +0 -0
  80. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/declare.py +0 -0
  81. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/errors.py +0 -0
  82. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/expression.py +0 -0
  83. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/instance.py +0 -0
  84. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/jobs.py +0 -0
  85. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/lineage.py +0 -0
  86. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/logging.py +0 -0
  87. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/migrate.py +0 -0
  88. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/objectref.py +0 -0
  89. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/preview.py +0 -0
  90. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/storage_adapter.py +0 -0
  91. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/types.py +0 -0
  92. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/user_tables.py +0 -0
  93. {datajoint-2.2.4 → datajoint-2.3.1}/src/datajoint/utils.py +0 -0
  94. {datajoint-2.2.4 → datajoint-2.3.1}/tests/__init__.py +0 -0
  95. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/__init__.py +0 -0
  96. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Course.csv +0 -0
  97. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/CurrentTerm.csv +0 -0
  98. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Department.csv +0 -0
  99. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Enroll.csv +0 -0
  100. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Grade.csv +0 -0
  101. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Section.csv +0 -0
  102. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Student.csv +0 -0
  103. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/StudentMajor.csv +0 -0
  104. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/data/Term.csv +0 -0
  105. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_aggr_regressions.py +0 -0
  106. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_alter.py +0 -0
  107. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_attach.py +0 -0
  108. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_blob.py +0 -0
  109. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_blob_matlab.py +0 -0
  110. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_cascading_delete.py +0 -0
  111. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_cli.py +0 -0
  112. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_codec_chaining.py +0 -0
  113. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_codecs.py +0 -0
  114. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_connection.py +0 -0
  115. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_declare.py +0 -0
  116. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_dependencies.py +0 -0
  117. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_erd.py +0 -0
  118. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_fetch.py +0 -0
  119. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_fetch_same.py +0 -0
  120. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_foreign_keys.py +0 -0
  121. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_groupby.py +0 -0
  122. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_hidden_job_metadata.py +0 -0
  123. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_insert.py +0 -0
  124. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_jobs.py +0 -0
  125. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_json.py +0 -0
  126. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_multi_backend.py +0 -0
  127. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_nan.py +0 -0
  128. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_npy_codec.py +0 -0
  129. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_privileges.py +0 -0
  130. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_reconnection.py +0 -0
  131. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_relation.py +0 -0
  132. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_relation_u.py +0 -0
  133. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_relational_operand.py +0 -0
  134. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_schema.py +0 -0
  135. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_schema_keywords.py +0 -0
  136. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_skip_duplicates.py +0 -0
  137. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_type_aliases.py +0 -0
  138. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_university.py +0 -0
  139. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_update1.py +0 -0
  140. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_utils.py +0 -0
  141. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_uuid.py +0 -0
  142. {datajoint-2.2.4 → datajoint-2.3.1}/tests/integration/test_virtual_module.py +0 -0
  143. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema.py +0 -0
  144. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_advanced.py +0 -0
  145. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_aggr_regress.py +0 -0
  146. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_alter.py +0 -0
  147. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_codecs.py +0 -0
  148. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_external.py +0 -0
  149. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_object.py +0 -0
  150. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_privileges.py +0 -0
  151. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_simple.py +0 -0
  152. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_type_aliases.py +0 -0
  153. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_university.py +0 -0
  154. {datajoint-2.2.4 → datajoint-2.3.1}/tests/schema_uuid.py +0 -0
  155. {datajoint-2.2.4 → datajoint-2.3.1}/tests/test_package.py +0 -0
  156. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/__init__.py +0 -0
  157. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_condition.py +0 -0
  158. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_connection_warning.py +0 -0
  159. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_declare_hidden_attribute.py +0 -0
  160. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_fetch_compat.py +0 -0
  161. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_lazy_imports.py +0 -0
  162. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_pk_rules.py +0 -0
  163. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_settings.py +0 -0
  164. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_storage_adapter.py +0 -0
  165. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_storage_urls.py +0 -0
  166. {datajoint-2.2.4 → datajoint-2.3.1}/tests/unit/test_thread_safe.py +0 -0
@@ -28,7 +28,7 @@ categories:
28
28
  - title: '📝 Documentation'
29
29
  label: 'documentation'
30
30
  change-template: '- $TITLE(#$NUMBER)@$AUTHOR'
31
- change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
31
+ change-title-escapes: '\<*_&@#' # escape @ and # so decorator/issue refs in PR titles (e.g. `@schema`) don't render as spurious mentions/links in release notes.
32
32
  template: |
33
33
  $CHANGES
34
34
 
@@ -27,6 +27,18 @@ on:
27
27
  jobs:
28
28
  test:
29
29
  runs-on: ubuntu-latest
30
+ strategy:
31
+ fail-fast: false
32
+ matrix:
33
+ # Python: exercise both ends of the supported range (requires-python
34
+ # >=3.10,<3.15); version-pinned pixi environments live in pyproject.toml
35
+ # under [tool.pixi.environments].
36
+ environment: [test-py310, test-py314]
37
+ # MySQL: exercise both LTS lines we support / that AWS RDS runs. 8.0 uses
38
+ # the datajoint image (SSL configured); 8.4 uses the official image (TLS
39
+ # tests already skip outside external containers). See #1497.
40
+ mysql-image: ["datajoint/mysql:8.0", "mysql:8.4"]
41
+ name: test (${{ matrix.environment }}, mysql=${{ matrix.mysql-image }})
30
42
  steps:
31
43
  - uses: actions/checkout@v4
32
44
 
@@ -35,9 +47,12 @@ jobs:
35
47
  with:
36
48
  cache: true
37
49
  locked: false
50
+ environments: ${{ matrix.environment }}
38
51
 
39
52
  - name: Run tests
40
- run: pixi run -e test test-cov
53
+ env:
54
+ DJ_TEST_MYSQL_IMAGE: ${{ matrix.mysql-image }}
55
+ run: pixi run -e ${{ matrix.environment }} test-cov
41
56
 
42
57
  # Unit tests run without containers (faster feedback)
43
58
  unit-tests:
@@ -45,4 +45,6 @@ repos:
45
45
  - types-PyMySQL
46
46
  - types-tqdm
47
47
  - pandas-stubs
48
- - numpy
48
+ # numpy>=2.2 stubs use PEP 695 `type` statements, which mypy rejects
49
+ # when type-checking against python_version 3.10 (the project floor).
50
+ - numpy<2.2
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datajoint
3
- Version: 2.2.4
3
+ Version: 2.3.1
4
4
  Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
5
5
  Project-URL: Homepage, https://docs.datajoint.com/
6
6
  Project-URL: Documentation, https://docs.datajoint.com/
@@ -207,11 +207,16 @@ Classifier: Intended Audience :: Healthcare Industry
207
207
  Classifier: Intended Audience :: Science/Research
208
208
  Classifier: License :: OSI Approved :: Apache Software License
209
209
  Classifier: Programming Language :: Python
210
+ Classifier: Programming Language :: Python :: 3.10
211
+ Classifier: Programming Language :: Python :: 3.11
212
+ Classifier: Programming Language :: Python :: 3.12
213
+ Classifier: Programming Language :: Python :: 3.13
214
+ Classifier: Programming Language :: Python :: 3.14
210
215
  Classifier: Topic :: Scientific/Engineering
211
216
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
212
217
  Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
213
218
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
214
- Requires-Python: <3.14,>=3.10
219
+ Requires-Python: <3.15,>=3.10
215
220
  Requires-Dist: deepdiff
216
221
  Requires-Dist: fsspec>=2023.1.0
217
222
  Requires-Dist: networkx
@@ -267,7 +272,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
267
272
  - **Tables represent workflow steps** — Each table is a step in your pipeline
268
273
  - **Foreign keys encode dependencies** — Parent tables must be populated before child tables
269
274
  - **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
270
- - **Results are immutable** — Full provenance and reproducibility
275
+ - **Results are immutable** — Full lineage and reproducibility
271
276
 
272
277
  **Documentation:** https://docs.datajoint.com
273
278
 
@@ -5,7 +5,7 @@ DataJoint is a framework for scientific data pipelines based on the **Relational
5
5
  - **Tables represent workflow steps** — Each table is a step in your pipeline
6
6
  - **Foreign keys encode dependencies** — Parent tables must be populated before child tables
7
7
  - **Computations are declarative** — Define *what* to compute; DataJoint handles *when*
8
- - **Results are immutable** — Full provenance and reproducibility
8
+ - **Results are immutable** — Full lineage and reproducibility
9
9
 
10
10
  **Documentation:** https://docs.datajoint.com
11
11
 
@@ -19,7 +19,7 @@ dependencies = [
19
19
  "packaging",
20
20
  ]
21
21
 
22
- requires-python = ">=3.10,<3.14"
22
+ requires-python = ">=3.10,<3.15"
23
23
  authors = [
24
24
  {name = "Dimitri Yatsenko", email = "dimitri@datajoint.com"},
25
25
  {name = "Thinh Nguyen", email = "thinh@datajoint.com"},
@@ -57,6 +57,11 @@ keywords = [
57
57
  # https://pypi.org/classifiers/
58
58
  classifiers = [
59
59
  "Programming Language :: Python",
60
+ "Programming Language :: Python :: 3.10",
61
+ "Programming Language :: Python :: 3.11",
62
+ "Programming Language :: Python :: 3.12",
63
+ "Programming Language :: Python :: 3.13",
64
+ "Programming Language :: Python :: 3.14",
60
65
  "Development Status :: 5 - Production/Stable",
61
66
  "Intended Audience :: Science/Research",
62
67
  "Intended Audience :: Healthcare Industry",
@@ -250,10 +255,22 @@ datajoint = { path = ".", editable = true, extras = ["test"] }
250
255
  [tool.pixi.feature.dev.pypi-dependencies]
251
256
  datajoint = { path = ".", editable = true, extras = ["dev", "test"] }
252
257
 
258
+ # Pin the interpreter for the min/max supported versions so CI can exercise
259
+ # both ends of the requires-python range (see .github/workflows/test.yaml).
260
+ [tool.pixi.feature.py310.dependencies]
261
+ python = "3.10.*"
262
+
263
+ [tool.pixi.feature.py314.dependencies]
264
+ python = "3.14.*"
265
+
253
266
  [tool.pixi.environments]
254
267
  default = { solve-group = "default" }
255
268
  dev = { features = ["dev"], solve-group = "default" }
256
269
  test = { features = ["test"], solve-group = "default" }
270
+ # Version-pinned test environments (each solves independently — a solve-group
271
+ # cannot span different Python versions).
272
+ test-py310 = { features = ["test", "py310"] }
273
+ test-py314 = { features = ["test", "py314"] }
257
274
 
258
275
  [tool.pixi.tasks]
259
276
  # Tests use testcontainers - no manual setup required
@@ -265,7 +282,7 @@ services-down = "docker compose down"
265
282
  test-external = { cmd = "DJ_USE_EXTERNAL_CONTAINERS=1 pytest tests/", depends-on = ["services-up"] }
266
283
 
267
284
  [tool.pixi.dependencies]
268
- python = ">=3.10,<3.14"
285
+ python = ">=3.10,<3.15"
269
286
  graphviz = ">=13.1.2,<14"
270
287
 
271
288
  [tool.pixi.activation]
@@ -51,12 +51,15 @@ __all__ = [
51
51
  "get_codec",
52
52
  "ObjectRef",
53
53
  "NpyRef",
54
+ # SparkAdapter Codec Protocol
55
+ "SparkAdapter",
54
56
  # Storage Adapter API
55
57
  "StorageAdapter",
56
58
  "get_storage_adapter",
57
59
  # Other
58
60
  "errors",
59
61
  "migrate",
62
+ "deploy",
60
63
  "DataJointError",
61
64
  "ThreadSafetyError",
62
65
  "logger",
@@ -69,6 +72,7 @@ __all__ = [
69
72
  # =============================================================================
70
73
  from . import errors
71
74
  from . import migrate
75
+ from . import deploy
72
76
  from .codecs import (
73
77
  Codec,
74
78
  get_codec,
@@ -85,6 +89,7 @@ from .expression import AndList, Not, Top, U
85
89
  from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global_config, _check_thread_safe
86
90
  from .logging import logger
87
91
  from .objectref import ObjectRef
92
+ from .spark import SparkAdapter
88
93
  from .storage_adapter import StorageAdapter, get_storage_adapter
89
94
  from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
90
95
  from .autopopulate import AutoPopulate
@@ -279,7 +284,7 @@ _lazy_modules = {
279
284
  "diagram": (".diagram", None), # Return the module itself
280
285
  # cli imports click
281
286
  "cli": (".cli", "cli"),
282
- # gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
287
+ # gc — exposed lazily so `dj.gc.GarbageCollector(...)` works as documented in gc.py
283
288
  # and in the user docs (how-to/garbage-collection.md).
284
289
  "gc": (".gc", None), # Return the module itself
285
290
  }
@@ -850,6 +850,29 @@ class DatabaseAdapter(ABC):
850
850
  raise NotImplementedError
851
851
  ...
852
852
 
853
+ def find_upstream_schemas_sql(self, schemas_list: str) -> str:
854
+ """
855
+ Generate query to find schemas that the given schemas reference via FK.
856
+
857
+ Used to discover unloaded schemas that the loaded ones depend on
858
+ (the upstream / ancestor direction). Symmetric to
859
+ :meth:`find_downstream_schemas_sql`.
860
+
861
+ Parameters
862
+ ----------
863
+ schemas_list : str
864
+ Comma-separated, quoted schema names for an IN clause.
865
+
866
+ Returns
867
+ -------
868
+ str
869
+ SQL query returning rows with a single column ``schema_name``
870
+ containing distinct schema names that are referenced by the
871
+ given schemas.
872
+ """
873
+ raise NotImplementedError
874
+ ...
875
+
853
876
  @abstractmethod
854
877
  def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
855
878
  """
@@ -696,6 +696,16 @@ class MySQLAdapter(DatabaseAdapter):
696
696
  f"AND table_schema NOT IN ({schemas_list})"
697
697
  )
698
698
 
699
+ def find_upstream_schemas_sql(self, schemas_list: str) -> str:
700
+ """Find schemas that the given schemas reference via FK."""
701
+ return (
702
+ f"SELECT DISTINCT referenced_table_schema as schema_name "
703
+ f"FROM information_schema.key_column_usage "
704
+ f"WHERE table_schema IN ({schemas_list}) "
705
+ f"AND referenced_table_schema IS NOT NULL "
706
+ f"AND referenced_table_schema NOT IN ({schemas_list})"
707
+ )
708
+
699
709
  def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
700
710
  """Query to get FK constraint details from information_schema."""
701
711
  return (
@@ -861,6 +861,20 @@ class PostgreSQLAdapter(DatabaseAdapter):
861
861
  f"AND ns1.nspname NOT IN ({schemas_list})"
862
862
  )
863
863
 
864
+ def find_upstream_schemas_sql(self, schemas_list: str) -> str:
865
+ """Find schemas that the given schemas reference via FK."""
866
+ return (
867
+ f"SELECT DISTINCT ns2.nspname as schema_name "
868
+ f"FROM pg_constraint c "
869
+ f"JOIN pg_class cl1 ON c.conrelid = cl1.oid "
870
+ f"JOIN pg_namespace ns1 ON cl1.relnamespace = ns1.oid "
871
+ f"JOIN pg_class cl2 ON c.confrelid = cl2.oid "
872
+ f"JOIN pg_namespace ns2 ON cl2.relnamespace = ns2.oid "
873
+ f"WHERE c.contype = 'f' "
874
+ f"AND ns1.nspname IN ({schemas_list}) "
875
+ f"AND ns2.nspname NOT IN ({schemas_list})"
876
+ )
877
+
864
878
  def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
865
879
  """
866
880
  Query to get FK constraint details from information_schema.
@@ -1266,6 +1280,31 @@ class PostgreSQLAdapter(DatabaseAdapter):
1266
1280
  quoted_values = ", ".join(f"'{v}'" for v in values)
1267
1281
  return f"CREATE TYPE {self.quote_identifier(type_name)} AS ENUM ({quoted_values})"
1268
1282
 
1283
+ def replica_identity_ddl(self, full_table_name: str, mode: str) -> str:
1284
+ """
1285
+ Generate ALTER TABLE ... REPLICA IDENTITY statement.
1286
+
1287
+ Controls how much of the old row PostgreSQL writes to WAL on UPDATE/DELETE.
1288
+ ``"default"`` logs only primary-key columns; ``"full"`` logs the entire row.
1289
+ Required by some CDC tools (e.g. Databricks Lakehouse Sync) that need the
1290
+ full pre-image to drive Slowly-Changing-Dimension history.
1291
+
1292
+ The ALTER is metadata-only, instant, and idempotent — re-applying the same
1293
+ mode is a no-op at the storage layer.
1294
+
1295
+ Examples
1296
+ --------
1297
+ >>> adapter.replica_identity_ddl('"schema"."table"', 'full')
1298
+ 'ALTER TABLE "schema"."table" REPLICA IDENTITY FULL'
1299
+ >>> adapter.replica_identity_ddl('"schema"."table"', 'default')
1300
+ 'ALTER TABLE "schema"."table" REPLICA IDENTITY DEFAULT'
1301
+ """
1302
+ if mode not in ("default", "full"):
1303
+ from ..errors import DataJointError
1304
+
1305
+ raise DataJointError(f"Unsupported replica_identity mode: {mode!r}. Expected 'default' or 'full'.")
1306
+ return f"ALTER TABLE {full_table_name} REPLICA IDENTITY {mode.upper()}"
1307
+
1269
1308
  def get_pending_enum_ddl(self, schema_name: str) -> list[str]:
1270
1309
  """
1271
1310
  Get DDL statements for pending enum types and clear the pending list.
@@ -22,6 +22,14 @@ if TYPE_CHECKING:
22
22
 
23
23
  logger = logging.getLogger(__name__.split(".")[0])
24
24
 
25
+ # Parallel populate hands the live table and its open connection to workers by
26
+ # process inheritance, so it requires the `fork` start method. Python 3.14
27
+ # changed the default from `fork` to `forkserver`, which pickles the payload
28
+ # instead — that neither carries a live DB connection nor the table's internal
29
+ # state, and it deadlocks. Pin to `fork` where available (all POSIX platforms);
30
+ # fall back to the platform default elsewhere.
31
+ _MP_START_METHOD = "fork" if "fork" in mp.get_all_start_methods() else None
32
+
25
33
 
26
34
  # --- helper functions for multiprocessing --
27
35
 
@@ -30,7 +38,8 @@ def _initialize_populate(table: Table, jobs: Job | None, populate_kwargs: dict[s
30
38
  """
31
39
  Initialize a worker process for multiprocessing.
32
40
 
33
- Saves the unpickled table to the current process and reconnects to database.
41
+ Stores the inherited table on the worker process and reconnects to database
42
+ (the parent closes its connection before forking; each worker reopens one).
34
43
 
35
44
  Parameters
36
45
  ----------
@@ -89,6 +98,55 @@ class AutoPopulate:
89
98
  _key_source = None
90
99
  _allow_insert = False
91
100
  _jobs = None
101
+ _upstream = None # memoized upstream trace for the current make(); built lazily on first use
102
+ _upstream_key = None # current make() key; not None iff executing inside make()
103
+
104
+ @property
105
+ def upstream(self):
106
+ """
107
+ Pre-restricted ancestor view for the current ``make(self, key)`` call.
108
+
109
+ Inside ``make()``, ``self.upstream`` is a ``Diagram`` constructed via
110
+ :meth:`Diagram.trace(self & key) <datajoint.Diagram.trace>`. Use
111
+ ``self.upstream[T]`` to obtain a pre-restricted ``QueryExpression``
112
+ (or ``FreeTable``, when indexed by a string) for any ancestor of
113
+ ``self``.
114
+
115
+ Reading via ``self.upstream`` is the recommended pattern for the
116
+ make() reproducibility contract: the framework guarantees the
117
+ restriction matches the current ``key``, and indexing a non-ancestor
118
+ table raises ``DataJointError``. See
119
+ :doc:`reference/specs/autopopulate` for the contract.
120
+
121
+ Raises
122
+ ------
123
+ DataJointError
124
+ If accessed outside ``make()`` execution. To construct a trace
125
+ explicitly, use ``dj.Diagram.trace(self & key)``.
126
+
127
+ Examples
128
+ --------
129
+ ::
130
+
131
+ def make(self, key):
132
+ date = self.upstream[Session].fetch1("session_date")
133
+ traces = self.upstream[ExtractTraces].to_arrays("trace")
134
+ self.insert1({**key, "summary": compute(traces, date)})
135
+ """
136
+ if self._upstream_key is None:
137
+ raise DataJointError(
138
+ "self.upstream is only available inside make(). "
139
+ "Outside make(), construct a trace explicitly: "
140
+ "dj.Diagram.trace(self & key)."
141
+ )
142
+ if self._upstream is None:
143
+ # Build the trace lazily on first access and memoize it for this
144
+ # make() call, so make() calls that never read self.upstream pay
145
+ # nothing. See #1493.
146
+ from .diagram import Diagram
147
+
148
+ self._upstream = Diagram.trace(self & self._upstream_key)
149
+ return self._upstream
92
150
 
93
151
  class _JobsDescriptor:
94
152
  """Descriptor allowing jobs access on both class and instance."""
@@ -436,7 +494,9 @@ class AutoPopulate:
436
494
  if hasattr(self.connection._conn, "ctx"):
437
495
  del self.connection._conn.ctx
438
496
  with (
439
- mp.Pool(processes, _initialize_populate, (self, None, populate_kwargs)) as pool,
497
+ mp.get_context(_MP_START_METHOD).Pool(
498
+ processes, _initialize_populate, (self, None, populate_kwargs)
499
+ ) as pool,
440
500
  tqdm(desc="Processes: ", total=nkeys) if display_progress else contextlib.nullcontext() as progress_bar,
441
501
  ):
442
502
  for status in pool.imap(_call_populate1, keys, chunksize=1):
@@ -532,7 +592,9 @@ class AutoPopulate:
532
592
  if hasattr(self.connection._conn, "ctx"):
533
593
  del self.connection._conn.ctx # SSLContext is not pickleable
534
594
  with (
535
- mp.Pool(processes, _initialize_populate, (self, self.jobs, populate_kwargs)) as pool,
595
+ mp.get_context(_MP_START_METHOD).Pool(
596
+ processes, _initialize_populate, (self, self.jobs, populate_kwargs)
597
+ ) as pool,
536
598
  tqdm(desc="Processes: ", total=nkeys)
537
599
  if display_progress
538
600
  else contextlib.nullcontext() as progress_bar,
@@ -611,6 +673,12 @@ class AutoPopulate:
611
673
  logger.jobs(f"Making {key} -> {self.full_table_name}")
612
674
  self.__class__._allow_insert = True
613
675
 
676
+ # Record the key for a lazily-constructed upstream view. The trace is
677
+ # built on first `self.upstream` access (see the `upstream` property),
678
+ # so make() calls that never read upstream pay nothing. See #1493.
679
+ self._upstream_key = dict(key)
680
+ self._upstream = None
681
+
614
682
  try:
615
683
  if not is_generator:
616
684
  make(dict(key), **(make_kwargs or {}))
@@ -668,6 +736,11 @@ class AutoPopulate:
668
736
  return True
669
737
  finally:
670
738
  self.__class__._allow_insert = False
739
+ # Clear the per-make() upstream view so subsequent attribute
740
+ # access raises a clear error rather than silently using a
741
+ # stale trace from the previous make() call.
742
+ self._upstream = None
743
+ self._upstream_key = None
671
744
 
672
745
  def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]:
673
746
  """
@@ -782,8 +855,6 @@ class AutoPopulate:
782
855
 
783
856
  pk_condition = make_condition(self, key, set())
784
857
  self.connection.query(
785
- f"UPDATE {self.full_table_name} SET "
786
- "_job_start_time=%s, _job_duration=%s, _job_version=%s "
787
- f"WHERE {pk_condition}",
858
+ f"UPDATE {self.full_table_name} SET _job_start_time=%s, _job_duration=%s, _job_version=%s WHERE {pk_condition}",
788
859
  args=(start_time, duration, version[:64] if version else ""),
789
860
  )
@@ -21,7 +21,7 @@ class HashCodec(Codec):
21
21
  The database column stores JSON metadata: ``{hash, store, size}``.
22
22
  Duplicate content is automatically deduplicated across all tables.
23
23
 
24
- Deletion: Requires garbage collection via ``dj.gc.collect()``.
24
+ Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
25
25
 
26
26
  External only - requires @ modifier.
27
27
 
@@ -272,7 +272,7 @@ class NpyCodec(SchemaCodec):
272
272
  - Path: ``{schema}/{table}/{pk}/{attribute}.npy``
273
273
  - Database column: JSON with ``{path, store, dtype, shape}``
274
274
 
275
- Deletion: Requires garbage collection via ``dj.gc.collect()``.
275
+ Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
276
276
 
277
277
  See Also
278
278
  --------
@@ -53,7 +53,7 @@ class ObjectCodec(SchemaCodec):
53
53
 
54
54
  {store_root}/{schema}/{table}/{pk}/{field}/
55
55
 
56
- Deletion: Requires garbage collection via ``dj.gc.collect()``.
56
+ Deletion: Requires garbage collection via ``dj.gc.GarbageCollector``.
57
57
 
58
58
  Comparison with hash-addressed::
59
59
 
@@ -114,7 +114,7 @@ class SchemaCodec(Codec, register=False):
114
114
  Build schema-addressed storage path.
115
115
 
116
116
  Constructs a path that mirrors the database schema structure:
117
- ``{schema}/{table}/{pk_values}/{field}{ext}``
117
+ ``{schema_prefix}/{schema}/{table}/{pk_values}/{field}{ext}``
118
118
 
119
119
  Supports partitioning if configured in the store.
120
120
 
@@ -150,6 +150,7 @@ class SchemaCodec(Codec, register=False):
150
150
  spec = config.get_store_spec(store_name)
151
151
  partition_pattern = spec.get("partition_pattern")
152
152
  token_length = spec.get("token_length", 8)
153
+ schema_prefix = spec["schema_prefix"] # always present: settings applies the default
153
154
 
154
155
  return build_object_path(
155
156
  schema=schema,
@@ -159,6 +160,7 @@ class SchemaCodec(Codec, register=False):
159
160
  ext=ext,
160
161
  partition_pattern=partition_pattern,
161
162
  token_length=token_length,
163
+ schema_prefix=schema_prefix,
162
164
  )
163
165
 
164
166
  def _get_backend(self, store_name: str | None = None, config=None):
@@ -38,6 +38,7 @@ class MyTable(dj.Manual):
38
38
 
39
39
  from __future__ import annotations
40
40
 
41
+ import json
41
42
  import logging
42
43
  from abc import ABC, abstractmethod
43
44
  from typing import Any
@@ -219,6 +220,45 @@ class Codec(ABC):
219
220
  """
220
221
  pass
221
222
 
223
+ def referenced_paths(self, stored: Any) -> list[tuple[str, str | None]]:
224
+ """
225
+ Return the external store paths this stored value references.
226
+
227
+ Garbage collection and delete-time cleanup call this on the *stored*
228
+ (encoded) representation of a column value — the JSON/dict already in
229
+ the database — so discovery stays metadata-only (no download or decode).
230
+
231
+ The default recognizes DataJoint's standard external-storage metadata:
232
+ a dict (or JSON string) carrying a ``path`` and optional ``store``. This
233
+ covers hash-addressed (``<hash@>``/``<blob@>``/``<attach@>``) and
234
+ schema-addressed (``<object@>``/``<npy@>`` and any :class:`SchemaCodec`
235
+ subclass) storage with no extra work, and returns an empty list for
236
+ values that are not externally stored (e.g. in-table ``<blob>``).
237
+
238
+ Override only if your codec stores external references in a non-standard
239
+ shape. Returning the paths here is what lets garbage collection see a
240
+ custom codec's files as *referenced* rather than orphaned (see #1469).
241
+
242
+ Parameters
243
+ ----------
244
+ stored : any
245
+ The stored (encoded) value as read from the database column.
246
+
247
+ Returns
248
+ -------
249
+ list[tuple[str, str | None]]
250
+ ``(path, store_name)`` for each external artifact referenced.
251
+ """
252
+ value = stored
253
+ if isinstance(value, str):
254
+ try:
255
+ value = json.loads(value)
256
+ except (json.JSONDecodeError, TypeError, ValueError):
257
+ return []
258
+ if isinstance(value, dict) and "path" in value:
259
+ return [(value["path"], value.get("store"))]
260
+ return []
261
+
222
262
  def __repr__(self) -> str:
223
263
  return f"<{self.__class__.__name__}(name={self.name!r})>"
224
264
 
@@ -268,6 +268,20 @@ def assert_join_compatibility(
268
268
  lineage2 = expr2.heading[name].lineage
269
269
  # Semantic match requires both lineages to be non-None and equal
270
270
  if lineage1 is None or lineage2 is None or lineage1 != lineage2:
271
+ if lineage1 is None or lineage2 is None:
272
+ # Missing lineage usually means stale ~lineage rows that survived
273
+ # an upgrade or a partial declare. Decoration in 2.3+ refreshes
274
+ # lineage automatically, so this typically indicates a schema
275
+ # that has not been re-decorated since the upgrade.
276
+ raise DataJointError(
277
+ f"Cannot join on attribute `{name}`: lineage missing on "
278
+ f"one side ({lineage1} vs {lineage2}). This usually "
279
+ f"indicates a stale `~lineage` entry from an older "
280
+ f"DataJoint version or an incomplete declare. Run "
281
+ f"`schema.rebuild_lineage()` to recompute lineage from "
282
+ f"current FK definitions. If the lineages are genuinely "
283
+ f"different, use `.proj()` to rename one of the attributes."
284
+ )
271
285
  raise DataJointError(
272
286
  f"Cannot join on attribute `{name}`: "
273
287
  f"different lineages ({lineage1} vs {lineage2}). "
@@ -132,11 +132,13 @@ class Dependencies(nx.DiGraph):
132
132
  self._conn = connection
133
133
  self._node_alias_count = itertools.count()
134
134
  self._loaded = False
135
+ self._loaded_schemas = set() # schema names currently represented in the graph
135
136
  super().__init__(self)
136
137
 
137
138
  def clear(self) -> None:
138
139
  """Clear the graph and reset loaded state."""
139
140
  self._loaded = False
141
+ self._loaded_schemas = set()
140
142
  self._node_alias_count = itertools.count() # reset alias IDs for consistency
141
143
  super().clear()
142
144
 
@@ -165,6 +167,7 @@ class Dependencies(nx.DiGraph):
165
167
 
166
168
  # Build schema list for IN clause
167
169
  names = schema_names if schema_names is not None else set(self._conn.schemas)
170
+ self._loaded_schemas = set(names)
168
171
  if not names:
169
172
  self._loaded = True
170
173
  return
@@ -257,6 +260,45 @@ class Dependencies(nx.DiGraph):
257
260
  break
258
261
  known_schemas |= new_schemas
259
262
 
263
+ # Skip the expensive rebuild when the graph already contains every needed
264
+ # schema, so repeated calls within one operation don't reload the whole
265
+ # dependency tree. See #1493.
266
+ if self._loaded and known_schemas <= self._loaded_schemas:
267
+ return
268
+ self.load(force=True, schema_names=known_schemas)
269
+
270
+ def load_all_upstream(self) -> None:
271
+ """
272
+ Load dependencies including all upstream schemas referenced via FK chains.
273
+
274
+ Iteratively discovers schemas that the currently loaded schemas
275
+ reference, expanding the dependency graph until no new schemas
276
+ are found. This ensures that upstream restriction propagation
277
+ (``Diagram.trace()``) reaches all ancestor tables, including
278
+ those in schemas the user has not explicitly activated.
279
+
280
+ Called automatically by ``Diagram.trace()``. Symmetric to
281
+ :meth:`load_all_downstream`.
282
+ """
283
+ adapter = self._conn.adapter
284
+ known_schemas = set(self._conn.schemas)
285
+ if not known_schemas:
286
+ self.load()
287
+ return
288
+
289
+ while True:
290
+ schemas_list = ", ".join(adapter.quote_string(s) for s in known_schemas)
291
+ result = self._conn.query(adapter.find_upstream_schemas_sql(schemas_list))
292
+ new_schemas = {row[0] for row in result} - known_schemas
293
+ if not new_schemas:
294
+ break
295
+ known_schemas |= new_schemas
296
+
297
+ # Skip the expensive rebuild when the graph already contains every needed
298
+ # schema, so repeated Diagram.trace() calls within one populate() don't
299
+ # reload the whole dependency tree per key. See #1493.
300
+ if self._loaded and known_schemas <= self._loaded_schemas:
301
+ return
260
302
  self.load(force=True, schema_names=known_schemas)
261
303
 
262
304
  def topo_sort(self) -> list[str]: