datajoint 2.2.1__tar.gz → 2.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. {datajoint-2.2.1 → datajoint-2.2.3}/.pre-commit-config.yaml +0 -1
  2. {datajoint-2.2.1 → datajoint-2.2.3}/CONTRIBUTING.md +1 -1
  3. {datajoint-2.2.1 → datajoint-2.2.3}/PKG-INFO +3 -3
  4. {datajoint-2.2.1 → datajoint-2.2.3}/README.md +2 -2
  5. datajoint-2.2.3/images/pipeline.png +0 -0
  6. datajoint-2.2.3/images/pipeline.svg +118 -0
  7. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/__init__.py +7 -0
  8. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/adapters/mysql.py +3 -7
  9. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/codecs.py +56 -41
  10. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/connection.py +14 -0
  11. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/declare.py +8 -0
  12. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/gc.py +14 -8
  13. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/heading.py +10 -7
  14. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/migrate.py +14 -10
  15. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/settings.py +105 -43
  16. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/storage.py +14 -5
  17. datajoint-2.2.3/src/datajoint/storage_adapter.py +102 -0
  18. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/version.py +1 -1
  19. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_gc.py +118 -0
  20. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_json.py +4 -0
  21. datajoint-2.2.3/tests/unit/test_connection_warning.py +42 -0
  22. datajoint-2.2.3/tests/unit/test_declare_hidden_attribute.py +51 -0
  23. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_settings.py +119 -0
  24. datajoint-2.2.3/tests/unit/test_storage_adapter.py +313 -0
  25. datajoint-2.2.1/.devcontainer/Dockerfile +0 -14
  26. datajoint-2.2.1/.devcontainer/devcontainer.json +0 -6
  27. datajoint-2.2.1/.devcontainer/docker-compose.yml +0 -14
  28. datajoint-2.2.1/.vscode/launch.json +0 -16
  29. datajoint-2.2.1/images/pipeline.drawio +0 -1
  30. datajoint-2.2.1/images/pipeline.png +0 -0
  31. {datajoint-2.2.1 → datajoint-2.2.3}/.coveragerc +0 -0
  32. {datajoint-2.2.1 → datajoint-2.2.3}/.dockerignore +0 -0
  33. {datajoint-2.2.1 → datajoint-2.2.3}/.gitattributes +0 -0
  34. {datajoint-2.2.1 → datajoint-2.2.3}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
  35. {datajoint-2.2.1 → datajoint-2.2.3}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  36. {datajoint-2.2.1 → datajoint-2.2.3}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  37. {datajoint-2.2.1 → datajoint-2.2.3}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
  38. {datajoint-2.2.1 → datajoint-2.2.3}/.github/dependabot.yml +0 -0
  39. {datajoint-2.2.1 → datajoint-2.2.3}/.github/release_drafter.yaml +0 -0
  40. {datajoint-2.2.1 → datajoint-2.2.3}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
  41. {datajoint-2.2.1 → datajoint-2.2.3}/.github/workflows/draft_release.yaml +0 -0
  42. {datajoint-2.2.1 → datajoint-2.2.3}/.github/workflows/lint.yaml +0 -0
  43. {datajoint-2.2.1 → datajoint-2.2.3}/.github/workflows/post_draft_release_published.yaml +0 -0
  44. {datajoint-2.2.1 → datajoint-2.2.3}/.github/workflows/test.yaml +0 -0
  45. {datajoint-2.2.1 → datajoint-2.2.3}/.gitignore +0 -0
  46. {datajoint-2.2.1 → datajoint-2.2.3}/CHANGELOG-archive.md +0 -0
  47. {datajoint-2.2.1 → datajoint-2.2.3}/Dockerfile +0 -0
  48. {datajoint-2.2.1 → datajoint-2.2.3}/LICENSE +0 -0
  49. {datajoint-2.2.1 → datajoint-2.2.3}/RELEASE_MEMO.md +0 -0
  50. {datajoint-2.2.1 → datajoint-2.2.3}/activate.sh +0 -0
  51. {datajoint-2.2.1 → datajoint-2.2.3}/docker-compose.yaml +0 -0
  52. {datajoint-2.2.1 → datajoint-2.2.3}/pixi.lock +0 -0
  53. {datajoint-2.2.1 → datajoint-2.2.3}/pyproject.toml +0 -0
  54. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/adapters/__init__.py +0 -0
  55. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/adapters/base.py +0 -0
  56. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/adapters/postgres.py +0 -0
  57. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/autopopulate.py +0 -0
  58. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/blob.py +0 -0
  59. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/__init__.py +0 -0
  60. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/attach.py +0 -0
  61. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/blob.py +0 -0
  62. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/filepath.py +0 -0
  63. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/hash.py +0 -0
  64. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/npy.py +0 -0
  65. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/object.py +0 -0
  66. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/builtin_codecs/schema.py +0 -0
  67. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/cli.py +0 -0
  68. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/condition.py +0 -0
  69. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/dependencies.py +0 -0
  70. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/diagram.py +0 -0
  71. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/errors.py +0 -0
  72. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/expression.py +0 -0
  73. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/hash_registry.py +0 -0
  74. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/instance.py +0 -0
  75. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/jobs.py +0 -0
  76. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/lineage.py +0 -0
  77. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/logging.py +0 -0
  78. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/objectref.py +0 -0
  79. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/preview.py +0 -0
  80. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/schemas.py +0 -0
  81. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/staged_insert.py +0 -0
  82. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/table.py +0 -0
  83. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/types.py +0 -0
  84. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/user_tables.py +0 -0
  85. {datajoint-2.2.1 → datajoint-2.2.3}/src/datajoint/utils.py +0 -0
  86. {datajoint-2.2.1 → datajoint-2.2.3}/tests/__init__.py +0 -0
  87. {datajoint-2.2.1 → datajoint-2.2.3}/tests/conftest.py +0 -0
  88. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/__init__.py +0 -0
  89. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/Course.csv +0 -0
  90. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/CurrentTerm.csv +0 -0
  91. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/Department.csv +0 -0
  92. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/Enroll.csv +0 -0
  93. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/Grade.csv +0 -0
  94. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/Section.csv +0 -0
  95. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/Student.csv +0 -0
  96. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/StudentMajor.csv +0 -0
  97. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/data/Term.csv +0 -0
  98. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_aggr_regressions.py +0 -0
  99. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_alter.py +0 -0
  100. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_attach.py +0 -0
  101. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_autopopulate.py +0 -0
  102. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_blob.py +0 -0
  103. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_blob_matlab.py +0 -0
  104. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_cascade_delete.py +0 -0
  105. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_cascading_delete.py +0 -0
  106. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_cli.py +0 -0
  107. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_codec_chaining.py +0 -0
  108. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_codecs.py +0 -0
  109. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_connection.py +0 -0
  110. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_declare.py +0 -0
  111. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_dependencies.py +0 -0
  112. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_erd.py +0 -0
  113. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_fetch.py +0 -0
  114. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_fetch_same.py +0 -0
  115. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_foreign_keys.py +0 -0
  116. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_groupby.py +0 -0
  117. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_hash_storage.py +0 -0
  118. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_hidden_job_metadata.py +0 -0
  119. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_insert.py +0 -0
  120. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_jobs.py +0 -0
  121. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_multi_backend.py +0 -0
  122. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_nan.py +0 -0
  123. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_npy_codec.py +0 -0
  124. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_object.py +0 -0
  125. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_privileges.py +0 -0
  126. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_reconnection.py +0 -0
  127. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_relation.py +0 -0
  128. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_relation_u.py +0 -0
  129. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_relational_operand.py +0 -0
  130. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_schema.py +0 -0
  131. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_schema_keywords.py +0 -0
  132. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_semantic_matching.py +0 -0
  133. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_skip_duplicates.py +0 -0
  134. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_tls.py +0 -0
  135. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_type_aliases.py +0 -0
  136. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_university.py +0 -0
  137. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_update1.py +0 -0
  138. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_utils.py +0 -0
  139. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_uuid.py +0 -0
  140. {datajoint-2.2.1 → datajoint-2.2.3}/tests/integration/test_virtual_module.py +0 -0
  141. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema.py +0 -0
  142. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_advanced.py +0 -0
  143. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_aggr_regress.py +0 -0
  144. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_alter.py +0 -0
  145. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_codecs.py +0 -0
  146. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_external.py +0 -0
  147. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_object.py +0 -0
  148. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_privileges.py +0 -0
  149. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_simple.py +0 -0
  150. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_type_aliases.py +0 -0
  151. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_university.py +0 -0
  152. {datajoint-2.2.1 → datajoint-2.2.3}/tests/schema_uuid.py +0 -0
  153. {datajoint-2.2.1 → datajoint-2.2.3}/tests/test_package.py +0 -0
  154. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/__init__.py +0 -0
  155. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_adapters.py +0 -0
  156. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_codecs.py +0 -0
  157. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_condition.py +0 -0
  158. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_fetch_compat.py +0 -0
  159. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_lazy_imports.py +0 -0
  160. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_pk_rules.py +0 -0
  161. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_storage_urls.py +0 -0
  162. {datajoint-2.2.1 → datajoint-2.2.3}/tests/unit/test_thread_safe.py +0 -0
@@ -12,7 +12,6 @@ repos:
12
12
  - id: check-yaml
13
13
  exclude: 'docs/mkdocs.yaml' # exclude mkdocs.yaml since pymdownx.emoji !! usage
14
14
  - id: check-json
15
- exclude: '(.vscode|.devcontainer)' # exclude these since // was used for comments
16
15
  - id: check-toml
17
16
  - id: check-added-large-files
18
17
  - repo: https://github.com/codespell-project/codespell
@@ -56,7 +56,7 @@ export DOCKER_HOST=unix://$HOME/.docker/run/docker.sock
56
56
 
57
57
  ### PostgreSQL Backend
58
58
 
59
- DataJoint supports PostgreSQL 15+ as an alternative to MySQL 8+. To install the PostgreSQL driver:
59
+ DataJoint supports MySQL 8.0.13+ and PostgreSQL 15+ as production database backends. To install the PostgreSQL driver:
60
60
 
61
61
  ```bash
62
62
  pip install -e ".[postgres]" # Installs psycopg2-binary
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datajoint
3
- Version: 2.2.1
3
+ Version: 2.2.3
4
4
  Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
5
5
  Project-URL: Homepage, https://docs.datajoint.com/
6
6
  Project-URL: Documentation, https://docs.datajoint.com/
@@ -330,9 +330,9 @@ conda install -c conda-forge datajoint
330
330
 
331
331
  ## Example Pipeline
332
332
 
333
- ![pipeline](https://raw.githubusercontent.com/datajoint/datajoint-python/master/images/pipeline.png)
333
+ ![pipeline](images/pipeline.png)
334
334
 
335
- [Yatsenko et al., bioRxiv 2021](https://doi.org/10.1101/2021.03.30.437358)
335
+ **Cite DataJoint:** [Yatsenko et al., 2026](https://arxiv.org/abs/2602.16585) — RRID: [SCR_014543](https://scicrunch.org/resolver/SCR_014543)
336
336
 
337
337
  ## Resources
338
338
 
@@ -69,9 +69,9 @@ conda install -c conda-forge datajoint
69
69
 
70
70
  ## Example Pipeline
71
71
 
72
- ![pipeline](https://raw.githubusercontent.com/datajoint/datajoint-python/master/images/pipeline.png)
72
+ ![pipeline](images/pipeline.png)
73
73
 
74
- [Yatsenko et al., bioRxiv 2021](https://doi.org/10.1101/2021.03.30.437358)
74
+ **Cite DataJoint:** [Yatsenko et al., 2026](https://arxiv.org/abs/2602.16585) — RRID: [SCR_014543](https://scicrunch.org/resolver/SCR_014543)
75
75
 
76
76
  ## Resources
77
77
 
Binary file
@@ -0,0 +1,118 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1500 540"
2
+ font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif"
3
+ font-size="22">
4
+ <defs>
5
+ <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5"
6
+ markerWidth="7" markerHeight="7" orient="auto-start-reverse">
7
+ <path d="M0,0 L10,5 L0,10 z" fill="#666"/>
8
+ </marker>
9
+ </defs>
10
+
11
+ <!-- Opaque white background — keeps the whole illustration legible
12
+ on any page theme (e.g. GitHub dark-mode README rendering). -->
13
+ <rect width="100%" height="100%" fill="white"/>
14
+
15
+ <!-- ============ LEFT PANEL: Pipeline diagram ============ -->
16
+
17
+ <!-- Edges -->
18
+ <g stroke="#666" stroke-width="1.6" fill="none">
19
+ <line x1="200" y1="90" x2="200" y2="118" marker-end="url(#arrow)"/>
20
+ <line x1="200" y1="160" x2="200" y2="188" marker-end="url(#arrow)"/>
21
+ <line x1="200" y1="230" x2="200" y2="259" marker-end="url(#arrow)"/>
22
+ <line x1="215" y1="316" x2="232" y2="352" marker-end="url(#arrow)"/>
23
+ <line x1="320" y1="310" x2="294" y2="358" marker-end="url(#arrow)"/>
24
+ <line x1="250" y1="410" x2="250" y2="441" marker-end="url(#arrow)"/>
25
+ </g>
26
+
27
+ <!-- Manual: Subject -->
28
+ <rect x="135" y="50" width="130" height="40" rx="3"
29
+ fill="#7eca9c" stroke="none"/>
30
+ <text x="200" y="78" text-anchor="middle" fill="#1a1a1a" font-weight="500">Subject</text>
31
+
32
+ <!-- Manual: Session -->
33
+ <rect x="135" y="120" width="130" height="40" rx="3"
34
+ fill="#7eca9c" stroke="none"/>
35
+ <text x="200" y="148" text-anchor="middle" fill="#1a1a1a" font-weight="500">Session</text>
36
+
37
+ <!-- Manual: Scan -->
38
+ <rect x="135" y="190" width="130" height="40" rx="3"
39
+ fill="#7eca9c" stroke="none"/>
40
+ <text x="200" y="218" text-anchor="middle" fill="#1a1a1a" font-weight="500">Scan</text>
41
+
42
+ <!-- Imported: AverageFrame -->
43
+ <ellipse cx="200" cy="290" rx="90" ry="28"
44
+ fill="#7eb6ca" stroke="none"/>
45
+ <text x="200" y="298" text-anchor="middle" fill="#1a1a1a" font-weight="500">AverageFrame</text>
46
+
47
+ <!-- Lookup: SegmentationParams -->
48
+ <rect x="320" y="270" width="220" height="40" rx="3"
49
+ fill="#e8e8e8" stroke="none"/>
50
+ <text x="430" y="298" text-anchor="middle" fill="#1a1a1a" font-weight="500">SegmentationParams</text>
51
+
52
+ <!-- Computed: Segmentation (highlighted — origin of zoom callout) -->
53
+ <ellipse cx="250" cy="380" rx="100" ry="30"
54
+ fill="#ca7e7e" stroke="none"/>
55
+ <text x="250" y="388" text-anchor="middle" fill="#1a1a1a" font-weight="600">Segmentation</text>
56
+
57
+ <!-- Computed: Activity -->
58
+ <ellipse cx="250" cy="470" rx="74" ry="26"
59
+ fill="#ca7e7e" stroke="none"/>
60
+ <text x="250" y="478" text-anchor="middle" fill="#1a1a1a" font-weight="500">Activity</text>
61
+
62
+ <!-- ============ Zoom-in callout ============ -->
63
+ <g stroke="#999" stroke-width="1" stroke-dasharray="5,4" fill="none">
64
+ <line x1="340" y1="358" x2="580" y2="80"/>
65
+ <line x1="340" y1="402" x2="580" y2="470"/>
66
+ </g>
67
+
68
+ <!-- ============ RIGHT PANEL: Code snippet ============ -->
69
+ <!-- Indentation handled via per-line x offsets:
70
+ level 0 (no indent): x = 600
71
+ level 1 (4 spaces): x = 643
72
+ level 2 (8 spaces): x = 686
73
+ Char width @ 18px monospace ≈ 10.8 -->
74
+
75
+ <rect x="580" y="80" width="660" height="390" rx="4"
76
+ fill="white" stroke="#d0d0d0" stroke-width="1"/>
77
+
78
+ <g font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace"
79
+ font-size="18" fill="#24292e">
80
+ <text x="600" y="112"><tspan fill="#005cc5">@schema</tspan></text>
81
+ <text x="600" y="136"><tspan fill="#005cc5">class</tspan> <tspan fill="#6f42c1" font-weight="600">Segmentation</tspan>(dj.Computed):</text>
82
+ <text x="643" y="160">definition = <tspan fill="#22863a">"""</tspan></text>
83
+ <text x="643" y="184" fill="#22863a">-&gt; AverageFrame</text>
84
+ <text x="643" y="208" fill="#22863a">-&gt; SegmentationParams</text>
85
+ <text x="643" y="232" fill="#22863a">---</text>
86
+ <text x="643" y="256" fill="#22863a">num_cells<tspan dx="11">: int32</tspan></text>
87
+ <text x="643" y="280" fill="#22863a">cell_masks : &lt;blob@&gt;</text>
88
+ <text x="643" y="304" fill="#22863a">"""</text>
89
+ <text x="643" y="352"><tspan fill="#005cc5">def</tspan> <tspan fill="#6f42c1" font-weight="600">make</tspan>(self, key):</text>
90
+ <text x="686" y="376">frame = (AverageFrame &amp; key).fetch1(<tspan fill="#22863a">'frame'</tspan>)</text>
91
+ <text x="686" y="400">params = (SegmentationParams &amp; key).fetch1()</text>
92
+ <text x="686" y="424">masks, n = segment(frame, **params)</text>
93
+ <text x="686" y="448">self.insert1(dict(key, num_cells=n, cell_masks=masks))</text>
94
+ </g>
95
+
96
+ <!-- ============ Annotations ============ -->
97
+ <g font-style="italic" fill="#5a5a5a" font-size="19">
98
+ <text x="1260" y="112">database link</text>
99
+ <line x1="1255" y1="107" x2="690" y2="107"
100
+ stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
101
+
102
+ <text x="1260" y="136">table name</text>
103
+ <line x1="1255" y1="131" x2="855" y2="131"
104
+ stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
105
+
106
+ <text x="1260" y="208">dependency</text>
107
+ <line x1="1255" y1="203" x2="850" y2="203"
108
+ stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
109
+
110
+ <text x="1260" y="280">object-store attribute</text>
111
+ <line x1="1255" y1="275" x2="900" y2="275"
112
+ stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
113
+
114
+ <text x="1260" y="400">computation</text>
115
+ <line x1="1255" y1="395" x2="970" y2="395"
116
+ stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
117
+ </g>
118
+ </svg>
@@ -30,6 +30,8 @@ __all__ = [
30
30
  "list_schemas",
31
31
  "Table",
32
32
  "FreeTable",
33
+ "AutoPopulate",
34
+ "Job",
33
35
  "Manual",
34
36
  "Lookup",
35
37
  "Imported",
@@ -81,6 +83,8 @@ from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global
81
83
  from .logging import logger
82
84
  from .objectref import ObjectRef
83
85
  from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
86
+ from .autopopulate import AutoPopulate
87
+ from .jobs import Job
84
88
  from .table import FreeTable as _FreeTable, Table, ValidationResult
85
89
  from .user_tables import Computed, Imported, Lookup, Manual, Part
86
90
  from .version import __version__
@@ -271,6 +275,9 @@ _lazy_modules = {
271
275
  "diagram": (".diagram", None), # Return the module itself
272
276
  # cli imports click
273
277
  "cli": (".cli", "cli"),
278
+ # gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
279
+ # and in the user docs (how-to/garbage-collection.md).
280
+ "gc": (".gc", None), # Return the module itself
274
281
  }
275
282
 
276
283
 
@@ -423,7 +423,7 @@ class MySQLAdapter(DatabaseAdapter):
423
423
  fk_cols = ", ".join(self.quote_identifier(col) for col in fk["columns"])
424
424
  ref_cols = ", ".join(self.quote_identifier(col) for col in fk["ref_columns"])
425
425
  lines.append(
426
- f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols}) " f"ON UPDATE CASCADE ON DELETE RESTRICT"
426
+ f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols}) ON UPDATE CASCADE ON DELETE RESTRICT"
427
427
  )
428
428
 
429
429
  # Indexes
@@ -735,14 +735,10 @@ class MySQLAdapter(DatabaseAdapter):
735
735
  return result
736
736
 
737
737
  def get_indexes_sql(self, schema_name: str, table_name: str) -> str:
738
- """Query to get index definitions.
739
-
740
- Note: For MySQL 8.0+, EXPRESSION column contains the expression for
741
- functional indexes. COLUMN_NAME is NULL for such indexes.
742
- """
738
+ """Query to get index definitions. Functional indexes (NULL COLUMN_NAME) are skipped downstream."""
743
739
  return (
744
740
  f"SELECT INDEX_NAME as index_name, "
745
- f"COALESCE(COLUMN_NAME, CONCAT('(', EXPRESSION, ')')) as column_name, "
741
+ f"COLUMN_NAME as column_name, "
746
742
  f"NON_UNIQUE as non_unique, SEQ_IN_INDEX as seq_in_index "
747
743
  f"FROM information_schema.statistics "
748
744
  f"WHERE table_schema = {self.quote_string(schema_name)} "
@@ -8,29 +8,32 @@ semantics for complex Python objects.
8
8
  Codecs auto-register when subclassed - no decorator needed (Python 3.10+).
9
9
 
10
10
  Example:
11
- class GraphCodec(dj.Codec):
12
- name = "graph"
13
11
 
14
- def get_dtype(self, is_store: bool) -> str:
15
- return "<blob>"
12
+ ```python
13
+ class GraphCodec(dj.Codec):
14
+ name = "graph"
16
15
 
17
- def encode(self, graph, *, key=None, store_name=None):
18
- return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())}
19
-
20
- def decode(self, stored, *, key=None):
21
- import networkx as nx
22
- G = nx.Graph()
23
- G.add_nodes_from(stored['nodes'])
24
- G.add_edges_from(stored['edges'])
25
- return G
26
-
27
- # Then use in table definitions:
28
- class MyTable(dj.Manual):
29
- definition = '''
30
- id : uint16
31
- ---
32
- data : <graph>
33
- '''
16
+ def get_dtype(self, is_store: bool) -> str:
17
+ return "<blob>"
18
+
19
+ def encode(self, graph, *, key=None, store_name=None):
20
+ return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())}
21
+
22
+ def decode(self, stored, *, key=None):
23
+ import networkx as nx
24
+ G = nx.Graph()
25
+ G.add_nodes_from(stored['nodes'])
26
+ G.add_edges_from(stored['edges'])
27
+ return G
28
+
29
+ # Then use in table definitions:
30
+ class MyTable(dj.Manual):
31
+ definition = '''
32
+ id : uint16
33
+ ---
34
+ data : <graph>
35
+ '''
36
+ ```
34
37
  """
35
38
 
36
39
  from __future__ import annotations
@@ -85,20 +88,24 @@ class Codec(ABC):
85
88
  ... G.add_edges_from(stored['edges'])
86
89
  ... return G
87
90
 
88
- Use in table definitions::
91
+ Use in table definitions:
89
92
 
90
- class Connectivity(dj.Manual):
91
- definition = '''
92
- id : uint16
93
- ---
94
- graph_data : <graph>
95
- '''
93
+ ```python
94
+ class Connectivity(dj.Manual):
95
+ definition = '''
96
+ id : uint16
97
+ ---
98
+ graph_data : <graph>
99
+ '''
100
+ ```
96
101
 
97
- Skip auto-registration for abstract base classes::
102
+ Skip auto-registration for abstract base classes:
98
103
 
99
- class ExternalOnlyCodec(dj.Codec, register=False):
100
- '''Abstract base - not registered.'''
101
- ...
104
+ ```python
105
+ class ExternalOnlyCodec(dj.Codec, register=False):
106
+ '''Abstract base - not registered.'''
107
+ ...
108
+ ```
102
109
  """
103
110
 
104
111
  name: str | None = None # Must be set by concrete subclasses
@@ -520,18 +527,26 @@ def decode_attribute(attr, data, squeeze: bool = False, connection=None):
520
527
  Decode raw database value using attribute's codec or native type handling.
521
528
 
522
529
  This is the central decode function used by all fetch methods. It handles:
523
- - Codec chains (e.g., <blob@store> → <hash> → bytes)
530
+
531
+ - Codec chains (e.g., ``<blob@store>`` → ``<hash>`` → ``bytes``)
524
532
  - Native type conversions (JSON, UUID)
525
- - Object storage downloads (via config["download_path"])
533
+ - Object storage downloads (via ``config["download_path"]``)
526
534
 
527
- Args:
528
- attr: Attribute from the table's heading.
529
- data: Raw value fetched from the database.
530
- squeeze: If True, remove singleton dimensions from numpy arrays.
531
- connection: Connection instance for config access. If provided,
532
- ``connection._config`` is passed to codecs via the key dict.
535
+ Parameters
536
+ ----------
537
+ attr : Attribute
538
+ Attribute from the table's heading.
539
+ data : any
540
+ Raw value fetched from the database.
541
+ squeeze : bool, optional
542
+ If True, remove singleton dimensions from numpy arrays.
543
+ connection : Connection, optional
544
+ Connection instance for config access. If provided,
545
+ ``connection._config`` is passed to codecs via the key dict.
533
546
 
534
- Returns:
547
+ Returns
548
+ -------
549
+ any
535
550
  Decoded Python value.
536
551
  """
537
552
  import json
@@ -108,6 +108,18 @@ def conn(
108
108
  return conn.connection
109
109
 
110
110
 
111
+ def _warn_if_mariadb(version_str: str) -> None:
112
+ """Emit a UserWarning if `version_str` looks like MariaDB. No-op for MySQL."""
113
+ if "MariaDB" in version_str:
114
+ warnings.warn(
115
+ f"MariaDB is not officially supported by DataJoint 2.x "
116
+ f"(server reports {version_str}). Compatibility is best-effort "
117
+ f"and may break in future releases.",
118
+ UserWarning,
119
+ stacklevel=3,
120
+ )
121
+
122
+
111
123
  class EmulatedCursor:
112
124
  """acts like a cursor"""
113
125
 
@@ -221,6 +233,8 @@ class Connection:
221
233
  f"{self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}{db_str}"
222
234
  )
223
235
  self.connection_id = self.adapter.get_connection_id(self._conn)
236
+ if self.adapter.backend == "mysql":
237
+ _warn_if_mariadb(self.query("SELECT @@version").fetchone()[0])
224
238
  else:
225
239
  raise errors.LostConnectionError(
226
240
  f"Connection failed {self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}"
@@ -855,6 +855,14 @@ def compile_attribute(
855
855
  DataJointError
856
856
  If syntax is invalid, primary key is nullable, or blob has invalid default.
857
857
  """
858
+ if line.lstrip().startswith("_"):
859
+ raise DataJointError(
860
+ f'Attribute name in line "{line}" starts with an underscore. '
861
+ "Names with leading underscore are reserved for platform-managed "
862
+ "columns (e.g. _job_start_time, _singleton). Use a regular "
863
+ "attribute name; if you need to control visibility at the call "
864
+ "site, use proj()."
865
+ )
858
866
  try:
859
867
  match = attribute_parser.parse_string(line + "#", parse_all=True)
860
868
  except pp.ParseException as err:
@@ -229,11 +229,14 @@ def scan_hash_references(
229
229
  if verbose:
230
230
  logger.info(f" Scanning {table_name}.{attr_name}")
231
231
 
232
- # Fetch all values for this attribute
232
+ # Read raw JSON metadata via cursor — bypasses decode_attribute
233
+ # so we get the stored dict (PostgreSQL/JSONB) or JSON string
234
+ # (MySQL), not the decoded codec output. _extract_hash_refs
235
+ # handles both shapes.
233
236
  try:
234
- values = table.to_arrays(attr_name)
235
- for value in values:
236
- for path, ref_store in _extract_hash_refs(value):
237
+ cursor = table.proj(attr_name).cursor(as_dict=True)
238
+ for row in cursor:
239
+ for path, ref_store in _extract_hash_refs(row[attr_name]):
237
240
  # Filter by store if specified
238
241
  if store_name is None or ref_store == store_name:
239
242
  referenced.add(path)
@@ -291,11 +294,14 @@ def scan_schema_references(
291
294
  if verbose:
292
295
  logger.info(f" Scanning {table_name}.{attr_name}")
293
296
 
294
- # Fetch all values for this attribute
297
+ # Read raw JSON metadata via cursor — bypasses decode_attribute
298
+ # so we get the stored dict (PostgreSQL/JSONB) or JSON string
299
+ # (MySQL), not the decoded codec output. _extract_schema_refs
300
+ # handles both shapes.
295
301
  try:
296
- values = table.to_arrays(attr_name)
297
- for value in values:
298
- for path, ref_store in _extract_schema_refs(value):
302
+ cursor = table.proj(attr_name).cursor(as_dict=True)
303
+ for row in cursor:
304
+ for path, ref_store in _extract_schema_refs(row[attr_name]):
299
305
  # Filter by store if specified
300
306
  if store_name is None or ref_store == store_name:
301
307
  referenced.add(path)
@@ -508,8 +508,16 @@ class Heading:
508
508
  if category == "UUID":
509
509
  attr["uuid"] = True
510
510
  elif category in CORE_TYPE_NAMES:
511
- # Core type alias - already resolved in DB
512
- pass
511
+ # Core type alias - already resolved in DB.
512
+ # MariaDB-specific recovery: MariaDB stores `json` columns
513
+ # as `longtext` and reports them back that way through
514
+ # information_schema, so the DB-type-based detection above
515
+ # leaves attr["json"] False. The :json: comment marker
516
+ # survives this aliasing, so we recover the json flag here
517
+ # from the original declared type. No-op on MySQL/PostgreSQL
518
+ # (attr["json"] is already True from the regex match above).
519
+ if category == "JSON":
520
+ attr["json"] = True
513
521
 
514
522
  # Check primary key constraints
515
523
  if attr["in_key"] and (attr["is_blob"] or attr["json"]):
@@ -555,14 +563,9 @@ class Heading:
555
563
  adapter.get_indexes_sql(database, table_name),
556
564
  as_dict=True,
557
565
  ):
558
- # Note: adapter.get_indexes_sql() already filters out PRIMARY key
559
- # MySQL/PostgreSQL adapters return: index_name, column_name, non_unique
560
566
  index_name = item.get("index_name") or item.get("Key_name")
561
567
  seq = item.get("seq_in_index") or item.get("Seq_in_index") or len(keys[index_name]) + 1
562
568
  column = item.get("column_name") or item.get("Column_name")
563
- # MySQL EXPRESSION column stores escaped single quotes - unescape them
564
- if column:
565
- column = column.replace("\\'", "'")
566
569
  non_unique = item.get("non_unique") or item.get("Non_unique")
567
570
  nullable = item.get("nullable") or (item.get("Null", "NO").lower() == "yes")
568
571
 
@@ -7,8 +7,8 @@ explicit `<blob>` type declarations.
7
7
 
8
8
  .. note::
9
9
  This module is provided temporarily to assist with migration from pre-2.0.
10
- It will be deprecated in DataJoint 2.1 and removed in 2.3.
11
- Complete your migrations while on DataJoint 2.0.
10
+ It is scheduled for removal in DataJoint 2.4 or 2.5.
11
+ Complete your migrations while on DataJoint 2.3 or earlier.
12
12
 
13
13
  Note on Terminology
14
14
  -------------------
@@ -32,8 +32,8 @@ from .version import __version__
32
32
  # Show deprecation warning starting in 2.1
33
33
  if Version(__version__) >= Version("2.1"):
34
34
  warnings.warn(
35
- "datajoint.migrate is deprecated and will be removed in DataJoint 2.3. "
36
- "Complete your schema migrations before upgrading.",
35
+ "datajoint.migrate is deprecated and is scheduled for removal in DataJoint 2.4 or 2.5. "
36
+ "Complete your schema migrations while on DataJoint 2.3 or earlier.",
37
37
  DeprecationWarning,
38
38
  stacklevel=2,
39
39
  )
@@ -142,14 +142,14 @@ def analyze_columns(schema: Schema) -> dict:
142
142
  for (table_name,) in tables:
143
143
  # Get all columns for this table
144
144
  columns_query = """
145
- SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT
145
+ SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT, IS_NULLABLE
146
146
  FROM information_schema.COLUMNS
147
147
  WHERE TABLE_SCHEMA = %s
148
148
  AND TABLE_NAME = %s
149
149
  """
150
150
  columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
151
151
 
152
- for column_name, column_type, data_type, comment in columns:
152
+ for column_name, column_type, data_type, comment, is_nullable in columns:
153
153
  comment = comment or ""
154
154
 
155
155
  # Check if column already has a type label (starts with :type:)
@@ -167,6 +167,7 @@ def analyze_columns(schema: Schema) -> dict:
167
167
  "column": column_name,
168
168
  "native_type": column_type,
169
169
  "comment": comment,
170
+ "is_nullable": is_nullable == "YES",
170
171
  }
171
172
 
172
173
  if is_external:
@@ -270,9 +271,10 @@ def migrate_columns(
270
271
  new_comment_escaped = new_comment.replace("\\", "\\\\").replace("'", "\\'")
271
272
 
272
273
  # Generate ALTER TABLE statement
274
+ not_null = "" if col["is_nullable"] else " NOT NULL"
273
275
  sql = (
274
276
  f"ALTER TABLE `{db_name}`.`{table_name}` "
275
- f"MODIFY COLUMN `{col['column']}` {col['native_type']} "
277
+ f"MODIFY COLUMN `{col['column']}` {col['native_type']}{not_null} "
276
278
  f"COMMENT '{new_comment_escaped}'"
277
279
  )
278
280
  result["sql_statements"].append(sql)
@@ -365,7 +367,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
365
367
  for (table_name,) in tables:
366
368
  # Get column information for each table
367
369
  columns_query = """
368
- SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT
370
+ SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT, IS_NULLABLE
369
371
  FROM information_schema.COLUMNS
370
372
  WHERE TABLE_SCHEMA = %s
371
373
  AND TABLE_NAME = %s
@@ -374,7 +376,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
374
376
 
375
377
  columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
376
378
 
377
- for column_name, column_type, comment in columns:
379
+ for column_name, column_type, comment, is_nullable in columns:
378
380
  # Check if comment already has a codec type (starts with :type:)
379
381
  has_codec = comment and comment.startswith(":")
380
382
 
@@ -385,6 +387,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
385
387
  "column_type": column_type,
386
388
  "current_comment": comment or "",
387
389
  "needs_migration": not has_codec,
390
+ "is_nullable": is_nullable == "YES",
388
391
  }
389
392
  )
390
393
 
@@ -447,9 +450,10 @@ def generate_migration_sql(
447
450
  db_name, table_name = col["table_name"].split(".")
448
451
 
449
452
  # Generate ALTER TABLE statement
453
+ not_null = "" if col.get("is_nullable", True) else " NOT NULL"
450
454
  sql = (
451
455
  f"ALTER TABLE `{db_name}`.`{table_name}` "
452
- f"MODIFY COLUMN `{col['column_name']}` {col['column_type']} "
456
+ f"MODIFY COLUMN `{col['column_name']}` {col['column_type']}{not_null} "
453
457
  f"COMMENT '{new_comment_escaped}'"
454
458
  )
455
459
  sql_statements.append(sql)