datajoint 2.3.3__tar.gz → 2.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. {datajoint-2.3.3 → datajoint-2.3.4}/PKG-INFO +1 -1
  2. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/__init__.py +4 -1
  3. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/_version.py +2 -2
  4. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/base.py +0 -23
  5. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/mysql.py +0 -17
  6. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/postgres.py +0 -17
  7. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/autopopulate.py +9 -0
  8. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/__init__.py +2 -2
  9. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/attach.py +5 -3
  10. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/filepath.py +6 -4
  11. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/hash.py +15 -6
  12. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/npy.py +5 -4
  13. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/object.py +5 -4
  14. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/schema.py +48 -17
  15. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/codecs.py +85 -6
  16. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/declare.py +91 -50
  17. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/deploy.py +137 -1
  18. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/heading.py +7 -3
  19. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/migrate.py +36 -21
  20. datajoint-2.3.4/src/datajoint/provenance.py +140 -0
  21. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/schemas.py +3 -3
  22. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/settings.py +46 -0
  23. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/spark.py +2 -2
  24. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/table.py +69 -8
  25. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/user_tables.py +19 -1
  26. {datajoint-2.3.3 → datajoint-2.3.4}/tests/conftest.py +6 -1
  27. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_declare.py +28 -0
  28. datajoint-2.3.4/tests/integration/test_entry_provenance.py +276 -0
  29. datajoint-2.3.4/tests/integration/test_migrate_job_metadata.py +135 -0
  30. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_object.py +12 -12
  31. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_adapters.py +32 -19
  32. datajoint-2.3.4/tests/unit/test_codec_context.py +172 -0
  33. datajoint-2.3.4/tests/unit/test_declare_hidden_attribute.py +146 -0
  34. datajoint-2.3.4/tests/unit/test_provenance.py +193 -0
  35. datajoint-2.3.4/tests/unit/test_tier_aliases.py +80 -0
  36. datajoint-2.3.3/tests/unit/test_declare_hidden_attribute.py +0 -51
  37. {datajoint-2.3.3 → datajoint-2.3.4}/.coveragerc +0 -0
  38. {datajoint-2.3.3 → datajoint-2.3.4}/.dockerignore +0 -0
  39. {datajoint-2.3.3 → datajoint-2.3.4}/.gitattributes +0 -0
  40. {datajoint-2.3.3 → datajoint-2.3.4}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
  41. {datajoint-2.3.3 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  42. {datajoint-2.3.3 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  43. {datajoint-2.3.3 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
  44. {datajoint-2.3.3 → datajoint-2.3.4}/.github/dependabot.yml +0 -0
  45. {datajoint-2.3.3 → datajoint-2.3.4}/.github/release_drafter.yaml +0 -0
  46. {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
  47. {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/draft_release.yaml +0 -0
  48. {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/lint.yaml +0 -0
  49. {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/post_draft_release_published.yaml +0 -0
  50. {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/test.yaml +0 -0
  51. {datajoint-2.3.3 → datajoint-2.3.4}/.gitignore +0 -0
  52. {datajoint-2.3.3 → datajoint-2.3.4}/.pre-commit-config.yaml +0 -0
  53. {datajoint-2.3.3 → datajoint-2.3.4}/CHANGELOG-archive.md +0 -0
  54. {datajoint-2.3.3 → datajoint-2.3.4}/CONTRIBUTING.md +0 -0
  55. {datajoint-2.3.3 → datajoint-2.3.4}/Dockerfile +0 -0
  56. {datajoint-2.3.3 → datajoint-2.3.4}/LICENSE +0 -0
  57. {datajoint-2.3.3 → datajoint-2.3.4}/README.md +0 -0
  58. {datajoint-2.3.3 → datajoint-2.3.4}/RELEASE_MEMO.md +0 -0
  59. {datajoint-2.3.3 → datajoint-2.3.4}/activate.sh +0 -0
  60. {datajoint-2.3.3 → datajoint-2.3.4}/docker-compose.yaml +0 -0
  61. {datajoint-2.3.3 → datajoint-2.3.4}/images/pipeline.svg +0 -0
  62. {datajoint-2.3.3 → datajoint-2.3.4}/pixi.lock +0 -0
  63. {datajoint-2.3.3 → datajoint-2.3.4}/pyproject.toml +0 -0
  64. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/__init__.py +0 -0
  65. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/blob.py +0 -0
  66. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/blob.py +0 -0
  67. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/cli.py +0 -0
  68. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/condition.py +0 -0
  69. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/connection.py +0 -0
  70. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/dependencies.py +0 -0
  71. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/diagram.py +0 -0
  72. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/errors.py +0 -0
  73. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/expression.py +0 -0
  74. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/gc.py +0 -0
  75. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/hash_registry.py +0 -0
  76. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/instance.py +0 -0
  77. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/jobs.py +0 -0
  78. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/lineage.py +0 -0
  79. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/logging.py +0 -0
  80. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/objectref.py +0 -0
  81. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/preview.py +0 -0
  82. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/staged_insert.py +0 -0
  83. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/storage.py +0 -0
  84. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/storage_adapter.py +0 -0
  85. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/types.py +0 -0
  86. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/utils.py +0 -0
  87. {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/version.py +0 -0
  88. {datajoint-2.3.3 → datajoint-2.3.4}/tests/__init__.py +0 -0
  89. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/__init__.py +0 -0
  90. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Course.csv +0 -0
  91. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/CurrentTerm.csv +0 -0
  92. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Department.csv +0 -0
  93. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Enroll.csv +0 -0
  94. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Grade.csv +0 -0
  95. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Section.csv +0 -0
  96. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Student.csv +0 -0
  97. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/StudentMajor.csv +0 -0
  98. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Term.csv +0 -0
  99. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_aggr_regressions.py +0 -0
  100. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_alter.py +0 -0
  101. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_attach.py +0 -0
  102. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_autopopulate.py +0 -0
  103. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_blob.py +0 -0
  104. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_blob_matlab.py +0 -0
  105. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cascade_delete.py +0 -0
  106. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cascade_integrity.py +0 -0
  107. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cascading_delete.py +0 -0
  108. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cli.py +0 -0
  109. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_codec_chaining.py +0 -0
  110. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_codecs.py +0 -0
  111. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_connection.py +0 -0
  112. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_dependencies.py +0 -0
  113. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram.py +0 -0
  114. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_bundle_edge.py +0 -0
  115. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_edge_weight.py +0 -0
  116. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_nocontext.py +0 -0
  117. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_style.py +0 -0
  118. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_fetch.py +0 -0
  119. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_fetch_same.py +0 -0
  120. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_fk_index.py +0 -0
  121. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_foreign_keys.py +0 -0
  122. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_gc.py +0 -0
  123. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_groupby.py +0 -0
  124. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_hash_storage.py +0 -0
  125. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_hidden_job_metadata.py +0 -0
  126. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_insert.py +0 -0
  127. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_jobs.py +0 -0
  128. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_json.py +0 -0
  129. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_multi_backend.py +0 -0
  130. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_nan.py +0 -0
  131. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_npy_codec.py +0 -0
  132. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_privileges.py +0 -0
  133. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_reconnection.py +0 -0
  134. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_relation.py +0 -0
  135. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_relation_u.py +0 -0
  136. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_relational_operand.py +0 -0
  137. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_schema.py +0 -0
  138. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_schema_keywords.py +0 -0
  139. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_semantic_matching.py +0 -0
  140. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_skip_duplicates.py +0 -0
  141. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_tls.py +0 -0
  142. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_trace.py +0 -0
  143. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_type_aliases.py +0 -0
  144. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_university.py +0 -0
  145. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_update1.py +0 -0
  146. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_utils.py +0 -0
  147. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_uuid.py +0 -0
  148. {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_virtual_module.py +0 -0
  149. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema.py +0 -0
  150. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_advanced.py +0 -0
  151. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_aggr_regress.py +0 -0
  152. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_alter.py +0 -0
  153. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_codecs.py +0 -0
  154. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_external.py +0 -0
  155. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_object.py +0 -0
  156. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_privileges.py +0 -0
  157. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_simple.py +0 -0
  158. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_type_aliases.py +0 -0
  159. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_university.py +0 -0
  160. {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_uuid.py +0 -0
  161. {datajoint-2.3.3 → datajoint-2.3.4}/tests/test_package.py +0 -0
  162. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/__init__.py +0 -0
  163. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_codecs.py +0 -0
  164. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_condition.py +0 -0
  165. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_connection_warning.py +0 -0
  166. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_ddl_substitution.py +0 -0
  167. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_deploy.py +0 -0
  168. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_fetch_compat.py +0 -0
  169. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_gc.py +0 -0
  170. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_lazy_imports.py +0 -0
  171. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_logging.py +0 -0
  172. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_pk_rules.py +0 -0
  173. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_settings.py +0 -0
  174. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_spark.py +0 -0
  175. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_storage_adapter.py +0 -0
  176. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_storage_urls.py +0 -0
  177. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_thread_safe.py +0 -0
  178. {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_type_patterns.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: datajoint
3
- Version: 2.3.3
3
+ Version: 2.3.4
4
4
  Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
5
5
  Project-URL: Homepage, https://docs.datajoint.com/
6
6
  Project-URL: Documentation, https://docs.datajoint.com/
@@ -33,9 +33,12 @@ __all__ = [
33
33
  "AutoPopulate",
34
34
  "Job",
35
35
  "Manual",
36
+ "Entry",
36
37
  "Lookup",
37
38
  "Imported",
39
+ "Ingest",
38
40
  "Computed",
41
+ "Compute",
39
42
  "Part",
40
43
  "Not",
41
44
  "AndList",
@@ -95,7 +98,7 @@ from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
95
98
  from .autopopulate import AutoPopulate
96
99
  from .jobs import Job
97
100
  from .table import FreeTable as _FreeTable, Table, ValidationResult
98
- from .user_tables import Computed, Imported, Lookup, Manual, Part
101
+ from .user_tables import Compute, Computed, Entry, Imported, Ingest, Lookup, Manual, Part
99
102
  from .version import __version__
100
103
 
101
104
  # =============================================================================
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '2.3.3'
22
- __version_tuple__ = version_tuple = (2, 3, 3)
21
+ __version__ = version = '2.3.4'
22
+ __version_tuple__ = version_tuple = (2, 3, 4)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -1317,29 +1317,6 @@ class DatabaseAdapter(ABC):
1317
1317
  """
1318
1318
  ...
1319
1319
 
1320
- @abstractmethod
1321
- def job_metadata_columns(self) -> list[str]:
1322
- """
1323
- Return job metadata column definitions for Computed/Imported tables.
1324
-
1325
- Returns
1326
- -------
1327
- list[str]
1328
- List of column definition strings (fully formatted with quotes).
1329
-
1330
- Examples
1331
- --------
1332
- MySQL:
1333
- ["`_job_start_time` datetime(3) DEFAULT NULL",
1334
- "`_job_duration` float DEFAULT NULL",
1335
- "`_job_version` varchar(64) DEFAULT ''"]
1336
- PostgreSQL:
1337
- ['"_job_start_time" timestamp DEFAULT NULL',
1338
- '"_job_duration" real DEFAULT NULL',
1339
- '"_job_version" varchar(64) DEFAULT \'\'']
1340
- """
1341
- ...
1342
-
1343
1320
  # =========================================================================
1344
1321
  # Error Translation
1345
1322
  # =========================================================================
@@ -996,23 +996,6 @@ class MySQLAdapter(DatabaseAdapter):
996
996
  """
997
997
  return None # MySQL uses inline enum
998
998
 
999
- def job_metadata_columns(self) -> list[str]:
1000
- """
1001
- Return MySQL-specific job metadata column definitions.
1002
-
1003
- Examples
1004
- --------
1005
- >>> adapter.job_metadata_columns()
1006
- ["`_job_start_time` datetime(3) DEFAULT NULL",
1007
- "`_job_duration` float DEFAULT NULL",
1008
- "`_job_version` varchar(64) DEFAULT ''"]
1009
- """
1010
- return [
1011
- "`_job_start_time` datetime(3) DEFAULT NULL",
1012
- "`_job_duration` float DEFAULT NULL",
1013
- "`_job_version` varchar(64) DEFAULT ''",
1014
- ]
1015
-
1016
999
  # =========================================================================
1017
1000
  # Error Translation
1018
1001
  # =========================================================================
@@ -1349,23 +1349,6 @@ class PostgreSQLAdapter(DatabaseAdapter):
1349
1349
  self._pending_enum_types = {}
1350
1350
  return ddl_statements
1351
1351
 
1352
- def job_metadata_columns(self) -> list[str]:
1353
- """
1354
- Return PostgreSQL-specific job metadata column definitions.
1355
-
1356
- Examples
1357
- --------
1358
- >>> adapter.job_metadata_columns()
1359
- ['"_job_start_time" timestamp DEFAULT NULL',
1360
- '"_job_duration" real DEFAULT NULL',
1361
- '"_job_version" varchar(64) DEFAULT \\'\\'']
1362
- """
1363
- return [
1364
- '"_job_start_time" timestamp DEFAULT NULL',
1365
- '"_job_duration" real DEFAULT NULL',
1366
- "\"_job_version\" varchar(64) DEFAULT ''",
1367
- ]
1368
-
1369
1352
  # =========================================================================
1370
1353
  # Error Translation
1371
1354
  # =========================================================================
@@ -11,6 +11,7 @@ import signal
11
11
  import traceback
12
12
  from typing import TYPE_CHECKING, Any, Generator
13
13
 
14
+ from . import provenance
14
15
  from .errors import DataJointError, LostConnectionError
15
16
  from .expression import AndList, QueryExpression
16
17
 
@@ -684,6 +685,13 @@ class AutoPopulate:
684
685
  self._upstream_key = dict(key)
685
686
  self._upstream = None
686
687
 
688
+ # Rows this make() writes into Entry tables that carry no foreign key back
689
+ # here -- the fan-out ingestion pattern -- record the ingesting table and
690
+ # key, which is what makes such a row traceable without one.
691
+ from .jobs import _get_job_version
692
+
693
+ prov_token = provenance.set_ingesting(self.full_table_name, key, _get_job_version(self.connection._config))
694
+
687
695
  try:
688
696
  if not is_generator:
689
697
  make(dict(key), **(make_kwargs or {}))
@@ -740,6 +748,7 @@ class AutoPopulate:
740
748
  jobs.complete(key, duration=duration)
741
749
  return True
742
750
  finally:
751
+ provenance.reset_ingesting(prov_token)
743
752
  self.__class__._allow_insert = False
744
753
  # Clear the per-make() upstream state: `_upstream = None` invalidates
745
754
  # the memoized Diagram; `_upstream_key = None` restores the "outside
@@ -29,14 +29,14 @@ Example - Creating a Custom Codec:
29
29
  def get_dtype(self, is_store: bool) -> str:
30
30
  return "<blob>" # Compose with blob for serialization
31
31
 
32
- def encode(self, graph, *, key=None, store_name=None):
32
+ def encode(self, graph, *, key=None, context=None, store_name=None):
33
33
  # Convert graph to a serializable format
34
34
  return {
35
35
  'nodes': list(graph.nodes(data=True)),
36
36
  'edges': list(graph.edges(data=True)),
37
37
  }
38
38
 
39
- def decode(self, stored, *, key=None):
39
+ def decode(self, stored, *, key=None, context=None):
40
40
  # Reconstruct graph from stored format
41
41
  G = nx.Graph()
42
42
  G.add_nodes_from(stored['nodes'])
@@ -50,7 +50,9 @@ class AttachCodec(Codec):
50
50
  """Return bytes for in-table, <hash> for in-store storage."""
51
51
  return "<hash>" if is_store else "bytes"
52
52
 
53
- def encode(self, value: Any, *, key: dict | None = None, store_name: str | None = None) -> bytes:
53
+ def encode(
54
+ self, value: Any, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
55
+ ) -> bytes:
54
56
  """
55
57
  Read file and encode as filename + contents.
56
58
 
@@ -80,7 +82,7 @@ class AttachCodec(Codec):
80
82
  contents = path.read_bytes()
81
83
  return filename.encode("utf-8") + b"\x00" + contents
82
84
 
83
- def decode(self, stored: bytes, *, key: dict | None = None) -> str:
85
+ def decode(self, stored: bytes, *, key: dict | None = None, context: dict | None = None) -> str:
84
86
  """
85
87
  Extract file to download path and return local path.
86
88
 
@@ -104,7 +106,7 @@ class AttachCodec(Codec):
104
106
  contents = stored[null_pos + 1 :]
105
107
 
106
108
  # Write to download path
107
- config = (key or {}).get("_config")
109
+ config = self._codec_config(key, context)
108
110
  if config is None:
109
111
  from ..settings import config # type: ignore[assignment]
110
112
  assert config is not None
@@ -74,7 +74,9 @@ class FilepathCodec(Codec):
74
74
  )
75
75
  return "json"
76
76
 
77
- def encode(self, value: Any, *, key: dict | None = None, store_name: str | None = None) -> dict:
77
+ def encode(
78
+ self, value: Any, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
79
+ ) -> dict:
78
80
  """
79
81
  Store path reference as JSON metadata.
80
82
 
@@ -104,7 +106,7 @@ class FilepathCodec(Codec):
104
106
 
105
107
  from ..hash_registry import get_store_backend
106
108
 
107
- config = (key or {}).get("_config")
109
+ config = self._codec_config(key, context)
108
110
  if config is None:
109
111
  from ..settings import config # type: ignore[assignment]
110
112
  assert config is not None
@@ -168,7 +170,7 @@ class FilepathCodec(Codec):
168
170
  "timestamp": datetime.now(timezone.utc).isoformat(),
169
171
  }
170
172
 
171
- def decode(self, stored: dict, *, key: dict | None = None) -> Any:
173
+ def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> Any:
172
174
  """
173
175
  Create ObjectRef handle for lazy access.
174
176
 
@@ -187,7 +189,7 @@ class FilepathCodec(Codec):
187
189
  from ..objectref import ObjectRef
188
190
  from ..hash_registry import get_store_backend
189
191
 
190
- config = (key or {}).get("_config")
192
+ config = self._codec_config(key, context)
191
193
  store_name = stored.get("store")
192
194
  backend = get_store_backend(store_name, config=config)
193
195
  return ObjectRef.from_json(stored, backend=backend)
@@ -57,7 +57,9 @@ class HashCodec(Codec):
57
57
  raise DataJointError("<hash> requires @ (in-store storage only)")
58
58
  return "json"
59
59
 
60
- def encode(self, value: bytes, *, key: dict | None = None, store_name: str | None = None) -> dict:
60
+ def encode(
61
+ self, value: bytes, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
62
+ ) -> dict:
61
63
  """
62
64
  Store content and return metadata.
63
65
 
@@ -66,7 +68,11 @@ class HashCodec(Codec):
66
68
  value : bytes
67
69
  Raw bytes to store.
68
70
  key : dict, optional
69
- Context dict with ``_schema`` for path isolation.
71
+ Primary key values. Before 2.3.4 this also carried the schema name
72
+ under ``_schema``; that is still read when ``context`` omits it.
73
+ context : dict, optional
74
+ Connection context. ``schema`` isolates the stored path and
75
+ ``config`` resolves the store.
70
76
  store_name : str, optional
71
77
  Store to use. If None, uses default store.
72
78
 
@@ -77,11 +83,14 @@ class HashCodec(Codec):
77
83
  """
78
84
  from ..hash_registry import put_hash
79
85
 
80
- schema_name = (key or {}).get("_schema", "unknown")
81
- config = (key or {}).get("_config")
86
+ # Same precedence as `_extract_context`. Reading `key` alone would store
87
+ # under "unknown" for any caller that passes `context` with a key holding
88
+ # only primary-key values -- which is every caller once 2.4 lands.
89
+ schema_name = (context or {}).get("schema", (key or {}).get("_schema", "unknown"))
90
+ config = self._codec_config(key, context)
82
91
  return put_hash(value, schema_name=schema_name, store_name=store_name, config=config)
83
92
 
84
- def decode(self, stored: dict, *, key: dict | None = None) -> bytes:
93
+ def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> bytes:
85
94
  """
86
95
  Retrieve content using stored metadata.
87
96
 
@@ -99,7 +108,7 @@ class HashCodec(Codec):
99
108
  """
100
109
  from ..hash_registry import get_hash
101
110
 
102
- config = (key or {}).get("_config")
111
+ config = self._codec_config(key, context)
103
112
  return get_hash(stored, config=config)
104
113
 
105
114
  def validate(self, value: Any) -> None:
@@ -312,6 +312,7 @@ class NpyCodec(SchemaCodec):
312
312
  value: Any,
313
313
  *,
314
314
  key: dict | None = None,
315
+ context: dict | None = None,
315
316
  store_name: str | None = None,
316
317
  ) -> dict:
317
318
  """
@@ -337,8 +338,8 @@ class NpyCodec(SchemaCodec):
337
338
  import numpy as np
338
339
 
339
340
  # Extract context using inherited helper
340
- schema, table, field, primary_key = self._extract_context(key)
341
- config = (key or {}).get("_config")
341
+ schema, table, field, primary_key = self._extract_context(key, context)
342
+ config = self._codec_config(key, context)
342
343
 
343
344
  # Build schema-addressed storage path
344
345
  path, _ = self._build_path(schema, table, field, primary_key, ext=".npy", store_name=store_name, config=config)
@@ -360,7 +361,7 @@ class NpyCodec(SchemaCodec):
360
361
  "shape": list(value.shape),
361
362
  }
362
363
 
363
- def decode(self, stored: dict, *, key: dict | None = None) -> NpyRef:
364
+ def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> NpyRef:
364
365
  """
365
366
  Create lazy NpyRef from stored metadata.
366
367
 
@@ -376,6 +377,6 @@ class NpyCodec(SchemaCodec):
376
377
  NpyRef
377
378
  Lazy array reference with metadata access and numpy integration.
378
379
  """
379
- config = (key or {}).get("_config")
380
+ config = self._codec_config(key, context)
380
381
  backend = self._get_backend(stored.get("store"), config=config)
381
382
  return NpyRef(stored, backend)
@@ -81,6 +81,7 @@ class ObjectCodec(SchemaCodec):
81
81
  value: Any,
82
82
  *,
83
83
  key: dict | None = None,
84
+ context: dict | None = None,
84
85
  store_name: str | None = None,
85
86
  ) -> dict:
86
87
  """
@@ -105,8 +106,8 @@ class ObjectCodec(SchemaCodec):
105
106
  from pathlib import Path
106
107
 
107
108
  # Extract context using inherited helper
108
- schema, table, field, primary_key = self._extract_context(key)
109
- config = (key or {}).get("_config")
109
+ schema, table, field, primary_key = self._extract_context(key, context)
110
+ config = self._codec_config(key, context)
110
111
 
111
112
  # Check for pre-computed metadata (from staged insert)
112
113
  if isinstance(value, dict) and "path" in value:
@@ -177,7 +178,7 @@ class ObjectCodec(SchemaCodec):
177
178
 
178
179
  return metadata
179
180
 
180
- def decode(self, stored: dict, *, key: dict | None = None) -> Any:
181
+ def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> Any:
181
182
  """
182
183
  Create ObjectRef handle for lazy access.
183
184
 
@@ -195,7 +196,7 @@ class ObjectCodec(SchemaCodec):
195
196
  """
196
197
  from ..objectref import ObjectRef
197
198
 
198
- config = (key or {}).get("_config")
199
+ config = self._codec_config(key, context)
199
200
  backend = self._get_backend(stored.get("store"), config=config)
200
201
  return ObjectRef.from_json(stored, backend=backend)
201
202
 
@@ -4,6 +4,8 @@ Schema-addressed storage base class.
4
4
 
5
5
  from __future__ import annotations
6
6
 
7
+ import warnings
8
+
7
9
  from ..codecs import Codec
8
10
  from ..errors import DataJointError
9
11
 
@@ -24,14 +26,27 @@ class SchemaCodec(Codec, register=False):
24
26
  - ``validate()``: Validate input values
25
27
 
26
28
  Helper Methods:
27
- - ``_extract_context()``: Parse key dict into schema/table/field/pk
29
+ - ``_extract_context()``: Parse key/context into schema/table/field/pk
30
+ - ``_codec_config()``: Read the calling connection's config
28
31
  - ``_build_path()``: Construct storage path from context
29
32
  - ``_get_backend()``: Get storage backend by name
30
33
 
31
- Both helpers take a ``config`` and fall back to the global ``dj.config``
32
- without one. Read it off ``key["_config"]`` and pass it through, as below: it
33
- is the calling connection's config, and in a process holding connections for
34
- several users the global one belongs to none of them.
34
+ ``_build_path`` and ``_get_backend`` take a ``config`` and fall back to the
35
+ global ``dj.config`` without one. Always pass the calling connection's
36
+ config: in a process holding connections for several users, the global one
37
+ belongs to none of them, and the fallback resolves a different store
38
+ silently rather than raising.
39
+
40
+ Since 2.3.4 that config arrives in an explicit ``context`` argument rather
41
+ than hidden among the primary key values. **Declare ``context=None`` in
42
+ ``encode`` and ``decode`` and pass it to the helpers**, as the example below
43
+ does.
44
+
45
+ The old underscore keys in ``key`` still work and are still populated, so a
46
+ codec written before 2.3.4 keeps running. They are removed in 2.4.
47
+ ``_extract_context`` warns when it falls back to them; ``_codec_config`` and
48
+ a direct ``key["_config"]`` do not, so the warning is a prompt rather than a
49
+ complete inventory of what still reads them.
35
50
 
36
51
  Comparison with Hash-addressed:
37
52
  - **Schema-addressed** (this): Path from schema structure, no dedup
@@ -42,9 +57,9 @@ class SchemaCodec(Codec, register=False):
42
57
  class MyCodec(SchemaCodec):
43
58
  name = "my"
44
59
 
45
- def encode(self, value, *, key=None, store_name=None):
46
- schema, table, field, pk = self._extract_context(key)
47
- config = (key or {}).get("_config")
60
+ def encode(self, value, *, key=None, context=None, store_name=None):
61
+ schema, table, field, pk = self._extract_context(key, context)
62
+ config = self._codec_config(key, context)
48
63
  path, _ = self._build_path(
49
64
  schema, table, field, pk, ext=".dat",
50
65
  store_name=store_name, config=config,
@@ -53,8 +68,8 @@ class SchemaCodec(Codec, register=False):
53
68
  backend.put_buffer(serialize(value), path)
54
69
  return {"path": path, "store": store_name, ...}
55
70
 
56
- def decode(self, stored, *, key=None):
57
- config = (key or {}).get("_config")
71
+ def decode(self, stored, *, key=None, context=None):
72
+ config = self._codec_config(key, context)
58
73
  backend = self._get_backend(stored.get("store"), config=config)
59
74
  return MyRef(stored, backend)
60
75
 
@@ -88,15 +103,21 @@ class SchemaCodec(Codec, register=False):
88
103
  raise DataJointError(f"<{self.name}> requires @ (store only)")
89
104
  return "json"
90
105
 
91
- def _extract_context(self, key: dict | None) -> tuple[str, str, str, dict]:
106
+ def _extract_context(self, key: dict | None, context: dict | None = None) -> tuple[str, str, str, dict]:
92
107
  """
93
- Extract schema, table, field, and primary key from context dict.
108
+ Extract schema, table, field, and primary key.
94
109
 
95
110
  Parameters
96
111
  ----------
97
112
  key : dict or None
98
- Context dict with ``_schema``, ``_table``, ``_field``,
99
- and primary key values.
113
+ Primary key values. Before 2.3.4 this also carried connection
114
+ context under ``_schema``, ``_table``, ``_field`` and ``_config``;
115
+ those keys are still populated and still read, with a
116
+ ``DeprecationWarning``, when ``context`` is not supplied.
117
+ context : dict or None
118
+ Connection context with ``schema``, ``table``, ``field`` and
119
+ ``config``. Pass the ``context`` argument your ``encode``/``decode``
120
+ received.
100
121
 
101
122
  Returns
102
123
  -------
@@ -104,9 +125,19 @@ class SchemaCodec(Codec, register=False):
104
125
  ``(schema, table, field, primary_key)``
105
126
  """
106
127
  key = dict(key) if key else {}
107
- schema = key.pop("_schema", "unknown")
108
- table = key.pop("_table", "unknown")
109
- field = key.pop("_field", "data")
128
+ if context is None and any(k.startswith("_") for k in key):
129
+ warnings.warn(
130
+ "Reading connection context from the `key` dict is deprecated and will "
131
+ "be removed in DataJoint 2.4. Accept a `context` argument in encode()/decode() "
132
+ "and pass it to _extract_context(key, context). See "
133
+ "https://github.com/datajoint/datajoint-python/issues/1550",
134
+ DeprecationWarning,
135
+ stacklevel=2,
136
+ )
137
+ context = context or {}
138
+ schema = context.get("schema", key.pop("_schema", "unknown"))
139
+ table = context.get("table", key.pop("_table", "unknown"))
140
+ field = context.get("field", key.pop("_field", "data"))
110
141
  primary_key = {k: v for k, v in key.items() if not k.startswith("_")}
111
142
  return schema, table, field, primary_key
112
143
 
@@ -16,10 +16,10 @@ class GraphCodec(dj.Codec):
16
16
  def get_dtype(self, is_store: bool) -> str:
17
17
  return "<blob>"
18
18
 
19
- def encode(self, graph, *, key=None, store_name=None):
19
+ def encode(self, graph, *, key=None, context=None, store_name=None):
20
20
  return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())}
21
21
 
22
- def decode(self, stored, *, key=None):
22
+ def decode(self, stored, *, key=None, context=None):
23
23
  import networkx as nx
24
24
  G = nx.Graph()
25
25
  G.add_nodes_from(stored['nodes'])
@@ -38,6 +38,8 @@ class MyTable(dj.Manual):
38
38
 
39
39
  from __future__ import annotations
40
40
 
41
+ import functools
42
+ import inspect
41
43
  import json
42
44
  import logging
43
45
  from abc import ABC, abstractmethod
@@ -79,10 +81,10 @@ class Codec(ABC):
79
81
  ... def get_dtype(self, is_store: bool) -> str:
80
82
  ... return "<blob>"
81
83
  ...
82
- ... def encode(self, graph, *, key=None, store_name=None):
84
+ ... def encode(self, graph, *, key=None, context=None, store_name=None):
83
85
  ... return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())}
84
86
  ...
85
- ... def decode(self, stored, *, key=None):
87
+ ... def decode(self, stored, *, key=None, context=None):
86
88
  ... import networkx as nx
87
89
  ... G = nx.Graph()
88
90
  ... G.add_nodes_from(stored['nodes'])
@@ -176,6 +178,18 @@ class Codec(ABC):
176
178
  -------
177
179
  any
178
180
  Value in the format expected by the dtype.
181
+
182
+ Notes
183
+ -----
184
+ **Declare ``context`` as well**: ``encode(self, value, *, key=None,
185
+ context=None, store_name=None)``. It carries ``schema``, ``table``,
186
+ ``field`` and ``config`` — the calling connection's configuration, which
187
+ :meth:`_codec_config` reads and which store resolution needs.
188
+
189
+ Before 2.3.4 those four arrived inside ``key`` under underscore-prefixed
190
+ names. That path still works and DataJoint still populates it, so a codec
191
+ written against it keeps running — but it is deprecated and removed in
192
+ 2.4. Write new codecs against ``context``.
179
193
  """
180
194
  ...
181
195
 
@@ -195,9 +209,45 @@ class Codec(ABC):
195
209
  -------
196
210
  any
197
211
  The reconstructed Python object.
212
+
213
+ Notes
214
+ -----
215
+ **Declare ``context`` as well**: ``decode(self, stored, *, key=None,
216
+ context=None)``. On this path it carries ``config`` only — the stored
217
+ metadata already holds the location, so ``schema``, ``table`` and
218
+ ``field`` are not resolved again. Pass it to :meth:`_codec_config` for
219
+ the same reason :meth:`encode` does.
198
220
  """
199
221
  ...
200
222
 
223
+ @staticmethod
224
+ def _codec_config(key: dict | None = None, context: dict | None = None):
225
+ """
226
+ Return the calling connection's config, or None.
227
+
228
+ Always thread the result into ``_build_path`` and ``_get_backend``. Those
229
+ helpers fall back to the global ``dj.config`` without it, which in a
230
+ process holding connections for several users belongs to none of them and
231
+ resolves a different store silently rather than raising.
232
+
233
+ Prefers ``context["config"]``. Falls back to ``key["_config"]``, the
234
+ pre-2.3.4 location, which DataJoint still populates.
235
+
236
+ Parameters
237
+ ----------
238
+ key : dict, optional
239
+ The ``key`` argument the codec received.
240
+ context : dict, optional
241
+ The ``context`` argument the codec received, if it declares one.
242
+
243
+ Returns
244
+ -------
245
+ Config or None
246
+ """
247
+ if context and context.get("config") is not None:
248
+ return context["config"]
249
+ return (key or {}).get("_config")
250
+
201
251
  def validate(self, value: Any) -> None:
202
252
  """
203
253
  Validate a value before encoding.
@@ -562,6 +612,28 @@ def lookup_codec(codec_spec: str) -> tuple[Codec, str | None]:
562
612
  # =============================================================================
563
613
 
564
614
 
615
+ @functools.lru_cache(maxsize=None)
616
+ def _accepts_kwarg(func, name: str) -> bool:
617
+ """
618
+ Whether ``func`` declares a keyword parameter ``name``.
619
+
620
+ Used to offer optional arguments -- ``store_name``, ``context`` -- only to
621
+ codecs whose signature declares them, so a codec written before either
622
+ existed is called exactly as it was.
623
+
624
+ Cached on the underlying function: ``inspect.signature`` costs several
625
+ times more than a small ``encode`` call, and this runs per attribute per
626
+ row. Pass the unbound function (``type(codec).encode``), which is stable
627
+ per class, rather than a bound method, which is not.
628
+
629
+ Introspection rather than calling and catching ``TypeError``: an ``encode``
630
+ body serializes and uploads, so a ``TypeError`` raised inside it would be
631
+ indistinguishable from an unexpected-keyword error at the call site, and
632
+ retrying would both mask the real failure and repeat the upload.
633
+ """
634
+ return name in inspect.signature(func).parameters
635
+
636
+
565
637
  def decode_attribute(attr, data, squeeze: bool = False, connection=None):
566
638
  """
567
639
  Decode raw database value using attribute's codec or native type handling.
@@ -617,14 +689,21 @@ def decode_attribute(attr, data, squeeze: bool = False, connection=None):
617
689
  elif final_dtype.lower() == "binary(16)":
618
690
  data = uuid_module.UUID(bytes=data)
619
691
 
620
- # Build decode key with config if connection is available
692
+ # Build decode key with config if connection is available. The
693
+ # underscore key stays for codecs written against it; `context` carries
694
+ # the same config to codecs that declare one -- see #1550.
621
695
  decode_key = None
696
+ decode_context = None
622
697
  if connection is not None:
623
698
  decode_key = {"_config": connection._config}
699
+ decode_context = {"config": connection._config}
624
700
 
625
701
  # Apply decoders in reverse order: innermost first, then outermost
626
702
  for codec in reversed(type_chain):
627
- data = codec.decode(data, key=decode_key)
703
+ if _accepts_kwarg(type(codec).decode, "context"):
704
+ data = codec.decode(data, key=decode_key, context=decode_context)
705
+ else:
706
+ data = codec.decode(data, key=decode_key)
628
707
 
629
708
  # Squeeze arrays if requested
630
709
  if squeeze and isinstance(data, np.ndarray):