datajoint 2.2.3__tar.gz → 2.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. {datajoint-2.2.3 → datajoint-2.3.0}/PKG-INFO +2 -1
  2. {datajoint-2.2.3 → datajoint-2.3.0}/pyproject.toml +1 -0
  3. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/__init__.py +9 -0
  4. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/adapters/base.py +23 -0
  5. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/adapters/mysql.py +10 -0
  6. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/adapters/postgres.py +39 -0
  7. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/autopopulate.py +84 -0
  8. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/condition.py +14 -0
  9. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/dependencies.py +29 -0
  10. datajoint-2.3.0/src/datajoint/deploy.py +185 -0
  11. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/diagram.py +451 -21
  12. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/expression.py +6 -0
  13. datajoint-2.3.0/src/datajoint/provenance.py +206 -0
  14. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/schemas.py +17 -0
  15. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/settings.py +15 -4
  16. datajoint-2.3.0/src/datajoint/spark.py +92 -0
  17. datajoint-2.3.0/src/datajoint/staged_insert.py +274 -0
  18. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/table.py +62 -2
  19. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/version.py +1 -1
  20. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_autopopulate.py +239 -0
  21. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_cascade_delete.py +187 -0
  22. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_object.py +98 -0
  23. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_semantic_matching.py +150 -0
  24. datajoint-2.3.0/tests/integration/test_strict_provenance.py +322 -0
  25. datajoint-2.3.0/tests/integration/test_trace.py +391 -0
  26. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_adapters.py +17 -0
  27. datajoint-2.3.0/tests/unit/test_deploy.py +172 -0
  28. datajoint-2.3.0/tests/unit/test_spark.py +105 -0
  29. datajoint-2.2.3/src/datajoint/staged_insert.py +0 -310
  30. {datajoint-2.2.3 → datajoint-2.3.0}/.coveragerc +0 -0
  31. {datajoint-2.2.3 → datajoint-2.3.0}/.dockerignore +0 -0
  32. {datajoint-2.2.3 → datajoint-2.3.0}/.gitattributes +0 -0
  33. {datajoint-2.2.3 → datajoint-2.3.0}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
  34. {datajoint-2.2.3 → datajoint-2.3.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  35. {datajoint-2.2.3 → datajoint-2.3.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  36. {datajoint-2.2.3 → datajoint-2.3.0}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
  37. {datajoint-2.2.3 → datajoint-2.3.0}/.github/dependabot.yml +0 -0
  38. {datajoint-2.2.3 → datajoint-2.3.0}/.github/release_drafter.yaml +0 -0
  39. {datajoint-2.2.3 → datajoint-2.3.0}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
  40. {datajoint-2.2.3 → datajoint-2.3.0}/.github/workflows/draft_release.yaml +0 -0
  41. {datajoint-2.2.3 → datajoint-2.3.0}/.github/workflows/lint.yaml +0 -0
  42. {datajoint-2.2.3 → datajoint-2.3.0}/.github/workflows/post_draft_release_published.yaml +0 -0
  43. {datajoint-2.2.3 → datajoint-2.3.0}/.github/workflows/test.yaml +0 -0
  44. {datajoint-2.2.3 → datajoint-2.3.0}/.gitignore +0 -0
  45. {datajoint-2.2.3 → datajoint-2.3.0}/.pre-commit-config.yaml +0 -0
  46. {datajoint-2.2.3 → datajoint-2.3.0}/CHANGELOG-archive.md +0 -0
  47. {datajoint-2.2.3 → datajoint-2.3.0}/CONTRIBUTING.md +0 -0
  48. {datajoint-2.2.3 → datajoint-2.3.0}/Dockerfile +0 -0
  49. {datajoint-2.2.3 → datajoint-2.3.0}/LICENSE +0 -0
  50. {datajoint-2.2.3 → datajoint-2.3.0}/README.md +0 -0
  51. {datajoint-2.2.3 → datajoint-2.3.0}/RELEASE_MEMO.md +0 -0
  52. {datajoint-2.2.3 → datajoint-2.3.0}/activate.sh +0 -0
  53. {datajoint-2.2.3 → datajoint-2.3.0}/docker-compose.yaml +0 -0
  54. {datajoint-2.2.3 → datajoint-2.3.0}/images/pipeline.png +0 -0
  55. {datajoint-2.2.3 → datajoint-2.3.0}/images/pipeline.svg +0 -0
  56. {datajoint-2.2.3 → datajoint-2.3.0}/pixi.lock +0 -0
  57. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/adapters/__init__.py +0 -0
  58. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/blob.py +0 -0
  59. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/__init__.py +0 -0
  60. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/attach.py +0 -0
  61. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/blob.py +0 -0
  62. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/filepath.py +0 -0
  63. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/hash.py +0 -0
  64. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/npy.py +0 -0
  65. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/object.py +0 -0
  66. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/builtin_codecs/schema.py +0 -0
  67. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/cli.py +0 -0
  68. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/codecs.py +0 -0
  69. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/connection.py +0 -0
  70. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/declare.py +0 -0
  71. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/errors.py +0 -0
  72. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/gc.py +0 -0
  73. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/hash_registry.py +0 -0
  74. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/heading.py +0 -0
  75. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/instance.py +0 -0
  76. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/jobs.py +0 -0
  77. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/lineage.py +0 -0
  78. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/logging.py +0 -0
  79. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/migrate.py +0 -0
  80. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/objectref.py +0 -0
  81. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/preview.py +0 -0
  82. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/storage.py +0 -0
  83. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/storage_adapter.py +0 -0
  84. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/types.py +0 -0
  85. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/user_tables.py +0 -0
  86. {datajoint-2.2.3 → datajoint-2.3.0}/src/datajoint/utils.py +0 -0
  87. {datajoint-2.2.3 → datajoint-2.3.0}/tests/__init__.py +0 -0
  88. {datajoint-2.2.3 → datajoint-2.3.0}/tests/conftest.py +0 -0
  89. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/__init__.py +0 -0
  90. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/Course.csv +0 -0
  91. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/CurrentTerm.csv +0 -0
  92. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/Department.csv +0 -0
  93. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/Enroll.csv +0 -0
  94. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/Grade.csv +0 -0
  95. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/Section.csv +0 -0
  96. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/Student.csv +0 -0
  97. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/StudentMajor.csv +0 -0
  98. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/data/Term.csv +0 -0
  99. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_aggr_regressions.py +0 -0
  100. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_alter.py +0 -0
  101. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_attach.py +0 -0
  102. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_blob.py +0 -0
  103. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_blob_matlab.py +0 -0
  104. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_cascading_delete.py +0 -0
  105. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_cli.py +0 -0
  106. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_codec_chaining.py +0 -0
  107. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_codecs.py +0 -0
  108. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_connection.py +0 -0
  109. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_declare.py +0 -0
  110. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_dependencies.py +0 -0
  111. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_erd.py +0 -0
  112. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_fetch.py +0 -0
  113. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_fetch_same.py +0 -0
  114. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_foreign_keys.py +0 -0
  115. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_gc.py +0 -0
  116. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_groupby.py +0 -0
  117. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_hash_storage.py +0 -0
  118. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_hidden_job_metadata.py +0 -0
  119. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_insert.py +0 -0
  120. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_jobs.py +0 -0
  121. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_json.py +0 -0
  122. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_multi_backend.py +0 -0
  123. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_nan.py +0 -0
  124. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_npy_codec.py +0 -0
  125. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_privileges.py +0 -0
  126. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_reconnection.py +0 -0
  127. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_relation.py +0 -0
  128. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_relation_u.py +0 -0
  129. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_relational_operand.py +0 -0
  130. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_schema.py +0 -0
  131. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_schema_keywords.py +0 -0
  132. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_skip_duplicates.py +0 -0
  133. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_tls.py +0 -0
  134. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_type_aliases.py +0 -0
  135. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_university.py +0 -0
  136. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_update1.py +0 -0
  137. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_utils.py +0 -0
  138. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_uuid.py +0 -0
  139. {datajoint-2.2.3 → datajoint-2.3.0}/tests/integration/test_virtual_module.py +0 -0
  140. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema.py +0 -0
  141. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_advanced.py +0 -0
  142. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_aggr_regress.py +0 -0
  143. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_alter.py +0 -0
  144. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_codecs.py +0 -0
  145. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_external.py +0 -0
  146. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_object.py +0 -0
  147. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_privileges.py +0 -0
  148. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_simple.py +0 -0
  149. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_type_aliases.py +0 -0
  150. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_university.py +0 -0
  151. {datajoint-2.2.3 → datajoint-2.3.0}/tests/schema_uuid.py +0 -0
  152. {datajoint-2.2.3 → datajoint-2.3.0}/tests/test_package.py +0 -0
  153. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/__init__.py +0 -0
  154. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_codecs.py +0 -0
  155. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_condition.py +0 -0
  156. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_connection_warning.py +0 -0
  157. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_declare_hidden_attribute.py +0 -0
  158. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_fetch_compat.py +0 -0
  159. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_lazy_imports.py +0 -0
  160. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_pk_rules.py +0 -0
  161. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_settings.py +0 -0
  162. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_storage_adapter.py +0 -0
  163. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_storage_urls.py +0 -0
  164. {datajoint-2.2.3 → datajoint-2.3.0}/tests/unit/test_thread_safe.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datajoint
3
- Version: 2.2.3
3
+ Version: 2.3.0
4
4
  Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
5
5
  Project-URL: Homepage, https://docs.datajoint.com/
6
6
  Project-URL: Documentation, https://docs.datajoint.com/
@@ -216,6 +216,7 @@ Requires-Dist: deepdiff
216
216
  Requires-Dist: fsspec>=2023.1.0
217
217
  Requires-Dist: networkx
218
218
  Requires-Dist: numpy
219
+ Requires-Dist: packaging
219
220
  Requires-Dist: pandas
220
221
  Requires-Dist: pydantic-settings>=2.0.0
221
222
  Requires-Dist: pydot
@@ -16,6 +16,7 @@ dependencies = [
16
16
  "pydot",
17
17
  "fsspec>=2023.1.0",
18
18
  "pydantic-settings>=2.0.0",
19
+ "packaging",
19
20
  ]
20
21
 
21
22
  requires-python = ">=3.10,<3.14"
@@ -51,9 +51,15 @@ __all__ = [
51
51
  "get_codec",
52
52
  "ObjectRef",
53
53
  "NpyRef",
54
+ # SparkAdapter Codec Protocol
55
+ "SparkAdapter",
56
+ # Storage Adapter API
57
+ "StorageAdapter",
58
+ "get_storage_adapter",
54
59
  # Other
55
60
  "errors",
56
61
  "migrate",
62
+ "deploy",
57
63
  "DataJointError",
58
64
  "ThreadSafetyError",
59
65
  "logger",
@@ -66,6 +72,7 @@ __all__ = [
66
72
  # =============================================================================
67
73
  from . import errors
68
74
  from . import migrate
75
+ from . import deploy
69
76
  from .codecs import (
70
77
  Codec,
71
78
  get_codec,
@@ -82,6 +89,8 @@ from .expression import AndList, Not, Top, U
82
89
  from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global_config, _check_thread_safe
83
90
  from .logging import logger
84
91
  from .objectref import ObjectRef
92
+ from .spark import SparkAdapter
93
+ from .storage_adapter import StorageAdapter, get_storage_adapter
85
94
  from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
86
95
  from .autopopulate import AutoPopulate
87
96
  from .jobs import Job
@@ -850,6 +850,29 @@ class DatabaseAdapter(ABC):
850
850
  raise NotImplementedError
851
851
  ...
852
852
 
853
+ def find_upstream_schemas_sql(self, schemas_list: str) -> str:
854
+ """
855
+ Generate query to find schemas that the given schemas reference via FK.
856
+
857
+ Used to discover unloaded schemas that the loaded ones depend on
858
+ (the upstream / ancestor direction). Symmetric to
859
+ :meth:`find_downstream_schemas_sql`.
860
+
861
+ Parameters
862
+ ----------
863
+ schemas_list : str
864
+ Comma-separated, quoted schema names for an IN clause.
865
+
866
+ Returns
867
+ -------
868
+ str
869
+ SQL query returning rows with a single column ``schema_name``
870
+ containing distinct schema names that are referenced by the
871
+ given schemas.
872
+ """
873
+ raise NotImplementedError
874
+ ...
875
+
853
876
  @abstractmethod
854
877
  def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
855
878
  """
@@ -696,6 +696,16 @@ class MySQLAdapter(DatabaseAdapter):
696
696
  f"AND table_schema NOT IN ({schemas_list})"
697
697
  )
698
698
 
699
+ def find_upstream_schemas_sql(self, schemas_list: str) -> str:
700
+ """Find schemas that the given schemas reference via FK."""
701
+ return (
702
+ f"SELECT DISTINCT referenced_table_schema as schema_name "
703
+ f"FROM information_schema.key_column_usage "
704
+ f"WHERE table_schema IN ({schemas_list}) "
705
+ f"AND referenced_table_schema IS NOT NULL "
706
+ f"AND referenced_table_schema NOT IN ({schemas_list})"
707
+ )
708
+
699
709
  def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
700
710
  """Query to get FK constraint details from information_schema."""
701
711
  return (
@@ -861,6 +861,20 @@ class PostgreSQLAdapter(DatabaseAdapter):
861
861
  f"AND ns1.nspname NOT IN ({schemas_list})"
862
862
  )
863
863
 
864
+ def find_upstream_schemas_sql(self, schemas_list: str) -> str:
865
+ """Find schemas that the given schemas reference via FK."""
866
+ return (
867
+ f"SELECT DISTINCT ns2.nspname as schema_name "
868
+ f"FROM pg_constraint c "
869
+ f"JOIN pg_class cl1 ON c.conrelid = cl1.oid "
870
+ f"JOIN pg_namespace ns1 ON cl1.relnamespace = ns1.oid "
871
+ f"JOIN pg_class cl2 ON c.confrelid = cl2.oid "
872
+ f"JOIN pg_namespace ns2 ON cl2.relnamespace = ns2.oid "
873
+ f"WHERE c.contype = 'f' "
874
+ f"AND ns1.nspname IN ({schemas_list}) "
875
+ f"AND ns2.nspname NOT IN ({schemas_list})"
876
+ )
877
+
864
878
  def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
865
879
  """
866
880
  Query to get FK constraint details from information_schema.
@@ -1266,6 +1280,31 @@ class PostgreSQLAdapter(DatabaseAdapter):
1266
1280
  quoted_values = ", ".join(f"'{v}'" for v in values)
1267
1281
  return f"CREATE TYPE {self.quote_identifier(type_name)} AS ENUM ({quoted_values})"
1268
1282
 
1283
+ def replica_identity_ddl(self, full_table_name: str, mode: str) -> str:
1284
+ """
1285
+ Generate ALTER TABLE ... REPLICA IDENTITY statement.
1286
+
1287
+ Controls how much of the old row PostgreSQL writes to WAL on UPDATE/DELETE.
1288
+ ``"default"`` logs only primary-key columns; ``"full"`` logs the entire row.
1289
+ Required by some CDC tools (e.g. Databricks Lakehouse Sync) that need the
1290
+ full pre-image to drive Slowly-Changing-Dimension history.
1291
+
1292
+ The ALTER is metadata-only, instant, and idempotent — re-applying the same
1293
+ mode is a no-op at the storage layer.
1294
+
1295
+ Examples
1296
+ --------
1297
+ >>> adapter.replica_identity_ddl('"schema"."table"', 'full')
1298
+ 'ALTER TABLE "schema"."table" REPLICA IDENTITY FULL'
1299
+ >>> adapter.replica_identity_ddl('"schema"."table"', 'default')
1300
+ 'ALTER TABLE "schema"."table" REPLICA IDENTITY DEFAULT'
1301
+ """
1302
+ if mode not in ("default", "full"):
1303
+ from ..errors import DataJointError
1304
+
1305
+ raise DataJointError(f"Unsupported replica_identity mode: {mode!r}. Expected 'default' or 'full'.")
1306
+ return f"ALTER TABLE {full_table_name} REPLICA IDENTITY {mode.upper()}"
1307
+
1269
1308
  def get_pending_enum_ddl(self, schema_name: str) -> list[str]:
1270
1309
  """
1271
1310
  Get DDL statements for pending enum types and clear the pending list.
@@ -89,6 +89,46 @@ class AutoPopulate:
89
89
  _key_source = None
90
90
  _allow_insert = False
91
91
  _jobs = None
92
+ _upstream = None # set per-make() by _populate_one; see `upstream` property below
93
+
94
+ @property
95
+ def upstream(self):
96
+ """
97
+ Pre-restricted ancestor view for the current ``make(self, key)`` call.
98
+
99
+ Inside ``make()``, ``self.upstream`` is a ``Diagram`` constructed via
100
+ :meth:`Diagram.trace(self & key) <datajoint.Diagram.trace>`. Use
101
+ ``self.upstream[T]`` to obtain a pre-restricted ``QueryExpression``
102
+ (or ``FreeTable``, when indexed by a string) for any ancestor of
103
+ ``self``.
104
+
105
+ Reading via ``self.upstream`` is the provenance-safe pattern: the
106
+ framework guarantees the restriction matches the current ``key``,
107
+ and indexing a non-ancestor table raises ``DataJointError``. See
108
+ :doc:`reference/specs/provenance` for the contract.
109
+
110
+ Raises
111
+ ------
112
+ DataJointError
113
+ If accessed outside ``make()`` execution. To construct a trace
114
+ explicitly, use ``dj.Diagram.trace(self & key)``.
115
+
116
+ Examples
117
+ --------
118
+ ::
119
+
120
+ def make(self, key):
121
+ date = self.upstream[Session].fetch1("session_date")
122
+ traces = self.upstream[ExtractTraces].to_arrays("trace")
123
+ self.insert1({**key, "summary": compute(traces, date)})
124
+ """
125
+ if self._upstream is None:
126
+ raise DataJointError(
127
+ "self.upstream is only available inside make(). "
128
+ "Outside make(), construct a trace explicitly: "
129
+ "dj.Diagram.trace(self & key)."
130
+ )
131
+ return self._upstream
92
132
 
93
133
  class _JobsDescriptor:
94
134
  """Descriptor allowing jobs access on both class and instance."""
@@ -611,6 +651,41 @@ class AutoPopulate:
611
651
  logger.jobs(f"Making {key} -> {self.full_table_name}")
612
652
  self.__class__._allow_insert = True
613
653
 
654
+ # Pre-construct the upstream view for this make() call. Lazy — only
655
+ # `dj.Diagram.trace(self & key)` runs here (graph copy); the
656
+ # expensive SQL fetch fires when the user accesses self.upstream[T].
657
+ from .diagram import Diagram
658
+
659
+ self._upstream = Diagram.trace(self & dict(key))
660
+
661
+ # If strict_provenance is on, push the active-make context so the
662
+ # runtime gates in expression.cursor / table.insert can check this
663
+ # make()'s reads and writes. The context is popped in the finally
664
+ # block below.
665
+ strict_token = None
666
+ if self.connection._config.get("strict_provenance", False):
667
+ from .provenance import push_strict_make_context
668
+ from .user_tables import Part
669
+
670
+ allowed_tables = set(self._upstream._cascade_restrictions.keys()) | {self.full_table_name}
671
+ # Add Part tables of self to the allowed set. Use class __dict__
672
+ # (not dir/getattr) to avoid triggering descriptors like the
673
+ # _JobsDescriptor that lazy-declares the ~~ job table.
674
+ for cls in type(self).__mro__:
675
+ for attr_name, attr in cls.__dict__.items():
676
+ if attr_name.startswith("_"):
677
+ continue
678
+ if isinstance(attr, type) and issubclass(attr, Part):
679
+ # Instantiate to get full_table_name resolved against
680
+ # this schema. The Part class is already attached via
681
+ # @schema decoration of the master.
682
+ try:
683
+ part_ftn = attr().full_table_name
684
+ allowed_tables.add(part_ftn)
685
+ except Exception:
686
+ pass
687
+ strict_token = push_strict_make_context(self, frozenset(allowed_tables), dict(key))
688
+
614
689
  try:
615
690
  if not is_generator:
616
691
  make(dict(key), **(make_kwargs or {}))
@@ -668,6 +743,15 @@ class AutoPopulate:
668
743
  return True
669
744
  finally:
670
745
  self.__class__._allow_insert = False
746
+ # Clear the per-make() upstream view so subsequent attribute
747
+ # access raises a clear error rather than silently using a
748
+ # stale trace from the previous make() call.
749
+ self._upstream = None
750
+ # Pop the strict-make context, if any.
751
+ if strict_token is not None:
752
+ from .provenance import pop_strict_make_context
753
+
754
+ pop_strict_make_context(strict_token)
671
755
 
672
756
  def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]:
673
757
  """
@@ -268,6 +268,20 @@ def assert_join_compatibility(
268
268
  lineage2 = expr2.heading[name].lineage
269
269
  # Semantic match requires both lineages to be non-None and equal
270
270
  if lineage1 is None or lineage2 is None or lineage1 != lineage2:
271
+ if lineage1 is None or lineage2 is None:
272
+ # Missing lineage usually means stale ~lineage rows that survived
273
+ # an upgrade or a partial declare. Decoration in 2.3+ refreshes
274
+ # lineage automatically, so this typically indicates a schema
275
+ # that has not been re-decorated since the upgrade.
276
+ raise DataJointError(
277
+ f"Cannot join on attribute `{name}`: lineage missing on "
278
+ f"one side ({lineage1} vs {lineage2}). This usually "
279
+ f"indicates a stale `~lineage` entry from an older "
280
+ f"DataJoint version or an incomplete declare. Run "
281
+ f"`schema.rebuild_lineage()` to recompute lineage from "
282
+ f"current FK definitions. If the lineages are genuinely "
283
+ f"different, use `.proj()` to rename one of the attributes."
284
+ )
271
285
  raise DataJointError(
272
286
  f"Cannot join on attribute `{name}`: "
273
287
  f"different lineages ({lineage1} vs {lineage2}). "
@@ -259,6 +259,35 @@ class Dependencies(nx.DiGraph):
259
259
 
260
260
  self.load(force=True, schema_names=known_schemas)
261
261
 
262
+ def load_all_upstream(self) -> None:
263
+ """
264
+ Load dependencies including all upstream schemas referenced via FK chains.
265
+
266
+ Iteratively discovers schemas that the currently loaded schemas
267
+ reference, expanding the dependency graph until no new schemas
268
+ are found. This ensures that upstream restriction propagation
269
+ (``Diagram.trace()``) reaches all ancestor tables, including
270
+ those in schemas the user has not explicitly activated.
271
+
272
+ Called automatically by ``Diagram.trace()``. Symmetric to
273
+ :meth:`load_all_downstream`.
274
+ """
275
+ adapter = self._conn.adapter
276
+ known_schemas = set(self._conn.schemas)
277
+ if not known_schemas:
278
+ self.load()
279
+ return
280
+
281
+ while True:
282
+ schemas_list = ", ".join(adapter.quote_string(s) for s in known_schemas)
283
+ result = self._conn.query(adapter.find_upstream_schemas_sql(schemas_list))
284
+ new_schemas = {row[0] for row in result} - known_schemas
285
+ if not new_schemas:
286
+ break
287
+ known_schemas |= new_schemas
288
+
289
+ self.load(force=True, schema_names=known_schemas)
290
+
262
291
  def topo_sort(self) -> list[str]:
263
292
  """
264
293
  Return table names in topological order.
@@ -0,0 +1,185 @@
1
+ """
2
+ Deployment-time operations for configuring an existing DataJoint pipeline.
3
+
4
+ This module hosts idempotent operational helpers — things you run as part of a
5
+ deploy hook to configure a schema for its environment, distinct from
6
+ :mod:`datajoint.migrate` which handles one-shot schema/state evolution.
7
+
8
+ The boundary between the two:
9
+
10
+ - :mod:`datajoint.migrate` — fix legacy state, evolve a schema definition,
11
+ retroactive corrections. Cadence: one-shot. Examples: ``migrate_columns``,
12
+ ``add_job_metadata_columns``, ``rebuild_lineage``.
13
+ - :mod:`datajoint.deploy` — configure an environment for a consumer's
14
+ requirements (CDC tools, replication, role grants, performance tuning).
15
+ Cadence: re-runnable, idempotent. Examples: :func:`set_replica_identity`.
16
+
17
+ Functions in this module should be safe to call repeatedly from a deploy hook
18
+ without accumulating side effects.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from typing import TYPE_CHECKING, Any, Literal, Union
24
+
25
+ from .errors import DataJointError
26
+
27
+ if TYPE_CHECKING:
28
+ from .schemas import _Schema
29
+ from .table import Table
30
+
31
+ TargetType = Union["_Schema", type["Table"], "Table"]
32
+
33
+
34
+ def set_replica_identity(
35
+ target: "TargetType",
36
+ mode: Literal["default", "full"] = "full",
37
+ dry_run: bool = True,
38
+ ) -> dict:
39
+ """
40
+ Apply ``ALTER TABLE ... REPLICA IDENTITY <mode>`` to a schema or table on PostgreSQL.
41
+
42
+ ``REPLICA IDENTITY`` controls how much of the **old row** PostgreSQL writes to
43
+ the write-ahead log on UPDATE/DELETE. Under ``DEFAULT``, only primary-key
44
+ columns appear in WAL; under ``FULL``, the entire old row does.
45
+
46
+ Why this exists
47
+ ---------------
48
+ Some change-data-capture (CDC) consumers require the full row pre-image to
49
+ drive their downstream models. The canonical example is **Databricks
50
+ Lakehouse Sync**: tables without ``REPLICA IDENTITY FULL`` are silently
51
+ skipped by the sync — no error, just missing data downstream. Other CDC
52
+ tools (Debezium, ClickHouse ClickPipes, Azure CDC) work fine with
53
+ ``DEFAULT`` when tables have a primary key; only Databricks mandates
54
+ ``FULL``.
55
+
56
+ This helper is the **operational** way to apply the setting. It is not a
57
+ migration: there's no legacy state being fixed; the setting is simply a
58
+ property of the deployment environment, and a fresh declare in a new
59
+ environment may need it re-applied. It is idempotent — re-applying the
60
+ same mode is a no-op at the storage layer — so it is safe to call from a
61
+ deploy hook on every release.
62
+
63
+ Cost
64
+ ----
65
+ The ALTER itself is metadata-only and instant, but requires a brief
66
+ ``AccessExclusiveLock`` on each table — it will block behind in-flight
67
+ writes/reads on a busy table. Run during a quiet window on actively-
68
+ ingested tables.
69
+
70
+ The ongoing cost is in WAL volume after the change: UPDATE/DELETE on
71
+ tables with FULL log the entire old row, which can be sizable on tables
72
+ with TOASTed bytea columns. For DataJoint's typical insert-append
73
+ workload, this cost is negligible. The notable scenario is bulk
74
+ ``delete()`` on tables with ``<blob>`` columns — a transient WAL burst
75
+ proportional to the deleted-row payload size.
76
+
77
+ Partial-failure semantics
78
+ -------------------------
79
+ If ``connection.query(ddl)`` raises on table N of M, the first N-1
80
+ tables are already modified at the storage layer but the exception
81
+ propagates without returning the partial summary. The operation is
82
+ idempotent, so re-running brings the remaining tables into compliance.
83
+
84
+ Compliance considerations
85
+ -------------------------
86
+ Under ``DEFAULT``, only primary-key values appear in WAL. Under ``FULL``,
87
+ entire rows do — including any PHI/PII/sensitive columns. For self-hosted
88
+ PostgreSQL with unrestricted WAL access this is a real consideration; for
89
+ managed PostgreSQL with logical replication confined to a specific
90
+ subscriber (Lakebase, RDS), WAL stays inside the managed environment's
91
+ security boundary. Apply intentionally.
92
+
93
+ Parameters
94
+ ----------
95
+ target : Schema or Table
96
+ A :class:`datajoint.Schema` (all user tables) or a
97
+ :class:`datajoint.Table` class/instance (just that table).
98
+ mode : str, default ``"full"``
99
+ ``"default"`` (PK only, minimal WAL) or ``"full"`` (entire row).
100
+ dry_run : bool, default ``True``
101
+ If True, collect the DDL statements but do not execute. Set to False
102
+ to actually apply.
103
+
104
+ Returns
105
+ -------
106
+ dict
107
+ - ``tables_analyzed`` (int): number of tables considered.
108
+ - ``tables_modified`` (int): number of tables on which the ALTER ran.
109
+ Always 0 when ``dry_run=True``.
110
+ - ``ddl`` (list[str]): the DDL statements that were (or would be) executed.
111
+
112
+ Raises
113
+ ------
114
+ DataJointError
115
+ If the target's backend is not PostgreSQL, or if ``mode`` is not one of
116
+ ``"default"`` / ``"full"``.
117
+
118
+ Examples
119
+ --------
120
+ >>> from datajoint.deploy import set_replica_identity
121
+ >>> # Preview
122
+ >>> set_replica_identity(my_schema, mode="full", dry_run=True)
123
+ {'tables_analyzed': 12, 'tables_modified': 0, 'ddl': ['ALTER TABLE "ms"."t1" REPLICA IDENTITY FULL', ...]}
124
+ >>> # Apply
125
+ >>> set_replica_identity(my_schema, mode="full", dry_run=False)
126
+ {'tables_analyzed': 12, 'tables_modified': 12, 'ddl': [...]}
127
+ >>> # Single table
128
+ >>> set_replica_identity(MyTable, mode="full", dry_run=False)
129
+
130
+ See Also
131
+ --------
132
+ PostgreSQL: `Logical Replication — Replica Identity
133
+ <https://www.postgresql.org/docs/current/logical-replication-publication.html>`_.
134
+ Databricks: `Lakehouse Sync
135
+ <https://docs.databricks.com/aws/en/oltp/projects/lakehouse-sync>`_.
136
+ """
137
+ mode_normalized = mode.lower() if isinstance(mode, str) else mode
138
+ if mode_normalized not in ("default", "full"):
139
+ raise DataJointError(f"mode must be 'default' or 'full'; got {mode!r}")
140
+ mode = mode_normalized # type: ignore[assignment]
141
+
142
+ from .schemas import _Schema
143
+ from .table import Table
144
+
145
+ if isinstance(target, _Schema):
146
+ connection = target.connection
147
+ if connection is None:
148
+ raise DataJointError("Schema has no active connection.")
149
+ adapter = connection.adapter
150
+ if target.database is None:
151
+ raise DataJointError("Schema is not activated. Call schema.activate(...) before set_replica_identity().")
152
+ tables = [adapter.make_full_table_name(target.database, t) for t in target.list_tables()]
153
+ elif isinstance(target, type) and issubclass(target, Table):
154
+ instance = target()
155
+ connection = instance.connection
156
+ if connection is None:
157
+ raise DataJointError(f"Table {target.__name__} has no active connection.")
158
+ adapter = connection.adapter
159
+ tables = [instance.full_table_name]
160
+ elif isinstance(target, Table):
161
+ connection = target.connection
162
+ if connection is None:
163
+ raise DataJointError(f"Table {type(target).__name__} has no active connection.")
164
+ adapter = connection.adapter
165
+ tables = [target.full_table_name]
166
+ else:
167
+ raise DataJointError(f"target must be a Schema or Table class/instance; got {type(target).__name__}")
168
+
169
+ if not hasattr(adapter, "replica_identity_ddl"):
170
+ raise DataJointError(
171
+ f"set_replica_identity is PostgreSQL-only; the {adapter.backend} adapter does not support REPLICA IDENTITY."
172
+ )
173
+
174
+ result: dict[str, Any] = {
175
+ "tables_analyzed": len(tables),
176
+ "tables_modified": 0,
177
+ "ddl": [],
178
+ }
179
+ for full_name in tables:
180
+ ddl = adapter.replica_identity_ddl(full_name, mode) # type: ignore[attr-defined]
181
+ result["ddl"].append(ddl)
182
+ if not dry_run:
183
+ connection.query(ddl)
184
+ result["tables_modified"] += 1
185
+ return result