datajoint 2.2.4__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datajoint-2.2.4 → datajoint-2.3.0}/PKG-INFO +1 -1
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/__init__.py +5 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/adapters/base.py +23 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/adapters/mysql.py +10 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/adapters/postgres.py +39 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/autopopulate.py +84 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/condition.py +14 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/dependencies.py +29 -0
- datajoint-2.3.0/src/datajoint/deploy.py +185 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/diagram.py +451 -21
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/expression.py +6 -0
- datajoint-2.3.0/src/datajoint/provenance.py +206 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/schemas.py +17 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/settings.py +11 -0
- datajoint-2.3.0/src/datajoint/spark.py +92 -0
- datajoint-2.3.0/src/datajoint/staged_insert.py +274 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/table.py +62 -2
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/version.py +1 -1
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_autopopulate.py +239 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_cascade_delete.py +187 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_object.py +98 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_semantic_matching.py +150 -0
- datajoint-2.3.0/tests/integration/test_strict_provenance.py +322 -0
- datajoint-2.3.0/tests/integration/test_trace.py +391 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_adapters.py +17 -0
- datajoint-2.3.0/tests/unit/test_deploy.py +172 -0
- datajoint-2.3.0/tests/unit/test_spark.py +105 -0
- datajoint-2.2.4/src/datajoint/staged_insert.py +0 -310
- {datajoint-2.2.4 → datajoint-2.3.0}/.coveragerc +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.dockerignore +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.gitattributes +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/dependabot.yml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/release_drafter.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/workflows/draft_release.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/workflows/lint.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/workflows/post_draft_release_published.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.github/workflows/test.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.gitignore +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/.pre-commit-config.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/CHANGELOG-archive.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/CONTRIBUTING.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/Dockerfile +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/LICENSE +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/README.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/RELEASE_MEMO.md +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/activate.sh +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/docker-compose.yaml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/images/pipeline.png +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/images/pipeline.svg +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/pixi.lock +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/pyproject.toml +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/adapters/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/blob.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/attach.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/blob.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/filepath.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/hash.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/npy.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/object.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/builtin_codecs/schema.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/cli.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/codecs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/connection.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/declare.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/errors.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/gc.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/hash_registry.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/heading.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/instance.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/jobs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/lineage.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/logging.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/migrate.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/objectref.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/preview.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/storage.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/storage_adapter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/types.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/user_tables.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/src/datajoint/utils.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/conftest.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/Course.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/CurrentTerm.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/Department.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/Enroll.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/Grade.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/Section.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/Student.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/StudentMajor.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/data/Term.csv +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_aggr_regressions.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_alter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_attach.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_blob.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_blob_matlab.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_cascading_delete.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_cli.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_codec_chaining.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_codecs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_connection.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_declare.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_dependencies.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_erd.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_fetch.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_fetch_same.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_foreign_keys.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_gc.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_groupby.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_hash_storage.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_hidden_job_metadata.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_insert.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_jobs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_json.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_multi_backend.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_nan.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_npy_codec.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_privileges.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_reconnection.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_relation.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_relation_u.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_relational_operand.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_schema.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_schema_keywords.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_skip_duplicates.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_tls.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_type_aliases.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_university.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_update1.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_utils.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_uuid.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/integration/test_virtual_module.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_advanced.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_aggr_regress.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_alter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_codecs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_external.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_object.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_privileges.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_simple.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_type_aliases.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_university.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/schema_uuid.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/test_package.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/__init__.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_codecs.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_condition.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_connection_warning.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_declare_hidden_attribute.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_fetch_compat.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_lazy_imports.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_pk_rules.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_settings.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_storage_adapter.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_storage_urls.py +0 -0
- {datajoint-2.2.4 → datajoint-2.3.0}/tests/unit/test_thread_safe.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: datajoint
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
|
|
5
5
|
Project-URL: Homepage, https://docs.datajoint.com/
|
|
6
6
|
Project-URL: Documentation, https://docs.datajoint.com/
|
|
@@ -51,12 +51,15 @@ __all__ = [
|
|
|
51
51
|
"get_codec",
|
|
52
52
|
"ObjectRef",
|
|
53
53
|
"NpyRef",
|
|
54
|
+
# SparkAdapter Codec Protocol
|
|
55
|
+
"SparkAdapter",
|
|
54
56
|
# Storage Adapter API
|
|
55
57
|
"StorageAdapter",
|
|
56
58
|
"get_storage_adapter",
|
|
57
59
|
# Other
|
|
58
60
|
"errors",
|
|
59
61
|
"migrate",
|
|
62
|
+
"deploy",
|
|
60
63
|
"DataJointError",
|
|
61
64
|
"ThreadSafetyError",
|
|
62
65
|
"logger",
|
|
@@ -69,6 +72,7 @@ __all__ = [
|
|
|
69
72
|
# =============================================================================
|
|
70
73
|
from . import errors
|
|
71
74
|
from . import migrate
|
|
75
|
+
from . import deploy
|
|
72
76
|
from .codecs import (
|
|
73
77
|
Codec,
|
|
74
78
|
get_codec,
|
|
@@ -85,6 +89,7 @@ from .expression import AndList, Not, Top, U
|
|
|
85
89
|
from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global_config, _check_thread_safe
|
|
86
90
|
from .logging import logger
|
|
87
91
|
from .objectref import ObjectRef
|
|
92
|
+
from .spark import SparkAdapter
|
|
88
93
|
from .storage_adapter import StorageAdapter, get_storage_adapter
|
|
89
94
|
from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
|
|
90
95
|
from .autopopulate import AutoPopulate
|
|
@@ -850,6 +850,29 @@ class DatabaseAdapter(ABC):
|
|
|
850
850
|
raise NotImplementedError
|
|
851
851
|
...
|
|
852
852
|
|
|
853
|
+
def find_upstream_schemas_sql(self, schemas_list: str) -> str:
|
|
854
|
+
"""
|
|
855
|
+
Generate query to find schemas that the given schemas reference via FK.
|
|
856
|
+
|
|
857
|
+
Used to discover unloaded schemas that the loaded ones depend on
|
|
858
|
+
(the upstream / ancestor direction). Symmetric to
|
|
859
|
+
:meth:`find_downstream_schemas_sql`.
|
|
860
|
+
|
|
861
|
+
Parameters
|
|
862
|
+
----------
|
|
863
|
+
schemas_list : str
|
|
864
|
+
Comma-separated, quoted schema names for an IN clause.
|
|
865
|
+
|
|
866
|
+
Returns
|
|
867
|
+
-------
|
|
868
|
+
str
|
|
869
|
+
SQL query returning rows with a single column ``schema_name``
|
|
870
|
+
containing distinct schema names that are referenced by the
|
|
871
|
+
given schemas.
|
|
872
|
+
"""
|
|
873
|
+
raise NotImplementedError
|
|
874
|
+
...
|
|
875
|
+
|
|
853
876
|
@abstractmethod
|
|
854
877
|
def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
|
|
855
878
|
"""
|
|
@@ -696,6 +696,16 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
696
696
|
f"AND table_schema NOT IN ({schemas_list})"
|
|
697
697
|
)
|
|
698
698
|
|
|
699
|
+
def find_upstream_schemas_sql(self, schemas_list: str) -> str:
|
|
700
|
+
"""Find schemas that the given schemas reference via FK."""
|
|
701
|
+
return (
|
|
702
|
+
f"SELECT DISTINCT referenced_table_schema as schema_name "
|
|
703
|
+
f"FROM information_schema.key_column_usage "
|
|
704
|
+
f"WHERE table_schema IN ({schemas_list}) "
|
|
705
|
+
f"AND referenced_table_schema IS NOT NULL "
|
|
706
|
+
f"AND referenced_table_schema NOT IN ({schemas_list})"
|
|
707
|
+
)
|
|
708
|
+
|
|
699
709
|
def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
|
|
700
710
|
"""Query to get FK constraint details from information_schema."""
|
|
701
711
|
return (
|
|
@@ -861,6 +861,20 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
861
861
|
f"AND ns1.nspname NOT IN ({schemas_list})"
|
|
862
862
|
)
|
|
863
863
|
|
|
864
|
+
def find_upstream_schemas_sql(self, schemas_list: str) -> str:
|
|
865
|
+
"""Find schemas that the given schemas reference via FK."""
|
|
866
|
+
return (
|
|
867
|
+
f"SELECT DISTINCT ns2.nspname as schema_name "
|
|
868
|
+
f"FROM pg_constraint c "
|
|
869
|
+
f"JOIN pg_class cl1 ON c.conrelid = cl1.oid "
|
|
870
|
+
f"JOIN pg_namespace ns1 ON cl1.relnamespace = ns1.oid "
|
|
871
|
+
f"JOIN pg_class cl2 ON c.confrelid = cl2.oid "
|
|
872
|
+
f"JOIN pg_namespace ns2 ON cl2.relnamespace = ns2.oid "
|
|
873
|
+
f"WHERE c.contype = 'f' "
|
|
874
|
+
f"AND ns1.nspname IN ({schemas_list}) "
|
|
875
|
+
f"AND ns2.nspname NOT IN ({schemas_list})"
|
|
876
|
+
)
|
|
877
|
+
|
|
864
878
|
def get_constraint_info_sql(self, constraint_name: str, schema_name: str, table_name: str) -> str:
|
|
865
879
|
"""
|
|
866
880
|
Query to get FK constraint details from information_schema.
|
|
@@ -1266,6 +1280,31 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
1266
1280
|
quoted_values = ", ".join(f"'{v}'" for v in values)
|
|
1267
1281
|
return f"CREATE TYPE {self.quote_identifier(type_name)} AS ENUM ({quoted_values})"
|
|
1268
1282
|
|
|
1283
|
+
def replica_identity_ddl(self, full_table_name: str, mode: str) -> str:
|
|
1284
|
+
"""
|
|
1285
|
+
Generate ALTER TABLE ... REPLICA IDENTITY statement.
|
|
1286
|
+
|
|
1287
|
+
Controls how much of the old row PostgreSQL writes to WAL on UPDATE/DELETE.
|
|
1288
|
+
``"default"`` logs only primary-key columns; ``"full"`` logs the entire row.
|
|
1289
|
+
Required by some CDC tools (e.g. Databricks Lakehouse Sync) that need the
|
|
1290
|
+
full pre-image to drive Slowly-Changing-Dimension history.
|
|
1291
|
+
|
|
1292
|
+
The ALTER is metadata-only, instant, and idempotent — re-applying the same
|
|
1293
|
+
mode is a no-op at the storage layer.
|
|
1294
|
+
|
|
1295
|
+
Examples
|
|
1296
|
+
--------
|
|
1297
|
+
>>> adapter.replica_identity_ddl('"schema"."table"', 'full')
|
|
1298
|
+
'ALTER TABLE "schema"."table" REPLICA IDENTITY FULL'
|
|
1299
|
+
>>> adapter.replica_identity_ddl('"schema"."table"', 'default')
|
|
1300
|
+
'ALTER TABLE "schema"."table" REPLICA IDENTITY DEFAULT'
|
|
1301
|
+
"""
|
|
1302
|
+
if mode not in ("default", "full"):
|
|
1303
|
+
from ..errors import DataJointError
|
|
1304
|
+
|
|
1305
|
+
raise DataJointError(f"Unsupported replica_identity mode: {mode!r}. Expected 'default' or 'full'.")
|
|
1306
|
+
return f"ALTER TABLE {full_table_name} REPLICA IDENTITY {mode.upper()}"
|
|
1307
|
+
|
|
1269
1308
|
def get_pending_enum_ddl(self, schema_name: str) -> list[str]:
|
|
1270
1309
|
"""
|
|
1271
1310
|
Get DDL statements for pending enum types and clear the pending list.
|
|
@@ -89,6 +89,46 @@ class AutoPopulate:
|
|
|
89
89
|
_key_source = None
|
|
90
90
|
_allow_insert = False
|
|
91
91
|
_jobs = None
|
|
92
|
+
_upstream = None # set per-make() by _populate_one; see `upstream` property below
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def upstream(self):
|
|
96
|
+
"""
|
|
97
|
+
Pre-restricted ancestor view for the current ``make(self, key)`` call.
|
|
98
|
+
|
|
99
|
+
Inside ``make()``, ``self.upstream`` is a ``Diagram`` constructed via
|
|
100
|
+
:meth:`Diagram.trace(self & key) <datajoint.Diagram.trace>`. Use
|
|
101
|
+
``self.upstream[T]`` to obtain a pre-restricted ``QueryExpression``
|
|
102
|
+
(or ``FreeTable``, when indexed by a string) for any ancestor of
|
|
103
|
+
``self``.
|
|
104
|
+
|
|
105
|
+
Reading via ``self.upstream`` is the provenance-safe pattern: the
|
|
106
|
+
framework guarantees the restriction matches the current ``key``,
|
|
107
|
+
and indexing a non-ancestor table raises ``DataJointError``. See
|
|
108
|
+
:doc:`reference/specs/provenance` for the contract.
|
|
109
|
+
|
|
110
|
+
Raises
|
|
111
|
+
------
|
|
112
|
+
DataJointError
|
|
113
|
+
If accessed outside ``make()`` execution. To construct a trace
|
|
114
|
+
explicitly, use ``dj.Diagram.trace(self & key)``.
|
|
115
|
+
|
|
116
|
+
Examples
|
|
117
|
+
--------
|
|
118
|
+
::
|
|
119
|
+
|
|
120
|
+
def make(self, key):
|
|
121
|
+
date = self.upstream[Session].fetch1("session_date")
|
|
122
|
+
traces = self.upstream[ExtractTraces].to_arrays("trace")
|
|
123
|
+
self.insert1({**key, "summary": compute(traces, date)})
|
|
124
|
+
"""
|
|
125
|
+
if self._upstream is None:
|
|
126
|
+
raise DataJointError(
|
|
127
|
+
"self.upstream is only available inside make(). "
|
|
128
|
+
"Outside make(), construct a trace explicitly: "
|
|
129
|
+
"dj.Diagram.trace(self & key)."
|
|
130
|
+
)
|
|
131
|
+
return self._upstream
|
|
92
132
|
|
|
93
133
|
class _JobsDescriptor:
|
|
94
134
|
"""Descriptor allowing jobs access on both class and instance."""
|
|
@@ -611,6 +651,41 @@ class AutoPopulate:
|
|
|
611
651
|
logger.jobs(f"Making {key} -> {self.full_table_name}")
|
|
612
652
|
self.__class__._allow_insert = True
|
|
613
653
|
|
|
654
|
+
# Pre-construct the upstream view for this make() call. Lazy — only
|
|
655
|
+
# `dj.Diagram.trace(self & key)` runs here (graph copy); the
|
|
656
|
+
# expensive SQL fetch fires when the user accesses self.upstream[T].
|
|
657
|
+
from .diagram import Diagram
|
|
658
|
+
|
|
659
|
+
self._upstream = Diagram.trace(self & dict(key))
|
|
660
|
+
|
|
661
|
+
# If strict_provenance is on, push the active-make context so the
|
|
662
|
+
# runtime gates in expression.cursor / table.insert can check this
|
|
663
|
+
# make()'s reads and writes. The context is popped in the finally
|
|
664
|
+
# block below.
|
|
665
|
+
strict_token = None
|
|
666
|
+
if self.connection._config.get("strict_provenance", False):
|
|
667
|
+
from .provenance import push_strict_make_context
|
|
668
|
+
from .user_tables import Part
|
|
669
|
+
|
|
670
|
+
allowed_tables = set(self._upstream._cascade_restrictions.keys()) | {self.full_table_name}
|
|
671
|
+
# Add Part tables of self to the allowed set. Use class __dict__
|
|
672
|
+
# (not dir/getattr) to avoid triggering descriptors like the
|
|
673
|
+
# _JobsDescriptor that lazy-declares the ~~ job table.
|
|
674
|
+
for cls in type(self).__mro__:
|
|
675
|
+
for attr_name, attr in cls.__dict__.items():
|
|
676
|
+
if attr_name.startswith("_"):
|
|
677
|
+
continue
|
|
678
|
+
if isinstance(attr, type) and issubclass(attr, Part):
|
|
679
|
+
# Instantiate to get full_table_name resolved against
|
|
680
|
+
# this schema. The Part class is already attached via
|
|
681
|
+
# @schema decoration of the master.
|
|
682
|
+
try:
|
|
683
|
+
part_ftn = attr().full_table_name
|
|
684
|
+
allowed_tables.add(part_ftn)
|
|
685
|
+
except Exception:
|
|
686
|
+
pass
|
|
687
|
+
strict_token = push_strict_make_context(self, frozenset(allowed_tables), dict(key))
|
|
688
|
+
|
|
614
689
|
try:
|
|
615
690
|
if not is_generator:
|
|
616
691
|
make(dict(key), **(make_kwargs or {}))
|
|
@@ -668,6 +743,15 @@ class AutoPopulate:
|
|
|
668
743
|
return True
|
|
669
744
|
finally:
|
|
670
745
|
self.__class__._allow_insert = False
|
|
746
|
+
# Clear the per-make() upstream view so subsequent attribute
|
|
747
|
+
# access raises a clear error rather than silently using a
|
|
748
|
+
# stale trace from the previous make() call.
|
|
749
|
+
self._upstream = None
|
|
750
|
+
# Pop the strict-make context, if any.
|
|
751
|
+
if strict_token is not None:
|
|
752
|
+
from .provenance import pop_strict_make_context
|
|
753
|
+
|
|
754
|
+
pop_strict_make_context(strict_token)
|
|
671
755
|
|
|
672
756
|
def progress(self, *restrictions: Any, display: bool = False) -> tuple[int, int]:
|
|
673
757
|
"""
|
|
@@ -268,6 +268,20 @@ def assert_join_compatibility(
|
|
|
268
268
|
lineage2 = expr2.heading[name].lineage
|
|
269
269
|
# Semantic match requires both lineages to be non-None and equal
|
|
270
270
|
if lineage1 is None or lineage2 is None or lineage1 != lineage2:
|
|
271
|
+
if lineage1 is None or lineage2 is None:
|
|
272
|
+
# Missing lineage usually means stale ~lineage rows that survived
|
|
273
|
+
# an upgrade or a partial declare. Decoration in 2.3+ refreshes
|
|
274
|
+
# lineage automatically, so this typically indicates a schema
|
|
275
|
+
# that has not been re-decorated since the upgrade.
|
|
276
|
+
raise DataJointError(
|
|
277
|
+
f"Cannot join on attribute `{name}`: lineage missing on "
|
|
278
|
+
f"one side ({lineage1} vs {lineage2}). This usually "
|
|
279
|
+
f"indicates a stale `~lineage` entry from an older "
|
|
280
|
+
f"DataJoint version or an incomplete declare. Run "
|
|
281
|
+
f"`schema.rebuild_lineage()` to recompute lineage from "
|
|
282
|
+
f"current FK definitions. If the lineages are genuinely "
|
|
283
|
+
f"different, use `.proj()` to rename one of the attributes."
|
|
284
|
+
)
|
|
271
285
|
raise DataJointError(
|
|
272
286
|
f"Cannot join on attribute `{name}`: "
|
|
273
287
|
f"different lineages ({lineage1} vs {lineage2}). "
|
|
@@ -259,6 +259,35 @@ class Dependencies(nx.DiGraph):
|
|
|
259
259
|
|
|
260
260
|
self.load(force=True, schema_names=known_schemas)
|
|
261
261
|
|
|
262
|
+
def load_all_upstream(self) -> None:
|
|
263
|
+
"""
|
|
264
|
+
Load dependencies including all upstream schemas referenced via FK chains.
|
|
265
|
+
|
|
266
|
+
Iteratively discovers schemas that the currently loaded schemas
|
|
267
|
+
reference, expanding the dependency graph until no new schemas
|
|
268
|
+
are found. This ensures that upstream restriction propagation
|
|
269
|
+
(``Diagram.trace()``) reaches all ancestor tables, including
|
|
270
|
+
those in schemas the user has not explicitly activated.
|
|
271
|
+
|
|
272
|
+
Called automatically by ``Diagram.trace()``. Symmetric to
|
|
273
|
+
:meth:`load_all_downstream`.
|
|
274
|
+
"""
|
|
275
|
+
adapter = self._conn.adapter
|
|
276
|
+
known_schemas = set(self._conn.schemas)
|
|
277
|
+
if not known_schemas:
|
|
278
|
+
self.load()
|
|
279
|
+
return
|
|
280
|
+
|
|
281
|
+
while True:
|
|
282
|
+
schemas_list = ", ".join(adapter.quote_string(s) for s in known_schemas)
|
|
283
|
+
result = self._conn.query(adapter.find_upstream_schemas_sql(schemas_list))
|
|
284
|
+
new_schemas = {row[0] for row in result} - known_schemas
|
|
285
|
+
if not new_schemas:
|
|
286
|
+
break
|
|
287
|
+
known_schemas |= new_schemas
|
|
288
|
+
|
|
289
|
+
self.load(force=True, schema_names=known_schemas)
|
|
290
|
+
|
|
262
291
|
def topo_sort(self) -> list[str]:
|
|
263
292
|
"""
|
|
264
293
|
Return table names in topological order.
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Deployment-time operations for configuring an existing DataJoint pipeline.
|
|
3
|
+
|
|
4
|
+
This module hosts idempotent operational helpers — things you run as part of a
|
|
5
|
+
deploy hook to configure a schema for its environment, distinct from
|
|
6
|
+
:mod:`datajoint.migrate` which handles one-shot schema/state evolution.
|
|
7
|
+
|
|
8
|
+
The boundary between the two:
|
|
9
|
+
|
|
10
|
+
- :mod:`datajoint.migrate` — fix legacy state, evolve a schema definition,
|
|
11
|
+
retroactive corrections. Cadence: one-shot. Examples: ``migrate_columns``,
|
|
12
|
+
``add_job_metadata_columns``, ``rebuild_lineage``.
|
|
13
|
+
- :mod:`datajoint.deploy` — configure an environment for a consumer's
|
|
14
|
+
requirements (CDC tools, replication, role grants, performance tuning).
|
|
15
|
+
Cadence: re-runnable, idempotent. Examples: :func:`set_replica_identity`.
|
|
16
|
+
|
|
17
|
+
Functions in this module should be safe to call repeatedly from a deploy hook
|
|
18
|
+
without accumulating side effects.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from typing import TYPE_CHECKING, Any, Literal, Union
|
|
24
|
+
|
|
25
|
+
from .errors import DataJointError
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from .schemas import _Schema
|
|
29
|
+
from .table import Table
|
|
30
|
+
|
|
31
|
+
TargetType = Union["_Schema", type["Table"], "Table"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def set_replica_identity(
|
|
35
|
+
target: "TargetType",
|
|
36
|
+
mode: Literal["default", "full"] = "full",
|
|
37
|
+
dry_run: bool = True,
|
|
38
|
+
) -> dict:
|
|
39
|
+
"""
|
|
40
|
+
Apply ``ALTER TABLE ... REPLICA IDENTITY <mode>`` to a schema or table on PostgreSQL.
|
|
41
|
+
|
|
42
|
+
``REPLICA IDENTITY`` controls how much of the **old row** PostgreSQL writes to
|
|
43
|
+
the write-ahead log on UPDATE/DELETE. Under ``DEFAULT``, only primary-key
|
|
44
|
+
columns appear in WAL; under ``FULL``, the entire old row does.
|
|
45
|
+
|
|
46
|
+
Why this exists
|
|
47
|
+
---------------
|
|
48
|
+
Some change-data-capture (CDC) consumers require the full row pre-image to
|
|
49
|
+
drive their downstream models. The canonical example is **Databricks
|
|
50
|
+
Lakehouse Sync**: tables without ``REPLICA IDENTITY FULL`` are silently
|
|
51
|
+
skipped by the sync — no error, just missing data downstream. Other CDC
|
|
52
|
+
tools (Debezium, ClickHouse ClickPipes, Azure CDC) work fine with
|
|
53
|
+
``DEFAULT`` when tables have a primary key; only Databricks mandates
|
|
54
|
+
``FULL``.
|
|
55
|
+
|
|
56
|
+
This helper is the **operational** way to apply the setting. It is not a
|
|
57
|
+
migration: there's no legacy state being fixed; the setting is simply a
|
|
58
|
+
property of the deployment environment, and a fresh declare in a new
|
|
59
|
+
environment may need it re-applied. It is idempotent — re-applying the
|
|
60
|
+
same mode is a no-op at the storage layer — so it is safe to call from a
|
|
61
|
+
deploy hook on every release.
|
|
62
|
+
|
|
63
|
+
Cost
|
|
64
|
+
----
|
|
65
|
+
The ALTER itself is metadata-only and instant, but requires a brief
|
|
66
|
+
``AccessExclusiveLock`` on each table — it will block behind in-flight
|
|
67
|
+
writes/reads on a busy table. Run during a quiet window on actively-
|
|
68
|
+
ingested tables.
|
|
69
|
+
|
|
70
|
+
The ongoing cost is in WAL volume after the change: UPDATE/DELETE on
|
|
71
|
+
tables with FULL log the entire old row, which can be sizable on tables
|
|
72
|
+
with TOASTed bytea columns. For DataJoint's typical insert-append
|
|
73
|
+
workload, this cost is negligible. The notable scenario is bulk
|
|
74
|
+
``delete()`` on tables with ``<blob>`` columns — a transient WAL burst
|
|
75
|
+
proportional to the deleted-row payload size.
|
|
76
|
+
|
|
77
|
+
Partial-failure semantics
|
|
78
|
+
-------------------------
|
|
79
|
+
If ``connection.query(ddl)`` raises on table N of M, the first N-1
|
|
80
|
+
tables are already modified at the storage layer but the exception
|
|
81
|
+
propagates without returning the partial summary. The operation is
|
|
82
|
+
idempotent, so re-running brings the remaining tables into compliance.
|
|
83
|
+
|
|
84
|
+
Compliance considerations
|
|
85
|
+
-------------------------
|
|
86
|
+
Under ``DEFAULT``, only primary-key values appear in WAL. Under ``FULL``,
|
|
87
|
+
entire rows do — including any PHI/PII/sensitive columns. For self-hosted
|
|
88
|
+
PostgreSQL with unrestricted WAL access this is a real consideration; for
|
|
89
|
+
managed PostgreSQL with logical replication confined to a specific
|
|
90
|
+
subscriber (Lakebase, RDS), WAL stays inside the managed environment's
|
|
91
|
+
security boundary. Apply intentionally.
|
|
92
|
+
|
|
93
|
+
Parameters
|
|
94
|
+
----------
|
|
95
|
+
target : Schema or Table
|
|
96
|
+
A :class:`datajoint.Schema` (all user tables) or a
|
|
97
|
+
:class:`datajoint.Table` class/instance (just that table).
|
|
98
|
+
mode : str, default ``"full"``
|
|
99
|
+
``"default"`` (PK only, minimal WAL) or ``"full"`` (entire row).
|
|
100
|
+
dry_run : bool, default ``True``
|
|
101
|
+
If True, collect the DDL statements but do not execute. Set to False
|
|
102
|
+
to actually apply.
|
|
103
|
+
|
|
104
|
+
Returns
|
|
105
|
+
-------
|
|
106
|
+
dict
|
|
107
|
+
- ``tables_analyzed`` (int): number of tables considered.
|
|
108
|
+
- ``tables_modified`` (int): number of tables on which the ALTER ran.
|
|
109
|
+
Always 0 when ``dry_run=True``.
|
|
110
|
+
- ``ddl`` (list[str]): the DDL statements that were (or would be) executed.
|
|
111
|
+
|
|
112
|
+
Raises
|
|
113
|
+
------
|
|
114
|
+
DataJointError
|
|
115
|
+
If the target's backend is not PostgreSQL, or if ``mode`` is not one of
|
|
116
|
+
``"default"`` / ``"full"``.
|
|
117
|
+
|
|
118
|
+
Examples
|
|
119
|
+
--------
|
|
120
|
+
>>> from datajoint.deploy import set_replica_identity
|
|
121
|
+
>>> # Preview
|
|
122
|
+
>>> set_replica_identity(my_schema, mode="full", dry_run=True)
|
|
123
|
+
{'tables_analyzed': 12, 'tables_modified': 0, 'ddl': ['ALTER TABLE "ms"."t1" REPLICA IDENTITY FULL', ...]}
|
|
124
|
+
>>> # Apply
|
|
125
|
+
>>> set_replica_identity(my_schema, mode="full", dry_run=False)
|
|
126
|
+
{'tables_analyzed': 12, 'tables_modified': 12, 'ddl': [...]}
|
|
127
|
+
>>> # Single table
|
|
128
|
+
>>> set_replica_identity(MyTable, mode="full", dry_run=False)
|
|
129
|
+
|
|
130
|
+
See Also
|
|
131
|
+
--------
|
|
132
|
+
PostgreSQL: `Logical Replication — Replica Identity
|
|
133
|
+
<https://www.postgresql.org/docs/current/logical-replication-publication.html>`_.
|
|
134
|
+
Databricks: `Lakehouse Sync
|
|
135
|
+
<https://docs.databricks.com/aws/en/oltp/projects/lakehouse-sync>`_.
|
|
136
|
+
"""
|
|
137
|
+
mode_normalized = mode.lower() if isinstance(mode, str) else mode
|
|
138
|
+
if mode_normalized not in ("default", "full"):
|
|
139
|
+
raise DataJointError(f"mode must be 'default' or 'full'; got {mode!r}")
|
|
140
|
+
mode = mode_normalized # type: ignore[assignment]
|
|
141
|
+
|
|
142
|
+
from .schemas import _Schema
|
|
143
|
+
from .table import Table
|
|
144
|
+
|
|
145
|
+
if isinstance(target, _Schema):
|
|
146
|
+
connection = target.connection
|
|
147
|
+
if connection is None:
|
|
148
|
+
raise DataJointError("Schema has no active connection.")
|
|
149
|
+
adapter = connection.adapter
|
|
150
|
+
if target.database is None:
|
|
151
|
+
raise DataJointError("Schema is not activated. Call schema.activate(...) before set_replica_identity().")
|
|
152
|
+
tables = [adapter.make_full_table_name(target.database, t) for t in target.list_tables()]
|
|
153
|
+
elif isinstance(target, type) and issubclass(target, Table):
|
|
154
|
+
instance = target()
|
|
155
|
+
connection = instance.connection
|
|
156
|
+
if connection is None:
|
|
157
|
+
raise DataJointError(f"Table {target.__name__} has no active connection.")
|
|
158
|
+
adapter = connection.adapter
|
|
159
|
+
tables = [instance.full_table_name]
|
|
160
|
+
elif isinstance(target, Table):
|
|
161
|
+
connection = target.connection
|
|
162
|
+
if connection is None:
|
|
163
|
+
raise DataJointError(f"Table {type(target).__name__} has no active connection.")
|
|
164
|
+
adapter = connection.adapter
|
|
165
|
+
tables = [target.full_table_name]
|
|
166
|
+
else:
|
|
167
|
+
raise DataJointError(f"target must be a Schema or Table class/instance; got {type(target).__name__}")
|
|
168
|
+
|
|
169
|
+
if not hasattr(adapter, "replica_identity_ddl"):
|
|
170
|
+
raise DataJointError(
|
|
171
|
+
f"set_replica_identity is PostgreSQL-only; the {adapter.backend} adapter does not support REPLICA IDENTITY."
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
result: dict[str, Any] = {
|
|
175
|
+
"tables_analyzed": len(tables),
|
|
176
|
+
"tables_modified": 0,
|
|
177
|
+
"ddl": [],
|
|
178
|
+
}
|
|
179
|
+
for full_name in tables:
|
|
180
|
+
ddl = adapter.replica_identity_ddl(full_name, mode) # type: ignore[attr-defined]
|
|
181
|
+
result["ddl"].append(ddl)
|
|
182
|
+
if not dry_run:
|
|
183
|
+
connection.query(ddl)
|
|
184
|
+
result["tables_modified"] += 1
|
|
185
|
+
return result
|