datajoint 2.3.3__tar.gz → 2.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datajoint-2.3.3 → datajoint-2.3.4}/PKG-INFO +1 -1
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/__init__.py +4 -1
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/_version.py +2 -2
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/base.py +0 -23
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/mysql.py +0 -17
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/postgres.py +0 -17
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/autopopulate.py +9 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/__init__.py +2 -2
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/attach.py +5 -3
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/filepath.py +6 -4
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/hash.py +15 -6
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/npy.py +5 -4
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/object.py +5 -4
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/schema.py +48 -17
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/codecs.py +85 -6
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/declare.py +91 -50
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/deploy.py +137 -1
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/heading.py +7 -3
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/migrate.py +36 -21
- datajoint-2.3.4/src/datajoint/provenance.py +140 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/schemas.py +3 -3
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/settings.py +46 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/spark.py +2 -2
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/table.py +69 -8
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/user_tables.py +19 -1
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/conftest.py +6 -1
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_declare.py +28 -0
- datajoint-2.3.4/tests/integration/test_entry_provenance.py +276 -0
- datajoint-2.3.4/tests/integration/test_migrate_job_metadata.py +135 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_object.py +12 -12
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_adapters.py +32 -19
- datajoint-2.3.4/tests/unit/test_codec_context.py +172 -0
- datajoint-2.3.4/tests/unit/test_declare_hidden_attribute.py +146 -0
- datajoint-2.3.4/tests/unit/test_provenance.py +193 -0
- datajoint-2.3.4/tests/unit/test_tier_aliases.py +80 -0
- datajoint-2.3.3/tests/unit/test_declare_hidden_attribute.py +0 -51
- {datajoint-2.3.3 → datajoint-2.3.4}/.coveragerc +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.dockerignore +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.gitattributes +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/dependabot.yml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/release_drafter.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/draft_release.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/lint.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/post_draft_release_published.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.github/workflows/test.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.gitignore +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/.pre-commit-config.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/CHANGELOG-archive.md +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/CONTRIBUTING.md +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/Dockerfile +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/LICENSE +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/README.md +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/RELEASE_MEMO.md +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/activate.sh +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/docker-compose.yaml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/images/pipeline.svg +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/pixi.lock +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/pyproject.toml +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/adapters/__init__.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/blob.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/builtin_codecs/blob.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/cli.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/condition.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/connection.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/dependencies.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/diagram.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/errors.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/expression.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/gc.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/hash_registry.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/instance.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/jobs.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/lineage.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/logging.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/objectref.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/preview.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/staged_insert.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/storage.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/storage_adapter.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/types.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/utils.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/src/datajoint/version.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/__init__.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/__init__.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Course.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/CurrentTerm.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Department.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Enroll.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Grade.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Section.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Student.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/StudentMajor.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/data/Term.csv +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_aggr_regressions.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_alter.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_attach.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_autopopulate.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_blob.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_blob_matlab.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cascade_delete.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cascade_integrity.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cascading_delete.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_cli.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_codec_chaining.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_codecs.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_connection.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_dependencies.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_bundle_edge.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_edge_weight.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_nocontext.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_diagram_style.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_fetch.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_fetch_same.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_fk_index.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_foreign_keys.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_gc.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_groupby.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_hash_storage.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_hidden_job_metadata.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_insert.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_jobs.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_json.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_multi_backend.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_nan.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_npy_codec.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_privileges.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_reconnection.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_relation.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_relation_u.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_relational_operand.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_schema.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_schema_keywords.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_semantic_matching.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_skip_duplicates.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_tls.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_trace.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_type_aliases.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_university.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_update1.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_utils.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_uuid.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/integration/test_virtual_module.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_advanced.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_aggr_regress.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_alter.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_codecs.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_external.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_object.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_privileges.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_simple.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_type_aliases.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_university.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/schema_uuid.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/test_package.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/__init__.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_codecs.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_condition.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_connection_warning.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_ddl_substitution.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_deploy.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_fetch_compat.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_gc.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_lazy_imports.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_logging.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_pk_rules.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_settings.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_spark.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_storage_adapter.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_storage_urls.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_thread_safe.py +0 -0
- {datajoint-2.3.3 → datajoint-2.3.4}/tests/unit/test_type_patterns.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: datajoint
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.4
|
|
4
4
|
Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
|
|
5
5
|
Project-URL: Homepage, https://docs.datajoint.com/
|
|
6
6
|
Project-URL: Documentation, https://docs.datajoint.com/
|
|
@@ -33,9 +33,12 @@ __all__ = [
|
|
|
33
33
|
"AutoPopulate",
|
|
34
34
|
"Job",
|
|
35
35
|
"Manual",
|
|
36
|
+
"Entry",
|
|
36
37
|
"Lookup",
|
|
37
38
|
"Imported",
|
|
39
|
+
"Ingest",
|
|
38
40
|
"Computed",
|
|
41
|
+
"Compute",
|
|
39
42
|
"Part",
|
|
40
43
|
"Not",
|
|
41
44
|
"AndList",
|
|
@@ -95,7 +98,7 @@ from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
|
|
|
95
98
|
from .autopopulate import AutoPopulate
|
|
96
99
|
from .jobs import Job
|
|
97
100
|
from .table import FreeTable as _FreeTable, Table, ValidationResult
|
|
98
|
-
from .user_tables import Computed, Imported, Lookup, Manual, Part
|
|
101
|
+
from .user_tables import Compute, Computed, Entry, Imported, Ingest, Lookup, Manual, Part
|
|
99
102
|
from .version import __version__
|
|
100
103
|
|
|
101
104
|
# =============================================================================
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '2.3.
|
|
22
|
-
__version_tuple__ = version_tuple = (2, 3,
|
|
21
|
+
__version__ = version = '2.3.4'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 3, 4)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -1317,29 +1317,6 @@ class DatabaseAdapter(ABC):
|
|
|
1317
1317
|
"""
|
|
1318
1318
|
...
|
|
1319
1319
|
|
|
1320
|
-
@abstractmethod
|
|
1321
|
-
def job_metadata_columns(self) -> list[str]:
|
|
1322
|
-
"""
|
|
1323
|
-
Return job metadata column definitions for Computed/Imported tables.
|
|
1324
|
-
|
|
1325
|
-
Returns
|
|
1326
|
-
-------
|
|
1327
|
-
list[str]
|
|
1328
|
-
List of column definition strings (fully formatted with quotes).
|
|
1329
|
-
|
|
1330
|
-
Examples
|
|
1331
|
-
--------
|
|
1332
|
-
MySQL:
|
|
1333
|
-
["`_job_start_time` datetime(3) DEFAULT NULL",
|
|
1334
|
-
"`_job_duration` float DEFAULT NULL",
|
|
1335
|
-
"`_job_version` varchar(64) DEFAULT ''"]
|
|
1336
|
-
PostgreSQL:
|
|
1337
|
-
['"_job_start_time" timestamp DEFAULT NULL',
|
|
1338
|
-
'"_job_duration" real DEFAULT NULL',
|
|
1339
|
-
'"_job_version" varchar(64) DEFAULT \'\'']
|
|
1340
|
-
"""
|
|
1341
|
-
...
|
|
1342
|
-
|
|
1343
1320
|
# =========================================================================
|
|
1344
1321
|
# Error Translation
|
|
1345
1322
|
# =========================================================================
|
|
@@ -996,23 +996,6 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
996
996
|
"""
|
|
997
997
|
return None # MySQL uses inline enum
|
|
998
998
|
|
|
999
|
-
def job_metadata_columns(self) -> list[str]:
|
|
1000
|
-
"""
|
|
1001
|
-
Return MySQL-specific job metadata column definitions.
|
|
1002
|
-
|
|
1003
|
-
Examples
|
|
1004
|
-
--------
|
|
1005
|
-
>>> adapter.job_metadata_columns()
|
|
1006
|
-
["`_job_start_time` datetime(3) DEFAULT NULL",
|
|
1007
|
-
"`_job_duration` float DEFAULT NULL",
|
|
1008
|
-
"`_job_version` varchar(64) DEFAULT ''"]
|
|
1009
|
-
"""
|
|
1010
|
-
return [
|
|
1011
|
-
"`_job_start_time` datetime(3) DEFAULT NULL",
|
|
1012
|
-
"`_job_duration` float DEFAULT NULL",
|
|
1013
|
-
"`_job_version` varchar(64) DEFAULT ''",
|
|
1014
|
-
]
|
|
1015
|
-
|
|
1016
999
|
# =========================================================================
|
|
1017
1000
|
# Error Translation
|
|
1018
1001
|
# =========================================================================
|
|
@@ -1349,23 +1349,6 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
1349
1349
|
self._pending_enum_types = {}
|
|
1350
1350
|
return ddl_statements
|
|
1351
1351
|
|
|
1352
|
-
def job_metadata_columns(self) -> list[str]:
|
|
1353
|
-
"""
|
|
1354
|
-
Return PostgreSQL-specific job metadata column definitions.
|
|
1355
|
-
|
|
1356
|
-
Examples
|
|
1357
|
-
--------
|
|
1358
|
-
>>> adapter.job_metadata_columns()
|
|
1359
|
-
['"_job_start_time" timestamp DEFAULT NULL',
|
|
1360
|
-
'"_job_duration" real DEFAULT NULL',
|
|
1361
|
-
'"_job_version" varchar(64) DEFAULT \\'\\'']
|
|
1362
|
-
"""
|
|
1363
|
-
return [
|
|
1364
|
-
'"_job_start_time" timestamp DEFAULT NULL',
|
|
1365
|
-
'"_job_duration" real DEFAULT NULL',
|
|
1366
|
-
"\"_job_version\" varchar(64) DEFAULT ''",
|
|
1367
|
-
]
|
|
1368
|
-
|
|
1369
1352
|
# =========================================================================
|
|
1370
1353
|
# Error Translation
|
|
1371
1354
|
# =========================================================================
|
|
@@ -11,6 +11,7 @@ import signal
|
|
|
11
11
|
import traceback
|
|
12
12
|
from typing import TYPE_CHECKING, Any, Generator
|
|
13
13
|
|
|
14
|
+
from . import provenance
|
|
14
15
|
from .errors import DataJointError, LostConnectionError
|
|
15
16
|
from .expression import AndList, QueryExpression
|
|
16
17
|
|
|
@@ -684,6 +685,13 @@ class AutoPopulate:
|
|
|
684
685
|
self._upstream_key = dict(key)
|
|
685
686
|
self._upstream = None
|
|
686
687
|
|
|
688
|
+
# Rows this make() writes into Entry tables that carry no foreign key back
|
|
689
|
+
# here -- the fan-out ingestion pattern -- record the ingesting table and
|
|
690
|
+
# key, which is what makes such a row traceable without one.
|
|
691
|
+
from .jobs import _get_job_version
|
|
692
|
+
|
|
693
|
+
prov_token = provenance.set_ingesting(self.full_table_name, key, _get_job_version(self.connection._config))
|
|
694
|
+
|
|
687
695
|
try:
|
|
688
696
|
if not is_generator:
|
|
689
697
|
make(dict(key), **(make_kwargs or {}))
|
|
@@ -740,6 +748,7 @@ class AutoPopulate:
|
|
|
740
748
|
jobs.complete(key, duration=duration)
|
|
741
749
|
return True
|
|
742
750
|
finally:
|
|
751
|
+
provenance.reset_ingesting(prov_token)
|
|
743
752
|
self.__class__._allow_insert = False
|
|
744
753
|
# Clear the per-make() upstream state: `_upstream = None` invalidates
|
|
745
754
|
# the memoized Diagram; `_upstream_key = None` restores the "outside
|
|
@@ -29,14 +29,14 @@ Example - Creating a Custom Codec:
|
|
|
29
29
|
def get_dtype(self, is_store: bool) -> str:
|
|
30
30
|
return "<blob>" # Compose with blob for serialization
|
|
31
31
|
|
|
32
|
-
def encode(self, graph, *, key=None, store_name=None):
|
|
32
|
+
def encode(self, graph, *, key=None, context=None, store_name=None):
|
|
33
33
|
# Convert graph to a serializable format
|
|
34
34
|
return {
|
|
35
35
|
'nodes': list(graph.nodes(data=True)),
|
|
36
36
|
'edges': list(graph.edges(data=True)),
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
def decode(self, stored, *, key=None):
|
|
39
|
+
def decode(self, stored, *, key=None, context=None):
|
|
40
40
|
# Reconstruct graph from stored format
|
|
41
41
|
G = nx.Graph()
|
|
42
42
|
G.add_nodes_from(stored['nodes'])
|
|
@@ -50,7 +50,9 @@ class AttachCodec(Codec):
|
|
|
50
50
|
"""Return bytes for in-table, <hash> for in-store storage."""
|
|
51
51
|
return "<hash>" if is_store else "bytes"
|
|
52
52
|
|
|
53
|
-
def encode(
|
|
53
|
+
def encode(
|
|
54
|
+
self, value: Any, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
|
|
55
|
+
) -> bytes:
|
|
54
56
|
"""
|
|
55
57
|
Read file and encode as filename + contents.
|
|
56
58
|
|
|
@@ -80,7 +82,7 @@ class AttachCodec(Codec):
|
|
|
80
82
|
contents = path.read_bytes()
|
|
81
83
|
return filename.encode("utf-8") + b"\x00" + contents
|
|
82
84
|
|
|
83
|
-
def decode(self, stored: bytes, *, key: dict | None = None) -> str:
|
|
85
|
+
def decode(self, stored: bytes, *, key: dict | None = None, context: dict | None = None) -> str:
|
|
84
86
|
"""
|
|
85
87
|
Extract file to download path and return local path.
|
|
86
88
|
|
|
@@ -104,7 +106,7 @@ class AttachCodec(Codec):
|
|
|
104
106
|
contents = stored[null_pos + 1 :]
|
|
105
107
|
|
|
106
108
|
# Write to download path
|
|
107
|
-
config = (key
|
|
109
|
+
config = self._codec_config(key, context)
|
|
108
110
|
if config is None:
|
|
109
111
|
from ..settings import config # type: ignore[assignment]
|
|
110
112
|
assert config is not None
|
|
@@ -74,7 +74,9 @@ class FilepathCodec(Codec):
|
|
|
74
74
|
)
|
|
75
75
|
return "json"
|
|
76
76
|
|
|
77
|
-
def encode(
|
|
77
|
+
def encode(
|
|
78
|
+
self, value: Any, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
|
|
79
|
+
) -> dict:
|
|
78
80
|
"""
|
|
79
81
|
Store path reference as JSON metadata.
|
|
80
82
|
|
|
@@ -104,7 +106,7 @@ class FilepathCodec(Codec):
|
|
|
104
106
|
|
|
105
107
|
from ..hash_registry import get_store_backend
|
|
106
108
|
|
|
107
|
-
config = (key
|
|
109
|
+
config = self._codec_config(key, context)
|
|
108
110
|
if config is None:
|
|
109
111
|
from ..settings import config # type: ignore[assignment]
|
|
110
112
|
assert config is not None
|
|
@@ -168,7 +170,7 @@ class FilepathCodec(Codec):
|
|
|
168
170
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
169
171
|
}
|
|
170
172
|
|
|
171
|
-
def decode(self, stored: dict, *, key: dict | None = None) -> Any:
|
|
173
|
+
def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> Any:
|
|
172
174
|
"""
|
|
173
175
|
Create ObjectRef handle for lazy access.
|
|
174
176
|
|
|
@@ -187,7 +189,7 @@ class FilepathCodec(Codec):
|
|
|
187
189
|
from ..objectref import ObjectRef
|
|
188
190
|
from ..hash_registry import get_store_backend
|
|
189
191
|
|
|
190
|
-
config = (key
|
|
192
|
+
config = self._codec_config(key, context)
|
|
191
193
|
store_name = stored.get("store")
|
|
192
194
|
backend = get_store_backend(store_name, config=config)
|
|
193
195
|
return ObjectRef.from_json(stored, backend=backend)
|
|
@@ -57,7 +57,9 @@ class HashCodec(Codec):
|
|
|
57
57
|
raise DataJointError("<hash> requires @ (in-store storage only)")
|
|
58
58
|
return "json"
|
|
59
59
|
|
|
60
|
-
def encode(
|
|
60
|
+
def encode(
|
|
61
|
+
self, value: bytes, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
|
|
62
|
+
) -> dict:
|
|
61
63
|
"""
|
|
62
64
|
Store content and return metadata.
|
|
63
65
|
|
|
@@ -66,7 +68,11 @@ class HashCodec(Codec):
|
|
|
66
68
|
value : bytes
|
|
67
69
|
Raw bytes to store.
|
|
68
70
|
key : dict, optional
|
|
69
|
-
|
|
71
|
+
Primary key values. Before 2.3.4 this also carried the schema name
|
|
72
|
+
under ``_schema``; that is still read when ``context`` omits it.
|
|
73
|
+
context : dict, optional
|
|
74
|
+
Connection context. ``schema`` isolates the stored path and
|
|
75
|
+
``config`` resolves the store.
|
|
70
76
|
store_name : str, optional
|
|
71
77
|
Store to use. If None, uses default store.
|
|
72
78
|
|
|
@@ -77,11 +83,14 @@ class HashCodec(Codec):
|
|
|
77
83
|
"""
|
|
78
84
|
from ..hash_registry import put_hash
|
|
79
85
|
|
|
80
|
-
|
|
81
|
-
|
|
86
|
+
# Same precedence as `_extract_context`. Reading `key` alone would store
|
|
87
|
+
# under "unknown" for any caller that passes `context` with a key holding
|
|
88
|
+
# only primary-key values -- which is every caller once 2.4 lands.
|
|
89
|
+
schema_name = (context or {}).get("schema", (key or {}).get("_schema", "unknown"))
|
|
90
|
+
config = self._codec_config(key, context)
|
|
82
91
|
return put_hash(value, schema_name=schema_name, store_name=store_name, config=config)
|
|
83
92
|
|
|
84
|
-
def decode(self, stored: dict, *, key: dict | None = None) -> bytes:
|
|
93
|
+
def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> bytes:
|
|
85
94
|
"""
|
|
86
95
|
Retrieve content using stored metadata.
|
|
87
96
|
|
|
@@ -99,7 +108,7 @@ class HashCodec(Codec):
|
|
|
99
108
|
"""
|
|
100
109
|
from ..hash_registry import get_hash
|
|
101
110
|
|
|
102
|
-
config = (key
|
|
111
|
+
config = self._codec_config(key, context)
|
|
103
112
|
return get_hash(stored, config=config)
|
|
104
113
|
|
|
105
114
|
def validate(self, value: Any) -> None:
|
|
@@ -312,6 +312,7 @@ class NpyCodec(SchemaCodec):
|
|
|
312
312
|
value: Any,
|
|
313
313
|
*,
|
|
314
314
|
key: dict | None = None,
|
|
315
|
+
context: dict | None = None,
|
|
315
316
|
store_name: str | None = None,
|
|
316
317
|
) -> dict:
|
|
317
318
|
"""
|
|
@@ -337,8 +338,8 @@ class NpyCodec(SchemaCodec):
|
|
|
337
338
|
import numpy as np
|
|
338
339
|
|
|
339
340
|
# Extract context using inherited helper
|
|
340
|
-
schema, table, field, primary_key = self._extract_context(key)
|
|
341
|
-
config = (key
|
|
341
|
+
schema, table, field, primary_key = self._extract_context(key, context)
|
|
342
|
+
config = self._codec_config(key, context)
|
|
342
343
|
|
|
343
344
|
# Build schema-addressed storage path
|
|
344
345
|
path, _ = self._build_path(schema, table, field, primary_key, ext=".npy", store_name=store_name, config=config)
|
|
@@ -360,7 +361,7 @@ class NpyCodec(SchemaCodec):
|
|
|
360
361
|
"shape": list(value.shape),
|
|
361
362
|
}
|
|
362
363
|
|
|
363
|
-
def decode(self, stored: dict, *, key: dict | None = None) -> NpyRef:
|
|
364
|
+
def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> NpyRef:
|
|
364
365
|
"""
|
|
365
366
|
Create lazy NpyRef from stored metadata.
|
|
366
367
|
|
|
@@ -376,6 +377,6 @@ class NpyCodec(SchemaCodec):
|
|
|
376
377
|
NpyRef
|
|
377
378
|
Lazy array reference with metadata access and numpy integration.
|
|
378
379
|
"""
|
|
379
|
-
config = (key
|
|
380
|
+
config = self._codec_config(key, context)
|
|
380
381
|
backend = self._get_backend(stored.get("store"), config=config)
|
|
381
382
|
return NpyRef(stored, backend)
|
|
@@ -81,6 +81,7 @@ class ObjectCodec(SchemaCodec):
|
|
|
81
81
|
value: Any,
|
|
82
82
|
*,
|
|
83
83
|
key: dict | None = None,
|
|
84
|
+
context: dict | None = None,
|
|
84
85
|
store_name: str | None = None,
|
|
85
86
|
) -> dict:
|
|
86
87
|
"""
|
|
@@ -105,8 +106,8 @@ class ObjectCodec(SchemaCodec):
|
|
|
105
106
|
from pathlib import Path
|
|
106
107
|
|
|
107
108
|
# Extract context using inherited helper
|
|
108
|
-
schema, table, field, primary_key = self._extract_context(key)
|
|
109
|
-
config = (key
|
|
109
|
+
schema, table, field, primary_key = self._extract_context(key, context)
|
|
110
|
+
config = self._codec_config(key, context)
|
|
110
111
|
|
|
111
112
|
# Check for pre-computed metadata (from staged insert)
|
|
112
113
|
if isinstance(value, dict) and "path" in value:
|
|
@@ -177,7 +178,7 @@ class ObjectCodec(SchemaCodec):
|
|
|
177
178
|
|
|
178
179
|
return metadata
|
|
179
180
|
|
|
180
|
-
def decode(self, stored: dict, *, key: dict | None = None) -> Any:
|
|
181
|
+
def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> Any:
|
|
181
182
|
"""
|
|
182
183
|
Create ObjectRef handle for lazy access.
|
|
183
184
|
|
|
@@ -195,7 +196,7 @@ class ObjectCodec(SchemaCodec):
|
|
|
195
196
|
"""
|
|
196
197
|
from ..objectref import ObjectRef
|
|
197
198
|
|
|
198
|
-
config = (key
|
|
199
|
+
config = self._codec_config(key, context)
|
|
199
200
|
backend = self._get_backend(stored.get("store"), config=config)
|
|
200
201
|
return ObjectRef.from_json(stored, backend=backend)
|
|
201
202
|
|
|
@@ -4,6 +4,8 @@ Schema-addressed storage base class.
|
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import warnings
|
|
8
|
+
|
|
7
9
|
from ..codecs import Codec
|
|
8
10
|
from ..errors import DataJointError
|
|
9
11
|
|
|
@@ -24,14 +26,27 @@ class SchemaCodec(Codec, register=False):
|
|
|
24
26
|
- ``validate()``: Validate input values
|
|
25
27
|
|
|
26
28
|
Helper Methods:
|
|
27
|
-
- ``_extract_context()``: Parse key
|
|
29
|
+
- ``_extract_context()``: Parse key/context into schema/table/field/pk
|
|
30
|
+
- ``_codec_config()``: Read the calling connection's config
|
|
28
31
|
- ``_build_path()``: Construct storage path from context
|
|
29
32
|
- ``_get_backend()``: Get storage backend by name
|
|
30
33
|
|
|
31
|
-
|
|
32
|
-
without one.
|
|
33
|
-
|
|
34
|
-
|
|
34
|
+
``_build_path`` and ``_get_backend`` take a ``config`` and fall back to the
|
|
35
|
+
global ``dj.config`` without one. Always pass the calling connection's
|
|
36
|
+
config: in a process holding connections for several users, the global one
|
|
37
|
+
belongs to none of them, and the fallback resolves a different store
|
|
38
|
+
silently rather than raising.
|
|
39
|
+
|
|
40
|
+
Since 2.3.4 that config arrives in an explicit ``context`` argument rather
|
|
41
|
+
than hidden among the primary key values. **Declare ``context=None`` in
|
|
42
|
+
``encode`` and ``decode`` and pass it to the helpers**, as the example below
|
|
43
|
+
does.
|
|
44
|
+
|
|
45
|
+
The old underscore keys in ``key`` still work and are still populated, so a
|
|
46
|
+
codec written before 2.3.4 keeps running. They are removed in 2.4.
|
|
47
|
+
``_extract_context`` warns when it falls back to them; ``_codec_config`` and
|
|
48
|
+
a direct ``key["_config"]`` do not, so the warning is a prompt rather than a
|
|
49
|
+
complete inventory of what still reads them.
|
|
35
50
|
|
|
36
51
|
Comparison with Hash-addressed:
|
|
37
52
|
- **Schema-addressed** (this): Path from schema structure, no dedup
|
|
@@ -42,9 +57,9 @@ class SchemaCodec(Codec, register=False):
|
|
|
42
57
|
class MyCodec(SchemaCodec):
|
|
43
58
|
name = "my"
|
|
44
59
|
|
|
45
|
-
def encode(self, value, *, key=None, store_name=None):
|
|
46
|
-
schema, table, field, pk = self._extract_context(key)
|
|
47
|
-
config = (key
|
|
60
|
+
def encode(self, value, *, key=None, context=None, store_name=None):
|
|
61
|
+
schema, table, field, pk = self._extract_context(key, context)
|
|
62
|
+
config = self._codec_config(key, context)
|
|
48
63
|
path, _ = self._build_path(
|
|
49
64
|
schema, table, field, pk, ext=".dat",
|
|
50
65
|
store_name=store_name, config=config,
|
|
@@ -53,8 +68,8 @@ class SchemaCodec(Codec, register=False):
|
|
|
53
68
|
backend.put_buffer(serialize(value), path)
|
|
54
69
|
return {"path": path, "store": store_name, ...}
|
|
55
70
|
|
|
56
|
-
def decode(self, stored, *, key=None):
|
|
57
|
-
config = (key
|
|
71
|
+
def decode(self, stored, *, key=None, context=None):
|
|
72
|
+
config = self._codec_config(key, context)
|
|
58
73
|
backend = self._get_backend(stored.get("store"), config=config)
|
|
59
74
|
return MyRef(stored, backend)
|
|
60
75
|
|
|
@@ -88,15 +103,21 @@ class SchemaCodec(Codec, register=False):
|
|
|
88
103
|
raise DataJointError(f"<{self.name}> requires @ (store only)")
|
|
89
104
|
return "json"
|
|
90
105
|
|
|
91
|
-
def _extract_context(self, key: dict | None) -> tuple[str, str, str, dict]:
|
|
106
|
+
def _extract_context(self, key: dict | None, context: dict | None = None) -> tuple[str, str, str, dict]:
|
|
92
107
|
"""
|
|
93
|
-
Extract schema, table, field, and primary key
|
|
108
|
+
Extract schema, table, field, and primary key.
|
|
94
109
|
|
|
95
110
|
Parameters
|
|
96
111
|
----------
|
|
97
112
|
key : dict or None
|
|
98
|
-
|
|
99
|
-
|
|
113
|
+
Primary key values. Before 2.3.4 this also carried connection
|
|
114
|
+
context under ``_schema``, ``_table``, ``_field`` and ``_config``;
|
|
115
|
+
those keys are still populated and still read, with a
|
|
116
|
+
``DeprecationWarning``, when ``context`` is not supplied.
|
|
117
|
+
context : dict or None
|
|
118
|
+
Connection context with ``schema``, ``table``, ``field`` and
|
|
119
|
+
``config``. Pass the ``context`` argument your ``encode``/``decode``
|
|
120
|
+
received.
|
|
100
121
|
|
|
101
122
|
Returns
|
|
102
123
|
-------
|
|
@@ -104,9 +125,19 @@ class SchemaCodec(Codec, register=False):
|
|
|
104
125
|
``(schema, table, field, primary_key)``
|
|
105
126
|
"""
|
|
106
127
|
key = dict(key) if key else {}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
128
|
+
if context is None and any(k.startswith("_") for k in key):
|
|
129
|
+
warnings.warn(
|
|
130
|
+
"Reading connection context from the `key` dict is deprecated and will "
|
|
131
|
+
"be removed in DataJoint 2.4. Accept a `context` argument in encode()/decode() "
|
|
132
|
+
"and pass it to _extract_context(key, context). See "
|
|
133
|
+
"https://github.com/datajoint/datajoint-python/issues/1550",
|
|
134
|
+
DeprecationWarning,
|
|
135
|
+
stacklevel=2,
|
|
136
|
+
)
|
|
137
|
+
context = context or {}
|
|
138
|
+
schema = context.get("schema", key.pop("_schema", "unknown"))
|
|
139
|
+
table = context.get("table", key.pop("_table", "unknown"))
|
|
140
|
+
field = context.get("field", key.pop("_field", "data"))
|
|
110
141
|
primary_key = {k: v for k, v in key.items() if not k.startswith("_")}
|
|
111
142
|
return schema, table, field, primary_key
|
|
112
143
|
|
|
@@ -16,10 +16,10 @@ class GraphCodec(dj.Codec):
|
|
|
16
16
|
def get_dtype(self, is_store: bool) -> str:
|
|
17
17
|
return "<blob>"
|
|
18
18
|
|
|
19
|
-
def encode(self, graph, *, key=None, store_name=None):
|
|
19
|
+
def encode(self, graph, *, key=None, context=None, store_name=None):
|
|
20
20
|
return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())}
|
|
21
21
|
|
|
22
|
-
def decode(self, stored, *, key=None):
|
|
22
|
+
def decode(self, stored, *, key=None, context=None):
|
|
23
23
|
import networkx as nx
|
|
24
24
|
G = nx.Graph()
|
|
25
25
|
G.add_nodes_from(stored['nodes'])
|
|
@@ -38,6 +38,8 @@ class MyTable(dj.Manual):
|
|
|
38
38
|
|
|
39
39
|
from __future__ import annotations
|
|
40
40
|
|
|
41
|
+
import functools
|
|
42
|
+
import inspect
|
|
41
43
|
import json
|
|
42
44
|
import logging
|
|
43
45
|
from abc import ABC, abstractmethod
|
|
@@ -79,10 +81,10 @@ class Codec(ABC):
|
|
|
79
81
|
... def get_dtype(self, is_store: bool) -> str:
|
|
80
82
|
... return "<blob>"
|
|
81
83
|
...
|
|
82
|
-
... def encode(self, graph, *, key=None, store_name=None):
|
|
84
|
+
... def encode(self, graph, *, key=None, context=None, store_name=None):
|
|
83
85
|
... return {'nodes': list(graph.nodes()), 'edges': list(graph.edges())}
|
|
84
86
|
...
|
|
85
|
-
... def decode(self, stored, *, key=None):
|
|
87
|
+
... def decode(self, stored, *, key=None, context=None):
|
|
86
88
|
... import networkx as nx
|
|
87
89
|
... G = nx.Graph()
|
|
88
90
|
... G.add_nodes_from(stored['nodes'])
|
|
@@ -176,6 +178,18 @@ class Codec(ABC):
|
|
|
176
178
|
-------
|
|
177
179
|
any
|
|
178
180
|
Value in the format expected by the dtype.
|
|
181
|
+
|
|
182
|
+
Notes
|
|
183
|
+
-----
|
|
184
|
+
**Declare ``context`` as well**: ``encode(self, value, *, key=None,
|
|
185
|
+
context=None, store_name=None)``. It carries ``schema``, ``table``,
|
|
186
|
+
``field`` and ``config`` — the calling connection's configuration, which
|
|
187
|
+
:meth:`_codec_config` reads and which store resolution needs.
|
|
188
|
+
|
|
189
|
+
Before 2.3.4 those four arrived inside ``key`` under underscore-prefixed
|
|
190
|
+
names. That path still works and DataJoint still populates it, so a codec
|
|
191
|
+
written against it keeps running — but it is deprecated and removed in
|
|
192
|
+
2.4. Write new codecs against ``context``.
|
|
179
193
|
"""
|
|
180
194
|
...
|
|
181
195
|
|
|
@@ -195,9 +209,45 @@ class Codec(ABC):
|
|
|
195
209
|
-------
|
|
196
210
|
any
|
|
197
211
|
The reconstructed Python object.
|
|
212
|
+
|
|
213
|
+
Notes
|
|
214
|
+
-----
|
|
215
|
+
**Declare ``context`` as well**: ``decode(self, stored, *, key=None,
|
|
216
|
+
context=None)``. On this path it carries ``config`` only — the stored
|
|
217
|
+
metadata already holds the location, so ``schema``, ``table`` and
|
|
218
|
+
``field`` are not resolved again. Pass it to :meth:`_codec_config` for
|
|
219
|
+
the same reason :meth:`encode` does.
|
|
198
220
|
"""
|
|
199
221
|
...
|
|
200
222
|
|
|
223
|
+
@staticmethod
|
|
224
|
+
def _codec_config(key: dict | None = None, context: dict | None = None):
|
|
225
|
+
"""
|
|
226
|
+
Return the calling connection's config, or None.
|
|
227
|
+
|
|
228
|
+
Always thread the result into ``_build_path`` and ``_get_backend``. Those
|
|
229
|
+
helpers fall back to the global ``dj.config`` without it, which in a
|
|
230
|
+
process holding connections for several users belongs to none of them and
|
|
231
|
+
resolves a different store silently rather than raising.
|
|
232
|
+
|
|
233
|
+
Prefers ``context["config"]``. Falls back to ``key["_config"]``, the
|
|
234
|
+
pre-2.3.4 location, which DataJoint still populates.
|
|
235
|
+
|
|
236
|
+
Parameters
|
|
237
|
+
----------
|
|
238
|
+
key : dict, optional
|
|
239
|
+
The ``key`` argument the codec received.
|
|
240
|
+
context : dict, optional
|
|
241
|
+
The ``context`` argument the codec received, if it declares one.
|
|
242
|
+
|
|
243
|
+
Returns
|
|
244
|
+
-------
|
|
245
|
+
Config or None
|
|
246
|
+
"""
|
|
247
|
+
if context and context.get("config") is not None:
|
|
248
|
+
return context["config"]
|
|
249
|
+
return (key or {}).get("_config")
|
|
250
|
+
|
|
201
251
|
def validate(self, value: Any) -> None:
|
|
202
252
|
"""
|
|
203
253
|
Validate a value before encoding.
|
|
@@ -562,6 +612,28 @@ def lookup_codec(codec_spec: str) -> tuple[Codec, str | None]:
|
|
|
562
612
|
# =============================================================================
|
|
563
613
|
|
|
564
614
|
|
|
615
|
+
@functools.lru_cache(maxsize=None)
|
|
616
|
+
def _accepts_kwarg(func, name: str) -> bool:
|
|
617
|
+
"""
|
|
618
|
+
Whether ``func`` declares a keyword parameter ``name``.
|
|
619
|
+
|
|
620
|
+
Used to offer optional arguments -- ``store_name``, ``context`` -- only to
|
|
621
|
+
codecs whose signature declares them, so a codec written before either
|
|
622
|
+
existed is called exactly as it was.
|
|
623
|
+
|
|
624
|
+
Cached on the underlying function: ``inspect.signature`` costs several
|
|
625
|
+
times more than a small ``encode`` call, and this runs per attribute per
|
|
626
|
+
row. Pass the unbound function (``type(codec).encode``), which is stable
|
|
627
|
+
per class, rather than a bound method, which is not.
|
|
628
|
+
|
|
629
|
+
Introspection rather than calling and catching ``TypeError``: an ``encode``
|
|
630
|
+
body serializes and uploads, so a ``TypeError`` raised inside it would be
|
|
631
|
+
indistinguishable from an unexpected-keyword error at the call site, and
|
|
632
|
+
retrying would both mask the real failure and repeat the upload.
|
|
633
|
+
"""
|
|
634
|
+
return name in inspect.signature(func).parameters
|
|
635
|
+
|
|
636
|
+
|
|
565
637
|
def decode_attribute(attr, data, squeeze: bool = False, connection=None):
|
|
566
638
|
"""
|
|
567
639
|
Decode raw database value using attribute's codec or native type handling.
|
|
@@ -617,14 +689,21 @@ def decode_attribute(attr, data, squeeze: bool = False, connection=None):
|
|
|
617
689
|
elif final_dtype.lower() == "binary(16)":
|
|
618
690
|
data = uuid_module.UUID(bytes=data)
|
|
619
691
|
|
|
620
|
-
# Build decode key with config if connection is available
|
|
692
|
+
# Build decode key with config if connection is available. The
|
|
693
|
+
# underscore key stays for codecs written against it; `context` carries
|
|
694
|
+
# the same config to codecs that declare one -- see #1550.
|
|
621
695
|
decode_key = None
|
|
696
|
+
decode_context = None
|
|
622
697
|
if connection is not None:
|
|
623
698
|
decode_key = {"_config": connection._config}
|
|
699
|
+
decode_context = {"config": connection._config}
|
|
624
700
|
|
|
625
701
|
# Apply decoders in reverse order: innermost first, then outermost
|
|
626
702
|
for codec in reversed(type_chain):
|
|
627
|
-
|
|
703
|
+
if _accepts_kwarg(type(codec).decode, "context"):
|
|
704
|
+
data = codec.decode(data, key=decode_key, context=decode_context)
|
|
705
|
+
else:
|
|
706
|
+
data = codec.decode(data, key=decode_key)
|
|
628
707
|
|
|
629
708
|
# Squeeze arrays if requested
|
|
630
709
|
if squeeze and isinstance(data, np.ndarray):
|