datajoint 2.3.2__tar.gz → 2.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/workflows/test.yaml +35 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/CONTRIBUTING.md +12 -12
- {datajoint-2.3.2 → datajoint-2.3.4}/PKG-INFO +3 -3
- {datajoint-2.3.2 → datajoint-2.3.4}/README.md +1 -1
- datajoint-2.3.4/images/pipeline.svg +165 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/__init__.py +4 -1
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/_version.py +2 -2
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/adapters/base.py +51 -23
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/adapters/mysql.py +0 -17
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/adapters/postgres.py +16 -17
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/autopopulate.py +9 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/__init__.py +2 -2
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/attach.py +5 -3
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/filepath.py +6 -4
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/hash.py +15 -6
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/npy.py +5 -4
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/object.py +5 -4
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/schema.py +55 -14
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/codecs.py +85 -6
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/declare.py +200 -60
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/deploy.py +137 -1
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/diagram.py +427 -165
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/heading.py +12 -4
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/migrate.py +64 -30
- datajoint-2.3.4/src/datajoint/provenance.py +140 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/schemas.py +3 -3
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/settings.py +66 -5
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/spark.py +2 -2
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/storage.py +42 -20
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/table.py +129 -14
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/user_tables.py +19 -1
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/conftest.py +6 -1
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_declare.py +116 -0
- datajoint-2.3.2/tests/integration/test_erd.py → datajoint-2.3.4/tests/integration/test_diagram.py +23 -9
- datajoint-2.3.4/tests/integration/test_diagram_bundle_edge.py +211 -0
- datajoint-2.3.4/tests/integration/test_diagram_edge_weight.py +125 -0
- datajoint-2.3.4/tests/integration/test_diagram_nocontext.py +54 -0
- datajoint-2.3.4/tests/integration/test_diagram_style.py +197 -0
- datajoint-2.3.4/tests/integration/test_entry_provenance.py +276 -0
- datajoint-2.3.4/tests/integration/test_fk_index.py +170 -0
- datajoint-2.3.4/tests/integration/test_migrate_job_metadata.py +135 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_multi_backend.py +85 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_object.py +12 -12
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_adapters.py +32 -19
- datajoint-2.3.4/tests/unit/test_codec_context.py +172 -0
- datajoint-2.3.4/tests/unit/test_ddl_substitution.py +16 -0
- datajoint-2.3.4/tests/unit/test_declare_hidden_attribute.py +146 -0
- datajoint-2.3.4/tests/unit/test_gc.py +35 -0
- datajoint-2.3.4/tests/unit/test_provenance.py +193 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_settings.py +31 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_storage_adapter.py +84 -1
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_storage_urls.py +15 -0
- datajoint-2.3.4/tests/unit/test_tier_aliases.py +80 -0
- datajoint-2.3.4/tests/unit/test_type_patterns.py +160 -0
- datajoint-2.3.2/images/pipeline.png +0 -0
- datajoint-2.3.2/images/pipeline.svg +0 -118
- datajoint-2.3.2/tests/unit/test_declare_hidden_attribute.py +0 -51
- {datajoint-2.3.2 → datajoint-2.3.4}/.coveragerc +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.dockerignore +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.gitattributes +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/dependabot.yml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/release_drafter.yaml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/workflows/draft_release.yaml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/workflows/lint.yaml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.github/workflows/post_draft_release_published.yaml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.gitignore +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/.pre-commit-config.yaml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/CHANGELOG-archive.md +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/Dockerfile +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/LICENSE +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/RELEASE_MEMO.md +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/activate.sh +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/docker-compose.yaml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/pixi.lock +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/pyproject.toml +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/adapters/__init__.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/blob.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/builtin_codecs/blob.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/cli.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/condition.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/connection.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/dependencies.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/errors.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/expression.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/gc.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/hash_registry.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/instance.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/jobs.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/lineage.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/logging.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/objectref.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/preview.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/staged_insert.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/storage_adapter.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/types.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/utils.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/src/datajoint/version.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/__init__.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/__init__.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/Course.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/CurrentTerm.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/Department.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/Enroll.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/Grade.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/Section.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/Student.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/StudentMajor.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/data/Term.csv +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_aggr_regressions.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_alter.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_attach.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_autopopulate.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_blob.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_blob_matlab.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_cascade_delete.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_cascade_integrity.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_cascading_delete.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_cli.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_codec_chaining.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_codecs.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_connection.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_dependencies.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_fetch.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_fetch_same.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_foreign_keys.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_gc.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_groupby.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_hash_storage.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_hidden_job_metadata.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_insert.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_jobs.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_json.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_nan.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_npy_codec.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_privileges.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_reconnection.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_relation.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_relation_u.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_relational_operand.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_schema.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_schema_keywords.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_semantic_matching.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_skip_duplicates.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_tls.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_trace.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_type_aliases.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_university.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_update1.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_utils.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_uuid.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/integration/test_virtual_module.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_advanced.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_aggr_regress.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_alter.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_codecs.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_external.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_object.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_privileges.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_simple.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_type_aliases.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_university.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/schema_uuid.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/test_package.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/__init__.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_codecs.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_condition.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_connection_warning.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_deploy.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_fetch_compat.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_lazy_imports.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_logging.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_pk_rules.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_spark.py +0 -0
- {datajoint-2.3.2 → datajoint-2.3.4}/tests/unit/test_thread_safe.py +0 -0
|
@@ -68,3 +68,38 @@ jobs:
|
|
|
68
68
|
|
|
69
69
|
- name: Run unit tests
|
|
70
70
|
run: pixi run -e test pytest tests/unit -v
|
|
71
|
+
|
|
72
|
+
# Windows unit tests: guard OS-specific behavior (path separators, etc.) that
|
|
73
|
+
# the Linux jobs above cannot catch — e.g. #1520, where file-protocol paths
|
|
74
|
+
# rendered with native backslashes broke garbage collection on Windows.
|
|
75
|
+
# pixi targets linux/osx only (see [tool.pixi] platforms), so this leg uses
|
|
76
|
+
# pip. Unit tests need no containers, so no Docker/DB is required on Windows.
|
|
77
|
+
unit-tests-windows:
|
|
78
|
+
runs-on: windows-latest
|
|
79
|
+
strategy:
|
|
80
|
+
fail-fast: false
|
|
81
|
+
matrix:
|
|
82
|
+
# Exercise both ends of the supported range (requires-python >=3.10,<3.15).
|
|
83
|
+
python-version: ["3.10", "3.14"]
|
|
84
|
+
name: unit-tests-windows (py${{ matrix.python-version }})
|
|
85
|
+
steps:
|
|
86
|
+
- uses: actions/checkout@v4
|
|
87
|
+
with:
|
|
88
|
+
fetch-depth: 0 # hatch-vcs derives the version from git history/tags
|
|
89
|
+
|
|
90
|
+
- name: Set up Python
|
|
91
|
+
uses: actions/setup-python@v5
|
|
92
|
+
with:
|
|
93
|
+
python-version: ${{ matrix.python-version }}
|
|
94
|
+
|
|
95
|
+
# NOTE: pip resolves `[project.optional-dependencies].test`, which is a
|
|
96
|
+
# DIFFERENT set than the pixi Linux legs' `[dependency-groups].test`
|
|
97
|
+
# (e.g. it omits graphviz). A unit test importing a dep present in only
|
|
98
|
+
# one set would then pass on Linux but error on Windows (or vice versa) —
|
|
99
|
+
# a dep-set mismatch that reads like an OS bug. Keep the two `test` sets
|
|
100
|
+
# in sync when unit-test dependencies change.
|
|
101
|
+
- name: Install package with test extras
|
|
102
|
+
run: pip install -e ".[test]"
|
|
103
|
+
|
|
104
|
+
- name: Run unit tests
|
|
105
|
+
run: pytest tests/unit -v
|
|
@@ -16,13 +16,13 @@ git clone https://github.com/datajoint/datajoint-python.git
|
|
|
16
16
|
cd datajoint-python
|
|
17
17
|
|
|
18
18
|
# Run tests (containers managed automatically)
|
|
19
|
-
pixi run test
|
|
19
|
+
pixi run -e test test
|
|
20
20
|
|
|
21
21
|
# Run with coverage
|
|
22
|
-
pixi run test-cov
|
|
22
|
+
pixi run -e test test-cov
|
|
23
23
|
|
|
24
24
|
# Run pre-commit hooks
|
|
25
|
-
pixi run pre-commit run --all-files
|
|
25
|
+
pixi run -e dev pre-commit run --all-files
|
|
26
26
|
```
|
|
27
27
|
|
|
28
28
|
### Alternative: Using pip
|
|
@@ -41,8 +41,8 @@ Tests use [testcontainers](https://testcontainers.com/) to automatically manage
|
|
|
41
41
|
Integration tests are **backend-parameterized** — tests using the `backend` fixture run automatically against both MySQL and PostgreSQL.
|
|
42
42
|
|
|
43
43
|
```bash
|
|
44
|
-
pixi run test
|
|
45
|
-
pixi run test-cov
|
|
44
|
+
pixi run -e test test # All tests (both backends)
|
|
45
|
+
pixi run -e test test-cov # With coverage
|
|
46
46
|
pixi run -e test pytest tests/unit/ # Unit tests only
|
|
47
47
|
pixi run -e test pytest tests/integration/test_blob.py -v # Specific file
|
|
48
48
|
pixi run -e test pytest -m mysql # MySQL tests only
|
|
@@ -69,12 +69,12 @@ Tests automatically spin up both MySQL and PostgreSQL containers via testcontain
|
|
|
69
69
|
```bash
|
|
70
70
|
# MySQL + MinIO
|
|
71
71
|
docker compose up -d db minio
|
|
72
|
-
DJ_USE_EXTERNAL_CONTAINERS=1 pixi run test
|
|
72
|
+
DJ_USE_EXTERNAL_CONTAINERS=1 pixi run -e test test
|
|
73
73
|
docker compose down
|
|
74
74
|
|
|
75
75
|
# MySQL + PostgreSQL + MinIO
|
|
76
76
|
docker compose up -d db postgres minio
|
|
77
|
-
DJ_USE_EXTERNAL_CONTAINERS=1 pixi run test
|
|
77
|
+
DJ_USE_EXTERNAL_CONTAINERS=1 pixi run -e test test
|
|
78
78
|
docker compose down
|
|
79
79
|
```
|
|
80
80
|
|
|
@@ -91,8 +91,8 @@ docker compose --profile test up djtest --build
|
|
|
91
91
|
Hooks run automatically on `git commit`. All must pass.
|
|
92
92
|
|
|
93
93
|
```bash
|
|
94
|
-
pixi run pre-commit install
|
|
95
|
-
pixi run pre-commit run --all-files # Run manually
|
|
94
|
+
pixi run -e dev pre-commit install # First time only
|
|
95
|
+
pixi run -e dev pre-commit run --all-files # Run manually
|
|
96
96
|
```
|
|
97
97
|
|
|
98
98
|
Hooks include: **ruff** (lint/format), **codespell**, YAML/JSON/TOML validation.
|
|
@@ -101,9 +101,9 @@ Hooks include: **ruff** (lint/format), **codespell**, YAML/JSON/TOML validation.
|
|
|
101
101
|
|
|
102
102
|
## Before Submitting a PR
|
|
103
103
|
|
|
104
|
-
1. `pixi run test` — All tests pass
|
|
105
|
-
2. `pixi run pre-commit run --all-files` — Hooks pass
|
|
106
|
-
3. `pixi run test-cov` — Coverage maintained
|
|
104
|
+
1. `pixi run -e test test` — All tests pass
|
|
105
|
+
2. `pixi run -e dev pre-commit run --all-files` — Hooks pass
|
|
106
|
+
3. `pixi run -e test test-cov` — Coverage maintained
|
|
107
107
|
|
|
108
108
|
---
|
|
109
109
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: datajoint
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.4
|
|
4
4
|
Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
|
|
5
5
|
Project-URL: Homepage, https://docs.datajoint.com/
|
|
6
6
|
Project-URL: Documentation, https://docs.datajoint.com/
|
|
@@ -336,7 +336,7 @@ conda install -c conda-forge datajoint
|
|
|
336
336
|
|
|
337
337
|
## Example Pipeline
|
|
338
338
|
|
|
339
|
-

|
|
340
340
|
|
|
341
341
|
**Cite DataJoint:** [Yatsenko et al., 2026](https://arxiv.org/abs/2602.16585) — RRID: [SCR_014543](https://scicrunch.org/resolver/SCR_014543)
|
|
342
342
|
|
|
@@ -69,7 +69,7 @@ conda install -c conda-forge datajoint
|
|
|
69
69
|
|
|
70
70
|
## Example Pipeline
|
|
71
71
|
|
|
72
|
-

|
|
73
73
|
|
|
74
74
|
**Cite DataJoint:** [Yatsenko et al., 2026](https://arxiv.org/abs/2602.16585) — RRID: [SCR_014543](https://scicrunch.org/resolver/SCR_014543)
|
|
75
75
|
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1500 700"
|
|
2
|
+
font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif"
|
|
3
|
+
font-size="22">
|
|
4
|
+
<!-- Adaptive theming (dj.Diagram "auto" mode): the figure renders in the
|
|
5
|
+
light palette by default and remaps to the dark palette under
|
|
6
|
+
prefers-color-scheme: dark. Colors are unique per role, so attribute
|
|
7
|
+
selectors override the inline presentation attributes unambiguously;
|
|
8
|
+
the three whites (page, code panel, part fill) are disambiguated by
|
|
9
|
+
class since they map to different darks. -->
|
|
10
|
+
<style>
|
|
11
|
+
@media (prefers-color-scheme: dark) {
|
|
12
|
+
.bg { fill: #161A21; }
|
|
13
|
+
.panel { fill: #1E232C; }
|
|
14
|
+
.part { fill: #1E232C; }
|
|
15
|
+
/* node fills */
|
|
16
|
+
[fill="#E7F3EC"] { fill: #16281F; }
|
|
17
|
+
[fill="#E2ECFA"] { fill: #152538; }
|
|
18
|
+
[fill="#F2F4F7"] { fill: #242832; }
|
|
19
|
+
[fill="#FBEAEC"] { fill: #331A1F; }
|
|
20
|
+
[fill="#F3F5F8"] { fill: #2A313D; }
|
|
21
|
+
/* strokes (nodes, cluster, edges) */
|
|
22
|
+
[stroke="#2F7D5B"] { stroke: #4FA97F; }
|
|
23
|
+
[stroke="#2A5FA5"] { stroke: #5E92D6; }
|
|
24
|
+
[stroke="#A9B1BD"] { stroke: #8A93A1; }
|
|
25
|
+
[stroke="#B23A48"] { stroke: #D0687A; }
|
|
26
|
+
[stroke="#9AA6B8"] { stroke: #7B879B; }
|
|
27
|
+
[stroke="#E2E6EC"] { stroke: #39414E; }
|
|
28
|
+
[stroke="#3A424F"] { stroke: #AEB6C2; }
|
|
29
|
+
/* node text */
|
|
30
|
+
[fill="#1B5138"] { fill: #BCE6CF; }
|
|
31
|
+
[fill="#123A6D"] { fill: #C3DAF6; }
|
|
32
|
+
[fill="#495261"] { fill: #C9CFD9; }
|
|
33
|
+
[fill="#7C2430"] { fill: #F3C2CB; }
|
|
34
|
+
[fill="#46536B"] { fill: #C4CCDB; }
|
|
35
|
+
/* code panel + annotations */
|
|
36
|
+
[fill="#24292e"] { fill: #c9d1d9; }
|
|
37
|
+
[fill="#005cc5"] { fill: #79c0ff; }
|
|
38
|
+
[fill="#6f42c1"] { fill: #d2a8ff; }
|
|
39
|
+
[fill="#22863a"] { fill: #7ee787; }
|
|
40
|
+
[fill="#5a5a5a"] { fill: #8b949e; }
|
|
41
|
+
[stroke="#d0d0d0"] { stroke: #30363d; }
|
|
42
|
+
[stroke="#bbb"] { stroke: #484f58; }
|
|
43
|
+
[stroke="#c9ced6"] { stroke: #484f58; }
|
|
44
|
+
}
|
|
45
|
+
</style>
|
|
46
|
+
|
|
47
|
+
<!-- Page background (adaptive; light default keeps GitHub/README legible even
|
|
48
|
+
where prefers-color-scheme is not honored). -->
|
|
49
|
+
<rect class="bg" width="100%" height="100%" fill="white"/>
|
|
50
|
+
|
|
51
|
+
<!-- ============ LEFT PANEL: pipeline diagram (dj.Diagram notation) ============
|
|
52
|
+
Node styling mirrors the modernized dj.Diagram light theme:
|
|
53
|
+
Manual rectangle fill #E7F3EC stroke #2F7D5B
|
|
54
|
+
Lookup rectangle fill #F2F4F7 stroke #A9B1BD
|
|
55
|
+
Imported oval fill #E2ECFA stroke #2A5FA5
|
|
56
|
+
Computed oval fill #FBEAEC stroke #B23A48
|
|
57
|
+
Part rectangle fill #FFFFFF stroke #9AA6B8
|
|
58
|
+
Edge thickness encodes cardinality: thick = 1:1 (the FK is the child's
|
|
59
|
+
whole primary key), thin = one-to-many. A master and its parts sit in a
|
|
60
|
+
subtle rounded entity cluster. -->
|
|
61
|
+
|
|
62
|
+
<!-- master-part entity cluster (drawn behind its members) -->
|
|
63
|
+
<rect x="142" y="350" width="216" height="152" rx="14"
|
|
64
|
+
fill="#F3F5F8" stroke="#E2E6EC" stroke-width="1.4"/>
|
|
65
|
+
|
|
66
|
+
<!-- Edges (no arrowheads — vertical flow implies direction, as dj.Diagram renders) -->
|
|
67
|
+
<g stroke="#3A424F" fill="none">
|
|
68
|
+
<line x1="200" y1="84" x2="200" y2="110" stroke-width="1.4"/>
|
|
69
|
+
<line x1="200" y1="154" x2="200" y2="180" stroke-width="1.4"/>
|
|
70
|
+
<line x1="200" y1="224" x2="200" y2="262" stroke-width="3"/> <!-- Scan->AverageFrame: 1:1, thick -->
|
|
71
|
+
<line x1="210" y1="316" x2="244" y2="366" stroke-width="1.4"/>
|
|
72
|
+
<line x1="432" y1="300" x2="304" y2="380" stroke-width="1.4"/>
|
|
73
|
+
<line x1="250" y1="422" x2="250" y2="448" stroke-width="1.4"/>
|
|
74
|
+
</g>
|
|
75
|
+
|
|
76
|
+
<!-- Manual: Subject -->
|
|
77
|
+
<rect x="135" y="40" width="130" height="44" rx="6" fill="#E7F3EC" stroke="#2F7D5B" stroke-width="1.6"/>
|
|
78
|
+
<text x="200" y="69" text-anchor="middle" fill="#1B5138" font-weight="600">Subject</text>
|
|
79
|
+
|
|
80
|
+
<!-- Manual: Session -->
|
|
81
|
+
<rect x="135" y="110" width="130" height="44" rx="6" fill="#E7F3EC" stroke="#2F7D5B" stroke-width="1.6"/>
|
|
82
|
+
<text x="200" y="139" text-anchor="middle" fill="#1B5138" font-weight="600">Session</text>
|
|
83
|
+
|
|
84
|
+
<!-- Manual: Scan -->
|
|
85
|
+
<rect x="135" y="180" width="130" height="44" rx="6" fill="#E7F3EC" stroke="#2F7D5B" stroke-width="1.6"/>
|
|
86
|
+
<text x="200" y="209" text-anchor="middle" fill="#1B5138" font-weight="600">Scan</text>
|
|
87
|
+
|
|
88
|
+
<!-- Imported: AverageFrame -->
|
|
89
|
+
<ellipse cx="200" cy="290" rx="94" ry="28" fill="#E2ECFA" stroke="#2A5FA5" stroke-width="1.6"/>
|
|
90
|
+
<text x="200" y="298" text-anchor="middle" fill="#123A6D" font-weight="600">AverageFrame</text>
|
|
91
|
+
|
|
92
|
+
<!-- Lookup: SegmentationParams -->
|
|
93
|
+
<rect x="327" y="256" width="236" height="44" rx="6" fill="#F2F4F7" stroke="#A9B1BD" stroke-width="1.6"/>
|
|
94
|
+
<text x="445" y="285" text-anchor="middle" fill="#495261" font-weight="600">SegmentationParams</text>
|
|
95
|
+
|
|
96
|
+
<!-- Computed: Segmentation (master) -->
|
|
97
|
+
<ellipse cx="250" cy="392" rx="102" ry="30" fill="#FBEAEC" stroke="#B23A48" stroke-width="1.6"/>
|
|
98
|
+
<text x="250" y="400" text-anchor="middle" fill="#7C2430" font-weight="700">Segmentation</text>
|
|
99
|
+
|
|
100
|
+
<!-- Part: Activity -->
|
|
101
|
+
<rect class="part" x="176" y="448" width="148" height="42" rx="6" fill="#FFFFFF" stroke="#9AA6B8" stroke-width="1.6"/>
|
|
102
|
+
<text x="250" y="476" text-anchor="middle" fill="#46536B" font-weight="600">Activity</text>
|
|
103
|
+
|
|
104
|
+
<!-- zoom connectors: the code panel is the Segmentation master-part group -->
|
|
105
|
+
<g stroke="#c9ced6" stroke-width="1" stroke-dasharray="5,4" fill="none">
|
|
106
|
+
<line x1="358" y1="356" x2="588" y2="62"/>
|
|
107
|
+
<line x1="358" y1="496" x2="588" y2="658"/>
|
|
108
|
+
</g>
|
|
109
|
+
|
|
110
|
+
<!-- ============ RIGHT PANEL: code snippet ============
|
|
111
|
+
Indentation via per-line x offsets @ 16px monospace (~9.6 px/char):
|
|
112
|
+
L0 x=610, L1 x=648, L2 x=686, L3 x=724 -->
|
|
113
|
+
<rect class="panel" x="588" y="60" width="627" height="600" rx="6" fill="white" stroke="#d0d0d0" stroke-width="1"/>
|
|
114
|
+
|
|
115
|
+
<g font-family="ui-monospace, SFMono-Regular, Menlo, Consolas, monospace"
|
|
116
|
+
font-size="16" fill="#24292e">
|
|
117
|
+
<text x="610" y="92"><tspan fill="#005cc5">@schema</tspan></text>
|
|
118
|
+
<text x="610" y="113"><tspan fill="#005cc5">class</tspan> <tspan fill="#6f42c1" font-weight="600">Segmentation</tspan>(dj.Computed):</text>
|
|
119
|
+
<text x="648" y="134">definition = <tspan fill="#22863a">"""</tspan></text>
|
|
120
|
+
<text x="648" y="155" fill="#22863a">-> AverageFrame</text>
|
|
121
|
+
<text x="648" y="176" fill="#22863a">-> SegmentationParams</text>
|
|
122
|
+
<text x="648" y="197" fill="#22863a">---</text>
|
|
123
|
+
<text x="648" y="218" fill="#22863a">num_cells : int32</text>
|
|
124
|
+
<text x="648" y="239" fill="#22863a">"""</text>
|
|
125
|
+
|
|
126
|
+
<text x="648" y="281"><tspan fill="#005cc5">class</tspan> <tspan fill="#6f42c1" font-weight="600">Activity</tspan>(dj.Part):</text>
|
|
127
|
+
<text x="686" y="302">definition = <tspan fill="#22863a">"""</tspan></text>
|
|
128
|
+
<text x="686" y="323" fill="#22863a">-> master</text>
|
|
129
|
+
<text x="686" y="344" fill="#22863a">roi_id : int32</text>
|
|
130
|
+
<text x="686" y="365" fill="#22863a">---</text>
|
|
131
|
+
<text x="686" y="386" fill="#22863a">cell_mask : <blob@></text>
|
|
132
|
+
<text x="686" y="407" fill="#22863a">trace : <blob@></text>
|
|
133
|
+
<text x="686" y="428" fill="#22863a">"""</text>
|
|
134
|
+
|
|
135
|
+
<text x="648" y="470"><tspan fill="#005cc5">def</tspan> <tspan fill="#6f42c1" font-weight="600">make</tspan>(self, key):</text>
|
|
136
|
+
<text x="686" y="491">frame = (AverageFrame & key).fetch1(<tspan fill="#22863a">'frame'</tspan>)</text>
|
|
137
|
+
<text x="686" y="512">params = (SegmentationParams & key).fetch1()</text>
|
|
138
|
+
<text x="686" y="533">masks, traces = segment(frame, **params)</text>
|
|
139
|
+
<text x="686" y="554">self.insert1(dict(key, num_cells=len(masks)))</text>
|
|
140
|
+
<text x="686" y="575">self.Activity.insert(</text>
|
|
141
|
+
<text x="724" y="596">dict(key, roi_id=i, cell_mask=m, trace=t)</text>
|
|
142
|
+
<text x="724" y="617"><tspan fill="#005cc5">for</tspan> i, (m, t) <tspan fill="#005cc5">in</tspan> enumerate(zip(masks, traces)))</text>
|
|
143
|
+
</g>
|
|
144
|
+
|
|
145
|
+
<!-- ============ Annotations ============ -->
|
|
146
|
+
<g font-style="italic" fill="#5a5a5a" font-size="18">
|
|
147
|
+
<text x="1235" y="92">database link</text>
|
|
148
|
+
<line x1="1230" y1="87" x2="700" y2="87" stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
|
|
149
|
+
|
|
150
|
+
<text x="1235" y="113">table name</text>
|
|
151
|
+
<line x1="1230" y1="108" x2="905" y2="108" stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
|
|
152
|
+
|
|
153
|
+
<text x="1235" y="155">dependency</text>
|
|
154
|
+
<line x1="1230" y1="150" x2="820" y2="150" stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
|
|
155
|
+
|
|
156
|
+
<text x="1235" y="281">part table</text>
|
|
157
|
+
<line x1="1230" y1="276" x2="835" y2="276" stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
|
|
158
|
+
|
|
159
|
+
<text x="1235" y="386">object-store attribute</text>
|
|
160
|
+
<line x1="1230" y1="381" x2="905" y2="381" stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
|
|
161
|
+
|
|
162
|
+
<text x="1235" y="491">computation</text>
|
|
163
|
+
<line x1="1230" y1="486" x2="985" y2="486" stroke="#bbb" stroke-width="0.7" stroke-dasharray="3,3"/>
|
|
164
|
+
</g>
|
|
165
|
+
</svg>
|
|
@@ -33,9 +33,12 @@ __all__ = [
|
|
|
33
33
|
"AutoPopulate",
|
|
34
34
|
"Job",
|
|
35
35
|
"Manual",
|
|
36
|
+
"Entry",
|
|
36
37
|
"Lookup",
|
|
37
38
|
"Imported",
|
|
39
|
+
"Ingest",
|
|
38
40
|
"Computed",
|
|
41
|
+
"Compute",
|
|
39
42
|
"Part",
|
|
40
43
|
"Not",
|
|
41
44
|
"AndList",
|
|
@@ -95,7 +98,7 @@ from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
|
|
|
95
98
|
from .autopopulate import AutoPopulate
|
|
96
99
|
from .jobs import Job
|
|
97
100
|
from .table import FreeTable as _FreeTable, Table, ValidationResult
|
|
98
|
-
from .user_tables import Computed, Imported, Lookup, Manual, Part
|
|
101
|
+
from .user_tables import Compute, Computed, Entry, Imported, Ingest, Lookup, Manual, Part
|
|
99
102
|
from .version import __version__
|
|
100
103
|
|
|
101
104
|
# =============================================================================
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '2.3.
|
|
22
|
-
__version_tuple__ = version_tuple = (2, 3,
|
|
21
|
+
__version__ = version = '2.3.4'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 3, 4)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -617,6 +617,57 @@ class DatabaseAdapter(ABC):
|
|
|
617
617
|
"""
|
|
618
618
|
return True # Default for MySQL, override in PostgreSQL
|
|
619
619
|
|
|
620
|
+
def get_pending_enum_ddl(self, schema_name: str) -> list[str]:
|
|
621
|
+
"""
|
|
622
|
+
DDL for backend types that must exist before the columns using them,
|
|
623
|
+
clearing the pending list as it reads.
|
|
624
|
+
|
|
625
|
+
Backends that spell column types inline (MySQL) have none. PostgreSQL
|
|
626
|
+
overrides this to emit CREATE TYPE for enums registered while parsing.
|
|
627
|
+
|
|
628
|
+
Parameters
|
|
629
|
+
----------
|
|
630
|
+
schema_name : str
|
|
631
|
+
Schema used to qualify the type names.
|
|
632
|
+
|
|
633
|
+
Returns
|
|
634
|
+
-------
|
|
635
|
+
list[str]
|
|
636
|
+
Empty for MySQL; CREATE TYPE statements for PostgreSQL.
|
|
637
|
+
"""
|
|
638
|
+
return []
|
|
639
|
+
|
|
640
|
+
@property
|
|
641
|
+
def supports_column_position(self) -> bool:
|
|
642
|
+
"""
|
|
643
|
+
Whether ALTER TABLE can place a column at a position (``AFTER x``).
|
|
644
|
+
|
|
645
|
+
MySQL supports it. PostgreSQL has no such clause and always appends,
|
|
646
|
+
so the position is dropped rather than emitted.
|
|
647
|
+
|
|
648
|
+
Returns
|
|
649
|
+
-------
|
|
650
|
+
bool
|
|
651
|
+
True for MySQL, False for PostgreSQL.
|
|
652
|
+
"""
|
|
653
|
+
return True # Default for MySQL, override in PostgreSQL
|
|
654
|
+
|
|
655
|
+
@property
|
|
656
|
+
def auto_indexes_foreign_keys(self) -> bool:
|
|
657
|
+
"""
|
|
658
|
+
Whether this backend implicitly indexes a foreign key's referencing
|
|
659
|
+
(child) columns as part of enforcing the constraint.
|
|
660
|
+
|
|
661
|
+
MySQL/InnoDB does; PostgreSQL does not (and offers no server setting to
|
|
662
|
+
make it), so DataJoint must emit an explicit index on that backend.
|
|
663
|
+
|
|
664
|
+
Returns
|
|
665
|
+
-------
|
|
666
|
+
bool
|
|
667
|
+
True for MySQL, False for PostgreSQL.
|
|
668
|
+
"""
|
|
669
|
+
return True # Default for MySQL, override in PostgreSQL
|
|
670
|
+
|
|
620
671
|
def create_index_ddl(
|
|
621
672
|
self,
|
|
622
673
|
full_table_name: str,
|
|
@@ -1266,29 +1317,6 @@ class DatabaseAdapter(ABC):
|
|
|
1266
1317
|
"""
|
|
1267
1318
|
...
|
|
1268
1319
|
|
|
1269
|
-
@abstractmethod
|
|
1270
|
-
def job_metadata_columns(self) -> list[str]:
|
|
1271
|
-
"""
|
|
1272
|
-
Return job metadata column definitions for Computed/Imported tables.
|
|
1273
|
-
|
|
1274
|
-
Returns
|
|
1275
|
-
-------
|
|
1276
|
-
list[str]
|
|
1277
|
-
List of column definition strings (fully formatted with quotes).
|
|
1278
|
-
|
|
1279
|
-
Examples
|
|
1280
|
-
--------
|
|
1281
|
-
MySQL:
|
|
1282
|
-
["`_job_start_time` datetime(3) DEFAULT NULL",
|
|
1283
|
-
"`_job_duration` float DEFAULT NULL",
|
|
1284
|
-
"`_job_version` varchar(64) DEFAULT ''"]
|
|
1285
|
-
PostgreSQL:
|
|
1286
|
-
['"_job_start_time" timestamp DEFAULT NULL',
|
|
1287
|
-
'"_job_duration" real DEFAULT NULL',
|
|
1288
|
-
'"_job_version" varchar(64) DEFAULT \'\'']
|
|
1289
|
-
"""
|
|
1290
|
-
...
|
|
1291
|
-
|
|
1292
1320
|
# =========================================================================
|
|
1293
1321
|
# Error Translation
|
|
1294
1322
|
# =========================================================================
|
|
@@ -996,23 +996,6 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
996
996
|
"""
|
|
997
997
|
return None # MySQL uses inline enum
|
|
998
998
|
|
|
999
|
-
def job_metadata_columns(self) -> list[str]:
|
|
1000
|
-
"""
|
|
1001
|
-
Return MySQL-specific job metadata column definitions.
|
|
1002
|
-
|
|
1003
|
-
Examples
|
|
1004
|
-
--------
|
|
1005
|
-
>>> adapter.job_metadata_columns()
|
|
1006
|
-
["`_job_start_time` datetime(3) DEFAULT NULL",
|
|
1007
|
-
"`_job_duration` float DEFAULT NULL",
|
|
1008
|
-
"`_job_version` varchar(64) DEFAULT ''"]
|
|
1009
|
-
"""
|
|
1010
|
-
return [
|
|
1011
|
-
"`_job_start_time` datetime(3) DEFAULT NULL",
|
|
1012
|
-
"`_job_duration` float DEFAULT NULL",
|
|
1013
|
-
"`_job_version` varchar(64) DEFAULT ''",
|
|
1014
|
-
]
|
|
1015
|
-
|
|
1016
999
|
# =========================================================================
|
|
1017
1000
|
# Error Translation
|
|
1018
1001
|
# =========================================================================
|
|
@@ -720,6 +720,22 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
720
720
|
"""
|
|
721
721
|
return False
|
|
722
722
|
|
|
723
|
+
@property
|
|
724
|
+
def supports_column_position(self) -> bool:
|
|
725
|
+
"""
|
|
726
|
+
PostgreSQL has no ``AFTER`` clause in ALTER TABLE; added columns are
|
|
727
|
+
always appended.
|
|
728
|
+
"""
|
|
729
|
+
return False
|
|
730
|
+
|
|
731
|
+
@property
|
|
732
|
+
def auto_indexes_foreign_keys(self) -> bool:
|
|
733
|
+
"""
|
|
734
|
+
PostgreSQL never indexes a foreign key's referencing columns
|
|
735
|
+
automatically, so DataJoint emits an explicit (coverage-aware) index.
|
|
736
|
+
"""
|
|
737
|
+
return False
|
|
738
|
+
|
|
723
739
|
# =========================================================================
|
|
724
740
|
# Introspection
|
|
725
741
|
# =========================================================================
|
|
@@ -1333,23 +1349,6 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
1333
1349
|
self._pending_enum_types = {}
|
|
1334
1350
|
return ddl_statements
|
|
1335
1351
|
|
|
1336
|
-
def job_metadata_columns(self) -> list[str]:
|
|
1337
|
-
"""
|
|
1338
|
-
Return PostgreSQL-specific job metadata column definitions.
|
|
1339
|
-
|
|
1340
|
-
Examples
|
|
1341
|
-
--------
|
|
1342
|
-
>>> adapter.job_metadata_columns()
|
|
1343
|
-
['"_job_start_time" timestamp DEFAULT NULL',
|
|
1344
|
-
'"_job_duration" real DEFAULT NULL',
|
|
1345
|
-
'"_job_version" varchar(64) DEFAULT \\'\\'']
|
|
1346
|
-
"""
|
|
1347
|
-
return [
|
|
1348
|
-
'"_job_start_time" timestamp DEFAULT NULL',
|
|
1349
|
-
'"_job_duration" real DEFAULT NULL',
|
|
1350
|
-
"\"_job_version\" varchar(64) DEFAULT ''",
|
|
1351
|
-
]
|
|
1352
|
-
|
|
1353
1352
|
# =========================================================================
|
|
1354
1353
|
# Error Translation
|
|
1355
1354
|
# =========================================================================
|
|
@@ -11,6 +11,7 @@ import signal
|
|
|
11
11
|
import traceback
|
|
12
12
|
from typing import TYPE_CHECKING, Any, Generator
|
|
13
13
|
|
|
14
|
+
from . import provenance
|
|
14
15
|
from .errors import DataJointError, LostConnectionError
|
|
15
16
|
from .expression import AndList, QueryExpression
|
|
16
17
|
|
|
@@ -684,6 +685,13 @@ class AutoPopulate:
|
|
|
684
685
|
self._upstream_key = dict(key)
|
|
685
686
|
self._upstream = None
|
|
686
687
|
|
|
688
|
+
# Rows this make() writes into Entry tables that carry no foreign key back
|
|
689
|
+
# here -- the fan-out ingestion pattern -- record the ingesting table and
|
|
690
|
+
# key, which is what makes such a row traceable without one.
|
|
691
|
+
from .jobs import _get_job_version
|
|
692
|
+
|
|
693
|
+
prov_token = provenance.set_ingesting(self.full_table_name, key, _get_job_version(self.connection._config))
|
|
694
|
+
|
|
687
695
|
try:
|
|
688
696
|
if not is_generator:
|
|
689
697
|
make(dict(key), **(make_kwargs or {}))
|
|
@@ -740,6 +748,7 @@ class AutoPopulate:
|
|
|
740
748
|
jobs.complete(key, duration=duration)
|
|
741
749
|
return True
|
|
742
750
|
finally:
|
|
751
|
+
provenance.reset_ingesting(prov_token)
|
|
743
752
|
self.__class__._allow_insert = False
|
|
744
753
|
# Clear the per-make() upstream state: `_upstream = None` invalidates
|
|
745
754
|
# the memoized Diagram; `_upstream_key = None` restores the "outside
|
|
@@ -29,14 +29,14 @@ Example - Creating a Custom Codec:
|
|
|
29
29
|
def get_dtype(self, is_store: bool) -> str:
|
|
30
30
|
return "<blob>" # Compose with blob for serialization
|
|
31
31
|
|
|
32
|
-
def encode(self, graph, *, key=None, store_name=None):
|
|
32
|
+
def encode(self, graph, *, key=None, context=None, store_name=None):
|
|
33
33
|
# Convert graph to a serializable format
|
|
34
34
|
return {
|
|
35
35
|
'nodes': list(graph.nodes(data=True)),
|
|
36
36
|
'edges': list(graph.edges(data=True)),
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
def decode(self, stored, *, key=None):
|
|
39
|
+
def decode(self, stored, *, key=None, context=None):
|
|
40
40
|
# Reconstruct graph from stored format
|
|
41
41
|
G = nx.Graph()
|
|
42
42
|
G.add_nodes_from(stored['nodes'])
|
|
@@ -50,7 +50,9 @@ class AttachCodec(Codec):
|
|
|
50
50
|
"""Return bytes for in-table, <hash> for in-store storage."""
|
|
51
51
|
return "<hash>" if is_store else "bytes"
|
|
52
52
|
|
|
53
|
-
def encode(
|
|
53
|
+
def encode(
|
|
54
|
+
self, value: Any, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
|
|
55
|
+
) -> bytes:
|
|
54
56
|
"""
|
|
55
57
|
Read file and encode as filename + contents.
|
|
56
58
|
|
|
@@ -80,7 +82,7 @@ class AttachCodec(Codec):
|
|
|
80
82
|
contents = path.read_bytes()
|
|
81
83
|
return filename.encode("utf-8") + b"\x00" + contents
|
|
82
84
|
|
|
83
|
-
def decode(self, stored: bytes, *, key: dict | None = None) -> str:
|
|
85
|
+
def decode(self, stored: bytes, *, key: dict | None = None, context: dict | None = None) -> str:
|
|
84
86
|
"""
|
|
85
87
|
Extract file to download path and return local path.
|
|
86
88
|
|
|
@@ -104,7 +106,7 @@ class AttachCodec(Codec):
|
|
|
104
106
|
contents = stored[null_pos + 1 :]
|
|
105
107
|
|
|
106
108
|
# Write to download path
|
|
107
|
-
config = (key
|
|
109
|
+
config = self._codec_config(key, context)
|
|
108
110
|
if config is None:
|
|
109
111
|
from ..settings import config # type: ignore[assignment]
|
|
110
112
|
assert config is not None
|
|
@@ -74,7 +74,9 @@ class FilepathCodec(Codec):
|
|
|
74
74
|
)
|
|
75
75
|
return "json"
|
|
76
76
|
|
|
77
|
-
def encode(
|
|
77
|
+
def encode(
|
|
78
|
+
self, value: Any, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
|
|
79
|
+
) -> dict:
|
|
78
80
|
"""
|
|
79
81
|
Store path reference as JSON metadata.
|
|
80
82
|
|
|
@@ -104,7 +106,7 @@ class FilepathCodec(Codec):
|
|
|
104
106
|
|
|
105
107
|
from ..hash_registry import get_store_backend
|
|
106
108
|
|
|
107
|
-
config = (key
|
|
109
|
+
config = self._codec_config(key, context)
|
|
108
110
|
if config is None:
|
|
109
111
|
from ..settings import config # type: ignore[assignment]
|
|
110
112
|
assert config is not None
|
|
@@ -168,7 +170,7 @@ class FilepathCodec(Codec):
|
|
|
168
170
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
169
171
|
}
|
|
170
172
|
|
|
171
|
-
def decode(self, stored: dict, *, key: dict | None = None) -> Any:
|
|
173
|
+
def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> Any:
|
|
172
174
|
"""
|
|
173
175
|
Create ObjectRef handle for lazy access.
|
|
174
176
|
|
|
@@ -187,7 +189,7 @@ class FilepathCodec(Codec):
|
|
|
187
189
|
from ..objectref import ObjectRef
|
|
188
190
|
from ..hash_registry import get_store_backend
|
|
189
191
|
|
|
190
|
-
config = (key
|
|
192
|
+
config = self._codec_config(key, context)
|
|
191
193
|
store_name = stored.get("store")
|
|
192
194
|
backend = get_store_backend(store_name, config=config)
|
|
193
195
|
return ObjectRef.from_json(stored, backend=backend)
|
|
@@ -57,7 +57,9 @@ class HashCodec(Codec):
|
|
|
57
57
|
raise DataJointError("<hash> requires @ (in-store storage only)")
|
|
58
58
|
return "json"
|
|
59
59
|
|
|
60
|
-
def encode(
|
|
60
|
+
def encode(
|
|
61
|
+
self, value: bytes, *, key: dict | None = None, context: dict | None = None, store_name: str | None = None
|
|
62
|
+
) -> dict:
|
|
61
63
|
"""
|
|
62
64
|
Store content and return metadata.
|
|
63
65
|
|
|
@@ -66,7 +68,11 @@ class HashCodec(Codec):
|
|
|
66
68
|
value : bytes
|
|
67
69
|
Raw bytes to store.
|
|
68
70
|
key : dict, optional
|
|
69
|
-
|
|
71
|
+
Primary key values. Before 2.3.4 this also carried the schema name
|
|
72
|
+
under ``_schema``; that is still read when ``context`` omits it.
|
|
73
|
+
context : dict, optional
|
|
74
|
+
Connection context. ``schema`` isolates the stored path and
|
|
75
|
+
``config`` resolves the store.
|
|
70
76
|
store_name : str, optional
|
|
71
77
|
Store to use. If None, uses default store.
|
|
72
78
|
|
|
@@ -77,11 +83,14 @@ class HashCodec(Codec):
|
|
|
77
83
|
"""
|
|
78
84
|
from ..hash_registry import put_hash
|
|
79
85
|
|
|
80
|
-
|
|
81
|
-
|
|
86
|
+
# Same precedence as `_extract_context`. Reading `key` alone would store
|
|
87
|
+
# under "unknown" for any caller that passes `context` with a key holding
|
|
88
|
+
# only primary-key values -- which is every caller once 2.4 lands.
|
|
89
|
+
schema_name = (context or {}).get("schema", (key or {}).get("_schema", "unknown"))
|
|
90
|
+
config = self._codec_config(key, context)
|
|
82
91
|
return put_hash(value, schema_name=schema_name, store_name=store_name, config=config)
|
|
83
92
|
|
|
84
|
-
def decode(self, stored: dict, *, key: dict | None = None) -> bytes:
|
|
93
|
+
def decode(self, stored: dict, *, key: dict | None = None, context: dict | None = None) -> bytes:
|
|
85
94
|
"""
|
|
86
95
|
Retrieve content using stored metadata.
|
|
87
96
|
|
|
@@ -99,7 +108,7 @@ class HashCodec(Codec):
|
|
|
99
108
|
"""
|
|
100
109
|
from ..hash_registry import get_hash
|
|
101
110
|
|
|
102
|
-
config = (key
|
|
111
|
+
config = self._codec_config(key, context)
|
|
103
112
|
return get_hash(stored, config=config)
|
|
104
113
|
|
|
105
114
|
def validate(self, value: Any) -> None:
|