datajoint 2.3.5__tar.gz → 2.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {datajoint-2.3.5 → datajoint-2.3.6}/PKG-INFO +2 -2
- {datajoint-2.3.5 → datajoint-2.3.6}/pixi.lock +3 -3
- {datajoint-2.3.5 → datajoint-2.3.6}/pyproject.toml +1 -1
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/__init__.py +12 -4
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/_version.py +2 -2
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/adapters/mysql.py +78 -14
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/adapters/postgres.py +77 -9
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/blob.py +28 -5
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/attach.py +69 -25
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/filepath.py +41 -9
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/npy.py +76 -7
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/object.py +2 -1
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/condition.py +48 -9
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/connection.py +48 -28
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/declare.py +207 -4
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/instance.py +2 -1
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/objectref.py +50 -2
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/settings.py +230 -27
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/storage.py +92 -5
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/table.py +26 -8
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/utils.py +13 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_alter.py +67 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_declare.py +1 -1
- datajoint-2.3.6/tests/integration/test_path_confinement.py +183 -0
- datajoint-2.3.6/tests/integration/test_restriction_values.py +205 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_tls.py +5 -14
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_adapters.py +2 -0
- datajoint-2.3.6/tests/unit/test_blob_decompression.py +73 -0
- datajoint-2.3.6/tests/unit/test_condition.py +420 -0
- datajoint-2.3.6/tests/unit/test_foreign_key_reference.py +172 -0
- datajoint-2.3.6/tests/unit/test_npy_cache.py +128 -0
- datajoint-2.3.6/tests/unit/test_path_confinement.py +432 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_settings.py +329 -0
- datajoint-2.3.6/tests/unit/test_tls.py +532 -0
- datajoint-2.3.5/tests/unit/test_condition.py +0 -95
- {datajoint-2.3.5 → datajoint-2.3.6}/.coveragerc +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.dockerignore +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.gitattributes +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/dependabot.yml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/release_drafter.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/workflows/draft_release.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/workflows/lint.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/workflows/post_draft_release_published.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.github/workflows/test.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.gitignore +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/.pre-commit-config.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/CHANGELOG-archive.md +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/CONTRIBUTING.md +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/Dockerfile +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/LICENSE +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/README.md +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/RELEASE_MEMO.md +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/activate.sh +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/docker-compose.yaml +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/images/pipeline.svg +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/adapters/__init__.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/adapters/base.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/autopopulate.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/__init__.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/blob.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/hash.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/builtin_codecs/schema.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/cli.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/codecs.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/dependencies.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/deploy.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/diagram.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/errors.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/expression.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/gc.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/hash_registry.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/heading.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/jobs.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/lineage.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/logging.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/migrate.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/preview.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/provenance.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/schemas.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/spark.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/staged_insert.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/storage_adapter.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/types.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/user_tables.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/src/datajoint/version.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/__init__.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/conftest.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/__init__.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/Course.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/CurrentTerm.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/Department.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/Enroll.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/Grade.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/Section.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/Student.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/StudentMajor.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/data/Term.csv +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_aggr_regressions.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_alter_rename.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_attach.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_autopopulate.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_blob.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_blob_matlab.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_cascade_delete.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_cascade_integrity.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_cascade_overdelete.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_cascading_delete.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_cli.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_codec_chaining.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_codecs.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_connection.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_dependencies.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_diagram.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_diagram_bundle_edge.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_diagram_edge_weight.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_diagram_nocontext.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_diagram_style.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_drop_shared_enum.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_entry_provenance.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_fetch.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_fetch_same.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_fk_index.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_foreign_keys.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_gc.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_groupby.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_hash_storage.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_hidden_job_metadata.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_insert.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_jobs.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_jobs_refresh.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_json.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_migrate_job_metadata.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_multi_backend.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_nan.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_npy_codec.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_object.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_privileges.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_reconnection.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_relation.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_relation_u.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_relational_operand.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_schema.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_schema_keywords.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_semantic_matching.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_skip_duplicates.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_trace.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_type_aliases.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_university.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_update1.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_utils.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_uuid.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/integration/test_virtual_module.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_advanced.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_aggr_regress.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_alter.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_codecs.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_external.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_object.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_privileges.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_simple.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_type_aliases.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_university.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/schema_uuid.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/test_package.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/__init__.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_codec_context.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_codecs.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_connection_warning.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_ddl_substitution.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_declare_hidden_attribute.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_deploy.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_fetch_compat.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_gc.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_lazy_imports.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_logging.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_pk_rules.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_provenance.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_spark.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_storage_adapter.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_storage_urls.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_thread_safe.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_tier_aliases.py +0 -0
- {datajoint-2.3.5 → datajoint-2.3.6}/tests/unit/test_type_patterns.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: datajoint
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.6
|
|
4
4
|
Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
|
|
5
5
|
Project-URL: Homepage, https://docs.datajoint.com/
|
|
6
6
|
Project-URL: Documentation, https://docs.datajoint.com/
|
|
@@ -227,7 +227,7 @@ Requires-Dist: packaging
|
|
|
227
227
|
Requires-Dist: pandas
|
|
228
228
|
Requires-Dist: pydantic-settings>=2.0.0
|
|
229
229
|
Requires-Dist: pydot
|
|
230
|
-
Requires-Dist: pymysql>=
|
|
230
|
+
Requires-Dist: pymysql>=1.1.1
|
|
231
231
|
Requires-Dist: pyparsing
|
|
232
232
|
Requires-Dist: tqdm
|
|
233
233
|
Provides-Extra: arrow
|
|
@@ -4610,8 +4610,8 @@ packages:
|
|
|
4610
4610
|
requires_python: '>=3.8'
|
|
4611
4611
|
- pypi: ./
|
|
4612
4612
|
name: datajoint
|
|
4613
|
-
version: 2.3.
|
|
4614
|
-
sha256:
|
|
4613
|
+
version: 2.3.6.dev0+g1ba8b9fee.d20261109
|
|
4614
|
+
sha256: dd2067138ee3cc7fa3d5514c2dae081be471d14279a27e7b8289ba2876c2ce61
|
|
4615
4615
|
requires_dist:
|
|
4616
4616
|
- deepdiff
|
|
4617
4617
|
- deepdiff<9 ; python_full_version >= '3.15'
|
|
@@ -4622,7 +4622,7 @@ packages:
|
|
|
4622
4622
|
- pandas
|
|
4623
4623
|
- pydantic-settings>=2.0.0
|
|
4624
4624
|
- pydot
|
|
4625
|
-
- pymysql>=
|
|
4625
|
+
- pymysql>=1.1.1
|
|
4626
4626
|
- pyparsing
|
|
4627
4627
|
- tqdm
|
|
4628
4628
|
- pyarrow>=14.0.0 ; extra == 'arrow'
|
|
@@ -7,7 +7,7 @@ name = "datajoint"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
dependencies = [
|
|
9
9
|
"numpy",
|
|
10
|
-
"pymysql>=
|
|
10
|
+
"pymysql>=1.1.1", # CVE-2024-36039
|
|
11
11
|
"deepdiff",
|
|
12
12
|
# deepdiff 9 requires cachebox<6, which ships no Python 3.15 wheels and cannot
|
|
13
13
|
# be built for 3.15 (PyO3 0.27). Drop once deepdiff accepts cachebox 6.
|
|
@@ -134,7 +134,11 @@ def conn(
|
|
|
134
134
|
reset : bool, optional
|
|
135
135
|
If True, reset existing connection. Default False.
|
|
136
136
|
use_tls : bool or dict, optional
|
|
137
|
-
TLS encryption option
|
|
137
|
+
TLS encryption option: True (required), False (no TLS), None (from
|
|
138
|
+
config; TLS if the server supports it by default), or dict (TLS
|
|
139
|
+
required; ``ca``, ``cert``, ``key``; with ``ca``, the server
|
|
140
|
+
certificate is verified). If the singleton connection exists, it is
|
|
141
|
+
reconnected with these unless it already satisfies them.
|
|
138
142
|
|
|
139
143
|
Returns
|
|
140
144
|
-------
|
|
@@ -152,10 +156,12 @@ def conn(
|
|
|
152
156
|
_check_thread_safe()
|
|
153
157
|
|
|
154
158
|
# If reset requested, always recreate
|
|
155
|
-
# If credentials provided and no singleton exists, create one
|
|
156
|
-
# If credentials provided and singleton exists, return existing singleton
|
|
159
|
+
# If credentials or use_tls provided and no singleton exists, create one
|
|
160
|
+
# If credentials provided and singleton exists, return existing singleton,
|
|
161
|
+
# reconnected with use_tls unless it already satisfies use_tls
|
|
157
162
|
if reset or (
|
|
158
|
-
instance_module._singleton_connection is None
|
|
163
|
+
instance_module._singleton_connection is None
|
|
164
|
+
and (host is not None or user is not None or password is not None or use_tls is not None)
|
|
159
165
|
):
|
|
160
166
|
# Use provided values or fall back to config
|
|
161
167
|
host = host if host is not None else _global_config.database.host
|
|
@@ -177,6 +183,8 @@ def conn(
|
|
|
177
183
|
)
|
|
178
184
|
|
|
179
185
|
instance_module._singleton_connection = Connection(host, user, password, port, use_tls, config_override=_global_config)
|
|
186
|
+
elif instance_module._singleton_connection is not None:
|
|
187
|
+
instance_module._singleton_connection._ensure_tls(use_tls)
|
|
180
188
|
|
|
181
189
|
return _get_singleton_connection()
|
|
182
190
|
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '2.3.
|
|
22
|
-
__version_tuple__ = version_tuple = (2, 3,
|
|
21
|
+
__version__ = version = '2.3.6'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 3, 6)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -7,9 +7,11 @@ type mapping, error translation, and connection management.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import ssl
|
|
10
11
|
from typing import Any
|
|
11
12
|
|
|
12
13
|
import pymysql as client
|
|
14
|
+
from pymysql.constants import CLIENT, CR
|
|
13
15
|
|
|
14
16
|
from .. import errors
|
|
15
17
|
from .base import DatabaseAdapter
|
|
@@ -45,6 +47,51 @@ SQL_TO_CORE_MAP = {
|
|
|
45
47
|
}
|
|
46
48
|
|
|
47
49
|
|
|
50
|
+
class _TLSUnavailableError(client.err.OperationalError):
|
|
51
|
+
"""TLS is required but the server does not offer it."""
|
|
52
|
+
|
|
53
|
+
def __init__(self):
|
|
54
|
+
super().__init__(
|
|
55
|
+
CR.CR_SSL_CONNECTION_ERROR,
|
|
56
|
+
"TLS is required (use_tls) but the server does not support it",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class _TLSRequiredConnection(client.connections.Connection):
|
|
61
|
+
"""
|
|
62
|
+
PyMySQL connection that does not authenticate without TLS.
|
|
63
|
+
|
|
64
|
+
Depending on its version, PyMySQL continues without TLS or fails with a
|
|
65
|
+
handshake error when TLS is configured and the server does not offer it.
|
|
66
|
+
This check runs after the server greeting, before the credentials are sent.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
tls_required = True
|
|
70
|
+
|
|
71
|
+
def _request_authentication(self):
|
|
72
|
+
if not self.server_capabilities & CLIENT.SSL:
|
|
73
|
+
if self.tls_required:
|
|
74
|
+
raise _TLSUnavailableError()
|
|
75
|
+
# Continue without TLS: do not announce TLS in the handshake response
|
|
76
|
+
self.ssl = False
|
|
77
|
+
self.client_flag &= ~CLIENT.SSL
|
|
78
|
+
super()._request_authentication()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class _TLSPreferredConnection(_TLSRequiredConnection):
|
|
82
|
+
"""PyMySQL connection that uses TLS when the server offers it and continues without TLS otherwise."""
|
|
83
|
+
|
|
84
|
+
tls_required = False
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _unverified_tls_context() -> ssl.SSLContext:
|
|
88
|
+
"""TLS context that encrypts without verifying the server certificate."""
|
|
89
|
+
context = ssl.create_default_context()
|
|
90
|
+
context.check_hostname = False
|
|
91
|
+
context.verify_mode = ssl.CERT_NONE
|
|
92
|
+
return context
|
|
93
|
+
|
|
94
|
+
|
|
48
95
|
class MySQLAdapter(DatabaseAdapter):
|
|
49
96
|
"""MySQL database adapter implementation."""
|
|
50
97
|
|
|
@@ -76,7 +123,13 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
76
123
|
**kwargs : Any
|
|
77
124
|
Additional MySQL-specific parameters:
|
|
78
125
|
- ssl: TLS/SSL configuration dict (deprecated, use use_tls)
|
|
79
|
-
- use_tls: bool or dict - DataJoint's SSL parameter (preferred)
|
|
126
|
+
- use_tls: bool or dict - DataJoint's SSL parameter (preferred).
|
|
127
|
+
None: TLS if the server offers it, otherwise no TLS; server
|
|
128
|
+
certificate not verified.
|
|
129
|
+
True: TLS required, server certificate not verified.
|
|
130
|
+
dict: TLS required, configured with PyMySQL ``ssl`` keys
|
|
131
|
+
(``ca``, ``cert``, ``key``, ``check_hostname``, ...); with ``ca``,
|
|
132
|
+
the server certificate is verified. False: TLS disabled.
|
|
80
133
|
- charset: Character set (default from kwargs)
|
|
81
134
|
|
|
82
135
|
Returns
|
|
@@ -86,9 +139,11 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
86
139
|
"""
|
|
87
140
|
# Handle both ssl (old) and use_tls (new) parameter names
|
|
88
141
|
ssl_config = kwargs.get("use_tls", kwargs.get("ssl"))
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
142
|
+
tls_required = ssl_config is not None
|
|
143
|
+
# PyMySQL treats an empty dict as "no TLS"; None, True, and {} use TLS
|
|
144
|
+
# without verifying the server certificate.
|
|
145
|
+
if ssl_config is None or ssl_config is True or (isinstance(ssl_config, dict) and not ssl_config):
|
|
146
|
+
ssl_config = _unverified_tls_context()
|
|
92
147
|
charset = kwargs.get("charset", "")
|
|
93
148
|
|
|
94
149
|
# Prepare connection parameters
|
|
@@ -107,13 +162,22 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
107
162
|
if ssl_config is False:
|
|
108
163
|
# Explicitly disable SSL
|
|
109
164
|
conn_params["ssl_disabled"] = True
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
165
|
+
return client.connect(**conn_params)
|
|
166
|
+
|
|
167
|
+
conn_params["ssl"] = ssl_config
|
|
168
|
+
# Explicitly enable SSL by setting ssl_disabled=False
|
|
169
|
+
conn_params["ssl_disabled"] = False
|
|
170
|
+
if not tls_required:
|
|
171
|
+
# use_tls=None: one connection, with TLS if the server offers it
|
|
172
|
+
return _TLSPreferredConnection(**conn_params)
|
|
173
|
+
|
|
174
|
+
# Require SSL, configured by the dict or SSLContext
|
|
175
|
+
connection = _TLSRequiredConnection(**conn_params)
|
|
176
|
+
# Checked again in case a PyMySQL version bypasses the hook above
|
|
177
|
+
if not connection.server_capabilities & CLIENT.SSL:
|
|
178
|
+
connection.close()
|
|
179
|
+
raise _TLSUnavailableError()
|
|
180
|
+
return connection
|
|
117
181
|
|
|
118
182
|
def close(self, connection: Any) -> None:
|
|
119
183
|
"""Close the MySQL connection."""
|
|
@@ -219,8 +283,8 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
219
283
|
str
|
|
220
284
|
Quoted and escaped string literal.
|
|
221
285
|
"""
|
|
222
|
-
# Use pymysql's escape_string
|
|
223
|
-
escaped = client.converters.escape_string(value)
|
|
286
|
+
# Use pymysql's escape_string, but double quotes so the literal stays closed even under NO_BACKSLASH_ESCAPES
|
|
287
|
+
escaped = client.converters.escape_string(value).replace("\\'", "''")
|
|
224
288
|
return f"'{escaped}'"
|
|
225
289
|
|
|
226
290
|
def get_master_table_name(self, part_table: str) -> str | None:
|
|
@@ -872,7 +936,7 @@ class MySQLAdapter(DatabaseAdapter):
|
|
|
872
936
|
"""
|
|
873
937
|
quoted_col = self.quote_identifier(column)
|
|
874
938
|
return_clause = f" returning {return_type}" if return_type else ""
|
|
875
|
-
return f"json_value({quoted_col}, _utf8mb4'$.
|
|
939
|
+
return f"json_value({quoted_col}, _utf8mb4{self.quote_string('$.' + path)}{return_clause})"
|
|
876
940
|
|
|
877
941
|
def translate_expression(self, expr: str) -> str:
|
|
878
942
|
"""
|
|
@@ -7,6 +7,7 @@ type mapping, error translation, and connection management.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
+
import logging
|
|
10
11
|
import re
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
@@ -20,6 +21,53 @@ except ImportError:
|
|
|
20
21
|
from .. import errors
|
|
21
22
|
from .base import DatabaseAdapter
|
|
22
23
|
|
|
24
|
+
logger = logging.getLogger(__name__.split(".")[0])
|
|
25
|
+
|
|
26
|
+
# use_tls dict keys (PyMySQL ssl names, as for MySQL) → libpq connection parameters
|
|
27
|
+
_LIBPQ_TLS_KEYS = {"ca": "sslrootcert", "cert": "sslcert", "key": "sslkey", "password": "sslpassword"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _tls_rejected(error: Exception) -> bool:
|
|
31
|
+
"""
|
|
32
|
+
Whether a connection attempt with sslmode=require was refused before authentication
|
|
33
|
+
because of SSL: the server does not support SSL, or pg_hba.conf has no entry
|
|
34
|
+
for SSL connections from this client.
|
|
35
|
+
|
|
36
|
+
psycopg2 does not report a SQLSTATE for connection failures, so the libpq and
|
|
37
|
+
server messages are matched. Authentication failures do not match.
|
|
38
|
+
"""
|
|
39
|
+
message = str(error)
|
|
40
|
+
if "server does not support SSL" in message:
|
|
41
|
+
return True
|
|
42
|
+
# SQLSTATE 28000 from pg_hba.conf; "SSL encryption" (PostgreSQL 14+) or "SSL on" (earlier)
|
|
43
|
+
return "pg_hba.conf" in message and ("SSL encryption" in message or "SSL on" in message)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _libpq_tls_params(use_tls: dict) -> dict[str, Any]:
|
|
47
|
+
"""
|
|
48
|
+
Map a ``use_tls`` dict to libpq SSL connection parameters.
|
|
49
|
+
|
|
50
|
+
With ``ca``, the server certificate is verified against it: sslmode
|
|
51
|
+
``verify-full`` (certificate and host name), or ``verify-ca`` when
|
|
52
|
+
``check_hostname`` is False. Without ``ca``, sslmode is ``require``.
|
|
53
|
+
"""
|
|
54
|
+
params = {_LIBPQ_TLS_KEYS[k]: v for k, v in use_tls.items() if k in _LIBPQ_TLS_KEYS and v is not None}
|
|
55
|
+
supported = set(_LIBPQ_TLS_KEYS)
|
|
56
|
+
if "sslrootcert" in params:
|
|
57
|
+
params["sslmode"] = "verify-full" if use_tls.get("check_hostname", True) else "verify-ca"
|
|
58
|
+
supported.add("check_hostname")
|
|
59
|
+
else:
|
|
60
|
+
params["sslmode"] = "require"
|
|
61
|
+
# Keys set to None or False request nothing; other unused keys are reported
|
|
62
|
+
ignored = sorted(k for k in set(use_tls) - supported if use_tls[k] not in (None, False))
|
|
63
|
+
if ignored:
|
|
64
|
+
logger.warning(
|
|
65
|
+
"use_tls keys ignored for PostgreSQL: %s. Supported: ca, cert, key, password, and check_hostname with ca.",
|
|
66
|
+
", ".join(ignored),
|
|
67
|
+
)
|
|
68
|
+
return params
|
|
69
|
+
|
|
70
|
+
|
|
23
71
|
# Core type mapping: DataJoint core types → PostgreSQL types
|
|
24
72
|
CORE_TYPE_MAP = {
|
|
25
73
|
"int64": "bigint",
|
|
@@ -90,8 +138,15 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
90
138
|
**kwargs : Any
|
|
91
139
|
Additional PostgreSQL-specific parameters:
|
|
92
140
|
- dbname: Database name
|
|
93
|
-
- sslmode: SSL mode ('disable', 'allow', 'prefer', 'require'
|
|
94
|
-
|
|
141
|
+
- sslmode: SSL mode ('disable', 'allow', 'prefer', 'require',
|
|
142
|
+
'verify-ca', 'verify-full')
|
|
143
|
+
- use_tls: bool or dict - DataJoint's SSL parameter (converted to sslmode).
|
|
144
|
+
True: 'require'. False: 'disable'.
|
|
145
|
+
None: 'require', then 'disable' only if the server does not support
|
|
146
|
+
SSL or pg_hba.conf does not accept SSL connections from this client.
|
|
147
|
+
dict: ``ca``, ``cert``, ``key``, ``password`` map to sslrootcert,
|
|
148
|
+
sslcert, sslkey, sslpassword; sslmode is 'verify-full' with ``ca``
|
|
149
|
+
('verify-ca' if ``check_hostname`` is False), otherwise 'require'.
|
|
95
150
|
- connect_timeout: Connection timeout in seconds
|
|
96
151
|
|
|
97
152
|
Returns
|
|
@@ -105,28 +160,41 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
105
160
|
# Handle use_tls parameter (from DataJoint Connection)
|
|
106
161
|
# Convert to PostgreSQL's sslmode
|
|
107
162
|
use_tls = kwargs.get("use_tls")
|
|
163
|
+
tls_params = {}
|
|
108
164
|
if "sslmode" in kwargs:
|
|
109
165
|
# Explicit sslmode takes precedence
|
|
110
166
|
sslmode = kwargs["sslmode"]
|
|
111
167
|
elif use_tls is False:
|
|
112
168
|
# use_tls=False → disable SSL
|
|
113
169
|
sslmode = "disable"
|
|
114
|
-
elif
|
|
115
|
-
# use_tls=
|
|
170
|
+
elif isinstance(use_tls, dict):
|
|
171
|
+
# use_tls=dict → require SSL, verified when a CA is given
|
|
172
|
+
tls_params = _libpq_tls_params(use_tls)
|
|
173
|
+
sslmode = tls_params.pop("sslmode")
|
|
174
|
+
elif use_tls is True:
|
|
175
|
+
# use_tls=True → require SSL
|
|
116
176
|
sslmode = "require"
|
|
117
177
|
else:
|
|
118
|
-
# use_tls=None (default) →
|
|
119
|
-
sslmode
|
|
178
|
+
# use_tls=None (default) → require SSL, then fall back below. libpq's
|
|
179
|
+
# sslmode=prefer also retries without SSL after an authentication failure.
|
|
180
|
+
sslmode = "require"
|
|
120
181
|
|
|
121
|
-
|
|
182
|
+
params = dict(
|
|
122
183
|
host=host,
|
|
123
184
|
port=port,
|
|
124
185
|
user=user,
|
|
125
186
|
password=password,
|
|
126
187
|
dbname=dbname,
|
|
127
|
-
sslmode=sslmode,
|
|
128
188
|
connect_timeout=connect_timeout,
|
|
189
|
+
**tls_params,
|
|
129
190
|
)
|
|
191
|
+
try:
|
|
192
|
+
conn = client.connect(sslmode=sslmode, **params)
|
|
193
|
+
except client.OperationalError as error:
|
|
194
|
+
if "sslmode" in kwargs or use_tls is not None or not _tls_rejected(error):
|
|
195
|
+
raise
|
|
196
|
+
logger.debug("Connecting without SSL: %s", str(error).strip())
|
|
197
|
+
conn = client.connect(sslmode="disable", **params)
|
|
130
198
|
# DataJoint manages transactions explicitly via start_transaction()
|
|
131
199
|
# Set autocommit=True to avoid implicit transactions
|
|
132
200
|
conn.autocommit = True
|
|
@@ -1115,7 +1183,7 @@ class PostgreSQLAdapter(DatabaseAdapter):
|
|
|
1115
1183
|
path_parts.extend(indices)
|
|
1116
1184
|
else:
|
|
1117
1185
|
path_parts.append(part)
|
|
1118
|
-
path_args = ", ".join(
|
|
1186
|
+
path_args = ", ".join(self.quote_string(part) for part in path_parts)
|
|
1119
1187
|
expr = f"jsonb_extract_path_text({quoted_col}, {path_args})"
|
|
1120
1188
|
# Add cast if return type specified
|
|
1121
1189
|
if return_type:
|
|
@@ -10,6 +10,7 @@ from __future__ import annotations
|
|
|
10
10
|
|
|
11
11
|
import collections
|
|
12
12
|
import datetime
|
|
13
|
+
import sys
|
|
13
14
|
import uuid
|
|
14
15
|
import zlib
|
|
15
16
|
from decimal import Decimal
|
|
@@ -58,7 +59,32 @@ serialize_lookup = {
|
|
|
58
59
|
}
|
|
59
60
|
|
|
60
61
|
|
|
61
|
-
|
|
62
|
+
def _zlib_decompress(data, size):
|
|
63
|
+
"""
|
|
64
|
+
Decompress zlib ``data`` that is declared to expand to exactly ``size`` bytes.
|
|
65
|
+
|
|
66
|
+
Output is capped at ``size`` bytes, so a payload that expands beyond its
|
|
67
|
+
declared size is rejected without being expanded in full.
|
|
68
|
+
"""
|
|
69
|
+
if size > sys.maxsize:
|
|
70
|
+
raise DataJointError(f"Blob size mismatch: declared size {size} is not supported")
|
|
71
|
+
decompressor = zlib.decompressobj()
|
|
72
|
+
try:
|
|
73
|
+
# max_length=0 would mean "no limit", so ask for at least one byte
|
|
74
|
+
blob = decompressor.decompress(data, max(size, 1))
|
|
75
|
+
if not decompressor.eof:
|
|
76
|
+
# check whether the stream holds more than the declared size
|
|
77
|
+
blob += decompressor.decompress(decompressor.unconsumed_tail, 1)
|
|
78
|
+
except zlib.error as error:
|
|
79
|
+
raise DataJointError(f"Corrupt compressed blob: {error}") from None
|
|
80
|
+
if len(blob) > size:
|
|
81
|
+
raise DataJointError(f"Blob size mismatch: expected {size} bytes, compressed data holds more")
|
|
82
|
+
if len(blob) != size or not decompressor.eof:
|
|
83
|
+
raise DataJointError(f"Blob size mismatch: expected {size}, got {len(blob)}")
|
|
84
|
+
return blob
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
compression = {b"ZL123\0": _zlib_decompress}
|
|
62
88
|
|
|
63
89
|
# runtime setting to read integers as 32-bit to read blobs created by the 32-bit
|
|
64
90
|
# version of the mYm library for MATLAB
|
|
@@ -161,10 +187,7 @@ class Blob:
|
|
|
161
187
|
else:
|
|
162
188
|
self._pos += len(prefix)
|
|
163
189
|
blob_size = self.read_value()
|
|
164
|
-
|
|
165
|
-
if len(blob) != blob_size:
|
|
166
|
-
raise DataJointError(f"Blob size mismatch: expected {blob_size}, got {len(blob)}")
|
|
167
|
-
self._blob = blob
|
|
190
|
+
self._blob = compression[prefix](self._blob[self._pos :], int(blob_size))
|
|
168
191
|
self._pos = 0
|
|
169
192
|
blob_format = self.read_zero_terminated_string()
|
|
170
193
|
if blob_format in ("mYm", "dj0"):
|
|
@@ -4,9 +4,46 @@ File attachment codec with filename preservation.
|
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import os
|
|
8
|
+
import stat
|
|
9
|
+
from pathlib import Path
|
|
7
10
|
from typing import Any
|
|
8
11
|
|
|
9
12
|
from ..codecs import Codec
|
|
13
|
+
from ..errors import DataJointError
|
|
14
|
+
from ..utils import is_plain_file_name
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _check_file_name(filename: str) -> None:
|
|
18
|
+
"""
|
|
19
|
+
Raise unless ``filename`` is a plain file name with no directory part.
|
|
20
|
+
|
|
21
|
+
``encode`` stores only the base name of the attached file. A stored name
|
|
22
|
+
with path separators (``/`` or ``\\``), ``..``, a NUL byte, or (on Windows)
|
|
23
|
+
a drive or stream marker ``:`` is refused.
|
|
24
|
+
"""
|
|
25
|
+
if not is_plain_file_name(filename):
|
|
26
|
+
raise DataJointError(f"<attach> stored file name {filename!r} is not a plain file name.")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _has_contents(path: Path, contents: bytes) -> bool:
|
|
30
|
+
"""
|
|
31
|
+
Check whether ``path`` is a regular file holding exactly ``contents``.
|
|
32
|
+
|
|
33
|
+
Symbolic links, devices, and pipes are never opened, and a file is read only
|
|
34
|
+
when its size matches.
|
|
35
|
+
"""
|
|
36
|
+
try:
|
|
37
|
+
st = os.lstat(path)
|
|
38
|
+
except FileNotFoundError:
|
|
39
|
+
return False
|
|
40
|
+
if not stat.S_ISREG(st.st_mode) or st.st_size != len(contents):
|
|
41
|
+
return False
|
|
42
|
+
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0))
|
|
43
|
+
with os.fdopen(fd, "rb") as f:
|
|
44
|
+
if not stat.S_ISREG(os.fstat(f.fileno()).st_mode):
|
|
45
|
+
return False
|
|
46
|
+
return f.read(len(contents) + 1) == contents
|
|
10
47
|
|
|
11
48
|
|
|
12
49
|
class AttachCodec(Codec):
|
|
@@ -19,7 +56,9 @@ class AttachCodec(Codec):
|
|
|
19
56
|
- ``<attach@store>``: Stored in specific named store
|
|
20
57
|
|
|
21
58
|
The filename is preserved and the file is extracted to the configured
|
|
22
|
-
download path on fetch.
|
|
59
|
+
download path on fetch. Only the base name is kept, and fetch refuses a
|
|
60
|
+
stored name that is not a plain file name, so extracted files always land
|
|
61
|
+
directly in the download path.
|
|
23
62
|
|
|
24
63
|
Example::
|
|
25
64
|
|
|
@@ -70,8 +109,6 @@ class AttachCodec(Codec):
|
|
|
70
109
|
bytes
|
|
71
110
|
Filename (UTF-8) + null byte + file contents.
|
|
72
111
|
"""
|
|
73
|
-
from pathlib import Path
|
|
74
|
-
|
|
75
112
|
path = Path(value)
|
|
76
113
|
if not path.exists():
|
|
77
114
|
raise FileNotFoundError(f"Attachment file not found: {path}")
|
|
@@ -97,13 +134,17 @@ class AttachCodec(Codec):
|
|
|
97
134
|
-------
|
|
98
135
|
str
|
|
99
136
|
Path to extracted file.
|
|
100
|
-
"""
|
|
101
|
-
from pathlib import Path
|
|
102
137
|
|
|
138
|
+
Raises
|
|
139
|
+
------
|
|
140
|
+
DataJointError
|
|
141
|
+
If the stored file name is not a plain file name.
|
|
142
|
+
"""
|
|
103
143
|
# Split on first null byte
|
|
104
144
|
null_pos = stored.index(b"\x00")
|
|
105
145
|
filename = stored[:null_pos].decode("utf-8")
|
|
106
146
|
contents = stored[null_pos + 1 :]
|
|
147
|
+
_check_file_name(filename)
|
|
107
148
|
|
|
108
149
|
# Write to download path
|
|
109
150
|
config = self._codec_config(key, context)
|
|
@@ -113,28 +154,31 @@ class AttachCodec(Codec):
|
|
|
113
154
|
download_path = Path(config.get("download_path", "."))
|
|
114
155
|
download_path.mkdir(parents=True, exist_ok=True)
|
|
115
156
|
local_path = download_path / filename
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
157
|
+
if local_path.parent != download_path:
|
|
158
|
+
raise DataJointError(f"<attach> file name {filename!r} leads outside the download path.")
|
|
159
|
+
|
|
160
|
+
# Reuse a file with identical content; otherwise add a numeric suffix until
|
|
161
|
+
# a free name is found. New files are created exclusively, never through an
|
|
162
|
+
# existing file or link.
|
|
163
|
+
stem = local_path.stem
|
|
164
|
+
suffix = local_path.suffix
|
|
165
|
+
counter = 0
|
|
166
|
+
while True:
|
|
167
|
+
if _has_contents(local_path, contents):
|
|
168
|
+
return str(local_path)
|
|
169
|
+
if not os.path.lexists(local_path):
|
|
170
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
|
|
171
|
+
try:
|
|
172
|
+
fd = os.open(local_path, flags, 0o666)
|
|
173
|
+
except FileExistsError:
|
|
174
|
+
continue # created concurrently; check it again
|
|
175
|
+
with os.fdopen(fd, "wb") as f:
|
|
176
|
+
f.write(contents)
|
|
177
|
+
return str(local_path)
|
|
178
|
+
counter += 1
|
|
179
|
+
local_path = download_path / f"{stem}_{counter}{suffix}"
|
|
134
180
|
|
|
135
181
|
def validate(self, value: Any) -> None:
|
|
136
182
|
"""Validate that value is a valid file path."""
|
|
137
|
-
from pathlib import Path
|
|
138
|
-
|
|
139
183
|
if not isinstance(value, (str, Path)):
|
|
140
184
|
raise TypeError(f"<attach> expects a file path, got {type(value).__name__}")
|
|
@@ -59,6 +59,14 @@ class FilepathCodec(Codec):
|
|
|
59
59
|
Paths cannot start with the store's reserved ``hash_prefix`` or
|
|
60
60
|
``schema_prefix`` sections (defaults ``_hash/`` and ``_schema/``).
|
|
61
61
|
|
|
62
|
+
Store Boundary:
|
|
63
|
+
Paths must stay inside the store location: paths with ``..``
|
|
64
|
+
components and, for ``file`` stores, absolute paths outside the
|
|
65
|
+
location are rejected on insert and on access. An absolute path
|
|
66
|
+
inside a ``file`` store's location is recorded relative to it. Set
|
|
67
|
+
``dj.config.allow_filepath_outside_store`` to accept references that
|
|
68
|
+
point outside the store.
|
|
69
|
+
|
|
62
70
|
Warning:
|
|
63
71
|
The file must exist in the store at the specified path.
|
|
64
72
|
DataJoint does not manage the lifecycle of referenced files.
|
|
@@ -99,12 +107,16 @@ class FilepathCodec(Codec):
|
|
|
99
107
|
------
|
|
100
108
|
ValueError
|
|
101
109
|
If path uses the store's reserved hash/schema sections.
|
|
110
|
+
DataJointError
|
|
111
|
+
If path leads outside the store location (see
|
|
112
|
+
``allow_filepath_outside_store``).
|
|
102
113
|
FileNotFoundError
|
|
103
114
|
If file does not exist in the store.
|
|
104
115
|
"""
|
|
116
|
+
import posixpath
|
|
105
117
|
from datetime import datetime, timezone
|
|
106
118
|
|
|
107
|
-
from ..
|
|
119
|
+
from ..storage import path_in_location
|
|
108
120
|
|
|
109
121
|
config = self._codec_config(key, context)
|
|
110
122
|
if config is None:
|
|
@@ -117,8 +129,16 @@ class FilepathCodec(Codec):
|
|
|
117
129
|
# Use filepath_default if no store specified (filepath is not part of OAS)
|
|
118
130
|
spec = config.get_store_spec(store_name, use_filepath_default=True)
|
|
119
131
|
|
|
120
|
-
#
|
|
121
|
-
|
|
132
|
+
# Reject paths outside the store before any prefix checks. An absolute
|
|
133
|
+
# path inside a file store's location is recorded relative to it.
|
|
134
|
+
backend = self._get_backend(store_name, config)
|
|
135
|
+
backend._full_path(path)
|
|
136
|
+
if spec.get("protocol") == "file":
|
|
137
|
+
path = path_in_location(spec["location"], path) or path
|
|
138
|
+
|
|
139
|
+
# Validate path doesn't use reserved sections (hash and schema).
|
|
140
|
+
# Normalize first so that "./_hash/x" or "//_hash/x" are caught too.
|
|
141
|
+
path_normalized = posixpath.normpath(path.replace("\\", "/")).lstrip("/")
|
|
122
142
|
reserved_prefixes = []
|
|
123
143
|
|
|
124
144
|
hash_prefix = spec.get("hash_prefix")
|
|
@@ -129,10 +149,11 @@ class FilepathCodec(Codec):
|
|
|
129
149
|
if schema_prefix:
|
|
130
150
|
reserved_prefixes.append(("schema_prefix", schema_prefix))
|
|
131
151
|
|
|
132
|
-
# Check if path starts with any reserved prefix
|
|
152
|
+
# Check if path starts with any reserved prefix. Compare without case:
|
|
153
|
+
# on case-insensitive file systems "_HASH/x" is the same as "_hash/x".
|
|
133
154
|
for prefix_name, prefix_value in reserved_prefixes:
|
|
134
155
|
prefix_normalized = prefix_value.strip("/") + "/"
|
|
135
|
-
if path_normalized.startswith(prefix_normalized):
|
|
156
|
+
if (path_normalized + "/").casefold().startswith(prefix_normalized.casefold()):
|
|
136
157
|
raise ValueError(
|
|
137
158
|
f"<filepath@> cannot use reserved section '{prefix_value}' ({prefix_name}). "
|
|
138
159
|
f"This section is managed by DataJoint. "
|
|
@@ -147,7 +168,6 @@ class FilepathCodec(Codec):
|
|
|
147
168
|
raise ValueError(f"<filepath@> must use prefix '{filepath_prefix}' (filepath_prefix). Got path: {path}")
|
|
148
169
|
|
|
149
170
|
# Verify file exists
|
|
150
|
-
backend = get_store_backend(store_name, config=config)
|
|
151
171
|
if not backend.exists(path):
|
|
152
172
|
raise FileNotFoundError(f"File not found in store '{store_name or 'default'}': {path}")
|
|
153
173
|
|
|
@@ -187,13 +207,25 @@ class FilepathCodec(Codec):
|
|
|
187
207
|
Handle for accessing the file.
|
|
188
208
|
"""
|
|
189
209
|
from ..objectref import ObjectRef
|
|
190
|
-
from ..hash_registry import get_store_backend
|
|
191
210
|
|
|
192
211
|
config = self._codec_config(key, context)
|
|
193
|
-
|
|
194
|
-
backend = get_store_backend(store_name, config=config)
|
|
212
|
+
backend = self._get_backend(stored.get("store"), config)
|
|
195
213
|
return ObjectRef.from_json(stored, backend=backend)
|
|
196
214
|
|
|
215
|
+
@staticmethod
|
|
216
|
+
def _get_backend(store_name: str | None, config):
|
|
217
|
+
"""
|
|
218
|
+
Get the store backend, confined to the store location unless
|
|
219
|
+
``allow_filepath_outside_store`` is set.
|
|
220
|
+
"""
|
|
221
|
+
from ..hash_registry import get_store_backend
|
|
222
|
+
|
|
223
|
+
if config is None:
|
|
224
|
+
from ..settings import config # type: ignore[assignment]
|
|
225
|
+
backend = get_store_backend(store_name, config=config)
|
|
226
|
+
backend.confine_paths = not config.get("allow_filepath_outside_store", False)
|
|
227
|
+
return backend
|
|
228
|
+
|
|
197
229
|
def validate(self, value: Any) -> None:
|
|
198
230
|
"""Validate that value is a path string or Path object."""
|
|
199
231
|
from pathlib import Path
|