datajoint 2.2.0__tar.gz → 2.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. {datajoint-2.2.0 → datajoint-2.2.2}/CONTRIBUTING.md +38 -2
  2. {datajoint-2.2.0 → datajoint-2.2.2}/PKG-INFO +2 -2
  3. {datajoint-2.2.0 → datajoint-2.2.2}/README.md +1 -1
  4. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/__init__.py +7 -0
  5. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/mysql.py +3 -7
  6. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/connection.py +57 -21
  7. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/declare.py +8 -0
  8. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/gc.py +14 -8
  9. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/heading.py +10 -7
  10. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/migrate.py +10 -6
  11. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/schemas.py +7 -0
  12. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/settings.py +34 -14
  13. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/storage.py +14 -5
  14. datajoint-2.2.2/src/datajoint/storage_adapter.py +109 -0
  15. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/version.py +1 -1
  16. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_gc.py +118 -0
  17. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_json.py +4 -0
  18. datajoint-2.2.2/tests/unit/test_connection_warning.py +42 -0
  19. datajoint-2.2.2/tests/unit/test_declare_hidden_attribute.py +51 -0
  20. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_settings.py +65 -0
  21. datajoint-2.2.2/tests/unit/test_storage_adapter.py +313 -0
  22. {datajoint-2.2.0 → datajoint-2.2.2}/.coveragerc +0 -0
  23. {datajoint-2.2.0 → datajoint-2.2.2}/.devcontainer/Dockerfile +0 -0
  24. {datajoint-2.2.0 → datajoint-2.2.2}/.devcontainer/devcontainer.json +0 -0
  25. {datajoint-2.2.0 → datajoint-2.2.2}/.devcontainer/docker-compose.yml +0 -0
  26. {datajoint-2.2.0 → datajoint-2.2.2}/.dockerignore +0 -0
  27. {datajoint-2.2.0 → datajoint-2.2.2}/.gitattributes +0 -0
  28. {datajoint-2.2.0 → datajoint-2.2.2}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
  29. {datajoint-2.2.0 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  30. {datajoint-2.2.0 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  31. {datajoint-2.2.0 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
  32. {datajoint-2.2.0 → datajoint-2.2.2}/.github/dependabot.yml +0 -0
  33. {datajoint-2.2.0 → datajoint-2.2.2}/.github/release_drafter.yaml +0 -0
  34. {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
  35. {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/draft_release.yaml +0 -0
  36. {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/lint.yaml +0 -0
  37. {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/post_draft_release_published.yaml +0 -0
  38. {datajoint-2.2.0 → datajoint-2.2.2}/.github/workflows/test.yaml +0 -0
  39. {datajoint-2.2.0 → datajoint-2.2.2}/.gitignore +0 -0
  40. {datajoint-2.2.0 → datajoint-2.2.2}/.pre-commit-config.yaml +0 -0
  41. {datajoint-2.2.0 → datajoint-2.2.2}/.vscode/launch.json +0 -0
  42. {datajoint-2.2.0 → datajoint-2.2.2}/CHANGELOG-archive.md +0 -0
  43. {datajoint-2.2.0 → datajoint-2.2.2}/Dockerfile +0 -0
  44. {datajoint-2.2.0 → datajoint-2.2.2}/LICENSE +0 -0
  45. {datajoint-2.2.0 → datajoint-2.2.2}/RELEASE_MEMO.md +0 -0
  46. {datajoint-2.2.0 → datajoint-2.2.2}/activate.sh +0 -0
  47. {datajoint-2.2.0 → datajoint-2.2.2}/docker-compose.yaml +0 -0
  48. {datajoint-2.2.0 → datajoint-2.2.2}/images/pipeline.drawio +0 -0
  49. {datajoint-2.2.0 → datajoint-2.2.2}/images/pipeline.png +0 -0
  50. {datajoint-2.2.0 → datajoint-2.2.2}/pixi.lock +0 -0
  51. {datajoint-2.2.0 → datajoint-2.2.2}/pyproject.toml +0 -0
  52. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/__init__.py +0 -0
  53. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/base.py +0 -0
  54. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/adapters/postgres.py +0 -0
  55. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/autopopulate.py +0 -0
  56. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/blob.py +0 -0
  57. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/__init__.py +0 -0
  58. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/attach.py +0 -0
  59. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/blob.py +0 -0
  60. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/filepath.py +0 -0
  61. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/hash.py +0 -0
  62. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/npy.py +0 -0
  63. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/object.py +0 -0
  64. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/builtin_codecs/schema.py +0 -0
  65. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/cli.py +0 -0
  66. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/codecs.py +0 -0
  67. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/condition.py +0 -0
  68. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/dependencies.py +0 -0
  69. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/diagram.py +0 -0
  70. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/errors.py +0 -0
  71. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/expression.py +0 -0
  72. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/hash_registry.py +0 -0
  73. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/instance.py +0 -0
  74. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/jobs.py +0 -0
  75. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/lineage.py +0 -0
  76. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/logging.py +0 -0
  77. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/objectref.py +0 -0
  78. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/preview.py +0 -0
  79. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/staged_insert.py +0 -0
  80. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/table.py +0 -0
  81. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/types.py +0 -0
  82. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/user_tables.py +0 -0
  83. {datajoint-2.2.0 → datajoint-2.2.2}/src/datajoint/utils.py +0 -0
  84. {datajoint-2.2.0 → datajoint-2.2.2}/tests/__init__.py +0 -0
  85. {datajoint-2.2.0 → datajoint-2.2.2}/tests/conftest.py +0 -0
  86. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/__init__.py +0 -0
  87. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Course.csv +0 -0
  88. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/CurrentTerm.csv +0 -0
  89. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Department.csv +0 -0
  90. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Enroll.csv +0 -0
  91. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Grade.csv +0 -0
  92. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Section.csv +0 -0
  93. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Student.csv +0 -0
  94. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/StudentMajor.csv +0 -0
  95. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/data/Term.csv +0 -0
  96. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_aggr_regressions.py +0 -0
  97. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_alter.py +0 -0
  98. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_attach.py +0 -0
  99. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_autopopulate.py +0 -0
  100. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_blob.py +0 -0
  101. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_blob_matlab.py +0 -0
  102. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_cascade_delete.py +0 -0
  103. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_cascading_delete.py +0 -0
  104. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_cli.py +0 -0
  105. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_codec_chaining.py +0 -0
  106. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_codecs.py +0 -0
  107. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_connection.py +0 -0
  108. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_declare.py +0 -0
  109. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_dependencies.py +0 -0
  110. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_erd.py +0 -0
  111. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_fetch.py +0 -0
  112. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_fetch_same.py +0 -0
  113. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_foreign_keys.py +0 -0
  114. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_groupby.py +0 -0
  115. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_hash_storage.py +0 -0
  116. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_hidden_job_metadata.py +0 -0
  117. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_insert.py +0 -0
  118. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_jobs.py +0 -0
  119. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_multi_backend.py +0 -0
  120. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_nan.py +0 -0
  121. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_npy_codec.py +0 -0
  122. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_object.py +0 -0
  123. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_privileges.py +0 -0
  124. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_reconnection.py +0 -0
  125. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_relation.py +0 -0
  126. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_relation_u.py +0 -0
  127. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_relational_operand.py +0 -0
  128. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_schema.py +0 -0
  129. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_schema_keywords.py +0 -0
  130. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_semantic_matching.py +0 -0
  131. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_skip_duplicates.py +0 -0
  132. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_tls.py +0 -0
  133. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_type_aliases.py +0 -0
  134. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_university.py +0 -0
  135. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_update1.py +0 -0
  136. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_utils.py +0 -0
  137. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_uuid.py +0 -0
  138. {datajoint-2.2.0 → datajoint-2.2.2}/tests/integration/test_virtual_module.py +0 -0
  139. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema.py +0 -0
  140. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_advanced.py +0 -0
  141. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_aggr_regress.py +0 -0
  142. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_alter.py +0 -0
  143. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_codecs.py +0 -0
  144. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_external.py +0 -0
  145. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_object.py +0 -0
  146. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_privileges.py +0 -0
  147. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_simple.py +0 -0
  148. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_type_aliases.py +0 -0
  149. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_university.py +0 -0
  150. {datajoint-2.2.0 → datajoint-2.2.2}/tests/schema_uuid.py +0 -0
  151. {datajoint-2.2.0 → datajoint-2.2.2}/tests/test_package.py +0 -0
  152. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/__init__.py +0 -0
  153. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_adapters.py +0 -0
  154. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_codecs.py +0 -0
  155. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_condition.py +0 -0
  156. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_fetch_compat.py +0 -0
  157. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_lazy_imports.py +0 -0
  158. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_pk_rules.py +0 -0
  159. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_storage_urls.py +0 -0
  160. {datajoint-2.2.0 → datajoint-2.2.2}/tests/unit/test_thread_safe.py +0 -0
@@ -36,13 +36,17 @@ pytest tests/
36
36
 
37
37
  ## Running Tests
38
38
 
39
- Tests use [testcontainers](https://testcontainers.com/) to automatically manage MySQL and MinIO containers. No manual `docker-compose up` required.
39
+ Tests use [testcontainers](https://testcontainers.com/) to automatically manage MySQL, PostgreSQL, and MinIO containers. No manual `docker-compose up` required.
40
+
41
+ Integration tests are **backend-parameterized** — tests using the `backend` fixture run automatically against both MySQL and PostgreSQL.
40
42
 
41
43
  ```bash
42
- pixi run test # All tests
44
+ pixi run test # All tests (both backends)
43
45
  pixi run test-cov # With coverage
44
46
  pixi run -e test pytest tests/unit/ # Unit tests only
45
47
  pixi run -e test pytest tests/integration/test_blob.py -v # Specific file
48
+ pixi run -e test pytest -m mysql # MySQL tests only
49
+ pixi run -e test pytest -m postgresql # PostgreSQL tests only
46
50
  ```
47
51
 
48
52
  **macOS Docker Desktop users:** If tests fail to connect:
@@ -50,12 +54,28 @@ pixi run -e test pytest tests/integration/test_blob.py -v # Specific file
50
54
  export DOCKER_HOST=unix://$HOME/.docker/run/docker.sock
51
55
  ```
52
56
 
57
+ ### PostgreSQL Backend
58
+
59
+ DataJoint supports MySQL 8.0.13+ and PostgreSQL 15+ as production database backends. To install the PostgreSQL driver:
60
+
61
+ ```bash
62
+ pip install -e ".[postgres]" # Installs psycopg2-binary
63
+ ```
64
+
65
+ Tests automatically spin up both MySQL and PostgreSQL containers via testcontainers. Backend-parameterized tests (those using the `backend` fixture in `tests/conftest.py`) run against both backends to ensure feature parity.
66
+
53
67
  ### External Containers (for debugging)
54
68
 
55
69
  ```bash
70
+ # MySQL + MinIO
56
71
  docker compose up -d db minio
57
72
  DJ_USE_EXTERNAL_CONTAINERS=1 pixi run test
58
73
  docker compose down
74
+
75
+ # MySQL + PostgreSQL + MinIO
76
+ docker compose up -d db postgres minio
77
+ DJ_USE_EXTERNAL_CONTAINERS=1 pixi run test
78
+ docker compose down
59
79
  ```
60
80
 
61
81
  ### Full Docker
@@ -91,12 +111,28 @@ Hooks include: **ruff** (lint/format), **codespell**, YAML/JSON/TOML validation.
91
111
 
92
112
  For `DJ_USE_EXTERNAL_CONTAINERS=1`:
93
113
 
114
+ ### MySQL
115
+
94
116
  | Variable | Default | Description |
95
117
  |----------|---------|-------------|
96
118
  | `DJ_HOST` | `localhost` | MySQL hostname |
97
119
  | `DJ_PORT` | `3306` | MySQL port |
98
120
  | `DJ_USER` | `root` | MySQL username |
99
121
  | `DJ_PASS` | `password` | MySQL password |
122
+
123
+ ### PostgreSQL
124
+
125
+ | Variable | Default | Description |
126
+ |----------|---------|-------------|
127
+ | `DJ_PG_HOST` | `localhost` | PostgreSQL hostname |
128
+ | `DJ_PG_PORT` | `5432` | PostgreSQL port |
129
+ | `DJ_PG_USER` | `postgres` | PostgreSQL username |
130
+ | `DJ_PG_PASS` | `password` | PostgreSQL password |
131
+
132
+ ### Object Storage
133
+
134
+ | Variable | Default | Description |
135
+ |----------|---------|-------------|
100
136
  | `S3_ENDPOINT` | `localhost:9000` | MinIO endpoint |
101
137
 
102
138
  ---
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datajoint
3
- Version: 2.2.0
3
+ Version: 2.2.2
4
4
  Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
5
5
  Project-URL: Homepage, https://docs.datajoint.com/
6
6
  Project-URL: Documentation, https://docs.datajoint.com/
@@ -346,4 +346,4 @@ conda install -c conda-forge datajoint
346
346
 
347
347
  ## Contributing
348
348
 
349
- See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines.
349
+ See [CONTRIBUTING.md](https://github.com/datajoint/datajoint-python/blob/master/CONTRIBUTING.md) for development setup and guidelines.
@@ -85,4 +85,4 @@ conda install -c conda-forge datajoint
85
85
 
86
86
  ## Contributing
87
87
 
88
- See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines.
88
+ See [CONTRIBUTING.md](https://github.com/datajoint/datajoint-python/blob/master/CONTRIBUTING.md) for development setup and guidelines.
@@ -30,6 +30,8 @@ __all__ = [
30
30
  "list_schemas",
31
31
  "Table",
32
32
  "FreeTable",
33
+ "AutoPopulate",
34
+ "Job",
33
35
  "Manual",
34
36
  "Lookup",
35
37
  "Imported",
@@ -81,6 +83,8 @@ from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global
81
83
  from .logging import logger
82
84
  from .objectref import ObjectRef
83
85
  from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
86
+ from .autopopulate import AutoPopulate
87
+ from .jobs import Job
84
88
  from .table import FreeTable as _FreeTable, Table, ValidationResult
85
89
  from .user_tables import Computed, Imported, Lookup, Manual, Part
86
90
  from .version import __version__
@@ -271,6 +275,9 @@ _lazy_modules = {
271
275
  "diagram": (".diagram", None), # Return the module itself
272
276
  # cli imports click
273
277
  "cli": (".cli", "cli"),
278
+ # gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
279
+ # and in the user docs (how-to/garbage-collection.md).
280
+ "gc": (".gc", None), # Return the module itself
274
281
  }
275
282
 
276
283
 
@@ -423,7 +423,7 @@ class MySQLAdapter(DatabaseAdapter):
423
423
  fk_cols = ", ".join(self.quote_identifier(col) for col in fk["columns"])
424
424
  ref_cols = ", ".join(self.quote_identifier(col) for col in fk["ref_columns"])
425
425
  lines.append(
426
- f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols}) " f"ON UPDATE CASCADE ON DELETE RESTRICT"
426
+ f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols}) ON UPDATE CASCADE ON DELETE RESTRICT"
427
427
  )
428
428
 
429
429
  # Indexes
@@ -735,14 +735,10 @@ class MySQLAdapter(DatabaseAdapter):
735
735
  return result
736
736
 
737
737
  def get_indexes_sql(self, schema_name: str, table_name: str) -> str:
738
- """Query to get index definitions.
739
-
740
- Note: For MySQL 8.0+, EXPRESSION column contains the expression for
741
- functional indexes. COLUMN_NAME is NULL for such indexes.
742
- """
738
+ """Query to get index definitions. Functional indexes (NULL COLUMN_NAME) are skipped downstream."""
743
739
  return (
744
740
  f"SELECT INDEX_NAME as index_name, "
745
- f"COALESCE(COLUMN_NAME, CONCAT('(', EXPRESSION, ')')) as column_name, "
741
+ f"COLUMN_NAME as column_name, "
746
742
  f"NON_UNIQUE as non_unique, SEQ_IN_INDEX as seq_in_index "
747
743
  f"FROM information_schema.statistics "
748
744
  f"WHERE table_schema = {self.quote_string(schema_name)} "
@@ -108,6 +108,18 @@ def conn(
108
108
  return conn.connection
109
109
 
110
110
 
111
+ def _warn_if_mariadb(version_str: str) -> None:
112
+ """Emit a UserWarning if `version_str` looks like MariaDB. No-op for MySQL."""
113
+ if "MariaDB" in version_str:
114
+ warnings.warn(
115
+ f"MariaDB is not officially supported by DataJoint 2.x "
116
+ f"(server reports {version_str}). Compatibility is best-effort "
117
+ f"and may break in future releases.",
118
+ UserWarning,
119
+ stacklevel=3,
120
+ )
121
+
122
+
111
123
  class EmulatedCursor:
112
124
  """acts like a cursor"""
113
125
 
@@ -168,6 +180,7 @@ class Connection:
168
180
  port: int | None = None,
169
181
  use_tls: bool | dict | None = None,
170
182
  *,
183
+ database_name: str | None = None,
171
184
  backend: str | None = None,
172
185
  config_override: "Config | None" = None,
173
186
  ) -> None:
@@ -180,7 +193,9 @@ class Connection:
180
193
  port = int(port)
181
194
  elif port is None:
182
195
  port = self._config["database.port"]
183
- self.conn_info = dict(host=host, port=port, user=user, passwd=password)
196
+ if database_name is None:
197
+ database_name = self._config.get("database.name")
198
+ self.conn_info = dict(host=host, port=port, user=user, passwd=password, database_name=database_name)
184
199
  if use_tls is not False:
185
200
  # use_tls can be: None (auto-detect), True (enable), False (disable), or dict (custom config)
186
201
  if isinstance(use_tls, dict):
@@ -201,12 +216,29 @@ class Connection:
201
216
  backend = self._config["database.backend"]
202
217
  self.adapter = get_adapter(backend)
203
218
 
219
+ if database_name and self.adapter.backend == "mysql":
220
+ warnings.warn(
221
+ "database.name is set but the MySQL backend does not support database selection. "
222
+ "This setting only applies to PostgreSQL connections.",
223
+ UserWarning,
224
+ stacklevel=2,
225
+ )
226
+
204
227
  self.connect()
205
228
  if self.is_connected:
206
- logger.info("DataJoint {version} connected to {user}@{host}:{port}".format(version=__version__, **self.conn_info))
229
+ db = self.conn_info.get("database_name")
230
+ db_str = f"/{db}" if db else ""
231
+ logger.info(
232
+ f"DataJoint {__version__} connected to "
233
+ f"{self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}{db_str}"
234
+ )
207
235
  self.connection_id = self.adapter.get_connection_id(self._conn)
236
+ if self.adapter.backend == "mysql":
237
+ _warn_if_mariadb(self.query("SELECT @@version").fetchone()[0])
208
238
  else:
209
- raise errors.LostConnectionError("Connection failed {user}@{host}:{port}".format(**self.conn_info))
239
+ raise errors.LostConnectionError(
240
+ f"Connection failed {self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}"
241
+ )
210
242
  self._in_transaction = False
211
243
  self.schemas = dict()
212
244
  self.dependencies = Dependencies(self)
@@ -216,22 +248,33 @@ class Connection:
216
248
 
217
249
  def __repr__(self):
218
250
  connected = "connected" if self.is_connected else "disconnected"
219
- return "DataJoint connection ({connected}) {user}@{host}:{port}".format(connected=connected, **self.conn_info)
251
+ user = self.conn_info["user"]
252
+ host = self.conn_info["host"]
253
+ port = self.conn_info["port"]
254
+ db = self.conn_info.get("database_name")
255
+ db_str = f"/{db}" if db else ""
256
+ return f"DataJoint connection ({connected}) {user}@{host}:{port}{db_str}"
257
+
258
+ def _build_connect_kwargs(self, use_tls=None):
259
+ """Build kwargs dict for adapter.connect()."""
260
+ kwargs = dict(
261
+ host=self.conn_info["host"],
262
+ port=self.conn_info["port"],
263
+ user=self.conn_info["user"],
264
+ password=self.conn_info["passwd"],
265
+ charset=self._config["connection.charset"],
266
+ use_tls=use_tls if use_tls is not None else self.conn_info.get("ssl"),
267
+ )
268
+ if self.conn_info.get("database_name"):
269
+ kwargs["dbname"] = self.conn_info["database_name"]
270
+ return kwargs
220
271
 
221
272
  def connect(self) -> None:
222
273
  """Establish or re-establish connection to the database server."""
223
274
  with warnings.catch_warnings():
224
275
  warnings.filterwarnings("ignore", ".*deprecated.*")
225
276
  try:
226
- # Use adapter to create connection
227
- self._conn = self.adapter.connect(
228
- host=self.conn_info["host"],
229
- port=self.conn_info["port"],
230
- user=self.conn_info["user"],
231
- password=self.conn_info["passwd"],
232
- charset=self._config["connection.charset"],
233
- use_tls=self.conn_info.get("ssl"),
234
- )
277
+ self._conn = self.adapter.connect(**self._build_connect_kwargs())
235
278
  except Exception as ssl_error:
236
279
  # If SSL fails, retry without SSL (if it was auto-detected)
237
280
  if self.conn_info.get("ssl_input") is None:
@@ -240,14 +283,7 @@ class Connection:
240
283
  "To require SSL, set use_tls=True explicitly.",
241
284
  ssl_error,
242
285
  )
243
- self._conn = self.adapter.connect(
244
- host=self.conn_info["host"],
245
- port=self.conn_info["port"],
246
- user=self.conn_info["user"],
247
- password=self.conn_info["passwd"],
248
- charset=self._config["connection.charset"],
249
- use_tls=False, # Explicitly disable SSL for fallback
250
- )
286
+ self._conn = self.adapter.connect(**self._build_connect_kwargs(use_tls=False))
251
287
  else:
252
288
  raise
253
289
  self._is_closed = False # Mark as connected after successful connection
@@ -855,6 +855,14 @@ def compile_attribute(
855
855
  DataJointError
856
856
  If syntax is invalid, primary key is nullable, or blob has invalid default.
857
857
  """
858
+ if line.lstrip().startswith("_"):
859
+ raise DataJointError(
860
+ f'Attribute name in line "{line}" starts with an underscore. '
861
+ "Names with leading underscore are reserved for platform-managed "
862
+ "columns (e.g. _job_start_time, _singleton). Use a regular "
863
+ "attribute name; if you need to control visibility at the call "
864
+ "site, use proj()."
865
+ )
858
866
  try:
859
867
  match = attribute_parser.parse_string(line + "#", parse_all=True)
860
868
  except pp.ParseException as err:
@@ -229,11 +229,14 @@ def scan_hash_references(
229
229
  if verbose:
230
230
  logger.info(f" Scanning {table_name}.{attr_name}")
231
231
 
232
- # Fetch all values for this attribute
232
+ # Read raw JSON metadata via cursor — bypasses decode_attribute
233
+ # so we get the stored dict (PostgreSQL/JSONB) or JSON string
234
+ # (MySQL), not the decoded codec output. _extract_hash_refs
235
+ # handles both shapes.
233
236
  try:
234
- values = table.to_arrays(attr_name)
235
- for value in values:
236
- for path, ref_store in _extract_hash_refs(value):
237
+ cursor = table.proj(attr_name).cursor(as_dict=True)
238
+ for row in cursor:
239
+ for path, ref_store in _extract_hash_refs(row[attr_name]):
237
240
  # Filter by store if specified
238
241
  if store_name is None or ref_store == store_name:
239
242
  referenced.add(path)
@@ -291,11 +294,14 @@ def scan_schema_references(
291
294
  if verbose:
292
295
  logger.info(f" Scanning {table_name}.{attr_name}")
293
296
 
294
- # Fetch all values for this attribute
297
+ # Read raw JSON metadata via cursor — bypasses decode_attribute
298
+ # so we get the stored dict (PostgreSQL/JSONB) or JSON string
299
+ # (MySQL), not the decoded codec output. _extract_schema_refs
300
+ # handles both shapes.
295
301
  try:
296
- values = table.to_arrays(attr_name)
297
- for value in values:
298
- for path, ref_store in _extract_schema_refs(value):
302
+ cursor = table.proj(attr_name).cursor(as_dict=True)
303
+ for row in cursor:
304
+ for path, ref_store in _extract_schema_refs(row[attr_name]):
299
305
  # Filter by store if specified
300
306
  if store_name is None or ref_store == store_name:
301
307
  referenced.add(path)
@@ -508,8 +508,16 @@ class Heading:
508
508
  if category == "UUID":
509
509
  attr["uuid"] = True
510
510
  elif category in CORE_TYPE_NAMES:
511
- # Core type alias - already resolved in DB
512
- pass
511
+ # Core type alias - already resolved in DB.
512
+ # MariaDB-specific recovery: MariaDB stores `json` columns
513
+ # as `longtext` and reports them back that way through
514
+ # information_schema, so the DB-type-based detection above
515
+ # leaves attr["json"] False. The :json: comment marker
516
+ # survives this aliasing, so we recover the json flag here
517
+ # from the original declared type. No-op on MySQL/PostgreSQL
518
+ # (attr["json"] is already True from the regex match above).
519
+ if category == "JSON":
520
+ attr["json"] = True
513
521
 
514
522
  # Check primary key constraints
515
523
  if attr["in_key"] and (attr["is_blob"] or attr["json"]):
@@ -555,14 +563,9 @@ class Heading:
555
563
  adapter.get_indexes_sql(database, table_name),
556
564
  as_dict=True,
557
565
  ):
558
- # Note: adapter.get_indexes_sql() already filters out PRIMARY key
559
- # MySQL/PostgreSQL adapters return: index_name, column_name, non_unique
560
566
  index_name = item.get("index_name") or item.get("Key_name")
561
567
  seq = item.get("seq_in_index") or item.get("Seq_in_index") or len(keys[index_name]) + 1
562
568
  column = item.get("column_name") or item.get("Column_name")
563
- # MySQL EXPRESSION column stores escaped single quotes - unescape them
564
- if column:
565
- column = column.replace("\\'", "'")
566
569
  non_unique = item.get("non_unique") or item.get("Non_unique")
567
570
  nullable = item.get("nullable") or (item.get("Null", "NO").lower() == "yes")
568
571
 
@@ -142,14 +142,14 @@ def analyze_columns(schema: Schema) -> dict:
142
142
  for (table_name,) in tables:
143
143
  # Get all columns for this table
144
144
  columns_query = """
145
- SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT
145
+ SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT, IS_NULLABLE
146
146
  FROM information_schema.COLUMNS
147
147
  WHERE TABLE_SCHEMA = %s
148
148
  AND TABLE_NAME = %s
149
149
  """
150
150
  columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
151
151
 
152
- for column_name, column_type, data_type, comment in columns:
152
+ for column_name, column_type, data_type, comment, is_nullable in columns:
153
153
  comment = comment or ""
154
154
 
155
155
  # Check if column already has a type label (starts with :type:)
@@ -167,6 +167,7 @@ def analyze_columns(schema: Schema) -> dict:
167
167
  "column": column_name,
168
168
  "native_type": column_type,
169
169
  "comment": comment,
170
+ "is_nullable": is_nullable == "YES",
170
171
  }
171
172
 
172
173
  if is_external:
@@ -270,9 +271,10 @@ def migrate_columns(
270
271
  new_comment_escaped = new_comment.replace("\\", "\\\\").replace("'", "\\'")
271
272
 
272
273
  # Generate ALTER TABLE statement
274
+ not_null = "" if col["is_nullable"] else " NOT NULL"
273
275
  sql = (
274
276
  f"ALTER TABLE `{db_name}`.`{table_name}` "
275
- f"MODIFY COLUMN `{col['column']}` {col['native_type']} "
277
+ f"MODIFY COLUMN `{col['column']}` {col['native_type']}{not_null} "
276
278
  f"COMMENT '{new_comment_escaped}'"
277
279
  )
278
280
  result["sql_statements"].append(sql)
@@ -365,7 +367,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
365
367
  for (table_name,) in tables:
366
368
  # Get column information for each table
367
369
  columns_query = """
368
- SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT
370
+ SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT, IS_NULLABLE
369
371
  FROM information_schema.COLUMNS
370
372
  WHERE TABLE_SCHEMA = %s
371
373
  AND TABLE_NAME = %s
@@ -374,7 +376,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
374
376
 
375
377
  columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
376
378
 
377
- for column_name, column_type, comment in columns:
379
+ for column_name, column_type, comment, is_nullable in columns:
378
380
  # Check if comment already has a codec type (starts with :type:)
379
381
  has_codec = comment and comment.startswith(":")
380
382
 
@@ -385,6 +387,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
385
387
  "column_type": column_type,
386
388
  "current_comment": comment or "",
387
389
  "needs_migration": not has_codec,
390
+ "is_nullable": is_nullable == "YES",
388
391
  }
389
392
  )
390
393
 
@@ -447,9 +450,10 @@ def generate_migration_sql(
447
450
  db_name, table_name = col["table_name"].split(".")
448
451
 
449
452
  # Generate ALTER TABLE statement
453
+ not_null = "" if col.get("is_nullable", True) else " NOT NULL"
450
454
  sql = (
451
455
  f"ALTER TABLE `{db_name}`.`{table_name}` "
452
- f"MODIFY COLUMN `{col['column_name']}` {col['column_type']} "
456
+ f"MODIFY COLUMN `{col['column_name']}` {col['column_type']}{not_null} "
453
457
  f"COMMENT '{new_comment_escaped}'"
454
458
  )
455
459
  sql_statements.append(sql)
@@ -172,6 +172,13 @@ class _Schema:
172
172
  self.connection = connection
173
173
  if self.connection is None:
174
174
  self.connection = _get_singleton_connection()
175
+ if self.connection._config.get("database.database_prefix"):
176
+ warnings.warn(
177
+ "database_prefix is deprecated and will be removed in DataJoint 2.3. "
178
+ "Use database.name to select a PostgreSQL database instead.",
179
+ DeprecationWarning,
180
+ stacklevel=2,
181
+ )
175
182
  self.database = schema_name
176
183
  if create_schema is not None:
177
184
  self.create_schema = create_schema
@@ -65,6 +65,7 @@ ENV_VAR_MAPPING = {
65
65
  "database.password": "DJ_PASS",
66
66
  "database.backend": "DJ_BACKEND",
67
67
  "database.port": "DJ_PORT",
68
+ "database.name": "DJ_DATABASE_NAME",
68
69
  "database.database_prefix": "DJ_DATABASE_PREFIX",
69
70
  "database.create_tables": "DJ_CREATE_TABLES",
70
71
  "loglevel": "DJ_LOG_LEVEL",
@@ -196,13 +197,17 @@ class DatabaseSettings(BaseSettings):
196
197
  description="Database backend: 'mysql' or 'postgresql'",
197
198
  )
198
199
  port: int | None = Field(default=None, validation_alias="DJ_PORT")
200
+ name: str | None = Field(
201
+ default=None,
202
+ validation_alias="DJ_DATABASE_NAME",
203
+ description="Database name for PostgreSQL connections. Defaults to 'postgres' if not set.",
204
+ )
199
205
  reconnect: bool = True
200
206
  use_tls: bool | None = Field(default=None, validation_alias="DJ_USE_TLS")
201
207
  database_prefix: str = Field(
202
208
  default="",
203
209
  validation_alias="DJ_DATABASE_PREFIX",
204
- description="Prefix for database/schema names. "
205
- "Not automatically applied; use dj.config.database.database_prefix when creating schemas.",
210
+ description="Deprecated. Use database.name instead.",
206
211
  )
207
212
  create_tables: bool = Field(
208
213
  default=True,
@@ -415,24 +420,29 @@ class Config(BaseSettings):
415
420
 
416
421
  spec = dict(self.stores[store])
417
422
 
418
- # Set defaults for optional fields (common to all protocols)
419
- spec.setdefault("subfolding", None) # No subfolding by default
420
- spec.setdefault("partition_pattern", None) # No partitioning by default
421
- spec.setdefault("token_length", 8) # Default token length
422
-
423
- # Set defaults for storage section prefixes
424
- spec.setdefault("hash_prefix", "_hash") # Hash-addressed storage section
425
- spec.setdefault("schema_prefix", "_schema") # Schema-addressed storage section
426
- spec.setdefault("filepath_prefix", None) # Filepath storage (unrestricted by default)
423
+ self._apply_common_store_defaults(spec)
427
424
 
428
425
  # Validate protocol
429
426
  protocol = spec.get("protocol", "").lower()
430
427
  supported_protocols = ("file", "s3", "gcs", "azure")
431
428
  if protocol not in supported_protocols:
432
- raise DataJointError(
433
- f'Missing or invalid protocol in config.stores["{store}"]. '
434
- f"Supported protocols: {', '.join(supported_protocols)}"
429
+ from .storage_adapter import get_storage_adapter
430
+
431
+ adapter = get_storage_adapter(protocol)
432
+ if adapter is None:
433
+ raise DataJointError(
434
+ f'Unknown protocol "{protocol}" in config.stores["{store}"]. '
435
+ f"Built-in: {', '.join(supported_protocols)}. "
436
+ f"Install a plugin package for additional protocols."
437
+ )
438
+ adapter.validate_spec(spec)
439
+ self._validate_prefix_separation(
440
+ store_name=store,
441
+ hash_prefix=spec.get("hash_prefix"),
442
+ schema_prefix=spec.get("schema_prefix"),
443
+ filepath_prefix=spec.get("filepath_prefix"),
435
444
  )
445
+ return spec
436
446
 
437
447
  # Set protocol-specific defaults
438
448
  if protocol == "s3":
@@ -577,6 +587,16 @@ class Config(BaseSettings):
577
587
  f"Storage section prefixes must be mutually exclusive."
578
588
  )
579
589
 
590
+ @staticmethod
591
+ def _apply_common_store_defaults(spec: dict[str, Any]) -> None:
592
+ """Apply defaults shared by every store protocol (built-in and plugin)."""
593
+ spec.setdefault("subfolding", None)
594
+ spec.setdefault("partition_pattern", None)
595
+ spec.setdefault("token_length", 8)
596
+ spec.setdefault("hash_prefix", "_hash")
597
+ spec.setdefault("schema_prefix", "_schema")
598
+ spec.setdefault("filepath_prefix", None)
599
+
580
600
  def load(self, filename: str | Path) -> None:
581
601
  """
582
602
  Load settings from a JSON file.
@@ -330,6 +330,15 @@ class StorageBackend:
330
330
  self._fs = self._create_filesystem()
331
331
  return self._fs
332
332
 
333
+ def _require_adapter(self):
334
+ """Look up a registered storage adapter, raising if none is registered."""
335
+ from .storage_adapter import get_storage_adapter
336
+
337
+ adapter = get_storage_adapter(self.protocol)
338
+ if adapter is None:
339
+ raise errors.DataJointError(f"Unsupported storage protocol: {self.protocol}")
340
+ return adapter
341
+
333
342
  def _create_filesystem(self) -> fsspec.AbstractFileSystem:
334
343
  """Create fsspec filesystem based on protocol."""
335
344
  if self.protocol == "file":
@@ -368,7 +377,7 @@ class StorageBackend:
368
377
  )
369
378
 
370
379
  else:
371
- raise errors.DataJointError(f"Unsupported storage protocol: {self.protocol}")
380
+ return self._require_adapter().create_filesystem(self.spec)
372
381
 
373
382
  def _full_path(self, path: str | PurePosixPath) -> str:
374
383
  """
@@ -397,12 +406,13 @@ class StorageBackend:
397
406
  if location:
398
407
  return f"{bucket}/{location}/{path}"
399
408
  return f"{bucket}/{path}"
400
- else:
401
- # Local filesystem - prepend location if specified
409
+ elif self.protocol == "file":
402
410
  location = self.spec.get("location", "")
403
411
  if location:
404
412
  return str(Path(location) / path)
405
413
  return path
414
+ else:
415
+ return self._require_adapter().full_path(self.spec, path)
406
416
 
407
417
  def get_url(self, path: str | PurePosixPath) -> str:
408
418
  """
@@ -448,8 +458,7 @@ class StorageBackend:
448
458
  elif self.protocol == "azure":
449
459
  return f"az://{full_path}"
450
460
  else:
451
- # Fallback: use protocol prefix
452
- return f"{self.protocol}://{full_path}"
461
+ return self._require_adapter().get_url(self.spec, full_path)
453
462
 
454
463
  def put_file(self, local_path: str | Path, remote_path: str | PurePosixPath, metadata: dict | None = None) -> None:
455
464
  """