datajoint 2.2.1__tar.gz → 2.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. {datajoint-2.2.1 → datajoint-2.2.2}/CONTRIBUTING.md +1 -1
  2. {datajoint-2.2.1 → datajoint-2.2.2}/PKG-INFO +1 -1
  3. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/__init__.py +7 -0
  4. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/adapters/mysql.py +3 -7
  5. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/connection.py +14 -0
  6. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/declare.py +8 -0
  7. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/gc.py +14 -8
  8. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/heading.py +10 -7
  9. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/migrate.py +10 -6
  10. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/settings.py +27 -12
  11. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/storage.py +14 -5
  12. datajoint-2.2.2/src/datajoint/storage_adapter.py +109 -0
  13. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/version.py +1 -1
  14. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_gc.py +118 -0
  15. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_json.py +4 -0
  16. datajoint-2.2.2/tests/unit/test_connection_warning.py +42 -0
  17. datajoint-2.2.2/tests/unit/test_declare_hidden_attribute.py +51 -0
  18. datajoint-2.2.2/tests/unit/test_storage_adapter.py +313 -0
  19. {datajoint-2.2.1 → datajoint-2.2.2}/.coveragerc +0 -0
  20. {datajoint-2.2.1 → datajoint-2.2.2}/.devcontainer/Dockerfile +0 -0
  21. {datajoint-2.2.1 → datajoint-2.2.2}/.devcontainer/devcontainer.json +0 -0
  22. {datajoint-2.2.1 → datajoint-2.2.2}/.devcontainer/docker-compose.yml +0 -0
  23. {datajoint-2.2.1 → datajoint-2.2.2}/.dockerignore +0 -0
  24. {datajoint-2.2.1 → datajoint-2.2.2}/.gitattributes +0 -0
  25. {datajoint-2.2.1 → datajoint-2.2.2}/.github/DISCUSSION_TEMPLATE/rfc.yml +0 -0
  26. {datajoint-2.2.1 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  27. {datajoint-2.2.1 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  28. {datajoint-2.2.1 → datajoint-2.2.2}/.github/ISSUE_TEMPLATE/improvement_request.md +0 -0
  29. {datajoint-2.2.1 → datajoint-2.2.2}/.github/dependabot.yml +0 -0
  30. {datajoint-2.2.1 → datajoint-2.2.2}/.github/release_drafter.yaml +0 -0
  31. {datajoint-2.2.1 → datajoint-2.2.2}/.github/workflows/close_inactive_issues_prs.yaml +0 -0
  32. {datajoint-2.2.1 → datajoint-2.2.2}/.github/workflows/draft_release.yaml +0 -0
  33. {datajoint-2.2.1 → datajoint-2.2.2}/.github/workflows/lint.yaml +0 -0
  34. {datajoint-2.2.1 → datajoint-2.2.2}/.github/workflows/post_draft_release_published.yaml +0 -0
  35. {datajoint-2.2.1 → datajoint-2.2.2}/.github/workflows/test.yaml +0 -0
  36. {datajoint-2.2.1 → datajoint-2.2.2}/.gitignore +0 -0
  37. {datajoint-2.2.1 → datajoint-2.2.2}/.pre-commit-config.yaml +0 -0
  38. {datajoint-2.2.1 → datajoint-2.2.2}/.vscode/launch.json +0 -0
  39. {datajoint-2.2.1 → datajoint-2.2.2}/CHANGELOG-archive.md +0 -0
  40. {datajoint-2.2.1 → datajoint-2.2.2}/Dockerfile +0 -0
  41. {datajoint-2.2.1 → datajoint-2.2.2}/LICENSE +0 -0
  42. {datajoint-2.2.1 → datajoint-2.2.2}/README.md +0 -0
  43. {datajoint-2.2.1 → datajoint-2.2.2}/RELEASE_MEMO.md +0 -0
  44. {datajoint-2.2.1 → datajoint-2.2.2}/activate.sh +0 -0
  45. {datajoint-2.2.1 → datajoint-2.2.2}/docker-compose.yaml +0 -0
  46. {datajoint-2.2.1 → datajoint-2.2.2}/images/pipeline.drawio +0 -0
  47. {datajoint-2.2.1 → datajoint-2.2.2}/images/pipeline.png +0 -0
  48. {datajoint-2.2.1 → datajoint-2.2.2}/pixi.lock +0 -0
  49. {datajoint-2.2.1 → datajoint-2.2.2}/pyproject.toml +0 -0
  50. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/adapters/__init__.py +0 -0
  51. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/adapters/base.py +0 -0
  52. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/adapters/postgres.py +0 -0
  53. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/autopopulate.py +0 -0
  54. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/blob.py +0 -0
  55. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/__init__.py +0 -0
  56. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/attach.py +0 -0
  57. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/blob.py +0 -0
  58. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/filepath.py +0 -0
  59. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/hash.py +0 -0
  60. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/npy.py +0 -0
  61. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/object.py +0 -0
  62. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/builtin_codecs/schema.py +0 -0
  63. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/cli.py +0 -0
  64. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/codecs.py +0 -0
  65. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/condition.py +0 -0
  66. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/dependencies.py +0 -0
  67. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/diagram.py +0 -0
  68. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/errors.py +0 -0
  69. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/expression.py +0 -0
  70. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/hash_registry.py +0 -0
  71. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/instance.py +0 -0
  72. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/jobs.py +0 -0
  73. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/lineage.py +0 -0
  74. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/logging.py +0 -0
  75. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/objectref.py +0 -0
  76. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/preview.py +0 -0
  77. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/schemas.py +0 -0
  78. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/staged_insert.py +0 -0
  79. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/table.py +0 -0
  80. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/types.py +0 -0
  81. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/user_tables.py +0 -0
  82. {datajoint-2.2.1 → datajoint-2.2.2}/src/datajoint/utils.py +0 -0
  83. {datajoint-2.2.1 → datajoint-2.2.2}/tests/__init__.py +0 -0
  84. {datajoint-2.2.1 → datajoint-2.2.2}/tests/conftest.py +0 -0
  85. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/__init__.py +0 -0
  86. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/Course.csv +0 -0
  87. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/CurrentTerm.csv +0 -0
  88. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/Department.csv +0 -0
  89. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/Enroll.csv +0 -0
  90. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/Grade.csv +0 -0
  91. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/Section.csv +0 -0
  92. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/Student.csv +0 -0
  93. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/StudentMajor.csv +0 -0
  94. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/data/Term.csv +0 -0
  95. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_aggr_regressions.py +0 -0
  96. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_alter.py +0 -0
  97. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_attach.py +0 -0
  98. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_autopopulate.py +0 -0
  99. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_blob.py +0 -0
  100. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_blob_matlab.py +0 -0
  101. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_cascade_delete.py +0 -0
  102. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_cascading_delete.py +0 -0
  103. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_cli.py +0 -0
  104. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_codec_chaining.py +0 -0
  105. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_codecs.py +0 -0
  106. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_connection.py +0 -0
  107. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_declare.py +0 -0
  108. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_dependencies.py +0 -0
  109. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_erd.py +0 -0
  110. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_fetch.py +0 -0
  111. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_fetch_same.py +0 -0
  112. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_foreign_keys.py +0 -0
  113. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_groupby.py +0 -0
  114. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_hash_storage.py +0 -0
  115. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_hidden_job_metadata.py +0 -0
  116. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_insert.py +0 -0
  117. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_jobs.py +0 -0
  118. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_multi_backend.py +0 -0
  119. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_nan.py +0 -0
  120. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_npy_codec.py +0 -0
  121. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_object.py +0 -0
  122. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_privileges.py +0 -0
  123. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_reconnection.py +0 -0
  124. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_relation.py +0 -0
  125. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_relation_u.py +0 -0
  126. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_relational_operand.py +0 -0
  127. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_schema.py +0 -0
  128. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_schema_keywords.py +0 -0
  129. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_semantic_matching.py +0 -0
  130. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_skip_duplicates.py +0 -0
  131. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_tls.py +0 -0
  132. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_type_aliases.py +0 -0
  133. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_university.py +0 -0
  134. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_update1.py +0 -0
  135. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_utils.py +0 -0
  136. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_uuid.py +0 -0
  137. {datajoint-2.2.1 → datajoint-2.2.2}/tests/integration/test_virtual_module.py +0 -0
  138. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema.py +0 -0
  139. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_advanced.py +0 -0
  140. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_aggr_regress.py +0 -0
  141. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_alter.py +0 -0
  142. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_codecs.py +0 -0
  143. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_external.py +0 -0
  144. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_object.py +0 -0
  145. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_privileges.py +0 -0
  146. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_simple.py +0 -0
  147. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_type_aliases.py +0 -0
  148. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_university.py +0 -0
  149. {datajoint-2.2.1 → datajoint-2.2.2}/tests/schema_uuid.py +0 -0
  150. {datajoint-2.2.1 → datajoint-2.2.2}/tests/test_package.py +0 -0
  151. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/__init__.py +0 -0
  152. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_adapters.py +0 -0
  153. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_codecs.py +0 -0
  154. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_condition.py +0 -0
  155. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_fetch_compat.py +0 -0
  156. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_lazy_imports.py +0 -0
  157. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_pk_rules.py +0 -0
  158. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_settings.py +0 -0
  159. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_storage_urls.py +0 -0
  160. {datajoint-2.2.1 → datajoint-2.2.2}/tests/unit/test_thread_safe.py +0 -0
@@ -56,7 +56,7 @@ export DOCKER_HOST=unix://$HOME/.docker/run/docker.sock
56
56
 
57
57
  ### PostgreSQL Backend
58
58
 
59
- DataJoint supports PostgreSQL 15+ as an alternative to MySQL 8+. To install the PostgreSQL driver:
59
+ DataJoint supports MySQL 8.0.13+ and PostgreSQL 15+ as production database backends. To install the PostgreSQL driver:
60
60
 
61
61
  ```bash
62
62
  pip install -e ".[postgres]" # Installs psycopg2-binary
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datajoint
3
- Version: 2.2.1
3
+ Version: 2.2.2
4
4
  Summary: DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
5
5
  Project-URL: Homepage, https://docs.datajoint.com/
6
6
  Project-URL: Documentation, https://docs.datajoint.com/
@@ -30,6 +30,8 @@ __all__ = [
30
30
  "list_schemas",
31
31
  "Table",
32
32
  "FreeTable",
33
+ "AutoPopulate",
34
+ "Job",
33
35
  "Manual",
34
36
  "Lookup",
35
37
  "Imported",
@@ -81,6 +83,8 @@ from .instance import Instance, _ConfigProxy, _get_singleton_connection, _global
81
83
  from .logging import logger
82
84
  from .objectref import ObjectRef
83
85
  from .schemas import _Schema, VirtualModule, list_schemas, virtual_schema
86
+ from .autopopulate import AutoPopulate
87
+ from .jobs import Job
84
88
  from .table import FreeTable as _FreeTable, Table, ValidationResult
85
89
  from .user_tables import Computed, Imported, Lookup, Manual, Part
86
90
  from .version import __version__
@@ -271,6 +275,9 @@ _lazy_modules = {
271
275
  "diagram": (".diagram", None), # Return the module itself
272
276
  # cli imports click
273
277
  "cli": (".cli", "cli"),
278
+ # gc — exposed lazily so `dj.gc.scan(...)` works as documented in gc.py
279
+ # and in the user docs (how-to/garbage-collection.md).
280
+ "gc": (".gc", None), # Return the module itself
274
281
  }
275
282
 
276
283
 
@@ -423,7 +423,7 @@ class MySQLAdapter(DatabaseAdapter):
423
423
  fk_cols = ", ".join(self.quote_identifier(col) for col in fk["columns"])
424
424
  ref_cols = ", ".join(self.quote_identifier(col) for col in fk["ref_columns"])
425
425
  lines.append(
426
- f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols}) " f"ON UPDATE CASCADE ON DELETE RESTRICT"
426
+ f"FOREIGN KEY ({fk_cols}) REFERENCES {fk['ref_table']} ({ref_cols}) ON UPDATE CASCADE ON DELETE RESTRICT"
427
427
  )
428
428
 
429
429
  # Indexes
@@ -735,14 +735,10 @@ class MySQLAdapter(DatabaseAdapter):
735
735
  return result
736
736
 
737
737
  def get_indexes_sql(self, schema_name: str, table_name: str) -> str:
738
- """Query to get index definitions.
739
-
740
- Note: For MySQL 8.0+, EXPRESSION column contains the expression for
741
- functional indexes. COLUMN_NAME is NULL for such indexes.
742
- """
738
+ """Query to get index definitions. Functional indexes (NULL COLUMN_NAME) are skipped downstream."""
743
739
  return (
744
740
  f"SELECT INDEX_NAME as index_name, "
745
- f"COALESCE(COLUMN_NAME, CONCAT('(', EXPRESSION, ')')) as column_name, "
741
+ f"COLUMN_NAME as column_name, "
746
742
  f"NON_UNIQUE as non_unique, SEQ_IN_INDEX as seq_in_index "
747
743
  f"FROM information_schema.statistics "
748
744
  f"WHERE table_schema = {self.quote_string(schema_name)} "
@@ -108,6 +108,18 @@ def conn(
108
108
  return conn.connection
109
109
 
110
110
 
111
+ def _warn_if_mariadb(version_str: str) -> None:
112
+ """Emit a UserWarning if `version_str` looks like MariaDB. No-op for MySQL."""
113
+ if "MariaDB" in version_str:
114
+ warnings.warn(
115
+ f"MariaDB is not officially supported by DataJoint 2.x "
116
+ f"(server reports {version_str}). Compatibility is best-effort "
117
+ f"and may break in future releases.",
118
+ UserWarning,
119
+ stacklevel=3,
120
+ )
121
+
122
+
111
123
  class EmulatedCursor:
112
124
  """acts like a cursor"""
113
125
 
@@ -221,6 +233,8 @@ class Connection:
221
233
  f"{self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}{db_str}"
222
234
  )
223
235
  self.connection_id = self.adapter.get_connection_id(self._conn)
236
+ if self.adapter.backend == "mysql":
237
+ _warn_if_mariadb(self.query("SELECT @@version").fetchone()[0])
224
238
  else:
225
239
  raise errors.LostConnectionError(
226
240
  f"Connection failed {self.conn_info['user']}@{self.conn_info['host']}:{self.conn_info['port']}"
@@ -855,6 +855,14 @@ def compile_attribute(
855
855
  DataJointError
856
856
  If syntax is invalid, primary key is nullable, or blob has invalid default.
857
857
  """
858
+ if line.lstrip().startswith("_"):
859
+ raise DataJointError(
860
+ f'Attribute name in line "{line}" starts with an underscore. '
861
+ "Names with leading underscore are reserved for platform-managed "
862
+ "columns (e.g. _job_start_time, _singleton). Use a regular "
863
+ "attribute name; if you need to control visibility at the call "
864
+ "site, use proj()."
865
+ )
858
866
  try:
859
867
  match = attribute_parser.parse_string(line + "#", parse_all=True)
860
868
  except pp.ParseException as err:
@@ -229,11 +229,14 @@ def scan_hash_references(
229
229
  if verbose:
230
230
  logger.info(f" Scanning {table_name}.{attr_name}")
231
231
 
232
- # Fetch all values for this attribute
232
+ # Read raw JSON metadata via cursor — bypasses decode_attribute
233
+ # so we get the stored dict (PostgreSQL/JSONB) or JSON string
234
+ # (MySQL), not the decoded codec output. _extract_hash_refs
235
+ # handles both shapes.
233
236
  try:
234
- values = table.to_arrays(attr_name)
235
- for value in values:
236
- for path, ref_store in _extract_hash_refs(value):
237
+ cursor = table.proj(attr_name).cursor(as_dict=True)
238
+ for row in cursor:
239
+ for path, ref_store in _extract_hash_refs(row[attr_name]):
237
240
  # Filter by store if specified
238
241
  if store_name is None or ref_store == store_name:
239
242
  referenced.add(path)
@@ -291,11 +294,14 @@ def scan_schema_references(
291
294
  if verbose:
292
295
  logger.info(f" Scanning {table_name}.{attr_name}")
293
296
 
294
- # Fetch all values for this attribute
297
+ # Read raw JSON metadata via cursor — bypasses decode_attribute
298
+ # so we get the stored dict (PostgreSQL/JSONB) or JSON string
299
+ # (MySQL), not the decoded codec output. _extract_schema_refs
300
+ # handles both shapes.
295
301
  try:
296
- values = table.to_arrays(attr_name)
297
- for value in values:
298
- for path, ref_store in _extract_schema_refs(value):
302
+ cursor = table.proj(attr_name).cursor(as_dict=True)
303
+ for row in cursor:
304
+ for path, ref_store in _extract_schema_refs(row[attr_name]):
299
305
  # Filter by store if specified
300
306
  if store_name is None or ref_store == store_name:
301
307
  referenced.add(path)
@@ -508,8 +508,16 @@ class Heading:
508
508
  if category == "UUID":
509
509
  attr["uuid"] = True
510
510
  elif category in CORE_TYPE_NAMES:
511
- # Core type alias - already resolved in DB
512
- pass
511
+ # Core type alias - already resolved in DB.
512
+ # MariaDB-specific recovery: MariaDB stores `json` columns
513
+ # as `longtext` and reports them back that way through
514
+ # information_schema, so the DB-type-based detection above
515
+ # leaves attr["json"] False. The :json: comment marker
516
+ # survives this aliasing, so we recover the json flag here
517
+ # from the original declared type. No-op on MySQL/PostgreSQL
518
+ # (attr["json"] is already True from the regex match above).
519
+ if category == "JSON":
520
+ attr["json"] = True
513
521
 
514
522
  # Check primary key constraints
515
523
  if attr["in_key"] and (attr["is_blob"] or attr["json"]):
@@ -555,14 +563,9 @@ class Heading:
555
563
  adapter.get_indexes_sql(database, table_name),
556
564
  as_dict=True,
557
565
  ):
558
- # Note: adapter.get_indexes_sql() already filters out PRIMARY key
559
- # MySQL/PostgreSQL adapters return: index_name, column_name, non_unique
560
566
  index_name = item.get("index_name") or item.get("Key_name")
561
567
  seq = item.get("seq_in_index") or item.get("Seq_in_index") or len(keys[index_name]) + 1
562
568
  column = item.get("column_name") or item.get("Column_name")
563
- # MySQL EXPRESSION column stores escaped single quotes - unescape them
564
- if column:
565
- column = column.replace("\\'", "'")
566
569
  non_unique = item.get("non_unique") or item.get("Non_unique")
567
570
  nullable = item.get("nullable") or (item.get("Null", "NO").lower() == "yes")
568
571
 
@@ -142,14 +142,14 @@ def analyze_columns(schema: Schema) -> dict:
142
142
  for (table_name,) in tables:
143
143
  # Get all columns for this table
144
144
  columns_query = """
145
- SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT
145
+ SELECT COLUMN_NAME, COLUMN_TYPE, DATA_TYPE, COLUMN_COMMENT, IS_NULLABLE
146
146
  FROM information_schema.COLUMNS
147
147
  WHERE TABLE_SCHEMA = %s
148
148
  AND TABLE_NAME = %s
149
149
  """
150
150
  columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
151
151
 
152
- for column_name, column_type, data_type, comment in columns:
152
+ for column_name, column_type, data_type, comment, is_nullable in columns:
153
153
  comment = comment or ""
154
154
 
155
155
  # Check if column already has a type label (starts with :type:)
@@ -167,6 +167,7 @@ def analyze_columns(schema: Schema) -> dict:
167
167
  "column": column_name,
168
168
  "native_type": column_type,
169
169
  "comment": comment,
170
+ "is_nullable": is_nullable == "YES",
170
171
  }
171
172
 
172
173
  if is_external:
@@ -270,9 +271,10 @@ def migrate_columns(
270
271
  new_comment_escaped = new_comment.replace("\\", "\\\\").replace("'", "\\'")
271
272
 
272
273
  # Generate ALTER TABLE statement
274
+ not_null = "" if col["is_nullable"] else " NOT NULL"
273
275
  sql = (
274
276
  f"ALTER TABLE `{db_name}`.`{table_name}` "
275
- f"MODIFY COLUMN `{col['column']}` {col['native_type']} "
277
+ f"MODIFY COLUMN `{col['column']}` {col['native_type']}{not_null} "
276
278
  f"COMMENT '{new_comment_escaped}'"
277
279
  )
278
280
  result["sql_statements"].append(sql)
@@ -365,7 +367,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
365
367
  for (table_name,) in tables:
366
368
  # Get column information for each table
367
369
  columns_query = """
368
- SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT
370
+ SELECT COLUMN_NAME, COLUMN_TYPE, COLUMN_COMMENT, IS_NULLABLE
369
371
  FROM information_schema.COLUMNS
370
372
  WHERE TABLE_SCHEMA = %s
371
373
  AND TABLE_NAME = %s
@@ -374,7 +376,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
374
376
 
375
377
  columns = connection.query(columns_query, args=(schema.database, table_name)).fetchall()
376
378
 
377
- for column_name, column_type, comment in columns:
379
+ for column_name, column_type, comment, is_nullable in columns:
378
380
  # Check if comment already has a codec type (starts with :type:)
379
381
  has_codec = comment and comment.startswith(":")
380
382
 
@@ -385,6 +387,7 @@ def analyze_blob_columns(schema: Schema) -> list[dict]:
385
387
  "column_type": column_type,
386
388
  "current_comment": comment or "",
387
389
  "needs_migration": not has_codec,
390
+ "is_nullable": is_nullable == "YES",
388
391
  }
389
392
  )
390
393
 
@@ -447,9 +450,10 @@ def generate_migration_sql(
447
450
  db_name, table_name = col["table_name"].split(".")
448
451
 
449
452
  # Generate ALTER TABLE statement
453
+ not_null = "" if col.get("is_nullable", True) else " NOT NULL"
450
454
  sql = (
451
455
  f"ALTER TABLE `{db_name}`.`{table_name}` "
452
- f"MODIFY COLUMN `{col['column_name']}` {col['column_type']} "
456
+ f"MODIFY COLUMN `{col['column_name']}` {col['column_type']}{not_null} "
453
457
  f"COMMENT '{new_comment_escaped}'"
454
458
  )
455
459
  sql_statements.append(sql)
@@ -420,24 +420,29 @@ class Config(BaseSettings):
420
420
 
421
421
  spec = dict(self.stores[store])
422
422
 
423
- # Set defaults for optional fields (common to all protocols)
424
- spec.setdefault("subfolding", None) # No subfolding by default
425
- spec.setdefault("partition_pattern", None) # No partitioning by default
426
- spec.setdefault("token_length", 8) # Default token length
427
-
428
- # Set defaults for storage section prefixes
429
- spec.setdefault("hash_prefix", "_hash") # Hash-addressed storage section
430
- spec.setdefault("schema_prefix", "_schema") # Schema-addressed storage section
431
- spec.setdefault("filepath_prefix", None) # Filepath storage (unrestricted by default)
423
+ self._apply_common_store_defaults(spec)
432
424
 
433
425
  # Validate protocol
434
426
  protocol = spec.get("protocol", "").lower()
435
427
  supported_protocols = ("file", "s3", "gcs", "azure")
436
428
  if protocol not in supported_protocols:
437
- raise DataJointError(
438
- f'Missing or invalid protocol in config.stores["{store}"]. '
439
- f"Supported protocols: {', '.join(supported_protocols)}"
429
+ from .storage_adapter import get_storage_adapter
430
+
431
+ adapter = get_storage_adapter(protocol)
432
+ if adapter is None:
433
+ raise DataJointError(
434
+ f'Unknown protocol "{protocol}" in config.stores["{store}"]. '
435
+ f"Built-in: {', '.join(supported_protocols)}. "
436
+ f"Install a plugin package for additional protocols."
437
+ )
438
+ adapter.validate_spec(spec)
439
+ self._validate_prefix_separation(
440
+ store_name=store,
441
+ hash_prefix=spec.get("hash_prefix"),
442
+ schema_prefix=spec.get("schema_prefix"),
443
+ filepath_prefix=spec.get("filepath_prefix"),
440
444
  )
445
+ return spec
441
446
 
442
447
  # Set protocol-specific defaults
443
448
  if protocol == "s3":
@@ -582,6 +587,16 @@ class Config(BaseSettings):
582
587
  f"Storage section prefixes must be mutually exclusive."
583
588
  )
584
589
 
590
+ @staticmethod
591
+ def _apply_common_store_defaults(spec: dict[str, Any]) -> None:
592
+ """Apply defaults shared by every store protocol (built-in and plugin)."""
593
+ spec.setdefault("subfolding", None)
594
+ spec.setdefault("partition_pattern", None)
595
+ spec.setdefault("token_length", 8)
596
+ spec.setdefault("hash_prefix", "_hash")
597
+ spec.setdefault("schema_prefix", "_schema")
598
+ spec.setdefault("filepath_prefix", None)
599
+
585
600
  def load(self, filename: str | Path) -> None:
586
601
  """
587
602
  Load settings from a JSON file.
@@ -330,6 +330,15 @@ class StorageBackend:
330
330
  self._fs = self._create_filesystem()
331
331
  return self._fs
332
332
 
333
+ def _require_adapter(self):
334
+ """Look up a registered storage adapter, raising if none is registered."""
335
+ from .storage_adapter import get_storage_adapter
336
+
337
+ adapter = get_storage_adapter(self.protocol)
338
+ if adapter is None:
339
+ raise errors.DataJointError(f"Unsupported storage protocol: {self.protocol}")
340
+ return adapter
341
+
333
342
  def _create_filesystem(self) -> fsspec.AbstractFileSystem:
334
343
  """Create fsspec filesystem based on protocol."""
335
344
  if self.protocol == "file":
@@ -368,7 +377,7 @@ class StorageBackend:
368
377
  )
369
378
 
370
379
  else:
371
- raise errors.DataJointError(f"Unsupported storage protocol: {self.protocol}")
380
+ return self._require_adapter().create_filesystem(self.spec)
372
381
 
373
382
  def _full_path(self, path: str | PurePosixPath) -> str:
374
383
  """
@@ -397,12 +406,13 @@ class StorageBackend:
397
406
  if location:
398
407
  return f"{bucket}/{location}/{path}"
399
408
  return f"{bucket}/{path}"
400
- else:
401
- # Local filesystem - prepend location if specified
409
+ elif self.protocol == "file":
402
410
  location = self.spec.get("location", "")
403
411
  if location:
404
412
  return str(Path(location) / path)
405
413
  return path
414
+ else:
415
+ return self._require_adapter().full_path(self.spec, path)
406
416
 
407
417
  def get_url(self, path: str | PurePosixPath) -> str:
408
418
  """
@@ -448,8 +458,7 @@ class StorageBackend:
448
458
  elif self.protocol == "azure":
449
459
  return f"az://{full_path}"
450
460
  else:
451
- # Fallback: use protocol prefix
452
- return f"{self.protocol}://{full_path}"
461
+ return self._require_adapter().get_url(self.spec, full_path)
453
462
 
454
463
  def put_file(self, local_path: str | Path, remote_path: str | PurePosixPath, metadata: dict | None = None) -> None:
455
464
  """
@@ -0,0 +1,109 @@
1
+ """Plugin system for third-party storage protocols.
2
+
3
+ Third-party packages register adapters via entry points::
4
+
5
+ [project.entry-points."datajoint.storage"]
6
+ myprotocol = "my_package:MyStorageAdapter"
7
+
8
+ The adapter is auto-discovered when DataJoint encounters the protocol name
9
+ in a store configuration. No explicit import is needed.
10
+ """
11
+
12
+ from abc import ABC, abstractmethod
13
+ from typing import Any
14
+ import logging
15
+
16
+ import fsspec
17
+
18
+ from . import errors
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+
23
+ class StorageAdapter(ABC):
24
+ """Base class for storage protocol adapters.
25
+
26
+ Subclass this and declare an entry point to add a new storage protocol
27
+ to DataJoint. At minimum, implement ``create_filesystem`` and set
28
+ ``protocol``, ``required_keys``, and ``allowed_keys``.
29
+ """
30
+
31
+ protocol: str
32
+ required_keys: tuple[str, ...] = ()
33
+ allowed_keys: tuple[str, ...] = ()
34
+
35
+ @abstractmethod
36
+ def create_filesystem(self, spec: dict[str, Any]) -> fsspec.AbstractFileSystem:
37
+ """Return an fsspec filesystem instance for this protocol."""
38
+ ...
39
+
40
+ def validate_spec(self, spec: dict[str, Any]) -> None:
41
+ """Validate protocol-specific config fields."""
42
+ missing = [k for k in self.required_keys if k not in spec]
43
+ if missing:
44
+ raise errors.DataJointError(f'{self.protocol} store is missing: {", ".join(missing)}')
45
+ all_allowed = set(self.allowed_keys) | _COMMON_STORE_KEYS
46
+ invalid = [k for k in spec if k not in all_allowed]
47
+ if invalid:
48
+ raise errors.DataJointError(f'Invalid key(s) for {self.protocol}: {", ".join(invalid)}')
49
+
50
+ def full_path(self, spec: dict[str, Any], relpath: str) -> str:
51
+ """Construct storage path from a relative path."""
52
+ location = spec.get("location", "")
53
+ return f"{location}/{relpath}" if location else relpath
54
+
55
+ def get_url(self, spec: dict[str, Any], path: str) -> str:
56
+ """Return a display URL for the stored object."""
57
+ return f"{self.protocol}://{path}"
58
+
59
+
60
+ _COMMON_STORE_KEYS = frozenset(
61
+ {
62
+ "protocol",
63
+ "location",
64
+ "subfolding",
65
+ "partition_pattern",
66
+ "token_length",
67
+ "hash_prefix",
68
+ "schema_prefix",
69
+ "filepath_prefix",
70
+ "stage",
71
+ }
72
+ )
73
+
74
+ _adapter_registry: dict[str, StorageAdapter] = {}
75
+ _adapters_loaded: bool = False
76
+
77
+
78
+ def get_storage_adapter(protocol: str) -> StorageAdapter | None:
79
+ """Look up a registered storage adapter by protocol name."""
80
+ global _adapters_loaded
81
+ if not _adapters_loaded:
82
+ _discover_adapters()
83
+ _adapters_loaded = True
84
+ return _adapter_registry.get(protocol)
85
+
86
+
87
+ def _discover_adapters() -> None:
88
+ """Load storage adapters from datajoint.storage entry points."""
89
+ try:
90
+ from importlib.metadata import entry_points
91
+ except ImportError:
92
+ logger.debug("importlib.metadata not available, skipping adapter discovery")
93
+ return
94
+
95
+ try:
96
+ eps = entry_points(group="datajoint.storage")
97
+ except TypeError:
98
+ eps = entry_points().get("datajoint.storage", [])
99
+
100
+ for ep in eps:
101
+ if ep.name in _adapter_registry:
102
+ continue
103
+ try:
104
+ adapter_cls = ep.load()
105
+ adapter = adapter_cls()
106
+ _adapter_registry[adapter.protocol] = adapter
107
+ logger.debug(f"Loaded storage adapter: {adapter.protocol}")
108
+ except Exception as e:
109
+ logger.warning(f"Failed to load storage adapter '{ep.name}': {e}")
@@ -1,4 +1,4 @@
1
1
  # version bump auto managed by Github Actions:
2
2
  # label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit)
3
3
  # manually set this version will be eventually overwritten by the above actions
4
- __version__ = "2.2.1"
4
+ __version__ = "2.2.2"
@@ -4,12 +4,43 @@ Tests for garbage collection (gc.py).
4
4
 
5
5
  from unittest.mock import MagicMock, patch
6
6
 
7
+ import numpy as np
7
8
  import pytest
8
9
 
10
+ import datajoint as dj
9
11
  from datajoint import gc
10
12
  from datajoint.errors import DataJointError
11
13
 
12
14
 
15
+ # Tables used by TestScanWithLiveData. Defined at module scope so dj.Schema's
16
+ # context resolution can find them by class name; bound to a schema inside
17
+ # each fixture (see schema(...) calls below).
18
+
19
+
20
+ class GcBlobTest(dj.Manual):
21
+ definition = """
22
+ rid : int
23
+ ---
24
+ payload : <blob@local>
25
+ """
26
+
27
+
28
+ class GcNpyTest(dj.Manual):
29
+ definition = """
30
+ rid : int
31
+ ---
32
+ waveform : <npy@local>
33
+ """
34
+
35
+
36
+ class GcObjectTest(dj.Manual):
37
+ definition = """
38
+ rid : int
39
+ ---
40
+ results : <object@local>
41
+ """
42
+
43
+
13
44
  class TestUsesHashStorage:
14
45
  """Tests for _uses_hash_storage helper function."""
15
46
 
@@ -347,3 +378,90 @@ class TestFormatStats:
347
378
  assert "Schema paths: 1" in result
348
379
  assert "2.00 MB" in result
349
380
  assert "Errors: 2" in result
381
+
382
+
383
+ class TestScanWithLiveData:
384
+ """End-to-end tests for gc.scan() against real schemas with external storage.
385
+
386
+ Exercises the full production path:
387
+ scan_*_references → table.proj(attr).cursor() → raw JSON metadata.
388
+
389
+ These are the regression tests that would have caught issue #1442
390
+ (silent type mismatch when scan helpers iterated decoded codec outputs
391
+ instead of raw stored metadata).
392
+ """
393
+
394
+ @pytest.fixture
395
+ def schema_blob(self, connection_test, prefix, mock_stores):
396
+ schema_name = f"{prefix}_test_gc_e2e_blob"
397
+ schema = dj.Schema(
398
+ schema_name,
399
+ context={"GcBlobTest": GcBlobTest},
400
+ connection=connection_test,
401
+ )
402
+ schema(GcBlobTest)
403
+ yield schema
404
+ schema.drop()
405
+
406
+ @pytest.fixture
407
+ def schema_npy(self, connection_test, prefix, mock_stores):
408
+ schema_name = f"{prefix}_test_gc_e2e_npy"
409
+ schema = dj.Schema(
410
+ schema_name,
411
+ context={"GcNpyTest": GcNpyTest},
412
+ connection=connection_test,
413
+ )
414
+ schema(GcNpyTest)
415
+ yield schema
416
+ schema.drop()
417
+
418
+ @pytest.fixture
419
+ def schema_object(self, connection_test, prefix, mock_stores):
420
+ schema_name = f"{prefix}_test_gc_e2e_object"
421
+ schema = dj.Schema(
422
+ schema_name,
423
+ context={"GcObjectTest": GcObjectTest},
424
+ connection=connection_test,
425
+ )
426
+ schema(GcObjectTest)
427
+ yield schema
428
+ schema.drop()
429
+
430
+ def test_scan_finds_active_blob_reference(self, schema_blob):
431
+ """scan() must report hash_referenced >= 1 for a populated <blob@> column.
432
+
433
+ Decoded value type returned by BlobCodec.decode is numpy.ndarray, which
434
+ does not satisfy `_extract_hash_refs`'s dict/JSON-string check — this
435
+ test fails before the cursor-based fix in scan_hash_references.
436
+ """
437
+ GcBlobTest.insert1({"rid": 1, "payload": np.arange(64, dtype="uint8")})
438
+
439
+ stats = gc.scan(schema_blob, store_name="local")
440
+
441
+ assert stats["hash_referenced"] >= 1, f"scan should find the active <blob@> reference; got {stats}"
442
+
443
+ def test_scan_finds_active_npy_reference(self, schema_npy):
444
+ """scan() must report schema_paths_referenced >= 1 for a populated <npy@> column.
445
+
446
+ Decoded value type returned by NpyCodec.decode is NpyRef (lazy handle),
447
+ which does not satisfy `_extract_schema_refs`'s dict check — this test
448
+ fails before the cursor-based fix in scan_schema_references.
449
+ """
450
+ GcNpyTest.insert1({"rid": 1, "waveform": np.arange(64, dtype="float32")})
451
+
452
+ stats = gc.scan(schema_npy, store_name="local")
453
+
454
+ assert stats["schema_paths_referenced"] >= 1, f"scan should find the active <npy@> reference; got {stats}"
455
+
456
+ def test_scan_finds_active_object_reference(self, schema_object):
457
+ """scan() must report schema_paths_referenced >= 1 for a populated <object@> column.
458
+
459
+ Decoded value type returned by ObjectCodec.decode is ObjectRef (lazy
460
+ handle), which does not satisfy `_extract_schema_refs`'s dict check —
461
+ this test fails before the cursor-based fix in scan_schema_references.
462
+ """
463
+ GcObjectTest.insert1({"rid": 1, "results": b"hello-gc-test"})
464
+
465
+ stats = gc.scan(schema_object, store_name="local")
466
+
467
+ assert stats["schema_paths_referenced"] >= 1, f"scan should find the active <object@> reference; got {stats}"
@@ -119,6 +119,10 @@ def test_insert_update(schema_json):
119
119
  assert not q
120
120
 
121
121
 
122
+ @pytest.mark.skip(
123
+ reason="Functional indexes are not currently round-tripped through Heading.indexes; "
124
+ "describe() drops them. Re-enable when functional-index introspection is restored."
125
+ )
122
126
  def test_describe(schema_json):
123
127
  rel = Team()
124
128
  context = inspect.currentframe().f_globals