spinta 0.2.dev20__py3-none-any.whl → 0.2.dev22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. spinta/auth.py +33 -26
  2. spinta/backends/helpers.py +234 -38
  3. spinta/backends/memory/commands/init.py +3 -3
  4. spinta/backends/memory/commands/manifest.py +1 -2
  5. spinta/backends/memory/commands/read.py +7 -7
  6. spinta/backends/postgresql/commands/bootstrap.py +16 -0
  7. spinta/backends/postgresql/commands/init.py +10 -11
  8. spinta/backends/postgresql/commands/manifest.py +5 -9
  9. spinta/backends/postgresql/commands/migrate/constants.py +1 -1
  10. spinta/backends/postgresql/commands/migrate/migrate.py +26 -118
  11. spinta/backends/postgresql/commands/migrate/model.py +233 -153
  12. spinta/backends/postgresql/commands/migrate/types/array.py +55 -29
  13. spinta/backends/postgresql/commands/migrate/types/datatype.py +45 -34
  14. spinta/backends/postgresql/commands/migrate/types/file.py +27 -18
  15. spinta/backends/postgresql/commands/migrate/types/ref.py +112 -88
  16. spinta/backends/postgresql/commands/migrate/types/string.py +20 -10
  17. spinta/backends/postgresql/commands/migrate/types/text.py +21 -9
  18. spinta/backends/postgresql/commands/redirect.py +5 -4
  19. spinta/backends/postgresql/commands/summary.py +22 -18
  20. spinta/backends/postgresql/commands/wipe.py +13 -10
  21. spinta/backends/postgresql/components.py +6 -4
  22. spinta/backends/postgresql/helpers/__init__.py +15 -0
  23. spinta/backends/postgresql/helpers/changes.py +5 -5
  24. spinta/backends/postgresql/helpers/migrate/actions.py +237 -121
  25. spinta/backends/postgresql/helpers/migrate/cast.py +67 -0
  26. spinta/backends/postgresql/helpers/migrate/migrate.py +367 -430
  27. spinta/backends/postgresql/helpers/migrate/name.py +219 -0
  28. spinta/backends/postgresql/helpers/name.py +18 -42
  29. spinta/backends/postgresql/helpers/redirect.py +5 -5
  30. spinta/backends/postgresql/types/array/init.py +9 -8
  31. spinta/backends/postgresql/types/file/init.py +7 -6
  32. spinta/backends/postgresql/types/ref/init.py +21 -6
  33. spinta/cli/admin.py +3 -0
  34. spinta/cli/helpers/admin/scripts/changelog.py +16 -16
  35. spinta/cli/helpers/admin/scripts/deduplicate.py +7 -7
  36. spinta/cli/helpers/script/components.py +13 -0
  37. spinta/cli/helpers/script/core.py +27 -6
  38. spinta/cli/helpers/upgrade/components.py +1 -0
  39. spinta/cli/helpers/upgrade/registry.py +15 -0
  40. spinta/cli/helpers/upgrade/scripts/backends/postgresql/comments.py +37 -28
  41. spinta/cli/helpers/upgrade/scripts/backends/postgresql/schemas.py +314 -0
  42. spinta/cli/helpers/upgrade/scripts/redirect.py +3 -3
  43. spinta/cli/main.py +0 -1
  44. spinta/cli/migrate.py +0 -16
  45. spinta/cli/upgrade.py +4 -0
  46. spinta/commands/__init__.py +0 -23
  47. spinta/components.py +2 -1
  48. spinta/config.py +3 -4
  49. spinta/datasets/backends/dataframe/backends/json/commands/read.py +1 -1
  50. spinta/datasets/backends/dataframe/commands/read.py +5 -0
  51. spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py +37 -7
  52. spinta/datasets/backends/helpers.py +25 -2
  53. spinta/datasets/backends/sql/ufuncs/query/ufuncs.py +0 -6
  54. spinta/datasets/backends/sql/ufuncs/result/ufuncs.py +0 -37
  55. spinta/exceptions.py +6 -0
  56. spinta/manifests/backend/commands/load.py +1 -3
  57. spinta/manifests/backend/commands/manifest.py +0 -13
  58. spinta/manifests/dict/commands/load.py +0 -2
  59. spinta/manifests/helpers.py +0 -12
  60. spinta/manifests/internal/commands/load.py +13 -31
  61. spinta/manifests/internal_sql/commands/load.py +0 -2
  62. spinta/manifests/memory/commands/load.py +0 -2
  63. spinta/manifests/open_api/commands/load.py +0 -2
  64. spinta/manifests/rdf/commands/load.py +0 -2
  65. spinta/manifests/sql/commands/load.py +0 -4
  66. spinta/manifests/sql/helpers.py +11 -3
  67. spinta/manifests/tabular/commands/load.py +0 -4
  68. spinta/manifests/tabular/constants.py +1 -0
  69. spinta/manifests/xsd/commands/load.py +0 -4
  70. spinta/manifests/yaml/commands/load.py +13 -34
  71. spinta/manifests/yaml/commands/manifest.py +0 -11
  72. spinta/manifests/yaml/helpers.py +1 -11
  73. spinta/nodes.py +0 -4
  74. spinta/testing/client.py +0 -2
  75. spinta/testing/migration.py +196 -75
  76. spinta/testing/pytest.py +21 -4
  77. spinta/types/config.py +5 -0
  78. spinta/types/datatype.py +6 -0
  79. spinta/types/partial/link.py +30 -15
  80. spinta/types/ref/link.py +3 -1
  81. spinta/types/text/helpers.py +1 -1
  82. spinta/ufuncs/querybuilder/ufuncs.py +9 -0
  83. spinta/ufuncs/resultbuilder/ufuncs.py +109 -2
  84. {spinta-0.2.dev20.dist-info → spinta-0.2.dev22.dist-info}/METADATA +1 -1
  85. {spinta-0.2.dev20.dist-info → spinta-0.2.dev22.dist-info}/RECORD +89 -96
  86. spinta/backends/mongo/commands/freeze.py +0 -15
  87. spinta/backends/postgresql/commands/freeze.py +0 -190
  88. spinta/backends/postgresql/types/array/freeze.py +0 -86
  89. spinta/backends/postgresql/types/file/freeze.py +0 -102
  90. spinta/backends/postgresql/types/object/freeze.py +0 -57
  91. spinta/backends/postgresql/types/ref/freeze.py +0 -83
  92. spinta/manifests/backend/commands/freeze.py +0 -10
  93. spinta/manifests/yaml/commands/freeze.py +0 -50
  94. spinta/migrations/__init__.py +0 -23
  95. spinta/migrations/schema/alembic.py +0 -119
  96. /spinta/{migrations/schema/__init__.py → manifests/xsd2/commands/load.py} +0 -0
  97. {spinta-0.2.dev20.dist-info → spinta-0.2.dev22.dist-info}/WHEEL +0 -0
  98. {spinta-0.2.dev20.dist-info → spinta-0.2.dev22.dist-info}/entry_points.txt +0 -0
  99. {spinta-0.2.dev20.dist-info → spinta-0.2.dev22.dist-info}/licenses/LICENSE +0 -0
spinta/auth.py CHANGED
@@ -58,6 +58,7 @@ from spinta.exceptions import (
58
58
  InvalidClientsKeymapStructure,
59
59
  InvalidScopes,
60
60
  InvalidClientFileFormat,
61
+ ModelNotFound,
61
62
  )
62
63
  from spinta.utils import passwords
63
64
  from spinta.utils.config import get_clients_path, get_keymap_path, get_id_path, get_helpers_path
@@ -858,40 +859,46 @@ def authorized(
858
859
  scope_formatter: ScopeFormatterFunc = None,
859
860
  ):
860
861
  config: Config = context.get("config")
862
+
863
+ # Disable access to nodes that have lower access level than config.access
864
+ if config.access > node.access:
865
+ if throw:
866
+ raise ModelNotFound(model=node.name)
867
+ return False
868
+
861
869
  token = context.get("auth.token")
862
- # Unauthorized clients can only access open nodes.
863
- unauthorized = token.get_client_id() == get_default_auth_client_id(context)
864
- open_node = node.access >= Access.open
865
- if unauthorized and not open_node:
870
+
871
+ # Unauthenticated clients can only access nodes if spinta config.access is open.
872
+ unauthenticated = token.get_client_id() == get_default_auth_client_id(context)
873
+
874
+ if unauthenticated and config.access < Access.open:
866
875
  if throw:
867
876
  raise AuthorizedClientsOnly()
868
877
  else:
869
878
  return False
870
879
 
871
- # Private nodes can only be accessed with explicit node scope.
880
+ # Add explicit node scope
872
881
  scopes = [node]
873
882
 
874
- # Protected and higher level nodes can be accessed with parent nodes scopes.
875
- if node.access > Access.private:
876
- ns = None
877
-
878
- if isinstance(node, Property):
879
- # Hidden nodes also require explicit scope.
880
- # XXX: `hidden` parameter should only be used for API control, not
881
- # access control. See docs.
882
- if not node.hidden:
883
- scopes.append(node.model)
884
- scopes.append(node.model.ns)
885
- ns = node.model.ns
886
- elif isinstance(node, Model):
887
- scopes.append(node.ns)
888
- ns = node.ns
889
- elif isinstance(node, Namespace):
890
- ns = node
891
-
892
- # Add all parent namespace scopes too.
893
- if ns:
894
- scopes.extend(ns.parents())
883
+ # Add parent node scopes
884
+ ns = None
885
+ if isinstance(node, Property):
886
+ # Hidden nodes also require explicit scope.
887
+ # XXX: `hidden` parameter should only be used for API control, not
888
+ # access control. See docs.
889
+ if not node.hidden:
890
+ scopes.append(node.model)
891
+ scopes.append(node.model.ns)
892
+ ns = node.model.ns
893
+ elif isinstance(node, Model):
894
+ scopes.append(node.ns)
895
+ ns = node.ns
896
+ elif isinstance(node, Namespace):
897
+ ns = node
898
+
899
+ # Add all parent namespace scopes too.
900
+ if ns:
901
+ scopes.extend(ns.parents())
895
902
 
896
903
  # Build scope names.
897
904
  scope_formatter = scope_formatter or config.scope_formatter
@@ -1,12 +1,14 @@
1
+ from __future__ import annotations
2
+
3
+ import dataclasses
1
4
  from typing import Any
2
- from typing import Dict
3
5
  from typing import Iterable
4
6
  from typing import Iterator
5
- from typing import List
6
- from typing import Optional
7
- from typing import Tuple
8
7
  from typing import TypeVar
9
- from typing import Union
8
+
9
+ import sqlalchemy as sa
10
+
11
+ from multipledispatch import dispatch
10
12
 
11
13
  from spinta import commands
12
14
  from spinta import exceptions
@@ -14,6 +16,7 @@ from spinta import spyna
14
16
  from spinta.auth import authorized
15
17
  from spinta.backends import Backend
16
18
  from spinta.backends.components import SelectTree
19
+ from spinta.backends.postgresql.helpers import get_pg_name
17
20
  from spinta.commands import build_full_response
18
21
  from spinta.components import Config, DataItem
19
22
  from spinta.core.enums import Action
@@ -28,6 +31,11 @@ from spinta.utils.data import take
28
31
  from spinta.backends.constants import TableType, BackendOrigin
29
32
 
30
33
 
34
+ from sqlalchemy.dialects import postgresql
35
+
36
+ pg_identifier_preparer = postgresql.dialect().identifier_preparer
37
+
38
+
31
39
  def validate_and_return_transaction(context: Context, backend: Backend, **kwargs):
32
40
  if not backend.available:
33
41
  backend.available = commands.wait(context, backend)
@@ -51,7 +59,7 @@ def validate_and_return_begin(context: Context, backend: Backend, **kwargs):
51
59
 
52
60
 
53
61
  def load_backend(
54
- context: Context, component: Component, name: str, origin: BackendOrigin, data: Dict[str, str]
62
+ context: Context, component: Component, name: str, origin: BackendOrigin, data: dict[str, str]
55
63
  ) -> Backend:
56
64
  config = context.get("config")
57
65
  type_ = data.get("type")
@@ -77,7 +85,7 @@ def load_backend(
77
85
  def get_select_tree(
78
86
  context: Context,
79
87
  action: Action,
80
- select: Optional[List[str]],
88
+ select: list[str] | None,
81
89
  ) -> SelectTree:
82
90
  if isinstance(select, dict):
83
91
  select = list(select.keys())
@@ -92,8 +100,8 @@ def get_select_tree(
92
100
  def _apply_always_show_id(
93
101
  context: Context,
94
102
  action: Action,
95
- select: Optional[List[str]],
96
- ) -> Optional[List[str]]:
103
+ select: list[str] | None,
104
+ ) -> list[str] | None:
97
105
  if action in (Action.GETALL, Action.SEARCH):
98
106
  config = context.get("config")
99
107
  if config.always_show_id:
@@ -106,18 +114,18 @@ def _apply_always_show_id(
106
114
 
107
115
  def get_select_prop_names(
108
116
  context: Context,
109
- node: Union[Model, Property, DataType],
110
- props: Dict[str, Property],
117
+ node: Model | Property | DataType,
118
+ props: dict[str, Property],
111
119
  action: Action,
112
120
  select: SelectTree,
113
121
  *,
114
122
  # If False, do not check if client has access to this property.
115
123
  auth: bool = True,
116
124
  # Allowed reserved property names.
117
- reserved: List[str] = None,
125
+ reserved: list[str] = None,
118
126
  # If False, do not include Denorm type props
119
127
  include_denorm_props: bool = True,
120
- ) -> List[str]:
128
+ ) -> list[str]:
121
129
  known = set(reserved or []) | set(take(props))
122
130
  check_unknown_props(node, select, known)
123
131
 
@@ -138,13 +146,13 @@ def get_select_prop_names(
138
146
 
139
147
  def select_model_props(
140
148
  model: Model,
141
- prop_names: List[str],
149
+ prop_names: list[str],
142
150
  value: dict,
143
151
  select: SelectTree,
144
- reserved: List[str],
152
+ reserved: list[str],
145
153
  ) -> Iterator[
146
- Tuple[
147
- Union[Property, str],
154
+ tuple[
155
+ Property | str,
148
156
  Any,
149
157
  SelectTree,
150
158
  ]
@@ -170,16 +178,16 @@ T = TypeVar("T")
170
178
 
171
179
 
172
180
  def select_props(
173
- node: Union[Namespace, Model, Property],
181
+ node: Namespace | Model | Property,
174
182
  keys: Iterable[str],
175
- props: Dict[str, Property],
176
- value: Dict[str, T],
183
+ props: dict[str, Property],
184
+ value: dict[str, T],
177
185
  select: SelectTree,
178
186
  *,
179
187
  reserved: bool = True,
180
188
  ) -> Iterator[
181
- Tuple[
182
- Union[Property, str],
189
+ tuple[
190
+ Property | str,
183
191
  T,
184
192
  SelectTree,
185
193
  ]
@@ -191,15 +199,15 @@ def select_props(
191
199
 
192
200
 
193
201
  def select_only_props(
194
- node: Union[Namespace, Model, Property],
202
+ node: Namespace | Model | Property,
195
203
  keys: Iterable[str],
196
- props: Dict[str, Property],
204
+ props: dict[str, Property],
197
205
  select: SelectTree,
198
206
  *,
199
207
  reserved: bool = True,
200
208
  ) -> Iterator[
201
- Tuple[
202
- Union[Property, str],
209
+ tuple[
210
+ Property | str,
203
211
  SelectTree,
204
212
  ]
205
213
  ]:
@@ -211,9 +219,9 @@ def select_only_props(
211
219
 
212
220
  def _select_prop(
213
221
  key: str,
214
- props: Dict[str, Property],
215
- node: Union[Namespace, Model, Property],
216
- ) -> Optional[Property]:
222
+ props: dict[str, Property],
223
+ node: Namespace | Model | Property,
224
+ ) -> Property | None:
217
225
  if not (prop := props.get(key)) or prop.hidden:
218
226
  return None
219
227
 
@@ -222,12 +230,12 @@ def _select_prop(
222
230
 
223
231
  def select_keys(
224
232
  keys: Iterable[str],
225
- value: Dict[str, T],
233
+ value: dict[str, T],
226
234
  select: SelectTree,
227
235
  *,
228
236
  reserved: bool = True,
229
237
  ) -> Iterator[
230
- Tuple[
238
+ tuple[
231
239
  str,
232
240
  T,
233
241
  SelectTree,
@@ -253,7 +261,7 @@ def select_only_keys(
253
261
  *,
254
262
  reserved: bool = True,
255
263
  ) -> Iterator[
256
- Tuple[
264
+ tuple[
257
265
  str,
258
266
  SelectTree,
259
267
  ]
@@ -280,8 +288,8 @@ def select_only_keys(
280
288
  # FIXME: We should check select list at the very beginning of
281
289
  # request, not when returning results.
282
290
  def check_unknown_props(
283
- node: Union[Model, Property, DataType],
284
- select: Optional[Iterable[str]],
291
+ node: Model | Property | DataType,
292
+ select: Iterable[str] | None,
285
293
  known: Iterable[str],
286
294
  ):
287
295
  unknown_properties = set(select or []) - set(known) - {"*"}
@@ -291,7 +299,7 @@ def check_unknown_props(
291
299
  )
292
300
 
293
301
 
294
- def flat_select_to_nested(select: Optional[List[str]]) -> SelectTree:
302
+ def flat_select_to_nested(select: list[str] | None) -> SelectTree:
295
303
  """
296
304
  >>> flat_select_to_nested(None)
297
305
 
@@ -316,7 +324,7 @@ def flat_select_to_nested(select: Optional[List[str]]) -> SelectTree:
316
324
  return res
317
325
 
318
326
 
319
- def get_model_reserved_props(action: Action, include_page: bool) -> List[str]:
327
+ def get_model_reserved_props(action: Action, include_page: bool) -> list[str]:
320
328
  if action == Action.GETALL:
321
329
  reserved = ["_type", "_id", "_revision"]
322
330
  elif action == Action.SEARCH:
@@ -332,12 +340,145 @@ def get_model_reserved_props(action: Action, include_page: bool) -> List[str]:
332
340
  return reserved
333
341
 
334
342
 
335
- def get_ns_reserved_props(action: Action) -> List[str]:
343
+ def get_ns_reserved_props(action: Action) -> list[str]:
336
344
  return []
337
345
 
338
346
 
347
+ @dataclasses.dataclass
348
+ class TableIdentifier:
349
+ """
350
+ Represents a table identifier across logical (app) and PostgreSQL layers.
351
+
352
+ It builds derived names used for internal logic and SQL queries, including
353
+ schema-qualified and escaped identifiers.
354
+
355
+ Attributes:
356
+ schema (str | None): Logical schema/namespace (e.g. "datasets/gov/rc").
357
+ base_name (str): Base table name (e.g. "Building").
358
+ table_type (TableType): Table type suffix (default: TableType.MAIN).
359
+ table_arg (str | None): Optional argument appended used for table types that require property.
360
+ default_pg_schema (str | None): Fallback PG schema if schema is not given.
361
+
362
+ logical_name (str): Computed name (base + type + optional arg).
363
+ Example: "Building/:list/apartments"
364
+ logical_qualified_name (str): Logical name with schema (dataset).
365
+ Example: "datasets/gov/rc/Building/:list/apartments"
366
+
367
+ pg_table_name (str): PG-safe (compressed) table name from logical_name.
368
+ pg_schema_name (str | None): PG-safe (compressed) schema name.
369
+ pg_qualified_name (str): PG-safe (compressed) schema with table name.
370
+ Example: "datasets/gov/rc.Building/:list/apartments" (unescaped).
371
+ pg_escaped_qualified_name (str): Quoted version of pg_qualified_name, used for queries.
372
+ Example: '"datasets/gov/rc"."Building/:list/apartments"' (escaped).
373
+
374
+ Example:
375
+ >>> TableIdentifier("datasets/gov/rc", "Buildings", TableType.LIST, "apartments")
376
+ # logical_qualified_name: "datasets/gov/rc/Buildings/:list/apartments"
377
+ # pg_qualified_name: "datasets/gov/rc.Building/:list/apartments"
378
+
379
+ >>> TableIdentifier("datasets/gov/rc", "Buildings")
380
+ # logical_qualified_name: "datasets/gov/rc/Buildings"
381
+ # pg_qualified_name: "datasets/gov/rc.Building"
382
+
383
+ >>> TableIdentifier("datasets/gov/rc/very/long/dataset/name/that/does/not/fit/withing/limits", "Buildings")
384
+ # logical_qualified_name: "datasets/gov/rc/very/long/dataset/name/that/does/not/fit/withing/limits/Buildings"
385
+ # pg_qualified_name: "datasets/gov/rc/very/long/dataset/nam_e5985b69_t/withing/limits.Building"
386
+ """
387
+
388
+ schema: str | None
389
+ base_name: str
390
+ table_type: TableType = dataclasses.field(default=TableType.MAIN)
391
+ table_arg: str | None = dataclasses.field(default=None)
392
+ default_pg_schema: str | None = dataclasses.field(default=None)
393
+
394
+ logical_name: str = dataclasses.field(init=False)
395
+ # Name with namespace connected with '/', like it is used with Model class
396
+ logical_qualified_name: str = dataclasses.field(init=False)
397
+
398
+ pg_table_name: str = dataclasses.field(init=False)
399
+ pg_schema_name: str | None = dataclasses.field(init=False)
400
+ # Used for hashed schema and table names
401
+ pg_qualified_name: str = dataclasses.field(init=False)
402
+ # Escaped qualified name, used for queries
403
+ pg_escaped_qualified_name: str = dataclasses.field(init=False)
404
+
405
+ def __post_init__(self):
406
+ self.logical_name = self.base_name + self.table_type.value
407
+ if self.table_arg:
408
+ self.logical_name += "/" + self.table_arg
409
+
410
+ self.logical_qualified_name = f"{self.schema}/{self.logical_name}" if self.schema else self.logical_name
411
+
412
+ self.pg_table_name = get_pg_name(self.logical_name)
413
+ self.pg_schema_name = get_pg_name(self.schema) if self.schema else self.default_pg_schema
414
+ self.pg_qualified_name = (
415
+ f"{self.pg_schema_name}.{self.pg_table_name}" if self.pg_schema_name else self.pg_table_name
416
+ )
417
+ self.pg_escaped_qualified_name = (
418
+ f"{pg_identifier_preparer.quote(self.pg_schema_name)}.{pg_identifier_preparer.quote(self.pg_table_name)}"
419
+ if self.pg_schema_name
420
+ else pg_identifier_preparer.quote(self.pg_table_name)
421
+ )
422
+
423
+ def change_table_type(self, new_type: TableType, table_arg: str | None = None) -> "TableIdentifier":
424
+ return dataclasses.replace(self, table_type=new_type, table_arg=table_arg)
425
+
426
+ def apply_removed_prefix(self, remove_model_only: bool = False) -> "TableIdentifier":
427
+ if remove_model_only or not self.table_arg:
428
+ if not self.base_name.startswith("__"):
429
+ return dataclasses.replace(self, base_name=f"__{self.base_name}")
430
+ return self
431
+
432
+ if not self.table_arg.startswith("__"):
433
+ return dataclasses.replace(self, table_arg=f"__{self.table_arg}")
434
+ return self
435
+
436
+
437
+ @dispatch(str)
438
+ def get_table_identifier(item: str, **kwargs) -> TableIdentifier:
439
+ schema, model_name, table_type, table_arg = split_logical_name(item)
440
+ return TableIdentifier(schema=schema, base_name=model_name, table_type=table_type, table_arg=table_arg, **kwargs)
441
+
442
+
443
+ @dispatch(sa.Table)
444
+ def get_table_identifier(item: sa.Table, **kwargs) -> TableIdentifier:
445
+ if not item.comment:
446
+ return TableIdentifier(schema=item.schema, base_name=item.name, **kwargs)
447
+ if item.schema not in ("public", None):
448
+ return get_table_identifier(item.comment, **kwargs)
449
+
450
+ schema, model_name, table_type, table_arg = split_logical_name(item.comment)
451
+ return TableIdentifier(
452
+ schema=None,
453
+ base_name=f"{schema}/{model_name}" if schema else model_name,
454
+ table_type=table_type,
455
+ table_arg=table_arg,
456
+ **kwargs,
457
+ )
458
+
459
+
460
+ @dispatch((Model, Property))
461
+ def get_table_identifier(node: Model | Property, **kwargs) -> TableIdentifier:
462
+ return get_table_identifier(node, TableType.MAIN, **kwargs)
463
+
464
+
465
+ @dispatch((Model, Property), TableType)
466
+ def get_table_identifier(
467
+ node: Model | Property, table_type: TableType, table_arg: str = None, **kwargs
468
+ ) -> TableIdentifier:
469
+ model = node if isinstance(node, Model) else node.model
470
+
471
+ schema = model.ns.name if model.ns else None
472
+ base_name = model.get_name_without_ns()
473
+
474
+ if isinstance(node, Property) and table_type in (TableType.LIST, TableType.FILE):
475
+ table_arg = node.place
476
+
477
+ return TableIdentifier(schema, base_name, table_type, table_arg, **kwargs)
478
+
479
+
339
480
  def get_table_name(
340
- node: Union[Model, Property],
481
+ node: Model | Property,
341
482
  ttype: TableType = TableType.MAIN,
342
483
  ) -> str:
343
484
  if isinstance(node, Model):
@@ -351,6 +492,27 @@ def get_table_name(
351
492
  return name
352
493
 
353
494
 
495
+ def split_table_name(full_name: str) -> tuple[str | None, str]:
496
+ parts = full_name.split(".", maxsplit=1)
497
+ if len(parts) == 1:
498
+ return None, parts[0]
499
+ return parts[0], parts[1]
500
+
501
+
502
+ def split_logical_name(full_name: str) -> tuple[str | None, str, TableType, str | None]:
503
+ base_name, table_type, property_name = extract_table_data_from_logical_name(full_name)
504
+ parts = base_name.split("/")
505
+ if len(parts) == 1:
506
+ return None, parts[0], table_type, property_name
507
+
508
+ for i, part in enumerate(parts):
509
+ if part[0].isupper() or (part[:2] == "__" and part[2].isupper()):
510
+ namespace = "/".join(parts[:i])
511
+ model = "/".join(parts[i:])
512
+ return namespace, model, table_type, property_name
513
+ return None, base_name, table_type, property_name
514
+
515
+
354
516
  def load_query_builder_class(config: Config, backend: Backend):
355
517
  if backend.query_builder_type is None:
356
518
  return
@@ -400,3 +562,37 @@ def prepare_response(
400
562
  else:
401
563
  resp = {}
402
564
  return resp
565
+
566
+
567
+ def extract_table_data_from_logical_name(table_name: str) -> tuple[str | None, TableType | None, str | None]:
568
+ """
569
+ Extracts the main table name, table type, and an optional property suffix from a logical
570
+ table name string. It parses the given logical table name and determines whether it belongs
571
+ to the main table or some specific table type. If a specific type is found, it splits the
572
+ table name into its components.
573
+
574
+ Parameters:
575
+ table_name (str): The logical table name string that needs to be processed.
576
+
577
+ Returns:
578
+ tuple: A tuple containing:
579
+ - str | None: The main table name.
580
+ - TableType | None: The type of the table, which can be `MAIN` or other enum members of
581
+ `TableType`.
582
+ - str | None: A property suffix string if present in the logical table name, or
583
+ None otherwise.
584
+ """
585
+ if "/:" not in table_name:
586
+ return table_name, TableType.MAIN, None
587
+
588
+ for table_type in TableType:
589
+ if table_type is TableType.MAIN:
590
+ continue
591
+
592
+ if table_type.value in table_name:
593
+ data = table_name.split(table_type.value, 1)
594
+ if data[1]:
595
+ return data[0], table_type, data[1][1:] # skip /property slash
596
+ return data[0], table_type, None
597
+
598
+ return None, None, None
@@ -2,12 +2,12 @@ from spinta import commands
2
2
  from spinta.components import Context
3
3
  from spinta.manifests.components import Manifest
4
4
  from spinta.backends.constants import TableType
5
- from spinta.backends.helpers import get_table_name
5
+ from spinta.backends.helpers import get_table_identifier
6
6
  from spinta.backends.memory.components import Memory
7
7
 
8
8
 
9
9
  @commands.prepare.register(Context, Memory, Manifest)
10
10
  def prepare(context: Context, backend: Memory, manifest: Manifest, **kwargs):
11
11
  for model in commands.get_models(context, manifest).values():
12
- backend.create(get_table_name(model))
13
- backend.create(get_table_name(model, TableType.CHANGELOG))
12
+ backend.create(get_table_identifier(model).logical_qualified_name)
13
+ backend.create(get_table_identifier(model, TableType.CHANGELOG).logical_qualified_name)
@@ -16,7 +16,6 @@ def load(
16
16
  backend: Memory,
17
17
  *,
18
18
  into: Manifest = None,
19
- freezed: bool = True,
20
19
  ) -> None:
21
20
  for source in manifest.sync:
22
- commands.load(context, source, into=into or manifest, freezed=freezed)
21
+ commands.load(context, source, into=into or manifest)
@@ -7,13 +7,13 @@ from spinta.components import Model
7
7
  from spinta.typing import ObjectData
8
8
  from spinta.backends.memory.components import Memory
9
9
  from spinta.backends.constants import TableType
10
- from spinta.backends.helpers import get_table_name
10
+ from spinta.backends.helpers import get_table_identifier
11
11
 
12
12
 
13
13
  @commands.getall.register(Context, Model, Memory)
14
14
  def getall(context: Context, model: Model, db: Memory, *, query: Expr = None, **kwargs) -> Iterator[ObjectData]:
15
- table = get_table_name(model)
16
- return db.data[table].values()
15
+ table = get_table_identifier(model)
16
+ return db.data[table.logical_qualified_name].values()
17
17
 
18
18
 
19
19
  @commands.getone.register(Context, Model, Memory)
@@ -24,8 +24,8 @@ def getone(
24
24
  *,
25
25
  id_: str,
26
26
  ) -> ObjectData:
27
- table = get_table_name(model)
28
- return db.data[table][id_]
27
+ table = get_table_identifier(model)
28
+ return db.data[table.logical_qualified_name][id_]
29
29
 
30
30
 
31
31
  @commands.changes.register(Context, Model, Memory)
@@ -38,5 +38,5 @@ def changes(
38
38
  limit: int = 100,
39
39
  offset: int = -10,
40
40
  ):
41
- table = get_table_name(model, TableType.CHANGELOG)
42
- return db.data[table].values()
41
+ table = get_table_identifier(model, TableType.CHANGELOG)
42
+ return db.data[table.logical_qualified_name].values()
@@ -2,6 +2,8 @@ from spinta import commands
2
2
  from spinta.components import Context
3
3
  from spinta.backends.postgresql.components import PostgreSQL
4
4
 
5
+ import sqlalchemy as sa
6
+
5
7
 
6
8
  @commands.bootstrap.register(Context, PostgreSQL)
7
9
  def bootstrap(context: Context, backend: PostgreSQL):
@@ -10,5 +12,19 @@ def bootstrap(context: Context, backend: PostgreSQL):
10
12
  # line.
11
13
  # TODO: update appropriate rows in _schema and save `applied` date
12
14
  # of schema migration
15
+ validated_schemas = []
13
16
  with backend.engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
17
+ for table in backend.tables.values():
18
+ if not (schema := table.schema):
19
+ continue
20
+
21
+ if schema in validated_schemas:
22
+ continue
23
+
24
+ if conn.dialect.has_schema(conn, schema):
25
+ validated_schemas.append(schema)
26
+ continue
27
+
28
+ conn.execute(sa.schema.CreateSchema(schema))
29
+ validated_schemas.append(schema)
14
30
  backend.schema.create_all(conn, checkfirst=True)
@@ -5,18 +5,17 @@ from sqlalchemy.dialects.postgresql import JSONB, UUID
5
5
 
6
6
  from spinta import commands
7
7
  from spinta.backends.constants import TableType
8
- from spinta.backends.helpers import get_table_name
8
+ from spinta.backends.helpers import get_table_identifier
9
9
  from spinta.backends.postgresql.components import PostgreSQL
10
10
  from spinta.backends.postgresql.constants import UNSUPPORTED_TYPES
11
11
  from spinta.backends.postgresql.helpers import get_column_name
12
12
  from spinta.backends.postgresql.helpers.changes import get_changes_table
13
- from spinta.backends.postgresql.helpers.name import PG_NAMING_CONVENTION, get_pg_table_name, get_pg_column_name
13
+ from spinta.backends.postgresql.helpers.name import get_pg_column_name
14
14
  from spinta.backends.postgresql.helpers.redirect import get_redirect_table
15
15
  from spinta.backends.postgresql.helpers.type import get_column_type
16
16
  from spinta.components import Context, Model
17
17
  from spinta.manifests.components import Manifest
18
18
  from spinta.types.datatype import DataType, PrimaryKey, Ref
19
- from spinta.utils.sqlalchemy import Convention
20
19
 
21
20
 
22
21
  @overload
@@ -31,9 +30,8 @@ def prepare(context: Context, backend: PostgreSQL, manifest: Manifest, **kwargs)
31
30
  @overload
32
31
  @commands.prepare.register(Context, PostgreSQL, Model)
33
32
  def prepare(context: Context, backend: PostgreSQL, model: Model, ignore_duplicate: bool = False, **kwargs):
34
- table_name = get_table_name(model)
35
- main_table_name = get_pg_table_name(table_name)
36
- if table_name in backend.tables and ignore_duplicate:
33
+ table_identifier = get_table_identifier(model)
34
+ if table_identifier.logical_qualified_name in backend.tables and ignore_duplicate:
37
35
  return
38
36
 
39
37
  columns = []
@@ -67,13 +65,14 @@ def prepare(context: Context, backend: PostgreSQL, model: Model, ignore_duplicat
67
65
  # Create main table.
68
66
  pkey_type = commands.get_primary_key_type(context, backend)
69
67
  main_table = sa.Table(
70
- main_table_name,
68
+ table_identifier.pg_table_name,
71
69
  backend.schema,
72
70
  sa.Column(get_pg_column_name("_txn"), pkey_type, index=True, comment="_txn"),
73
71
  sa.Column(get_pg_column_name("_created"), sa.DateTime, comment="_created"),
74
72
  sa.Column(get_pg_column_name("_updated"), sa.DateTime, comment="_updated"),
75
73
  *columns,
76
- comment=table_name,
74
+ schema=table_identifier.pg_schema_name,
75
+ comment=table_identifier.logical_qualified_name,
77
76
  )
78
77
  backend.add_table(main_table, model)
79
78
 
@@ -115,6 +114,7 @@ def prepare(context: Context, backend: PostgreSQL, dtype: DataType, **kwargs):
115
114
  "uri": sa.String,
116
115
  "denorm": sa.String,
117
116
  "uuid": UUID(as_uuid=True),
117
+ "unknown": sa.Text,
118
118
  }
119
119
 
120
120
  if dtype.name not in types:
@@ -136,13 +136,12 @@ def prepare(context: Context, backend: PostgreSQL, dtype: PrimaryKey, **kwargs):
136
136
  base = dtype.prop.model.base
137
137
  column_name = get_pg_column_name("_id")
138
138
  if base and commands.identifiable(base):
139
- ref_table = get_pg_table_name(base.parent)
139
+ ref_table_identifier = get_table_identifier(base.parent)
140
140
  return [
141
141
  sa.Column(column_name, pkey_type, primary_key=True, comment="_id"),
142
142
  sa.ForeignKeyConstraint(
143
143
  [column_name],
144
- [f"{ref_table}._id"],
145
- name=PG_NAMING_CONVENTION[Convention.FK] % {"table_name": ref_table, "column_0_N_name": column_name},
144
+ [f"{ref_table_identifier.pg_qualified_name}._id"],
146
145
  ),
147
146
  ]
148
147
  return sa.Column(column_name, pkey_type, primary_key=True, comment="_id")