spinta 0.2.dev20__py3-none-any.whl → 0.2.dev21__py3-none-any.whl

spinta/backends/helpers.py

@@ -1,12 +1,14 @@
+from __future__ import annotations
+
+import dataclasses
 from typing import Any
-from typing import Dict
 from typing import Iterable
 from typing import Iterator
-from typing import List
-from typing import Optional
-from typing import Tuple
 from typing import TypeVar
-from typing import Union
+
+import sqlalchemy as sa
+
+from multipledispatch import dispatch
 
 from spinta import commands
 from spinta import exceptions
@@ -14,6 +16,7 @@ from spinta import spyna
 from spinta.auth import authorized
 from spinta.backends import Backend
 from spinta.backends.components import SelectTree
+from spinta.backends.postgresql.helpers import get_pg_name
 from spinta.commands import build_full_response
 from spinta.components import Config, DataItem
 from spinta.core.enums import Action
@@ -28,6 +31,11 @@ from spinta.utils.data import take
 from spinta.backends.constants import TableType, BackendOrigin
 
 
+from sqlalchemy.dialects import postgresql
+
+pg_identifier_preparer = postgresql.dialect().identifier_preparer
+
+
 def validate_and_return_transaction(context: Context, backend: Backend, **kwargs):
     if not backend.available:
         backend.available = commands.wait(context, backend)
@@ -51,7 +59,7 @@ def validate_and_return_begin(context: Context, backend: Backend, **kwargs):
 
 
 def load_backend(
-    context: Context, component: Component, name: str, origin: BackendOrigin, data: Dict[str, str]
+    context: Context, component: Component, name: str, origin: BackendOrigin, data: dict[str, str]
 ) -> Backend:
     config = context.get("config")
     type_ = data.get("type")
@@ -77,7 +85,7 @@ def load_backend(
 def get_select_tree(
     context: Context,
     action: Action,
-    select: Optional[List[str]],
+    select: list[str] | None,
 ) -> SelectTree:
     if isinstance(select, dict):
         select = list(select.keys())
@@ -92,8 +100,8 @@ def get_select_tree(
 def _apply_always_show_id(
     context: Context,
     action: Action,
-    select: Optional[List[str]],
-) -> Optional[List[str]]:
+    select: list[str] | None,
+) -> list[str] | None:
     if action in (Action.GETALL, Action.SEARCH):
         config = context.get("config")
         if config.always_show_id:
@@ -106,18 +114,18 @@ def _apply_always_show_id(
 
 def get_select_prop_names(
     context: Context,
-    node: Union[Model, Property, DataType],
-    props: Dict[str, Property],
+    node: Model | Property | DataType,
+    props: dict[str, Property],
     action: Action,
     select: SelectTree,
     *,
     # If False, do not check if client has access to this property.
     auth: bool = True,
     # Allowed reserved property names.
-    reserved: List[str] = None,
+    reserved: list[str] = None,
     # If False, do not include Denorm type props
     include_denorm_props: bool = True,
-) -> List[str]:
+) -> list[str]:
     known = set(reserved or []) | set(take(props))
     check_unknown_props(node, select, known)
 
@@ -138,13 +146,13 @@ def get_select_prop_names(
 
 def select_model_props(
     model: Model,
-    prop_names: List[str],
+    prop_names: list[str],
     value: dict,
     select: SelectTree,
-    reserved: List[str],
+    reserved: list[str],
 ) -> Iterator[
-    Tuple[
-        Union[Property, str],
+    tuple[
+        Property | str,
         Any,
         SelectTree,
     ]
@@ -170,16 +178,16 @@ T = TypeVar("T")
 
 
 def select_props(
-    node: Union[Namespace, Model, Property],
+    node: Namespace | Model | Property,
     keys: Iterable[str],
-    props: Dict[str, Property],
-    value: Dict[str, T],
+    props: dict[str, Property],
+    value: dict[str, T],
     select: SelectTree,
     *,
     reserved: bool = True,
 ) -> Iterator[
-    Tuple[
-        Union[Property, str],
+    tuple[
+        Property | str,
         T,
         SelectTree,
     ]
@@ -191,15 +199,15 @@ def select_props(
 
 
 def select_only_props(
-    node: Union[Namespace, Model, Property],
+    node: Namespace | Model | Property,
     keys: Iterable[str],
-    props: Dict[str, Property],
+    props: dict[str, Property],
     select: SelectTree,
     *,
     reserved: bool = True,
 ) -> Iterator[
-    Tuple[
-        Union[Property, str],
+    tuple[
+        Property | str,
         SelectTree,
     ]
 ]:
@@ -211,9 +219,9 @@ def select_only_props(
 
 def _select_prop(
     key: str,
-    props: Dict[str, Property],
-    node: Union[Namespace, Model, Property],
-) -> Optional[Property]:
+    props: dict[str, Property],
+    node: Namespace | Model | Property,
+) -> Property | None:
     if not (prop := props.get(key)) or prop.hidden:
         return None
 
@@ -222,12 +230,12 @@ def _select_prop(
 
 def select_keys(
     keys: Iterable[str],
-    value: Dict[str, T],
+    value: dict[str, T],
     select: SelectTree,
     *,
     reserved: bool = True,
 ) -> Iterator[
-    Tuple[
+    tuple[
         str,
         T,
         SelectTree,
@@ -253,7 +261,7 @@ def select_only_keys(
     *,
     reserved: bool = True,
 ) -> Iterator[
-    Tuple[
+    tuple[
         str,
         SelectTree,
     ]
@@ -280,8 +288,8 @@ def select_only_keys(
 # FIXME: We should check select list at the very beginning of
 #        request, not when returning results.
 def check_unknown_props(
-    node: Union[Model, Property, DataType],
-    select: Optional[Iterable[str]],
+    node: Model | Property | DataType,
+    select: Iterable[str] | None,
     known: Iterable[str],
 ):
     unknown_properties = set(select or []) - set(known) - {"*"}
@@ -291,7 +299,7 @@ def check_unknown_props(
         )
 
 
-def flat_select_to_nested(select: Optional[List[str]]) -> SelectTree:
+def flat_select_to_nested(select: list[str] | None) -> SelectTree:
     """
     >>> flat_select_to_nested(None)
 
@@ -316,7 +324,7 @@ def flat_select_to_nested(select: Optional[List[str]]) -> SelectTree:
     return res
 
 
-def get_model_reserved_props(action: Action, include_page: bool) -> List[str]:
+def get_model_reserved_props(action: Action, include_page: bool) -> list[str]:
     if action == Action.GETALL:
         reserved = ["_type", "_id", "_revision"]
     elif action == Action.SEARCH:
@@ -332,12 +340,145 @@ def get_model_reserved_props(action: Action, include_page: bool) -> List[str]:
     return reserved
 
 
-def get_ns_reserved_props(action: Action) -> List[str]:
+def get_ns_reserved_props(action: Action) -> list[str]:
     return []
 
 
+@dataclasses.dataclass
+class TableIdentifier:
+    """
+    Represents a table identifier across logical (app) and PostgreSQL layers.
+
+    It builds derived names used for internal logic and SQL queries, including
+    schema-qualified and escaped identifiers.
+
+    Attributes:
+        schema (str | None): Logical schema/namespace (e.g. "datasets/gov/rc").
+        base_name (str): Base table name (e.g. "Building").
+        table_type (TableType): Table type suffix (default: TableType.MAIN).
+        table_arg (str | None): Optional argument appended used for table types that require property.
+        default_pg_schema (str | None): Fallback PG schema if schema is not given.
+
+        logical_name (str): Computed name (base + type + optional arg).
+            Example: "Building/:list/apartments"
+        logical_qualified_name (str): Logical name with schema (dataset).
+            Example: "datasets/gov/rc/Building/:list/apartments"
+
+        pg_table_name (str): PG-safe (compressed) table name from logical_name.
+        pg_schema_name (str | None): PG-safe (compressed) schema name.
+        pg_qualified_name (str): PG-safe (compressed) schema with table name.
+            Example: "datasets/gov/rc.Building/:list/apartments" (unescaped).
+        pg_escaped_qualified_name (str): Quoted version of pg_qualified_name, used for queries.
+            Example: '"datasets/gov/rc"."Building/:list/apartments"' (escaped).
+
+    Example:
+        >>> TableIdentifier("datasets/gov/rc", "Buildings", TableType.LIST, "apartments")
+        # logical_qualified_name: "datasets/gov/rc/Buildings/:list/apartments"
+        # pg_qualified_name: "datasets/gov/rc.Building/:list/apartments"
+
+        >>> TableIdentifier("datasets/gov/rc", "Buildings")
+        # logical_qualified_name: "datasets/gov/rc/Buildings"
+        # pg_qualified_name: "datasets/gov/rc.Building"
+
+        >>> TableIdentifier("datasets/gov/rc/very/long/dataset/name/that/does/not/fit/withing/limits", "Buildings")
+        # logical_qualified_name: "datasets/gov/rc/very/long/dataset/name/that/does/not/fit/withing/limits/Buildings"
+        # pg_qualified_name: "datasets/gov/rc/very/long/dataset/nam_e5985b69_t/withing/limits.Building"
+    """
+
+    schema: str | None
+    base_name: str
+    table_type: TableType = dataclasses.field(default=TableType.MAIN)
+    table_arg: str | None = dataclasses.field(default=None)
+    default_pg_schema: str | None = dataclasses.field(default=None)
+
+    logical_name: str = dataclasses.field(init=False)
+    # Name with namespace connected with '/', like it is used with Model class
+    logical_qualified_name: str = dataclasses.field(init=False)
+
+    pg_table_name: str = dataclasses.field(init=False)
+    pg_schema_name: str | None = dataclasses.field(init=False)
+    # Used for hashed schema and table names
+    pg_qualified_name: str = dataclasses.field(init=False)
+    # Escaped qualified name, used for queries
+    pg_escaped_qualified_name: str = dataclasses.field(init=False)
+
+    def __post_init__(self):
+        self.logical_name = self.base_name + self.table_type.value
+        if self.table_arg:
+            self.logical_name += "/" + self.table_arg
+
+        self.logical_qualified_name = f"{self.schema}/{self.logical_name}" if self.schema else self.logical_name
+
+        self.pg_table_name = get_pg_name(self.logical_name)
+        self.pg_schema_name = get_pg_name(self.schema) if self.schema else self.default_pg_schema
+        self.pg_qualified_name = (
+            f"{self.pg_schema_name}.{self.pg_table_name}" if self.pg_schema_name else self.pg_table_name
+        )
+        self.pg_escaped_qualified_name = (
+            f"{pg_identifier_preparer.quote(self.pg_schema_name)}.{pg_identifier_preparer.quote(self.pg_table_name)}"
+            if self.pg_schema_name
+            else pg_identifier_preparer.quote(self.pg_table_name)
+        )
+
+    def change_table_type(self, new_type: TableType, table_arg: str | None = None) -> "TableIdentifier":
+        return dataclasses.replace(self, table_type=new_type, table_arg=table_arg)
+
+    def apply_removed_prefix(self, remove_model_only: bool = False) -> "TableIdentifier":
+        if remove_model_only or not self.table_arg:
+            if not self.base_name.startswith("__"):
+                return dataclasses.replace(self, base_name=f"__{self.base_name}")
+            return self
+
+        if not self.table_arg.startswith("__"):
+            return dataclasses.replace(self, table_arg=f"__{self.table_arg}")
+        return self
+
+
+@dispatch(str)
+def get_table_identifier(item: str, **kwargs) -> TableIdentifier:
+    schema, model_name, table_type, table_arg = split_logical_name(item)
+    return TableIdentifier(schema=schema, base_name=model_name, table_type=table_type, table_arg=table_arg, **kwargs)
+
+
+@dispatch(sa.Table)
+def get_table_identifier(item: sa.Table, **kwargs) -> TableIdentifier:
+    if not item.comment:
+        return TableIdentifier(schema=item.schema, base_name=item.name, **kwargs)
+    if item.schema not in ("public", None):
+        return get_table_identifier(item.comment, **kwargs)
+
+    schema, model_name, table_type, table_arg = split_logical_name(item.comment)
+    return TableIdentifier(
+        schema=None,
+        base_name=f"{schema}/{model_name}" if schema else model_name,
+        table_type=table_type,
+        table_arg=table_arg,
+        **kwargs,
+    )
+
+
+@dispatch((Model, Property))
+def get_table_identifier(node: Model | Property, **kwargs) -> TableIdentifier:
+    return get_table_identifier(node, TableType.MAIN, **kwargs)
+
+
+@dispatch((Model, Property), TableType)
+def get_table_identifier(
+    node: Model | Property, table_type: TableType, table_arg: str = None, **kwargs
+) -> TableIdentifier:
+    model = node if isinstance(node, Model) else node.model
+
+    schema = model.ns.name if model.ns else None
+    base_name = model.get_name_without_ns()
+
+    if isinstance(node, Property) and table_type in (TableType.LIST, TableType.FILE):
+        table_arg = node.place
+
+    return TableIdentifier(schema, base_name, table_type, table_arg, **kwargs)
+
+
 def get_table_name(
-    node: Union[Model, Property],
+    node: Model | Property,
     ttype: TableType = TableType.MAIN,
 ) -> str:
     if isinstance(node, Model):
@@ -351,6 +492,27 @@ def get_table_name(
     return name
 
 
+def split_table_name(full_name: str) -> tuple[str | None, str]:
+    parts = full_name.split(".", maxsplit=1)
+    if len(parts) == 1:
+        return None, parts[0]
+    return parts[0], parts[1]
+
+
+def split_logical_name(full_name: str) -> tuple[str | None, str, TableType, str | None]:
+    base_name, table_type, property_name = extract_table_data_from_logical_name(full_name)
+    parts = base_name.split("/")
+    if len(parts) == 1:
+        return None, parts[0], table_type, property_name
+
+    for i, part in enumerate(parts):
+        if part[0].isupper() or (part[:2] == "__" and part[2].isupper()):
+            namespace = "/".join(parts[:i])
+            model = "/".join(parts[i:])
+            return namespace, model, table_type, property_name
+    return None, base_name, table_type, property_name
+
+
 def load_query_builder_class(config: Config, backend: Backend):
     if backend.query_builder_type is None:
         return
@@ -400,3 +562,37 @@ def prepare_response(
     else:
         resp = {}
     return resp
+
+
+def extract_table_data_from_logical_name(table_name: str) -> tuple[str | None, TableType | None, str | None]:
+    """
+    Extracts the main table name, table type, and an optional property suffix from a logical
+    table name string. It parses the given logical table name and determines whether it belongs
+    to the main table or some specific table type. If a specific type is found, it splits the
+    table name into its components.
+
+    Parameters:
+        table_name (str): The logical table name string that needs to be processed.
+
+    Returns:
+        tuple: A tuple containing:
+            - str | None: The main table name.
+            - TableType | None: The type of the table, which can be `MAIN` or other enum members of
+              `TableType`.
+            - str | None: A property suffix string if present in the logical table name, or
+              None otherwise.
+    """
+    if "/:" not in table_name:
+        return table_name, TableType.MAIN, None
+
+    for table_type in TableType:
+        if table_type is TableType.MAIN:
+            continue
+
+        if table_type.value in table_name:
+            data = table_name.split(table_type.value, 1)
+            if data[1]:
+                return data[0], table_type, data[1][1:]  # skip /property slash
+            return data[0], table_type, None
+
+    return None, None, None

spinta/backends/memory/commands/init.py

@@ -2,12 +2,12 @@ from spinta import commands
 from spinta.components import Context
 from spinta.manifests.components import Manifest
 from spinta.backends.constants import TableType
-from spinta.backends.helpers import get_table_name
+from spinta.backends.helpers import get_table_identifier
 from spinta.backends.memory.components import Memory
 
 
 @commands.prepare.register(Context, Memory, Manifest)
 def prepare(context: Context, backend: Memory, manifest: Manifest, **kwargs):
     for model in commands.get_models(context, manifest).values():
-        backend.create(get_table_name(model))
-        backend.create(get_table_name(model, TableType.CHANGELOG))
+        backend.create(get_table_identifier(model).logical_qualified_name)
+        backend.create(get_table_identifier(model, TableType.CHANGELOG).logical_qualified_name)

spinta/backends/memory/commands/manifest.py

@@ -16,7 +16,6 @@ def load(
     backend: Memory,
     *,
     into: Manifest = None,
-    freezed: bool = True,
 ) -> None:
     for source in manifest.sync:
-        commands.load(context, source, into=into or manifest, freezed=freezed)
+        commands.load(context, source, into=into or manifest)

spinta/backends/memory/commands/read.py

@@ -7,13 +7,13 @@ from spinta.components import Model
 from spinta.typing import ObjectData
 from spinta.backends.memory.components import Memory
 from spinta.backends.constants import TableType
-from spinta.backends.helpers import get_table_name
+from spinta.backends.helpers import get_table_identifier
 
 
 @commands.getall.register(Context, Model, Memory)
 def getall(context: Context, model: Model, db: Memory, *, query: Expr = None, **kwargs) -> Iterator[ObjectData]:
-    table = get_table_name(model)
-    return db.data[table].values()
+    table = get_table_identifier(model)
+    return db.data[table.logical_qualified_name].values()
 
 
 @commands.getone.register(Context, Model, Memory)
@@ -24,8 +24,8 @@ def getone(
     *,
     id_: str,
 ) -> ObjectData:
-    table = get_table_name(model)
-    return db.data[table][id_]
+    table = get_table_identifier(model)
+    return db.data[table.logical_qualified_name][id_]
 
 
 @commands.changes.register(Context, Model, Memory)
@@ -38,5 +38,5 @@ def changes(
     limit: int = 100,
     offset: int = -10,
 ):
-    table = get_table_name(model, TableType.CHANGELOG)
-    return db.data[table].values()
+    table = get_table_identifier(model, TableType.CHANGELOG)
+    return db.data[table.logical_qualified_name].values()

spinta/backends/postgresql/commands/bootstrap.py

@@ -2,6 +2,8 @@ from spinta import commands
 from spinta.components import Context
 from spinta.backends.postgresql.components import PostgreSQL
 
+import sqlalchemy as sa
+
 
 @commands.bootstrap.register(Context, PostgreSQL)
 def bootstrap(context: Context, backend: PostgreSQL):
@@ -10,5 +12,19 @@ def bootstrap(context: Context, backend: PostgreSQL):
     #      line.
     # TODO: update appropriate rows in _schema and save `applied` date
     #       of schema migration
+    validated_schemas = []
     with backend.engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
+        for table in backend.tables.values():
+            if not (schema := table.schema):
+                continue
+
+            if schema in validated_schemas:
+                continue
+
+            if conn.dialect.has_schema(conn, schema):
+                validated_schemas.append(schema)
+                continue
+
+            conn.execute(sa.schema.CreateSchema(schema))
+            validated_schemas.append(schema)
         backend.schema.create_all(conn, checkfirst=True)

spinta/backends/postgresql/commands/init.py

@@ -5,18 +5,17 @@ from sqlalchemy.dialects.postgresql import JSONB, UUID
 
 from spinta import commands
 from spinta.backends.constants import TableType
-from spinta.backends.helpers import get_table_name
+from spinta.backends.helpers import get_table_identifier
 from spinta.backends.postgresql.components import PostgreSQL
 from spinta.backends.postgresql.constants import UNSUPPORTED_TYPES
 from spinta.backends.postgresql.helpers import get_column_name
 from spinta.backends.postgresql.helpers.changes import get_changes_table
-from spinta.backends.postgresql.helpers.name import PG_NAMING_CONVENTION, get_pg_table_name, get_pg_column_name
+from spinta.backends.postgresql.helpers.name import get_pg_column_name
 from spinta.backends.postgresql.helpers.redirect import get_redirect_table
 from spinta.backends.postgresql.helpers.type import get_column_type
 from spinta.components import Context, Model
 from spinta.manifests.components import Manifest
 from spinta.types.datatype import DataType, PrimaryKey, Ref
-from spinta.utils.sqlalchemy import Convention
 
 
 @overload
@@ -31,9 +30,8 @@ def prepare(context: Context, backend: PostgreSQL, manifest: Manifest, **kwargs)
 @overload
 @commands.prepare.register(Context, PostgreSQL, Model)
 def prepare(context: Context, backend: PostgreSQL, model: Model, ignore_duplicate: bool = False, **kwargs):
-    table_name = get_table_name(model)
-    main_table_name = get_pg_table_name(table_name)
-    if table_name in backend.tables and ignore_duplicate:
+    table_identifier = get_table_identifier(model)
+    if table_identifier.logical_qualified_name in backend.tables and ignore_duplicate:
         return
 
     columns = []
@@ -67,13 +65,14 @@ def prepare(context: Context, backend: PostgreSQL, model: Model, ignore_duplicat
     # Create main table.
     pkey_type = commands.get_primary_key_type(context, backend)
     main_table = sa.Table(
-        main_table_name,
+        table_identifier.pg_table_name,
         backend.schema,
         sa.Column(get_pg_column_name("_txn"), pkey_type, index=True, comment="_txn"),
         sa.Column(get_pg_column_name("_created"), sa.DateTime, comment="_created"),
         sa.Column(get_pg_column_name("_updated"), sa.DateTime, comment="_updated"),
         *columns,
-        comment=table_name,
+        schema=table_identifier.pg_schema_name,
+        comment=table_identifier.logical_qualified_name,
     )
     backend.add_table(main_table, model)
 
@@ -115,6 +114,7 @@ def prepare(context: Context, backend: PostgreSQL, dtype: DataType, **kwargs):
         "uri": sa.String,
         "denorm": sa.String,
         "uuid": UUID(as_uuid=True),
+        "unknown": sa.Text,
     }
 
     if dtype.name not in types:
@@ -136,13 +136,12 @@ def prepare(context: Context, backend: PostgreSQL, dtype: PrimaryKey, **kwargs):
     base = dtype.prop.model.base
     column_name = get_pg_column_name("_id")
     if base and commands.identifiable(base):
-        ref_table = get_pg_table_name(base.parent)
+        ref_table_identifier = get_table_identifier(base.parent)
         return [
             sa.Column(column_name, pkey_type, primary_key=True, comment="_id"),
             sa.ForeignKeyConstraint(
                 [column_name],
-                [f"{ref_table}._id"],
-                name=PG_NAMING_CONVENTION[Convention.FK] % {"table_name": ref_table, "column_0_N_name": column_name},
+                [f"{ref_table_identifier.pg_qualified_name}._id"],
             ),
         ]
     return sa.Column(column_name, pkey_type, primary_key=True, comment="_id")

spinta/backends/postgresql/commands/manifest.py

@@ -18,19 +18,18 @@ def load(
     backend: PostgreSQL,
     *,
     into: Manifest = None,
-    freezed: bool = True,
 ) -> None:
     if manifest.backend.bootstrapped():
         if into:
             log.info(
-                'Loading manifest %r into %r from %r backend.',
+                "Loading manifest %r into %r from %r backend.",
                 manifest.name,
                 into.name,
                 manifest.backend.name,
             )
         else:
             log.info(
-                'Loading manifest %r from %r backend.',
+                "Loading manifest %r from %r backend.",
                 manifest.name,
                 manifest.backend.name,
             )
@@ -49,12 +48,9 @@ def load(
         )
 
         target = into or manifest
-        if not commands.has_model(context, target, '_schema'):
-            store = context.get('store')
+        if not commands.has_model(context, target, "_schema"):
+            store = context.get("store")
             commands.load(context, store.internal, into=target)
 
         for source in manifest.sync:
-            commands.load(context, source, into=into or manifest, freezed=freezed)
-
-
-
+            commands.load(context, source, into=into or manifest)

spinta/backends/postgresql/commands/migrate/constants.py

@@ -1 +1 @@
-EXCLUDED_MODELS = "spatial_ref_sys"
+EXCLUDED_MODELS = ["spatial_ref_sys"]