spinta 0.2.dev18__py3-none-any.whl → 0.2.dev19__py3-none-any.whl

spinta/__init__.py

@@ -1,3 +1,5 @@
 import importlib.metadata
 
 __version__ = importlib.metadata.version(__name__)
+
+HTTP_URL_PREFIXES = ("http://", "https://")

spinta/backends/postgresql/commands/summary.py

@@ -6,7 +6,7 @@ from spinta.backends.helpers import get_table_name
 from spinta.backends.postgresql.helpers.name import get_pg_table_name, get_pg_column_name
 from spinta.core.ufuncs import Expr
 from spinta.types.datatype import Integer, Number, Boolean, String, Date, DateTime, Time, Ref
-from spinta import commands
+from spinta import commands, HTTP_URL_PREFIXES
 from spinta.components import Context, Property
 from spinta.components import Model
 from spinta.exceptions import NotFoundError, NotImplementedFeature, InvalidRequestQuery
@@ -322,7 +322,7 @@ def summary(context: Context, dtype: Ref, backend: PostgreSQL, **kwargs):
         prefixes = dtype.model.external.dataset.prefixes
         label = None
         if uri and ":" in uri:
-            if uri.startswith(("http://", "https://")):
+            if uri.startswith(HTTP_URL_PREFIXES):
                 label = uri
             else:
                 split = uri.split(":")

spinta/manifests/dict/components.py

@@ -1,7 +1,8 @@
+import dataclasses
 from enum import Enum
 
-
 from spinta.manifests.components import Manifest
+from spinta.manifests.helpers import TypeDetector
 
 
 class DictFormat(Enum):
@@ -30,3 +31,103 @@ class XmlManifest(DictManifest):
     @staticmethod
     def detect_from_path(path: str) -> bool:
         return path.endswith(".xml")
+
+
+@dataclasses.dataclass
+class _MappedProperties:
+    name: str
+    source: str
+    extra: str
+    type_detector: TypeDetector
+
+
+@dataclasses.dataclass
+class _MappedModels:
+    name: str
+    source: str
+    properties: dict[str, _MappedProperties]
+
+
+@dataclasses.dataclass
+class _MappedDataset:
+    dataset: str
+    resource: str
+    models: dict[str, dict[str, _MappedModels]]
+
+
+@dataclasses.dataclass
+class _MappingMeta:
+    is_blank_node: bool
+    blank_node_name: str
+    blank_node_source: str
+    seperator: str
+    recursive_descent: str
+    remove_array_suffix: bool
+    model_source_prefix: str
+    check_namespace: bool
+    namespace_prefixes: dict[str, list[str]]
+    namespace_seperator: str
+
+    @classmethod
+    def get_for(cls, manifest_type: DictFormat) -> "_MappingMeta":
+        if manifest_type == DictFormat.JSON:
+            mapping_meta = _MappingMeta.for_json()
+        elif manifest_type in (DictFormat.XML, DictFormat.HTML):
+            mapping_meta = _MappingMeta.for_xml()
+        else:
+            mapping_meta = _MappingMeta.default()
+
+        return mapping_meta
+
+    @classmethod
+    def for_json(cls) -> "_MappingMeta":
+        return cls(
+            is_blank_node=False,
+            blank_node_name="model1",
+            blank_node_source=".",
+            seperator=".",
+            recursive_descent=".",
+            model_source_prefix="",
+            namespace_seperator=":",
+            remove_array_suffix=False,
+            check_namespace=False,
+            namespace_prefixes={},
+        )
+
+    @classmethod
+    def for_xml(cls) -> "_MappingMeta":
+        return cls(
+            is_blank_node=False,
+            blank_node_name="model1",
+            blank_node_source=".",
+            seperator="/",
+            recursive_descent="/..",
+            model_source_prefix="/",
+            namespace_seperator=":",
+            remove_array_suffix=True,
+            check_namespace=True,
+            namespace_prefixes={"xmlns": ["xmlns", "@xmlns"]},
+        )
+
+    @classmethod
+    def default(cls) -> "_MappingMeta":
+        return cls(
+            is_blank_node=False,
+            blank_node_name="model1",
+            blank_node_source=".",
+            seperator="",
+            recursive_descent="",
+            model_source_prefix="",
+            namespace_seperator=":",
+            remove_array_suffix=False,
+            check_namespace=False,
+            namespace_prefixes={},
+        )
+
+
+@dataclasses.dataclass
+class _MappingScope:
+    parent_scope: str
+    model_scope: str
+    model_name: str
+    property_name: str

spinta/manifests/dict/helpers.py

@@ -4,70 +4,59 @@ from copy import deepcopy
 
 import requests
 import xmltodict
-from typing import List, Dict, Union, TypedDict, Any, Tuple, Callable
-
-from spinta.manifests.dict.components import DictFormat
+from typing import Any, Iterator
+
+from spinta import HTTP_URL_PREFIXES
+from spinta.manifests.components import ManifestSchema
+from spinta.manifests.dict.components import (
+    DictFormat,
+    _MappedDataset,
+    _MappingMeta,
+    _MappingScope,
+    _MappedModels,
+    _MappedProperties,
+)
 from spinta.manifests.helpers import TypeDetector
 from spinta.utils.itertools import first_dict_value, first_dict_key
 from spinta.utils.naming import Deduplicator, to_model_name, to_property_name
 
 
-def read_schema(manifest_type: DictFormat, path: str, dataset_name: str):
-    if path.startswith(("http://", "https://")):
+def read_schema(manifest_type: DictFormat, path: str, dataset_name: str) -> Iterator[ManifestSchema]:
+    if path.startswith(HTTP_URL_PREFIXES):
         value = requests.get(path).text
     else:
         with pathlib.Path(path).open(encoding="utf-8-sig") as f:
             value = f.read()
 
     converted = {}
-    mapping_meta = _MappingMeta(
-        is_blank_node=False,
-        blank_node_name="model1",
-        blank_node_source=".",
-        seperator="",
-        recursive_descent="",
-        model_source_prefix="",
-        namespace_seperator=":",
-        remove_array_suffix=False,
-        check_namespace=False,
-        namespace_prefixes={},
-        detect_model=is_model,
-    )
+    mapping_meta = _MappingMeta.get_for(manifest_type)
+
     if manifest_type == DictFormat.JSON:
         converted = json.loads(value)
-        mapping_meta["seperator"] = "."
-        mapping_meta["recursive_descent"] = "."
-
     elif manifest_type in (DictFormat.XML, DictFormat.HTML):
         converted = xmltodict.parse(value, cdata_key="text()")
-        mapping_meta["seperator"] = "/"
-        mapping_meta["recursive_descent"] = "/.."
-        mapping_meta["model_source_prefix"] = "/"
-        mapping_meta["remove_array_suffix"] = True
-        mapping_meta["check_namespace"] = True
-        mapping_meta["namespace_prefixes"] = {"xmlns": ["xmlns", "@xmlns"]}
 
     namespaces = list(extract_namespaces(converted, mapping_meta))
     converted = _fix_for_blank_nodes(converted)
     prefixes = {}
     for i, (key, value) in enumerate(namespaces):
         prefixes[key] = {"type": "prefix", "name": key, "uri": value, "eid": i}
-    dataset_structure: _MappedDataset = {
-        "dataset": dataset_name if dataset_name else "dataset",
-        "resource": "resource",
-        "models": {},
-    }
-    mapping_meta["is_blank_node"] = is_blank_node(converted)
+    dataset_structure = _MappedDataset(
+        dataset=dataset_name if dataset_name else "dataset",
+        resource="resource",
+        models={},
+    )
+    mapping_meta.is_blank_node = is_blank_node(converted)
     create_type_detectors(dataset_structure, converted, mapping_meta)
 
     yield (
         None,
         {
             "type": "dataset",
-            "name": dataset_structure["dataset"],
+            "name": dataset_structure.dataset,
             "prefixes": prefixes,
             "resources": {
-                dataset_structure["resource"]: {
+                dataset_structure.resource: {
                     "type": f"dask/{manifest_type.value}",
                     "external": path,
                 },
@@ -78,38 +67,38 @@ def read_schema(manifest_type: DictFormat, path: str, dataset_name: str):
 
     dedup_model = Deduplicator("{}")
     blank_model = ""
-    mapped_models: Dict[Tuple, str] = {}
+    mapped_models: dict[tuple, str] = {}
 
-    for model in dataset_structure["models"].values():
+    for model in dataset_structure.models.values():
         for model_source, m in model.items():
-            new_model = to_model_name(_name_without_namespace(m["name"], mapping_meta, prefixes))
+            new_model = to_model_name(_name_without_namespace(m.name, mapping_meta, prefixes))
             new_model = dedup_model(new_model)
-            mapped_models[(m["name"], model_source)] = new_model
+            mapped_models[(m.name, model_source)] = new_model
 
-    for model in dataset_structure["models"].values():
+    for model in dataset_structure.models.values():
         for model_source, m in model.items():
-            if model_source == mapping_meta["blank_node_source"]:
-                blank_model = mapped_models[(m["name"], model_source)]
+            if model_source == mapping_meta.blank_node_source:
+                blank_model = mapped_models[(m.name, model_source)]
             dedup_prop = Deduplicator("_{}")
             converted_props = {}
-            for prop in m["properties"].values():
-                new_prop = to_property_name(_name_without_namespace(prop["name"], mapping_meta, prefixes))
+            for prop in m.properties.values():
+                new_prop = to_property_name(_name_without_namespace(prop.name, mapping_meta, prefixes))
                 new_prop = dedup_prop(new_prop)
                 extra = {}
-                type_detector = prop["type_detector"]
+                type_detector = prop.type_detector
                 prop_type = type_detector.get_type()
                 if prop_type == "ref":
-                    if prop["name"] in dataset_structure["models"].keys():
+                    if prop.name in dataset_structure.models.keys():
                         model_name = _name_without_namespace(
-                            mapped_models[prop["name"], prop["extra"]], mapping_meta, prefixes
+                            mapped_models[prop.name, prop.extra], mapping_meta, prefixes
                         )
-                        ref_model = f"{dataset_structure['dataset']}/{model_name}"
+                        ref_model = f"{dataset_structure.dataset}/{model_name}"
                     else:
-                        ref_model = f"{dataset_structure['dataset']}/{blank_model}"
+                        ref_model = f"{dataset_structure.dataset}/{blank_model}"
                     extra = {"model": ref_model}
                 converted_props[new_prop] = {
                     "type": prop_type,
-                    "external": {"name": prop["source"]},
+                    "external": {"name": prop.source},
                     "description": "",
                     "required": type_detector.required,
                     "unique": type_detector.unique,
@@ -122,19 +111,19 @@ def read_schema(manifest_type: DictFormat, path: str, dataset_name: str):
                     copied["given_name"] = f"{new_prop}[]"
                     converted_props[new_prop]["items"] = copied
 
-            fixed_external_source = m["source"]
-            if mapping_meta["remove_array_suffix"]:
+            fixed_external_source = m.source
+            if mapping_meta.remove_array_suffix:
                 fixed_external_source = fixed_external_source.replace("[]", "")
             if not fixed_external_source.startswith("."):
-                fixed_external_source = f"{mapping_meta['model_source_prefix']}{fixed_external_source}"
+                fixed_external_source = f"{mapping_meta.model_source_prefix}{fixed_external_source}"
             yield (
                 None,
                 {
                     "type": "model",
-                    "name": f"{dataset_structure['dataset']}/{mapped_models[(m['name'], model_source)]}",
+                    "name": f"{dataset_structure.dataset}/{mapped_models[(m.name, model_source)]}",
                     "external": {
-                        "dataset": dataset_structure["dataset"],
-                        "resource": dataset_structure["resource"],
+                        "dataset": dataset_structure.dataset,
+                        "resource": dataset_structure.resource,
                         "name": fixed_external_source,
                     },
                     "description": "",
@@ -147,80 +136,43 @@ def is_single_item_dict(value: dict) -> bool:
     return len(value) == 1
 
 
-def _fix_for_blank_nodes(values: Any):
-    return_values = values
-    if isinstance(values, dict) and is_single_item_dict(values):
-        result = {}
-        temp_dict = values
-        while is_single_item_dict(temp_dict):
-            first_key = first_dict_key(temp_dict)
-            temp_dict = temp_dict[first_key]
-
-            if not isinstance(temp_dict, dict):
-                return return_values
-
-            result[first_key] = temp_dict if is_single_item_dict(temp_dict) else [temp_dict]
+def _fix_for_blank_nodes(values: Any) -> Any:
+    if not (isinstance(values, dict) and is_single_item_dict(values)):
+        return values
 
-        return_values = result
-    return return_values
-
-
-class _MappedProperties(TypedDict):
-    name: str
-    source: str
-    extra: str
-    type_detector: TypeDetector
-
-
-class _MappedModels(TypedDict):
-    name: str
-    source: str
-    properties: Dict[str, _MappedProperties]
+    return_values = values
+    temp_dict = values
+    result = {}
 
+    while is_single_item_dict(temp_dict):
+        first_key = first_dict_key(temp_dict)
+        temp_dict = temp_dict[first_key]
 
-class _MappedDataset(TypedDict):
-    dataset: str
-    resource: str
-    models: Dict[str, Dict[str, _MappedModels]]
+        if not isinstance(temp_dict, dict):
+            return return_values
 
+        result[first_key] = temp_dict if is_single_item_dict(temp_dict) else [temp_dict]
 
-class _MappingMeta(TypedDict):
-    is_blank_node: bool
-    blank_node_name: str
-    blank_node_source: str
-    seperator: str
-    recursive_descent: str
-    remove_array_suffix: bool
-    model_source_prefix: str
-    check_namespace: bool
-    namespace_prefixes: dict
-    namespace_seperator: str
-    detect_model: Callable
+    return_values = result
+    return return_values
 
 
-class _MappingScope(TypedDict):
-    parent_scope: str
-    model_scope: str
-    model_name: str
-    property_name: str
+def _name_without_namespace(name: str, mapping_meta: _MappingMeta, prefixes: dict) -> str:
+    if mapping_meta.namespace_seperator not in name:
+        return name
 
+    for prefix in prefixes.keys():
+        if (namespace := f"{prefix}{mapping_meta.namespace_seperator}") in name:
+            return name.replace(namespace, "")
 
-def _name_without_namespace(name: str, mapping_meta: _MappingMeta, prefixes: dict):
-    if mapping_meta["namespace_seperator"] in name:
-        for prefix in prefixes.keys():
-            namespace = f"{prefix}{mapping_meta['namespace_seperator']}"
-            if namespace in name:
-                return name.replace(namespace, "")
     return name
 
 
-def is_model(data):
-    if isinstance(data, list) and is_list_of_dicts(data):
-        return True
-    return False
+def is_model(data: Any) -> bool:
+    return isinstance(data, list) and is_list_of_dicts(data)
 
 
-def _find_parent_key(data, string):
+def _find_parent_key(data: dict[str, list[str]], string: str) -> str | None:
     for key, values in data.items():
         for value in values:
             if string.startswith(value):
@@ -228,29 +180,33 @@ def _find_parent_key(data, string):
     return None
 
 
-def extract_namespaces(data: Any, mapping_meta: _MappingMeta):
-    if mapping_meta["check_namespace"]:
-        if isinstance(data, dict):
-            keys_to_remove = {}
-            for key, value in data.items():
-                parent = _find_parent_key(mapping_meta["namespace_prefixes"], key)
-                if parent is not None:
-                    if key not in keys_to_remove.keys():
-                        return_key = key.split(mapping_meta["namespace_seperator"])
-                        if len(return_key) == 1:
-                            return_key = [parent]
-                        keys_to_remove[key] = (return_key[-1], value)
-            for key, value in keys_to_remove.items():
-                del data[key]
-                yield value
-            for value in data.values():
-                yield from extract_namespaces(value, mapping_meta)
-        elif isinstance(data, list):
-            for item in data:
-                yield from extract_namespaces(item, mapping_meta)
-
-
-def nested_prop_names(new_values: list, values: dict, root: str, seperator: str):
+def extract_namespaces(data: Any, mapping_meta: _MappingMeta) -> Iterator[Any]:
+    if not mapping_meta.check_namespace:
+        return
+
+    if isinstance(data, dict):
+        keys_to_remove = {}
+        for key, value in data.items():
+            parent = _find_parent_key(mapping_meta.namespace_prefixes, key)
+            if parent is not None and key not in keys_to_remove.keys():
+                return_key = key.split(mapping_meta.namespace_seperator)
+                if len(return_key) == 1:
+                    return_key = [parent]
+                keys_to_remove[key] = (return_key[-1], value)
+
+        for key, value in keys_to_remove.items():
+            del data[key]
+            yield value
+
+        for value in data.values():
+            yield from extract_namespaces(value, mapping_meta)
+
+    elif isinstance(data, list):
+        for item in data:
+            yield from extract_namespaces(item, mapping_meta)
+
+
+def nested_prop_names(new_values: list, values: dict, root: str, seperator: str) -> None:
     for key, value in values.items():
         if isinstance(value, dict):
             nested_prop_names(new_values, value, f"{root}{seperator}{key}", seperator)
@@ -263,136 +219,150 @@ def nested_prop_names(new_values: list, values: dict, root: str, seperator: str)
 
 def check_missing_prop_required(
     dataset: _MappedDataset, values: dict, mapping_scope: _MappingScope, mapping_meta: _MappingMeta
-):
-    if mapping_scope["model_scope"] == "":
-        model_name = (
-            mapping_scope["model_name"] if mapping_scope["model_name"] != "" else first_dict_key(dataset["models"])
-        )
-        model_source = _create_name_with_prefix(mapping_scope["parent_scope"], mapping_meta["seperator"], model_name)
-        if mapping_scope["model_name"] == "" and mapping_meta["is_blank_node"]:
-            model_name = mapping_meta["blank_node_name"]
-            model_source = mapping_meta["blank_node_source"]
-        key_values = []
-        for k, v in values.items():
-            new_val = _create_name_with_prefix(mapping_scope["model_scope"], mapping_meta["seperator"], k)
-            if isinstance(v, dict):
-                nested_prop_names(key_values, v, new_val, mapping_meta["seperator"])
-            elif isinstance(v, list):
-                if not is_list_of_dicts(v):
-                    key_values.append(new_val)
-            else:
-                key_values.append(new_val)
-        if key_values:
-            filtered_models = dataset["models"][model_name][model_source]["properties"].keys()
-            subtracted = set(filtered_models) - set(key_values)
-            for prop in subtracted:
-                type_detector = dataset["models"][model_name][model_source]["properties"][prop]["type_detector"]
-                if type_detector.required and type_detector.type != "ref":
-                    type_detector.required = False
+) -> None:
+    if mapping_scope.model_scope != "":
+        return
+
+    model_name = mapping_scope.model_name if mapping_scope.model_name != "" else first_dict_key(dataset.models)
+    model_source = _create_name_with_prefix(mapping_scope.parent_scope, mapping_meta.seperator, model_name)
+    if mapping_scope.model_name == "" and mapping_meta.is_blank_node:
+        model_name = mapping_meta.blank_node_name
+        model_source = mapping_meta.blank_node_source
 
+    key_values = []
+    for key, value in values.items():
+        new_value = _create_name_with_prefix(mapping_scope.model_scope, mapping_meta.seperator, key)
+        if isinstance(value, dict):
+            nested_prop_names(key_values, value, new_value, mapping_meta.seperator)
+        elif isinstance(value, list):
+            if not is_list_of_dicts(value):
+                key_values.append(new_value)
+        else:
+            key_values.append(new_value)
+
+    if not key_values:
+        return
+
+    filtered_models = dataset.models[model_name][model_source].properties.keys()
+    subtracted = set(filtered_models) - set(key_values)
+    for prop in subtracted:
+        type_detector = dataset.models[model_name][model_source].properties[prop].type_detector
+        if type_detector.required and type_detector.type != "ref":
+            type_detector.required = False
 
-def run_type_detectors(dataset: _MappedDataset, values: dict, mapping_scope: _MappingScope, mapping_meta: _MappingMeta):
+
+def run_type_detectors(
+    dataset: _MappedDataset, values: dict, mapping_scope: _MappingScope, mapping_meta: _MappingMeta
+) -> None:
     if isinstance(values, list):
         for item in values:
             run_type_detectors(dataset, item, mapping_scope, mapping_meta)
+
     elif isinstance(values, dict):
         check_missing_prop_required(dataset, values, mapping_scope, mapping_meta)
         for key, value in values.items():
-            new_mapping_scope = _MappingScope(
-                parent_scope=mapping_scope["parent_scope"],
-                model_name=mapping_scope["model_name"],
-                model_scope=mapping_scope["model_scope"],
-                property_name=key,
-            )
-            if mapping_meta["detect_model"](value):
-                new_mapping_scope["model_name"] = key
+            if is_model(value):
                 parent_scope = _create_name_with_prefix(
-                    new_mapping_scope["parent_scope"],
-                    mapping_meta["seperator"],
-                    "" if mapping_scope["model_name"] == "" else f"{mapping_scope['model_name']}[]",
+                    prefix=_create_name_with_prefix(
+                        prefix=mapping_scope.parent_scope,
+                        seperator=mapping_meta.seperator,
+                        value="" if mapping_scope.model_name == "" else f"{mapping_scope.model_name}[]",
+                    ),
+                    seperator=mapping_meta.seperator,
+                    value=mapping_scope.model_scope,
                 )
-                parent_scope = _create_name_with_prefix(
-                    parent_scope, mapping_meta["seperator"], new_mapping_scope["model_scope"]
+                new_mapping_scope = _MappingScope(
+                    parent_scope=parent_scope,
+                    model_name=key,
+                    model_scope="",
+                    property_name=key,
                 )
-                new_mapping_scope["model_scope"] = ""
-                new_mapping_scope["parent_scope"] = parent_scope
                 run_type_detectors(dataset, value, new_mapping_scope, mapping_meta)
             else:
+                new_mapping_scope = _MappingScope(
+                    parent_scope=mapping_scope.parent_scope,
+                    model_name=mapping_scope.model_name,
+                    model_scope=mapping_scope.model_scope,
+                    property_name=key,
+                )
                 if isinstance(value, dict):
-                    new_mapping_scope["model_scope"] = _create_name_with_prefix(
-                        new_mapping_scope["model_scope"], mapping_meta["seperator"], key
+                    new_mapping_scope.model_scope = _create_name_with_prefix(
+                        prefix=new_mapping_scope.model_scope,
+                        seperator=mapping_meta.seperator,
+                        value=key,
                     )
                     run_type_detectors(dataset, value, new_mapping_scope, mapping_meta)
                 else:
                     _detect_type(dataset, new_mapping_scope, mapping_meta, value)
 
 
-def _detect_type(dataset: _MappedDataset, mapping_scope: _MappingScope, mapping_meta: _MappingMeta, value: Any):
-    prop_name = _create_name_with_prefix(
-        mapping_scope["model_scope"], mapping_meta["seperator"], mapping_scope["property_name"]
-    )
-    model_name = mapping_scope["model_name"]
+def _detect_type(dataset: _MappedDataset, mapping_scope: _MappingScope, mapping_meta: _MappingMeta, value: Any) -> None:
+    prop_name = _create_name_with_prefix(mapping_scope.model_scope, mapping_meta.seperator, mapping_scope.property_name)
+    model_name = mapping_scope.model_name
     model_source = _create_name_with_prefix(
-        mapping_scope["parent_scope"], mapping_meta["seperator"], mapping_scope["model_name"]
+        mapping_scope.parent_scope, mapping_meta.seperator, mapping_scope.model_name
     )
     if model_name == "":
-        model_name = mapping_meta["blank_node_name"]
-        model_source = mapping_meta["blank_node_source"]
-    dataset["models"][model_name][model_source]["properties"][prop_name]["type_detector"].detect(value)
+        model_name = mapping_meta.blank_node_name
+        model_source = mapping_meta.blank_node_source
+    dataset.models[model_name][model_source].properties[prop_name].type_detector.detect(value)
 
 
-def is_list_of_dicts(lst: List) -> bool:
-    for item in lst:
-        if not isinstance(item, dict):
-            return False
-    return True
+def is_list_of_dicts(data: list) -> bool:
+    return all(isinstance(item, dict) for item in data)
 
 
 def setup_model_type_detectors(
-    dataset: _MappedDataset, values: Union[dict, list], mapping_scope: _MappingScope, mapping_meta: _MappingMeta
-):
+    dataset: _MappedDataset, values: dict | list, mapping_scope: _MappingScope, mapping_meta: _MappingMeta
+) -> None:
     if isinstance(values, list):
         for item in values:
             setup_model_type_detectors(dataset, item, mapping_scope, mapping_meta)
+
     elif isinstance(values, dict):
         for key, value in values.items():
-            new_mapping_scope = _MappingScope(
-                parent_scope=mapping_scope["parent_scope"],
-                model_name=mapping_scope["model_name"],
-                model_scope=mapping_scope["model_scope"],
-                property_name=key,
-            )
-            if mapping_meta["detect_model"](value):
-                new_mapping_scope["model_name"] = key
+            if is_model(value):
                 parent_scope = _create_name_with_prefix(
-                    new_mapping_scope["parent_scope"],
-                    mapping_meta["seperator"],
-                    "" if mapping_scope["model_name"] == "" else f"{mapping_scope['model_name']}[]",
+                    prefix=_create_name_with_prefix(
+                        prefix=mapping_scope.parent_scope,
+                        seperator=mapping_meta.seperator,
+                        value="" if mapping_scope.model_name == "" else f"{mapping_scope.model_name}[]",
+                    ),
+                    seperator=mapping_meta.seperator,
+                    value=mapping_scope.model_scope,
                 )
-                parent_scope = _create_name_with_prefix(
-                    parent_scope, mapping_meta["seperator"], new_mapping_scope["model_scope"]
-                )
-                new_mapping_scope["model_scope"] = ""
-                new_mapping_scope["parent_scope"] = parent_scope
 
+                new_mapping_scope = _MappingScope(
+                    parent_scope=parent_scope,
+                    model_name=key,
+                    model_scope="",
+                    property_name=key,
+                )
                 setup_model_type_detectors(dataset, value, new_mapping_scope, mapping_meta)
 
-                old_mapping_scope = new_mapping_scope.copy()
-                old_mapping_scope["property_name"] = mapping_scope["property_name"]
+                old_mapping_scope = deepcopy(new_mapping_scope)
+                old_mapping_scope.property_name = mapping_scope.property_name
 
-                if old_mapping_scope["property_name"] == "" and mapping_meta["is_blank_node"]:
-                    old_mapping_scope["property_name"] = "parent"
+                if old_mapping_scope.property_name == "" and mapping_meta.is_blank_node:
+                    old_mapping_scope.property_name = "parent"
                     set_type_detector(dataset, old_mapping_scope, mapping_meta, is_ref=True)
-                elif old_mapping_scope["property_name"] != "":
+                elif old_mapping_scope.property_name != "":
                     set_type_detector(dataset, old_mapping_scope, mapping_meta, is_ref=True)
+
             else:
+                new_mapping_scope = _MappingScope(
+                    parent_scope=mapping_scope.parent_scope,
+                    model_name=mapping_scope.model_name,
+                    model_scope=mapping_scope.model_scope,
+                    property_name=key,
+                )
                 if isinstance(value, list):
                     set_type_detector(dataset, new_mapping_scope, mapping_meta, is_array=True)
                 elif isinstance(value, dict):
-                    new_mapping_scope["model_scope"] = _create_name_with_prefix(
-                        new_mapping_scope["model_scope"], mapping_meta["seperator"], key
+                    new_mapping_scope.model_scope = _create_name_with_prefix(
+                        new_mapping_scope.model_scope, mapping_meta.seperator, key
                     )
-                    new_mapping_scope["property_name"] = mapping_scope["property_name"]
+                    new_mapping_scope.property_name = mapping_scope.property_name
                     setup_model_type_detectors(dataset, value, new_mapping_scope, mapping_meta)
                 else:
                     set_type_detector(dataset, new_mapping_scope, mapping_meta)
@@ -401,18 +371,22 @@ def setup_model_type_detectors(
 def _create_name_with_prefix(prefix: str, seperator: str, value: str) -> str:
     if value == "":
         return prefix
-    return value if prefix == "" else f"{prefix}{seperator}{value}"
+    if prefix == "":
+        return value
+
+    return f"{prefix}{seperator}{value}"
 
 
-def create_type_detectors(dataset: _MappedDataset, values: Any, mapping_meta: _MappingMeta):
+def create_type_detectors(dataset: _MappedDataset, values: Any, mapping_meta: _MappingMeta) -> None:
     mapping_scope = _MappingScope(parent_scope="", model_scope="", model_name="", property_name="")
     setup_model_type_detectors(dataset, values, mapping_scope, mapping_meta)
     run_type_detectors(dataset, values, mapping_scope, mapping_meta)
 
 
-def is_blank_node(values: Union[list, dict]) -> bool:
+def is_blank_node(values: list | dict) -> bool:
     if isinstance(values, list):
         return True
+
     if isinstance(values, dict):
         temp_dict = values
         first_value = first_dict_value(temp_dict)
@@ -429,6 +403,7 @@ def is_blank_node(values: Union[list, dict]) -> bool:
             for key, value in values.items():
                 if not isinstance(value, list):
                     return True
+
     return False
 
 
@@ -438,75 +413,74 @@ def set_type_detector(
     mapping_meta: _MappingMeta,
     is_ref: bool = False,
     is_array: bool = False,
-):
-    if mapping_scope["property_name"] != "":
-        model_name = mapping_scope["model_name"]
-        model_source = _create_name_with_prefix(mapping_scope["parent_scope"], mapping_meta["seperator"], model_name)
-        prop_name = _create_name_with_prefix(
-            mapping_scope["model_scope"], mapping_meta["seperator"], mapping_scope["property_name"]
-        )
-        prop_source = prop_name
-        if model_name == "" and mapping_meta["is_blank_node"]:
-            model_name = mapping_meta["blank_node_name"]
-            model_source = mapping_meta["blank_node_source"]
-        extra = ""
-        if is_ref:
-            extra = ""
-            split = mapping_scope["parent_scope"].split(mapping_meta["seperator"])
-            count = 0
-            for i in reversed(split):
-                if i.endswith("[]"):
-                    half = mapping_scope["parent_scope"].split(i)
-                    extra = i[:-2] if half[0] == "" else f"{half[0]}{i[:-2]}"
-                    break
-                elif i != "":
-                    count += 1
-            prop_source = f"..{mapping_meta['recursive_descent'] * count}"
-
-        if model_name in dataset["models"].keys():
-            if model_source in dataset["models"][model_name].keys():
-                if prop_name not in dataset["models"][model_name][model_source]["properties"].keys():
-                    dataset["models"][model_name][model_source]["properties"][prop_name] = {
-                        "name": prop_name,
-                        "source": prop_source,
-                        "type_detector": TypeDetector(),
-                        "extra": extra,
-                    }
-            else:
-                dataset["models"][model_name][model_source] = {
-                    "name": model_name,
-                    "source": model_source,
-                    "properties": {
-                        prop_name: {
-                            "name": prop_name,
-                            "source": prop_source,
-                            "type_detector": TypeDetector(),
-                            "extra": extra,
-                        }
-                    },
-                }
+) -> None:
+    if mapping_scope.property_name == "":
+        return
+
+    model_name = mapping_scope.model_name
+    model_source = _create_name_with_prefix(mapping_scope.parent_scope, mapping_meta.seperator, model_name)
+    prop_name = _create_name_with_prefix(mapping_scope.model_scope, mapping_meta.seperator, mapping_scope.property_name)
+    prop_source = prop_name
+    if model_name == "" and mapping_meta.is_blank_node:
+        model_name = mapping_meta.blank_node_name
+        model_source = mapping_meta.blank_node_source
+    extra = ""
+    if is_ref:
+        split = mapping_scope.parent_scope.split(mapping_meta.seperator)
+        count = 0
+        for i in reversed(split):
+            if i.endswith("[]"):
+                half = mapping_scope.parent_scope.split(i)
+                extra = i[:-2] if half[0] == "" else f"{half[0]}{i[:-2]}"
+                break
+            elif i != "":
+                count += 1
+        prop_source = f"..{mapping_meta.recursive_descent * count}"
+
+    if model_name in dataset.models.keys():
+        if model_source in dataset.models[model_name].keys():
+            if prop_name not in dataset.models[model_name][model_source].properties.keys():
+                dataset.models[model_name][model_source].properties[prop_name] = _MappedProperties(
+                    name=prop_name,
+                    source=prop_source,
+                    type_detector=TypeDetector(),
+                    extra=extra,
+                )
         else:
-            dataset["models"][model_name] = {
-                model_source: {
-                    "name": model_name,
-                    "source": model_source,
-                    "properties": {
-                        prop_name: {
-                            "name": prop_name,
-                            "source": prop_source,
-                            "type_detector": TypeDetector(),
-                            "extra": extra,
-                        }
-                    },
-                }
-            }
-
-        type_detector = dataset["models"][model_name][model_source]["properties"][prop_name]["type_detector"]
-        if is_array:
-            type_detector.array = True
-            type_detector.unique = False
-            type_detector.required = False
-        if is_ref:
-            type_detector.type = "ref"
-            type_detector.unique = False
-            type_detector.required = False
+            dataset.models[model_name][model_source] = _MappedModels(
+                name=model_name,
+                source=model_source,
+                properties={
+                    prop_name: _MappedProperties(
+                        name=prop_name,
+                        source=prop_source,
+                        type_detector=TypeDetector(),
+                        extra=extra,
+                    )
+                },
+            )
+    else:
+        dataset.models[model_name] = {
+            model_source: _MappedModels(
+                name=model_name,
+                source=model_source,
+                properties={
+                    prop_name: _MappedProperties(
+                        name=prop_name,
+                        source=prop_source,
+                        type_detector=TypeDetector(),
+                        extra=extra,
+                    )
+                },
+            )
+        }
+
+    type_detector = dataset.models[model_name][model_source].properties[prop_name].type_detector
+    if is_array:
+        type_detector.array = True
+        type_detector.unique = False
+        type_detector.required = False
+    if is_ref:
+        type_detector.type = "ref"
+        type_detector.unique = False
+        type_detector.required = False

spinta/manifests/helpers.py

@@ -8,7 +8,7 @@ from typing import Type
 
 import jsonpatch
 
-from spinta import commands
+from spinta import commands, HTTP_URL_PREFIXES
 from spinta.backends.constants import BackendOrigin
 from spinta.backends.helpers import load_backend
 from spinta.components import Model
@@ -324,7 +324,7 @@ def get_manifest_from_type(rc: RawConfig, type_: str) -> Type[Manifest]:
 
 
 def check_manifest_path(manifest: Manifest, path: str) -> None:
-    if not path.startswith(("http://", "https://")) and not pathlib.Path(path).exists():
+    if not path.startswith(HTTP_URL_PREFIXES) and not pathlib.Path(path).exists():
         raise ManifestFileDoesNotExist(manifest, path=path)
 
 
@@ -432,7 +432,7 @@ class TypeDetector:
         self.type = new_type
 
     def _assert_url(self, value: str):
-        if value.startswith(("http://", "https://")):
+        if value.startswith(HTTP_URL_PREFIXES):
             self.type = "url"
         else:
             self.type = ""

spinta/manifests/sql/helpers.py

@@ -1,4 +1,5 @@
 import dataclasses
+import logging
 from operator import itemgetter
 from typing import Any, TypedDict
 from typing import Dict
@@ -28,6 +29,9 @@ from spinta.utils.naming import to_model_name
 from spinta.utils.naming import to_property_name
 
 
+logger = logging.getLogger(__name__)
+
+
 def read_schema(context: Context, path: str, prepare: str = None, dataset_name: str = ""):
     engine = sa.create_engine(path)
     schema = None
@@ -236,6 +240,7 @@ def _read_props(
         )
 
 
+UNKNOWN_TYPE = "UNKNOWN"
 TYPES = [
     (sa.Boolean, "boolean"),
     (sa.Date, "date"),
@@ -273,12 +278,13 @@ class _Column(TypedDict):
     type: TypeEngine
 
 
-def _get_column_type(column: _Column, table: str = None) -> str:
+def _get_column_type(column: _Column, table: str | None = None) -> str:
     column_type: TypeEngine = column["type"]
     for cls, name in TYPES:
         if isinstance(column_type, cls):
             return name
-    raise TypeError(f"Unknown type {column_type!r} of column {column!r} in table {table!r}.")
+    logger.warning(f"Unknown type {column_type!r} of column {column!r} in table {table!r}. Column will be skipped.")
+    return UNKNOWN_TYPE
 
 
 class _Ref(NamedTuple):

spinta/nodes.py

@@ -14,6 +14,7 @@ from spinta.components import Context
 from spinta.components import EntryId
 from spinta.components import Node
 from spinta.manifests.components import Manifest
+from spinta.manifests.sql.helpers import UNKNOWN_TYPE
 from spinta.manifests.tabular.constants import DataTypeEnum
 from spinta.utils.schema import NA
 from spinta.utils.schema import resolve_schema
@@ -221,6 +222,9 @@ def load_model_properties(
     model.leafprops = {}
     model.properties = {}
     for name, params in data.items():
+        # Do not load properties with unmapped types.
+        if params["type"] == UNKNOWN_TYPE:
+            continue
         prop = Prop()
         prop.name = name
         prop.place = name

spinta/testing/pytest.py

@@ -97,8 +97,12 @@ def mongo(rc):
     if dsn and db:
         import pymongo
 
-        client = pymongo.MongoClient(dsn)
-        client.drop_database(db)
+        try:
+            client = pymongo.MongoClient(dsn, serverSelectionTimeoutMS=2000)
+            client.server_info()  # force connection check
+            client.drop_database(db)
+        except pymongo.errors.ServerSelectionTimeoutError:
+            pass  # MongoDB not running, nothing to clean up
 
 
 @pytest.fixture(scope="session")

{spinta-0.2.dev18.dist-info → spinta-0.2.dev19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev18
+Version: 0.2.dev19
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev18.dist-info → spinta-0.2.dev19.dist-info}/RECORD

@@ -1,4 +1,4 @@
-spinta/__init__.py,sha256=IILIKzO2OO60Rhsh3SqhdKHDf7XI_RLRwzztBGZZmHY,78
+spinta/__init__.py,sha256=OLVp_IMRnqoX_Bxzh2exv1p4N0NSTPTBVVjhX74qFCU,123
 spinta/__main__.py,sha256=3n1t_DsFmpQZyAtqPrfXyHAPjpBoS-Oj9hPPFSi2IPw,70
 spinta/accesslog/__init__.py,sha256=svgEqf0U1ykvxQ61pRuAzadwdn7TnQriQ4a9A_aMoKI,5938
 spinta/accesslog/file.py,sha256=h1KcGsbIPoI2ZzXXQ2b8eZFOP0EvkTudfWDmwy25zlg,2150
@@ -107,7 +107,7 @@ spinta/backends/postgresql/commands/migrate/types/string.py,sha256=C2jo9AH8YTLlc
 spinta/backends/postgresql/commands/migrate/types/text.py,sha256=E8Pa4GpCIq2JLNRxkkF5YIs4hfpqy4AVQgGXAfplzoI,5767
 spinta/backends/postgresql/commands/read.py,sha256=h_kMFdb3-wl5i-3yh6b4dr-09GVNgUos65NDWMf0Igk,2908
 spinta/backends/postgresql/commands/redirect.py,sha256=nc1cHobA3JsmXn3h5otKNIU-a1us3ouTNKjhr6N7zMk,1810
-spinta/backends/postgresql/commands/summary.py,sha256=opYAn1cD_VIsgcM-YGcT1xJj6AaQCBuovQm5VPEHh24,17572
+spinta/backends/postgresql/commands/summary.py,sha256=bwbLVMB1MVAHpGrRRLy38PpNedUt-se3AHx4pdhQjX4,17585
 spinta/backends/postgresql/commands/validate.py,sha256=Q6NEK3vIkd1k4jLg9pGRKKw6PfelS52m-BORGg4nO_Y,615
 spinta/backends/postgresql/commands/wait.py,sha256=Yr_NF3hprX-KhKAqRcddrnRJZazsCWud3yYYI1fi2yg,647
 spinta/backends/postgresql/commands/wipe.py,sha256=o0mZuYVr5xbQfO3lqxpBcVtiZE0FqOJyAm738XGf36Q,2279
@@ -522,9 +522,9 @@ spinta/manifests/dict/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
 spinta/manifests/dict/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/dict/commands/configure.py,sha256=P74uhCZKdr6nc70gBBkr1tvsrv5JjKowMx7QQMwzJVA,537
 spinta/manifests/dict/commands/load.py,sha256=eILuiDz5mSrB93fdxseYl8gZNUtUXwVnQJQ06CatrTg,1417
-spinta/manifests/dict/components.py,sha256=9G3tBzyf2JdvJVQoukQqSI-0Q-Idf7YLauhYzS4XItc,646
-spinta/manifests/dict/helpers.py,sha256=v8Zwa-87INdHIF50Bw5XmddzojAOLUhr5BE-34gH0_E,20994
-spinta/manifests/helpers.py,sha256=qCwauueoCzIwck1s4njqBXQrogk4bonot9WOX-foifE,14611
+spinta/manifests/dict/components.py,sha256=fY7IKoArTrPhKaMukwXYP3IfYdJETHRzhUaNrAt4xWs,3281
+spinta/manifests/dict/helpers.py,sha256=AzLQMMdvwvcAb7XWgmJtvDgVEyazN0GgPS9A6DRX3oo,19235
+spinta/manifests/helpers.py,sha256=k_QBpWeWeMTIA9oSN-y5QjUcwJV8-CEL1fVJCGqAKGI,14618
 spinta/manifests/internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/internal/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/internal/commands/configure.py,sha256=pNSEwita8LCnn1r7Xs6BiijEd9LW8kqa24oIjeDAP-Y,462
@@ -567,7 +567,7 @@ spinta/manifests/sql/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
 spinta/manifests/sql/commands/configure.py,sha256=j_ypYAV9o0pTS_qazw1oBCuRohSXfGPs_tmrDUIEVNk,539
 spinta/manifests/sql/commands/load.py,sha256=xHdoo7dWHZ9RtCUo7J0R89DyFFFUKG7XV-0vVgE9SHo,1806
 spinta/manifests/sql/components.py,sha256=lG44u_ufsqDm3kjP8_zshYg87J5S_UVzYWJ5Ea74G1w,661
-spinta/manifests/sql/helpers.py,sha256=doJ7XAXCUFrhc0JWbhRGLFeDVVDz-wLRWgtYw5GQZ5s,14720
+spinta/manifests/sql/helpers.py,sha256=LgnWWU4P2UMVC9zp0Dpwk_5ZnLCXQnlPWv4Jp8aApk8,14853
 spinta/manifests/tabular/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/tabular/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/tabular/commands/bootstrap.py,sha256=hdINMkcwBKXlpKPPXXJqdMjWpLb5WEigf-RfEFcEUkE,438
@@ -610,7 +610,7 @@ spinta/naming/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/naming/components.py,sha256=TnuJg3fB9ftLOV6C2QI2lI-e3_OhlFdpI-qWW0wDeS4,210
 spinta/naming/helpers.py,sha256=tmey39J56_4YQ-jvsVInon4NJkyzM6i91hN_RMtt9kM,4599
 spinta/naming/ufuncts.py,sha256=gRGgyt_jjqn0-QHLc7BWGtJRb4qdaBCc8I90mdzJeYc,1334
-spinta/nodes.py,sha256=1SU_vvZB0P67-waZFs46BYuD9AB5nKgQie1mmUS_bjc,7898
+spinta/nodes.py,sha256=UQJU0rQK2avVUkvQ6oXz3GQkZIvVAd0zaXJQpp5XTlg,8070
 spinta/renderer.py,sha256=D0hlDOFFdjfjZ4x6WV9pK5Lozs0fbpLYd-IcOmtNoR0,761
 spinta/spyna.py,sha256=lwBjhpxKoZtnEMmIvvGlCy0yLKfqmU575F-Axr2K5t4,11528
 spinta/templates/base.html,sha256=tRMf48yngJ-zPDP9mP-Q9TkQW4VXIXkdqpodzwFGF0w,2092
@@ -629,7 +629,7 @@ spinta/testing/dtypes.py,sha256=5qGBuCjDJywTJSvJmvA62DvY8H-dJFti441wtGkrvO0,6288
 spinta/testing/manifest.py,sha256=HCrgCu9QMry_iNtuGkQI7-hp6m5xmtSMUTzwpW0Z6y8,6121
 spinta/testing/migration.py,sha256=q-fQqjTtTZKdJKagYFYAspkGQHhenYoO4O2x-Amamdw,7979
 spinta/testing/push.py,sha256=t_2PpWQI-95_rNUhvBZTHtOQhaIpTO0nXHd5nigI6mM,531
-spinta/testing/pytest.py,sha256=NwCXiX36V2YxZ_1Hbg38ewj77fKEaobX1Gcf5d26C98,10261
+spinta/testing/pytest.py,sha256=ouXo_G8N3IEGAQZ74YlJhAzW7KPbv-20_6a0bc1OeXY,10492
 spinta/testing/request.py,sha256=vJ5uJASCE21w9WFwvYaB5zDHfL3d744LNwLi6kk8fXQ,2605
 spinta/testing/tabular.py,sha256=_AEi6Lf_rcIlWO-55XfAYx-1sdHSBtyFMlsXGg1C8DE,1114
 spinta/testing/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -748,8 +748,8 @@ spinta/utils/tree.py,sha256=iF8eOSBagUoDmdJGNQgsYB_gshsFajmiUALXqU9luHE,591
 spinta/utils/types.py,sha256=lfYSxKGPuPeUsO14d2OYodtbRY3zsa-o-z8HveVH3t0,801
 spinta/utils/units.py,sha256=CFFLv1NHYsoSSzwiar3zXYmt4m3sccW5niUgkZQgo3k,747
 spinta/utils/url.py,sha256=b6sqQEpmCdT3oV4vGDzXnN8w415InAYjIW_o2djhQS8,2950
-spinta-0.2.dev18.dist-info/METADATA,sha256=j6sJ8LCHnWqtnSLRfX9YEDVBmms2np9CdPF9rglQvO0,10207
-spinta-0.2.dev18.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-spinta-0.2.dev18.dist-info/entry_points.txt,sha256=-jdsOQZcMu3rUOwgIJNS3gZS4rwWPACuXXy128F676w,46
-spinta-0.2.dev18.dist-info/licenses/LICENSE,sha256=JKmjfBLapeFWNI_qdVr5bXGlsuMPa6nRarKPK5davKw,1071
-spinta-0.2.dev18.dist-info/RECORD,,
+spinta-0.2.dev19.dist-info/METADATA,sha256=uWFAtzdrYe0mMg-4oVZH6LKnB9hB8aG-cdhgK62AqIY,10207
+spinta-0.2.dev19.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+spinta-0.2.dev19.dist-info/entry_points.txt,sha256=-jdsOQZcMu3rUOwgIJNS3gZS4rwWPACuXXy128F676w,46
+spinta-0.2.dev19.dist-info/licenses/LICENSE,sha256=JKmjfBLapeFWNI_qdVr5bXGlsuMPa6nRarKPK5davKw,1071
+spinta-0.2.dev19.dist-info/RECORD,,