spinta 0.2.dev17__tar.gz → 0.2.dev19__tar.gz

{spinta-0.2.dev17 → spinta-0.2.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev17
+Version: 0.2.dev19
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev17 → spinta-0.2.dev19}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev17"
+version = "0.2dev19"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

{spinta-0.2.dev17 → spinta-0.2.dev19}/spinta/__init__.py

@@ -1,3 +1,5 @@
 import importlib.metadata
 
 __version__ = importlib.metadata.version(__name__)
+
+HTTP_URL_PREFIXES = ("http://", "https://")

{spinta-0.2.dev17 → spinta-0.2.dev19}/spinta/backends/postgresql/commands/summary.py

@@ -6,7 +6,7 @@ from spinta.backends.helpers import get_table_name
 from spinta.backends.postgresql.helpers.name import get_pg_table_name, get_pg_column_name
 from spinta.core.ufuncs import Expr
 from spinta.types.datatype import Integer, Number, Boolean, String, Date, DateTime, Time, Ref
-from spinta import commands
+from spinta import commands, HTTP_URL_PREFIXES
 from spinta.components import Context, Property
 from spinta.components import Model
 from spinta.exceptions import NotFoundError, NotImplementedFeature, InvalidRequestQuery
@@ -322,7 +322,7 @@ def summary(context: Context, dtype: Ref, backend: PostgreSQL, **kwargs):
         prefixes = dtype.model.external.dataset.prefixes
         label = None
         if uri and ":" in uri:
-            if uri.startswith(("http://", "https://")):
+            if uri.startswith(HTTP_URL_PREFIXES):
                 label = uri
             else:
                 split = uri.split(":")

{spinta-0.2.dev17 → spinta-0.2.dev19}/spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py

@@ -15,7 +15,7 @@ from spinta.datasets.backends.dataframe.ufuncs.query.components import (
 )
 from spinta.datasets.components import Param
 from spinta.datasets.utils import iterparams
-from spinta.exceptions import PropertyNotFound, NotImplementedFeature, SourceCannotBeList
+from spinta.exceptions import PropertyNotFound, NotImplementedFeature, SourceCannotBeList, SourceOrPrepareNotAllowed
 from spinta.types.datatype import DataType, PrimaryKey, Ref
 from spinta.types.text.components import Text
 from spinta.ufuncs.components import ForeignProperty
@@ -188,9 +188,12 @@ def select(env: DaskDataFrameQueryBuilder, prop: Property) -> Selected:
             #      tag:resolving_private_properties_in_prepare_context
             result = env.call("select", prop.dtype, result)
         elif prop.external.prepare is not NA:
-            # property without external name and with `prepare` is already evaluated
-            # so just use evaluated value
-            result = Selected(prop=prop, prep=prop.external.prepare)
+            # property without external name may be evaluated already, if it is use the value
+            if isinstance(prop.external.prepare, Expr):
+                result = env(this=prop).resolve(prop.external.prepare)
+                result = env.call("select", prop.dtype, result)
+            else:
+                result = Selected(prop=prop, prep=prop.external.prepare)
         elif prop.external and prop.external.name:
             # If prepare is not given, then take value from `source`.
             result = env.call("select", prop.dtype)
@@ -339,7 +342,7 @@ def select(
 ) -> Selected:
     # TODO need join for this to work
     return Selected(
-        item=dtype.prop.name,
+        item=dtype.prop.external.name,
         prop=dtype.prop,
     )
 
@@ -458,3 +461,13 @@ def eval_(env: DaskDataFrameQueryBuilder, param: Param) -> Iterator[str]:
     )
 
     return resolved_values
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, Bind, Bind, name="getattr")
+def getattr_(env: DaskDataFrameQueryBuilder, obj: Bind, attr: Bind) -> Any:
+    return GetAttr(obj, attr)
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, Bind, Bind, Bind, name="getattr")
+def getattr_(env: DaskDataFrameQueryBuilder, source: Bind, obj: Bind, attr: Bind) -> Any:
+    raise SourceOrPrepareNotAllowed(source=str(source))

{spinta-0.2.dev17 → spinta-0.2.dev19}/spinta/exceptions.py

@@ -1215,3 +1215,9 @@ class EnumPrepareMissing(UserError):
     template = """
         Enum {enum} is missing a required value in the prepare column.
     """
+
+
+class SourceOrPrepareNotAllowed(UserError):
+    template = """
+        The source {source} was not expected. Delete it from the manifest or update the prepare function to allow it.    
+    """

spinta-0.2.dev19/spinta/manifests/dict/components.py

@@ -0,0 +1,133 @@
+import dataclasses
+from enum import Enum
+
+from spinta.manifests.components import Manifest
+from spinta.manifests.helpers import TypeDetector
+
+
+class DictFormat(Enum):
+    JSON = "json"
+    XML = "xml"
+    HTML = "html"
+
+
+class DictManifest(Manifest):
+    format: DictFormat = None
+
+
+class JsonManifest(DictManifest):
+    type = "json"
+    format: DictFormat = DictFormat.JSON
+
+    @staticmethod
+    def detect_from_path(path: str) -> bool:
+        return path.endswith(".json") and not path.startswith("openapi+file://")
+
+
+class XmlManifest(DictManifest):
+    type = "xml"
+    format: DictFormat = DictFormat.XML
+
+    @staticmethod
+    def detect_from_path(path: str) -> bool:
+        return path.endswith(".xml")
+
+
+@dataclasses.dataclass
+class _MappedProperties:
+    name: str
+    source: str
+    extra: str
+    type_detector: TypeDetector
+
+
+@dataclasses.dataclass
+class _MappedModels:
+    name: str
+    source: str
+    properties: dict[str, _MappedProperties]
+
+
+@dataclasses.dataclass
+class _MappedDataset:
+    dataset: str
+    resource: str
+    models: dict[str, dict[str, _MappedModels]]
+
+
+@dataclasses.dataclass
+class _MappingMeta:
+    is_blank_node: bool
+    blank_node_name: str
+    blank_node_source: str
+    seperator: str
+    recursive_descent: str
+    remove_array_suffix: bool
+    model_source_prefix: str
+    check_namespace: bool
+    namespace_prefixes: dict[str, list[str]]
+    namespace_seperator: str
+
+    @classmethod
+    def get_for(cls, manifest_type: DictFormat) -> "_MappingMeta":
+        if manifest_type == DictFormat.JSON:
+            mapping_meta = _MappingMeta.for_json()
+        elif manifest_type in (DictFormat.XML, DictFormat.HTML):
+            mapping_meta = _MappingMeta.for_xml()
+        else:
+            mapping_meta = _MappingMeta.default()
+
+        return mapping_meta
+
+    @classmethod
+    def for_json(cls) -> "_MappingMeta":
+        return cls(
+            is_blank_node=False,
+            blank_node_name="model1",
+            blank_node_source=".",
+            seperator=".",
+            recursive_descent=".",
+            model_source_prefix="",
+            namespace_seperator=":",
+            remove_array_suffix=False,
+            check_namespace=False,
+            namespace_prefixes={},
+        )
+
+    @classmethod
+    def for_xml(cls) -> "_MappingMeta":
+        return cls(
+            is_blank_node=False,
+            blank_node_name="model1",
+            blank_node_source=".",
+            seperator="/",
+            recursive_descent="/..",
+            model_source_prefix="/",
+            namespace_seperator=":",
+            remove_array_suffix=True,
+            check_namespace=True,
+            namespace_prefixes={"xmlns": ["xmlns", "@xmlns"]},
+        )
+
+    @classmethod
+    def default(cls) -> "_MappingMeta":
+        return cls(
+            is_blank_node=False,
+            blank_node_name="model1",
+            blank_node_source=".",
+            seperator="",
+            recursive_descent="",
+            model_source_prefix="",
+            namespace_seperator=":",
+            remove_array_suffix=False,
+            check_namespace=False,
+            namespace_prefixes={},
+        )
+
+
+@dataclasses.dataclass
+class _MappingScope:
+    parent_scope: str
+    model_scope: str
+    model_name: str
+    property_name: str