spinta 0.2.dev16__tar.gz → 0.2.dev18__tar.gz

{spinta-0.2.dev16 → spinta-0.2.dev18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev16
+Version: 0.2.dev18
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev16 → spinta-0.2.dev18}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev16"
+version = "0.2dev18"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/datasets/backends/dataframe/backends/soap/commands/read.py

@@ -12,7 +12,7 @@ from spinta.core.ufuncs import Expr
 from spinta.datasets.backends.dataframe.backends.soap.components import Soap
 from spinta.datasets.backends.dataframe.backends.soap.ufuncs.components import SoapQueryBuilder
 from spinta.datasets.backends.dataframe.backends.soap.ufuncs.ufuncs import MakeCDATA
-from spinta.datasets.backends.dataframe.commands.read import parametrize_bases, get_dask_dataframe_meta, dask_get_all
+from spinta.datasets.backends.dataframe.commands.read import parametrize_bases, dask_get_all, get_dask_dataframe_meta
 from spinta.datasets.backends.dataframe.ufuncs.query.components import DaskDataFrameQueryBuilder
 from spinta.datasets.components import Resource
 from spinta.dimensions.param.components import ResolvedParams

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/datasets/backends/dataframe/backends/xml/commands/read.py

@@ -12,9 +12,9 @@ from spinta.core.ufuncs import Expr
 from spinta.datasets.backends.dataframe.backends.xml.components import Xml
 from spinta.datasets.backends.dataframe.commands.read import (
     parametrize_bases,
-    get_dask_dataframe_meta,
     dask_get_all,
     get_pkeys_if_ref,
+    get_dask_dataframe_meta,
 )
 from spinta.datasets.backends.helpers import is_file_path
 from spinta.dimensions.param.components import ResolvedParams
@@ -132,9 +132,13 @@ def getall(
     builder.update(model=model, params={param.name: param for param in resource.params}, url_query_params=query)
 
     props = {}
-    for prop in model.properties.values():
+
+    for prop in model.flatprops.values():
         if prop.external and prop.external.name:
-            props[prop.name] = {"source": prop.external.name, "pkeys": get_pkeys_if_ref(prop)}
+            props[prop.place] = {
+                "source": prop.external.name,
+                "pkeys": get_pkeys_if_ref(prop),
+            }
 
     meta = get_dask_dataframe_meta(model)
 

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/datasets/backends/dataframe/commands/read.py

@@ -32,6 +32,8 @@ from spinta.ufuncs.resultbuilder.components import ResultBuilder
 from spinta.utils.data import take
 from spinta.utils.schema import NA
 
+OBJECT_DTYPE = "object"
+
 
 def _resolve_expr(context: Context, row: Any, sel: Selected, params: dict) -> Any:
     if sel.item is None:
@@ -192,9 +194,9 @@ def get_pkeys_if_ref(prop: Property) -> list[str]:
 
 def get_dask_dataframe_meta(model: Model):
     dask_meta = {}
-    for prop in model.properties.values():
+    for prop in model.flatprops.values():
         if prop.external and prop.external.name:
-            dask_meta[prop.external.name] = spinta_to_np_dtype(prop.dtype)
+            dask_meta[prop.external.name] = OBJECT_DTYPE
     return dask_meta
 
 

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py

@@ -15,7 +15,7 @@ from spinta.datasets.backends.dataframe.ufuncs.query.components import (
 )
 from spinta.datasets.components import Param
 from spinta.datasets.utils import iterparams
-from spinta.exceptions import PropertyNotFound, NotImplementedFeature, SourceCannotBeList
+from spinta.exceptions import PropertyNotFound, NotImplementedFeature, SourceCannotBeList, SourceOrPrepareNotAllowed
 from spinta.types.datatype import DataType, PrimaryKey, Ref
 from spinta.types.text.components import Text
 from spinta.ufuncs.components import ForeignProperty
@@ -188,9 +188,12 @@ def select(env: DaskDataFrameQueryBuilder, prop: Property) -> Selected:
             #      tag:resolving_private_properties_in_prepare_context
             result = env.call("select", prop.dtype, result)
         elif prop.external.prepare is not NA:
-            # property without external name and with `prepare` is already evaluated
-            # so just use evaluated value
-            result = Selected(prop=prop, prep=prop.external.prepare)
+            # property without external name may be evaluated already, if it is use the value
+            if isinstance(prop.external.prepare, Expr):
+                result = env(this=prop).resolve(prop.external.prepare)
+                result = env.call("select", prop.dtype, result)
+            else:
+                result = Selected(prop=prop, prep=prop.external.prepare)
         elif prop.external and prop.external.name:
             # If prepare is not given, then take value from `source`.
             result = env.call("select", prop.dtype)
@@ -339,7 +342,7 @@ def select(
 ) -> Selected:
     # TODO need join for this to work
     return Selected(
-        item=dtype.prop.name,
+        item=dtype.prop.external.name,
         prop=dtype.prop,
     )
 
@@ -458,3 +461,13 @@ def eval_(env: DaskDataFrameQueryBuilder, param: Param) -> Iterator[str]:
     )
 
     return resolved_values
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, Bind, Bind, name="getattr")
+def getattr_(env: DaskDataFrameQueryBuilder, obj: Bind, attr: Bind) -> Any:
+    return GetAttr(obj, attr)
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, Bind, Bind, Bind, name="getattr")
+def getattr_(env: DaskDataFrameQueryBuilder, source: Bind, obj: Bind, attr: Bind) -> Any:
+    raise SourceOrPrepareNotAllowed(source=str(source))

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/dimensions/enum/helpers.py

@@ -19,10 +19,11 @@ from spinta.dimensions.enum.components import EnumFormula
 from spinta.dimensions.enum.components import EnumItem
 from spinta.dimensions.enum.components import EnumValue
 from spinta.dimensions.enum.components import Enums
-from spinta.exceptions import ValueNotInEnum
+from spinta.exceptions import ValueNotInEnum, EnumPrepareMissing
 from spinta.manifests.components import Manifest
 from spinta.manifests.tabular.components import EnumRow
 from spinta.nodes import load_node
+from spinta.types.datatype import String
 from spinta.utils.schema import NA
 
 
@@ -35,7 +36,11 @@ def _load_enum_item(
     parent = parents[0]
     item = load_node(context, item, data, parent=parent)
     item = cast(EnumItem, item)
-    if item.prepare is not NA:
+
+    if item.prepare is NA:
+        if hasattr(parent, "dtype") and not isinstance(parent.dtype, String):
+            raise EnumPrepareMissing(enum=item.source)
+    else:
         ast = item.prepare
         expr = asttoexpr(ast)
         env = EnumFormula(

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/exceptions.py

@@ -1209,3 +1209,15 @@ class UnsupportedDataTypeConfiguration(UserError):
     template = """
         DataType {data_type!r} currently does not support custom type assignment in configuration.
     """
+
+
+class EnumPrepareMissing(UserError):
+    template = """
+        Enum {enum} is missing a required value in the prepare column.
+    """
+
+
+class SourceOrPrepareNotAllowed(UserError):
+    template = """
+        The source {source} was not expected. Delete it from the manifest or update the prepare function to allow it.    
+    """

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/manifests/open_api/helpers.py

@@ -353,8 +353,13 @@ def read_open_api_manifest(path: Path) -> Generator[tuple[None, dict]]:
     yield from get_dataset_schemas(data, dataset_prefix)
 
 
-def create_openapi_manifest(manifest: ManifestPath) -> dict:
+def create_openapi_manifest(manifest: ManifestPath, **kwargs: Any) -> dict:
     """Create OpenAPI manifest from manifest data"""
 
-    generator = OpenAPIGenerator()
+    main_dataset_name = kwargs.get("main_dataset_name")
+    api_version = kwargs.get("api_version")
+    generator = OpenAPIGenerator(
+        main_dataset_name=main_dataset_name,
+        api_version=api_version,
+    )
     return generator.generate_spec(manifest)

{spinta-0.2.dev16 → spinta-0.2.dev18}/spinta/manifests/open_api/openapi_config.py

@@ -18,16 +18,6 @@ SERVERS = [
     {"url": "get.data.gov.lt", "description": "Data access server"},
 ]
 
-CHANGE_SCHEMA_EXAMPLE = {
-    "_cid": {"type": "integer", "example": 11},
-    "_id": {"type": "string", "format": "uuidv4", "example": "abdd1245-bbf9-4085-9366-f11c0f737c1d"},
-    "_rev": {"type": "string", "format": "uuidv4", "example": "16dabe62-61e9-4549-a6bd-07cecfbc3508"},
-    "_txn": {"type": "string", "example": "792a5029-63c9-4c07-995c-cbc063aaac2c"},
-    "_created": {"type": "string", "format": "date-time", "example": "2021-07-30T14:03:14.645198"},
-    "_op": {"type": "string", "enum": ["insert", "patch", "delete"]},
-    "_objectType": {"type": "string", "example": "ExampleModel"},
-}
-
 PROPERTY_EXAMPLE = {
     "string": "Example string",
     "integer": 42,
@@ -223,22 +213,6 @@ PATHS_CONFIG = {
             },
         },
     },
-    "/{model_name}/:changes/{cid}": {
-        "parameters": ["cid", "traceparent", "tracestate", "If-None-Match", "Accept-Language"],
-        "get": {
-            "security": [{}],
-            "summary": "Get all object changes since given {cid} (change id).",
-            "description": "Get latest changes to a table.\n\nIf {cid} is not given, return changes, since very first available\nchange.\n\nIf {cid} is gven, return only changes, since given change id, including\nchange id itself.\n\nThis API can return changes, that were returned previously, client\nshould be responsible for checking if a change was received previously\nor not.\n\nLast change id is included in the request, in order for clients to check\nif last change id matches change received by client. If last change\ndoes not match, then client should do a full synce, because if last\nchange id does not match, that means, that a data migration or some\nother alterations to data were made, which requires to do a full sync.\n",
-            "operationId": "getChanges",
-            "responses": {
-                "200": {"description": "OK", "content": {"application/json": {"schema": "changes"}}},
-                "400": {"$ref": "error400"},
-                "404": {"$ref": "error404"},
-                "500": {"$ref": "error500"},
-                "503": {"$ref": "error503"},
-            },
-        },
-    },
 }
 
 RESPONSE_COMPONENTS = {
@@ -393,13 +367,6 @@ PARAMETER_COMPONENTS = {
         "description": "Subresource of an object.\n\nAll lower case, words separated with `_` symbol.",
         "schema": {"type": "string", "examples": ["cities"]},
     },
-    "cid": {
-        "name": "cid",
-        "in": "path",
-        "required": True,
-        "description": "Change id.\n\nUsed for incremental changes API, to get next changes after given change id.",
-        "schema": {"type": "string"},
-    },
 }
 
 COMMON_SCHEMAS = {
@@ -408,32 +375,11 @@ COMMON_SCHEMAS = {
         "description": "For objects that have been deleted during change, `type` value is changed to `absent`",
         "properties": {"type": {"type": "string", "enum": ["absent"]}},
     },
-    "boolean": {
-        "oneOf": [
-            {"type": "string", "description": "Maturity level < 3", "examples": [True, 1, "taip"]},
-            {"type": "boolean", "description": "Maturity level >= 3"},
-        ],
-        "description": "Logical value of true or false, depending on maturity level this value can be expressed in a non standard true/false values if maturity level is lower than 3",
-    },
-    "integer": {
-        "type": "integer",
-        "description": "A value of a whole number",
-        "minimum": -2147483648,
-        "maximum": 2147483647,
-    },
-    "number": {
-        "type": "number",
-        "description": "A value of a real number, based on Floating-Point Arithmetic (IEEE 754), with a decimal point marked with `.`. Whole number can be up to 6 characters in length.",
-    },
     "binary": {
         "type": "string",
         "description": "Binary string of data. A single set should not exceed 1G",
         "pattern": "^[0-1]+$",
     },
-    "string": {
-        "type": "string",
-        "description": "Non natural language strings of characters. Should be provided based on UTF-8 encoding and should not exceed 1G",
-    },
     "text": {"type": "string", "description": "Natural language text."},
     "datetime": {
         "type": "string",
@@ -508,10 +454,6 @@ COMMON_SCHEMAS = {
         "type": "string",
         "description": "Uniform Resource Identifier. Used to provide an identifier of an external resource, in an RDF data model it is subject identifier.",
     },
-    "array": {
-        "type": "array",
-        "description": "Array of data. It is recommended to avoid this type and instead use `backref`",
-    },
     "backref": {
         "type": "string",
         "description": "Backwards link showing that another model has a link to this one. This item does not hold any data",