spinta 0.2.dev14__tar.gz → 0.2.dev16__tar.gz

{spinta-0.2.dev14 → spinta-0.2.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev14
+Version: 0.2.dev16
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev14 → spinta-0.2.dev16}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev14"
+version = "0.2dev16"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

spinta-0.2.dev16/spinta/cli/__init__.py

@@ -0,0 +1 @@
+REQUEST_TIMEOUT = 10

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/cli/helpers/sync/controllers/dataset.py

@@ -21,7 +21,7 @@ from requests.models import Response
 
 from spinta.cli.helpers.sync.enum import ResourceType
 from spinta.cli.helpers.sync.api_helpers import validate_api_response
-
+from spinta.cli import REQUEST_TIMEOUT
 
 DEFAULT_RESOURCE = ResourceType.DATASET
 RESOURCE_TYPE_MAPPER = {
@@ -43,6 +43,7 @@ def get_resources(
         f"{base_path}/Dataset/",
         headers=headers,
         params=query_parameters,
+        timeout=REQUEST_TIMEOUT,
     )
     if validate_response:
         validate_api_response(response, {HTTPStatus.OK, HTTPStatus.NOT_FOUND}, "Get resource")
@@ -71,6 +72,7 @@ def create_resource(
         f"{base_path}/Dataset/",
         headers=headers,
         data=data,
+        timeout=REQUEST_TIMEOUT,
     )
     if validate_response:
         validate_api_response(response, {HTTPStatus.CREATED}, "Create resource")

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/cli/helpers/sync/controllers/distribution.py

@@ -6,6 +6,7 @@ import requests
 
 from spinta.cli.helpers.sync import CONTENT_TYPE_TEXT_CSV
 from spinta.cli.helpers.sync.api_helpers import validate_api_response
+from spinta.cli import REQUEST_TIMEOUT
 
 
 def create_distribution(
@@ -23,5 +24,6 @@ def create_distribution(
             "title": distribution_name,
         },
         files={"file": (f"{distribution_name}.csv", BytesIO(file_bytes), CONTENT_TYPE_TEXT_CSV)},
+        timeout=REQUEST_TIMEOUT,
     )
     validate_api_response(response, {HTTPStatus.CREATED}, "Create distribution")

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/cli/helpers/sync/controllers/dsa.py

@@ -5,6 +5,7 @@ import requests
 
 from spinta.cli.helpers.sync import CONTENT_TYPE_TEXT_CSV
 from spinta.cli.helpers.sync.api_helpers import validate_api_response
+from spinta.cli import REQUEST_TIMEOUT
 from spinta.exceptions import NotImplementedFeature
 
 
@@ -13,6 +14,7 @@ def create_dsa(base_path: str, headers: dict[str, str], dataset_id: str, content
         f"{base_path}/Dataset/{dataset_id}/dsa/",
         headers={"Content-Type": CONTENT_TYPE_TEXT_CSV, **headers},
         data=content,
+        timeout=REQUEST_TIMEOUT,
     )
     validate_api_response(response, {HTTPStatus.NO_CONTENT}, "Create DSA")
 
@@ -21,6 +23,7 @@ def get_dsa(base_path: str, headers: dict[str, str], dataset_id: str) -> str:
     response = requests.get(
         f"{base_path}/Dataset/{dataset_id}/dsa",
         headers={"Content-Type": CONTENT_TYPE_TEXT_CSV, **headers},
+        timeout=REQUEST_TIMEOUT,
     )
     validate_api_response(response, {HTTPStatus.OK}, "Get DSA")
     return response.content.decode()
@@ -28,7 +31,7 @@ def get_dsa(base_path: str, headers: dict[str, str], dataset_id: str) -> str:
 
 def update_dsa(base_path: str, headers: dict[str, str], dataset_id: str) -> None:
     # TODO: Unfinished: implement w/ https://github.com/atviriduomenys/katalogas/issues/1600.
-    response = requests.put(f"{base_path}/Dataset/{dataset_id}/dsa/", headers=headers)
+    response = requests.put(f"{base_path}/Dataset/{dataset_id}/dsa/", headers=headers, timeout=REQUEST_TIMEOUT)
     raise NotImplementedFeature(
         status=response.status_code,
         dataset_id=dataset_id,

spinta-0.2.dev16/spinta/cli/helpers/sync/controllers/synchronization/connection.py

@@ -0,0 +1,18 @@
+from http import HTTPStatus
+
+import requests
+from requests.models import Response
+
+from spinta.cli.helpers.sync.api_helpers import validate_api_response
+from spinta.cli import REQUEST_TIMEOUT
+
+
+def connection_check(base_path: str, headers: dict[str, str], data: dict[str, str]) -> Response:
+    response = requests.post(
+        f"{base_path}/Connection/check",
+        headers=headers,
+        data=data,
+        timeout=REQUEST_TIMEOUT,
+    )
+    validate_api_response(response, {HTTPStatus.NO_CONTENT}, "Connection check")
+    return response

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/cli/manifest.py

@@ -31,35 +31,22 @@ app = Typer()
 @app.command(short_help="Copy manifest optionally transforming final copy")
 def copy(
     ctx: TyperContext,
-    source: bool = Option(True, help=(
-        "Do not copy external data source metadata"
-    )),
+    source: bool = Option(True, help=("Do not copy external data source metadata")),
     # TODO: Change `str` to `Access`
     #       https://github.com/tiangolo/typer/issues/151
-    access: str = Option('private', help=(
-        "Copy properties with at least specified access"
-    )),
-    format_names: bool = Option(False, help=(
-        "Reformat model and property names."
-    )),
-    output: Optional[str] = Option(None, '-o', '--output', help=(
-        "Output tabular manifest in a specified file"
-    )),
-    columns: Optional[str] = Option(None, '-c', '--columns', help=(
-        "Comma separated list of columns"
-    )),
-    order_by: Optional[str] = Option(None, help=(
-        "Order by a specified column (currently only access column is supported)"
-    )),
-    rename_duplicates: bool = Option(False, help=(
-        "Rename duplicate model names by adding number suffix"
-    )),
-    manifests: List[str] = Argument(None, help=(
-        "Source manifest files to copy from"
-    )),
+    access: str = Option("private", help=("Copy properties with at least specified access")),
+    format_names: bool = Option(False, help=("Reformat model and property names.")),
+    output: Optional[str] = Option(None, "-o", "--output", help=("Output tabular manifest in a specified file")),
+    columns: Optional[str] = Option(None, "-c", "--columns", help=("Comma separated list of columns")),
+    order_by: Optional[str] = Option(
+        None, help=("Order by a specified column (currently only access column is supported)")
+    ),
+    rename_duplicates: bool = Option(False, help=("Rename duplicate model names by adding number suffix")),
+    manifests: List[str] = Argument(None, help=("Source manifest files to copy from")),
 ):
     """Copy models from CSV manifest files into another CSV manifest file"""
     context: Context = ctx.obj
+    context = configure_context(context, manifests, check_names=not (format_names or rename_duplicates))
     copy_manifest(
         context,
         source=source,
@@ -189,7 +176,7 @@ def _read_and_return_rows(
         rename_duplicates=rename_duplicates,
         load_internal=False,
         verbose=verbose,
-        full_load=True
+        full_load=True,
     )
     if format_names:
         reformat_names(context, store.manifest)

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/cli/sync.py

@@ -2,11 +2,13 @@ import logging
 
 from typer import echo, Context as TyperContext
 
+import spinta
 from spinta.cli.helpers.sync.api_helpers import get_base_path_and_headers
 from spinta.cli.helpers.sync.controllers.data_service import get_data_service_children_dataset_ids
 from spinta.cli.helpers.sync.controllers.synchronization.manifest_catalog_to_agent import (
     execute_manifest_synchronization_catalog_to_agent,
 )
+from spinta.cli.helpers.sync.controllers.synchronization.connection import connection_check
 from spinta.cli.helpers.sync.helpers import (
     get_configuration_credentials,
     validate_credentials,
@@ -22,6 +24,12 @@ from spinta.manifests.tabular.helpers import render_tabular_manifest
 logger = logging.getLogger(__name__)
 
 
+def validate_connection(base_path: str, headers: dict[str, str]) -> None:
+    connection_check(base_path, headers, data={"spinta_version": spinta.__version__})
+    echo("✓ Connection with Catalog established successfully!")
+    return None
+
+
 def prepare_synchronization(ctx: TyperContext) -> tuple[Context, str, str]:
     data_source_name, manifest_path = load_configuration_values(ctx.obj)
     prepare_local_manifest_file(manifest_path)
@@ -42,6 +50,7 @@ def prepare_api_context(context: Context) -> tuple[str, dict[str, str], str]:
 def sync(ctx: TyperContext) -> None:
     context, data_source_name, manifest_path = prepare_synchronization(ctx)
     base_path, headers, agent_name = prepare_api_context(context)
+    validate_connection(base_path, headers)
 
     dataset_ids = get_data_service_children_dataset_ids(base_path, headers, agent_name)
     manifest = execute_manifest_synchronization_catalog_to_agent(

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/client.py

@@ -9,6 +9,7 @@ from typing import Optional
 import requests
 
 from spinta.auth import OWNER_READABLE_FILE
+from spinta.cli import REQUEST_TIMEOUT
 from spinta.exceptions import RemoteClientCredentialsNotFound
 from spinta.exceptions import RemoteClientCredentialsNotGiven
 from spinta.exceptions import RemoteClientScopesNotGiven
@@ -164,6 +165,7 @@ def get_access_token(creds: RemoteClientCredentials) -> str:
             "grant_type": "client_credentials",
             "scope": creds.scopes,
         },
+        timeout=REQUEST_TIMEOUT,
     )
     resp.raise_for_status()
     return resp.json()["access_token"]

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/components.py

@@ -1115,6 +1115,7 @@ class Config:
     sync_retry_count: int
     sync_retry_delay_range: tuple[float]
     languages: List[str]
+    # For CLI commands `spinta copy` and `spinta check --check-names`
     check_names: bool = False
     # MB
     max_api_file_size: int

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/manifests/open_api/openapi_generator.py

@@ -6,6 +6,7 @@ from typing import Any
 
 from spinta.core.context import configure_context, create_context
 from spinta.cli.manifest import _read_and_return_manifest
+from spinta.dimensions.enum.components import EnumItem
 from spinta.manifests.open_api.openapi_config import (
     BASE_TAGS,
     COMMON_SCHEMAS,
@@ -23,6 +24,8 @@ from spinta.manifests.open_api.openapi_config import (
     SERVERS,
     VERSION,
 )
+from spinta.types.datatype import DataType
+from spinta.utils.schema import NA
 
 
 @dataclass
@@ -74,6 +77,11 @@ class DataTypeHandler:
         """Check if property has enum values"""
         return hasattr(model_property, "enum") and model_property.enum
 
+    def get_enum_value(self, dtype: DataType, enum_item: EnumItem) -> Any:
+        """Converts enum value to property type"""
+        value = enum_item.prepare if enum_item.prepare and enum_item.prepare is not NA else enum_item.source
+        return dtype.load(value)
+
     def get_enum_values(self, model_property) -> list[str]:
         """Extract enum values from property"""
         if not self.is_enum_property(model_property):
@@ -81,7 +89,7 @@ class DataTypeHandler:
 
         enum = model_property.enum
         if isinstance(enum, dict):
-            return list(enum.keys())
+            return [self.get_enum_value(model_property.dtype, enum_value) for enum_value in enum.values()]
         else:
             return [enum_prop.strip('"') for enum_prop in enum]
 
@@ -92,7 +100,11 @@ class DataTypeHandler:
 
         if self.is_enum_property(model_property):
             enum_values = self.get_enum_values(model_property)
-            return {"type": "string", "enum": enum_values, "example": enum_values[0] if enum_values else "UNKNOWN"}
+            dtype_name = self.get_dtype_name(dtype)
+            return {
+                **copy.deepcopy(self.schema_registry.type_mapping.mappings.get(dtype_name, {"type": "string"})),
+                **{"enum": enum_values, "example": enum_values[0] if enum_values else "UNKNOWN"},
+            }
 
         if self.is_reference_type(dtype):
             return {

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/manifests/sql/helpers.py

@@ -382,15 +382,63 @@ def _create_dataset_for_schema(
 
 
 @cachetools.cached(cache=cachetools.LRUCache(maxsize=1024))
-def oracle_maintained_schemas(engine: Engine) -> Iterator[str]:
-    query = sa.text("""
-        SELECT USERNAME 
-        FROM ALL_USERS
-        WHERE ORACLE_MAINTAINED = 'Y'
-    """)
-    with engine.connect() as conn:
-        rows = conn.execute(query).fetchall()
-        return {r[0].upper() for r in rows}
+def oracle_maintained_schemas(engine: Engine) -> set[str]:
+    try:
+        query = sa.text("""
+            SELECT USERNAME 
+            FROM ALL_USERS
+            WHERE ORACLE_MAINTAINED = 'Y'
+        """)
+        with engine.connect() as conn:
+            rows = conn.execute(query).fetchall()
+            return {r[0].upper() for r in rows}
+    except sa.exc.DatabaseError as _:
+        # Fallback in case users do not have access to the ALL_USERS table or ORACLE_MAINTAINED column.
+
+        # https://docs.oracle.com/cd/E11882_01/server.112/e10575/tdpsg_user_accounts.htm#BABJGDJF
+        predefined_admin_accounts = {
+            "ANONYMOUS",
+            "CTXSYS",
+            "DBSNMP",
+            "EXFSYS",
+            "LBACSYS",
+            "MDSYS",
+            "MGMT_VIEW",
+            "OLAPSYS",
+            "ORDDATA",
+            "OWBSYS",
+            "ORDPLUGINS",
+            "ORDSYS",
+            "OUTLN",
+            "SI_INFORMTN_SCHEMA",
+            "SYS",
+            "SYSMAN",
+            "SYSTEM",
+            "WK_TEST",
+            "WKSYS",
+            "WKPROXY",
+            "WMSYS",
+            "XDB",
+            "OPS$ORACLE",
+        }
+
+        # https://docs.oracle.com/cd/E11882_01/server.112/e10575/tdpsg_user_accounts.htm#BABGJDJC
+        predefined_non_admin_accounts = {
+            "APEX_PUBLIC_USER",
+            "DIP",
+            "FLOWS_040100",
+            "FLOWS_FILES",
+            "MDDATA",
+            "ORACLE_OCM",
+            "SPATIAL_CSW_ADMIN_USR",
+            "SPATIAL_WFS_ADMIN_USR",
+            "XS$NULL",
+        }
+
+        # https://docs.oracle.com/cd/E11882_01/server.112/e10575/tdpsg_user_accounts.htm#TDPSG20024
+        predefined_sample_accounts = {"BI", "HR", "OE", "PM", "IX", "SH"}
+
+        return predefined_admin_accounts | predefined_non_admin_accounts | predefined_sample_accounts
 
 
 def is_internal_schema(engine: Engine, schema: str) -> bool:

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/manifests/tabular/helpers.py

@@ -2373,7 +2373,7 @@ def _property_to_tabular(
     access: Access = Access.private,
     order_by: ManifestColumn = None,
 ) -> Iterator[ManifestRow]:
-    if prop.name.startswith("_"):
+    if prop.name.startswith("_") and not prop.explicitly_given:
         return
 
     if prop.access is not None and prop.access < access:

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/types/datatype.py

@@ -152,6 +152,8 @@ class Integer(DataType):
 
         if isinstance(value, int) and not isinstance(value, bool):
             return value
+        elif isinstance(value, str) and (value.removeprefix("-").isdigit() or value.removeprefix("+").isdigit()):
+            return int(value)
         else:
             raise exceptions.InvalidValue(self, value=value)
 

{spinta-0.2.dev14 → spinta-0.2.dev16}/spinta/types/helpers.py

@@ -15,15 +15,20 @@ from spinta.utils.naming import is_valid_model_name, is_valid_property_name
 if TYPE_CHECKING:
     from spinta.types.datatype import DataType
 
+ALLOWED_RESERVED_PROPERTY_NAMES = {
+    "_id",
+    "_created",
+    "_updated",
+    "_revision",
+    "_label",
+}
 
 RESERVED_PROPERTY_NAMES = {
+    *ALLOWED_RESERVED_PROPERTY_NAMES,
     "_op",
     "_type",
-    "_id",
-    "_revision",
     "_txn",
     "_cid",
-    "_created",
     "_where",
     "_base",
     "_uri",
@@ -31,6 +36,8 @@ RESERVED_PROPERTY_NAMES = {
     "_same_as",
 }
 
+C_LANG = "C"
+
 
 def check_no_extra_keys(dtype: DataType, schema: Iterable, data: Iterable):
     unknown = set(data) - set(schema)
@@ -70,6 +77,15 @@ def check_model_name(context: Context, model: Model):
 def check_property_name(context: Context, prop: Property):
     config: Config = context.get("config")
     if config.check_names:
-        name = prop.name
-        if name not in RESERVED_PROPERTY_NAMES and not is_valid_property_name(name):
-            raise InvalidName(prop, name=name, type="property")
+        if prop.name.startswith("_"):
+            if prop.explicitly_given:
+                if prop.name in ALLOWED_RESERVED_PROPERTY_NAMES:
+                    return  #  prop.name atėjo iš DSA ir yra mažajame  sąraše
+            elif prop.name in RESERVED_PROPERTY_NAMES:
+                return  # prop.name sugeneruotas sistemos (nėra DSA) ir yra dyžiajame sąraše
+        elif is_valid_property_name(prop.name):
+            return
+        # "C" as default language: dataset1/City?select(id,name@lt,name@pl,name@C)
+        elif prop.name == C_LANG:
+            return
+        raise InvalidName(prop, name=prop.name, type="property")