spinta 0.2.dev10__tar.gz → 0.2.dev11__tar.gz

{spinta-0.2.dev10 → spinta-0.2.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev10
+Version: 0.2.dev11
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev10 → spinta-0.2.dev11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev10"
+version = "0.2dev11"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/auth.py

@@ -70,6 +70,11 @@ yml.indent(mapping=2, sequence=4, offset=2)
 yml.width = 80
 yml.explicit_start = False
 
+# File permission constants for sensitive authentication files
+OWNER_READABLE_FILE = 0o600  # rw------- (owner read/write only)
+OWNER_READABLE_DIR = 0o700  # rwx------ (owner read/write/execute only)
+WORLD_READABLE_FILE = 0o644  # rw-r--r-- (owner read/write, others read)
+
 # Cache limits
 CLIENT_FILE_CACHE_SIZE_LIMIT = 1000
 KEYMAP_CACHE_SIZE_LIMIT = 1
@@ -831,6 +836,7 @@ def gen_auth_server_keys(
 ) -> Tuple[pathlib.Path, pathlib.Path]:
     path = path / "keys"
     path.mkdir(exist_ok=True)
+    os.chmod(path, OWNER_READABLE_DIR)
 
     files = (
         path / "private.json",
@@ -855,10 +861,12 @@ def gen_auth_server_keys(
         with files[0].open("w") as f:
             result = JsonWebKey.import_key(private_key, {"kty": "RSA"})
             json.dump(result.as_dict(is_private=True), f, indent=4, ensure_ascii=False)
+        os.chmod(files[0], OWNER_READABLE_FILE)
 
         with files[1].open("w") as f:
             result = JsonWebKey.import_key(public_key, {"kty": "RSA"})
             json.dump(result.as_dict(), f, indent=4, ensure_ascii=False)
+        os.chmod(files[1], WORLD_READABLE_FILE)
 
     return files
 
@@ -916,6 +924,8 @@ def create_client_file(
         raise ClientWithNameAlreadyExists(client_name=name)
 
     os.makedirs(id_path / client_id[:2] / client_id[2:4], exist_ok=True)
+    os.chmod(id_path / client_id[:2], OWNER_READABLE_DIR)
+    os.chmod(id_path / client_id[:2] / client_id[2:4], OWNER_READABLE_DIR)
 
     secret = secret or passwords.gensecret(32)
     secret_hash = passwords.crypt(secret)
@@ -934,7 +944,9 @@ def create_client_file(
         write = data.copy()
         del write["client_secret"]
     yml.dump(write, client_file)
+    os.chmod(client_file, OWNER_READABLE_FILE)
     yml.dump(keymap, keymap_path)
+    os.chmod(keymap_path, OWNER_READABLE_FILE)
 
     return client_file, data
 
@@ -1004,6 +1016,7 @@ def update_client_file(
         }
 
         yml.dump(new_data, client_path)
+        os.chmod(client_path, OWNER_READABLE_FILE)
         if keymap:
             changed = False
             # Check if client changed name
@@ -1014,6 +1027,7 @@ def update_client_file(
 
             if changed:
                 yml.dump(keymap, keymap_path)
+                os.chmod(keymap_path, OWNER_READABLE_FILE)
         return new_data
     else:
         raise InvalidClientError(description="Invalid client id or secret")
@@ -1042,18 +1056,22 @@ def validate_id_path(id_path: pathlib.Path):
 def ensure_client_folders_exist(clients_path: pathlib.Path):
     # Ensure clients folder exist
     clients_path.mkdir(parents=True, exist_ok=True)
+    os.chmod(clients_path, OWNER_READABLE_DIR)
 
     # Ensure clients/helpers directory
     helpers_path = get_helpers_path(clients_path)
     helpers_path.mkdir(parents=True, exist_ok=True)
+    os.chmod(helpers_path, OWNER_READABLE_DIR)
 
     # Ensure clients/helpers/keymap.yml exists
     keymap_path = get_keymap_path(clients_path)
     keymap_path.touch(exist_ok=True)
+    os.chmod(keymap_path, OWNER_READABLE_FILE)
 
     # Ensure clients/id directory
     id_path = get_id_path(clients_path)
     id_path.mkdir(parents=True, exist_ok=True)
+    os.chmod(id_path, OWNER_READABLE_DIR)
 
 
 def _keymap_file_cache_key(path: pathlib.Path, *args, **kwargs):

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/commands/init.py

@@ -12,6 +12,7 @@ from spinta.backends.postgresql.helpers import get_column_name
 from spinta.backends.postgresql.helpers.changes import get_changes_table
 from spinta.backends.postgresql.helpers.name import PG_NAMING_CONVENTION, get_pg_table_name, get_pg_column_name
 from spinta.backends.postgresql.helpers.redirect import get_redirect_table
+from spinta.backends.postgresql.helpers.type import get_column_type
 from spinta.components import Context, Model
 from spinta.manifests.components import Manifest
 from spinta.types.datatype import DataType, PrimaryKey, Ref
@@ -118,7 +119,7 @@ def prepare(context: Context, backend: PostgreSQL, dtype: DataType, **kwargs):
 
     if dtype.name not in types:
         raise Exception(f"Unknown type {dtype.name!r} for property {prop.place!r}.")
-    column_type = types[dtype.name]
+    column_type = get_column_type(dtype, types[dtype.name])
     nullable = not dtype.required
     return sa.Column(get_pg_column_name(name), column_type, unique=dtype.unique, nullable=nullable, comment=name)
 

spinta-0.2.dev11/spinta/backends/postgresql/helpers/type.py

@@ -0,0 +1,13 @@
+from spinta.components import Context
+from spinta.backends.postgresql.components import PostgreSQL
+from spinta.exceptions import UnsupportedDataTypeConfiguration
+from spinta.types.datatype import DataType
+
+
+def validate_type_assignment(context: Context, backend: PostgreSQL, dtype: DataType):
+    if dtype.prop.external and dtype.prop.external.custom_type:
+        raise UnsupportedDataTypeConfiguration(dtype, data_type=dtype.name)
+
+
+def get_column_type(dtype: DataType, default: object):
+    return dtype.prop.external and dtype.prop.external.custom_type or default

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/helpers/validate.py

@@ -1,6 +1,5 @@
-from spinta.components import Context, Property
-from spinta.components import Context, Property, DataItem
 from spinta.backends.postgresql.components import PostgreSQL
+from spinta.components import Context, Property, DataItem
 
 
 def pg_check_unique_constraint(

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/array/init.py

@@ -4,6 +4,7 @@ from sqlalchemy.dialects.postgresql import JSONB
 
 from spinta import commands
 from spinta.backends.postgresql.helpers.name import get_pg_column_name
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context
 from spinta.types.datatype import Array
 from spinta.backends.constants import TableType
@@ -14,6 +15,8 @@ from spinta.backends.postgresql.helpers import get_pg_name
 
 @commands.prepare.register(Context, PostgreSQL, Array)
 def prepare(context: Context, backend: PostgreSQL, dtype: Array, **kwargs):
+    validate_type_assignment(context, backend, dtype)
+
     prop = dtype.prop
 
     columns = commands.prepare(context, backend, dtype.items, **kwargs)

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/array_backref/init.py

@@ -1,9 +1,11 @@
 from spinta import commands
 from spinta.backends.postgresql.components import PostgreSQL
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context
 from spinta.types.datatype import ArrayBackRef
 
 
 @commands.prepare.register(Context, PostgreSQL, ArrayBackRef)
 def prepare(context: Context, backend: PostgreSQL, dtype: ArrayBackRef, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     return None

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/backref/init.py

@@ -1,9 +1,11 @@
 from spinta import commands
 from spinta.backends.postgresql.components import PostgreSQL
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context
 from spinta.types.datatype import BackRef
 
 
 @commands.prepare.register(Context, PostgreSQL, BackRef)
 def prepare(context: Context, backend: PostgreSQL, dtype: BackRef, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     return None

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/denorm/init.py

@@ -1,4 +1,5 @@
 from spinta import commands
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context
 from spinta.types.datatype import Denorm
 from spinta.backends.postgresql.components import PostgreSQL
@@ -6,4 +7,5 @@ from spinta.backends.postgresql.components import PostgreSQL
 
 @commands.prepare.register(Context, PostgreSQL, Denorm)
 def prepare(context: Context, backend: PostgreSQL, dtype: Denorm, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     return

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/external_ref/init.py

@@ -2,6 +2,7 @@ import sqlalchemy as sa
 
 from spinta import commands
 from spinta.backends.postgresql.helpers.name import get_pg_column_name
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context
 from spinta.types.datatype import ExternalRef, Denorm
 from spinta.backends.postgresql.components import PostgreSQL
@@ -10,6 +11,7 @@ from spinta.backends.postgresql.helpers import get_column_name
 
 @commands.prepare.register(Context, PostgreSQL, ExternalRef)
 def prepare(context: Context, backend: PostgreSQL, dtype: ExternalRef, propagate=True, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     columns = []
     if not dtype.inherited:
         if dtype.model.given.pkeys or dtype.explicit:

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/file/init.py

@@ -4,6 +4,7 @@ from sqlalchemy.dialects.postgresql import BIGINT, ARRAY
 
 from spinta import commands
 from spinta.backends.postgresql.helpers.name import get_pg_column_name
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context
 from spinta.types.datatype import File
 from spinta.backends.constants import TableType, BackendFeatures
@@ -15,6 +16,7 @@ from spinta.backends.postgresql.helpers import get_column_name
 
 @commands.prepare.register(Context, PostgreSQL, File)
 def prepare(context: Context, backend: PostgreSQL, dtype: File, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     prop = dtype.prop
     model = prop.model
 

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/geometry/init.py

@@ -5,6 +5,7 @@ from spinta import commands
 from spinta.backends.postgresql.components import PostgreSQL
 from spinta.backends.postgresql.helpers import get_column_name
 from spinta.backends.postgresql.helpers.name import get_pg_column_name
+from spinta.backends.postgresql.helpers.type import get_column_type
 from spinta.components import Context
 from spinta.types.geometry.components import Geometry
 
@@ -17,7 +18,7 @@ def prepare(context: Context, backend: PostgreSQL, dtype: Geometry, **kwargs) ->
     # Have to disable spatial index and create our own, because Geoalchemy2 uses their own naming convention
     kwargs = {"geometry_type": dtype.geometry_type, "srid": dtype.srid, "spatial_index": False}
     kwargs = {k: v for k, v in kwargs.items() if v is not None}
-    column_type = ga.Geometry(**kwargs)
+    column_type = get_column_type(dtype, ga.Geometry(**kwargs))
     nullable = not dtype.required
     columns = [
         column := sa.Column(get_pg_column_name(name), column_type, nullable=nullable, comment=name),

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/object/init.py

@@ -1,4 +1,5 @@
 from spinta import commands
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context
 from spinta.types.datatype import Object
 from spinta.backends.postgresql.components import PostgreSQL
@@ -6,6 +7,7 @@ from spinta.backends.postgresql.components import PostgreSQL
 
 @commands.prepare.register(Context, PostgreSQL, Object)
 def prepare(context: Context, backend: PostgreSQL, dtype: Object, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     columns = []
     for prop in dtype.properties.values():
         if prop.name.startswith("_") and prop.name not in ("_revision",):

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/ref/init.py

@@ -7,12 +7,14 @@ from spinta import commands
 from spinta.backends.postgresql.components import PostgreSQL
 from spinta.backends.postgresql.helpers import get_column_name
 from spinta.backends.postgresql.helpers.name import get_pg_column_name, get_pg_table_name
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.components import Context, Property
 from spinta.types.datatype import Ref
 
 
 @commands.prepare.register(Context, PostgreSQL, Ref)
 def prepare(context: Context, backend: PostgreSQL, dtype: Ref, propagate: bool = True, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     pkey_type = commands.get_primary_key_type(context, backend)
     columns = []
 

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/backends/postgresql/types/text/init.py

@@ -4,6 +4,7 @@ from sqlalchemy.dialects.postgresql import JSONB
 from spinta import commands
 from spinta.backends.postgresql.components import PostgreSQL
 from spinta.backends.postgresql.helpers import get_column_name
+from spinta.backends.postgresql.helpers.type import validate_type_assignment
 from spinta.backends.postgresql.helpers.name import get_pg_column_name
 from spinta.components import Context
 from spinta.types.text.components import Text
@@ -11,6 +12,7 @@ from spinta.types.text.components import Text
 
 @commands.prepare.register(Context, PostgreSQL, Text)
 def prepare(context: Context, backend: PostgreSQL, dtype: Text, **kwargs):
+    validate_type_assignment(context, backend, dtype)
     prop = dtype.prop
     name = get_column_name(prop)
 

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/client.py

@@ -1,5 +1,6 @@
 import configparser
 import dataclasses
+import os
 import pathlib
 import urllib.parse
 from typing import List
@@ -7,6 +8,7 @@ from typing import Optional
 
 import requests
 
+from spinta.auth import OWNER_READABLE_FILE
 from spinta.exceptions import RemoteClientCredentialsNotFound
 from spinta.exceptions import RemoteClientCredentialsNotGiven
 from spinta.exceptions import RemoteClientScopesNotGiven
@@ -150,6 +152,7 @@ def add_client_credentials(
 
     with credsfile.open("w") as f:
         config.write(f)
+    os.chmod(credsfile, OWNER_READABLE_FILE)
 
 
 def get_access_token(creds: RemoteClientCredentials) -> str:

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/config.yml

@@ -130,6 +130,27 @@ manifests:
         file:
           type: io
 
+models:
+  type: object
+  keys:
+    type: string
+  values:
+    type: object
+    items:
+      backend:
+        type: ref
+        ref: backends
+      properties:
+        type: object
+        keys:
+          type: string
+        values:
+          type: object
+          keys:
+            type: string
+          values:
+            type: any
+
 manifest:
   type: ref
   ref: manifests

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/core/config.py

@@ -406,8 +406,6 @@ class RawConfig:
                     break
             else:
                 default = schema.get("default", NA)
-            if default is NA:
-                default = None
         return default, None
 
     def get_source_names(self) -> List[str]:

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/datasets/backends/sql/commands/column.py

@@ -24,7 +24,7 @@ def get_column(backend: Sql, data: tuple, **kwargs):
 @commands.get_column.register(Sql, DataType)
 def get_column(backend: Sql, dtype: DataType, table: sa.Table, **kwargs):
     prop = dtype.prop
-    if prop.external is None or not prop.external.name:
+    if not prop.external or not prop.external.name:
         raise NoExternalName(prop)
     if prop.external.name not in table.c:
         raise PropertyNotFound(
@@ -38,7 +38,7 @@ def get_column(backend: Sql, dtype: DataType, table: sa.Table, **kwargs):
 @commands.get_column.register(Sql, Ref)
 def get_column(backend: Sql, dtype: Ref, table: sa.Table, **kwargs):
     prop = dtype.prop
-    if prop.external is None or not prop.external.name and not dtype.inherited:
+    if not prop.external or not prop.external.name and not dtype.inherited:
         raise NoExternalName(prop)
 
     if prop.external.name:

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/datasets/backends/sql/components.py

@@ -44,7 +44,13 @@ class Sql(ExternalBackend):
             key = name
 
         if key not in self.schema.tables:
-            sa.Table(name, self.schema, autoload_with=self.engine)
+            schema = None
+            if "." in key:
+                split = key.split(".", 1)
+                schema = split[0]
+                name = split[1]
+
+            sa.Table(name, self.schema, schema=schema, autoload_with=self.engine)
 
         return self.schema.tables[key]
 

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/datasets/backends/sql/ufuncs/query/ufuncs.py

@@ -237,7 +237,7 @@ def _resolve_unresolved(env: SqlQueryBuilder, value: Any) -> Any:
 
 @ufunc.resolver(SqlQueryBuilder, Property)
 def _resolve_unresolved(env: SqlQueryBuilder, prop: Property) -> sa.Column:
-    if prop.external is None:
+    if not prop.external:
         raise NoExternalName(prop)
 
     if prop.external.name:
@@ -383,7 +383,8 @@ def select(env: SqlQueryBuilder, prop: Property) -> Selected:
     if prop.place not in env.resolved:
         if isinstance(prop.external, list):
             raise SourceCannotBeList(prop)
-        if prop.external.prepare is not NA:
+
+        if prop.external and prop.external.prepare is not NA:
             # If `prepare` formula is given, evaluate formula.
             if isinstance(prop.external.prepare, Expr):
                 result = env(this=prop).resolve(prop.external.prepare)

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/datasets/components.py

@@ -242,9 +242,13 @@ class Attribute(External):
     name: str  # property.source
     prepare: Expr = NA  # property.prepare
 
+    # Type object, that overwrites default types
+    custom_type: Any = None
+
     schema = {
         "prop": {"parent": True},
         "name": {"default": None},
         "prepare": {"type": "spyna", "default": NA},
         "type": {"type": "string"},
+        "custom_type": {"type": "object", "default": None},
     }

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/datasets/helpers.py

@@ -166,7 +166,7 @@ def get_ref_filters(
     seen = seen or []
 
     for prop in take(["_id", all], model.properties).values():
-        if prop.external is None or not prop.external.name:
+        if not prop.external or not prop.external.name:
             # Do not include properties, that has no external source, we can't
             # query data if there is no source.
             continue

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/datasets/inspect/helpers.py

@@ -225,13 +225,17 @@ def zipitems(
                 res[mapped_key] = [new_value]
                 continue
 
-            extension_list = []
-            for existing_value in res[mapped_key]:
-                if existing_value[1] is NA:
-                    existing_value[1] = value
-                else:
-                    extension_list.append([existing_value[0], value])
-            res[mapped_key] += extension_list
+            existing_list = res[mapped_key]
+
+            # 1) Fill ALL rows that still have NA on the right
+            na_rows = [row for row in existing_list if row[1] is NA]
+            if na_rows:
+                for row in na_rows:
+                    row[1] = value
+            else:
+                # 2) No NA slots left -> append exactly ONE new row
+                left_side = existing_list[0][0] if existing_list else NA
+                existing_list.append([left_side, value])
 
     yield from res.values()
 

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/exceptions.py

@@ -1169,3 +1169,35 @@ class MissingPostgresqlComments(UpgradeError):
         {table!r} does not have a comment.
         Please run `spinta upgrade postgresql_comments` to update all comments.
     """
+
+
+class InvalidCustomPropertyTypeConfiguration(UserError):
+    template = """
+        Unable to import custom property type: {custom_property_type!r}.
+    """
+
+
+class InvalidCustomPropertyTypeWithArgsConfiguration(UserError):
+    template = """
+        Unable to import custom property type: {custom_property_type!r}.
+        
+        Using args: {args}.
+    """
+
+
+class UnknownConfigurationParameters(UserError):
+    template = """
+        {config_type} {config_object!r} configuration contains unknown parameters: {unknown_params}.
+    """
+
+
+class MissingConfigurationParameter(UserError):
+    template = """
+        {config_type} {config_object!r} configuration is missing parameter: {missing_params!r}.
+    """
+
+
+class UnsupportedDataTypeConfiguration(UserError):
+    template = """
+        DataType {data_type!r} currently does not support custom type assignment in configuration.
+    """

{spinta-0.2.dev10 → spinta-0.2.dev11}/spinta/manifests/commands/check.py

@@ -1,5 +1,7 @@
 from spinta import commands
 from spinta.components import Context
+from spinta.core.config import RawConfig
+from spinta.exceptions import ModelNotFound
 from spinta.manifests.components import Manifest, get_manifest_object_names
 
 
@@ -8,3 +10,10 @@ def check(context: Context, manifest: Manifest):
     for node in get_manifest_object_names():
         for obj in commands.get_nodes(context, manifest, node).values():
             check(context, obj)
+
+    # Check if the configuration does not contain unknown models
+    rc: RawConfig = context.get("rc")
+    if rc.has("models"):
+        for name in rc.keys("models"):
+            if not commands.has_model(context, manifest, name):
+                raise ModelNotFound(manifest, model=name)