spinta 0.2.dev1__tar.gz → 0.2.dev3__tar.gz

{spinta-0.2.dev1 → spinta-0.2.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: spinta
-Version: 0.2.dev1
+Version: 0.2.dev3
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 Home-page: https://github.com/atviriduomenys/spinta
 License: MIT
@@ -20,22 +20,27 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Database
 Classifier: Topic :: Database :: Database Engines/Servers
 Classifier: Topic :: Database :: Front-Ends
+Provides-Extra: all
+Provides-Extra: http
+Provides-Extra: migrations
 Requires-Dist: GeoAlchemy2
 Requires-Dist: Shapely
 Requires-Dist: XlsxWriter
 Requires-Dist: aiofiles
 Requires-Dist: aiohttp
-Requires-Dist: alembic (>=1.11.0,<1.12.0)
+Requires-Dist: alembic (>=1.11.0,<1.12.0) ; extra == "migrations" or extra == "all"
 Requires-Dist: asyncpg
 Requires-Dist: authlib (>=0.11,<0.12)
+Requires-Dist: cachetools
 Requires-Dist: click
 Requires-Dist: dask[dataframe]
 Requires-Dist: docutils (>=0.21.2,<0.22.0)
 Requires-Dist: fsspec
-Requires-Dist: gunicorn
+Requires-Dist: gunicorn ; extra == "http" or extra == "all"
 Requires-Dist: jinja2
 Requires-Dist: jsonpatch
 Requires-Dist: lark-parser
@@ -68,9 +73,10 @@ Requires-Dist: tqdm
 Requires-Dist: typer[all]
 Requires-Dist: ujson
 Requires-Dist: unidecode
-Requires-Dist: uvicorn
+Requires-Dist: uvicorn ; extra == "http" or extra == "all"
 Requires-Dist: xlrd
 Requires-Dist: xmltodict
+Requires-Dist: zeep (>=4.3.1,<5.0.0)
 Project-URL: Bug Tracker, https://github.com/atviriduomenys/spinta/issues
 Project-URL: Documentation, https://spinta.readthedocs.io/
 Project-URL: Repository, https://github.com/atviriduomenys/spinta

{spinta-0.2.dev1 → spinta-0.2.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2.dev1"
+version = "0.2.dev3"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"
@@ -67,11 +67,11 @@ click = "*"
 typer = { version = "*", extras = ["all"] }
 
 # API server dependencies
-gunicorn = "*"
-uvicorn = "*"
+gunicorn = { version = "*", optional = true }
+uvicorn = { version = "*", optional = true }
 
 # PostgreSQL backend dependencies
-alembic = "~1.11.0"
+alembic = { version = "~1.11.0", optional = true }
 asyncpg = "*"
 psycopg2-binary = "*"
 # FIXME: https://github.com/python-poetry/poetry/issues/4402
@@ -106,7 +106,14 @@ tabulate = "*"
 pyproj = "*"
 xmltodict = "*"
 sqlean-py = "*"
+cachetools = "*"
 docutils = "^0.21.2"
+zeep = "^4.3.1"
+
+[tool.poetry.extras]
+http = ["gunicorn", "uvicorn"]
+migrations = ["alembic"]
+all = ["gunicorn", "uvicorn", "alembic"]
 
 
 [tool.poetry.group.dev.dependencies]
@@ -120,6 +127,7 @@ pytest-mock = "*"
 responses = "*"
 snoop = "*"
 python-dotenv = "*"
+docutils = "*"
 
 # Starlette
 httpx = "*"

{spinta-0.2.dev1 → spinta-0.2.dev3}/spinta/accesslog/__init__.py

@@ -25,13 +25,14 @@ class AccessLog:
     reason: str = None
     url: str = None
     buffer_size: int = 100
-    format: str = None          # response format
-    content_type: str = None    # request content-type header
-    agent: str = None           # request user-agent header
-    txn: str = None             # request transaction id
-    start: float = None         # request start time in secodns
-    memory: int = None          # memory used in bytes at the start of request
-    scopes: List[str] = None    # list of scopes
+    format: str = None  # response format
+    content_type: str = None  # request content-type header
+    agent: str = None  # request user-agent header
+    txn: str = None  # request transaction id
+    start: float = None  # request start time in seconds
+    memory: int = None  # memory used in bytes at the start of request
+    scopes: List[str] = None  # list of scopes
+    token: str = None  # token used for request
 
     def __enter__(self):
         return self
@@ -75,7 +76,7 @@ class AccessLog:
             'format': self.format,
             'url': self.url,
             'client': self.client,
-            'scope': self.scopes,
+            'token': self.token,
             'reason': reason or self.reason,
             'agent': self.agent,
         }
@@ -100,6 +101,27 @@ class AccessLog:
 
         self.log(message)
 
+    def auth(
+        self,
+        *,
+        reason: str = None,
+    ):
+        message = {
+            'time': datetime.datetime.now(datetime.timezone.utc).isoformat(),
+            'type': 'auth',
+            'client': self.client,
+            'token': self.token,
+            'scope': self.scopes,
+            'reason': reason or self.reason,
+            'method': self.method,
+            'rctype': self.content_type,
+            'format': self.format,
+            'url': self.url,
+            'agent': self.agent,
+        }
+        message = {k: v for k, v in message.items() if v is not None}
+        self.log(message)
+
     def _get_time(self) -> float:
         "Get current time in seconds"
         return time.monotonic()
@@ -124,8 +146,10 @@ def load(context: Context, accesslog: AccessLog, token: Token):  # noqa
 
     client_id = token.get_client_id()
     config = context.get('config')
-    if config.scope_log and client_id != get_default_auth_client_id(context):
-        accesslog.scopes = token.get_scope()
+    if client_id != get_default_auth_client_id(context):
+        if config.scope_log:
+            accesslog.scopes = token.get_scope()
+        accesslog.token = token.get_jti()
 
 
 @overload

{spinta-0.2.dev1 → spinta-0.2.dev3}/spinta/api/__init__.py

@@ -1,3 +1,4 @@
+import json
 import uuid
 from typing import Type
 
@@ -18,13 +19,14 @@ from starlette.middleware import Middleware
 
 from spinta import components, commands
 from spinta.auth import AuthorizationServer, check_scope, query_client, get_clients_list, \
-    client_exists, create_client_file, delete_client_file, update_client_file, get_clients_path, Scopes
+    client_exists, create_client_file, delete_client_file, update_client_file, get_clients_path, Scopes, Token, \
+    authenticate_token
 from spinta.auth import ResourceProtector
 from spinta.auth import BearerTokenValidator
 from spinta.auth import get_auth_request
 from spinta.auth import get_auth_token
 from spinta.commands import prepare, get_version
-from spinta.components import Context
+from spinta.components import Context, UrlParams
 from spinta.exceptions import BaseError, MultipleErrors, error_response, InsufficientPermission, \
     UnknownPropertyInRequest, InsufficientPermissionForUpdate, EmptyPassword
 from spinta.middlewares import ContextMiddleware
@@ -60,18 +62,60 @@ async def version(request: Request):
 
 
 async def auth_token(request: Request):
-    auth_server = request.state.context.get('auth.server')
+    context = request.state.context
+    auth_server = context.get('auth.server')
     if auth_server.enabled():
-        return auth_server.create_token_response({
+        resp: JSONResponse = auth_server.create_token_response({
             'method': request.method,
             'url': str(request.url.replace(query='')),
             'body': dict(await request.form()),
             'headers': request.headers,
         })
+
+        payload = json.loads(resp.body.decode("utf-8"))
+        _auth_accesslog(
+            context,
+            request,
+            payload,
+            'json'
+        )
+
+        return resp
     else:
         raise NoAuthServer()
 
 
+def _auth_accesslog(
+    context: Context,
+    request: Request,
+    payload: dict,
+    output_format: str
+):
+    access_token = payload.get('access_token')
+    if not access_token:
+        return
+
+    token = payload['access_token']
+    type_ = payload['token_type']
+
+    resource_protector = context.get('auth.resource_protector')
+    token = authenticate_token(
+        resource_protector,
+        token=token,
+        type_=type_
+    )
+    params = UrlParams()
+    params.format = output_format
+    context.attach('accesslog', create_accesslog, context, loaders=(
+        context.get('store'),
+        request,
+        token,
+        params
+    ))
+    accesslog = context.get('accesslog')
+    accesslog.auth()
+
+
 def _auth_client_context(request: Request) -> Context:
     context: Context = request.state.context
     context.set('auth.request', get_auth_request({

{spinta-0.2.dev1 → spinta-0.2.dev3}/spinta/api/schema.py

@@ -2,13 +2,14 @@ import os
 import tempfile
 from uuid import UUID
 
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+
 from spinta import commands
 from spinta.auth import check_scope, Scopes
+from spinta.cli.helpers.migrate import MigrationConfig
 from spinta.cli.helpers.store import prepare_manifest
-from spinta.cli.helpers.migrate import MigrateRename, MigrateMeta
 from spinta.components import Context, UrlParams, Store, Model, Config, Property
-from starlette.requests import Request
-
 from spinta.core.context import configure_context, create_context
 from spinta.datasets.commands.check import check_dataset_name
 from spinta.datasets.inspect.helpers import zipitems
@@ -19,7 +20,6 @@ from spinta.manifests.components import ManifestPath, Manifest
 from spinta.manifests.tabular.helpers import datasets_to_tabular
 from spinta.utils.schema import NA
 from spinta.utils.types import is_str_uuid
-from starlette.responses import JSONResponse
 
 
 def _clean_up_file(file_path: str):
@@ -184,16 +184,15 @@ async def schema_api(context: Context, request: Request, params: UrlParams):
         validate_manifest(context, target_manifest, dataset_name)
 
         rename_data = create_migrate_rename_mapping(context, target_context, manifest, target_manifest, dataset_name)
-        migrate_meta = MigrateMeta(
+        migration_config = MigrationConfig(
             plan=False,
             autocommit=False,
-            rename=MigrateRename(
-                rename_src=rename_data
-            ),
+            rename_src=rename_data,
             datasets=[dataset_name],
-            migration_extension=(lambda: commands.update_manifest_dataset_schema(context, manifest, target_manifest))
+            migration_extension=(lambda: commands.update_manifest_dataset_schema(context, manifest, target_manifest)),
+            raise_error=True,
         )
-        commands.migrate(context, target_manifest, migrate_meta)
+        commands.migrate(context, target_manifest, migration_config)
         backend = manifest.backend
         commands.reload_backend_metadata(context, manifest, backend)
         reset_affected_objects(context, manifest, dataset_name)

{spinta-0.2.dev1 → spinta-0.2.dev3}/spinta/auth.py

@@ -1,48 +1,47 @@
 from __future__ import annotations
 
-import os
-from functools import lru_cache
-
-from multipledispatch import dispatch
-from typing import Set
-from typing import Type
-from typing import Union, List, Tuple
-
+import base64
 import datetime
 import enum
 import json
 import logging
-import time
+import os
 import pathlib
-import base64
+import time
+import uuid
+from threading import Lock
+from typing import Set
+from typing import Type
+from typing import Union, List, Tuple
 
 import ruamel.yaml
-
-from starlette.responses import JSONResponse
-from starlette.exceptions import HTTPException
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives.asymmetric import rsa
-
 from authlib.jose import jwk
 from authlib.jose import jwt
 from authlib.jose.errors import JoseError
+from authlib.oauth2 import OAuth2Error
 from authlib.oauth2 import OAuth2Request
 from authlib.oauth2 import rfc6749
 from authlib.oauth2 import rfc6750
 from authlib.oauth2.rfc6749 import grants
+from authlib.oauth2.rfc6749.errors import InvalidClientError, UnsupportedTokenTypeError
 from authlib.oauth2.rfc6750.errors import InsufficientScopeError
-from authlib.oauth2.rfc6749.errors import InvalidClientError
-from authlib.oauth2 import OAuth2Error
+from cachetools import cached, LRUCache
+from cachetools.keys import hashkey
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from multipledispatch import dispatch
+from starlette.exceptions import HTTPException
+from starlette.responses import JSONResponse
 
 from spinta.components import Config
+from spinta.components import Context, Namespace, Model, Property
 from spinta.components import ScopeFormatterFunc
-from spinta.core.enums import Access
-from spinta.components import Context, Action, Namespace, Model, Property
+from spinta.core.enums import Access, Action
+from spinta.exceptions import AuthorizedClientsOnly
+from spinta.exceptions import BasicAuthRequired
 from spinta.exceptions import InvalidToken, NoTokenValidationKey, ClientWithNameAlreadyExists, ClientAlreadyExists, \
     ClientsKeymapNotFound, ClientsIdFolderNotFound, InvalidClientsKeymapStructure, InvalidScopes, \
     InvalidClientFileFormat
-from spinta.exceptions import AuthorizedClientsOnly
-from spinta.exceptions import BasicAuthRequired
 from spinta.utils import passwords
 from spinta.utils.config import get_clients_path, get_keymap_path, get_id_path, get_helpers_path
 from spinta.utils.scopes import name_to_scope
@@ -56,6 +55,16 @@ yml.indent(mapping=2, sequence=4, offset=2)
 yml.width = 80
 yml.explicit_start = False
 
+# Cache limits
+CLIENT_FILE_CACHE_SIZE_LIMIT = 1000
+KEYMAP_CACHE_SIZE_LIMIT = 1
+DEFAULT_CLIENT_ID_CACHE_SIZE_LIMIT = 1
+
+
+class KeyType(enum.Enum):
+    public = 'public'
+    private = 'private'
+
 
 class Scopes(enum.Enum):
     """
@@ -189,7 +198,7 @@ class Client(rfc6749.ClientMixin):
         if unknown_scopes:
             log.warning(f"requested unknown scopes: %s", ', '.join(sorted(unknown_scopes)))
             unknown_scopes = ', '.join(sorted(unknown_scopes))
-            raise InvalidScopes(scopes = unknown_scopes)
+            raise InvalidScopes(scopes=unknown_scopes)
         else:
             return True
 
@@ -241,6 +250,9 @@ class Token(rfc6749.TokenMixin):
     def get_aud(self):  # Client.
         return self._token.get('aud', '')
 
+    def get_jti(self):
+        return self._token.get('jti', '')
+
     def get_client_id(self):
         return self.get_aud()
 
@@ -259,10 +271,21 @@ class AdminToken(rfc6749.TokenMixin):
     def get_aud(self):  # Client.
         return 'admin'
 
+    def get_jti(self):
+        return 'admin'
+
     def get_client_id(self):
         return self.get_aud()
 
 
+def authenticate_token(protector: ResourceProtector, token: str, type_: str) -> Token:
+    type_ = type_.lower()
+    if type_ not in protector.TOKEN_VALIDATORS:
+        raise UnsupportedTokenTypeError()
+
+    return protector.TOKEN_VALIDATORS[type_].authenticate_token(token)
+
+
 def get_auth_token(context: Context) -> Token:
     scope = None  # Scopes will be validated later using Token.check_scope
     request: OAuth2Request = context.get('auth.request')
@@ -333,11 +356,6 @@ def create_key_pair():
     return rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend())
 
 
-class KeyType(enum.Enum):
-    public = 'public'
-    private = 'private'
-
-
 def load_key(context: Context, key_type: KeyType, *, required: bool = True):
     key = None
     config = context.get('config')
@@ -397,6 +415,7 @@ def create_access_token(
     iat = int(time.time())
     exp = iat + expires_in
     scopes = ' '.join(sorted(scopes)) if scopes else ''
+    jti = str(uuid.uuid4())
     payload = {
         'iss': config.server_url,
         'sub': client,
@@ -404,39 +423,11 @@ def create_access_token(
         'iat': iat,
         'exp': exp,
         'scope': scopes,
+        'jti': jti
     }
     return jwt.encode(header, payload, private_key).decode('ascii')
 
 
-def query_client(path: pathlib.Path, client: str, is_name: bool = False) -> Client:
-    if is_name:
-        keymap_path = get_keymap_path(path)
-        validate_keymap_path(keymap_path)
-
-        data = _load_keymap_data(keymap_path)
-        if not client_name_exists(data, client):
-            raise (InvalidClientError(description='Invalid client name'))
-        client = data[client]
-    client_file = get_client_file_path(path, client)
-
-    id_path = get_id_path(path)
-    validate_id_path(id_path)
-
-    try:
-        data = yaml.load(client_file)
-    except FileNotFoundError:
-        raise (InvalidClientError(description='Invalid client id or secret'))
-    if not isinstance(data, dict):
-        raise InvalidClientFileFormat(client_file=client_file.name, client_file_type=type(data))
-    if not isinstance(data['scopes'], list):
-        raise Exception(f'Client {client_file} scopes must be list of scopes.')
-    client_id = data["client_id"]
-    client_name = data["client_name"] if ("client_name" in data.keys() and data["client_name"]) else None
-    client = Client(id_=client_id, name_=client_name, secret_hash=data['client_secret_hash'],
-                    scopes=data['scopes'])
-    return client
-
-
 def get_client_file_path(
     path: pathlib.Path,
     client: str
@@ -623,18 +614,6 @@ def client_exists(path: pathlib.Path, client: str) -> bool:
     return False
 
 
-def _load_keymap_data(keymap_path: pathlib.Path) -> dict:
-    keymap = yaml.load(keymap_path)
-    # This could mean keymap is empty, or keymap has bad yml structure
-    if keymap is None:
-        if os.stat(keymap_path).st_size == 0:
-            return {}
-        raise InvalidClientsKeymapStructure()
-    if not isinstance(keymap, dict):
-        raise InvalidClientsKeymapStructure()
-    return keymap
-
-
 @dispatch(pathlib.Path, str)
 def client_name_exists(path: pathlib.Path, client_name: str) -> bool:
     keymap_path = get_keymap_path(path)
@@ -747,7 +726,7 @@ def update_client_file(
         new_scopes = scopes if scopes is not None else client.scopes
 
         client_path = get_client_file_path(path, client_id)
-        keymap =  _load_keymap_data(keymap_path)
+        keymap = _load_keymap_data(keymap_path)
         if new_name != client.name:
             if client_name_exists(keymap, new_name):
                 raise ClientWithNameAlreadyExists(client_name=new_name)
@@ -798,13 +777,6 @@ def validate_id_path(id_path: pathlib.Path):
         raise ClientsIdFolderNotFound()
 
 
-# Get default auth client id using cache
-@lru_cache
-def get_default_auth_client_id(context: Context) -> str:
-    config: Config = context.get('config')
-    return get_client_id_from_name(get_clients_path(config.config_path), config.default_auth_client)
-
-
 def ensure_client_folders_exist(clients_path: pathlib.Path):
     # Ensure clients folder exist
     clients_path.mkdir(parents=True, exist_ok=True)
@@ -820,3 +792,103 @@ def ensure_client_folders_exist(clients_path: pathlib.Path):
     # Ensure clients/id directory
     id_path = get_id_path(clients_path)
     id_path.mkdir(parents=True, exist_ok=True)
+
+
+def _keymap_file_cache_key(path: pathlib.Path, *args, **kwargs):
+    """
+    Creates keymap file cache key using
+    keymap path and keymap file update time.
+    """
+    key = hashkey(path, *args, **kwargs)
+    time_ = os.path.getmtime(path)
+    key += tuple([time_])
+    return key
+
+
+def _default_client_id_cache_key(context: Context, *args, **kwargs):
+    """
+    Creates default client id cache key using
+    client folder path, default client name and keymap update time.
+    """
+    key = hashkey(*args, **kwargs)
+    config: Config = context.get('config')
+    path = get_clients_path(config.config_path)
+    client = config.default_auth_client
+    keymap_path = get_keymap_path(path)
+    validate_keymap_path(keymap_path)
+    time_ = os.path.getmtime(keymap_path)
+    key += tuple([path, client, time_])
+    return key
+
+
+def _client_file_cache_key(path: pathlib.Path, client: str, *args, is_name: bool = False, **kwargs):
+    """
+    Creates client file cache key using
+    client folder path, client id and client file update time.
+    """
+    id_path = get_id_path(path)
+    validate_id_path(id_path)
+
+    key = hashkey(path, client, *args, **kwargs)
+    if is_name:
+        client_id = get_client_id_from_name(path, client)
+        if client_id is None:
+            raise (InvalidClientError(description='Invalid client name'))
+
+        client = client_id
+
+    client_file = get_client_file_path(path, client)
+    if not client_file.exists():
+        raise (InvalidClientError(description='Invalid client id or secret'))
+
+    time_ = os.path.getmtime(client_file)
+    key += tuple([time_])
+    return key
+
+
+@cached(LRUCache(KEYMAP_CACHE_SIZE_LIMIT), key=_keymap_file_cache_key)
+def _load_keymap_data(keymap_path: pathlib.Path) -> dict:
+    keymap = yaml.load(keymap_path)
+    # This could mean keymap is empty, or keymap has bad yml structure
+    if keymap is None:
+        if os.stat(keymap_path).st_size == 0:
+            return {}
+        raise InvalidClientsKeymapStructure()
+    if not isinstance(keymap, dict):
+        raise InvalidClientsKeymapStructure()
+    return keymap
+
+
+@cached(LRUCache(DEFAULT_CLIENT_ID_CACHE_SIZE_LIMIT), key=_default_client_id_cache_key)
+def get_default_auth_client_id(context: Context) -> str:
+    config: Config = context.get('config')
+    return get_client_id_from_name(get_clients_path(config.config_path), config.default_auth_client)
+
+
+@cached(LRUCache(CLIENT_FILE_CACHE_SIZE_LIMIT), key=_client_file_cache_key, lock=Lock())
+def query_client(path: pathlib.Path, client: str, is_name: bool = False) -> Client:
+    if is_name:
+        client_id = get_client_id_from_name(path, client)
+        if client_id is None:
+            raise (InvalidClientError(description='Invalid client name'))
+
+        client = client_id
+    client_file = get_client_file_path(path, client)
+
+    id_path = get_id_path(path)
+    validate_id_path(id_path)
+
+    try:
+        data = yaml.load(client_file)
+    except FileNotFoundError:
+        raise (InvalidClientError(description='Invalid client id or secret'))
+    if not isinstance(data, dict):
+        raise InvalidClientFileFormat(client_file=client_file.name, client_file_type=type(data))
+    if not isinstance(data['scopes'], list):
+        raise Exception(f'Client {client_file} scopes must be list of scopes.')
+    client_id = data["client_id"]
+    client_name = data["client_name"] if ("client_name" in data.keys() and data["client_name"]) else None
+    client = Client(id_=client_id, name_=client_name, secret_hash=data['client_secret_hash'],
+                    scopes=data['scopes'])
+    return client
+