viur-core 3.9.0.dev3__tar.gz → 3.9.0.dev4__tar.gz

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: viur-core
-Version: 3.9.0.dev3
+Version: 3.9.0.dev4
 Summary: The core component of ViUR, a development framework for Google App Engine
 Author-email: Mausbrand Informationssysteme GmbH <devs@viur.dev>
 Maintainer-email: Jan Max Meyer <jm@mausbrand.de>

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/__init__.py

@@ -18,6 +18,7 @@ from types import ModuleType
 from viur.core import i18n, request, utils
 from viur.core.config import conf
 from viur.core.decorators import access, exposed, force_post, force_ssl, internal_exposed, skey
+from viur.core.request import before_request, after_request
 from viur.core.i18n import translate
 from viur.core.module import Method, Module
 import inspect
@@ -60,6 +61,8 @@ __all__ = [
     "PeriodicTask",
     # Decorators
     "access",
+    "after_request",
+    "before_request",
     "exposed",
     "force_post",
     "force_ssl",

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/bones/__init__.py

@@ -13,6 +13,7 @@ from .base import (
     UniqueValue,
 )
 from .boolean import BooleanBone
+from .code import CodeBone, JinjaBone, LogicsBone, PythonBone
 from .captcha import CaptchaBone
 from .color import ColorBone
 from .credential import CredentialBone
@@ -51,6 +52,10 @@ __all = [
     "BaseBone",
     "BooleanBone",
     "CaptchaBone",
+    "CodeBone",
+    "JinjaBone",
+    "LogicsBone",
+    "PythonBone",
     "CloneBehavior",
     "CloneStrategy",
     "ColorBone",

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/bones/base.py

@@ -798,18 +798,23 @@ class BaseBone(object):
         if self.languages and isinstance(self.required, (list, tuple)):
             missing = set(self.required).difference(filled_languages)
             if missing:
-                return [
+                result_errors = [
                     ReadFromClientError(ReadFromClientErrorSeverity.Empty, fieldPath=[lang])
                     for lang in missing
                 ]
+                self.after_from_client(skel, name, result_errors)
+                return result_errors or None
 
         if isEmpty:
-            return [ReadFromClientError(ReadFromClientErrorSeverity.Empty)]
+            result_errors = [ReadFromClientError(ReadFromClientErrorSeverity.Empty)]
+            self.after_from_client(skel, name, result_errors)
+            return result_errors or None
 
         # Check multiple constraints on demand
         if self.multiple and isinstance(self.multiple, MultipleConstraints):
             errors.extend(self._validate_multiple_contraints(self.multiple, skel, name))
 
+        self.after_from_client(skel, name, errors)
         return errors or None
 
     def _get_single_destinct_hash(self, value) -> t.Any:
@@ -1387,6 +1392,24 @@ class BaseBone(object):
         """
         pass
 
+    def after_from_client(self, skel: "SkeletonInstance", name: str, errors: list[ReadFromClientError]) -> None:
+        """
+        Called at the end of :meth:`fromClient` after ``skel[name]`` has been set and all
+        validation (including multiple-constraints) has run.
+
+        Override to post-process or normalize ``skel[name]`` in-place, or to add/remove
+        entries from ``errors``. Always called when the field was part of the submitted data
+        (i.e. ``skel[name]`` has been written), regardless of whether errors occurred.
+        The ``NotSet`` early-return (field absent from request) is the only case where this
+        hook is *not* called.
+
+        :param skel: The skeleton instance whose bone value was just read.
+        :param name: The attribute name of this bone within the skeleton.
+        :param errors: Mutable list of :class:`ReadFromClientError` collected so far.
+            Changes here affect the return value of :meth:`fromClient`.
+        """
+        pass
+
     def clone_value(self, skel: "SkeletonInstance", src_skel: "SkeletonInstance", bone_name: str) -> None:
         """Clone / Set the value for this bone depending on :attr:`clone_behavior`"""
         match self.clone_behavior.strategy:

viur_core-3.9.0.dev4/src/viur/core/bones/code.py

@@ -0,0 +1,95 @@
+import ast
+import jinja2
+import logics
+from viur.core.bones.raw import RawBone
+
+
+class CodeBone(RawBone):
+    """
+    Stores source code with optional language-specific syntax validation.
+
+    The ``syntax`` parameter sets the type suffix used by the frontend for syntax
+    highlighting, e.g. ``syntax="python"`` yields ``type = "raw.code.python"``.
+    ``type_suffix`` can override this to an arbitrary suffix.
+
+    Neither ``multiple`` nor ``languages`` are supported.
+    Setting ``validate=False`` disables any syntax validation in subclasses.
+    """
+
+    type = "raw.code"
+
+    def __init__(
+        self,
+        *,
+        indexed: bool = False,
+        languages=None,
+        multiple: bool = False,
+        syntax: str | None = None,
+        type_suffix: str = "",
+        validate: bool = True,
+        **kwargs,
+    ):
+        assert not multiple, "CodeBone does not support multiple values"
+        assert not languages, "CodeBone does not support language variants"
+        self.syntax = syntax
+        self.validate = validate
+        if self.syntax:
+            type_suffix = type_suffix or syntax
+        super().__init__(indexed=indexed, type_suffix=type_suffix, **kwargs)
+
+
+class LogicsBone(CodeBone):
+    """
+    Validates its value as a Logics expression (https://github.com/viur-framework/logics).
+    Uses Python syntax highlighting in the frontend.
+    """
+
+    def __init__(self, *, syntax: str = "logics", **kwargs):
+        super().__init__(syntax=syntax, type_suffix="python", **kwargs)
+
+    def singleValueFromClient(self, value, skel, bone_name, client_data):
+        if value := str(value or "").strip():
+            value += "\n"
+        return super().singleValueFromClient(value, skel, bone_name, client_data)
+
+    def isInvalid(self, value):
+        if self.validate and value:
+            try:
+                logics.Logics(value)
+            except logics.ParseException as e:
+                return str(e).replace("&eof", "end-of-expression")
+
+
+class JinjaBone(CodeBone):
+    """
+    Validates its value as a Jinja2 template.
+    """
+
+    def __init__(self, *, syntax: str = "jinja2", **kwargs):
+        super().__init__(syntax=syntax, **kwargs)
+
+    def isInvalid(self, value):
+        if self.validate and value:
+            env = jinja2.Environment()
+            try:
+                env.parse(value)
+            except jinja2.TemplateSyntaxError as e:
+                return f"Syntax error in line {e.lineno}: {e.message}"
+            except jinja2.TemplateError as e:
+                return f"General error: {e}"
+
+
+class PythonBone(CodeBone):
+    """
+    Validates its value as Python source code.
+    """
+
+    def __init__(self, *, syntax: str = "python", **kwargs):
+        super().__init__(syntax=syntax, **kwargs)
+
+    def isInvalid(self, value):
+        if self.validate and value:
+            try:
+                ast.parse(value)
+            except SyntaxError as e:
+                return f"Syntax error in line {e.lineno}: {e.msg}"

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/bones/email.py

@@ -45,6 +45,10 @@ class EmailBone(StringBone):
             assert account and subDomain and tld
             assert subDomain[0] != "."
             assert len(account) <= 64
+            # RFC 5321: local part must not start/end with a dot or contain consecutive dots
+            assert not account.startswith(".")
+            assert not account.endswith(".")
+            assert ".." not in account
         except (ValueError, AssertionError):
             is_valid = False
 

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/bones/relational.py

@@ -1107,7 +1107,7 @@ class RelationalBone(BaseBone):
 
         ref_skel_cache, using_skel_cache = self._getSkels()
         for idx, lang, value in self.iter_bone_value(skel, name):
-            if value is None:
+            if not value:
                 continue
             if value["dest"]:
                 get_values(ref_skel_cache, value["dest"])

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/bones/string.py

@@ -6,7 +6,7 @@ import typing as t
 import warnings
 from numbers import Number
 
-from viur.core import current, db, utils
+from viur.core import conf, current, db, utils
 from .base import ReadFromClientError, ReadFromClientErrorSeverity
 from .raw import RawBone
 
@@ -29,7 +29,7 @@ class StringBone(RawBone):
         max_length: int | None = 254,
         min_length: int | None = None,
         natural_sorting: bool | t.Callable = False,
-        escape_html: bool = True,
+        escape_html: bool | None = None,
         **kwargs
     ):
         """
@@ -46,6 +46,7 @@ class StringBone(RawBone):
             that creates the value for the index property.
         :param escape_html: Replace some characters in the string with HTML-safe sequences with
             using :meth:`utils.string.escape` for safe use in HTML.
+            Defaults to :attr:`conf.bone_string_escape_html` if not set explicitly.
         :param kwargs: Inherited arguments from the BaseBone.
         """
         # fixme: Remove in viur-core >= 4
@@ -71,7 +72,7 @@ class StringBone(RawBone):
         elif not natural_sorting:
             self.natural_sorting = None
         # else: keep self.natural_sorting as is
-        self.escape_html = escape_html
+        self.escape_html = conf.bone_string_escape_html if escape_html is None else escape_html
 
     def type_coerce_single_value(self, value: t.Any) -> str:
         """Convert a value to a string (if not already)

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/config.py

@@ -794,6 +794,9 @@ class Conf(ConfigType):
     bone_boolean_str2true: Multiple[str | int] = ("true", "yes", "1")
     """Allowed values that define a str to evaluate to true"""
 
+    bone_string_escape_html: bool = True
+    """Default escape_html setting for StringBone. Set to False to disable HTML escaping globally."""
+
     bone_html_default_allow: "HtmlBoneConfiguration" = {
         "validTags": [
             "a",

viur_core-3.9.0.dev4/src/viur/core/contrib/__init__.py

@@ -0,0 +1,43 @@
+"""
+viur.core.contrib — optional, reusable application-level components.
+
+This package contains self-contained components that are commonly needed
+but *not* required to run a ViUR application.  Components are opt-in;
+import only what you use.
+
+Available modules
+-----------------
+loginkey
+    :class:`~viur.core.contrib.loginkey.IndexedCredentialBone` and
+    :class:`~viur.core.contrib.loginkey.LoginKey` — a
+    :class:`~viur.core.modules.user.UserPrimaryAuthentication` that
+    authenticates users via a secret token stored in a Datastore-indexed
+    :class:`~viur.core.bones.CredentialBone`.  Suitable for "magic link"
+    style logins or machine-to-machine auth.
+
+Usage example::
+
+    from viur.core.modules.user import User
+    from viur.core.contrib.loginkey import LoginKey
+
+    class MyUser(User):
+        authenticationProviders = [LoginKey, ...]
+
+ratelimit
+    :class:`~viur.core.contrib.ratelimit.RequestRateLimit` — a
+    :class:`~viur.core.request.RequestValidator` that enforces per-IP /
+    per-user request-rate limits using App Engine Memcache.  Suitable for
+    global rate-limiting and basic DDoS mitigation at the WSGI boundary.
+
+Usage example::
+
+    from viur.core.request import Router
+    from viur.core.contrib.ratelimit import RequestRateLimit, TimeWindow
+
+    Router.requestValidators.append(
+        RequestRateLimit(
+            rate_for_guests=TimeWindow(limit=200, time_window=60),
+            rate_for_users=TimeWindow(limit=500, time_window=60),
+        )
+    )
+"""

viur_core-3.9.0.dev4/src/viur/core/contrib/loginkey.py

@@ -0,0 +1,115 @@
+"""
+Token-based ("magic link") primary authentication for ViUR user modules.
+
+``LoginKey`` authenticates a user by a secret token stored as an indexed
+``CredentialBone`` on the user skeleton.  The caller submits the token as a
+POST parameter; the handler looks up the matching user, validates the account
+state, and completes the authentication flow.
+
+Typical use cases include magic-link email logins, CLI tool authentication,
+and service-to-service auth where a shared secret is acceptable.
+
+Usage::
+
+    from viur.core.modules.user import User
+    from viur.core.contrib.loginkey import LoginKey
+
+    class MyUser(User):
+        authenticationProviders = [LoginKey, ...]
+
+.. warning::
+    An indexed :class:`~viur.core.bones.CredentialBone` allows any caller
+    with Datastore read access to enumerate users by key.  Only deploy this
+    in environments where that access is appropriately restricted, and always
+    use long (≥ 32 char), randomly generated tokens.
+"""
+import logging
+
+from viur.core import current, errors
+from viur.core.bones import CredentialBone
+from viur.core.decorators import exposed, force_post, force_ssl, skey
+from viur.core.modules.user import Status, UserPrimaryAuthentication
+from viur.core.ratelimit import RateLimit
+from viur.core.skeleton import SkeletonInstance
+
+logger = logging.getLogger(__name__)
+
+
+class IndexedCredentialBone(CredentialBone):
+    """A :class:`~viur.core.bones.CredentialBone` that is always Datastore-indexed.
+
+    Regular ``CredentialBone`` values are excluded from indexes for security.
+    This subclass forces indexing so that the value can be used as a filter
+    criterion (e.g. ``filter("login_key =", token)``).
+
+    .. note::
+        Accepting an indexed credential is a deliberate trade-off: it enables
+        server-side token lookup at the cost of exposing the value to anyone
+        with Datastore read access.  Only use this when that trade-off is
+        explicitly acceptable.
+    """
+
+    def serialize(self, skel: "SkeletonInstance", name: str, parentIndexed: bool) -> bool:
+        skel.dbEntity.exclude_from_indexes.discard(name)  # force index even though it's a credential
+        if name in skel.accessedValues and skel.accessedValues[name]:
+            skel.dbEntity[name] = skel.accessedValues[name]
+            return True
+        return False
+
+
+class LoginKey(UserPrimaryAuthentication):
+    """Primary authentication via a secret login token.
+
+    The token is stored in a ``login_key`` bone on the user skeleton
+    (added automatically by :meth:`patch_user_skel`).  Failed attempts are
+    rate-limited per IP address; successful logins are *not* counted against
+    the quota.
+
+    :cvar METHOD_NAME: HTTP header name used to identify this auth method.
+    :cvar loginRateLimit: Allows 12 failed attempts per minute per IP.
+    """
+
+    METHOD_NAME = "X-AUTH-LOGINKEY"
+    NAME = "LoginKey"
+
+    # 12 failed attempts per minute, IP-based
+    loginRateLimit = RateLimit("user.loginkey", 12, 1, "ip")
+
+    @classmethod
+    def patch_user_skel(cls, skel_cls):
+        skel_cls.login_key = IndexedCredentialBone(
+            descr="LoginKey",
+            params={"category": "Authentication"},
+            min_length=32,
+        )
+
+    @exposed
+    @force_ssl
+    @force_post
+    @skey()
+    def login(self, *, key: str, **kwargs):
+        if current.user.get():
+            return self._user_module.render.loginSucceeded()
+
+        self.loginRateLimit.assertQuotaIsAvailable()
+
+        user_skel = self._user_module.baseSkel()
+        user_skel = user_skel.all().filter("login_key =", key).getSkel()
+
+        is_okay = user_skel is not None
+        logger.debug(f"user found: {is_okay=}")
+
+        is_okay = is_okay and (user_skel["status"] or 0) >= Status.ACTIVE.value
+        logger.debug(f"account active: {is_okay=}")
+
+        is_okay = is_okay and len(str(user_skel.dbEntity["login_key"])) >= 32
+        logger.debug(f"key length ok: {is_okay=}")
+
+        is_okay = is_okay and ("root" not in user_skel["access"])
+        logger.debug(f"not root: {is_okay=}")
+
+        if not is_okay:
+            self.loginRateLimit.decrementQuota()  # only failed attempts count
+            raise errors.Unauthorized()
+
+        return self.next_or_finish(user_skel)

viur_core-3.9.0.dev4/src/viur/core/contrib/ratelimit.py

@@ -0,0 +1,123 @@
+"""
+Request-level rate limiter using App Engine Memcache.
+
+Registers as a :class:`~viur.core.request.RequestValidator` and therefore
+runs *before* any session, routing, or handler logic — making it the
+earliest possible place to shed excess traffic.
+
+Guests are identified by their IP address (IPv6 addresses are bucketed into
+/64 prefixes so that a single host cannot trivially rotate around the limit).
+Authenticated users are identified by their Datastore user key.
+
+Usage::
+
+    from viur.core.request import Router
+    from viur.core.contrib.ratelimit import RequestRateLimit, TimeWindow
+
+    Router.requestValidators.append(
+        RequestRateLimit(
+            rate_for_guests=TimeWindow(limit=200, time_window=60),
+            rate_for_users=TimeWindow(limit=500, time_window=60),
+        )
+    )
+"""
+import dataclasses
+import ipaddress
+import logging
+import time
+import typing as t
+
+from google.appengine.api.memcache import Client
+
+from viur.core import current
+from viur.core.request import RequestValidator, Router
+
+logger = logging.getLogger(__name__)
+
+_memcache = Client()
+
+
+@dataclasses.dataclass(frozen=True)
+class TimeWindow:
+    """Rate-limit budget for a single time window.
+
+    :param limit: Maximum number of requests allowed within *time_window*.
+    :param time_window: Length of the window in seconds.
+    """
+    limit: int
+    time_window: int
+
+
+class RequestRateLimit(RequestValidator):
+    """Global HTTP request rate limiter.
+
+    Enforces separate budgets for anonymous (guest) and authenticated
+    requests.  When the budget is exceeded the validator returns HTTP 429
+    and sets the ``Retry-After`` header so clients know when to retry.
+
+    :param rate_for_guests: Budget applied to unauthenticated requests.
+    :param rate_for_users: Budget applied to authenticated requests.
+    :param namespace: Memcache namespace used for all rate-limit keys.
+    """
+
+    name = "RequestRateLimit"
+
+    def __init__(
+        self,
+        rate_for_guests: TimeWindow = TimeWindow(limit=1000, time_window=60),
+        rate_for_users: TimeWindow = TimeWindow(limit=2000, time_window=60),
+        namespace: str = "viur_rate_limit",
+    ):
+        self.rate_for_guests = rate_for_guests
+        self.rate_for_users = rate_for_users
+        self.namespace = namespace
+
+    def validate(self, request: "Router") -> t.Optional[tuple[int, str, str]]:
+        if request.is_deferred:
+            return None  # Task Queue requests are always allowed
+
+        if user := current.user.get():
+            client_id = str(user["key"])
+            rate = self.rate_for_users
+        else:
+            client_id = self._get_request_ip()
+            rate = self.rate_for_guests
+
+        current_time = time.time() / rate.time_window
+        key = f"rate_limit:{client_id}:{int(current_time)}"
+
+        count = _memcache.get(key, namespace=self.namespace)
+        logger.debug(f"rate limit check: {client_id=} {count=} limit={rate.limit}")
+
+        if count is None:
+            _memcache.add(key, 1, time=rate.time_window, namespace=self.namespace)
+            return None
+
+        if count < rate.limit:
+            _memcache.incr(key, initial_value=1, namespace=self.namespace)
+            return None
+
+        # Budget exhausted — tell the client when the current window expires.
+        seconds_into_window = (current_time - int(current_time)) * rate.time_window
+        retry_after = int(rate.time_window - seconds_into_window)
+        request.response.headers["Retry-After"] = str(retry_after)
+        return 429, "Too Many Requests", "Too Many Requests. Please try again later."
+
+    @staticmethod
+    def _get_request_ip() -> str:
+        """Return a stable client identifier derived from the remote address.
+
+        IPv4 addresses are returned as-is.  For IPv6 the /64 network prefix
+        is returned so that a single host cannot trivially rotate its
+        interface identifier to bypass the limit.
+        """
+        raw = current.request.get().request.remote_addr
+        ip = ipaddress.ip_address(raw)
+
+        if isinstance(ip, ipaddress.IPv4Address):
+            return str(ip)
+
+        if isinstance(ip, ipaddress.IPv6Address):
+            return str(ipaddress.IPv6Network((ip, 64), strict=False))
+
+        raise NotImplementedError(f"Unsupported IP version: {ip!r}")

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/db/cache.py

@@ -1,7 +1,4 @@
 import datetime
-
-from google.appengine.ext.testbed import Testbed
-
 import logging
 import sys
 import typing as t
@@ -169,6 +166,7 @@ def check_for_memcache() -> bool:
 def init_testbed() -> None:
     global TESTBED
     if TESTBED is None and conf.instance.is_dev_server and conf.db.memcache_client:
+        from google.appengine.ext.testbed import Testbed
         TESTBED = Testbed()
         TESTBED.activate()
         TESTBED.init_memcache_stub()

{viur_core-3.9.0.dev3 → viur_core-3.9.0.dev4}/src/viur/core/db/transport.py

@@ -49,9 +49,15 @@ def get(keys: t.Union[Key, t.List[Key]]) -> t.Union[t.List[Entity], Entity, None
     if isinstance(keys, (list, set, tuple)):
         res_list = list(__client__.get_multi(keys))
         res_list.sort(key=lambda k: keys.index(k.key) if k else -1)
+        if conf.debug.trace_queries:
+            found = sum(1 for r in res_list if r is not None)
+            logging.info(f"db.get: {found}/{len(keys)} entities found")
         return res_list
 
-    return __client__.get(keys)
+    res = __client__.get(keys)
+    if conf.debug.trace_queries:
+        logging.info(f"db.get({keys}): {'found' if res is not None else 'not found'}")
+    return res
 
 
 @deprecated(version="3.8.0", reason="Use 'db.get' instead")
@@ -67,9 +73,15 @@ def put(entities: t.Union[Entity, t.List[Entity]]):
     """
     _write_to_access_log(entities)
     if isinstance(entities, Entity):
-        return __client__.put(entities)
-
-    return __client__.put_multi(entities=entities)
+        res = __client__.put(entities)
+        if conf.debug.trace_queries:
+            logging.info(f"db.put: saved {entities.key}")
+        return res
+
+    res = __client__.put_multi(entities=entities)
+    if conf.debug.trace_queries:
+        logging.info(f"db.put: saved {len(entities)} entities")
+    return res
 
 
 @deprecated(version="3.8.0", reason="Use 'db.put' instead")
@@ -82,12 +94,17 @@ def delete(keys: t.Union[Entity, t.List[Entity], Key, t.List[Key]]):
     Deletes the entities with the given key(s) from the datastore.
     :param keys: A Key (or a t.List of Keys) to delete
     """
-
     _write_to_access_log(keys)
     if not isinstance(keys, (set, list, tuple)):
-        return __client__.delete(keys)
-
-    return __client__.delete_multi(keys)
+        res = __client__.delete(keys)
+        if conf.debug.trace_queries:
+            logging.info(f"db.delete: deleted {keys}")
+        return res
+
+    res = __client__.delete_multi(keys)
+    if conf.debug.trace_queries:
+        logging.info(f"db.delete: deleted {len(keys)} keys")
+    return res
 
 
 @deprecated(version="3.8.0", reason="Use 'db.delete' instead")
@@ -212,6 +229,14 @@ def run_single_filter(query: QueryDefinition, limit: int, keys_only: bool) -> t.
     query.currentCursor = qryRes.next_page_token
     if hasInvertedOrderings:
         res.reverse()
+
+    if conf.debug.trace_queries:
+        distinct_on = f" distinct on {query.distinct}" if query.distinct else ""
+        logging.debug(
+            f"Queried {query.kind} with filter {query.filters} and orders {query.orders}{distinct_on}."
+            f" Returned {len(res)} results"
+        )
+
     return res