viur-core 3.8.0.dev7__tar.gz → 3.8.0.dev9__tar.gz

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: viur-core
-Version: 3.8.0.dev7
+Version: 3.8.0.dev9
 Summary: The core component of ViUR, a development framework for Google App Engine
 Author-email: Mausbrand Informationssysteme GmbH <devs@viur.dev>
 Maintainer-email: Jan Max Meyer <jm@mausbrand.de>
@@ -34,11 +34,10 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
-Requires-Python: >=3.10
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: appengine-python-standard~=1.0
@@ -57,6 +56,7 @@ Requires-Dist: googleapis-common-protos[grpc]~=1.0
 Requires-Dist: gunicorn~=23.0
 Requires-Dist: jinja2~=3.0
 Requires-Dist: jsonschema~=4.0
+Requires-Dist: logics-py==0.0.12
 Requires-Dist: pillow~=10.0
 Requires-Dist: pyotp~=2.0
 Requires-Dist: pytz~=2023.0
@@ -64,7 +64,7 @@ Requires-Dist: pyyaml~=6.0
 Requires-Dist: qrcode~=7.0
 Requires-Dist: requests~=2.0
 Requires-Dist: tzlocal~=5.0
-Requires-Dist: urllib3==1.26.18
+Requires-Dist: urllib3==1.26.20
 Requires-Dist: user-agents~=2.0
 Requires-Dist: webob~=1.0
 Provides-Extra: mailjet

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
     "gunicorn~=23.0",
     "jinja2~=3.0",
     "jsonschema~=4.0",
+    "logics-py==0.0.12",
     "pillow~=10.0",
     "pyotp~=2.0",
     "pytz~=2023.0",
@@ -29,11 +30,11 @@ dependencies = [
     "qrcode~=7.0",
     "requests~=2.0",
     "tzlocal~=5.0",
-    "urllib3==1.26.18", # for appengine-python-standard used by some projects (https://github.com/GoogleCloudPlatform/appengine-python-standard/blob/main/setup.py#L28)
+    "urllib3==1.26.20", # for appengine-python-standard (https://github.com/GoogleCloudPlatform/appengine-python-standard/blob/main/setup.py#L28)
     "user-agents~=2.0",
     "webob~=1.0",
 ]
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 authors = [
     { name = "Mausbrand Informationssysteme GmbH", email = "devs@viur.dev" },
 ]
@@ -49,9 +50,8 @@ classifiers = [
     "Intended Audience :: Developers",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Software Development :: Libraries :: Application Frameworks",
 ]
 

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/src/viur/core/bones/file.py

@@ -229,9 +229,10 @@ class FileBone(TreeLeafBone):
         super().postSavedHandler(skel, boneName, key)
         if (
             current.request.get().is_deferred
-            and "derived" in current.request_data.get().get("__update_relations_bones")
+            and "derived" in (current.request_data.get().get("__update_relations_bones") or ())
         ):
             return
+
         from viur.core.skeleton import RelSkel, Skeleton
 
         if issubclass(skel.skeletonCls, Skeleton):

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/src/viur/core/bones/record.py

@@ -31,7 +31,7 @@ class RecordBone(BaseBone):
         using: 'viur.core.skeleton.RelSkel' = None,
         **kwargs
     ):
-        from viur.core.skeleton import RelSkel
+        from viur.core.skeleton.relskel import RelSkel
         if not issubclass(using, RelSkel):
             raise ValueError("RecordBone requires for valid using-parameter (subclass of viur.core.skeleton.RelSkel)")
 

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/src/viur/core/config.py

@@ -849,8 +849,11 @@ class Conf(ConfigType):
     ]
     """Backward compatibility flags; Remove to enforce new style."""
 
-    db_engine: str = "viur.datastore"
-    """Database engine module"""
+    db_query_external_limit: int = 100
+    """Sets the maximum query limit allowed by external filters."""
+
+    db_query_default_limit: int = 30
+    """Sets the default query limit for all queries."""
 
     db_memcache_client: Client | None = None
     """If set, ViUR cache data for the db.get in the Memcache for faster access."""
@@ -1031,7 +1034,6 @@ class Conf(ConfigType):
         "viur.cacheEnvironmentKey": "cache_environment_key",
         "viur.contentSecurityPolicy": "content_security_policy",
         "viur.bone.boolean.str2true": "bone_boolean_str2true",
-        "viur.db.engine": "db_engine",
         "viur.errorHandler": "error_handler",
         "viur.static.embedSvg.path": "static_embed_svg_path",
         "viur.file.hmacKey": "file_hmac_key",

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/src/viur/core/db/cache.py

@@ -1,6 +1,9 @@
+import datetime
+
+from google.appengine.ext.testbed import Testbed
+
 import logging
 import sys
-import time
 import typing as t
 
 from viur.core.config import conf
@@ -8,9 +11,9 @@ from .types import Entity, Key
 
 MEMCACHE_MAX_BATCH_SIZE = 30
 MEMCACHE_NAMESPACE = "viur-datastore"
-MEMCACHE_TIMEOUT = 60 * 60
-MEMCACHE_MAX_SIZE = 1_000_000
-
+MEMCACHE_TIMEOUT: int | datetime.timedelta = datetime.timedelta(days=1)
+MEMCACHE_MAX_SIZE: t.Final[int] = 1_000_000
+TESTBED = None
 """
 
     This Module controls the Interaction with the Memcache from Google
@@ -18,12 +21,8 @@ MEMCACHE_MAX_SIZE = 1_000_000
     ..  code-block:: python
     # Example
     from viur.core import conf
-    if not conf.instance.is_dev_server:
-        from google.appengine.api.memcache import Client
-        conf.db_memcache_client = Client()
-    else:
-        conf.db_memcache_client = db.cache.LocalMemcache()
-
+    from google.appengine.api.memcache import Client
+    conf.db_memcache_client = Client()
 """
 
 __all__ = [
@@ -34,40 +33,63 @@ __all__ = [
     "get",
     "put",
     "delete",
-    "LocalMemcache",
+    "flush",
 ]
 
 
-def get(keys: t.Union[str, Key, t.List[str], t.List[Key]]) -> t.Dict[str, dict]:
+def get(keys: t.Union[Key, list[Key]], namespace: t.Optional[str] = None) -> t.Union[Entity, list[Entity], None]:
     """
     Reads data form the memcache.
     :param keys: Unique identifier(s) for one or more entry(s).
-    :return: A dict with the entry(s) that found in the memcache.
+    :param namespace: Optional namespace to use.
+    :return: The entity (or None if it has not been found), or a list of entities.
     """
     if not check_for_memcache():
-        return {}
-    if not isinstance(keys, list):
+        return None
+
+    namespace = namespace or MEMCACHE_NAMESPACE
+    single_request = not isinstance(keys, (list, tuple, set))
+    if single_request:
         keys = [keys]
     keys = [str(key) for key in keys]  # Enforce that all keys are strings
-    res = {}
+    cached_data = {}
+    result = []
     try:
         while keys:
-            res |= conf.db_memcache_client.get_multi(keys[:MEMCACHE_MAX_BATCH_SIZE], namespace=MEMCACHE_NAMESPACE)
+            cached_data |= conf.db_memcache_client.get_multi(keys[:MEMCACHE_MAX_BATCH_SIZE], namespace=namespace)
             keys = keys[MEMCACHE_MAX_BATCH_SIZE:]
     except Exception as e:
         logging.error(f"""Failed to get keys form the memcache with {e=}""")
-    return res
-
-
-def put(data: t.Union[Entity, t.Dict[Key, Entity], t.List[Entity]]):
+    for key, value in cached_data.items():
+        entity = Entity(Key(key))
+        entity |= value
+        result.append(entity)
+    if single_request:
+        return result[0] if result else None
+    return result if result else None
+
+
+def put(
+    data: t.Union[Entity, t.Dict[Key, Entity], t.Iterable[Entity]],
+    namespace: t.Optional[str] = None,
+    timeout: t.Optional[t.Union[int, datetime.timedelta]] = None
+) -> bool:
     """
     Writes Data to the memcache.
-
     :param data: Data to write
+    :param namespace: Optional namespace to use.
+    :param timeout: Optional timeout in seconds or a timedelta object.
+    :return: A boolean indicating success.
     """
     if not check_for_memcache():
-        return
-    if isinstance(data, list):
+        return False
+
+    namespace = namespace or MEMCACHE_NAMESPACE
+    timeout = timeout or MEMCACHE_TIMEOUT
+    if isinstance(timeout, datetime.timedelta):
+        timeout = timeout.total_seconds()
+
+    if isinstance(data, (list, tuple, set)):
         data = {item.key: item for item in data}
     elif isinstance(data, Entity):
         data = {data.key: data}
@@ -80,41 +102,48 @@ def put(data: t.Union[Entity, t.Dict[Key, Entity], t.List[Entity]]):
     try:
         while keys:
             data_batch = {key: data[key] for key in keys[:MEMCACHE_MAX_BATCH_SIZE]}
-            conf.db_memcache_client.set_multi(data_batch, namespace=MEMCACHE_NAMESPACE, time=MEMCACHE_TIMEOUT)
+            conf.db_memcache_client.set_multi(data_batch, namespace=namespace, time=timeout)
             keys = keys[MEMCACHE_MAX_BATCH_SIZE:]
+        return True
     except Exception as e:
         logging.error(f"""Failed to put data to the memcache with {e=}""")
+        return False
 
 
-def delete(keys: t.Union[str, Key, t.List[str], t.List[Key]]) -> None:
+def delete(keys: t.Union[Key, list[Key]], namespace: t.Optional[str] = None) -> None:
     """
     Deletes an Entry form memcache.
-
     :param keys: Unique identifier(s) for one or more entry(s).
+    :param namespace: Optional namespace to use.
     """
     if not check_for_memcache():
-        return
+        return None
+
+    namespace = namespace or MEMCACHE_NAMESPACE
     if not isinstance(keys, list):
         keys = [keys]
     keys = [str(key) for key in keys]  # Enforce that all keys are strings
     try:
         while keys:
-            conf.db_memcache_client.delete_multi(keys[:MEMCACHE_MAX_BATCH_SIZE], namespace=MEMCACHE_NAMESPACE)
+            conf.db_memcache_client.delete_multi(keys[:MEMCACHE_MAX_BATCH_SIZE], namespace=namespace)
             keys = keys[MEMCACHE_MAX_BATCH_SIZE:]
     except Exception as e:
         logging.error(f"""Failed to delete keys form the memcache with {e=}""")
 
 
-def flush():
+def flush() -> bool:
     """
     Deletes everything in memcache.
+    :return: A boolean indicating success.
     """
     if not check_for_memcache():
-        return
+        return False
     try:
         conf.db_memcache_client.flush_all()
     except Exception as e:
         logging.error(f"""Failed to flush the memcache with {e=}""")
+        return False
+    return True
 
 
 def get_size(obj: t.Any) -> int:
@@ -133,36 +162,13 @@ def check_for_memcache() -> bool:
     if conf.db_memcache_client is None:
         logging.warning(f"""conf.db_memcache_client is 'None'. It can not be used.""")
         return False
+    init_testbed()
     return True
 
 
-class LocalMemcache:
-    def __init__(self):
-        self._data = {}
-
-    def get_multi(self, keys: t.List[str], namespace: str = MEMCACHE_NAMESPACE):
-        self._data.setdefault(namespace, {})
-        res = {}
-        for key in keys:
-            if (data := self._data[namespace].get(key)) is not None:
-                if data["__lifetime__"]["last_seen"] + data["__lifetime__"]["timeout"] > time.time():
-                    res[key] = data["__data__"]
-                else:
-                    self._data[namespace].pop(key)
-        return res
-
-    def set_multi(self, data: t.Dict[str, t.Any], namespace: str = MEMCACHE_NAMESPACE, time: int = MEMCACHE_TIMEOUT):
-        self._data.setdefault(namespace, {})
-        for key, value in data.items():
-            self._data[namespace][key] = {}
-            self._data[namespace][key]["__data__"] = value
-            self._data[namespace][key]["__lifetime__"] = {"timeout": time, "last_seen": time.time()}
-
-    def delete_multi(self, keys: t.List[str] = [], namespace: str = MEMCACHE_NAMESPACE):
-        self._data.setdefault(namespace, {})
-        for key in keys:
-            if (data := self._data[namespace].get(key)) is not None:
-                self._data[namespace].pop(key)
-
-    def flush_all(self):
-        self._data.clear()
+def init_testbed() -> None:
+    global TESTBED
+    if TESTBED is None and conf.instance.is_dev_server and conf.db_memcache_client:
+        TESTBED = Testbed()
+        TESTBED.activate()
+        TESTBED.init_memcache_stub()

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/src/viur/core/db/query.py

@@ -151,9 +151,12 @@ class Query(object):
         """
         if self.srcSkel is None:
             raise NotImplementedError("This query has not been created using skel.all()")
+
         if self.queries is None:  # This query is already unsatisfiable and adding more constraints won't change this
             return self
+
         skel = self.srcSkel
+
         if "search" in filters:
             if self.srcSkel.customDatabaseAdapter and self.srcSkel.customDatabaseAdapter.providesFulltextSearch:
                 self._fulltextQueryString = str(filters["search"])
@@ -163,31 +166,50 @@ class Query(object):
                     % self.srcSkel.kindName
                 )
                 self.queries = None
+
         bones = [(y, x) for x, y in skel.items()]
+
         try:
             # Process filters first
             for bone, key in bones:
                 bone.buildDBFilter(key, skel, self, filters)
+
             # Parse orders
             for bone, key in bones:
                 bone.buildDBSort(key, skel, self, filters)
+
         except RuntimeError as e:
             logging.exception(e)
             self.queries = None
             return self
+
         startCursor = endCursor = None
+
         if "cursor" in filters and filters["cursor"] and filters["cursor"].lower() != "none":
             startCursor = filters["cursor"]
+
         if "endcursor" in filters and filters["endcursor"] and filters["endcursor"].lower() != "none":
             endCursor = filters["endcursor"]
+
         if startCursor or endCursor:
             self.setCursor(startCursor, endCursor)
-        if (
-                "limit" in filters
-                and str(filters["limit"]).isdigit()
-                and 0 < int(filters["limit"]) <= 100
-        ):
-            self.limit(int(filters["limit"]))
+
+        if limit := filters.get("limit"):
+            try:
+                limit = int(limit)
+
+                # disallow limit beyond conf.db_query_external_limit
+                if limit > conf.db_query_external_limit:
+                    limit = conf.db_query_external_limit
+
+                # forbid any limit < 0, which might bypass defaults
+                if limit < 0:
+                    limit = 0
+
+                self.limit(limit)
+            except ValueError:
+                pass  # ignore this
+
         return self
 
     def filter(self, prop: str, value: DATASTORE_BASE_TYPES | list[DATASTORE_BASE_TYPES]) -> t.Self:
@@ -373,19 +395,17 @@ class Query(object):
 
     def limit(self, limit: int) -> t.Self:
         """
-        Sets the query limit to *amount* entities in the result.
+        Sets the query limit to *limit* entities in the result.
 
-        Specifying a limit of 0 disables the limit (use with care!).
-
-        :param limit: The maximum number of entities.
+        :param limit: The maximum number of entities per batch.
         :returns: Returns the query itself for chaining.
         """
-        # TODO Add a check for the limit (<=100) ?
         if isinstance(self.queries, QueryDefinition):
             self.queries.limit = limit
         elif isinstance(self.queries, list):
             for query in self.queries:
                 query.limit = limit
+
         return self
 
     def distinctOn(self, keyList: t.List[str]) -> t.Self:
@@ -586,36 +606,46 @@ class Query(object):
         if self._fulltextQueryString:
             if utils.is_in_transaction():
                 raise ValueError("Can't run fulltextSearch inside transactions!")  # InvalidStateError FIXME!
+
             qryStr = self._fulltextQueryString
             self._fulltextQueryString = None  # Reset, so the adapter can still work with this query
             res = self.srcSkel.customDatabaseAdapter.fulltextSearch(qryStr, self)
+
             if not self.srcSkel.customDatabaseAdapter.fulltextSearchGuaranteesQueryConstrains:
                 # Search might yield results that are not included in the listfilter
                 if isinstance(self.queries, QueryDefinition):  # Just one
                     res = [x for x in res if _entryMatchesQuery(x, self.queries.filters)]
                 else:  # Multi-Query, must match at least one
                     res = [x for x in res if any([_entryMatchesQuery(x, y.filters) for y in self.queries])]
+
         elif isinstance(self.queries, list):
+            limit = limit if limit >= 0 else self.queries[0].limit
+
             # We have more than one query to run
             if self._calculateInternalMultiQueryLimit:
-                limit = self._calculateInternalMultiQueryLimit(self, limit if limit != -1 else self.queries[0].limit)
+                limit = self._calculateInternalMultiQueryLimit(self, limit)
+
             res = []
             # We run all queries first (preventing multiple round-trips to the server)
             for singleQuery in self.queries:
-                res.append(self._run_single_filter_query(singleQuery, limit if limit != -1 else singleQuery.limit))
+                res.append(self._run_single_filter_query(singleQuery, limit))
+
             # Wait for the actual results to arrive and convert the protobuffs to Entries
             res = [self._fixKind(x) for x in res]
             if self._customMultiQueryMerge:
                 # We have a custom merge function, use that
-                res = self._customMultiQueryMerge(self, res, limit if limit != -1 else self.queries[0].limit)
+                res = self._customMultiQueryMerge(self, res, limit)
             else:
                 # We must merge (and sort) the results ourself
                 res = self._merge_multi_query_results(res)
+
         else:  # We have just one single query
             res = self._fixKind(self._run_single_filter_query(
-                self.queries, limit if limit != -1 else self.queries.limit))
+                self.queries, limit if limit >= 0 else self.queries.limit))
+
         if res:
             self._lastEntry = res[-1]
+
         return res
 
     def count(self, up_to: int = 2 ** 63 - 1) -> int:
@@ -648,7 +678,6 @@ class Query(object):
         should be used.
 
         :param limit: Limits the query to the defined maximum entities.
-            A maximum value of 100 entries can be fetched at once.
 
         :raises: :exc:`BadFilterError` if a filter string is invalid
         :raises: :exc:`BadValueError` if a filter value is invalid.
@@ -659,19 +688,17 @@ class Query(object):
 
         if self.srcSkel is None:
             raise NotImplementedError("This query has not been created using skel.all()")
-        # limit = limit if limit != -1 else self._limit
-        if limit != -1 and not (0 < limit <= 100):
-            logging.error(("Limit", limit))
-            raise NotImplementedError(
-                "This query is not limited! You must specify an upper bound using limit() between 1 and 100")
 
         res = SkelList(self.srcSkel)
+
         for entity in self.run(limit):
             skel_instance = SkeletonInstance(self.srcSkel.skeletonCls, bone_map=self.srcSkel.boneMap)
             skel_instance.dbEntity = entity
             res.append(skel_instance)
+
         res.getCursor = lambda: self.getCursor()
         res.get_orders = lambda: self.get_orders()
+
         return res
 
     def iter(self) -> t.Iterator[Entity]:

{viur_core-3.8.0.dev7 → viur_core-3.8.0.dev9}/src/viur/core/db/types.py

@@ -7,7 +7,8 @@ import datetime
 import enum
 import typing as t
 from contextvars import ContextVar
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from ..config import conf
 
 from google.cloud.datastore import Entity as Datastore_entity, Key as Datastore_key
 
@@ -167,7 +168,7 @@ class QueryDefinition:
     distinct: t.Optional[list[str]] = None
     """If set, a list of fields that we should return distinct values of"""
 
-    limit: int = 30
+    limit: int = field(init=False)
     """The maximum amount of entities that should be returned"""
 
     startCursor: t.Optional[str] = None
@@ -178,3 +179,6 @@ class QueryDefinition:
 
     currentCursor: t.Optional[str] = None
     """Will be set after this query has been run, pointing after the last entity returned"""
+
+    def __post_init__(self):
+        self.limit = conf.db_query_default_limit