openmodule 13.0.2__tar.gz → 13.1.0rc0__tar.gz

{openmodule-13.0.2 → openmodule-13.1.0rc0}/AUTHORS

@@ -4,7 +4,6 @@ Matthias Mietschnig <m.mietschnig@arivo.co>
 Philipp Reitter <i@philipp.ninja>
 Philipp Reitter <p.reitter@accessio.at>
 Philipp Reitter <p.reitter@gmail.com>
-Stefan Bräuer <s.braeuer@ariv.co>
 Stefan Bräuer <s.braeuer@arivo.co>
 Thomas Senfter <t.senfter@accessio.at>
 Thomas Senfter <t.senfter@arivo.co>

{openmodule-13.0.2 → openmodule-13.1.0rc0}/ChangeLog

@@ -1,6 +1,20 @@
 CHANGES
 =======
 
+v13.1.0.rc0
+-----------
+
+* OM-61 Openmodule Anpassungen für Access Events
+* [OM-56] added enter\_time and leave\_time to vehicle and implemented presence message changes
+* fixes and refactoring of presencelistener
+* Resolve OM-78 improved KVStore performance
+* fixes INBOX-2510
+
+v13.0.3
+-------
+
+* added garage\_settings mode "barrier\_bypass" in settings models removal of some unused imports
+
 v13.0.2
 -------
 
@@ -128,8 +142,6 @@ v11.1.0.rc0
 * removing ="asdf" for strings
 * added decimal separator support for english
 * some testcases for csv exporter, incorrect default value + static missing [skip ci]
-* fixed comment
-* added csv\_export library (no testcases)
 * removed mock rpcs from schema
 
 v11.0.3
@@ -217,28 +229,3 @@ v8.1.2
 
 * dont register schemas for mock handlers
 * fixes a race condition in the io tests
-* update known issues
-
-v8.1.1
-------
-
-* adds multiprocessing logging to openmodule requirements
-
-v8.1.0
-------
-
-* changelog and breaking changes updated
-* \* Adds an IO Listener \* Adds a SigTERM interrupt handler which converts to KeyboardInterrupt so shutdown in Docker containers is easier \* The main Test Mixin was refactored to start a separate process and raise a Interrupt. The other test methods were removed due to being hacky and crashing tox/pytest from time to time. Since this only breaks test code we don't bump the major version. \* The openmodule now waits for a connection with the message broker on startup \* Fixes an issue in the RPCServerTestMixin
-* RPCServerTestMixin now only registers one random channel instead of all channels on the server. (Author: Philipp Reitter)
-* add infos to CountMessage
-* Added line to exception\_in\_function to call new \_signal\_in\_function if exception is a signal or KeyboardInterrupt
-
-v8.0.1
-------
-
-* fixed loading empty yamls fixes rpc server not logging info in debug log
-
-v8.0.0
-------
-
-* fix in ci job

{openmodule-13.0.2 → openmodule-13.1.0rc0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 13.0.2
+Version: 13.1.0rc0
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO

{openmodule-13.0.2 → openmodule-13.1.0rc0}/docs/access_service.md

@@ -21,6 +21,11 @@ You have to implement the `rpc_check_access` function and probably add some othe
 
 An example implementations is shown below.
 
+**Important:** Every Access Service has to add their human-readable description in the `AccessCheckResponse`.
+The default value of `readable_service_name` is `_("__READABLE_NAME")`,  so you do not have to add
+this variable explicitly.
+
+
 ```python
 def rpc_check_access(self, request: AccessCheckRequest, _) -> AccessCheckResponse:
     """
@@ -44,9 +49,28 @@ def rpc_check_access(self, request: AccessCheckRequest, _) -> AccessCheckRespons
 
     self.log.info(f"Found {len(accesses)} matching accesses "
                   f"where {len([a for a in accesses if a.accepted])} are valid")
-    return AccessCheckResponse(success=True, accesses=accesses)
+    return AccessCheckResponse(success=True, accesses=accesses, readable_service_name=_("Test Access Service"))
 ```
 
+### Access Service Reject Reasons
+
+In the enum `AccessCheckRejectReason` common reject reasons can be found.
+
+#### Reject Reason "wrong_gate"
+
+This reject reason is automatically set if the function `check_accesses_valid_at_gate()` is used.
+
+#### Reject Reason "wrong_time"
+
+If your access service has some time restriction the reject reason `wrong_time` can be returned.
+**Do not** use this for accesses with start and end (e.g. permanent access for a month)
+as these accesses should be ignored if outside start and end.
+
+#### Reject Reason "custom"
+
+If the given e.g. license plate should be rejected for a specific other reason the value `custom` should
+be used. If the reject reason `custom` is used then the string field `supplementary_infos` has to be set!
+
 ## AccessServiceWithSessions
 
 In addition to the Access Service stuff implement the session handling functions `check_in_session`, `check_out_session`

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/config.py

@@ -234,12 +234,13 @@ def testing():
 
 
 def database_folder() -> str:
-    path = string("DATABASE_FOLDER", f"/data/sqlite/")
     if testing():
-        path = "../sqlite/test/"
+        default_path = "../sqlite/test/"
     elif debug():
-        path = "../sqlite/debug/"
-    return path
+        default_path = "../sqlite/debug/"
+    else:
+        default_path = "/data/sqlite/"
+    return string("DATABASE_FOLDER", default_path)
 
 
 def run_checks() -> bool:

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/models/access_service.py

@@ -1,18 +1,12 @@
+from datetime import datetime
 from enum import Enum
-from typing import List, Optional, Dict, Union
+from typing import List, Optional, Dict
 
 from pydantic import Field, root_validator
 
 from openmodule.models.base import OpenModuleModel, timezone_validator
 from openmodule.models.vehicle import Medium, LPRMedium, MakeModel
-
-from dateutil import tz
-from dateutil.parser import parse
-from dateutil.rrule import rrulestr
-from dateutil.tz import UTC
-from pydantic import validator
-import re
-from datetime import datetime, tzinfo, timedelta
+from openmodule.utils.translation import _
 
 
 class AccessCategory(str, Enum):
@@ -72,6 +66,7 @@ class AccessCheckAccess(OpenModuleModel):
     category: AccessCategory           # category used for sorting and eventlog
     used_medium: Medium                # medium of the access
     access_data: Dict                  # complete access data, will be used only for display and debug purposes
+    supplementary_infos: Optional[str]  # field has to be if reject reason is "custom"
 
     accepted: bool                     # if access service decided access can enter
     reject_reason: Optional[AccessCheckRejectReason]  # only if not accepted: reason for not accepted
@@ -88,9 +83,12 @@ class AccessCheckAccess(OpenModuleModel):
         if values["category"] in [AccessCategory.shortterm, AccessCategory.unknown] \
                 and values["group_limit"] is not None:
             raise ValueError("group_limit must not be set for shortterm and unknown accesses")
+        if values.get("reject_reason") == AccessCheckRejectReason.custom and not values.get("supplementary_infos"):
+            raise ValueError("supplementary_infos must be set if reject_reason is custom")
         return values
 
 
 class AccessCheckResponse(OpenModuleModel):
     success: bool
-    accesses: List[AccessCheckAccess] = []     # list of all matched accesses (including already rejected ones)
+    accesses: List[AccessCheckAccess] = []  # list of all matched accesses (including already rejected ones)
+    readable_service_name: str = _("__READABLE_NAME")  # readable name of the service

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/models/presence.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import Optional, List
 
-from pydantic import Field
+from pydantic import Field, root_validator
 
 from openmodule.models.base import ZMQMessage, OpenModuleModel, Gateway, timezone_validator
 from openmodule.models.vehicle import Medium, LPRMedium, MakeModel, PresenceAllIds, EnterDirection, QRMedium
@@ -24,14 +24,23 @@ class PresenceBaseData(OpenModuleModel):
     make_model: Optional[MakeModel]
     all_ids: PresenceAllIds
     enter_direction: EnterDirection = EnterDirection.unknown
+    enter_time: datetime
 
     _tz_last_update = timezone_validator("last_update")
+    _tz_enter_time = timezone_validator("enter_time")
 
     class Config:
         # allows setting attributes both via the alias, and the field name.
         # is used to rename old variables which are hard to understand by their name (e.g. id -> medium id)
         allow_population_by_field_name = True
 
+    @root_validator(pre=True)
+    def set_enter_time_default(cls, values):
+        """for backward compatibility, set enter_time based on vehicle_id"""
+        if values.get("enter_time") is None:
+            values["enter_time"] = values["vehicle_id"] / 1e9  # timestamp from vehicle_id
+        return values
+
 
 class PresenceBaseMessage(PresenceBaseData, ZMQMessage):
     pass
@@ -58,6 +67,15 @@ class PresenceForwardMessage(PresenceBaseMessage):
 class PresenceLeaveMessage(PresenceBaseMessage):
     type: str = "leave"
     num_presents: int = Field(0, alias="num-presents")
+    leave_time: datetime = Field(..., alias="leave-time")
+
+    _tz_leave_time = timezone_validator("leave_time")
+
+    @root_validator(pre=True)
+    def set_leave_time_default(cls, values):
+        if values.get("leave_time") is None:
+            values["leave_time"] = values["timestamp"]  # timestamp from message timestamp
+        return values
 
 
 class PresenceEnterMessage(PresenceBaseMessage):

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/models/validation.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import List, Optional, Union
+from typing import List, Optional
 
 from openmodule.models.base import OpenModuleModel, base64_validator, CostEntryData
 

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/models/vehicle.py

@@ -1,7 +1,8 @@
+from datetime import datetime
 from enum import Enum
 from typing import Optional, List, Iterable, Union
 
-from openmodule.models.base import OpenModuleModel
+from openmodule.models.base import OpenModuleModel, timezone_validator
 
 
 def _truncate(x):
@@ -115,6 +116,11 @@ class Vehicle(OpenModuleModel):
     make_model: Optional[MakeModel]
     all_ids: PresenceAllIds = []
     enter_direction: EnterDirection = EnterDirection.unknown
+    enter_time: datetime
+    leave_time: Optional[datetime]
+
+    _tz_enter_time = timezone_validator("enter_time")
+    _tz_leave_time = timezone_validator("leave_time")
 
     def iterate_media(self) -> Iterable[Union[Medium, LPRMedium]]:
         """

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/rpc/client.py

@@ -203,3 +203,6 @@ class RPCClient:
         if entry:
             entry.response = response.response
             entry.ready.set()
+        else:
+            self.log.error("Received response after timeout from %s, type: %s, id: %s",
+                           response.name, response.type, response.rpc_id)

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/sentry.py

@@ -1,4 +1,3 @@
-import logging
 from socket import gethostname
 from typing import Optional, Dict, Any
 from typing import TYPE_CHECKING

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/threading.py

@@ -4,7 +4,6 @@ import os
 import sys
 from functools import partial
 
-import logging
 import time
 from zmq import ContextTerminated
 from openmodule.config import settings

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/utils/eventlog.py

@@ -1,13 +1,15 @@
+import logging
 from datetime import datetime
 from enum import Enum
 from typing import Optional, Dict, Union
 
 from pydantic import validator, root_validator
 
-from openmodule.config import testing
 from openmodule.core import core
 from openmodule.models.base import OpenModuleModel, ZMQMessage, timezone_validator
 
+log = logging.getLogger("Eventlog")
+
 
 class AnonymizationType(str, Enum):
     lpr = "lpr"  # anonymize to "***AB"
@@ -119,4 +121,6 @@ class EventlogMessage(ZMQMessage):
 
 
 def send_event(infos: EventInfo, message: str, **message_kwargs: MessageKwarg):
+    format_kwargs = {key: kwarg.value for key, kwarg in message_kwargs.items()}
+    log.info(f"Sending event: {message.format(**format_kwargs)}")
     EventlogMessage.create(infos=infos, message=message, **message_kwargs).send()

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/utils/kv_store.py

@@ -1,5 +1,4 @@
 import hashlib
-import json
 import logging
 import random
 import threading
@@ -7,6 +6,7 @@ import time
 import warnings
 from typing import Type, Optional, List, Union, Dict
 
+import orjson
 from sqlalchemy import Column, String, Integer
 from sqlalchemy.ext.declarative import declarative_base
 
@@ -41,12 +41,12 @@ class KVEntry:
         """
         raise NotImplementedError
 
-    def comparison_value(self) -> str:
+    def comparison_value(self) -> str:  # pragma: no cover
         """
-        return some hash or other string. Changed messages are only sent if this value changed for a key
-        if you don't want changed messages, use for efficiency:
+        Only used for KVStoreWithChangedNotification
+        return some hash or string. Changed messages are only sent if this value changed for a key
         """
-        return self.key
+        raise NotImplementedError
 
 
 class KVStore:
@@ -93,24 +93,24 @@ class KVStore:
         """RPCs for kv_sync are filtered by service name"""
         return request.service == self.service_name
 
-    def _kvs_check_etag_mismatched(self, request: KVSetRequest, current_kvs: List[KVEntry]):
+    def _kvs_check_etag_mismatched(self, request: KVSetRequest, current_kvs: Dict[str, int]):
         for kv in request.kvs:
-            for current in current_kvs:
-                if kv.key == current.key and kv.previous_e_tag == current.e_tag:
-                    break
-                if kv.key == current.key and kv.previous_e_tag is None:
-                    self.log.warning(f"Key exists but no previous ETag given for key {kv.key}")
-                    raise self.ETagMismatchException("No previous ETag")
-            else:
-                if kv.previous_e_tag is not None:
-                    self.log.warning(f"Key does not exists but previous ETag given for key {kv.key}")
-                    raise self.ETagMismatchException("Previous ETag given but not in database")
+            current_e_tag = current_kvs.get(kv.key)
+            if current_e_tag is None and kv.previous_e_tag is not None:
+                self.log.warning(f"Key does not exists but previous ETag given for key {kv.key}")
+                raise self.ETagMismatchException("Previous ETag given but not in database")
+            elif current_e_tag is not None and kv.previous_e_tag is None:
+                self.log.warning(f"Key exists but no previous ETag given for key {kv.key}")
+                raise self.ETagMismatchException("No previous ETag")
+            elif current_e_tag != kv.previous_e_tag:
+                self.log.warning(f"Previous ETag does not match for key {kv.key}")
+                raise self.ETagMismatchException("Previous ETag does not match")
 
     def _kvs_set_parse_kvs(self, kvs: List[KVSetRequestKV]) -> List[KVEntry]:
         to_set = []
         deprecation_warning = False
         for kv in kvs:
-            value = json.loads(kv.value)
+            value = orjson.loads(kv.value)
             if value is not None:
                 instances = self.database_table.parse_value(value)
                 # fallback for old parse_value() implementations
@@ -134,14 +134,6 @@ class KVStore:
                 to_set += instances
         return to_set
 
-    def _get_comparison_values(self, updated_kv_entries: List[KVEntry]):
-        # only used in KVStoreWithChangedNotification
-        pass
-
-    def _send_changed_notification(self, success_keys: List[str], old_values: Dict[str, str]):
-        # only used in KVStoreWithChangedNotification
-        pass
-
     def kvs_set(self, request: KVSetRequest, _) -> KVSetResponse:
         """
         This method is called by the server to set values in the database.
@@ -154,24 +146,23 @@ class KVStore:
         if not request.kvs:
             return KVSetResponse()
         with self.db() as session:
-            current_kvs = \
-                session.query(self.database_table).filter(self.database_table.key.in_([kv.key for kv in request.kvs]))
-            comparison_values = self._get_comparison_values(current_kvs)
+            current_kvs: Dict[str, int] = \
+                {kv[0]: kv[1]
+                 for kv in session.query(self.database_table).
+                 filter(self.database_table.key.in_([kv.key for kv in request.kvs])).
+                 with_entities(self.database_table.key, self.database_table.e_tag).all()}
             self._kvs_check_etag_mismatched(request, current_kvs)  # raises Exception on mismatch
             to_set = self._kvs_set_parse_kvs(request.kvs)
             try:
                 if current_kvs:
                     delete_query(session, session.query(self.database_table).filter(
-                        self.database_table.key.in_([kv.key for kv in current_kvs])))
+                        self.database_table.key.in_(list(current_kvs.keys()))))
                 if to_set:
                     session.add_all(to_set)
                 session.commit()
             except Exception as e:
                 session.rollback()
                 raise e
-
-        if request.kvs:
-            self._send_changed_notification([s.key for s in request.kvs], comparison_values)
         return KVSetResponse()
 
     def kvs_sync(self, request: KVSyncRequest, _) -> KVSyncResponse:
@@ -183,15 +174,20 @@ class KVStore:
         """
         self.log.debug("Syncing kvs for %s", request.service)
         with self.db() as session:
-            current_kvs = session.query(self.database_table).all()
-            changed_kvs = {key: current.e_tag
-                           for current in current_kvs for key, e_tag in request.kvs.items()
-                           if current.key == key and current.e_tag != e_tag}
-            missing_kvs = {key: e_tag for key, e_tag in request.kvs.items() if
-                           not any(current for current in current_kvs if current.key == key)}
-            additional_kvs = {current.key: current.e_tag for current in current_kvs
-                              if not any(key for key, _ in request.kvs.items() if key == current.key)}
-            return KVSyncResponse(additions=additional_kvs, changes=changed_kvs, missing=missing_kvs)
+            current_kvs: Dict[str, int] = \
+                {kv[0]: kv[1]
+                 for kv in session.query(self.database_table).
+                 with_entities(self.database_table.key, self.database_table.e_tag).all()}
+        changed_kvs = {}
+        missing_kvs = {}
+        for key, server_e_tag in request.kvs.items():
+            device_e_tag = current_kvs.pop(key, None)
+            if device_e_tag is None:
+                missing_kvs[key] = server_e_tag
+            elif device_e_tag != server_e_tag:
+                changed_kvs[key] = device_e_tag
+        additional_kvs = current_kvs
+        return KVSyncResponse(additions=additional_kvs, changes=changed_kvs, missing=missing_kvs)
 
     def _log_sync_rpc_response(self, rpc_entry: RPCClient.RPCEntry) -> bool:
         """
@@ -329,25 +325,55 @@ class KVStoreHandler:
 
 
 class KVStoreWithChangedNotification(KVStore):
-    def _get_comparison_values(self, updated_kv_entries: List[KVEntry]):
+    def _get_comparison_values(self, kv_entries: List[KVEntry]) -> Dict[str, str]:
         """
         make sure your db model has a meaningful `comparison_value` method
         """
-        raise NotImplementedError
+        return {kv.key: kv.comparison_value() for kv in kv_entries}
 
-    def _find_changed_kvs(self, success_keys: List[str], old_values: Dict[str, str]):
+    def _find_changed_kvs(self, new_values: Dict[str, str], old_values: Dict[str, str]) -> List[str]:
         changed = []
-        with self.db() as session:
-            kvs = session.query(self.database_table).filter(self.database_table.key.in_(success_keys)).all()
-            new_values = {kv.key: kv.comparison_value() for kv in kvs}
-            for key in success_keys:
-                if key not in old_values or key not in new_values or old_values[key] != new_values[key]:
-                    changed.append(key)
+        for key, new_value in new_values.items():
+            old_value = old_values.pop(key, None)
+            if old_value is None or old_value != new_value:
+                changed.append(key)
+        changed += list(old_values.keys())
         return changed
 
-    def _send_changed_notification(self, success_keys: List[str], old_values: Dict[str, str]):
+    def _send_changed_notification(self, new_values: Dict[str, str], old_values: Dict[str, str]): # pragma: no cover
         """
         consider using self._find_changed_kvs when implementing this
         also make sure your db model has a meaningful `comparison_value` method
         """
         raise NotImplementedError
+
+    def kvs_set(self, request: KVSetRequest, _) -> KVSetResponse:
+        """
+        This method is called by the server to set values in the database.
+        It must never be called from the device.
+        Database entries must only be created/edited/deleted by this method.
+        Entries are never changed, only deleted and recreated with new values and e_tag.
+        The e_tag of the current value of a key needs to match previous_e_tag in the request, otherwise the key is
+            rejected.
+        """
+        if not request.kvs:
+            return KVSetResponse()
+        with self.db() as session:
+            full_current_kvs: List[KVEntry] = session.query(self.database_table). \
+                filter(self.database_table.key.in_([kv.key for kv in request.kvs]))
+            comparison_values = self._get_comparison_values(full_current_kvs)
+            current_kvs = {kv.key: kv.e_tag for kv in full_current_kvs}
+            self._kvs_check_etag_mismatched(request, current_kvs)  # raises Exception on mismatch
+            to_set = self._kvs_set_parse_kvs(request.kvs)
+            try:
+                if current_kvs:
+                    delete_query(session, session.query(self.database_table).filter(
+                        self.database_table.key.in_(list(current_kvs.keys()))))
+                if to_set:
+                    session.add_all(to_set)
+                session.commit()
+                self._send_changed_notification(self._get_comparison_values(to_set), comparison_values)
+            except Exception as e:
+                session.rollback()
+                raise e
+        return KVSetResponse()

{openmodule-13.0.2 → openmodule-13.1.0rc0}/openmodule/utils/matching.py

@@ -1,5 +1,4 @@
 import glob
-import threading
 import unicodedata
 from io import StringIO