tinyjukebox 0.47.0__py3-none-any.whl

tinyjukebox/data/jsondata.py

@@ -0,0 +1,218 @@
+"""JSON file with schema validation and a backup copy of the previous version."""
+
+import contextlib
+import json
+import logging
+import os
+import threading
+import time
+from typing import Any
+
+import jsonschema
+
+from tinyjukebox.core.logging import get_child
+from tinyjukebox.data import degraded
+
+__all__ = ["JsonData", "tolerant"]
+
+
+def tolerant(schema: Any) -> Any:
+    """``schema`` without ``"additionalProperties": false``: a field that a newer version added is accepted."""
+    if isinstance(schema, list):
+        return [tolerant(item) for item in schema]
+    if not isinstance(schema, dict):
+        return schema
+    return {
+        key: tolerant(value) for key, value in schema.items() if not (key == "additionalProperties" and value is False)
+    }
+
+
+class JsonData:
+    """Content of a JSON file (``data``), validated against ``schema``.
+
+    Every write moves the current file to ``backup_filename`` first. If the regular file is missing, empty or
+    invalid on start, the backup is used and written back to the regular file. Without a usable file and backup,
+    ``FileNotFoundError`` is raised.
+
+    ``data`` may be changed by the owner; :meth:`store_data` writes it (only if it changed). :meth:`store` replaces
+    and writes the data in one step - the collections use it, so that concurrent writers never see half built data.
+
+    ``entries``: the file is a collection - an object whose values are entries of the same kind (tags, targets,
+    activities), and the box works with what it can read of it (docs/developer/library.md, section 4.21):
+
+    - Every entry is checked by itself. One that is damaged, or of a kind this version does not know, is set aside
+      (``aside``) instead of making the whole file unusable: it is not in ``data``, and it is written back unchanged
+      with every write - this version never deletes what a newer one wrote. An entry of ``data`` with the same key
+      replaces it.
+    - A field this version does not know is accepted; the owner keeps it (``extra`` of the entries).
+    - If neither the file nor its backup can be read, both are kept under a name of their own (``.damaged-<time>``)
+      and the collection starts empty instead of not starting.
+
+    What is set aside is noted in :mod:`tinyjukebox.data.degraded`.
+    """
+
+    def __init__(
+        self,
+        filename: str,
+        backup_filename: str,
+        schema: dict[str, Any],
+        logger: logging.Logger,
+        entries: bool = False,
+    ) -> None:
+        self._logger = get_child(logger, f"{type(self).__name__}.{filename}")
+        self.filename = filename
+        self.backup_filename = backup_filename
+        self._entries = entries
+        self._schema = tolerant(schema) if entries else schema
+        self._lock = threading.RLock()
+        self.last_string: str | None = None
+        self.aside: dict[str, Any] = {}
+        self.started_empty = False  # neither the file nor its backup could be read
+        self._source = os.path.basename(filename)
+
+        with self._lock:
+            read = self._read_data(self.filename)
+            if read is None:
+                self._logger.error("failed to load regular file - trying the backup file instead.")
+                read = self._read_data(self.backup_filename)
+                if read is None and not entries:
+                    raise FileNotFoundError(f"neither '{filename}' nor its backup '{backup_filename}' can be loaded.")
+                if read is None:
+                    read = self._start_empty()
+                self.data: dict[str, Any] = read[0]
+                self.aside = read[1]
+                self._logger.info("rewriting regular file with the content of the backup file.")
+                self.store_data(force_write=True, skip_backup=True)
+            else:
+                self.data, self.aside = read
+                self.last_string = self._serialize()
+            if entries:
+                self._note_aside()
+        self._logger.debug("__init__ - %s", self.data)
+
+    def _read_data(self, filename: str) -> tuple[dict[str, Any], dict[str, Any]] | None:
+        """The data of the file and the entries set aside - or None if the file cannot be used at all."""
+        self._logger.info("_read_data - from file %s", filename)
+        try:
+            with open(filename, encoding="utf-8") as infile:
+                data = json.load(infile)
+        except FileNotFoundError:
+            self._logger.error("_read_data - file %s not found.", filename)
+            return None
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            self._logger.error("_read_data - file %s is not valid json: %s", filename, e)
+            return None
+        if self._entries:
+            if not isinstance(data, dict):
+                self._logger.error("_read_data - file %s holds no object.", filename)
+                return None
+            return self.split(data)
+        if not self._validate_data(data):
+            self._logger.error("_read_data - content of file %s is not valid.", filename)
+            return None
+        return data, {}
+
+    def split(self, data: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
+        """The entries of a collection that this version can use, and the ones it sets aside."""
+        valid, aside = {}, {}
+        for key, entry in data.items():
+            error = self.entry_error(entry)
+            if error is None:
+                valid[key] = entry
+            else:
+                self._logger.warning("entry '%s' set aside: %s", key, error)
+                aside[key] = entry
+        return valid, aside
+
+    def entry_error(self, entry: Any) -> str | None:
+        """Why an entry of a collection cannot be used - None if it can."""
+        try:
+            jsonschema.validate(entry, self._schema["additionalProperties"])
+        except jsonschema.ValidationError as e:
+            return e.message
+        return None
+
+    def set_aside(self, key: str, entry: Any, why: str) -> None:
+        """The owner cannot use an entry that passed the schema: it is kept like a damaged one."""
+        with self._lock:
+            self._logger.warning("entry '%s' set aside: %s", key, why)
+            self.data.pop(key, None)
+            self.aside[key] = entry
+            self._note_aside()
+
+    def _note_aside(self) -> None:
+        message = "{}: entry '{}' cannot be used by this version and is left out"
+        degraded.replace(self._source, {key: message.format(self._source, key) for key in self.aside})
+
+    def _start_empty(self) -> tuple[dict[str, Any], dict[str, Any]]:
+        """Neither file nor backup can be read: both stay as they are under a name of their own - somebody may repair
+        them - and the collection starts empty. Nothing is overwritten."""
+        self.started_empty = True
+        stamp = time.strftime("%Y%m%d-%H%M%S")
+        kept = [name for name in (self.filename, self.backup_filename) if os.path.exists(name)]
+        for name in kept:
+            os.replace(name, f"{name}.damaged-{stamp}")
+        if not kept:  # nothing was lost: e.g. the positions of a box that has never played
+            self._logger.warning("%s: neither the file nor its backup exists; starting without entries", self._source)
+            return {}, {}
+        message = f"{self._source}: neither the file nor its backup can be read - kept as .damaged-{stamp}"
+        self._logger.error("%s; starting without its entries", message)
+        degraded.note(self._source + " (file)", "", message)
+        return {}, {}
+
+    def _validate_data(self, data: Any) -> bool:
+        try:
+            jsonschema.validate(data, self._schema)
+        except jsonschema.ValidationError as e:
+            self._logger.error("json validation error: %s", e.message)
+            return False
+        except jsonschema.SchemaError as e:
+            self._logger.error("json schema error: %s", e.message)
+            return False
+        return True
+
+    def _serialize(self) -> str:
+        aside = {key: entry for key, entry in self.aside.items() if key not in self.data}
+        return json.dumps({**self.data, **aside}, indent=4)
+
+    def store(self, data: dict[str, Any], aside: dict[str, Any] | None = None) -> None:
+        """Replace ``data`` - and the entries set aside, if given - and write it, atomically with respect to other
+        writers."""
+        with self._lock:
+            previous, self.data = self.data, data
+            previous_aside = self.aside
+            if aside is not None:
+                self.aside = aside
+            try:
+                self.store_data()
+            except ValueError:
+                self.data, self.aside = previous, previous_aside
+                raise
+            if aside is not None and self._entries:
+                self._note_aside()
+
+    def store_data(self, force_write: bool = False, skip_backup: bool = False) -> None:
+        """Write ``data`` if it changed (or ``force_write``); the previous file becomes the backup file.
+
+        :raises ValueError: if ``data`` does not match the schema.
+        """
+        with self._lock:
+            if not self._validate_data(self.data):
+                raise ValueError(f"store_data - data for '{self.filename}' does not match the schema.")
+            new_string = self._serialize()
+            if not force_write and new_string == self.last_string:
+                self._logger.debug("store_data - skipped, data not changed.")
+                return
+            # the box is switched off by pulling the plug: the new content is on the disk before it replaces anything -
+            # otherwise the file can be empty after a power loss
+            temp = self.filename + ".tmp"
+            with open(temp, "w", encoding="utf-8") as outfile:
+                outfile.write(new_string)
+                outfile.flush()
+                os.fsync(outfile.fileno())
+            if not skip_backup:
+                with contextlib.suppress(FileNotFoundError):  # the file does not exist yet
+                    os.replace(self.filename, self.backup_filename)
+            os.replace(temp, self.filename)  # the regular file is always complete - or missing (-> backup)
+            self.last_string = new_string
+            self._logger.debug("store_data - written to %s.", self.filename)

tinyjukebox/data/sanitycheck.py

@@ -0,0 +1,71 @@
+"""Consistency checks of tags, targets and activities."""
+
+import logging
+
+from tinyjukebox.core.logging import get_child
+from tinyjukebox.data.activities import Activities
+from tinyjukebox.data.common import is_unmapped
+from tinyjukebox.data.tags import Tags
+from tinyjukebox.data.targets import Targets
+
+__all__ = ["SanityCheck"]
+
+
+class SanityCheck:
+    """Checks every collection and the references between them; every problem is logged as warning."""
+
+    def __init__(self, logger: logging.Logger, targets: Targets, tags: Tags, activities: Activities) -> None:
+        self._logger = get_child(logger, type(self).__name__)
+        self.targets = targets
+        self.tags = tags
+        self.activities = activities
+
+    def sanity_check(self) -> bool:
+        targets_passed, targets_errors = self.targets.sanity_check()
+        if not targets_passed:
+            self._logger.warning("sanity_check - targets: %s", targets_errors)
+        # evaluate all checks (no short circuit) so that every problem gets logged
+        results = [
+            self.tags.sanity_check(),
+            targets_passed,
+            self.activities.sanity_check(),
+            self._cross_check_tags_and_activity(),
+            self._cross_check_tags_and_targets(),
+        ]
+        passed = all(results)
+        if not passed:
+            self._logger.warning("sanity_check - sanity check failed")
+        return passed
+
+    def _cross_check_tags_and_activity(self) -> bool:
+        uids_tags = {tag.tag_uid for tag in self.tags.all()}
+        uids_activity = set(self.activities.all())
+        only_in_activity = uids_activity - uids_tags
+        only_in_tags = uids_tags - uids_activity
+        if only_in_activity:
+            self._logger.warning(
+                "_cross_check_jsons - activity json contains uids that are not in tags json:%s", only_in_activity
+            )
+        if only_in_tags:
+            # activities are created on first playback - missing entries are expected for new or unmapped tags
+            self._logger.info(
+                "_cross_check_jsons - tags json contains uids that are not in activity json:%s", only_in_tags
+            )
+        return not only_in_activity
+
+    def _cross_check_tags_and_targets(self) -> bool:
+        passed = True
+        for tag in self.tags.all():
+            if is_unmapped(tag.target_id):
+                continue
+            target = self.targets.targets.get(str(tag.target_id))
+            if target is None or target.target_type != tag.target_type:
+                passed = False
+                target_type = tag.target_type.name if tag.target_type is not None else None
+                self._logger.warning(
+                    "_cross_check_jsons - tag '%s' contains not existing %s target_id '%s'",
+                    tag.tag_uid,
+                    target_type,
+                    tag.target_id,
+                )
+        return passed

tinyjukebox/data/tag.py

@@ -0,0 +1,98 @@
+"""An rfid tag and the target (album) it is linked to."""
+
+import logging
+from typing import Any
+
+from tinyjukebox.core.logging import get_child
+from tinyjukebox.data.common import NO_TARGET, TargetType, UpdateJson, extra_fields, is_valid_timestamp, now
+
+__all__ = ["Tag"]
+
+
+class Tag:
+    """Entry of ``tags.json``::
+
+        {"tag": {"uid": "1:2:3:4", "name": "Album A"}, "target": {"type": "album", "id": "7"}, "created": "..."}
+
+    ``target_id`` is ``-1`` as long as the tag is not linked to an album. Every change calls ``update_json`` of the
+    owning collection.
+    """
+
+    def __init__(self, logger: logging.Logger, tag: dict[str, Any], update_json: UpdateJson | None) -> None:
+        uid = tag.get("tag", {}).get("uid")
+        if not isinstance(uid, str) or not uid:
+            message = f"Tag.__init__ - tag.uid must be a non empty string (dict: {tag})."
+            logger.error(message)
+            raise ValueError(message)
+        self.tag_uid: str | None = uid
+        self._logger = get_child(logger, f"{type(self).__name__}.{uid}")
+        self._logger.debug("__init__ - %s", tag)
+        self._update_json = update_json
+        self.load(tag)
+
+    def load(self, tag: dict[str, Any]) -> None:
+        """Take over name, target and time stamp of ``tag`` (an entry of ``tags.json``) - nothing is written."""
+        self.tag_name: str = tag["tag"].get("name", "")
+        target = tag.get("target", {})
+        # fields of a newer version, on each level: kept and written back
+        self._extra = (extra_fields(tag, ("tag", "target", "created")), extra_fields(tag["tag"], ("uid", "name")),
+                       extra_fields(target, ("type", "id")))  # fmt: skip
+        type_name = str(target.get("type") or "album").lower()
+        try:
+            self.target_type: TargetType | None = TargetType[type_name]
+        except KeyError:
+            self._logger.error("__init__ - unknown target type %s.", type_name)
+            raise ValueError(f"unknown target type {type_name}.") from None
+        self.target_id: int | str | None = target.get("id", NO_TARGET)
+        self.created: str | None = tag.get("created") or now()
+
+    def _changed(self) -> None:
+        if self._update_json is not None:
+            self._update_json()
+
+    def set_target_type(self, target_type: TargetType) -> None:
+        if not isinstance(target_type, TargetType):
+            raise ValueError("set_target_type - target_type must be a TargetType.")
+        self.target_type = target_type
+        self._changed()
+
+    def set_target_id(self, target_id: int | str | None) -> None:
+        if target_id is None or not str(target_id):
+            self._logger.warning("set_target_id - target_id is None or empty.")
+            return
+        self.target_id = target_id
+        self._changed()
+
+    def set_tag_name(self, tag_name: str) -> None:
+        self.tag_name = tag_name
+        self._changed()
+
+    @classmethod
+    def create_tag(cls, logger: logging.Logger, tag_uid: str, update_json: UpdateJson | None) -> "Tag":
+        return Tag(logger, {"tag": {"uid": tag_uid}}, update_json)
+
+    def to_dict(self) -> dict[str, Any]:
+        extra, extra_tag, extra_target = self._extra
+        target_type = self.target_type.name if self.target_type is not None else None
+        return {
+            "tag": {"uid": self.tag_uid, "name": self.tag_name, **extra_tag},
+            "target": {"type": target_type, "id": self.target_id, **extra_target},
+            "created": self.created,
+            **extra,
+        }
+
+    def sanity_check(self) -> bool:
+        result = True
+        if not is_valid_timestamp(self.created):
+            self._logger.warning("sanity_check - created timestamp error '%s'", self.created)
+            result = False
+        if not self.tag_uid:
+            self._logger.warning("sanity_check - tag_uid not set.")
+            result = False
+        return result
+
+    def to_text(self) -> str:
+        target_type = self.target_type.name if self.target_type is not None else "None"
+        return (
+            f"tag: {self.tag_uid}/'{self.tag_name}' -> target: {target_type}/{self.target_id}. created: {self.created}"
+        )

tinyjukebox/data/tags.py

@@ -0,0 +1,160 @@
+"""All known rfid tags (``tags.json``)."""
+
+import logging
+import threading
+from collections.abc import Callable
+from typing import Any
+
+from tinyjukebox.core.logging import get_child
+from tinyjukebox.data.common import Listener, Mirror, TargetType, is_unmapped
+from tinyjukebox.data.jsondata import JsonData
+from tinyjukebox.data.tag import Tag
+from tinyjukebox.schema.tags import schema
+
+__all__ = ["Tags"]
+
+
+class Tags:
+    """Tags by uid. Changes are written to the json file immediately.
+
+    Tags are changed by the state machine (new tags), the rest api and the command line - concurrently. ``_lock``
+    serializes changes of the collection and the writing of the file; it is never held while waiting for anything
+    else. Readers in other threads use :meth:`all` (a snapshot) instead of iterating ``tags``.
+    """
+
+    def __init__(self, filename: str, logger: logging.Logger) -> None:
+        self._logger = get_child(logger, type(self).__name__)
+        self._lock = threading.RLock()
+        self._json = JsonData(filename, filename + "_", schema, self._logger, entries=True)
+        self._mirror = Mirror()
+        self.tags: dict[str, Tag] = {}
+        for key, tag in list(self._json.data.items()):
+            try:
+                self.tags[key] = Tag(self._logger, tag, self._update_json)
+            except ValueError as e:  # a tag this version cannot use does not keep the box from starting
+                self._json.set_aside(key, tag, str(e))
+
+    @property
+    def unavailable(self) -> set[str]:
+        """The uids of tags this version cannot use (docs/developer/library.md, section 4.21): known, but not
+        playable here - never taken for new tags."""
+        with self._lock:
+            return set(self._json.aside) - set(self.tags)
+
+    def _to_dict(self) -> dict[str, Any]:
+        return {key: tag.to_dict() for key, tag in self.tags.items()}
+
+    def _update_json(self) -> None:
+        with self._lock:
+            data = self._to_dict()
+            self._json.store(data)
+            self._mirror.changed(data)
+
+    def listen(self, listener: Listener) -> None:
+        """``listener(uid, entry)`` is called for every tag that changes from now on (``None``: deleted)."""
+        with self._lock:
+            self._mirror.listen(listener, self._to_dict())
+
+    def sync(self, get: Callable[[], dict[str, Any]]) -> bool:
+        """Make the tags what ``get()`` returns (the content of ``tags.json``), keeping the tags that stay. The
+        listener is not called. Returns whether something changed."""
+        with self._lock:
+            data, aside = self._json.split(get())
+            if data == self._to_dict() and aside == self._json.aside:
+                return False
+            for key in [key for key in self.tags if key not in data]:
+                del self.tags[key]
+            for key, entry in data.items():
+                try:
+                    if key in self.tags:
+                        self.tags[key].load(entry)
+                    else:
+                        self.tags[key] = Tag(self._logger, entry, self._update_json)
+                except ValueError as e:
+                    self._logger.warning("sync - tag '%s' set aside: %s", key, e)
+                    self.tags.pop(key, None)
+                    aside[key] = entry
+            data = self._to_dict()
+            self._json.store(data, aside)
+            self._mirror.replaced(data)
+            return True
+
+    def all(self) -> list[Tag]:
+        """Snapshot of all tags."""
+        with self._lock:
+            return list(self.tags.values())
+
+    def add_tag(self, uid: str | None) -> Tag:
+        """Add an unmapped tag. :raises ValueError: if the uid is empty or already known."""
+        if not uid:
+            message = "add_tag - uid must not be None or empty."
+            self._logger.error(message)
+            raise ValueError(message)
+        with self._lock:
+            if uid in self.tags or uid in self._json.aside:
+                message = f"add_tag - tag already exists with uid '{uid}'"
+                self._logger.error(message)
+                raise ValueError(message)
+            tag = Tag.create_tag(self._logger, uid, self._update_json)
+            self.tags[uid] = tag
+            self._update_json()
+        return tag
+
+    def delete_tag(self, uid: str | None) -> None:
+        with self._lock:
+            if uid not in self.tags:
+                self._logger.error("delete_tag - unknown tag %s.", uid)
+                raise ValueError(f"delete_tag - unknown tag {uid}.")
+            del self.tags[uid]
+            self._update_json()
+
+    def get_tag(self, uid: str | None) -> Tag:
+        """:raises ValueError: for unknown uids."""
+        tag = self.tags.get(uid) if uid is not None else None
+        if tag is None:
+            self._logger.warning("get_tag - unknown tag %s.", uid)
+            raise ValueError(f"get_tag - unknown tag {uid}.")
+        return tag
+
+    def get_used_target_ids(self, target_type: object) -> list[int | str]:
+        """Distinct ids of all targets of ``target_type`` linked to a tag - numeric ids sorted numerically first."""
+        if not isinstance(target_type, TargetType):
+            self._logger.error("get_used_target_ids - target_type must be a TargetType, got %r.", target_type)
+            raise ValueError("get_used_target_ids - target_type must be a TargetType.")
+        target_ids = {
+            tag.target_id
+            for tag in self.all()
+            if tag.target_type == target_type and tag.target_id is not None and not is_unmapped(tag.target_id)
+        }
+        return sorted(target_ids, key=lambda i: (0, int(i), "") if str(i).isdigit() else (1, 0, str(i)))
+
+    def assign_target(self, tag_uid: str, target_type: object, target_id: int | str | None) -> None:
+        """Link the tag with a target. :raises ValueError: for unknown tags or invalid target type/id."""
+        try:
+            tag = self.get_tag(tag_uid)
+        except ValueError:
+            self._logger.error("assign_target - unknown tag_uid %s.", tag_uid)
+            raise
+        if target_id is None or not str(target_id):
+            self._logger.error("assign_target - target_id is None or empty")
+            raise ValueError("assign_target - target_id is None or empty")
+        if not isinstance(target_type, TargetType):
+            self._logger.error("assign_target - target_type must be a TargetType, got %r.", target_type)
+            raise ValueError("assign_target - target_type must be a TargetType.")
+        with self._lock:
+            if tag.target_type != target_type:
+                tag.set_target_type(target_type)
+            tag.set_target_id(target_id)
+
+    def sanity_check(self) -> bool:
+        result = True
+        for key, tag in list(self.tags.items()):
+            if key != tag.tag_uid:
+                self._logger.warning("sanity_check - key '%s' != tag.tag_uid '%s'", key, tag.tag_uid)
+                result = False
+            if not tag.sanity_check():
+                result = False
+        return result
+
+    def to_text(self) -> str:
+        return "\n".join(f"key: {key}, tag: {tag.to_text()}" for key, tag in list(self.tags.items()))