dbetto 1.0.2__py3-none-any.whl

dbetto/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2022 Luigi Pertoldi <gipert@pm.me>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+from ._version import version as __version__
+from .attrsdict import AttrsDict
+from .textdb import TextDB
+from .time import str_to_datetime
+
+__all__ = [
+    "__version__",
+    "TextDB",
+    "AttrsDict",
+    "str_to_datetime",
+]

dbetto/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.0.2'
+__version_tuple__ = version_tuple = (1, 0, 2)

dbetto/_version.pyi

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+version: str
+version_tuple: tuple[int, int, int] | tuple[int, int, int, str, str]

dbetto/attrsdict.py

@@ -0,0 +1,242 @@
+# Copyright (C) 2022 Luigi Pertoldi <gipert@pm.me>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+log = logging.getLogger(__name__)
+
+
+class AttrsDict(dict):
+    """Access dictionary items as attributes.
+
+    Examples
+    --------
+    >>> d = AttrsDict({"key1": {"key2": 1}})
+    >>> d.key1.key2
+    1
+    >>> d1 = AttrsDict()
+    >>> d1["a"] = 1
+    >>> d1.a
+    1
+    """
+
+    def __init__(self, value: dict | None = None) -> None:
+        """Construct an :class:`.AttrsDict` object.
+
+        Note
+        ----
+        The input dictionary is copied.
+
+        Parameters
+        ----------
+        value
+            a :class:`dict` object to initialize the instance with.
+        """
+        if value is None:
+            super().__init__()
+        # can only be initialized with a dict
+        elif isinstance(value, dict):
+            for key in value:
+                self.__setitem__(key, value[key])
+        else:
+            msg = "expected dict"
+            raise TypeError(msg)
+
+        # attribute that holds cached remappings -- see map()
+        super().__setattr__("__cached_remaps__", {})
+
+    def __setitem__(self, key: str | int | float, value: Any) -> Any:
+        # convert dicts to AttrsDicts
+        if not isinstance(value, AttrsDict):
+            if isinstance(value, dict):
+                value = AttrsDict(value)  # this should make it recursive
+            # recurse lists
+            elif isinstance(value, list):
+                for i, el in enumerate(value):
+                    if isinstance(el, dict):
+                        value[i] = AttrsDict(el)  # this should make it recursive
+
+        super().__setitem__(key, value)
+
+        # if the key is a valid attribute name, create a new attribute
+        if isinstance(key, str) and key.isidentifier():
+            super().__setattr__(key, value)
+
+        # reset special __cached_remaps__ private attribute -- see map()
+        super().__setattr__("__cached_remaps__", {})
+
+    __setattr__ = __setitem__
+
+    def __getattr__(self, name: str) -> Any:
+        try:
+            super().__getattr__(name)
+        except AttributeError as exc:
+            msg = f"dictionary does not contain a '{name}' key"
+            raise AttributeError(msg) from exc
+
+    def map(self, label: str, unique: bool = True) -> AttrsDict:
+        """Remap dictionary according to an alternative unique label.
+
+        Loop over keys in the first level and search for key named `label` in
+        their values. If `label` is found and its value `newid` is unique,
+        create a mapping between `newid` and the first-level dictionary `obj`.
+        If `label` is of the form ``key.label``, ``label`` will be searched in
+        a dictionary keyed by ``key``. If the label is unique a dictionary of
+        dictionaries will be returned, if not unique and `unique` is false, a
+        dictionary will be returned where each entry is a dictionary of
+        dictionaries keyed by an arbitrary integer.
+
+        Parameters
+        ----------
+        label
+            game (key) at which the new label can be found. If nested in
+            dictionaries, use ``.`` to separate levels, e.g.
+            ``level1.level2.label``.
+        unique
+            bool specifying whether only unique keys are allowed. If true
+            will raise an error if the specified key is not unique.
+
+        Examples
+        --------
+        >>> d = AttrsDict({
+        ...   "a": {
+        ...     "id": 1,
+        ...     "group": {
+        ...       "id": 3,
+        ...     },
+        ...     "data": "x"
+        ...   },
+        ...   "b": {
+        ...     "id": 2,
+        ...     "group": {
+        ...       "id": 4,
+        ...     },
+        ...     "data": "y"
+        ...   },
+        ... })
+        >>> d.map("id")[1].data == "x"
+        True
+        >>> d.map("group.id")[4].data == "y"
+        True
+
+        Note
+        ----
+        No copy is performed, the returned dictionary is made of references to
+        the original objects.
+
+        Warning
+        -------
+        The result is cached internally for fast access after the first call.
+        If the dictionary is modified, the cache gets cleared.
+        """
+        # if this is a second call, return the cached result
+        if label in self.__cached_remaps__:
+            return self.__cached_remaps__[label]
+
+        splitk = label.split(".")
+        newmap = AttrsDict()
+        unique_tracker = True
+        # loop over values in the first level
+        for v in self.values():
+            # find the (nested) label value
+            newid = v
+            try:
+                for k in splitk:
+                    newid = newid[k]
+            # just skip if the label is not there
+            except (KeyError, TypeError, FileNotFoundError):
+                continue
+
+            if not isinstance(newid, (int, float, str)):
+                msg = f"'{label}' values are not all numbers or strings"
+                raise RuntimeError(msg)
+
+            if newid in newmap:
+                newkey = sorted(newmap[newid].keys())[-1] + 1
+                newmap[newid].update({newkey: v})
+                unique_tracker = False
+            else:
+                # add an item to the new dict with key equal to the value of the label
+                newmap[newid] = {0: v}
+
+        if unique is True and unique_tracker is False:
+            # complain if a label with the same value was already found
+            msg = f"'{label}' values are not unique"
+            raise RuntimeError(msg)
+
+        if unique_tracker is True:
+            newmap = AttrsDict({entry: newmap[entry][0] for entry in newmap})
+
+        if not newmap:
+            msg = f"could not find '{label}' anywhere in the dictionary"
+            raise ValueError(msg)
+
+        # cache it
+        self.__cached_remaps__[label] = newmap
+        return newmap
+
+    def group(self, label: str) -> AttrsDict:
+        """Group dictionary according to a `label`.
+
+        This is equivalent to :meth:`.map` with `unique` set to ``False``.
+
+        Parameters
+        ----------
+        label
+            name (key) at which the new label can be found. If nested in
+            dictionaries, use ``.`` to separate levels, e.g.
+            ``level1.level2.label``.
+
+        Examples
+        --------
+        >>> d = AttrsDict({
+        ...   "a": {
+        ...     "type": "A",
+        ...     "data": 1
+        ...   },
+        ...   "b": {
+        ...     "type": "A",
+        ...     "data": 2
+        ...   },
+        ...   "c": {
+        ...     "type": "B",
+        ...     "data": 3
+        ...   },
+        ... })
+        >>> d.group("type").keys()
+        dict_keys(['A', 'B'])
+        >>> d.group("type").A.values()
+        dict_values([{'type': 'A', 'data': 1}, {'type': 'A', 'data': 2}])
+        >>> d.group("type").B.values()
+        dict_values([{'type': 'B', 'data': 3}])
+        >>> d.group("type").A.map("data")[1]
+        {'type': 'A', 'data': 1}
+
+        See Also
+        --------
+        map
+        """
+        return self.map(label, unique=False)
+
+    # d |= other_d should still produce a valid AttrsDict
+    def __ior__(self, other: dict | AttrsDict) -> AttrsDict:
+        return AttrsDict(super().__ior__(other))
+
+    # d1 | d2 should still produce a valid AttrsDict
+    def __or__(self, other: dict | AttrsDict) -> AttrsDict:
+        return AttrsDict(super().__or__(other))

dbetto/catalog.py

@@ -0,0 +1,238 @@
+# Copyright (C) 2015 Oliver Schulz <oschulz@mpp.mpg.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import bisect
+import collections
+import copy
+import types
+from collections import namedtuple
+from pathlib import Path
+from string import Template
+
+import yaml
+
+from . import time, utils
+
+
+class PropsStream:
+    """Simple class to control loading of validity.yaml files"""
+
+    @staticmethod
+    def get(value):
+        if isinstance(value, str):
+            return PropsStream.read_from(value)
+
+        if isinstance(value, (collections.abc.Sequence, types.GeneratorType)):
+            return value
+
+        msg = f"Can't get PropsStream from value of type {type(value)}"
+        raise ValueError(msg)
+
+    @staticmethod
+    def read_from(file_name):
+        with Path(file_name).open(encoding="utf-8") as r:
+            file = yaml.safe_load(r)
+        file = sorted(file, key=lambda item: time.unix_time(item["valid_from"]))
+        yield from file
+
+
+class Catalog(namedtuple("Catalog", ["entries"])):
+    """Implementation of the `YAML metadata validity specification <https://legend-exp.github.io/legend-data-format-specs/dev/metadata/#Specifying-metadata-validity-in-time-(and-system)>`_."""
+
+    __slots__ = ()
+
+    class Entry(namedtuple("Entry", ["valid_from", "file"])):
+        """An entry in the validity file."""
+
+        __slots__ = ()
+
+    @staticmethod
+    def get(value):
+        if isinstance(value, Catalog):
+            return value
+
+        if isinstance(value, str):
+            return Catalog.read_from(value)
+
+        msg = f"Can't get Catalog from value of type {type(value)}"
+        raise ValueError(msg)
+
+    @staticmethod
+    def read_from(file_name):
+        """Read from a valdiity YAML file and build a Catalog object"""
+        entries = {}
+        for props in PropsStream.get(file_name):
+            timestamp = props["valid_from"]
+            system = "all" if props.get("category") is None else props["category"]
+            file_key = props["apply"]
+            if system not in entries:
+                entries[system] = []
+            mode = "append" if props.get("mode") is None else props["mode"]
+            mode = "reset" if len(entries[system]) == 0 else mode
+            if mode == "reset":
+                new = file_key
+            elif mode == "append":
+                new = entries[system][-1].file.copy() + file_key
+            elif mode == "remove":
+                new = entries[system][-1].file.copy()
+                for file in file_key:
+                    new.remove(file)
+            elif mode == "replace":
+                new = entries[system][-1].file.copy()
+                if len(file_key) != 2:
+                    msg = f"Invalid number of elements in replace mode: {len(file_key)}"
+                    raise ValueError(msg)
+                new.remove(file_key[0])
+                new += [file_key[1]]
+
+            else:
+                msg = f"Unknown mode for {timestamp}"
+                raise ValueError(msg)
+
+            if timestamp in [entry.valid_from for entry in entries[system]]:
+                msg = f"Duplicate timestamp: {timestamp}, use reset mode instead with a single entry"
+                raise ValueError(msg)
+            entries[system].append(Catalog.Entry(time.unix_time(timestamp), new))
+
+        for system in entries:
+            entries[system] = sorted(
+                entries[system], key=lambda entry: entry.valid_from
+            )
+        return Catalog(entries)
+
+    def valid_for(self, timestamp, system="all", allow_none=False):
+        """Get the valid entries for a given timestamp and system"""
+        if system in self.entries:
+            valid_from = [entry.valid_from for entry in self.entries[system]]
+            pos = bisect.bisect_right(valid_from, time.unix_time(timestamp))
+            if pos > 0:
+                return self.entries[system][pos - 1].file
+
+            if system != "all":
+                return self.valid_for(timestamp, system="all", allow_none=allow_none)
+
+            if allow_none:
+                return None
+
+            msg = f"No valid entries found for timestamp: {timestamp}, system: {system}"
+            raise RuntimeError(msg)
+
+        if system != "all":
+            return self.valid_for(timestamp, system="all", allow_none=allow_none)
+
+        if allow_none:
+            return None
+
+        msg = f"No entries found for system: {system}"
+        raise RuntimeError(msg)
+
+    @staticmethod
+    def get_files(catalog_file, timestamp, category="all"):
+        """Helper function to get the files for a given timestamp and category"""
+        catalog = Catalog.read_from(catalog_file)
+        return Catalog.valid_for(catalog, timestamp, category)
+
+
+class Props:
+    """Class to handle overwriting of dictionaries in cascade order"""
+
+    @staticmethod
+    def read_from(sources, subst_pathvar=False, trim_null=False):
+        def read_impl(sources):
+            if isinstance(sources, str):
+                file_name = sources
+                result = utils.load_dict(file_name)
+                if subst_pathvar:
+                    Props.subst_vars(
+                        result,
+                        var_values={"_": Path(file_name).parent},
+                        ignore_missing=True,
+                    )
+                return result
+
+            if isinstance(sources, list):
+                result = {}
+                for p in map(read_impl, sources):
+                    Props.add_to(result, p)
+                return result
+
+            msg = f"Can't run Props.read_from on sources-value of type {type(sources)}"
+            raise ValueError(msg)
+
+        result = read_impl(sources)
+        if trim_null:
+            Props.trim_null(result)
+        return result
+
+    @staticmethod
+    def write_to(file_name, obj, ftype: str | None = None):
+        utils.write_dict(file_name, obj, ftype)
+
+    @staticmethod
+    def add_to(props_a, props_b):
+        a = props_a
+        b = props_b
+
+        for key in b:
+            if key in a:
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    Props.add_to(a[key], b[key])
+                elif a[key] != b[key]:
+                    a[key] = copy.deepcopy(b[key])
+            else:
+                a[key] = copy.deepcopy(b[key])
+
+    @staticmethod
+    def trim_null(props_a):
+        a = props_a
+
+        for key in list(a.keys()):
+            if isinstance(a[key], dict):
+                Props.trim_null(a[key])
+            elif a[key] is None:
+                del a[key]
+
+    @staticmethod
+    def subst_vars(props, var_values=None, ignore_missing=False):
+        if not var_values:
+            var_values = {}
+
+        for key in props:
+            value = props[key]
+            if isinstance(value, str) and "$" in value:
+                new_value = None
+                if ignore_missing:
+                    new_value = Template(value).safe_substitute(var_values)
+                else:
+                    new_value = Template(value).substitute(var_values)
+
+                if new_value != value:
+                    props[key] = new_value
+            elif isinstance(value, list):
+                new_values = []
+                for val in value:
+                    if isinstance(val, str) and "$" in val:
+                        if ignore_missing:
+                            new_value = Template(val).safe_substitute(var_values)
+                        else:
+                            new_value = Template(val).substitute(var_values)
+                    else:
+                        new_value = val
+                    new_values.append(new_value)
+                if new_values != value:
+                    props[key] = new_values
+            elif isinstance(value, dict):
+                Props.subst_vars(value, var_values, ignore_missing)

dbetto/textdb.py

@@ -0,0 +1,353 @@
+# Copyright (C) 2022 Luigi Pertoldi <gipert@pm.me>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import sys
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+
+import yaml
+
+from . import utils
+from .attrsdict import AttrsDict
+from .catalog import Catalog, Props
+
+log = logging.getLogger(__name__)
+
+
+class TextDB:
+    """A simple text file database.
+
+    The database is represented on disk by a collection of text files
+    arbitrarily scattered in a filesystem. Subdirectories are also
+    :class:`.TextDB` objects. In memory, the database is represented as an
+    :class:`~.attrsdict.AttrsDict`.
+
+    Currently supported file formats are JSON and YAML.
+
+    Tip
+    ---
+    For large databases, a basic "lazy" mode is available. In this case, no
+    global scan of the filesystem is performed at initialization time. Once a
+    file is queried, it is also cached in the internal store for faster access.
+    Caution, this option is for advanced use (see warning message below).
+
+    Warning
+    -------
+    A manual call to :meth:`scan` is needed before most class methods (e.g.
+    iterating on the database files) can be properly used.
+
+    Examples
+    --------
+    >>> from dbetto import TextDB
+    >>> jdb = TextDB("path/to/dir")
+    >>> jdb["file1.json"]  # is a dict
+    >>> jdb["file1.yaml"]  # is a dict
+    >>> jdb["file1"]  # also works
+    >>> jdb["dir1"]  # TextDB instance
+    >>> jdb["dir1"]["file1"]  # nested file
+    >>> jdb["dir1/file1"]  # also works
+    >>> jdb.dir1.file # keys can be accessed as attributes
+    """
+
+    def __init__(
+        self, path: str | Path, lazy: str | bool = False, hidden: bool = False
+    ) -> None:
+        """Construct a :class:`.TextDB` object.
+
+        Parameters
+        ----------
+        path
+            path to the directory containing the database.
+        lazy
+            whether a database scan should be performed at initialization time.
+            if ``auto``, be non-lazy only if working in a python interactive
+            session.
+        hidden
+            ignore hidden (i.e. starting with ".") files of directories.
+        """
+        if isinstance(lazy, bool):
+            self.__lazy__ = lazy
+        elif lazy == "auto":
+            self.__lazy__ = not hasattr(sys, "ps1")
+        else:
+            msg = f"unrecognized value {lazy=}"
+            raise ValueError(msg)
+
+        self.__hidden__ = hidden
+        self.__path__ = Path(path).expanduser().resolve()
+
+        if not self.__path__.is_dir():
+            msg = "input path is not a valid directory"
+            raise ValueError(msg)
+
+        self.__store__ = AttrsDict()
+        self.__ftypes__ = {"json", "yaml"}
+
+        if not self.__lazy__:
+            self.scan()
+
+    @property
+    def __extensions__(self) -> set:
+        # determine list of supported file extensions
+        return set().union(
+            *[
+                exts
+                for ft, exts in utils.__file_extensions__.items()
+                if ft in self.__ftypes__
+            ]
+        )
+
+    def scan(self, recursive: bool = True, subdir: str = ".") -> None:
+        """Populate the database by walking the filesystem.
+
+        Parameters
+        ----------
+        recursive
+            if ``True``, recurse subdirectories.
+        subdir
+            restrict scan to path relative to the database location.
+        """
+        # recursive search or not?
+        _fcn = self.__path__.rglob if recursive else self.__path__.glob
+        # build file list
+        flist = (
+            p for p in _fcn(str(Path(subdir) / "*")) if p.suffix in self.__extensions__
+        )
+
+        for j in flist:
+            try:
+                self[j.with_suffix("")]
+            except (json.JSONDecodeError, yaml.YAMLError, ValueError) as e:
+                msg = f"could not scan file {j}, reason: {e!r}"
+                log.warning(msg)
+
+    def keys(self) -> list[str]:
+        return self.__store__.keys()
+
+    def items(self) -> Iterator[(str, TextDB | AttrsDict | list)]:
+        return self.__store__.items()
+
+    def on(
+        self, timestamp: str | datetime, pattern: str | None = None, system: str = "all"
+    ) -> AttrsDict | list:
+        """Query database in `time[, file pattern, system]`.
+
+        A (only one) valid ``validity.yaml`` file must exist in the directory
+        to specify a validity mapping. This functionality relies on the
+        :class:`.catalog.Catalog` class.
+
+        The YAML specification is documented at `this link
+        <https://legend-exp.github.io/legend-data-format-specs/dev/metadata/#Specifying-metadata-validity-in-time-(and-system)>`_.
+
+        The special ``$_`` string is expanded to the directory containing the
+        text files.
+
+        Parameters
+        ----------
+        timestamp
+            a :class:`~datetime.datetime` object or a string matching the
+            pattern ``YYYYmmddTHHMMSSZ``.
+        pattern
+            query by filename pattern.
+        system
+            query only a data taking "system" (e.g. 'all', 'phy', 'cal', 'lar', ...)
+        """
+        for ext in utils.__file_extensions__["yaml"]:
+            yml = self.__path__ / f"validity{ext}"
+            if yml.is_file():
+                break
+        if not yml.is_file():
+            msg = f"no validity.yaml / validity.yml file found in {self.__path__!s}"
+            raise RuntimeError(msg)
+
+        file_list = Catalog.get_files(str(yml), timestamp, system)
+        # select only files matching pattern if specified
+        if pattern is not None:
+            c = re.compile(pattern)
+            out_files = []
+            for file in file_list:
+                if c.match(file):
+                    out_files.append(file)
+            files = out_files
+        else:
+            files = file_list
+
+        # read files in and combine as necessary
+        db_ptr = self
+        if isinstance(files, list):
+            result = AttrsDict()
+            for file in files:
+                fp = self.__path__.rglob(file)
+                # TODO: what does this do exactly?
+                Props.add_to(result, db_ptr[next(iter(fp))])
+            db_ptr = result
+        else:
+            fp = self.__path__.rglob(files)
+            db_ptr = db_ptr[next(iter(fp))]
+        Props.subst_vars(db_ptr, var_values={"_": self.__path__})
+        return db_ptr
+
+    def map(self, label: str, unique: bool = True) -> AttrsDict:
+        """Remap dictionary according to a second unique `label`.
+
+        See Also
+        --------
+        .attrsdict.AttrsDict.map
+
+        Warning
+        -------
+        If the database is lazy, you must call :meth:`.scan` in advance to
+        populate it, otherwise mappings cannot be created.
+        """
+        return self.__store__.map(label, unique=unique)
+
+    def group(self, label: str) -> AttrsDict:
+        """Group dictionary according to a second unique `label`.
+
+        See Also
+        --------
+        .attrsdict.AttrsDict.group
+
+        Warning
+        -------
+        If the database is lazy, you must call :meth:`.scan` in advance to
+        populate it, otherwise groupings cannot be created.
+        """
+        return self.__store__.group(label)
+
+    def __getitem__(self, item: str | Path) -> TextDB | AttrsDict | list | None:
+        """Access files or directories in the database."""
+        # resolve relative paths / links, but keep it relative to self.__path__
+        item = Path(item)
+
+        if item.is_absolute() and item.is_relative_to(self.__path__):
+            item = item.expanduser().resolve().relative_to(self.__path__)
+        elif not item.is_absolute():
+            item = (
+                (self.__path__ / item).expanduser().resolve().relative_to(self.__path__)
+            )
+        else:
+            msg = f"{item} lies outside the database root path {self.__path__!s}"
+            raise ValueError(msg)
+
+        ext_list = "[" + "|".join(self.__extensions__) + "]"
+        msg = f"parsing directory or file{ext_list}: {item}"
+        log.debug(msg)
+
+        # now call this very function recursively to walk the directories to the file
+        db_ptr = self
+        for d in item.parts[:-1]:
+            db_ptr = db_ptr[d]
+            # check if we encountered hidden directory (must skip)
+            if not self.__hidden__ and db_ptr is None:
+                return None
+
+        # item_id should not contain any / at this point
+        # store file names without extension
+        item_id = item.stem
+        # skip if object is already in the store
+        if item_id not in db_ptr.__store__:
+            obj = db_ptr.__path__ / item.name
+
+            # do not consider hidden files
+            if not self.__hidden__ and obj.name.startswith("."):
+                return None
+
+            # if directory, construct another TextDB object
+            if obj.is_dir():
+                db_ptr.__store__[item_id] = TextDB(obj, lazy=self.__lazy__)
+
+            else:
+                # try to attach an extension if file cannot be found
+                # but check if there are multiple files that only differ in extension (unsupported)
+                found = True
+                if not obj.is_file():
+                    found = False
+                    for ext in self.__extensions__:
+                        if obj.with_suffix(ext).is_file():
+                            if found:
+                                msg = "the database cannot contain files that differ only in the extension"
+                                raise RuntimeError(msg)
+
+                            obj = obj.with_suffix(ext)
+                            found = True
+
+                if not found:
+                    msg = f"{obj.with_stem(ext_list)} is not a valid file or directory"
+                    raise FileNotFoundError(msg)
+
+                # if it's a valid file, construct an AttrsDict object
+                loaded = utils.load_dict(obj)
+                if isinstance(loaded, dict):
+                    loaded = AttrsDict(loaded)
+                    Props.subst_vars(loaded, var_values={"_": self.__path__})
+                else:  # must be a list, check if there are dicts inside to convert
+                    for i, el in enumerate(loaded):
+                        if isinstance(el, dict):
+                            loaded[i] = AttrsDict(el)
+                            Props.subst_vars(loaded[i], var_values={"_": self.__path__})
+
+                db_ptr.__store__[item_id] = loaded
+
+            # set also an attribute, if possible
+            if item_id.isidentifier():
+                db_ptr.__setattr__(item_id, db_ptr.__store__[item_id])
+
+        return db_ptr.__store__[item_id]
+
+    def __getattr__(self, name: str) -> TextDB | AttrsDict | list:
+        try:
+            return object.__getattribute__(self, name)
+        except AttributeError:
+            try:
+                return self.__getitem__(name)
+            except AttributeError as exc:
+                msg = f"file database does not contain '{name}'"
+                raise AttributeError(msg) from exc
+
+    # NOTE: self cannot stay a TextDB, since the class is characterized by a
+    # (unique) root directory. What would be the root directory of the merged
+    # TextDB?
+    def __ior__(self, other: TextDB) -> AttrsDict:
+        msg = "cannot merge TextDB in-place"
+        raise TypeError(msg)
+
+    # NOTE: returning a TextDB does not make much sense, see above
+    def __or__(self, other: TextDB) -> AttrsDict:
+        if isinstance(other, TextDB):
+            return self.__store__ | other.__store__
+
+        return self.__store__ | other
+
+    def __contains__(self, value: str) -> bool:
+        return self.__store__.__contains__(value)
+
+    def __len__(self) -> int:
+        return len(self.__store__)
+
+    def __iter__(self) -> Iterator:
+        return iter(self.__store__)
+
+    def __str__(self) -> str:
+        return str(self.__store__)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}('{self.__path__!s}')"

dbetto/time.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2022 Luigi Pertoldi <gipert@pm.me>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+from datetime import datetime
+
+
+def str_to_datetime(value):
+    """Convert a string in the format %Y%m%dT%H%M%SZ to :class:`datetime.datetime`."""
+    return datetime.strptime(value, "%Y%m%dT%H%M%SZ")
+
+
+def unix_time(value):
+    """Convert a string in the format %Y%m%dT%H%M%SZ or datetime object to Unix time value"""
+    if isinstance(value, str):
+        return datetime.timestamp(str_to_datetime(value))
+
+    if isinstance(value, datetime):
+        return datetime.timestamp(value)
+
+    msg = f"Can't convert type {type(value)} to unix time"
+    raise ValueError(msg)

dbetto/utils.py

@@ -0,0 +1,76 @@
+# Copyright (C) 2022 Luigi Pertoldi <gipert@pm.me>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import yaml
+
+log = logging.getLogger(__name__)
+
+__file_extensions__ = {"json": [".json"], "yaml": [".yaml", ".yml"]}
+
+
+def load_dict(fname: str, ftype: str | None = None) -> dict:
+    """Load a text file as a Python dict."""
+    fname = Path(fname)
+
+    # determine file type from extension
+    if ftype is None:
+        for _ftype, exts in __file_extensions__.items():
+            if fname.suffix in exts:
+                ftype = _ftype
+
+    msg = f"writing {ftype} dict to: {fname}"
+    log.debug(msg)
+
+    with fname.open(encoding="utf-8") as f:
+        if ftype == "json":
+            return json.load(f)
+        if ftype == "yaml":
+            return yaml.safe_load(f)
+
+        msg = f"unsupported file format {ftype}"
+        raise NotImplementedError(msg)
+
+
+def write_dict(fname: str, obj: dict, ftype: str | None = None) -> dict:
+    """Load a text file as a Python dict."""
+    fname = Path(fname)
+
+    # determine file type from extension
+    if ftype is None:
+        for _ftype, exts in __file_extensions__.items():
+            if fname.suffix in exts:
+                ftype = _ftype
+
+    msg = f"writing {ftype} dict to: {fname}"
+    log.debug(msg)
+
+    with fname.open("w", encoding="utf-8") as f:
+        if ftype == "json":
+            separators = (",", ":")
+            indent = 2
+            json.dump(obj, f, indent=indent, separators=separators)
+            f.write("\n")
+        elif ftype == "yaml":
+            yaml.dump(obj, f, sort_keys=False)
+
+        else:
+            msg = f"unsupported file format {ftype}"
+            raise NotImplementedError(msg)

dbetto-1.0.2.dist-info/METADATA

@@ -0,0 +1,120 @@
+Metadata-Version: 2.4
+Name: dbetto
+Version: 1.0.2
+Summary: A tiny text file based database
+Project-URL: Homepage, https://github.com/gipert/dbetto
+Project-URL: Bug Tracker, https://github.com/gipert/dbetto/issues
+Project-URL: Discussions, https://github.com/gipert/dbetto/discussions
+Project-URL: Changelog, https://github.com/gipert/dbetto/releases
+Author-email: Luigi Pertoldi <gipert@pm.me>
+License-File: LICENSE
+Classifier: Development Status :: 1 - Planning
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: pyyaml
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=3; extra == 'dev'
+Requires-Dist: pytest>=6; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: furo>=2023.08.17; extra == 'docs'
+Requires-Dist: myst-parser>=0.13; extra == 'docs'
+Requires-Dist: sphinx-autodoc-typehints; extra == 'docs'
+Requires-Dist: sphinx-copybutton; extra == 'docs'
+Requires-Dist: sphinx-inline-tabs; extra == 'docs'
+Requires-Dist: sphinx>=7.0; extra == 'docs'
+Provides-Extra: test
+Requires-Dist: pytest-cov>=3; extra == 'test'
+Requires-Dist: pytest>=6; extra == 'test'
+Description-Content-Type: text/markdown
+
+# 📂 dbetto
+
+A tiny text-based database.
+
+[![PyPI](https://img.shields.io/pypi/v/dbetto?logo=pypi)](https://pypi.org/project/dbetto/)
+![GitHub tag (latest by date)](https://img.shields.io/github/v/tag/gipert/dbetto?logo=git)
+[![GitHub Workflow Status](https://img.shields.io/github/checks-status/gipert/dbetto/main?label=main%20branch&logo=github)](https://github.com/gipert/dbetto/actions)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Codecov](https://img.shields.io/codecov/c/github/gipert/dbetto?logo=codecov)](https://app.codecov.io/gh/gipert/dbetto)
+![GitHub issues](https://img.shields.io/github/issues/gipert/dbetto?logo=github)
+![GitHub pull requests](https://img.shields.io/github/issues-pr/gipert/dbetto?logo=github)
+![License](https://img.shields.io/github/license/gipert/dbetto)
+[![Read the Docs](https://img.shields.io/readthedocs/dbetto?logo=readthedocs)](https://dbetto.readthedocs.io)
+
+_dbetto_ is a lightweight Python library that enables seamless access and
+management of **text-based databases** (JSON/YAML) scattered across your
+filesystem. Whether you're handling nested data structures or querying data by
+validity periods, _dbetto_ simplifies it all with an intuitive interface.
+
+The project was originally developed to efficiently manage metadata for the
+LEGEND experiment.
+
+### Key Features
+
+- Access JSON/YAML files like a Python dictionary
+- Attribute-style access with tab-completion for rapid query
+- Time-sensitive data querying with validity rules
+- Data remapping and grouping functionalities
+
+## Showcase
+
+Install from PyPI:
+
+```bash
+pip install dbetto
+```
+
+Access data arbitrarily structured as JSON or YAML files:
+
+```python
+>>> from dbetto import TextDB
+>>> db = TextDB("/path/to/data/folder")
+>>> db["dir1"]["file1.json"]["value"]
+1
+>>> db.dir1.file1.value
+1
+```
+
+Query data valid on a given time period:
+
+```python
+>>> from datetime import datetime
+>>> db.on(datetime(2023, 1, 10, 9, 53, 0)).value
+3
+```
+
+Map data according to alternative keys:
+
+```python
+>>> chmap = db["detectors.yaml"]
+>>> chmap.V05266A
+{'name': 'V05266A',
+ 'location': {'string': 1, 'position': 4},
+ 'daq': {'crate': 0,
+  'rawid': 1104003,
+ ...
+>>> chmap.map("daq.rawid")[1104003]
+{'name': 'V05266A',
+ 'location': {'string': 1, 'position': 4},
+ 'daq': {'crate': 0,
+  'rawid': 1104003}}
+>>> grouped = chmap.group("electronics.cc4.id")["C3"]
+>>> grouped.map("name").keys()
+dict_keys(['V02160A', 'V02160B', 'V05261B', 'V05266A', 'V05266B', 'V05268B', 'V05612A'])
+```
+
+Have a look at the [docs](https://dbetto.readthedocs.io) for more!

dbetto-1.0.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+dbetto/__init__.py,sha256=0-QwaMgvWS9HS45JNjTfoBKi6CwL365R8DMtcDbkWK8,952
+dbetto/_version.py,sha256=BGgbj7AHZwtVw29XgDIjcG5woR-f3jQy2X-ArcRT22E,411
+dbetto/_version.pyi,sha256=j5kbzfm6lOn8BzASXWjGIA1yT0OlHTWqlbyZ8Si_o0E,118
+dbetto/attrsdict.py,sha256=f3jcFwEA5ginIDQM0F8iRFNPx1RKi3wcY7VT0IbS0kQ,8044
+dbetto/catalog.py,sha256=rebVBYyCFAsJf7gDHZsoBfKGbLJ12iStDgPHJrCNX0I,8337
+dbetto/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dbetto/textdb.py,sha256=ILAez98Z5XTcXPEZxB7HFDBJ6fQi_56lKTe1PoJihN4,12638
+dbetto/time.py,sha256=nwSy1bSggZVFp1q-kjW4NIlrHBwlD9xRaSqgMiPdrZs,1297
+dbetto/utils.py,sha256=HG5Pe0AhVpsMTV-WMDgYNHLgnX_gfpHZqJ2-x4P7y4s,2359
+dbetto-1.0.2.dist-info/METADATA,sha256=dIKN9kKyaKtALTrslynvRxt5e92-9LQj1Hqc_qlGq5A,4517
+dbetto-1.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dbetto-1.0.2.dist-info/licenses/LICENSE,sha256=yO1J9FccKANTlBrPhCtWEiFW9nIJFT_AEqR86iinBf4,1054
+dbetto-1.0.2.dist-info/RECORD,,

dbetto-1.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

dbetto-1.0.2.dist-info/licenses/LICENSE

@@ -0,0 +1,19 @@
+Copyright 2024 Luigi Pertoldi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.