fresco 3.3.4__py3-none-any.whl → 3.9.0__py3-none-any.whl

fresco/options.py

@@ -12,12 +12,16 @@
 #     See the License for the specific language governing permissions and
 #     limitations under the License.
 #
+import contextlib
+import dataclasses
 import inspect
+import itertools
 import json
 import logging
 import typing as t
 import os
 import re
+from collections.abc import Mapping
 from decimal import Decimal
 from pathlib import Path
 from socket import gethostname
@@ -26,7 +30,6 @@ from typing import Callable
 from typing import Dict
 from typing import Iterable
 from typing import List
-from typing import Mapping
 from typing import Sequence
 from typing import Union
 
@@ -35,6 +38,8 @@ from fresco.exceptions import OptionsLoadedException
 __all__ = ["Options"]
 
 logger = logging.getLogger(__name__)
+known_suffixes = {"py", "sh", "rc", "txt", "cfg", "ini", "json", "toml", "conf"}
+priority_pattern = re.compile(r"([0-9]+)(?:-(.*))?$")
 
 
 class Options(dict):
@@ -96,7 +101,7 @@ class Options(dict):
         strict=True,
         dir=None,
         trigger_onload=True,
-    ):
+    ) -> "Options":
         """
         Find all files matching glob pattern ``sources`` and populates the
         options object from those with matching filenames containing ``tags``.
@@ -119,20 +124,43 @@ class Options(dict):
         format. Any other files will be interpreted as simple lists of
         ```key=value`` pairs.
 
-        Tags in filenames are delimited with periods,
-        for example ".env.production.py".
+        Filename format
+        ---------------
+
+        The general format of filenames is:
+
+        .. code-block:: shell
+
+            <base>(.<priority number>-)(<tags>)(<suffix>)
+
+        Example filenames:
+
+        .. code-block:: shell
+
+            # Just <base>
+            .env
+
+            # <base>.<suffix>
+            .settings.toml
+
+            # <base>.<tags>.<suffix>
+            .env.dev.local..py
+
+            # <base>.<priority>-<tags>.<suffix>
+            .env.100-dev.py
+
+        Priority number, if specified, is used to determine loading order.
+        Lower numbers are loaded first. Priority numbers must be positive
+        integers.
+
+        Tags are delimited with periods,
+        for example ``.env.production.py``.
         The filename ``setttings.dev.local.ini`` would be
         considered to have the tags ``('dev', 'local')``
 
         Where filename contain multiple tags, all tags must match for the file
         to be loaded.
 
-        Files are processed in the order that tags are specified in the
-        ``tags`` parameter, and then in lexicographical order.
-        For example, calling ``options.load(..., tags=["dev", "local"])`` would
-        cause a file named "settings.dev" to be loaded before one named
-        "settings.local".
-
         Tag names may contain the names of environment variable surrounded by
         braces, for example ``{USER}``. These will be substituted for the
         environment variable's value, with any dots or path separators replaced
@@ -143,8 +171,24 @@ class Options(dict):
 
         Files with the suffix ".sample" are unconditionally excluded.
 
-        Files are loaded in the order specified by ``tags``, then in filename
-        order. Environment variables, if requested, are loaded last.
+        Loading order
+        -------------
+
+        Files are loaded in the following order:
+
+        1. On priority number, from low to high. If the priority number is not
+           given, a priority of zero is assumed
+
+        2. Then in tag order, based on the ordering given in the
+           ``tags`` parameter
+
+           For example, calling ``options.load(..., tags=["dev", "local"])`` would
+           cause a file named "settings.dev" to be loaded before one named
+           "settings.local".
+
+        3. Finally in lexicographical order.
+
+        Environment variables, if requested, are loaded last.
 
         Example::
 
@@ -152,19 +196,12 @@ class Options(dict):
             opts.load(".env*", ["dev", "host-{hostname}", "local"])
 
         Would load options from files named ``.env``, ``.env.json``, ``.env.dev.py``
-        and ``.env.local.py.``
+        and ``.env.local.py``, in that order.
 
         """
         if self._is_loaded:
             raise OptionsLoadedException("Options have already been loaded")
 
-        tag_substitutions = os.environ.copy()
-        tag_substitutions["hostname"] = gethostname()
-        tag_substitutions = {
-            k: v.replace(".", "_").replace(os.pathsep, "_")
-            for k, v in tag_substitutions.items()
-        }
-
         candidates: List[Path] = []
         if dir is None:
             dir = Path(".")
@@ -181,50 +218,45 @@ class Options(dict):
                 if p.suffix.lower() != ".sample"
             )
 
-        if tags:
-            subbed_tags = []
-            for tag in tags:
-                try:
-                    subbed_tags.append(tag.format(**tag_substitutions))
-                except KeyError:
-                    pass
-            tags = subbed_tags
-            tagged_sources = []
-            for p in candidates:
-                candidate_tags = [t for t in str(p.name).split(".") if t][1:]
-                if len(candidate_tags) == 0:
-                    tagged_sources.append((candidate_tags, p))
-                else:
-                    # Ignore the final tag if it matches a common config file
-                    # extension
-                    if candidate_tags[-1].lower() in {
-                        "py",
-                        "sh",
-                        "rc",
-                        "txt",
-                        "cfg",
-                        "ini",
-                        "json",
-                        "toml",
-                        "conf",
-                    }:
-                        candidate_tags.pop()
-
-                    if all(t in tags for t in candidate_tags):
-                        tagged_sources.append((candidate_tags, p))
-            matched = [
-                ts[1]
-                for ts in sorted(
-                    tagged_sources,
-                    key=(
-                        lambda ts: ([], ts[1])
-                        if len(ts[0]) == 0
-                        else (sorted(tags.index(t) for t in ts[0]), ts[1])
-                    )
+        tag_substitutions = make_tag_substitutions()
+        subbed_tags: list[str] = []
+        for tag in tags:
+            try:
+                subbed_tags.append(tag.format(**tag_substitutions))
+            except KeyError:
+                pass
+        tags = subbed_tags
+        tagged_sources = []
+        for path in candidates:
+            priority = 0
+            filename = path.name
+            path_tags = []
+            path_tags = [t for t in str(filename).split(".") if t][1:]
+            if len(path_tags) > 0:
+                # Ignore the final tag if it matches a common config file
+                # extension
+                if path_tags[-1].lower() in known_suffixes:
+                    path_tags.pop()
+
+            if path_tags:
+                if m := priority_pattern.match(path_tags[0]):
+                    priority = int(m.group(1), 10)
+                    if m.group(2):
+                        path_tags[0] = m.group(2)
+                    else:
+                        path_tags = path_tags[1:]
+
+            if all(t in tags for t in path_tags):
+                tagged_sources.append(TaggedSource(priority, path_tags, path))
+            else:
+                excluded = [t for t in path_tags if t not in tags]
+                logger.debug(
+                    f"Ignoring {path} as one or more tag does not match: {excluded=}"
                 )
-            ]
-        else:
-            matched = candidates
+
+        matched = [
+            ts.path for ts in sorted(tagged_sources, key=tagged_source_sort_key(tags))
+        ]
 
         for path in matched:
             existing_keys = set(self.keys())
@@ -232,7 +264,7 @@ class Options(dict):
             if path.suffix == ".py":
                 self.update_from_file(str(path))
             elif path.suffix == ".toml":
-                import toml
+                import toml  # type: ignore
 
                 with path.open("r") as f:
                     self.update(toml.load(f))
@@ -253,16 +285,20 @@ class Options(dict):
                         )
                     )
 
-            if strict and existing_keys and set(self.keys()) != existing_keys:
-                raise AssertionError(
+            if existing_keys and set(self.keys()) != existing_keys:
+                error_msg = (
                     f"settings file {path} created undefined options: "
                     f"{set(self.keys()) - existing_keys}"
                 )
+                if strict:
+                    raise AssertionError(error_msg)
+                else:
+                    logger.warning(error_msg)
 
             if use_environ:
-                for k in self:
-                    if k in os.environ:
-                        self[k] = parse_value(self, os.environ[k])
+                self |= {
+                    k: parse_value(self, os.environ[k]) for k in self if k in os.environ
+                }
 
         if trigger_onload:
             self.do_loaded_callbacks()
@@ -278,9 +314,11 @@ class Options(dict):
         :param load_all: If true private symbols will also be loaded into the
                          options object.
         """
-        ns: Dict[str, Any] = {"__file__": path}
+        ns: Dict[str, Any] = {"__file__": path, "options": self}
         with open(path) as f:
             exec(f.read(), ns)
+        if ns.get("options") is self:
+            del ns["options"]
         self.update_from_dict(ns, load_all)
 
     def update_from_dict(self, d, load_all=False):
@@ -314,6 +352,42 @@ class Options(dict):
         self.update_from_dict(dict(inspect.getmembers(ob)), load_all)
 
 
+@contextlib.contextmanager
+def override_options(options, other=None, **kwargs):
+    """
+    Context manager that updates the given Options object with new values.
+    On exit, the old values will be restored.
+
+    This function is provided to assist with writing tests. It directly
+    modifies the given options object and does not prevent other threads from
+    accessing the modified values.
+    """
+    saved: list[tuple[str, t.Any]] = []
+    items: t.Iterable[tuple[str, t.Any]] = []
+    if isinstance(other, Mapping):
+        items = ((k, other[k]) for k in other.keys())
+
+    if kwargs:
+        items = itertools.chain(items, kwargs.items())
+
+    NOT_PRESENT = object()
+
+    for k, v in items:
+        if k in options:
+            saved.append((k, options[k]))
+        else:
+            saved.append((k, NOT_PRESENT))
+
+        options[k] = v
+
+    yield options
+    for k, v in saved:
+        if v is NOT_PRESENT:
+            del options[k]
+        else:
+            options[k] = v
+
+
 def parse_value(
     options: Mapping,
     v: str,
@@ -416,3 +490,47 @@ def dict_from_options(
     if recursive:
         return recursive_split(d, prefix[-1])
     return d
+
+
+@dataclasses.dataclass
+class TaggedSource:
+    priority: int
+    tags: list[str]
+    path: Path
+
+
+def tagged_source_sort_key(
+    tags: list[str],
+) -> Callable[[TaggedSource], tuple[int, list[int], str]]:
+    """
+    Return a function that can be used as the ``key`` argument of``sorted``,
+    that sorts TaggedSource objects based on
+    priority, tag order, then filename.
+    """
+
+    def _sort_key(ts: TaggedSource) -> tuple[int, list[int], str]:
+        sorted_tags = [tags.index(t) for t in ts.tags]
+        return (ts.priority, sorted_tags, str(ts.path))
+
+    return _sort_key
+
+
+def make_tag_substitutions():
+    """
+    Return a dict of substitutions to make in tag names, based on the current
+    hostname and environment variables
+
+    This permits tags to be specified as, for example,
+    '{hostname}' or '{FRESCO_PROFILE}'
+    and the associated value will be substituted in.
+
+    Path separators and dots (which separate tags) are replaced in substitution
+    values by an underscore.
+    """
+    substitutions = os.environ.copy()
+    substitutions["hostname"] = gethostname()
+    substitutions = {
+        k: v.replace(".", "_").replace(os.pathsep, "_")
+        for k, v in substitutions.items()
+    }
+    return substitutions

fresco/request.py

@@ -23,6 +23,7 @@ from urllib.parse import ParseResult
 from urllib.parse import quote
 from urllib.parse import urlparse
 from urllib.parse import urlunparse
+from decimal import Decimal
 import typing as t
 import datetime
 import json
@@ -32,6 +33,7 @@ import re
 from fresco import exceptions
 from fresco.cookie import parse_cookie_header
 from fresco.multidict import MultiDict
+from fresco.defaults import DEFAULT_CHARSET
 from fresco.types import QuerySpec
 from fresco.util.http import FileUpload
 from fresco.util.http import get_body_bytes
@@ -48,7 +50,15 @@ __all__ = "Request", "currentrequest"
 KB = 1024
 MB = 1024 * KB
 
-_marker = object()
+
+class _Marker:
+    ...
+
+
+_marker = _Marker()
+
+T1 = t.TypeVar("T1")
+T2 = t.TypeVar("T2")
 
 
 class Request(object):
@@ -86,7 +96,7 @@ class Request(object):
     environ: t.Dict
 
     #: Encoding used to decode WSGI parameters, notably PATH_INFO and form data
-    default_charset = fresco.DEFAULT_CHARSET
+    default_charset = DEFAULT_CHARSET
 
     #: The decoder class to use for JSON request payloads
     json_decoder_class = json.JSONDecoder
@@ -185,7 +195,7 @@ class Request(object):
                 "Payload contains data that could not be decoded using " + encoding
             )
 
-    def get_json(self, *args, **kwargs):
+    def get_json(self, *args, **kwargs) -> t.Any:
         """
         Return a decoded JSON request body.
 
@@ -196,12 +206,14 @@ class Request(object):
         by the client.
         """
         try:
-            return self.json_decoder_class(*args, **kwargs).decode(self.body)
+            return self.json_decoder_class(*args, **kwargs).decode(
+                self.body  # type: ignore
+            )
         except ValueError:
             raise exceptions.RequestParseError("Payload is not valid JSON")
 
     @property
-    def files(self):
+    def files(self) -> MultiDict:
         """
         Return a MultiDict of all ``FileUpload`` objects available.
         """
@@ -235,35 +247,64 @@ class Request(object):
 
         return self._query
 
-    def __getitem__(self, key, _marker=_marker):
+    def __getitem__(self, key: str) -> str:
         """
         Return the value of ``key`` from submitted form values.
         """
-        v = self.get(key, _marker)
-        if v is _marker:
+        v = self.get(key, default=None)
+        if v is None:
             raise KeyError(key)
         return v
 
-    def get(self, key, default=_marker, type=None):
+    @t.overload
+    def get(
+        self, key: str, default: T1, type: t.Callable[[str], T2] = str
+    ) -> t.Union[T1, T2, None]:
+        ...
+
+    @t.overload
+    def get(self, key: str, *, type: t.Callable[[str], T2] = str) -> t.Union[T2, None]:
+        ...
+
+    @t.overload
+    def get(
+        self, key: str, *, type: t.Callable[[str], T2] = str, required: t.Literal[True]
+    ) -> T2:
+        ...
+
+    def get(
+        self,
+        key: str,
+        default=_marker,
+        type: t.Callable[[str], T2] = str,
+        required: bool = False,
+    ):
         """
         Look up ``key`` in submitted form values.
 
-        :param type: The type to which the returned result should be converted
-        :param default: The value produced if the argument is missing
+        :param type:
+            The type to which the returned result should be converted
+
+            A :class:`fresco.exceptions.BadRequest` error is raised if ``type`` is
+            specified and the value cannot be converted to the given type
+
+        :param default:
+            The value to return if the key not present.
+
+        :param required:
+            If True, a missing key will cause a
+            :class:`fresco.exceptions.BadRequest` to be raised.
 
-        If ``type`` is specified and the value cannot be converted to the given
-        type, or the value is not present and and no default value has been
-        specified, then a :class:`fresco.exceptions.BadRequest` error is raised.
         """
-        default_specified = default is not _marker
-        value = self.form.get(key, default)
-        if value is default:
-            if default_specified:
-                return default
-            if type is None:
+        value = self.form.get(key, _marker)
+        if value is _marker:
+            if required:
+                raise exceptions.BadRequest()
+            if default is _marker:
                 return None
+            return default
 
-        if type is None:
+        if type is str:
             return value
         try:
             return type(value)
@@ -276,15 +317,97 @@ class Request(object):
         """
         return self.form.getlist(key)
 
-    def getint(self, key, default=_marker) -> int:
+    @t.overload
+    def getbool(self, key: str) -> t.Union[bool, None]:
+        ...
+
+    @t.overload
+    def getbool(self, key: str, default: T1) -> t.Union[bool, T1, None]:
+        ...
+
+    @t.overload
+    def getbool(self, key: str, *, required: t.Literal[True]) -> bool:
+        ...
+
+    def getbool(
+        self, key: str, default: T1 = _marker, required: bool = False
+    ) -> t.Union[bool, T1, None]:
         """
-        Return the named key, converted to an integer.
+        Return the named key, converted to a bool.
+        """
+        if required:
+            return self.get(key, type=bool, required=True)
+        else:
+            return self.get(key, default=default, type=bool)
+
+    @t.overload
+    def getdecimal(self, key: str) -> t.Union[Decimal, None]:
+        ...
+
+    @t.overload
+    def getdecimal(self, key: str, default: T1) -> t.Union[Decimal, T1, None]:
+        ...
 
-        If the key is not present or could not be converted to an int and no
-        default is specified, :class:`~fresco.exceptions.BadRequest` will be
-        raised.
+    @t.overload
+    def getdecimal(self, key: str, *, required: t.Literal[True]) -> Decimal:
+        ...
+
+    def getdecimal(
+        self, key: str, default: T1 = _marker, required: bool = False
+    ) -> t.Union[Decimal, T1, None]:
+        """
+        Return the named key, converted to a decimal.Decimal
         """
-        return self.get(key, default, type=int)
+        if required:
+            return self.get(key, type=Decimal, required=True)
+        else:
+            return self.get(key, default=default, type=Decimal)
+
+    @t.overload
+    def getfloat(self, key: str) -> t.Union[float, None]:
+        ...
+
+    @t.overload
+    def getfloat(self, key: str, default: T1) -> t.Union[float, T1, None]:
+        ...
+
+    @t.overload
+    def getfloat(self, key: str, *, required: t.Literal[True]) -> float:
+        ...
+
+    def getfloat(
+        self, key: str, default: T1 = _marker, required: bool = False
+    ) -> t.Union[float, T1, None]:
+        """
+        Return the named key, converted to a float.
+        """
+        if required:
+            return self.get(key, type=float, required=True)
+        else:
+            return self.get(key, default=default, type=float)
+
+    @t.overload
+    def getint(self, key: str) -> t.Union[int, None]:
+        ...
+
+    @t.overload
+    def getint(self, key: str, default: T1) -> t.Union[int, T1]:
+        ...
+
+    @t.overload
+    def getint(self, key: str, *, required: t.Literal[True]) -> int:
+        ...
+
+    def getint(
+        self, key: str, default: T1 = _marker, required: bool = False
+    ) -> t.Union[int, T1, None]:
+        """
+        Return the named key, converted to an integer.
+        """
+        if required:
+            return self.get(key, type=int, required=True)
+        else:
+            return self.get(key, default=default, type=int)
 
     def __contains__(self, key):
         """
@@ -293,9 +416,7 @@ class Request(object):
         return key in self.form
 
     @property
-    def now(  # type: ignore
-        self, now=datetime.datetime.now, utc=datetime.timezone.utc
-    ):
+    def now(self, now=datetime.datetime.now, utc=datetime.timezone.utc):  # type: ignore
         """
         Return a timezone-aware UTC datetime instance. The value returned is
         guaranteed to be constant throughout the lifetime of the request.
@@ -451,7 +572,7 @@ class Request(object):
         return self._parsed_url
 
     @property
-    def path_info(self, environ_to_str=environ_to_str) -> str:  # type: ignore
+    def path_info(self, environ_to_str=environ_to_str) -> str:
         """
         The PATH_INFO value as a string
 
@@ -463,9 +584,9 @@ class Request(object):
             raise exceptions.BadRequest
 
     @property
-    def script_name(self):
-        """\
-        The SCRIPT_NAME value as a unicode string
+    def script_name(self) -> str:
+        """
+        The SCRIPT_NAME value as a string
 
         Note that SCRIPT_NAME is already unquoted by the server
         """
@@ -477,12 +598,12 @@ class Request(object):
     @property
     def query_string(self) -> t.Optional[str]:
         """
-        The QUERY_STRING value as a unicode string
+        The QUERY_STRING value as a string
         """
         return self.environ.get("QUERY_STRING")
 
     @property
-    def referrer(self):
+    def referrer(self) -> t.Optional[str]:
         """
         Return the HTTP referer header, or ``None`` if this is not available.
         """

fresco/requestcontext.py

@@ -68,6 +68,9 @@ class RequestContext(object):
     def currentcontext(self) -> ContextDict:
         return self._contexts[self._ident_func()][-1]
 
+    def __contains__(self, item):
+        return item in self._contexts[self._ident_func()][-1]
+
     def __getattr__(self, item):
         try:
             return self._contexts[self._ident_func()][-1][item]