c2cwsgiutils 1.0.0.dev1__tar.gz → 5.2.1.dev197__tar.gz

{c2cwsgiutils-1.0.0.dev1 → c2cwsgiutils-5.2.1.dev197}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.1
 Name: c2cwsgiutils
-Version: 1.0.0.dev1
+Version: 5.2.1.dev197
 Summary: Common utilities for Camptocamp WSGI applications
 Home-page: https://github.com/camptocamp/c2cwsgiutils
 License: BSD-2-Clause
 Keywords: geo,gis,sqlalchemy,orm,wsgi
 Author: Camptocamp
 Author-email: info@camptocamp.com
-Requires-Python: >=3.8,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Plugins
 Classifier: Framework :: Pyramid
@@ -17,44 +17,48 @@ Classifier: License :: OSI Approved :: BSD License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Application
 Classifier: Typing :: Typed
+Provides-Extra: alembic
 Provides-Extra: all
 Provides-Extra: broadcast
+Provides-Extra: debug
 Provides-Extra: dev
 Provides-Extra: oauth2
+Provides-Extra: sentry
 Provides-Extra: standard
 Provides-Extra: test-images
-Requires-Dist: SQLAlchemy (>=1.4.0,<3.0.0) ; extra == "standard" or extra == "all"
-Requires-Dist: SQLAlchemy-Utils ; extra == "standard" or extra == "all"
-Requires-Dist: alembic ; extra == "standard" or extra == "all"
-Requires-Dist: boltons ; extra == "standard" or extra == "all"
-Requires-Dist: cee_syslog_handler ; extra == "standard" or extra == "all"
+Provides-Extra: tests
+Provides-Extra: webserver
+Requires-Dist: SQLAlchemy (>=1.4.0,<3.0.0)
+Requires-Dist: SQLAlchemy-Utils
+Requires-Dist: alembic ; extra == "standard" or extra == "alembic" or extra == "all"
+Requires-Dist: boltons ; extra == "tests" or extra == "all"
+Requires-Dist: cee_syslog_handler
 Requires-Dist: certifi
-Requires-Dist: cornice ; extra == "standard" or extra == "all"
-Requires-Dist: gunicorn ; extra == "standard" or extra == "all"
-Requires-Dist: lxml ; extra == "standard" or extra == "all"
-Requires-Dist: netifaces ; extra == "standard" or extra == "all"
-Requires-Dist: objgraph ; extra == "standard" or extra == "all"
+Requires-Dist: cornice
+Requires-Dist: gunicorn ; extra == "standard" or extra == "webserver" or extra == "all"
+Requires-Dist: lxml ; extra == "tests" or extra == "all"
+Requires-Dist: objgraph ; extra == "debug" or extra == "all"
 Requires-Dist: prometheus-client
-Requires-Dist: psycopg2 ; extra == "standard" or extra == "all"
-Requires-Dist: pyjwt ; extra == "oauth2" or extra == "all"
-Requires-Dist: pyramid ; extra == "standard" or extra == "all"
-Requires-Dist: pyramid-tm ; extra == "standard" or extra == "all"
+Requires-Dist: psycopg2
+Requires-Dist: pyjwt ; extra == "standard" or extra == "oauth2" or extra == "all"
+Requires-Dist: pyramid
+Requires-Dist: pyramid-tm
+Requires-Dist: pyramid_mako
 Requires-Dist: pyyaml
-Requires-Dist: redis ; extra == "standard" or extra == "broadcast" or extra == "all" or extra == "all"
+Requires-Dist: redis ; extra == "standard" or extra == "broadcast" or extra == "all"
 Requires-Dist: requests
-Requires-Dist: requests-oauthlib ; extra == "oauth2" or extra == "all"
+Requires-Dist: requests-oauthlib ; extra == "standard" or extra == "oauth2" or extra == "all"
 Requires-Dist: scikit-image ; extra == "test-images"
-Requires-Dist: sentry-sdk ; extra == "standard" or extra == "all"
-Requires-Dist: ujson ; extra == "standard" or extra == "all"
+Requires-Dist: sentry-sdk ; extra == "standard" or extra == "sentry" or extra == "all"
+Requires-Dist: ujson
 Requires-Dist: waitress ; extra == "dev" or extra == "all"
-Requires-Dist: zope.interface ; extra == "standard" or extra == "all"
-Requires-Dist: zope.sqlalchemy ; extra == "standard" or extra == "all"
+Requires-Dist: zope.interface
+Requires-Dist: zope.sqlalchemy
 Project-URL: Repository, https://github.com/camptocamp/c2cwsgiutils
 Description-Content-Type: text/markdown
 
@@ -120,7 +124,6 @@ ENV \
   LOG_TYPE=console \
   OTHER_LOG_LEVEL=WARNING \
   GUNICORN_LOG_LEVEL=WARNING \
-  GUNICORN_ACCESS_LOG_LEVEL=INFO \
   SQL_LOG_LEVEL=WARNING \
   C2CWSGIUTILS_LOG_LEVEL=WARNING \
   LOG_LEVEL=INFO
@@ -159,8 +162,6 @@ The related environment variables:
 - `SQL_LOG_LEVEL`: The SQL query log level, `WARNING`: no logs, `INFO`: logs the queries,
   `DEBUG` also logs the results, default is `WARNING`.
 - `GUNICORN_ERROR_LOG_LEVEL`: The Gunicorn error log level, default is `WARNING`.
-- `GUNICORN_ACCESS_LOG_LEVEL`: The Gunicorn access log level, the logs have the level `INFO`,
-  default is `WARNING`.
 - `C2CWSGIUTILS_CONFIG`: The fallback ini file to use by gunicorn, default is `production.ini`.
 - `C2CWSGIUTILS_LOG_LEVEL`: The c2c WSGI utils log level, default is `WARNING`.
 - `OTHER_LOG_LEVEL`: The log level for all the other logger, default is `WARNING`.
@@ -483,14 +484,14 @@ command line. Usually done in the [Dockerfile](acceptance_tests/app/Dockerfile)
 It will work in multi process mode with the limitation listed in the
 [`prometheus_client` documentation](https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn).
 
-To enable it you should provide the `PROMETHEUS_PORT` environment variable.
+To enable it you should provide the `C2CWSGIUTILS_PROMETHEUS_PORT` environment variable.
 For security reason, this port should not be exposed.
 
 We can customize it with the following environment variables:
 
-- `PROMETHEUS_PREFIX`: to customize the prefix, default is `c2cwsggiutils-`.
+- `C2C_PROMETHEUS_PREFIX`: to customize the prefix, default is `c2cwsggiutils-`.
 - `C2C_PROMETHEUS_PACKAGES` the packages that will be present in the version information, default is `c2cwsgiutils,pyramid,gunicorn,sqlalchemy`.
-- `C2C_PROMETHEUS_APPLICATION_PACKAGES` the packages that will be present in the version information as application.
+- `C2C_PROMETHEUS_APPLICATION_PACKAGE` the packages that will be present in the version information as application.
 
 And you should add in your `gunicorn.conf.py`:
 
@@ -524,7 +525,7 @@ In your `Dockerfile` you should add:
 
 ```dockerfile
 RUN mkdir -p /prometheus-metrics \
-    chmod a+rwx /prometheus-metrics
+    && chmod a+rwx /prometheus-metrics
 ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
 ```
 
@@ -757,6 +758,35 @@ To make a release:
 - Add the new branch name in the `.github/workflows/rebuild.yaml` and
   `.github/workflows/audit.yaml` files.
 
+## Testing
+
+### Screenshots
+
+To test the screenshots, you need to install `node` with `npm`, to do that add the following lines in your `Dockerfile`:
+
+```dockerfile
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    --mount=type=cache,target=/var/cache,sharing=locked \
+    . /etc/os-release \
+    && echo "deb https://deb.nodesource.com/node_18.x ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/nodesource.list \
+    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && apt-get update \
+    && apt-get install --assume-yes --no-install-recommends 'nodejs=18.*'
+```
+
+To do the image test call `check_screenshot` e.g.:
+
+```python
+def test_screenshot(app_connection):
+    image.check_screenshot(
+        app_connection.base_url + "my-path",
+        width=800,
+        height=600,
+        result_folder="results",
+        expected_filename=os.path.join(os.path.dirname(__file__), "my-check.expected.png"),
+    )
+```
+
 ## Contributing
 
 Install the pre-commit hooks:

{c2cwsgiutils-1.0.0.dev1 → c2cwsgiutils-5.2.1.dev197}/README.md

@@ -60,7 +60,6 @@ ENV \
   LOG_TYPE=console \
   OTHER_LOG_LEVEL=WARNING \
   GUNICORN_LOG_LEVEL=WARNING \
-  GUNICORN_ACCESS_LOG_LEVEL=INFO \
   SQL_LOG_LEVEL=WARNING \
   C2CWSGIUTILS_LOG_LEVEL=WARNING \
   LOG_LEVEL=INFO
@@ -99,8 +98,6 @@ The related environment variables:
 - `SQL_LOG_LEVEL`: The SQL query log level, `WARNING`: no logs, `INFO`: logs the queries,
   `DEBUG` also logs the results, default is `WARNING`.
 - `GUNICORN_ERROR_LOG_LEVEL`: The Gunicorn error log level, default is `WARNING`.
-- `GUNICORN_ACCESS_LOG_LEVEL`: The Gunicorn access log level, the logs have the level `INFO`,
-  default is `WARNING`.
 - `C2CWSGIUTILS_CONFIG`: The fallback ini file to use by gunicorn, default is `production.ini`.
 - `C2CWSGIUTILS_LOG_LEVEL`: The c2c WSGI utils log level, default is `WARNING`.
 - `OTHER_LOG_LEVEL`: The log level for all the other logger, default is `WARNING`.
@@ -423,14 +420,14 @@ command line. Usually done in the [Dockerfile](acceptance_tests/app/Dockerfile)
 It will work in multi process mode with the limitation listed in the
 [`prometheus_client` documentation](https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn).
 
-To enable it you should provide the `PROMETHEUS_PORT` environment variable.
+To enable it you should provide the `C2CWSGIUTILS_PROMETHEUS_PORT` environment variable.
 For security reason, this port should not be exposed.
 
 We can customize it with the following environment variables:
 
-- `PROMETHEUS_PREFIX`: to customize the prefix, default is `c2cwsggiutils-`.
+- `C2C_PROMETHEUS_PREFIX`: to customize the prefix, default is `c2cwsggiutils-`.
 - `C2C_PROMETHEUS_PACKAGES` the packages that will be present in the version information, default is `c2cwsgiutils,pyramid,gunicorn,sqlalchemy`.
-- `C2C_PROMETHEUS_APPLICATION_PACKAGES` the packages that will be present in the version information as application.
+- `C2C_PROMETHEUS_APPLICATION_PACKAGE` the packages that will be present in the version information as application.
 
 And you should add in your `gunicorn.conf.py`:
 
@@ -464,7 +461,7 @@ In your `Dockerfile` you should add:
 
 ```dockerfile
 RUN mkdir -p /prometheus-metrics \
-    chmod a+rwx /prometheus-metrics
+    && chmod a+rwx /prometheus-metrics
 ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
 ```
 
@@ -697,6 +694,35 @@ To make a release:
 - Add the new branch name in the `.github/workflows/rebuild.yaml` and
   `.github/workflows/audit.yaml` files.
 
+## Testing
+
+### Screenshots
+
+To test the screenshots, you need to install `node` with `npm`, to do that add the following lines in your `Dockerfile`:
+
+```dockerfile
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    --mount=type=cache,target=/var/cache,sharing=locked \
+    . /etc/os-release \
+    && echo "deb https://deb.nodesource.com/node_18.x ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/nodesource.list \
+    && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && apt-get update \
+    && apt-get install --assume-yes --no-install-recommends 'nodejs=18.*'
+```
+
+To do the image test call `check_screenshot` e.g.:
+
+```python
+def test_screenshot(app_connection):
+    image.check_screenshot(
+        app_connection.base_url + "my-path",
+        width=800,
+        height=600,
+        result_folder="results",
+        expected_filename=os.path.join(os.path.dirname(__file__), "my-check.expected.png"),
+    )
+```
+
 ## Contributing
 
 Install the pre-commit hooks:

{c2cwsgiutils-1.0.0.dev1 → c2cwsgiutils-5.2.1.dev197}/c2cwsgiutils/__init__.py

@@ -4,12 +4,12 @@ import os
 import re
 import sys
 from configparser import SectionProxy
-from typing import Any, Dict, Set
+from typing import Any
 
 LOG = logging.getLogger(__name__)
 
 
-def get_config_defaults() -> Dict[str, str]:
+def get_config_defaults() -> dict[str, str]:
     """
     Get the environment variables as defaults for configparser.
 
@@ -18,8 +18,8 @@ def get_config_defaults() -> Dict[str, str]:
 
     configparser interpretate the % then we need to escape them
     """
-    result: Dict[str, str] = {}
-    lowercase_keys: Set[str] = set()
+    result: dict[str, str] = {}
+    lowercase_keys: set[str] = set()
     for key, value in os.environ.items():
         if key.lower() in lowercase_keys:
             LOG.warning("The environment variable '%s' is duplicated with different case, ignoring", key)
@@ -29,9 +29,9 @@ def get_config_defaults() -> Dict[str, str]:
     return result
 
 
-def _create_handlers(config: configparser.ConfigParser) -> Dict[str, Any]:
+def _create_handlers(config: configparser.ConfigParser) -> dict[str, Any]:
     handlers = [k.strip() for k in config["handlers"]["keys"].split(",")]
-    d_handlers: Dict[str, Any] = {}
+    d_handlers: dict[str, Any] = {}
     stream_re = re.compile(r"\((.*?),\)")
     for hh in handlers:
         block = config[f"handler_{hh}"]
@@ -53,8 +53,8 @@ def _create_handlers(config: configparser.ConfigParser) -> Dict[str, Any]:
     return d_handlers
 
 
-def _filter_logger(block: SectionProxy) -> Dict[str, Any]:
-    out: Dict[str, Any] = {"level": block["level"]}
+def _filter_logger(block: SectionProxy) -> dict[str, Any]:
+    out: dict[str, Any] = {"level": block["level"]}
     handlers = block.get("handlers", "")
     if handlers != "":
         out["handlers"] = [block["handlers"]]
@@ -65,7 +65,7 @@ def _filter_logger(block: SectionProxy) -> Dict[str, Any]:
 # logging configuration
 # https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
 ###
-def get_logconfig_dict(filename: str) -> Dict[str, Any]:
+def get_logconfig_dict(filename: str) -> dict[str, Any]:
     """
     Create a logconfig dictionary based on the provided ini file.
 
@@ -76,8 +76,8 @@ def get_logconfig_dict(filename: str) -> Dict[str, Any]:
     loggers = [k.strip() for k in config["loggers"]["keys"].split(",")]
     formatters = [k.strip() for k in config["formatters"]["keys"].split(",")]
 
-    d_loggers: Dict[str, Any] = {}
-    root: Dict[str, Any] = {}
+    d_loggers: dict[str, Any] = {}
+    root: dict[str, Any] = {}
     for ll in loggers:
         block = config[f"logger_{ll}"]
         if ll == "root":
@@ -86,7 +86,7 @@ def get_logconfig_dict(filename: str) -> Dict[str, Any]:
         qualname = block["qualname"]
         d_loggers[qualname] = _filter_logger(block)
 
-    d_formatters: Dict[str, Any] = {}
+    d_formatters: dict[str, Any] = {}
     for ff in formatters:
         block = config[f"formatter_{ff}"]
         d_formatters[ff] = {

{c2cwsgiutils-1.0.0.dev1 → c2cwsgiutils-5.2.1.dev197}/c2cwsgiutils/acceptance/connection.py

@@ -1,9 +1,9 @@
 import re
+from collections.abc import Mapping, MutableMapping
 from enum import Enum
-from typing import Any, Mapping, MutableMapping, Optional, Union
+from typing import Any, Optional, Union
 
 import requests
-from lxml import etree  # nosec
 
 COLON_SPLIT_RE = re.compile(r"\s*,\s*")
 
@@ -82,6 +82,9 @@ class Connection:
         **kwargs: Any,
     ) -> Any:
         """Get the given URL (relative to the root of API)."""
+
+        from lxml import etree  # nosec
+
         with self.session.get(
             self.base_url + url,
             headers=self._merge_headers(headers, cors),

{c2cwsgiutils-1.0.0.dev1 → c2cwsgiutils-5.2.1.dev197}/c2cwsgiutils/acceptance/image.py

@@ -1,4 +1,6 @@
+import json
 import os
+import subprocess  # nosec
 from typing import TYPE_CHECKING, Any, Optional
 
 import numpy as np
@@ -42,6 +44,20 @@ def check_image_file(
     check_image(result_folder, result, expected_filename, level, generate_expected_image, use_mask)
 
 
+def normalize_image(image: NpNdarrayInt) -> NpNdarrayInt:
+    """
+    Normalize the image to be comparable.
+
+    - Remove the alpha channel
+    - Convert to uint8
+    """
+    if len(image.shape) == 3 and image.shape[2] == 4:
+        image = skimage.color.rgba2rgb(image)
+    if np.issubdtype(image.dtype, np.floating):
+        image = (image * 255).astype("uint8")
+    return image
+
+
 def check_image(
     result_folder: str,
     image_to_check: NpNdarrayInt,
@@ -80,11 +96,30 @@ def check_image(
     result_filename = os.path.join(result_folder, f"{image_file_basename}.result.png")
     diff_filename = os.path.join(result_folder, f"{image_file_basename}.diff.png")
 
+    image_to_check = normalize_image(image_to_check)
+
     mask = None
     if mask_filename is not None:
         mask = skimage.io.imread(mask_filename)
+
         assert mask is not None, "Wrong mask: " + mask_filename
-        image_to_check[mask == 0] = [255, 255, 255]
+
+        # Normalize the mask
+        if len(mask.shape) == 3 and mask.shape[2] == 3:
+            mask = skimage.color.rgb2gray(mask)
+
+        if len(mask.shape) == 3 and mask.shape[2] == 4:
+            mask = skimage.color.rgba2gray(mask)
+
+        if np.issubdtype(mask.dtype, np.floating):
+            mask = (mask * 255).astype("uint8")
+
+        assert ((0 < mask) & (mask < 255)).sum() == 0, "Mask should be only black and white image"
+
+        # Convert to boolean
+        mask = mask == 0
+
+        image_to_check[mask] = [255, 255, 255]
 
     if not os.path.exists(result_folder):
         os.makedirs(result_folder)
@@ -97,9 +132,10 @@ def check_image(
         assert False, "Expected image not found: " + expected_filename
     expected = skimage.io.imread(expected_filename)
     assert expected is not None, "Wrong image: " + expected_filename
+    expected = normalize_image(expected)
 
     if mask is not None:
-        expected[mask == 0] = [255, 255, 255]
+        expected[mask] = [255, 255, 255]
 
     score, diff = skimage.metrics.structural_similarity(
         expected, image_to_check, multichannel=True, full=True, channel_axis=2
@@ -115,3 +151,59 @@ def check_image(
         assert (
             score >= level
         ), f"{result_filename} != {expected_filename} => {diff_filename} ({score} < {level})"
+
+
+def check_screenshot(
+    url: str,
+    result_folder: str,
+    expected_filename: str,
+    width: int = 800,
+    height: int = 600,
+    headers: Optional[dict[str, str]] = None,
+    media: Optional[list[dict[str, str]]] = None,
+    level: float = 1.0,
+    generate_expected_image: bool = False,
+    use_mask: bool = True,
+) -> None:
+    """
+    Test that the screenshot of the `url` corresponds to the image `expected_filename`.
+
+    Requires nodejs with puppeteer and commander to be installed.
+    """
+
+    if headers is None:
+        headers = {}
+    if media is None:
+        media = []
+
+    if not os.path.exists(os.path.join(os.path.dirname(__file__), "node_modules")):
+        subprocess.run(["npm", "install"], cwd=os.path.dirname(__file__), check=True)  # nosec
+
+    image_file_basename = os.path.splitext(os.path.basename(expected_filename))[0]
+    if image_file_basename.endswith(".expected"):
+        image_file_basename = os.path.splitext(image_file_basename)[0]
+
+    result_folder = os.path.abspath(result_folder)
+    actual_filename = os.path.join(result_folder, f"{image_file_basename}.actual.png")
+    subprocess.run(  # nosec
+        [
+            "node",
+            "screenshot.js",
+            f"--url={url}",
+            f"--width={width}",
+            f"--height={height}",
+            f"--headers={json.dumps(headers)}",
+            f"--media={json.dumps(media)}",
+            f"--output={actual_filename}",
+        ],
+        cwd=os.path.dirname(__file__),
+        check=True,
+    )
+    check_image(
+        result_folder,
+        skimage.io.imread(actual_filename)[:, :, :3],
+        expected_filename,
+        level,
+        generate_expected_image,
+        use_mask,
+    )