c2cwsgiutils 5.1.7.dev20230901073305__py3-none-any.whl → 5.2.1.dev197__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. c2cwsgiutils/__init__.py +13 -13
  2. c2cwsgiutils/acceptance/connection.py +5 -2
  3. c2cwsgiutils/acceptance/image.py +98 -4
  4. c2cwsgiutils/acceptance/package-lock.json +1933 -0
  5. c2cwsgiutils/acceptance/package.json +7 -0
  6. c2cwsgiutils/acceptance/print.py +4 -4
  7. c2cwsgiutils/acceptance/screenshot.js +62 -0
  8. c2cwsgiutils/acceptance/utils.py +14 -22
  9. c2cwsgiutils/auth.py +4 -4
  10. c2cwsgiutils/broadcast/__init__.py +15 -7
  11. c2cwsgiutils/broadcast/interface.py +3 -2
  12. c2cwsgiutils/broadcast/local.py +3 -2
  13. c2cwsgiutils/broadcast/redis.py +8 -7
  14. c2cwsgiutils/client_info.py +5 -5
  15. c2cwsgiutils/config_utils.py +2 -1
  16. c2cwsgiutils/coverage_setup.py +2 -2
  17. c2cwsgiutils/db.py +58 -37
  18. c2cwsgiutils/db_maintenance_view.py +2 -1
  19. c2cwsgiutils/debug/_listeners.py +10 -9
  20. c2cwsgiutils/debug/_views.py +12 -11
  21. c2cwsgiutils/debug/utils.py +5 -5
  22. c2cwsgiutils/errors.py +7 -6
  23. c2cwsgiutils/health_check.py +96 -85
  24. c2cwsgiutils/index.py +90 -105
  25. c2cwsgiutils/loader.py +3 -3
  26. c2cwsgiutils/logging_view.py +3 -2
  27. c2cwsgiutils/models_graph.py +8 -6
  28. c2cwsgiutils/prometheus.py +175 -57
  29. c2cwsgiutils/pyramid.py +4 -2
  30. c2cwsgiutils/pyramid_logging.py +2 -1
  31. c2cwsgiutils/redis_stats.py +13 -11
  32. c2cwsgiutils/redis_utils.py +15 -14
  33. c2cwsgiutils/request_tracking/__init__.py +36 -30
  34. c2cwsgiutils/request_tracking/_sql.py +3 -1
  35. c2cwsgiutils/scripts/genversion.py +4 -4
  36. c2cwsgiutils/scripts/stats_db.py +130 -68
  37. c2cwsgiutils/scripts/test_print.py +1 -1
  38. c2cwsgiutils/sentry.py +2 -1
  39. c2cwsgiutils/setup_process.py +13 -17
  40. c2cwsgiutils/sql_profiler/_impl.py +12 -5
  41. c2cwsgiutils/sqlalchemylogger/README.md +48 -0
  42. c2cwsgiutils/sqlalchemylogger/_models.py +7 -4
  43. c2cwsgiutils/sqlalchemylogger/examples/example.py +15 -0
  44. c2cwsgiutils/sqlalchemylogger/handlers.py +11 -8
  45. c2cwsgiutils/static/favicon-16x16.png +0 -0
  46. c2cwsgiutils/static/favicon-32x32.png +0 -0
  47. c2cwsgiutils/stats_pyramid/__init__.py +7 -11
  48. c2cwsgiutils/stats_pyramid/_db_spy.py +14 -11
  49. c2cwsgiutils/stats_pyramid/_pyramid_spy.py +29 -20
  50. c2cwsgiutils/templates/index.html.mako +50 -0
  51. c2cwsgiutils/version.py +49 -16
  52. c2cwsgiutils-5.2.1.dev197.dist-info/LICENSE +22 -0
  53. {c2cwsgiutils-5.1.7.dev20230901073305.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/METADATA +187 -135
  54. c2cwsgiutils-5.2.1.dev197.dist-info/RECORD +67 -0
  55. {c2cwsgiutils-5.1.7.dev20230901073305.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/WHEEL +1 -2
  56. c2cwsgiutils-5.2.1.dev197.dist-info/entry_points.txt +21 -0
  57. c2cwsgiutils/acceptance/composition.py +0 -129
  58. c2cwsgiutils/metrics.py +0 -110
  59. c2cwsgiutils/scripts/check_es.py +0 -130
  60. c2cwsgiutils/scripts/coverage_report.py +0 -36
  61. c2cwsgiutils/stats.py +0 -355
  62. c2cwsgiutils/stats_pyramid/_views.py +0 -16
  63. c2cwsgiutils-5.1.7.dev20230901073305.data/scripts/c2cwsgiutils-run +0 -32
  64. c2cwsgiutils-5.1.7.dev20230901073305.dist-info/LICENSE.txt +0 -28
  65. c2cwsgiutils-5.1.7.dev20230901073305.dist-info/RECORD +0 -69
  66. c2cwsgiutils-5.1.7.dev20230901073305.dist-info/entry_points.txt +0 -25
  67. c2cwsgiutils-5.1.7.dev20230901073305.dist-info/top_level.txt +0 -2
  68. tests/acceptance/__init__.py +0 -0
  69. tests/acceptance/test_utils.py +0 -13
@@ -1,89 +1,73 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: c2cwsgiutils
3
- Version: 5.1.7.dev20230901073305
3
+ Version: 5.2.1.dev197
4
4
  Summary: Common utilities for Camptocamp WSGI applications
5
5
  Home-page: https://github.com/camptocamp/c2cwsgiutils
6
+ License: BSD-2-Clause
7
+ Keywords: geo,gis,sqlalchemy,orm,wsgi
6
8
  Author: Camptocamp
7
9
  Author-email: info@camptocamp.com
8
- License: FreeBSD
9
- Keywords: geo gis sqlalchemy orm wsgi
10
+ Requires-Python: >=3.9,<4.0
10
11
  Classifier: Development Status :: 5 - Production/Stable
11
12
  Classifier: Environment :: Plugins
12
13
  Classifier: Framework :: Pyramid
13
14
  Classifier: Intended Audience :: Developers
14
- Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Intended Audience :: Information Technology
16
+ Classifier: License :: OSI Approved :: BSD License
15
17
  Classifier: Operating System :: OS Independent
16
18
  Classifier: Programming Language :: Python
17
19
  Classifier: Programming Language :: Python :: 3
18
- Classifier: Programming Language :: Python :: 3.8
19
- Classifier: Intended Audience :: Information Technology
20
+ Classifier: Programming Language :: Python :: 3.9
21
+ Classifier: Programming Language :: Python :: 3.10
22
+ Classifier: Programming Language :: Python :: 3.11
20
23
  Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Application
21
24
  Classifier: Typing :: Typed
22
- Description-Content-Type: text/markdown
23
- License-File: LICENSE.txt
25
+ Provides-Extra: alembic
26
+ Provides-Extra: all
24
27
  Provides-Extra: broadcast
25
- Requires-Dist: redis ; extra == 'broadcast'
28
+ Provides-Extra: debug
29
+ Provides-Extra: dev
30
+ Provides-Extra: oauth2
31
+ Provides-Extra: sentry
26
32
  Provides-Extra: standard
27
- Requires-Dist: alembic ; extra == 'standard'
28
- Requires-Dist: boltons ; extra == 'standard'
29
- Requires-Dist: cornice ; extra == 'standard'
30
- Requires-Dist: gunicorn ; extra == 'standard'
31
- Requires-Dist: lxml ; extra == 'standard'
32
- Requires-Dist: netifaces ; extra == 'standard'
33
- Requires-Dist: objgraph ; extra == 'standard'
34
- Requires-Dist: pipfile ; extra == 'standard'
35
- Requires-Dist: psycopg2 ; extra == 'standard'
36
- Requires-Dist: pyramid ; extra == 'standard'
37
- Requires-Dist: pyramid-tm ; extra == 'standard'
38
- Requires-Dist: sentry-sdk ; extra == 'standard'
39
- Requires-Dist: requests ; extra == 'standard'
40
- Requires-Dist: ujson ; extra == 'standard'
41
- Requires-Dist: cee-syslog-handler ; extra == 'standard'
42
- Requires-Dist: sqlalchemy ; extra == 'standard'
43
- Requires-Dist: sqlalchemy-utils ; extra == 'standard'
44
- Requires-Dist: zope.interface ; extra == 'standard'
45
- Requires-Dist: zope.sqlalchemy ; extra == 'standard'
46
- Requires-Dist: pyyaml ; extra == 'standard'
47
- Requires-Dist: pyjwt ; extra == 'standard'
48
- Requires-Dist: requests-oauthlib ; extra == 'standard'
49
- Requires-Dist: waitress ; extra == 'standard'
50
- Requires-Dist: async-timeout ; extra == 'standard'
51
- Requires-Dist: certifi ; extra == 'standard'
52
- Requires-Dist: charset-normalizer ; extra == 'standard'
53
- Requires-Dist: deprecated ; extra == 'standard'
54
- Requires-Dist: graphviz ; extra == 'standard'
55
- Requires-Dist: greenlet ; extra == 'standard'
56
- Requires-Dist: hupper ; extra == 'standard'
57
- Requires-Dist: idna ; extra == 'standard'
58
- Requires-Dist: importlib-metadata ; extra == 'standard'
59
- Requires-Dist: importlib-resources ; extra == 'standard'
60
- Requires-Dist: mako ; extra == 'standard'
61
- Requires-Dist: markupsafe ; extra == 'standard'
62
- Requires-Dist: oauthlib ; extra == 'standard'
63
- Requires-Dist: packaging ; extra == 'standard'
64
- Requires-Dist: pastedeploy ; extra == 'standard'
65
- Requires-Dist: plaster ; extra == 'standard'
66
- Requires-Dist: plaster-pastedeploy ; extra == 'standard'
67
- Requires-Dist: pyparsing ; extra == 'standard'
68
- Requires-Dist: toml ; extra == 'standard'
69
- Requires-Dist: transaction ; extra == 'standard'
70
- Requires-Dist: translationstring ; extra == 'standard'
71
- Requires-Dist: urllib3 ; extra == 'standard'
72
- Requires-Dist: venusian ; extra == 'standard'
73
- Requires-Dist: webob ; extra == 'standard'
74
- Requires-Dist: wrapt ; extra == 'standard'
75
- Requires-Dist: zipp ; extra == 'standard'
76
- Requires-Dist: zope.deprecation ; extra == 'standard'
77
- Provides-Extra: test_images
78
- Requires-Dist: scikit-image ; extra == 'test_images'
33
+ Provides-Extra: test-images
34
+ Provides-Extra: tests
35
+ Provides-Extra: webserver
36
+ Requires-Dist: SQLAlchemy (>=1.4.0,<3.0.0)
37
+ Requires-Dist: SQLAlchemy-Utils
38
+ Requires-Dist: alembic ; extra == "standard" or extra == "alembic" or extra == "all"
39
+ Requires-Dist: boltons ; extra == "tests" or extra == "all"
40
+ Requires-Dist: cee_syslog_handler
41
+ Requires-Dist: certifi
42
+ Requires-Dist: cornice
43
+ Requires-Dist: gunicorn ; extra == "standard" or extra == "webserver" or extra == "all"
44
+ Requires-Dist: lxml ; extra == "tests" or extra == "all"
45
+ Requires-Dist: objgraph ; extra == "debug" or extra == "all"
46
+ Requires-Dist: prometheus-client
47
+ Requires-Dist: psycopg2
48
+ Requires-Dist: pyjwt ; extra == "standard" or extra == "oauth2" or extra == "all"
49
+ Requires-Dist: pyramid
50
+ Requires-Dist: pyramid-tm
51
+ Requires-Dist: pyramid_mako
52
+ Requires-Dist: pyyaml
53
+ Requires-Dist: redis ; extra == "standard" or extra == "broadcast" or extra == "all"
54
+ Requires-Dist: requests
55
+ Requires-Dist: requests-oauthlib ; extra == "standard" or extra == "oauth2" or extra == "all"
56
+ Requires-Dist: scikit-image ; extra == "test-images"
57
+ Requires-Dist: sentry-sdk ; extra == "standard" or extra == "sentry" or extra == "all"
58
+ Requires-Dist: ujson
59
+ Requires-Dist: waitress ; extra == "dev" or extra == "all"
60
+ Requires-Dist: zope.interface
61
+ Requires-Dist: zope.sqlalchemy
62
+ Project-URL: Repository, https://github.com/camptocamp/c2cwsgiutils
63
+ Description-Content-Type: text/markdown
79
64
 
80
65
  # Camptocamp WSGI utilities
81
66
 
82
67
  This is a Python 3 library (>=3.5) providing common tools for Camptocamp WSGI
83
68
  applications:
84
69
 
85
- - Provide a small framework for gathering performance statistics about
86
- a web application (statsd protocol)
70
+ - Provide prometheus metrics
87
71
  - Allow to use a master/slave PostgresQL configuration
88
72
  - Logging handler for CEE/UDP logs
89
73
  - An optional view to change runtime the log levels
@@ -122,7 +106,7 @@ You should install `c2cwsgiutils` with the tool you use to manage your pip depen
122
106
 
123
107
  In the `Dockerfile` you should add the following lines:
124
108
 
125
- ```
109
+ ```dockerfile
126
110
  # Generate the version file.
127
111
  RUN c2cwsgiutils-genversion $(git rev-parse HEAD)
128
112
 
@@ -130,20 +114,19 @@ CMD ["gunicorn", "--paste=/app/production.ini"]
130
114
 
131
115
  # Default values for the environment variables
132
116
  ENV \
133
- DEVELOPMENT=0 \
134
- SQLALCHEMY_POOL_RECYCLE=30 \
135
- SQLALCHEMY_POOL_SIZE=5 \
136
- SQLALCHEMY_MAX_OVERFLOW=25 \
137
- SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \
138
- SQLALCHEMY_SLAVE_POOL_SIZE=5 \
139
- SQLALCHEMY_SLAVE_MAX_OVERFLOW=25\
140
- LOG_TYPE=console \
141
- OTHER_LOG_LEVEL=WARNING \
142
- GUNICORN_LOG_LEVEL=WARNING \
143
- GUNICORN_ACCESS_LOG_LEVEL=INFO \
144
- SQL_LOG_LEVEL=WARNING \
145
- C2CWSGIUTILS_LOG_LEVEL=WARNING \
146
- LOG_LEVEL=INFO
117
+ DEVELOPMENT=0 \
118
+ SQLALCHEMY_POOL_RECYCLE=30 \
119
+ SQLALCHEMY_POOL_SIZE=5 \
120
+ SQLALCHEMY_MAX_OVERFLOW=25 \
121
+ SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \
122
+ SQLALCHEMY_SLAVE_POOL_SIZE=5 \
123
+ SQLALCHEMY_SLAVE_MAX_OVERFLOW=25\
124
+ LOG_TYPE=console \
125
+ OTHER_LOG_LEVEL=WARNING \
126
+ GUNICORN_LOG_LEVEL=WARNING \
127
+ SQL_LOG_LEVEL=WARNING \
128
+ C2CWSGIUTILS_LOG_LEVEL=WARNING \
129
+ LOG_LEVEL=INFO
147
130
  ```
148
131
 
149
132
  Add in your `main` function.
@@ -179,8 +162,6 @@ The related environment variables:
179
162
  - `SQL_LOG_LEVEL`: The SQL query log level, `WARNING`: no logs, `INFO`: logs the queries,
180
163
  `DEBUG` also logs the results, default is `WARNING`.
181
164
  - `GUNICORN_ERROR_LOG_LEVEL`: The Gunicorn error log level, default is `WARNING`.
182
- - `GUNICORN_ACCESS_LOG_LEVEL`: The Gunicorn access log level, the logs have the level `INFO`,
183
- default is `WARNING`.
184
165
  - `C2CWSGIUTILS_CONFIG`: The fallback ini file to use by gunicorn, default is `production.ini`.
185
166
  - `C2CWSGIUTILS_LOG_LEVEL`: The c2c WSGI utils log level, default is `WARNING`.
186
167
  - `OTHER_LOG_LEVEL`: The log level for all the other logger, default is `WARNING`.
@@ -349,47 +330,6 @@ The requests module is also patched to monitor requests done without timeout. In
349
330
  configure a default timeout with the `C2C_REQUESTS_DEFAULT_TIMEOUT` environment variable
350
331
  (`c2c.requests_default_timeout`). If no timeout and no default is specified, a warning is issued.
351
332
 
352
- ## Metrics
353
-
354
- To enable and configure the metrics framework, you can use:
355
-
356
- - STATS_VIEW (c2c.stats_view): if defined, will enable the stats view `{C2C_BASE_PATH}/stats.json`
357
- - STATSD_ADDRESS (c2c.statsd_address): if defined, send stats to the given statsd server
358
- - STATSD_PREFIX (c2c.statsd_prefix): prefix to add to every metric names
359
- - STATSD_USE_TAGS: If true, automatic metrics will use tags
360
- - STATSD*TAG*{tag_name}: To set a global tag for the service
361
-
362
- If enabled, some metrics are automatically generated:
363
-
364
- - {STATSD_PREFIX}.route.{verb}.{route_name}.{status}: The time to process a query (includes rendering)
365
- - {STATSD_PREFIX}.render.{verb}.{route_name}.{status}: The time to render a query
366
- - {STATSD_PREFIX}.sql.{query}: The time to execute the given SQL query (simplified and normalized)
367
- - {STATSD_PREFIX}.requests.{scheme}.{hostname}.{port}.{verb}.{status}: The time to execute HTTP requests to
368
- outside services (only the time between the start of sending of the request and when the header is
369
- back with a chunk of the body)
370
- - {STATSD_PREFIX}.redis.{command}: The time to execute the given Redis command
371
-
372
- You can manually measure the time spent on something like that:
373
-
374
- ```python
375
- from c2cwsgiutils import stats
376
- with stats.timer_context(['toto', 'tutu']):
377
- do_something()
378
- ```
379
-
380
- It will only add a timer event in case of success. If you want to measure both success and failures, do that:
381
-
382
- ```python
383
- from c2cwsgiutils import stats
384
- with stats.outcome_timer_context(['toto', 'tutu']):
385
- do_something()
386
- ```
387
-
388
- Other functions exists to generate metrics. Look at the `c2cwsgiutils.stats` module.
389
-
390
- Look at the `c2cwsgiutils-stats-db` utility if you want to generate statistics (gauges) about the
391
- row counts.
392
-
393
333
  ## SQL profiler
394
334
 
395
335
  The SQL profiler must be configured with the `C2C_SQL_PROFILER_ENABLED` environment variable. That enables a view
@@ -537,27 +477,100 @@ If the `/app/versions.json` exists, a view is added (`{C2C_BASE_PATH}/versions.j
537
477
  version of a app. This file is generated by calling the `c2cwsgiutils-genversion [$GIT_TAG] $GIT_HASH`
538
478
  command line. Usually done in the [Dockerfile](acceptance_tests/app/Dockerfile) of the WSGI application.
539
479
 
540
- ## Metrics
480
+ ## Prometheus
481
+
482
+ [Prometheus client](https://github.com/prometheus/client_python) is integrated in c2cwsgiutils.
483
+
484
+ It will work in multi process mode with the limitation listed in the
485
+ [`prometheus_client` documentation](https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn).
486
+
487
+ To enable it you should provide the `C2CWSGIUTILS_PROMETHEUS_PORT` environment variable.
488
+ For security reason, this port should not be exposed.
489
+
490
+ We can customize it with the following environment variables:
491
+
492
+ - `C2C_PROMETHEUS_PREFIX`: to customize the prefix, default is `c2cwsggiutils-`.
493
+ - `C2C_PROMETHEUS_PACKAGES` the packages that will be present in the version information, default is `c2cwsgiutils,pyramid,gunicorn,sqlalchemy`.
494
+ - `C2C_PROMETHEUS_APPLICATION_PACKAGE` the packages that will be present in the version information as application.
495
+
496
+ And you should add in your `gunicorn.conf.py`:
497
+
498
+ ```python
499
+ from prometheus_client import multiprocess
541
500
 
542
- The path `/metrics` provide some metrics for Prometheus.
543
- By default we have the `smap` `pss`, but we can easily add the `rss`, `size` or your custom settings:
544
501
 
545
- Example:
502
+ def on_starting(server):
503
+ from c2cwsgiutils import prometheus
546
504
 
505
+ del server
506
+
507
+ prometheus.start()
508
+
509
+
510
+ def post_fork(server, worker):
511
+ from c2cwsgiutils import prometheus
512
+
513
+ del server, worker
514
+
515
+ prometheus.cleanup()
516
+
517
+
518
+ def child_exit(server, worker):
519
+ del server
520
+
521
+ multiprocess.mark_process_dead(worker.pid)
547
522
  ```
548
- from import c2cwsgiutils.metrics import add_provider, Provider, MemoryMapProvider
549
523
 
550
- class CustomProvider(Provider):
551
- def __init__(self):
552
- super().__init__("my_metrics", "My Metric")
524
+ In your `Dockerfile` you should add:
553
525
 
554
- def get_data(self):
555
- return [({'metadata_key': 'matadata_value'}, metrics_value)]
526
+ ```dockerfile
527
+ RUN mkdir -p /prometheus-metrics \
528
+ && chmod a+rwx /prometheus-metrics
529
+ ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
530
+ ```
556
531
 
557
- add_provider(MemoryMapProvider('rss'))
558
- add_provider(CustomProvider())
532
+ ### Add custom metric collector
533
+
534
+ See [official documentation](https://github.com/prometheus/client_python#custom-collectors).
535
+
536
+ Related to the Unix process.
537
+
538
+ ```python
539
+ from c2cwsgiutils import broadcast, prometheus
540
+
541
+ prometheus.MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS.append("prometheus_collector_custom")
542
+ broadcast.subscribe("c2cwsgiutils_prometheus_collect_gc", _broadcast_collector_custom)
543
+ my_custom_collector_instance = MyCustomCollector()
544
+
545
+
546
+ def _broadcast_collector_custom() -> List[prometheus.SerializedGauge]:
547
+ """Get the collected GC gauges."""
548
+
549
+ return prometheus.serialize_collected_data(my_custom_collector_instance)
559
550
  ```
560
551
 
552
+ Related to the host, use that in the `gunicorn.conf.py`:
553
+
554
+ ```python
555
+ def on_starting(server):
556
+ from c2cwsgiutils import prometheus
557
+
558
+ del server
559
+
560
+ registry = CollectorRegistry()
561
+ registry.register(MyCollector())
562
+ prometheus.start(registry)
563
+ ```
564
+
565
+ ### Database metrics
566
+
567
+ Look at the `c2cwsgiutils-stats-db` utility if you want to generate statistics (gauges) about the
568
+ row counts.
569
+
570
+ ### Usage of metrics
571
+
572
+ With c2cwsgiutils each instance (Pod) has its own metrics, so we need to aggregate them to have the metrics for the service you probably need to use `sum by (<fields>) (<metric>)` to get the metric (especially for counters).
573
+
561
574
  ## Custom scripts
562
575
 
563
576
  To have the application initialized in a script you should use the
@@ -610,7 +623,7 @@ have dumps of a few things:
610
623
  - memory usage: `{C2C_BASE_PATH}/debug/memory?secret={C2C_SECRET}&limit=30&analyze_type=builtins.dict&python_internals_map=false`
611
624
  - object ref: `{C2C_BASE_PATH}/debug/show_refs.dot?secret={C2C_SECRET}&analyze_type=gunicorn.app.wsgiapp.WSGIApplication&analyze_id=12345&max_depth=3&too_many=10&filter=1024&no_extra_info&backrefs`
612
625
  `analyze_type` and `analyze_id` should not ve used toogether, you can use it like:
613
- ```
626
+ ```bash
614
627
  curl "<URL>" > /tmp/show_refs.dot
615
628
  dot -Lg -Tpng /tmp/show_refs.dot > /tmp/show_refs.png
616
629
  ```
@@ -671,7 +684,7 @@ client. In production mode, you can still get them by sending the secret defined
671
684
 
672
685
  If you want to use pyramid_debugtoolbar, you need to disable exception handling and configure it like that:
673
686
 
674
- ```
687
+ ```ini
675
688
  pyramid.includes =
676
689
  pyramid_debugtoolbar
677
690
  debugtoolbar.enabled = true
@@ -744,3 +757,42 @@ To make a release:
744
757
  - Tag the GIT commit.
745
758
  - Add the new branch name in the `.github/workflows/rebuild.yaml` and
746
759
  `.github/workflows/audit.yaml` files.
760
+
761
+ ## Testing
762
+
763
+ ### Screenshots
764
+
765
+ To test the screenshots, you need to install `node` with `npm`, to do that add the following lines in your `Dockerfile`:
766
+
767
+ ```dockerfile
768
+ RUN --mount=type=cache,target=/var/lib/apt/lists \
769
+ --mount=type=cache,target=/var/cache,sharing=locked \
770
+ . /etc/os-release \
771
+ && echo "deb https://deb.nodesource.com/node_18.x ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/nodesource.list \
772
+ && curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
773
+ && apt-get update \
774
+ && apt-get install --assume-yes --no-install-recommends 'nodejs=18.*'
775
+ ```
776
+
777
+ To do the image test call `check_screenshot` e.g.:
778
+
779
+ ```python
780
+ def test_screenshot(app_connection):
781
+ image.check_screenshot(
782
+ app_connection.base_url + "my-path",
783
+ width=800,
784
+ height=600,
785
+ result_folder="results",
786
+ expected_filename=os.path.join(os.path.dirname(__file__), "my-check.expected.png"),
787
+ )
788
+ ```
789
+
790
+ ## Contributing
791
+
792
+ Install the pre-commit hooks:
793
+
794
+ ```bash
795
+ pip install pre-commit
796
+ pre-commit install --allow-missing-config
797
+ ```
798
+
@@ -0,0 +1,67 @@
1
+ c2cwsgiutils/__init__.py,sha256=YHZ6OY8BvFrZAoBpbEWmAZ-60nqDer_CKHgY-LSHAKs,3986
2
+ c2cwsgiutils/acceptance/__init__.py,sha256=vjtpPfu0kbXUOYMx15Z8713IfPFZA9XnkUKkIFtVj_M,1500
3
+ c2cwsgiutils/acceptance/connection.py,sha256=WKD4yDDvOqpKTcwCn9cIK5W_AWxumGboe8Sel7ZeXHI,9146
4
+ c2cwsgiutils/acceptance/image.py,sha256=_LNKuUYnljZTyPoVqtOLvWb5HWFr_7mQXWQh2ow4o4Q,7200
5
+ c2cwsgiutils/acceptance/package-lock.json,sha256=Y50ESiYKsdaZsmTo448ScQa1Vrn8DnuDgpuEkSXSHIE,76668
6
+ c2cwsgiutils/acceptance/package.json,sha256=nQN96fhokGljwDhBoLn00rm9jSd7_U38qRs44oHY394,101
7
+ c2cwsgiutils/acceptance/print.py,sha256=j5K1c2Kn0eEnhgdbZNBVkdscK02pQhtPIh6lJzHMJcM,2323
8
+ c2cwsgiutils/acceptance/screenshot.js,sha256=pp8G5Bb04aDb1VX7PnLwBxVhaxN9p5PUU3iPWENVGoI,1899
9
+ c2cwsgiutils/acceptance/utils.py,sha256=-NgLlG_oQj3P_ZiK293RG7ZHPumg0WrDwo_APOI3WG4,1851
10
+ c2cwsgiutils/auth.py,sha256=73NkFu4mKeDwulECRabSFhpNUKoEfP-0M3JnTQ9lJ3s,9333
11
+ c2cwsgiutils/broadcast/__init__.py,sha256=Ae0qU6nP0G6Qa6Zi-vJK4Om8T9HKxmaz5-uOAJKZZdE,4360
12
+ c2cwsgiutils/broadcast/interface.py,sha256=jE8BSy9N7xnPmq5U0m852sFFhx46c7Uo9SyFJTCde9o,636
13
+ c2cwsgiutils/broadcast/local.py,sha256=24aIRdFOR2CXZfp_F2R_S1QW-yon3EyTM6TvljWVlP0,1083
14
+ c2cwsgiutils/broadcast/redis.py,sha256=sxBsYZtMNXMOck6a_Mcb3QZjjW4lPTxnImTfL1vv14g,5085
15
+ c2cwsgiutils/broadcast/utils.py,sha256=0fQZXPu3p_5LEJpGenJwiiMxECQjJhjZBjIkBk8h-ng,272
16
+ c2cwsgiutils/client_info.py,sha256=t35d8yI6MVDoB3i2yRKk0k-GmLP3v-WCis5JVzC6XFg,3044
17
+ c2cwsgiutils/config_utils.py,sha256=N_DPNRCBmeomhv2cJN5OQALrTAG29FCCk3VeugWrBwI,1523
18
+ c2cwsgiutils/coverage_setup.py,sha256=fES0sdhFy6oaeOCuP1qjjm7PQL9l_O8rUKZhRvRBRwQ,839
19
+ c2cwsgiutils/db.py,sha256=N_9zxxXYKiKwfNxs5KsunKYb_gUZQkTrlw6HZu5qsgQ,16174
20
+ c2cwsgiutils/db_maintenance_view.py,sha256=ejSNCv7vZIDjXctDTWTXEYTnUYFVVNIecFLeDlCkDBA,3076
21
+ c2cwsgiutils/debug/__init__.py,sha256=80zdAZnE9cwgQW1odE2aOauIxYsG5CQpWvHPcslRue8,1239
22
+ c2cwsgiutils/debug/_listeners.py,sha256=sXXHbPHQaSRMrzUC2ryiSDsBXuTdVbpQOxmMmhZ3n90,4378
23
+ c2cwsgiutils/debug/_views.py,sha256=zUeIshxBshBrlTz0p1I1LUi6HiJBOAAiDPrlOJsziQA,7522
24
+ c2cwsgiutils/debug/utils.py,sha256=TPlJC5qKeFnvbgq1xjlfrrRgDcV5kIR69IPJgNcIZQY,2311
25
+ c2cwsgiutils/errors.py,sha256=LUhZF3BW1i-v20x3EE-rZbT-TpdugpxiW7p5iCAu73Q,6723
26
+ c2cwsgiutils/health_check.py,sha256=O-GGS0aicTuMz4gqE-btk8ipzVi5allDe6sFuyOANOA,19977
27
+ c2cwsgiutils/index.py,sha256=jFThEi1sbsd2RnMNwhXpVAQUcpZSxW7ava9kxYwO5Zk,16703
28
+ c2cwsgiutils/loader.py,sha256=x_yHRTDzzlQ61fHonWnnG01xdqFuXpbGZMNN--tN25U,622
29
+ c2cwsgiutils/logging_view.py,sha256=W6-dFTz00hLt6BGJ8V3-4ip4AdxTyPG2W5vQjIuKiQs,3357
30
+ c2cwsgiutils/models_graph.py,sha256=laip8EdhI2hoGZVAotdrsgMwiNbwsJPjknKkRq1eEq0,2680
31
+ c2cwsgiutils/pretty_json.py,sha256=f1-oecFX9hub1nD32mmZRjOTIxhV8bVSt3Meqw68sNU,1698
32
+ c2cwsgiutils/profiler.py,sha256=3tIwoDSzOKQ06ug_U6j5VDR1BQ9auUOqdJRRLRhDoHw,739
33
+ c2cwsgiutils/prometheus.py,sha256=a09eTEmmhv_WENYx5I8jZUFdbUuVztE_lbMdOJNDm1U,6494
34
+ c2cwsgiutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
35
+ c2cwsgiutils/pyramid.py,sha256=H-b5abZvxt9MkMFjyWOjWzPa02YRhZR6LDiMCaYLq5s,1388
36
+ c2cwsgiutils/pyramid_logging.py,sha256=vVPrkAKZoIwb6SPf-omZDGyiKfElBf5yo_dPaIo-8ko,3730
37
+ c2cwsgiutils/redis_stats.py,sha256=triLtzryGXJKvCUw4TreYpF22BpUKPdrMp30ZGKsXwU,1545
38
+ c2cwsgiutils/redis_utils.py,sha256=dSe7qvnobCglKwDbYg4OwCxWxWg4migxmHbgClU2Qnw,4630
39
+ c2cwsgiutils/request_tracking/__init__.py,sha256=5SxMqBV3_w6SyykB1FFKug7XqzwdPhQdQOZ4VKUM4A8,4039
40
+ c2cwsgiutils/request_tracking/_sql.py,sha256=ME13tdXV2Vvhvscon9DbDUqehNWcn4uL75uqzcN5-Mg,577
41
+ c2cwsgiutils/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
+ c2cwsgiutils/scripts/genversion.py,sha256=GKmD08-H83hiKqXxy4odABuWqD5gADs_YEzM3TzHxEA,1985
43
+ c2cwsgiutils/scripts/stats_db.py,sha256=QNwtEKznCgOKg4ZsAFED9ISS-THgdwD2km_oVoII5XU,10317
44
+ c2cwsgiutils/scripts/test_print.py,sha256=UeOZa7jTazgEq5BRJD6lq-u9K6G4movf-sOVKTEs1cQ,2096
45
+ c2cwsgiutils/sentry.py,sha256=X-8Pd9wEi6EKMde50LxcdeM6LYBRyGnJpkPgEmFgqpg,5029
46
+ c2cwsgiutils/services.py,sha256=qz51oCZOC0Lj2_ig4UuHIm0ZZO3FfpFTxrXBWZ_oaNo,1567
47
+ c2cwsgiutils/setup_process.py,sha256=RKZGQdcKvHh_YU2mA-J5FQ01XNGj-SevcPNQB0439rs,3428
48
+ c2cwsgiutils/sql_profiler/__init__.py,sha256=lZYq83LYlm_P4uNMv0WU_B9Obl90YaNzkqWtteUHadg,876
49
+ c2cwsgiutils/sql_profiler/_impl.py,sha256=0UWSZDEm2E9Zeujd8UFRu0cIyyHBMxXxdmzt0cmuv0s,3702
50
+ c2cwsgiutils/sqlalchemylogger/README.md,sha256=WEyJSrBjedtX1FFrYiq4oMaWMt1fNxRkJYmJWnAoz3g,1552
51
+ c2cwsgiutils/sqlalchemylogger/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
52
+ c2cwsgiutils/sqlalchemylogger/_filters.py,sha256=OJQ9_WA-fd9fMZ7TUNFzHHTPI6msw2NVBl5RoeYFnGw,752
53
+ c2cwsgiutils/sqlalchemylogger/_models.py,sha256=A9SQ8AqUazCMemVjp5p_1x4bZG3LAYW9pOXT84FdNkE,1471
54
+ c2cwsgiutils/sqlalchemylogger/examples/example.py,sha256=n48dJdUi1FH1hfBMAbfHLGPSb1bOVD8pXMxXB57PnpQ,460
55
+ c2cwsgiutils/sqlalchemylogger/handlers.py,sha256=Qul-Bebmxqt4KvFU4TqjUGnbmpAVBviF5pSV3duUqG0,4801
56
+ c2cwsgiutils/static/favicon-16x16.png,sha256=LKk6RFvb3NlPIZdDfAodE8H9IN8KM6CMGnMx4vOHlUQ,887
57
+ c2cwsgiutils/static/favicon-32x32.png,sha256=i4ucx08zAZARd8e7JTMGK-gb5WcOmyuDN6IN4brsEOo,1216
58
+ c2cwsgiutils/stats_pyramid/__init__.py,sha256=7P10LjLv3c-ObEDGuYmRF_RFt7fRmO80ruqTGQAyC6w,747
59
+ c2cwsgiutils/stats_pyramid/_db_spy.py,sha256=ZGRdrI17Bdl3mzaLjfPyAaEW3KK8Pikrgi-0WmH7zCs,2917
60
+ c2cwsgiutils/stats_pyramid/_pyramid_spy.py,sha256=P212MGGl2VV_7UU4AXZA-rOuF7ouaONRklZwpas2wc8,3209
61
+ c2cwsgiutils/templates/index.html.mako,sha256=zrdMxncl3PFwdg1TVDKAjH1CaXw1m7yr9dj8Hi7rYGw,1362
62
+ c2cwsgiutils/version.py,sha256=z4of1DDr6J7PDw4AUOz31Gp63khgXf3JfiIaoWUM-9I,2870
63
+ c2cwsgiutils-5.2.1.dev197.dist-info/LICENSE,sha256=rM6IWxociA3daRkXnNLYOxGndT5fbs3BfVZCA2Xgt-g,1304
64
+ c2cwsgiutils-5.2.1.dev197.dist-info/METADATA,sha256=AXKlHCfO4_Uq--GCt6EpxznwArBUXtSO-IQrrlH-CH8,32515
65
+ c2cwsgiutils-5.2.1.dev197.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
66
+ c2cwsgiutils-5.2.1.dev197.dist-info/entry_points.txt,sha256=ujgqMTL1awN9qDg8WXmrF7m0fgR-hslUM6zKH86pvy0,703
67
+ c2cwsgiutils-5.2.1.dev197.dist-info/RECORD,,
@@ -1,5 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.37.1)
2
+ Generator: poetry-core 1.7.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
-
@@ -0,0 +1,21 @@
1
+ [console_scripts]
2
+ c2cwsgiutils-check-es=c2cwsgiutils.scripts.check_es:main
3
+ c2cwsgiutils-coverage-report=c2cwsgiutils.scripts.coverage_report:main
4
+ c2cwsgiutils-genversion=c2cwsgiutils.scripts.genversion:main
5
+ c2cwsgiutils-stats-db=c2cwsgiutils.scripts.stats_db:main
6
+ c2cwsgiutils-test-print=c2cwsgiutils.scripts.test_print:main
7
+
8
+ [paste.filter_factory]
9
+ client_info=c2cwsgiutils.client_info:filter_factory
10
+ sentry=c2cwsgiutils.sentry:filter_factory
11
+
12
+ [plaster.loader_factory]
13
+ c2c=c2cwsgiutils.loader:Loader
14
+ c2c+egg=c2cwsgiutils.loader:Loader
15
+ c2c+ini=c2cwsgiutils.loader:Loader
16
+
17
+ [plaster.wsgi_loader_factory]
18
+ c2c=c2cwsgiutils.loader:Loader
19
+ c2c+egg=c2cwsgiutils.loader:Loader
20
+ c2c+ini=c2cwsgiutils.loader:Loader
21
+
@@ -1,129 +0,0 @@
1
- import logging
2
- import os
3
- import subprocess # nosec
4
- import sys
5
- import time
6
- import warnings
7
- from typing import Any, Callable, Dict, List, Mapping, Optional, cast
8
-
9
- import netifaces
10
- from pyramid.request import Request
11
-
12
- from c2cwsgiutils.acceptance import utils
13
-
14
- LOG = logging.getLogger(__name__)
15
- logging.basicConfig(
16
- level=logging.DEBUG,
17
- format="TEST: %(asctime)-15s %(levelname)5s %(name)s %(message)s",
18
- stream=sys.stdout,
19
- )
20
- logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARN)
21
-
22
-
23
- def _try(what: Callable[[], Any], fail: bool = True, times: int = 5, delay: int = 10) -> Optional[Any]:
24
- for i in range(times):
25
- try:
26
- return what()
27
- except: # noqa: bare-except
28
- LOG.warning("Exception:", exc_info=True)
29
- if i + 1 == times and fail:
30
- raise
31
- time.sleep(delay)
32
- return None
33
-
34
-
35
- class Composition:
36
- """The Docker composition."""
37
-
38
- def __init__(self, request: Request, composition: str, coverage_paths: Optional[str] = None) -> None:
39
- warnings.warn("The c2cwsgiutils.acceptance.composition should be used only if it's relay needed.")
40
- self.cwd = os.path.dirname(composition)
41
- filename = os.path.basename(composition)
42
- self.docker_compose = ["docker-compose"]
43
- if filename != "docker-compose.yaml":
44
- self.docker_compose.append("--file=" + filename)
45
- self.coverage_paths = coverage_paths
46
- env = Composition._get_env()
47
- if os.environ.get("docker_start", "1") == "1":
48
- self.dc_try(["stop"], fail=False)
49
- self.dc_try(["rm", "-f"], fail=False)
50
- self.dc_try(["build"], fail=False)
51
- self.dc_try(["up", "-d"], fail=False)
52
-
53
- # Setup something that redirects the docker container logs to the test output
54
- log_watcher = subprocess.Popen( # nosec, pylint: disable=consider-using-with
55
- self.docker_compose + ["logs", "--follow", "--no-color"],
56
- cwd=self.cwd,
57
- env=env,
58
- stderr=subprocess.STDOUT,
59
- )
60
- request.addfinalizer(log_watcher.kill)
61
- if os.environ.get("docker_stop", "1") == "1":
62
- request.addfinalizer(self.stop_all)
63
-
64
- def dc(self, args: List[str], **kwargs: Any) -> str:
65
- return cast(
66
- str,
67
- subprocess.check_output( # nosec
68
- self.docker_compose + args,
69
- cwd=self.cwd,
70
- env=Composition._get_env(),
71
- stderr=subprocess.STDOUT,
72
- **kwargs,
73
- ).decode(),
74
- )
75
-
76
- def dc_try(self, args: List[str], **kwargs: Any) -> None:
77
- _try(
78
- lambda: self.dc(args),
79
- **kwargs,
80
- )
81
-
82
- def stop_all(self) -> None:
83
- self.dc_try(["stop"])
84
- if self.coverage_paths:
85
- target_dir = "/reports/"
86
- os.makedirs(target_dir, exist_ok=True)
87
- for path in self.coverage_paths:
88
- try:
89
- subprocess.check_call( # nosec
90
- ["docker", "cp", path, target_dir], stderr=subprocess.STDOUT
91
- )
92
- except Exception:
93
- self.dc(["ps"])
94
- raise
95
-
96
- def stop(self, container: str) -> None:
97
- self.dc_try(["stop", container])
98
-
99
- def restart(self, container: str) -> None:
100
- self.dc_try(["restart", container])
101
-
102
- def run(self, container: str, *command: str, **kwargs: Dict[str, Any]) -> str:
103
- return self.dc(
104
- ["run", "--rm", container] + list(command),
105
- **kwargs,
106
- )
107
-
108
- def exec(self, container: str, *command: str, **kwargs: Dict[str, Any]) -> str:
109
- return self.dc(["exec", "-T", container] + list(command), **kwargs)
110
-
111
- @staticmethod
112
- def _get_env() -> Mapping[str, str]:
113
- """
114
- Make sure the DOCKER_TAG environment variable.
115
-
116
- Used in the docker-compose.yaml file is correctly set when we call docker-compose.
117
- """
118
- env = dict(os.environ)
119
- if "DOCKER_TAG" not in env:
120
- env["DOCKER_TAG"] = "latest"
121
- if utils.in_docker():
122
- env["DOCKER_IP"] = netifaces.gateways()[netifaces.AF_INET][0][0]
123
- env["DOCKER_CB_HOST"] = env["DOCKER_IP"]
124
- else:
125
- env["DOCKER_IP"] = netifaces.ifaddresses("docker0")[netifaces.AF_INET][0]["addr"]
126
- env["DOCKER_CB_HOST"] = "localhost"
127
- default_iface = netifaces.gateways()[netifaces.AF_INET][0][1]
128
- env["TEST_IP"] = netifaces.ifaddresses(default_iface)[netifaces.AF_INET][0]["addr"]
129
- return env