c2cwsgiutils 5.1.7.dev20230901073305__py3-none-any.whl → 5.2.1.dev197__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cwsgiutils/__init__.py +13 -13
- c2cwsgiutils/acceptance/connection.py +5 -2
- c2cwsgiutils/acceptance/image.py +98 -4
- c2cwsgiutils/acceptance/package-lock.json +1933 -0
- c2cwsgiutils/acceptance/package.json +7 -0
- c2cwsgiutils/acceptance/print.py +4 -4
- c2cwsgiutils/acceptance/screenshot.js +62 -0
- c2cwsgiutils/acceptance/utils.py +14 -22
- c2cwsgiutils/auth.py +4 -4
- c2cwsgiutils/broadcast/__init__.py +15 -7
- c2cwsgiutils/broadcast/interface.py +3 -2
- c2cwsgiutils/broadcast/local.py +3 -2
- c2cwsgiutils/broadcast/redis.py +8 -7
- c2cwsgiutils/client_info.py +5 -5
- c2cwsgiutils/config_utils.py +2 -1
- c2cwsgiutils/coverage_setup.py +2 -2
- c2cwsgiutils/db.py +58 -37
- c2cwsgiutils/db_maintenance_view.py +2 -1
- c2cwsgiutils/debug/_listeners.py +10 -9
- c2cwsgiutils/debug/_views.py +12 -11
- c2cwsgiutils/debug/utils.py +5 -5
- c2cwsgiutils/errors.py +7 -6
- c2cwsgiutils/health_check.py +96 -85
- c2cwsgiutils/index.py +90 -105
- c2cwsgiutils/loader.py +3 -3
- c2cwsgiutils/logging_view.py +3 -2
- c2cwsgiutils/models_graph.py +8 -6
- c2cwsgiutils/prometheus.py +175 -57
- c2cwsgiutils/pyramid.py +4 -2
- c2cwsgiutils/pyramid_logging.py +2 -1
- c2cwsgiutils/redis_stats.py +13 -11
- c2cwsgiutils/redis_utils.py +15 -14
- c2cwsgiutils/request_tracking/__init__.py +36 -30
- c2cwsgiutils/request_tracking/_sql.py +3 -1
- c2cwsgiutils/scripts/genversion.py +4 -4
- c2cwsgiutils/scripts/stats_db.py +130 -68
- c2cwsgiutils/scripts/test_print.py +1 -1
- c2cwsgiutils/sentry.py +2 -1
- c2cwsgiutils/setup_process.py +13 -17
- c2cwsgiutils/sql_profiler/_impl.py +12 -5
- c2cwsgiutils/sqlalchemylogger/README.md +48 -0
- c2cwsgiutils/sqlalchemylogger/_models.py +7 -4
- c2cwsgiutils/sqlalchemylogger/examples/example.py +15 -0
- c2cwsgiutils/sqlalchemylogger/handlers.py +11 -8
- c2cwsgiutils/static/favicon-16x16.png +0 -0
- c2cwsgiutils/static/favicon-32x32.png +0 -0
- c2cwsgiutils/stats_pyramid/__init__.py +7 -11
- c2cwsgiutils/stats_pyramid/_db_spy.py +14 -11
- c2cwsgiutils/stats_pyramid/_pyramid_spy.py +29 -20
- c2cwsgiutils/templates/index.html.mako +50 -0
- c2cwsgiutils/version.py +49 -16
- c2cwsgiutils-5.2.1.dev197.dist-info/LICENSE +22 -0
- {c2cwsgiutils-5.1.7.dev20230901073305.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/METADATA +187 -135
- c2cwsgiutils-5.2.1.dev197.dist-info/RECORD +67 -0
- {c2cwsgiutils-5.1.7.dev20230901073305.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/WHEEL +1 -2
- c2cwsgiutils-5.2.1.dev197.dist-info/entry_points.txt +21 -0
- c2cwsgiutils/acceptance/composition.py +0 -129
- c2cwsgiutils/metrics.py +0 -110
- c2cwsgiutils/scripts/check_es.py +0 -130
- c2cwsgiutils/scripts/coverage_report.py +0 -36
- c2cwsgiutils/stats.py +0 -355
- c2cwsgiutils/stats_pyramid/_views.py +0 -16
- c2cwsgiutils-5.1.7.dev20230901073305.data/scripts/c2cwsgiutils-run +0 -32
- c2cwsgiutils-5.1.7.dev20230901073305.dist-info/LICENSE.txt +0 -28
- c2cwsgiutils-5.1.7.dev20230901073305.dist-info/RECORD +0 -69
- c2cwsgiutils-5.1.7.dev20230901073305.dist-info/entry_points.txt +0 -25
- c2cwsgiutils-5.1.7.dev20230901073305.dist-info/top_level.txt +0 -2
- tests/acceptance/__init__.py +0 -0
- tests/acceptance/test_utils.py +0 -13
{c2cwsgiutils-5.1.7.dev20230901073305.dist-info → c2cwsgiutils-5.2.1.dev197.dist-info}/METADATA
RENAMED
@@ -1,89 +1,73 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: c2cwsgiutils
|
3
|
-
Version: 5.1.
|
3
|
+
Version: 5.2.1.dev197
|
4
4
|
Summary: Common utilities for Camptocamp WSGI applications
|
5
5
|
Home-page: https://github.com/camptocamp/c2cwsgiutils
|
6
|
+
License: BSD-2-Clause
|
7
|
+
Keywords: geo,gis,sqlalchemy,orm,wsgi
|
6
8
|
Author: Camptocamp
|
7
9
|
Author-email: info@camptocamp.com
|
8
|
-
|
9
|
-
Keywords: geo gis sqlalchemy orm wsgi
|
10
|
+
Requires-Python: >=3.9,<4.0
|
10
11
|
Classifier: Development Status :: 5 - Production/Stable
|
11
12
|
Classifier: Environment :: Plugins
|
12
13
|
Classifier: Framework :: Pyramid
|
13
14
|
Classifier: Intended Audience :: Developers
|
14
|
-
Classifier:
|
15
|
+
Classifier: Intended Audience :: Information Technology
|
16
|
+
Classifier: License :: OSI Approved :: BSD License
|
15
17
|
Classifier: Operating System :: OS Independent
|
16
18
|
Classifier: Programming Language :: Python
|
17
19
|
Classifier: Programming Language :: Python :: 3
|
18
|
-
Classifier: Programming Language :: Python :: 3.
|
19
|
-
Classifier:
|
20
|
+
Classifier: Programming Language :: Python :: 3.9
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
20
23
|
Classifier: Topic :: Internet :: WWW/HTTP :: WSGI :: Application
|
21
24
|
Classifier: Typing :: Typed
|
22
|
-
|
23
|
-
|
25
|
+
Provides-Extra: alembic
|
26
|
+
Provides-Extra: all
|
24
27
|
Provides-Extra: broadcast
|
25
|
-
|
28
|
+
Provides-Extra: debug
|
29
|
+
Provides-Extra: dev
|
30
|
+
Provides-Extra: oauth2
|
31
|
+
Provides-Extra: sentry
|
26
32
|
Provides-Extra: standard
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
Requires-Dist:
|
31
|
-
Requires-Dist:
|
32
|
-
Requires-Dist:
|
33
|
-
Requires-Dist:
|
34
|
-
Requires-Dist:
|
35
|
-
Requires-Dist:
|
36
|
-
Requires-Dist:
|
37
|
-
Requires-Dist:
|
38
|
-
Requires-Dist:
|
39
|
-
Requires-Dist:
|
40
|
-
Requires-Dist:
|
41
|
-
Requires-Dist:
|
42
|
-
Requires-Dist:
|
43
|
-
Requires-Dist:
|
44
|
-
Requires-Dist:
|
45
|
-
Requires-Dist:
|
46
|
-
Requires-Dist: pyyaml
|
47
|
-
Requires-Dist:
|
48
|
-
Requires-Dist: requests
|
49
|
-
Requires-Dist:
|
50
|
-
Requires-Dist:
|
51
|
-
Requires-Dist:
|
52
|
-
Requires-Dist:
|
53
|
-
Requires-Dist:
|
54
|
-
Requires-Dist:
|
55
|
-
Requires-Dist:
|
56
|
-
|
57
|
-
|
58
|
-
Requires-Dist: importlib-metadata ; extra == 'standard'
|
59
|
-
Requires-Dist: importlib-resources ; extra == 'standard'
|
60
|
-
Requires-Dist: mako ; extra == 'standard'
|
61
|
-
Requires-Dist: markupsafe ; extra == 'standard'
|
62
|
-
Requires-Dist: oauthlib ; extra == 'standard'
|
63
|
-
Requires-Dist: packaging ; extra == 'standard'
|
64
|
-
Requires-Dist: pastedeploy ; extra == 'standard'
|
65
|
-
Requires-Dist: plaster ; extra == 'standard'
|
66
|
-
Requires-Dist: plaster-pastedeploy ; extra == 'standard'
|
67
|
-
Requires-Dist: pyparsing ; extra == 'standard'
|
68
|
-
Requires-Dist: toml ; extra == 'standard'
|
69
|
-
Requires-Dist: transaction ; extra == 'standard'
|
70
|
-
Requires-Dist: translationstring ; extra == 'standard'
|
71
|
-
Requires-Dist: urllib3 ; extra == 'standard'
|
72
|
-
Requires-Dist: venusian ; extra == 'standard'
|
73
|
-
Requires-Dist: webob ; extra == 'standard'
|
74
|
-
Requires-Dist: wrapt ; extra == 'standard'
|
75
|
-
Requires-Dist: zipp ; extra == 'standard'
|
76
|
-
Requires-Dist: zope.deprecation ; extra == 'standard'
|
77
|
-
Provides-Extra: test_images
|
78
|
-
Requires-Dist: scikit-image ; extra == 'test_images'
|
33
|
+
Provides-Extra: test-images
|
34
|
+
Provides-Extra: tests
|
35
|
+
Provides-Extra: webserver
|
36
|
+
Requires-Dist: SQLAlchemy (>=1.4.0,<3.0.0)
|
37
|
+
Requires-Dist: SQLAlchemy-Utils
|
38
|
+
Requires-Dist: alembic ; extra == "standard" or extra == "alembic" or extra == "all"
|
39
|
+
Requires-Dist: boltons ; extra == "tests" or extra == "all"
|
40
|
+
Requires-Dist: cee_syslog_handler
|
41
|
+
Requires-Dist: certifi
|
42
|
+
Requires-Dist: cornice
|
43
|
+
Requires-Dist: gunicorn ; extra == "standard" or extra == "webserver" or extra == "all"
|
44
|
+
Requires-Dist: lxml ; extra == "tests" or extra == "all"
|
45
|
+
Requires-Dist: objgraph ; extra == "debug" or extra == "all"
|
46
|
+
Requires-Dist: prometheus-client
|
47
|
+
Requires-Dist: psycopg2
|
48
|
+
Requires-Dist: pyjwt ; extra == "standard" or extra == "oauth2" or extra == "all"
|
49
|
+
Requires-Dist: pyramid
|
50
|
+
Requires-Dist: pyramid-tm
|
51
|
+
Requires-Dist: pyramid_mako
|
52
|
+
Requires-Dist: pyyaml
|
53
|
+
Requires-Dist: redis ; extra == "standard" or extra == "broadcast" or extra == "all"
|
54
|
+
Requires-Dist: requests
|
55
|
+
Requires-Dist: requests-oauthlib ; extra == "standard" or extra == "oauth2" or extra == "all"
|
56
|
+
Requires-Dist: scikit-image ; extra == "test-images"
|
57
|
+
Requires-Dist: sentry-sdk ; extra == "standard" or extra == "sentry" or extra == "all"
|
58
|
+
Requires-Dist: ujson
|
59
|
+
Requires-Dist: waitress ; extra == "dev" or extra == "all"
|
60
|
+
Requires-Dist: zope.interface
|
61
|
+
Requires-Dist: zope.sqlalchemy
|
62
|
+
Project-URL: Repository, https://github.com/camptocamp/c2cwsgiutils
|
63
|
+
Description-Content-Type: text/markdown
|
79
64
|
|
80
65
|
# Camptocamp WSGI utilities
|
81
66
|
|
82
67
|
This is a Python 3 library (>=3.5) providing common tools for Camptocamp WSGI
|
83
68
|
applications:
|
84
69
|
|
85
|
-
- Provide
|
86
|
-
a web application (statsd protocol)
|
70
|
+
- Provide prometheus metrics
|
87
71
|
- Allow to use a master/slave PostgresQL configuration
|
88
72
|
- Logging handler for CEE/UDP logs
|
89
73
|
- An optional view to change runtime the log levels
|
@@ -122,7 +106,7 @@ You should install `c2cwsgiutils` with the tool you use to manage your pip depen
|
|
122
106
|
|
123
107
|
In the `Dockerfile` you should add the following lines:
|
124
108
|
|
125
|
-
```
|
109
|
+
```dockerfile
|
126
110
|
# Generate the version file.
|
127
111
|
RUN c2cwsgiutils-genversion $(git rev-parse HEAD)
|
128
112
|
|
@@ -130,20 +114,19 @@ CMD ["gunicorn", "--paste=/app/production.ini"]
|
|
130
114
|
|
131
115
|
# Default values for the environment variables
|
132
116
|
ENV \
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
LOG_LEVEL=INFO
|
117
|
+
DEVELOPMENT=0 \
|
118
|
+
SQLALCHEMY_POOL_RECYCLE=30 \
|
119
|
+
SQLALCHEMY_POOL_SIZE=5 \
|
120
|
+
SQLALCHEMY_MAX_OVERFLOW=25 \
|
121
|
+
SQLALCHEMY_SLAVE_POOL_RECYCLE=30 \
|
122
|
+
SQLALCHEMY_SLAVE_POOL_SIZE=5 \
|
123
|
+
SQLALCHEMY_SLAVE_MAX_OVERFLOW=25\
|
124
|
+
LOG_TYPE=console \
|
125
|
+
OTHER_LOG_LEVEL=WARNING \
|
126
|
+
GUNICORN_LOG_LEVEL=WARNING \
|
127
|
+
SQL_LOG_LEVEL=WARNING \
|
128
|
+
C2CWSGIUTILS_LOG_LEVEL=WARNING \
|
129
|
+
LOG_LEVEL=INFO
|
147
130
|
```
|
148
131
|
|
149
132
|
Add in your `main` function.
|
@@ -179,8 +162,6 @@ The related environment variables:
|
|
179
162
|
- `SQL_LOG_LEVEL`: The SQL query log level, `WARNING`: no logs, `INFO`: logs the queries,
|
180
163
|
`DEBUG` also logs the results, default is `WARNING`.
|
181
164
|
- `GUNICORN_ERROR_LOG_LEVEL`: The Gunicorn error log level, default is `WARNING`.
|
182
|
-
- `GUNICORN_ACCESS_LOG_LEVEL`: The Gunicorn access log level, the logs have the level `INFO`,
|
183
|
-
default is `WARNING`.
|
184
165
|
- `C2CWSGIUTILS_CONFIG`: The fallback ini file to use by gunicorn, default is `production.ini`.
|
185
166
|
- `C2CWSGIUTILS_LOG_LEVEL`: The c2c WSGI utils log level, default is `WARNING`.
|
186
167
|
- `OTHER_LOG_LEVEL`: The log level for all the other logger, default is `WARNING`.
|
@@ -349,47 +330,6 @@ The requests module is also patched to monitor requests done without timeout. In
|
|
349
330
|
configure a default timeout with the `C2C_REQUESTS_DEFAULT_TIMEOUT` environment variable
|
350
331
|
(`c2c.requests_default_timeout`). If no timeout and no default is specified, a warning is issued.
|
351
332
|
|
352
|
-
## Metrics
|
353
|
-
|
354
|
-
To enable and configure the metrics framework, you can use:
|
355
|
-
|
356
|
-
- STATS_VIEW (c2c.stats_view): if defined, will enable the stats view `{C2C_BASE_PATH}/stats.json`
|
357
|
-
- STATSD_ADDRESS (c2c.statsd_address): if defined, send stats to the given statsd server
|
358
|
-
- STATSD_PREFIX (c2c.statsd_prefix): prefix to add to every metric names
|
359
|
-
- STATSD_USE_TAGS: If true, automatic metrics will use tags
|
360
|
-
- STATSD*TAG*{tag_name}: To set a global tag for the service
|
361
|
-
|
362
|
-
If enabled, some metrics are automatically generated:
|
363
|
-
|
364
|
-
- {STATSD_PREFIX}.route.{verb}.{route_name}.{status}: The time to process a query (includes rendering)
|
365
|
-
- {STATSD_PREFIX}.render.{verb}.{route_name}.{status}: The time to render a query
|
366
|
-
- {STATSD_PREFIX}.sql.{query}: The time to execute the given SQL query (simplified and normalized)
|
367
|
-
- {STATSD_PREFIX}.requests.{scheme}.{hostname}.{port}.{verb}.{status}: The time to execute HTTP requests to
|
368
|
-
outside services (only the time between the start of sending of the request and when the header is
|
369
|
-
back with a chunk of the body)
|
370
|
-
- {STATSD_PREFIX}.redis.{command}: The time to execute the given Redis command
|
371
|
-
|
372
|
-
You can manually measure the time spent on something like that:
|
373
|
-
|
374
|
-
```python
|
375
|
-
from c2cwsgiutils import stats
|
376
|
-
with stats.timer_context(['toto', 'tutu']):
|
377
|
-
do_something()
|
378
|
-
```
|
379
|
-
|
380
|
-
It will only add a timer event in case of success. If you want to measure both success and failures, do that:
|
381
|
-
|
382
|
-
```python
|
383
|
-
from c2cwsgiutils import stats
|
384
|
-
with stats.outcome_timer_context(['toto', 'tutu']):
|
385
|
-
do_something()
|
386
|
-
```
|
387
|
-
|
388
|
-
Other functions exists to generate metrics. Look at the `c2cwsgiutils.stats` module.
|
389
|
-
|
390
|
-
Look at the `c2cwsgiutils-stats-db` utility if you want to generate statistics (gauges) about the
|
391
|
-
row counts.
|
392
|
-
|
393
333
|
## SQL profiler
|
394
334
|
|
395
335
|
The SQL profiler must be configured with the `C2C_SQL_PROFILER_ENABLED` environment variable. That enables a view
|
@@ -537,27 +477,100 @@ If the `/app/versions.json` exists, a view is added (`{C2C_BASE_PATH}/versions.j
|
|
537
477
|
version of a app. This file is generated by calling the `c2cwsgiutils-genversion [$GIT_TAG] $GIT_HASH`
|
538
478
|
command line. Usually done in the [Dockerfile](acceptance_tests/app/Dockerfile) of the WSGI application.
|
539
479
|
|
540
|
-
##
|
480
|
+
## Prometheus
|
481
|
+
|
482
|
+
[Prometheus client](https://github.com/prometheus/client_python) is integrated in c2cwsgiutils.
|
483
|
+
|
484
|
+
It will work in multi process mode with the limitation listed in the
|
485
|
+
[`prometheus_client` documentation](https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn).
|
486
|
+
|
487
|
+
To enable it you should provide the `C2CWSGIUTILS_PROMETHEUS_PORT` environment variable.
|
488
|
+
For security reason, this port should not be exposed.
|
489
|
+
|
490
|
+
We can customize it with the following environment variables:
|
491
|
+
|
492
|
+
- `C2C_PROMETHEUS_PREFIX`: to customize the prefix, default is `c2cwsggiutils-`.
|
493
|
+
- `C2C_PROMETHEUS_PACKAGES` the packages that will be present in the version information, default is `c2cwsgiutils,pyramid,gunicorn,sqlalchemy`.
|
494
|
+
- `C2C_PROMETHEUS_APPLICATION_PACKAGE` the packages that will be present in the version information as application.
|
495
|
+
|
496
|
+
And you should add in your `gunicorn.conf.py`:
|
497
|
+
|
498
|
+
```python
|
499
|
+
from prometheus_client import multiprocess
|
541
500
|
|
542
|
-
The path `/metrics` provide some metrics for Prometheus.
|
543
|
-
By default we have the `smap` `pss`, but we can easily add the `rss`, `size` or your custom settings:
|
544
501
|
|
545
|
-
|
502
|
+
def on_starting(server):
|
503
|
+
from c2cwsgiutils import prometheus
|
546
504
|
|
505
|
+
del server
|
506
|
+
|
507
|
+
prometheus.start()
|
508
|
+
|
509
|
+
|
510
|
+
def post_fork(server, worker):
|
511
|
+
from c2cwsgiutils import prometheus
|
512
|
+
|
513
|
+
del server, worker
|
514
|
+
|
515
|
+
prometheus.cleanup()
|
516
|
+
|
517
|
+
|
518
|
+
def child_exit(server, worker):
|
519
|
+
del server
|
520
|
+
|
521
|
+
multiprocess.mark_process_dead(worker.pid)
|
547
522
|
```
|
548
|
-
from import c2cwsgiutils.metrics import add_provider, Provider, MemoryMapProvider
|
549
523
|
|
550
|
-
|
551
|
-
def __init__(self):
|
552
|
-
super().__init__("my_metrics", "My Metric")
|
524
|
+
In your `Dockerfile` you should add:
|
553
525
|
|
554
|
-
|
555
|
-
|
526
|
+
```dockerfile
|
527
|
+
RUN mkdir -p /prometheus-metrics \
|
528
|
+
&& chmod a+rwx /prometheus-metrics
|
529
|
+
ENV PROMETHEUS_MULTIPROC_DIR=/prometheus-metrics
|
530
|
+
```
|
556
531
|
|
557
|
-
|
558
|
-
|
532
|
+
### Add custom metric collector
|
533
|
+
|
534
|
+
See [official documentation](https://github.com/prometheus/client_python#custom-collectors).
|
535
|
+
|
536
|
+
Related to the Unix process.
|
537
|
+
|
538
|
+
```python
|
539
|
+
from c2cwsgiutils import broadcast, prometheus
|
540
|
+
|
541
|
+
prometheus.MULTI_PROCESS_COLLECTOR_BROADCAST_CHANNELS.append("prometheus_collector_custom")
|
542
|
+
broadcast.subscribe("c2cwsgiutils_prometheus_collect_gc", _broadcast_collector_custom)
|
543
|
+
my_custom_collector_instance = MyCustomCollector()
|
544
|
+
|
545
|
+
|
546
|
+
def _broadcast_collector_custom() -> List[prometheus.SerializedGauge]:
|
547
|
+
"""Get the collected GC gauges."""
|
548
|
+
|
549
|
+
return prometheus.serialize_collected_data(my_custom_collector_instance)
|
559
550
|
```
|
560
551
|
|
552
|
+
Related to the host, use that in the `gunicorn.conf.py`:
|
553
|
+
|
554
|
+
```python
|
555
|
+
def on_starting(server):
|
556
|
+
from c2cwsgiutils import prometheus
|
557
|
+
|
558
|
+
del server
|
559
|
+
|
560
|
+
registry = CollectorRegistry()
|
561
|
+
registry.register(MyCollector())
|
562
|
+
prometheus.start(registry)
|
563
|
+
```
|
564
|
+
|
565
|
+
### Database metrics
|
566
|
+
|
567
|
+
Look at the `c2cwsgiutils-stats-db` utility if you want to generate statistics (gauges) about the
|
568
|
+
row counts.
|
569
|
+
|
570
|
+
### Usage of metrics
|
571
|
+
|
572
|
+
With c2cwsgiutils each instance (Pod) has its own metrics, so we need to aggregate them to have the metrics for the service you probably need to use `sum by (<fields>) (<metric>)` to get the metric (especially for counters).
|
573
|
+
|
561
574
|
## Custom scripts
|
562
575
|
|
563
576
|
To have the application initialized in a script you should use the
|
@@ -610,7 +623,7 @@ have dumps of a few things:
|
|
610
623
|
- memory usage: `{C2C_BASE_PATH}/debug/memory?secret={C2C_SECRET}&limit=30&analyze_type=builtins.dict&python_internals_map=false`
|
611
624
|
- object ref: `{C2C_BASE_PATH}/debug/show_refs.dot?secret={C2C_SECRET}&analyze_type=gunicorn.app.wsgiapp.WSGIApplication&analyze_id=12345&max_depth=3&too_many=10&filter=1024&no_extra_info&backrefs`
|
612
625
|
`analyze_type` and `analyze_id` should not ve used toogether, you can use it like:
|
613
|
-
```
|
626
|
+
```bash
|
614
627
|
curl "<URL>" > /tmp/show_refs.dot
|
615
628
|
dot -Lg -Tpng /tmp/show_refs.dot > /tmp/show_refs.png
|
616
629
|
```
|
@@ -671,7 +684,7 @@ client. In production mode, you can still get them by sending the secret defined
|
|
671
684
|
|
672
685
|
If you want to use pyramid_debugtoolbar, you need to disable exception handling and configure it like that:
|
673
686
|
|
674
|
-
```
|
687
|
+
```ini
|
675
688
|
pyramid.includes =
|
676
689
|
pyramid_debugtoolbar
|
677
690
|
debugtoolbar.enabled = true
|
@@ -744,3 +757,42 @@ To make a release:
|
|
744
757
|
- Tag the GIT commit.
|
745
758
|
- Add the new branch name in the `.github/workflows/rebuild.yaml` and
|
746
759
|
`.github/workflows/audit.yaml` files.
|
760
|
+
|
761
|
+
## Testing
|
762
|
+
|
763
|
+
### Screenshots
|
764
|
+
|
765
|
+
To test the screenshots, you need to install `node` with `npm`, to do that add the following lines in your `Dockerfile`:
|
766
|
+
|
767
|
+
```dockerfile
|
768
|
+
RUN --mount=type=cache,target=/var/lib/apt/lists \
|
769
|
+
--mount=type=cache,target=/var/cache,sharing=locked \
|
770
|
+
. /etc/os-release \
|
771
|
+
&& echo "deb https://deb.nodesource.com/node_18.x ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/nodesource.list \
|
772
|
+
&& curl --silent https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
|
773
|
+
&& apt-get update \
|
774
|
+
&& apt-get install --assume-yes --no-install-recommends 'nodejs=18.*'
|
775
|
+
```
|
776
|
+
|
777
|
+
To do the image test call `check_screenshot` e.g.:
|
778
|
+
|
779
|
+
```python
|
780
|
+
def test_screenshot(app_connection):
|
781
|
+
image.check_screenshot(
|
782
|
+
app_connection.base_url + "my-path",
|
783
|
+
width=800,
|
784
|
+
height=600,
|
785
|
+
result_folder="results",
|
786
|
+
expected_filename=os.path.join(os.path.dirname(__file__), "my-check.expected.png"),
|
787
|
+
)
|
788
|
+
```
|
789
|
+
|
790
|
+
## Contributing
|
791
|
+
|
792
|
+
Install the pre-commit hooks:
|
793
|
+
|
794
|
+
```bash
|
795
|
+
pip install pre-commit
|
796
|
+
pre-commit install --allow-missing-config
|
797
|
+
```
|
798
|
+
|
@@ -0,0 +1,67 @@
|
|
1
|
+
c2cwsgiutils/__init__.py,sha256=YHZ6OY8BvFrZAoBpbEWmAZ-60nqDer_CKHgY-LSHAKs,3986
|
2
|
+
c2cwsgiutils/acceptance/__init__.py,sha256=vjtpPfu0kbXUOYMx15Z8713IfPFZA9XnkUKkIFtVj_M,1500
|
3
|
+
c2cwsgiutils/acceptance/connection.py,sha256=WKD4yDDvOqpKTcwCn9cIK5W_AWxumGboe8Sel7ZeXHI,9146
|
4
|
+
c2cwsgiutils/acceptance/image.py,sha256=_LNKuUYnljZTyPoVqtOLvWb5HWFr_7mQXWQh2ow4o4Q,7200
|
5
|
+
c2cwsgiutils/acceptance/package-lock.json,sha256=Y50ESiYKsdaZsmTo448ScQa1Vrn8DnuDgpuEkSXSHIE,76668
|
6
|
+
c2cwsgiutils/acceptance/package.json,sha256=nQN96fhokGljwDhBoLn00rm9jSd7_U38qRs44oHY394,101
|
7
|
+
c2cwsgiutils/acceptance/print.py,sha256=j5K1c2Kn0eEnhgdbZNBVkdscK02pQhtPIh6lJzHMJcM,2323
|
8
|
+
c2cwsgiutils/acceptance/screenshot.js,sha256=pp8G5Bb04aDb1VX7PnLwBxVhaxN9p5PUU3iPWENVGoI,1899
|
9
|
+
c2cwsgiutils/acceptance/utils.py,sha256=-NgLlG_oQj3P_ZiK293RG7ZHPumg0WrDwo_APOI3WG4,1851
|
10
|
+
c2cwsgiutils/auth.py,sha256=73NkFu4mKeDwulECRabSFhpNUKoEfP-0M3JnTQ9lJ3s,9333
|
11
|
+
c2cwsgiutils/broadcast/__init__.py,sha256=Ae0qU6nP0G6Qa6Zi-vJK4Om8T9HKxmaz5-uOAJKZZdE,4360
|
12
|
+
c2cwsgiutils/broadcast/interface.py,sha256=jE8BSy9N7xnPmq5U0m852sFFhx46c7Uo9SyFJTCde9o,636
|
13
|
+
c2cwsgiutils/broadcast/local.py,sha256=24aIRdFOR2CXZfp_F2R_S1QW-yon3EyTM6TvljWVlP0,1083
|
14
|
+
c2cwsgiutils/broadcast/redis.py,sha256=sxBsYZtMNXMOck6a_Mcb3QZjjW4lPTxnImTfL1vv14g,5085
|
15
|
+
c2cwsgiutils/broadcast/utils.py,sha256=0fQZXPu3p_5LEJpGenJwiiMxECQjJhjZBjIkBk8h-ng,272
|
16
|
+
c2cwsgiutils/client_info.py,sha256=t35d8yI6MVDoB3i2yRKk0k-GmLP3v-WCis5JVzC6XFg,3044
|
17
|
+
c2cwsgiutils/config_utils.py,sha256=N_DPNRCBmeomhv2cJN5OQALrTAG29FCCk3VeugWrBwI,1523
|
18
|
+
c2cwsgiutils/coverage_setup.py,sha256=fES0sdhFy6oaeOCuP1qjjm7PQL9l_O8rUKZhRvRBRwQ,839
|
19
|
+
c2cwsgiutils/db.py,sha256=N_9zxxXYKiKwfNxs5KsunKYb_gUZQkTrlw6HZu5qsgQ,16174
|
20
|
+
c2cwsgiutils/db_maintenance_view.py,sha256=ejSNCv7vZIDjXctDTWTXEYTnUYFVVNIecFLeDlCkDBA,3076
|
21
|
+
c2cwsgiutils/debug/__init__.py,sha256=80zdAZnE9cwgQW1odE2aOauIxYsG5CQpWvHPcslRue8,1239
|
22
|
+
c2cwsgiutils/debug/_listeners.py,sha256=sXXHbPHQaSRMrzUC2ryiSDsBXuTdVbpQOxmMmhZ3n90,4378
|
23
|
+
c2cwsgiutils/debug/_views.py,sha256=zUeIshxBshBrlTz0p1I1LUi6HiJBOAAiDPrlOJsziQA,7522
|
24
|
+
c2cwsgiutils/debug/utils.py,sha256=TPlJC5qKeFnvbgq1xjlfrrRgDcV5kIR69IPJgNcIZQY,2311
|
25
|
+
c2cwsgiutils/errors.py,sha256=LUhZF3BW1i-v20x3EE-rZbT-TpdugpxiW7p5iCAu73Q,6723
|
26
|
+
c2cwsgiutils/health_check.py,sha256=O-GGS0aicTuMz4gqE-btk8ipzVi5allDe6sFuyOANOA,19977
|
27
|
+
c2cwsgiutils/index.py,sha256=jFThEi1sbsd2RnMNwhXpVAQUcpZSxW7ava9kxYwO5Zk,16703
|
28
|
+
c2cwsgiutils/loader.py,sha256=x_yHRTDzzlQ61fHonWnnG01xdqFuXpbGZMNN--tN25U,622
|
29
|
+
c2cwsgiutils/logging_view.py,sha256=W6-dFTz00hLt6BGJ8V3-4ip4AdxTyPG2W5vQjIuKiQs,3357
|
30
|
+
c2cwsgiutils/models_graph.py,sha256=laip8EdhI2hoGZVAotdrsgMwiNbwsJPjknKkRq1eEq0,2680
|
31
|
+
c2cwsgiutils/pretty_json.py,sha256=f1-oecFX9hub1nD32mmZRjOTIxhV8bVSt3Meqw68sNU,1698
|
32
|
+
c2cwsgiutils/profiler.py,sha256=3tIwoDSzOKQ06ug_U6j5VDR1BQ9auUOqdJRRLRhDoHw,739
|
33
|
+
c2cwsgiutils/prometheus.py,sha256=a09eTEmmhv_WENYx5I8jZUFdbUuVztE_lbMdOJNDm1U,6494
|
34
|
+
c2cwsgiutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
35
|
+
c2cwsgiutils/pyramid.py,sha256=H-b5abZvxt9MkMFjyWOjWzPa02YRhZR6LDiMCaYLq5s,1388
|
36
|
+
c2cwsgiutils/pyramid_logging.py,sha256=vVPrkAKZoIwb6SPf-omZDGyiKfElBf5yo_dPaIo-8ko,3730
|
37
|
+
c2cwsgiutils/redis_stats.py,sha256=triLtzryGXJKvCUw4TreYpF22BpUKPdrMp30ZGKsXwU,1545
|
38
|
+
c2cwsgiutils/redis_utils.py,sha256=dSe7qvnobCglKwDbYg4OwCxWxWg4migxmHbgClU2Qnw,4630
|
39
|
+
c2cwsgiutils/request_tracking/__init__.py,sha256=5SxMqBV3_w6SyykB1FFKug7XqzwdPhQdQOZ4VKUM4A8,4039
|
40
|
+
c2cwsgiutils/request_tracking/_sql.py,sha256=ME13tdXV2Vvhvscon9DbDUqehNWcn4uL75uqzcN5-Mg,577
|
41
|
+
c2cwsgiutils/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
42
|
+
c2cwsgiutils/scripts/genversion.py,sha256=GKmD08-H83hiKqXxy4odABuWqD5gADs_YEzM3TzHxEA,1985
|
43
|
+
c2cwsgiutils/scripts/stats_db.py,sha256=QNwtEKznCgOKg4ZsAFED9ISS-THgdwD2km_oVoII5XU,10317
|
44
|
+
c2cwsgiutils/scripts/test_print.py,sha256=UeOZa7jTazgEq5BRJD6lq-u9K6G4movf-sOVKTEs1cQ,2096
|
45
|
+
c2cwsgiutils/sentry.py,sha256=X-8Pd9wEi6EKMde50LxcdeM6LYBRyGnJpkPgEmFgqpg,5029
|
46
|
+
c2cwsgiutils/services.py,sha256=qz51oCZOC0Lj2_ig4UuHIm0ZZO3FfpFTxrXBWZ_oaNo,1567
|
47
|
+
c2cwsgiutils/setup_process.py,sha256=RKZGQdcKvHh_YU2mA-J5FQ01XNGj-SevcPNQB0439rs,3428
|
48
|
+
c2cwsgiutils/sql_profiler/__init__.py,sha256=lZYq83LYlm_P4uNMv0WU_B9Obl90YaNzkqWtteUHadg,876
|
49
|
+
c2cwsgiutils/sql_profiler/_impl.py,sha256=0UWSZDEm2E9Zeujd8UFRu0cIyyHBMxXxdmzt0cmuv0s,3702
|
50
|
+
c2cwsgiutils/sqlalchemylogger/README.md,sha256=WEyJSrBjedtX1FFrYiq4oMaWMt1fNxRkJYmJWnAoz3g,1552
|
51
|
+
c2cwsgiutils/sqlalchemylogger/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
52
|
+
c2cwsgiutils/sqlalchemylogger/_filters.py,sha256=OJQ9_WA-fd9fMZ7TUNFzHHTPI6msw2NVBl5RoeYFnGw,752
|
53
|
+
c2cwsgiutils/sqlalchemylogger/_models.py,sha256=A9SQ8AqUazCMemVjp5p_1x4bZG3LAYW9pOXT84FdNkE,1471
|
54
|
+
c2cwsgiutils/sqlalchemylogger/examples/example.py,sha256=n48dJdUi1FH1hfBMAbfHLGPSb1bOVD8pXMxXB57PnpQ,460
|
55
|
+
c2cwsgiutils/sqlalchemylogger/handlers.py,sha256=Qul-Bebmxqt4KvFU4TqjUGnbmpAVBviF5pSV3duUqG0,4801
|
56
|
+
c2cwsgiutils/static/favicon-16x16.png,sha256=LKk6RFvb3NlPIZdDfAodE8H9IN8KM6CMGnMx4vOHlUQ,887
|
57
|
+
c2cwsgiutils/static/favicon-32x32.png,sha256=i4ucx08zAZARd8e7JTMGK-gb5WcOmyuDN6IN4brsEOo,1216
|
58
|
+
c2cwsgiutils/stats_pyramid/__init__.py,sha256=7P10LjLv3c-ObEDGuYmRF_RFt7fRmO80ruqTGQAyC6w,747
|
59
|
+
c2cwsgiutils/stats_pyramid/_db_spy.py,sha256=ZGRdrI17Bdl3mzaLjfPyAaEW3KK8Pikrgi-0WmH7zCs,2917
|
60
|
+
c2cwsgiutils/stats_pyramid/_pyramid_spy.py,sha256=P212MGGl2VV_7UU4AXZA-rOuF7ouaONRklZwpas2wc8,3209
|
61
|
+
c2cwsgiutils/templates/index.html.mako,sha256=zrdMxncl3PFwdg1TVDKAjH1CaXw1m7yr9dj8Hi7rYGw,1362
|
62
|
+
c2cwsgiutils/version.py,sha256=z4of1DDr6J7PDw4AUOz31Gp63khgXf3JfiIaoWUM-9I,2870
|
63
|
+
c2cwsgiutils-5.2.1.dev197.dist-info/LICENSE,sha256=rM6IWxociA3daRkXnNLYOxGndT5fbs3BfVZCA2Xgt-g,1304
|
64
|
+
c2cwsgiutils-5.2.1.dev197.dist-info/METADATA,sha256=AXKlHCfO4_Uq--GCt6EpxznwArBUXtSO-IQrrlH-CH8,32515
|
65
|
+
c2cwsgiutils-5.2.1.dev197.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
|
66
|
+
c2cwsgiutils-5.2.1.dev197.dist-info/entry_points.txt,sha256=ujgqMTL1awN9qDg8WXmrF7m0fgR-hslUM6zKH86pvy0,703
|
67
|
+
c2cwsgiutils-5.2.1.dev197.dist-info/RECORD,,
|
@@ -0,0 +1,21 @@
|
|
1
|
+
[console_scripts]
|
2
|
+
c2cwsgiutils-check-es=c2cwsgiutils.scripts.check_es:main
|
3
|
+
c2cwsgiutils-coverage-report=c2cwsgiutils.scripts.coverage_report:main
|
4
|
+
c2cwsgiutils-genversion=c2cwsgiutils.scripts.genversion:main
|
5
|
+
c2cwsgiutils-stats-db=c2cwsgiutils.scripts.stats_db:main
|
6
|
+
c2cwsgiutils-test-print=c2cwsgiutils.scripts.test_print:main
|
7
|
+
|
8
|
+
[paste.filter_factory]
|
9
|
+
client_info=c2cwsgiutils.client_info:filter_factory
|
10
|
+
sentry=c2cwsgiutils.sentry:filter_factory
|
11
|
+
|
12
|
+
[plaster.loader_factory]
|
13
|
+
c2c=c2cwsgiutils.loader:Loader
|
14
|
+
c2c+egg=c2cwsgiutils.loader:Loader
|
15
|
+
c2c+ini=c2cwsgiutils.loader:Loader
|
16
|
+
|
17
|
+
[plaster.wsgi_loader_factory]
|
18
|
+
c2c=c2cwsgiutils.loader:Loader
|
19
|
+
c2c+egg=c2cwsgiutils.loader:Loader
|
20
|
+
c2c+ini=c2cwsgiutils.loader:Loader
|
21
|
+
|
@@ -1,129 +0,0 @@
|
|
1
|
-
import logging
|
2
|
-
import os
|
3
|
-
import subprocess # nosec
|
4
|
-
import sys
|
5
|
-
import time
|
6
|
-
import warnings
|
7
|
-
from typing import Any, Callable, Dict, List, Mapping, Optional, cast
|
8
|
-
|
9
|
-
import netifaces
|
10
|
-
from pyramid.request import Request
|
11
|
-
|
12
|
-
from c2cwsgiutils.acceptance import utils
|
13
|
-
|
14
|
-
LOG = logging.getLogger(__name__)
|
15
|
-
logging.basicConfig(
|
16
|
-
level=logging.DEBUG,
|
17
|
-
format="TEST: %(asctime)-15s %(levelname)5s %(name)s %(message)s",
|
18
|
-
stream=sys.stdout,
|
19
|
-
)
|
20
|
-
logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARN)
|
21
|
-
|
22
|
-
|
23
|
-
def _try(what: Callable[[], Any], fail: bool = True, times: int = 5, delay: int = 10) -> Optional[Any]:
|
24
|
-
for i in range(times):
|
25
|
-
try:
|
26
|
-
return what()
|
27
|
-
except: # noqa: bare-except
|
28
|
-
LOG.warning("Exception:", exc_info=True)
|
29
|
-
if i + 1 == times and fail:
|
30
|
-
raise
|
31
|
-
time.sleep(delay)
|
32
|
-
return None
|
33
|
-
|
34
|
-
|
35
|
-
class Composition:
|
36
|
-
"""The Docker composition."""
|
37
|
-
|
38
|
-
def __init__(self, request: Request, composition: str, coverage_paths: Optional[str] = None) -> None:
|
39
|
-
warnings.warn("The c2cwsgiutils.acceptance.composition should be used only if it's relay needed.")
|
40
|
-
self.cwd = os.path.dirname(composition)
|
41
|
-
filename = os.path.basename(composition)
|
42
|
-
self.docker_compose = ["docker-compose"]
|
43
|
-
if filename != "docker-compose.yaml":
|
44
|
-
self.docker_compose.append("--file=" + filename)
|
45
|
-
self.coverage_paths = coverage_paths
|
46
|
-
env = Composition._get_env()
|
47
|
-
if os.environ.get("docker_start", "1") == "1":
|
48
|
-
self.dc_try(["stop"], fail=False)
|
49
|
-
self.dc_try(["rm", "-f"], fail=False)
|
50
|
-
self.dc_try(["build"], fail=False)
|
51
|
-
self.dc_try(["up", "-d"], fail=False)
|
52
|
-
|
53
|
-
# Setup something that redirects the docker container logs to the test output
|
54
|
-
log_watcher = subprocess.Popen( # nosec, pylint: disable=consider-using-with
|
55
|
-
self.docker_compose + ["logs", "--follow", "--no-color"],
|
56
|
-
cwd=self.cwd,
|
57
|
-
env=env,
|
58
|
-
stderr=subprocess.STDOUT,
|
59
|
-
)
|
60
|
-
request.addfinalizer(log_watcher.kill)
|
61
|
-
if os.environ.get("docker_stop", "1") == "1":
|
62
|
-
request.addfinalizer(self.stop_all)
|
63
|
-
|
64
|
-
def dc(self, args: List[str], **kwargs: Any) -> str:
|
65
|
-
return cast(
|
66
|
-
str,
|
67
|
-
subprocess.check_output( # nosec
|
68
|
-
self.docker_compose + args,
|
69
|
-
cwd=self.cwd,
|
70
|
-
env=Composition._get_env(),
|
71
|
-
stderr=subprocess.STDOUT,
|
72
|
-
**kwargs,
|
73
|
-
).decode(),
|
74
|
-
)
|
75
|
-
|
76
|
-
def dc_try(self, args: List[str], **kwargs: Any) -> None:
|
77
|
-
_try(
|
78
|
-
lambda: self.dc(args),
|
79
|
-
**kwargs,
|
80
|
-
)
|
81
|
-
|
82
|
-
def stop_all(self) -> None:
|
83
|
-
self.dc_try(["stop"])
|
84
|
-
if self.coverage_paths:
|
85
|
-
target_dir = "/reports/"
|
86
|
-
os.makedirs(target_dir, exist_ok=True)
|
87
|
-
for path in self.coverage_paths:
|
88
|
-
try:
|
89
|
-
subprocess.check_call( # nosec
|
90
|
-
["docker", "cp", path, target_dir], stderr=subprocess.STDOUT
|
91
|
-
)
|
92
|
-
except Exception:
|
93
|
-
self.dc(["ps"])
|
94
|
-
raise
|
95
|
-
|
96
|
-
def stop(self, container: str) -> None:
|
97
|
-
self.dc_try(["stop", container])
|
98
|
-
|
99
|
-
def restart(self, container: str) -> None:
|
100
|
-
self.dc_try(["restart", container])
|
101
|
-
|
102
|
-
def run(self, container: str, *command: str, **kwargs: Dict[str, Any]) -> str:
|
103
|
-
return self.dc(
|
104
|
-
["run", "--rm", container] + list(command),
|
105
|
-
**kwargs,
|
106
|
-
)
|
107
|
-
|
108
|
-
def exec(self, container: str, *command: str, **kwargs: Dict[str, Any]) -> str:
|
109
|
-
return self.dc(["exec", "-T", container] + list(command), **kwargs)
|
110
|
-
|
111
|
-
@staticmethod
|
112
|
-
def _get_env() -> Mapping[str, str]:
|
113
|
-
"""
|
114
|
-
Make sure the DOCKER_TAG environment variable.
|
115
|
-
|
116
|
-
Used in the docker-compose.yaml file is correctly set when we call docker-compose.
|
117
|
-
"""
|
118
|
-
env = dict(os.environ)
|
119
|
-
if "DOCKER_TAG" not in env:
|
120
|
-
env["DOCKER_TAG"] = "latest"
|
121
|
-
if utils.in_docker():
|
122
|
-
env["DOCKER_IP"] = netifaces.gateways()[netifaces.AF_INET][0][0]
|
123
|
-
env["DOCKER_CB_HOST"] = env["DOCKER_IP"]
|
124
|
-
else:
|
125
|
-
env["DOCKER_IP"] = netifaces.ifaddresses("docker0")[netifaces.AF_INET][0]["addr"]
|
126
|
-
env["DOCKER_CB_HOST"] = "localhost"
|
127
|
-
default_iface = netifaces.gateways()[netifaces.AF_INET][0][1]
|
128
|
-
env["TEST_IP"] = netifaces.ifaddresses(default_iface)[netifaces.AF_INET][0]["addr"]
|
129
|
-
return env
|