c2cgeoportal-geoportal 2.7.1.156__py2.py3-none-any.whl → 2.8.1.180__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- c2cgeoportal_geoportal/__init__.py +24 -14
- c2cgeoportal_geoportal/lib/authentication.py +10 -14
- c2cgeoportal_geoportal/lib/caching.py +8 -6
- c2cgeoportal_geoportal/lib/checker.py +10 -6
- c2cgeoportal_geoportal/lib/common_headers.py +5 -8
- c2cgeoportal_geoportal/lib/dbreflection.py +8 -8
- c2cgeoportal_geoportal/lib/filter_capabilities.py +5 -1
- c2cgeoportal_geoportal/lib/lingua_extractor.py +11 -12
- c2cgeoportal_geoportal/lib/loader.py +1 -1
- c2cgeoportal_geoportal/lib/oauth2.py +217 -100
- c2cgeoportal_geoportal/lib/wmstparsing.py +8 -12
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/Dockerfile +9 -11
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/development.ini +1 -1
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/gunicorn.conf.py +0 -2
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/requirements.txt +1 -1
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/webpack.api.js +6 -4
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/webpack.apps.js +1 -3
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/webpack.commons.js +1 -0
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/{{cookiecutter.package}}_geoportal/__init__.py +1 -6
- c2cgeoportal_geoportal/scaffolds/advance_update/{{cookiecutter.project}}/geoportal/CONST_Makefile +0 -20
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/.github/workflows/main.yaml +20 -6
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/.github/workflows/update_l10n.yaml +4 -3
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/Dockerfile +22 -22
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/Makefile +58 -2
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/build +48 -24
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/ci/config.yaml +2 -5
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/ci/docker-compose-check +25 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/ci/requirements.txt +1 -1
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose-db.yaml +26 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose-lib.yaml +53 -26
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose-qgis.yaml +23 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose.override.sample.yaml +0 -1
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose.yaml +3 -3
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/env.default +21 -2
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/env.project +9 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/geoportal/vars.yaml +38 -14
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/mapserver/data/Readme.txt +2 -2
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/mapserver/mapserver.conf +15 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/mapserver/mapserver.map.tmpl +2 -3
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A3_Landscape.jrxml +5 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A3_Portrait.jrxml +5 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A4_Landscape.jrxml +5 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A4_Portrait.jrxml +5 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/config.yaml.tmpl +6 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/pyproject.toml +4 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/run_alembic.sh +3 -5
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/scripts/db-backup +1 -1
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/scripts/db-restore +1 -1
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/spell-ignore-words.txt +2 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/tests/__init__.py +0 -0
- c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/tests/test_app.py +38 -0
- c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/.upgrade.yaml +2 -132
- c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_CHANGELOG.txt +210 -1097
- c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_create_template/tests/test_testapp.py +48 -0
- c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_config-schema.yaml +17 -15
- c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_vars.yaml +46 -2
- c2cgeoportal_geoportal/scripts/c2cupgrade.py +1 -2
- c2cgeoportal_geoportal/scripts/pcreate.py +8 -10
- c2cgeoportal_geoportal/scripts/theme2fts.py +58 -3
- c2cgeoportal_geoportal/views/__init__.py +1 -3
- c2cgeoportal_geoportal/views/dynamic.py +1 -1
- c2cgeoportal_geoportal/views/entry.py +2 -10
- c2cgeoportal_geoportal/views/fulltextsearch.py +1 -1
- c2cgeoportal_geoportal/views/geometry_processing.py +3 -3
- c2cgeoportal_geoportal/views/layers.py +10 -11
- c2cgeoportal_geoportal/views/login.py +63 -8
- c2cgeoportal_geoportal/views/mapserverproxy.py +2 -3
- c2cgeoportal_geoportal/views/ogcproxy.py +6 -2
- c2cgeoportal_geoportal/views/pdfreport.py +4 -4
- c2cgeoportal_geoportal/views/printproxy.py +2 -2
- c2cgeoportal_geoportal/views/profile.py +1 -1
- c2cgeoportal_geoportal/views/proxy.py +2 -4
- c2cgeoportal_geoportal/views/raster.py +2 -2
- c2cgeoportal_geoportal/views/resourceproxy.py +1 -1
- c2cgeoportal_geoportal/views/shortener.py +1 -2
- c2cgeoportal_geoportal/views/theme.py +97 -63
- c2cgeoportal_geoportal/views/tinyowsproxy.py +3 -12
- c2cgeoportal_geoportal/views/vector_tiles.py +1 -1
- {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/METADATA +21 -15
- {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/RECORD +96 -90
- {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/entry_points.txt +1 -0
- tests/__init__.py +3 -2
- tests/test_cachebuster.py +3 -3
- tests/test_caching.py +7 -7
- tests/test_checker.py +1 -1
- tests/test_decimaljson.py +1 -1
- tests/test_headerstween.py +1 -1
- tests/test_i18n.py +1 -1
- tests/test_init.py +14 -15
- tests/test_locale_negociator.py +4 -4
- tests/test_mapserverproxy_route_predicate.py +1 -2
- tests/test_raster.py +15 -15
- tests/test_wmstparsing.py +10 -10
- tests/xmlstr.py +1 -3
- c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/tools/extract-messages.js +0 -41
- {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/WHEEL +0 -0
- {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (c) 2011-
|
1
|
+
# Copyright (c) 2011-2024, Camptocamp SA
|
2
2
|
# All rights reserved.
|
3
3
|
|
4
4
|
# Redistribution and use in source and binary forms, with or without
|
@@ -28,9 +28,9 @@
|
|
28
28
|
|
29
29
|
import json
|
30
30
|
import logging
|
31
|
-
import secrets
|
32
31
|
import sys
|
33
32
|
import urllib.parse
|
33
|
+
from random import Random
|
34
34
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
35
35
|
|
36
36
|
import pyotp
|
@@ -88,7 +88,7 @@ class Login:
|
|
88
88
|
if not hasattr(self.request, "is_valid_referer"):
|
89
89
|
self.request.is_valid_referer = is_valid_referrer(self.request)
|
90
90
|
if not self.request.is_valid_referer:
|
91
|
-
LOG.info("Invalid
|
91
|
+
LOG.info("Invalid referrer for %s: %s", self.request.path_qs, repr(self.request.referrer))
|
92
92
|
|
93
93
|
@forbidden_view_config(renderer="login.html") # type: ignore
|
94
94
|
def loginform403(self) -> Union[Dict[str, Any], pyramid.response.Response]:
|
@@ -341,11 +341,12 @@ class Login:
|
|
341
341
|
|
342
342
|
@staticmethod
|
343
343
|
def generate_password() -> str:
|
344
|
-
|
344
|
+
allchars = "123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
345
|
+
rand = Random() # nosec
|
345
346
|
|
346
347
|
password = "" # nosec
|
347
348
|
for _ in range(8):
|
348
|
-
password +=
|
349
|
+
password += rand.choice(allchars)
|
349
350
|
|
350
351
|
return password
|
351
352
|
|
@@ -397,6 +398,36 @@ class Login:
|
|
397
398
|
|
398
399
|
return {"success": True}
|
399
400
|
|
401
|
+
@view_config(route_name="oauth2introspect") # type: ignore
|
402
|
+
def oauth2introspect(self) -> pyramid.response.Response:
|
403
|
+
LOG.debug(
|
404
|
+
"Call OAuth create_introspect_response with:\nurl: %s\nmethod: %s\nbody:\n%s",
|
405
|
+
self.request.current_route_url(_query=self.request.GET),
|
406
|
+
self.request.method,
|
407
|
+
self.request.body,
|
408
|
+
)
|
409
|
+
headers, body, status = oauth2.get_oauth_client(
|
410
|
+
self.request.registry.settings
|
411
|
+
).create_introspect_response(
|
412
|
+
self.request.current_route_url(_query=self.request.GET),
|
413
|
+
self.request.method,
|
414
|
+
self.request.body,
|
415
|
+
self.request.headers,
|
416
|
+
)
|
417
|
+
LOG.debug("OAuth create_introspect_response return status: %s", status)
|
418
|
+
|
419
|
+
# All requests to /token will return a json response, no redirection.
|
420
|
+
if status != 200:
|
421
|
+
if body:
|
422
|
+
raise exception_response(status, detail=body)
|
423
|
+
raise exception_response(status)
|
424
|
+
return set_common_headers(
|
425
|
+
self.request,
|
426
|
+
"login",
|
427
|
+
Cache.PRIVATE_NO,
|
428
|
+
response=Response(body, headers=headers.items()),
|
429
|
+
)
|
430
|
+
|
400
431
|
@view_config(route_name="oauth2token") # type: ignore
|
401
432
|
def oauth2token(self) -> pyramid.response.Response:
|
402
433
|
LOG.debug(
|
@@ -414,9 +445,6 @@ class Login:
|
|
414
445
|
)
|
415
446
|
LOG.debug("OAuth create_token_response return status: %s", status)
|
416
447
|
|
417
|
-
if hasattr(self.request, "tm"):
|
418
|
-
self.request.tm.commit()
|
419
|
-
|
420
448
|
# All requests to /token will return a json response, no redirection.
|
421
449
|
if status != 200:
|
422
450
|
if body:
|
@@ -429,6 +457,33 @@ class Login:
|
|
429
457
|
response=Response(body, headers=headers.items()),
|
430
458
|
)
|
431
459
|
|
460
|
+
@view_config(route_name="oauth2revoke_token") # type: ignore
|
461
|
+
def oauth2revoke_token(self) -> pyramid.response.Response:
|
462
|
+
LOG.debug(
|
463
|
+
"Call OAuth create_revocation_response with:\nurl: %s\nmethod: %s\nbody:\n%s",
|
464
|
+
self.request.create_revocation_response(_query=self.request.GET),
|
465
|
+
self.request.method,
|
466
|
+
self.request.body,
|
467
|
+
)
|
468
|
+
headers, body, status = oauth2.get_oauth_client(
|
469
|
+
self.request.registry.settings
|
470
|
+
).create_authorize_response(
|
471
|
+
self.request.current_route_url(_query=self.request.GET),
|
472
|
+
self.request.method,
|
473
|
+
self.request.body,
|
474
|
+
self.request.headers,
|
475
|
+
)
|
476
|
+
if status != 200:
|
477
|
+
if body:
|
478
|
+
raise exception_response(status, detail=body)
|
479
|
+
raise exception_response(status)
|
480
|
+
return set_common_headers(
|
481
|
+
self.request,
|
482
|
+
"login",
|
483
|
+
Cache.PRIVATE_NO,
|
484
|
+
response=Response(body, headers=headers.items()),
|
485
|
+
)
|
486
|
+
|
432
487
|
@view_config(route_name="oauth2loginform", renderer="login.html") # type: ignore
|
433
488
|
def oauth2loginform(self) -> Dict[str, Any]:
|
434
489
|
set_common_headers(self.request, "login", Cache.PUBLIC)
|
@@ -29,7 +29,7 @@
|
|
29
29
|
import logging
|
30
30
|
from typing import Any, Dict, Set
|
31
31
|
|
32
|
-
from pyramid.httpexceptions import HTTPFound, HTTPInternalServerError, HTTPUnauthorized
|
32
|
+
from pyramid.httpexceptions import HTTPForbidden, HTTPFound, HTTPInternalServerError, HTTPUnauthorized
|
33
33
|
from pyramid.request import Request
|
34
34
|
from pyramid.response import Response
|
35
35
|
from pyramid.view import view_config
|
@@ -67,7 +67,7 @@ class MapservProxy(OGCProxy):
|
|
67
67
|
raise HTTPUnauthorized(
|
68
68
|
headers={"WWW-Authenticate": 'Basic realm="Access to restricted layers"'}
|
69
69
|
)
|
70
|
-
raise
|
70
|
+
raise HTTPForbidden("Basic auth is not enabled")
|
71
71
|
|
72
72
|
# We have a user logged in. We need to set group_id and possible layer_name in the params. We set
|
73
73
|
# layer_name when either QUERY_PARAMS or LAYERS is set in the WMS params, i.e. for GetMap and
|
@@ -108,7 +108,6 @@ class MapservProxy(OGCProxy):
|
|
108
108
|
self.params = {}
|
109
109
|
else:
|
110
110
|
if self.ogc_server.type != main.OGCSERVER_TYPE_QGISSERVER or "user_id" not in self.params:
|
111
|
-
|
112
111
|
use_cache = self.lower_params["request"] in ("getlegendgraphic",)
|
113
112
|
|
114
113
|
# no user_id and role_id or cached queries
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (c) 2011-
|
1
|
+
# Copyright (c) 2011-2023, Camptocamp SA
|
2
2
|
# All rights reserved.
|
3
3
|
|
4
4
|
# Redistribution and use in source and binary forms, with or without
|
@@ -60,6 +60,8 @@ class OGCProxy(Proxy):
|
|
60
60
|
if "user_id" in self.params:
|
61
61
|
del self.params["user_id"]
|
62
62
|
|
63
|
+
main_ogc_server = self.request.registry.settings.get("main_ogc_server")
|
64
|
+
|
63
65
|
self.lower_params = self._get_lower_params(self.params)
|
64
66
|
|
65
67
|
# We need original case for OGCSERVER parameter value
|
@@ -69,11 +71,13 @@ class OGCProxy(Proxy):
|
|
69
71
|
self.ogc_server = self._get_ogcserver_byname(request.matchdict["ogcserver"])
|
70
72
|
elif "ogcserver" in self.lower_key_params:
|
71
73
|
self.ogc_server = self._get_ogcserver_byname(self.lower_key_params["ogcserver"])
|
74
|
+
elif main_ogc_server is not None:
|
75
|
+
self.ogc_server = self._get_ogcserver_byname(main_ogc_server)
|
72
76
|
elif not has_default_ogc_server:
|
73
77
|
raise HTTPBadRequest("The querystring argument 'ogcserver' is required")
|
74
78
|
|
75
79
|
@CACHE_REGION.cache_on_arguments() # type: ignore
|
76
|
-
def _get_ogcserver_byname(self, name: str) -> main.OGCServer:
|
80
|
+
def _get_ogcserver_byname(self, name: str) -> main.OGCServer:
|
77
81
|
try:
|
78
82
|
result = DBSession.query(main.OGCServer).filter(main.OGCServer.name == name).one()
|
79
83
|
DBSession.expunge(result)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (c) 2011-
|
1
|
+
# Copyright (c) 2011-2024, Camptocamp SA
|
2
2
|
# All rights reserved.
|
3
3
|
|
4
4
|
# Redistribution and use in source and binary forms, with or without
|
@@ -106,14 +106,14 @@ class PdfReport(OGCProxy):
|
|
106
106
|
self.layername = self.request.matchdict["layername"]
|
107
107
|
layer_config = self.config["layers"].get(self.layername)
|
108
108
|
|
109
|
+
if layer_config is None:
|
110
|
+
raise HTTPBadRequest("Layer not found: " + self.layername)
|
111
|
+
|
109
112
|
multiple = layer_config.get("multiple", False)
|
110
113
|
ids = self.request.matchdict["ids"]
|
111
114
|
if multiple:
|
112
115
|
ids = ids.split(",")
|
113
116
|
|
114
|
-
if layer_config is None:
|
115
|
-
raise HTTPBadRequest("Layer not found")
|
116
|
-
|
117
117
|
features_ids = (
|
118
118
|
[self.layername + "." + id_ for id_ in ids] if multiple else [self.layername + "." + ids]
|
119
119
|
)
|
@@ -68,8 +68,8 @@ class PrintProxy(Proxy):
|
|
68
68
|
)
|
69
69
|
|
70
70
|
response = self._build_response(resp, content, Cache.PRIVATE, "print")
|
71
|
-
# Mapfish print will check the
|
72
|
-
response.vary += ("Referer"
|
71
|
+
# Mapfish print will check the referrer header to return the capabilities.
|
72
|
+
response.vary += ("Referrer", "Referer")
|
73
73
|
return response
|
74
74
|
|
75
75
|
@CACHE_REGION.cache_on_arguments() # type: ignore
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
|
3
1
|
# Copyright (c) 2011-2024, Camptocamp SA
|
4
2
|
# All rights reserved.
|
5
3
|
|
@@ -47,7 +45,7 @@ CACHE_REGION = get_region("std")
|
|
47
45
|
|
48
46
|
|
49
47
|
class Proxy:
|
50
|
-
"""Some
|
48
|
+
"""Some methods used by all the proxy."""
|
51
49
|
|
52
50
|
def __init__(self, request: pyramid.request.Request):
|
53
51
|
self.request = request
|
@@ -252,7 +250,7 @@ class Proxy:
|
|
252
250
|
|
253
251
|
@staticmethod
|
254
252
|
def _get_lower_params(params: Dict[str, str]) -> Dict[str, str]:
|
255
|
-
return
|
253
|
+
return {k.lower(): str(v).lower() for k, v in params.items()}
|
256
254
|
|
257
255
|
def get_headers(self) -> Dict[str, str]:
|
258
256
|
headers: Dict[str, str] = self.request.headers
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (c) 2012-
|
1
|
+
# Copyright (c) 2012-2023, Camptocamp SA
|
2
2
|
# All rights reserved.
|
3
3
|
|
4
4
|
# Redistribution and use in source and binary forms, with or without
|
@@ -167,7 +167,7 @@ class Raster:
|
|
167
167
|
result = None if result == layer.get("nodata", dataset.nodata) else result
|
168
168
|
else:
|
169
169
|
LOG.debug(
|
170
|
-
"Out of index for layer: %s (%s),
|
170
|
+
"Out of index for layer: %s (%s), lon/lat: %dx%d, index: %dx%d, shape: %dx%d.",
|
171
171
|
name,
|
172
172
|
layer["file"],
|
173
173
|
lon,
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (c) 2013-
|
1
|
+
# Copyright (c) 2013-2023, Camptocamp SA
|
2
2
|
# All rights reserved.
|
3
3
|
|
4
4
|
# Redistribution and use in source and binary forms, with or without
|
@@ -71,7 +71,6 @@ class Shortener:
|
|
71
71
|
|
72
72
|
@view_config(route_name="shortener_create", renderer="json") # type: ignore
|
73
73
|
def create(self) -> Dict[str, str]:
|
74
|
-
|
75
74
|
if "url" not in self.request.params:
|
76
75
|
raise HTTPBadRequest("The parameter url is required")
|
77
76
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (c) 2011-
|
1
|
+
# Copyright (c) 2011-2024, Camptocamp SA
|
2
2
|
# All rights reserved.
|
3
3
|
|
4
4
|
# Redistribution and use in source and binary forms, with or without
|
@@ -38,6 +38,7 @@ from math import sqrt
|
|
38
38
|
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
|
39
39
|
|
40
40
|
import dogpile.cache.api
|
41
|
+
import pyramid.httpexceptions
|
41
42
|
import pyramid.request
|
42
43
|
import requests
|
43
44
|
import sqlalchemy
|
@@ -52,7 +53,7 @@ from sqlalchemy.orm.exc import NoResultFound
|
|
52
53
|
|
53
54
|
from c2cgeoportal_commons import models
|
54
55
|
from c2cgeoportal_commons.lib.url import Url, get_url2
|
55
|
-
from c2cgeoportal_commons.models import main
|
56
|
+
from c2cgeoportal_commons.models import cache_invalidate_cb, main
|
56
57
|
from c2cgeoportal_geoportal.lib import get_roles_id, get_typed, get_types_map, is_intranet
|
57
58
|
from c2cgeoportal_geoportal.lib.caching import get_region
|
58
59
|
from c2cgeoportal_geoportal.lib.common_headers import Cache, set_common_headers
|
@@ -63,27 +64,31 @@ from c2cgeoportal_geoportal.lib.layers import (
|
|
63
64
|
get_protected_layers_query,
|
64
65
|
)
|
65
66
|
from c2cgeoportal_geoportal.lib.wmstparsing import TimeInformation, parse_extent
|
66
|
-
from c2cgeoportal_geoportal.views import restrict_headers
|
67
67
|
from c2cgeoportal_geoportal.views.layers import get_layer_metadata
|
68
68
|
|
69
69
|
LOG = logging.getLogger(__name__)
|
70
70
|
CACHE_REGION = get_region("std")
|
71
|
+
CACHE_OGC_SERVER_REGION = get_region("ogc-server")
|
71
72
|
TIMEOUT = int(os.environ.get("C2CGEOPORTAL_THEME_TIMEOUT", "300"))
|
72
73
|
|
73
74
|
Metadata = Union[str, int, float, bool, List[Any], Dict[str, Any]]
|
74
75
|
|
75
76
|
|
76
|
-
def get_http_cached(
|
77
|
+
def get_http_cached(
|
78
|
+
http_options: Dict[str, Any], url: str, headers: Dict[str, str], cache: bool = True
|
79
|
+
) -> Tuple[bytes, str]:
|
77
80
|
"""Get the content of the URL with a cash (dogpile)."""
|
78
81
|
|
79
|
-
@
|
82
|
+
@CACHE_OGC_SERVER_REGION.cache_on_arguments() # type: ignore
|
80
83
|
def do_get_http_cached(url: str) -> Tuple[bytes, str]:
|
81
84
|
response = requests.get(url, headers=headers, timeout=TIMEOUT, **http_options)
|
82
85
|
response.raise_for_status()
|
83
|
-
LOG.info("Get
|
86
|
+
LOG.info("Get URL '%s' in %.1fs.", url, response.elapsed.total_seconds())
|
84
87
|
return response.content, response.headers.get("Content-Type", "")
|
85
88
|
|
86
|
-
|
89
|
+
if cache:
|
90
|
+
return do_get_http_cached(url) # type: ignore
|
91
|
+
return do_get_http_cached.refresh(url) # type: ignore
|
87
92
|
|
88
93
|
|
89
94
|
class DimensionInformation:
|
@@ -143,8 +148,6 @@ class Theme:
|
|
143
148
|
self.request = request
|
144
149
|
self.settings = request.registry.settings
|
145
150
|
self.http_options = self.settings.get("http_options", {})
|
146
|
-
self.headers_whitelist = self.settings.get("headers_whitelist", [])
|
147
|
-
self.headers_blacklist = self.settings.get("headers_blacklist", [])
|
148
151
|
self.metadata_type = get_types_map(
|
149
152
|
self.settings.get("admin_interface", {}).get("available_metadata", [])
|
150
153
|
)
|
@@ -179,14 +182,16 @@ class Theme:
|
|
179
182
|
return metadatas
|
180
183
|
|
181
184
|
async def _wms_getcap(
|
182
|
-
self, ogc_server: main.OGCServer, preload: bool = False
|
185
|
+
self, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True
|
183
186
|
) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]:
|
184
|
-
|
187
|
+
LOG.debug("Get the WMS Capabilities of '%s', preload: %s, cache: %s", ogc_server.name, preload, cache)
|
188
|
+
|
189
|
+
@CACHE_OGC_SERVER_REGION.cache_on_arguments() # type: ignore
|
185
190
|
def build_web_map_service(ogc_server_id: int) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]:
|
186
191
|
del ogc_server_id # Just for cache
|
187
192
|
|
188
193
|
if url is None:
|
189
|
-
raise RuntimeError("
|
194
|
+
raise RuntimeError("URL is None")
|
190
195
|
|
191
196
|
version = url.query.get("VERSION", "1.1.1")
|
192
197
|
layers = {}
|
@@ -220,16 +225,16 @@ class Theme:
|
|
220
225
|
}
|
221
226
|
|
222
227
|
del wms
|
223
|
-
LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
|
224
228
|
|
225
229
|
return {"layers": layers}, set()
|
226
230
|
|
227
|
-
|
228
|
-
|
229
|
-
|
231
|
+
if cache:
|
232
|
+
result = build_web_map_service.get(ogc_server.id)
|
233
|
+
if result != dogpile.cache.api.NO_VALUE:
|
234
|
+
return result # type: ignore
|
230
235
|
|
231
236
|
try:
|
232
|
-
url, content, errors = await self._wms_getcap_cached(ogc_server)
|
237
|
+
url, content, errors = await self._wms_getcap_cached(ogc_server, cache=cache)
|
233
238
|
except requests.exceptions.RequestException as exception:
|
234
239
|
error = (
|
235
240
|
f"Unable to get the WMS Capabilities for OGC server '{ogc_server.name}', "
|
@@ -240,10 +245,10 @@ class Theme:
|
|
240
245
|
if errors or preload:
|
241
246
|
return None, errors
|
242
247
|
|
243
|
-
return build_web_map_service(ogc_server.id) # type: ignore
|
248
|
+
return build_web_map_service.refresh(ogc_server.id) # type: ignore
|
244
249
|
|
245
250
|
async def _wms_getcap_cached(
|
246
|
-
self, ogc_server: main.OGCServer
|
251
|
+
self, ogc_server: main.OGCServer, cache: bool = True
|
247
252
|
) -> Tuple[Optional[Url], Optional[bytes], Set[str]]:
|
248
253
|
errors: Set[str] = set()
|
249
254
|
url = get_url2(f"The OGC server '{ogc_server.name}'", ogc_server.url, self.request, errors)
|
@@ -269,23 +274,15 @@ class Theme:
|
|
269
274
|
|
270
275
|
LOG.debug("Get WMS GetCapabilities for URL: %s", url)
|
271
276
|
|
272
|
-
|
273
|
-
headers = dict(self.request.headers)
|
277
|
+
headers = {}
|
274
278
|
|
275
279
|
# Add headers for Geoserver
|
276
280
|
if ogc_server.auth == main.OGCSERVER_AUTH_GEOSERVER:
|
277
281
|
headers["sec-username"] = "root"
|
278
282
|
headers["sec-roles"] = "root"
|
279
283
|
|
280
|
-
if url.hostname != "localhost" and "Host" in headers:
|
281
|
-
headers.pop("Host")
|
282
|
-
|
283
|
-
headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist)
|
284
|
-
|
285
284
|
try:
|
286
|
-
content, content_type =
|
287
|
-
None, get_http_cached, self.http_options, url, headers
|
288
|
-
)
|
285
|
+
content, content_type = get_http_cached(self.http_options, url.url(), headers, cache=cache)
|
289
286
|
except Exception:
|
290
287
|
error = f"Unable to GetCapabilities from URL {url}"
|
291
288
|
errors.add(error)
|
@@ -386,8 +383,6 @@ class Theme:
|
|
386
383
|
layer_info = {"id": layer.id, "name": layer.name, "metadata": self._get_metadata_list(layer, errors)}
|
387
384
|
if re.search("[/?#]", layer.name):
|
388
385
|
errors.add(f"The layer has an unsupported name '{layer.name}'.")
|
389
|
-
if isinstance(layer, main.LayerWMS) and re.search("[/?#]", layer.layer):
|
390
|
-
errors.add(f"The layer has an unsupported layers '{layer.layer}'.")
|
391
386
|
if layer.geo_table:
|
392
387
|
errors |= self._fill_editable(layer_info, layer)
|
393
388
|
if mixed:
|
@@ -858,7 +853,7 @@ class Theme:
|
|
858
853
|
return None if self.request.user is None else {role.id for role in self.request.user.roles}
|
859
854
|
|
860
855
|
async def _wfs_get_features_type(
|
861
|
-
self, wfs_url: Url,
|
856
|
+
self, wfs_url: Url, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True
|
862
857
|
) -> Tuple[Optional[etree.Element], Set[str]]:
|
863
858
|
errors = set()
|
864
859
|
|
@@ -872,23 +867,23 @@ class Theme:
|
|
872
867
|
}
|
873
868
|
)
|
874
869
|
|
875
|
-
LOG.debug(
|
870
|
+
LOG.debug(
|
871
|
+
"Get the WFS DescribeFeatureType of '%s', preload: %s, cache: %s", ogc_server.name, preload, cache
|
872
|
+
)
|
876
873
|
|
877
|
-
|
878
|
-
headers = dict(self.request.headers)
|
879
|
-
if wfs_url.hostname != "localhost" and "Host" in headers:
|
880
|
-
headers.pop("Host")
|
874
|
+
headers = {}
|
881
875
|
|
882
|
-
|
876
|
+
# Add headers for Geoserver
|
877
|
+
if ogc_server.auth == main.OGCSERVER_AUTH_GEOSERVER:
|
878
|
+
headers["sec-username"] = "root"
|
879
|
+
headers["sec-roles"] = "root"
|
883
880
|
|
884
881
|
try:
|
885
|
-
content, _ =
|
886
|
-
None, get_http_cached, self.http_options, wfs_url, headers
|
887
|
-
)
|
882
|
+
content, _ = get_http_cached(self.http_options, wfs_url.url(), headers, cache)
|
888
883
|
except requests.exceptions.RequestException as exception:
|
889
884
|
error = (
|
890
885
|
f"Unable to get WFS DescribeFeatureType from the URL '{wfs_url.url()}' for "
|
891
|
-
f"OGC server {
|
886
|
+
f"OGC server {ogc_server.name}, "
|
892
887
|
+ (
|
893
888
|
f"return the error: {exception.response.status_code} {exception.response.reason}"
|
894
889
|
if exception.response is not None
|
@@ -901,7 +896,7 @@ class Theme:
|
|
901
896
|
except Exception:
|
902
897
|
error = (
|
903
898
|
f"Unable to get WFS DescribeFeatureType from the URL {wfs_url} for "
|
904
|
-
f"OGC server {
|
899
|
+
f"OGC server {ogc_server.name}"
|
905
900
|
)
|
906
901
|
errors.add(error)
|
907
902
|
LOG.exception(error)
|
@@ -948,10 +943,10 @@ class Theme:
|
|
948
943
|
url_internal_wfs = url_wfs
|
949
944
|
return url_internal_wfs, url, url_wfs
|
950
945
|
|
951
|
-
async def
|
946
|
+
async def _preload(self, errors: Set[str]) -> None:
|
952
947
|
tasks = set()
|
953
948
|
for ogc_server in models.DBSession.query(main.OGCServer).all():
|
954
|
-
# Don't load unused OGC servers, required for
|
949
|
+
# Don't load unused OGC servers, required for landing page, because the related OGC server
|
955
950
|
# will be on error in those functions.
|
956
951
|
nb_layers = (
|
957
952
|
models.DBSession.query(sqlalchemy.func.count(main.LayerWMS.id))
|
@@ -963,26 +958,30 @@ class Theme:
|
|
963
958
|
LOG.debug("Preload OGC server '%s'", ogc_server.name)
|
964
959
|
url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors)
|
965
960
|
if url_internal_wfs is not None:
|
966
|
-
|
967
|
-
tasks.add(self._wfs_get_features_type(url_internal_wfs, ogc_server.name, True))
|
968
|
-
tasks.add(self._wms_getcap(ogc_server, True))
|
961
|
+
tasks.add(self.preload_ogc_server(ogc_server, url_internal_wfs))
|
969
962
|
|
970
963
|
await asyncio.gather(*tasks)
|
971
964
|
|
965
|
+
async def preload_ogc_server(
|
966
|
+
self, ogc_server: main.OGCServer, url_internal_wfs: Url, cache: bool = True
|
967
|
+
) -> None:
|
968
|
+
if ogc_server.wfs_support:
|
969
|
+
await self._get_features_attributes(url_internal_wfs, ogc_server, cache=cache)
|
970
|
+
await self._wms_getcap(ogc_server, False, cache=cache)
|
971
|
+
|
972
972
|
async def _get_features_attributes(
|
973
|
-
self, url_internal_wfs: Url,
|
973
|
+
self, url_internal_wfs: Url, ogc_server: main.OGCServer, cache: bool = True
|
974
974
|
) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]:
|
975
|
-
@
|
975
|
+
@CACHE_OGC_SERVER_REGION.cache_on_arguments() # type: ignore
|
976
976
|
def _get_features_attributes_cache(
|
977
977
|
url_internal_wfs: Url, ogc_server_name: str
|
978
978
|
) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]:
|
979
979
|
del url_internal_wfs # Just for cache
|
980
980
|
all_errors: Set[str] = set()
|
981
|
-
LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
|
982
981
|
if errors:
|
983
982
|
all_errors |= errors
|
984
983
|
return None, None, all_errors
|
985
|
-
assert feature_type
|
984
|
+
assert feature_type is not None
|
986
985
|
namespace: str = feature_type.attrib.get("targetNamespace")
|
987
986
|
types: Dict[Any, Dict[str, Any]] = {}
|
988
987
|
elements = {}
|
@@ -1041,9 +1040,8 @@ class Theme:
|
|
1041
1040
|
attributes[name] = types[type_]
|
1042
1041
|
elif (type_ == "Character") and (name + "Type") in types:
|
1043
1042
|
LOG.debug(
|
1044
|
-
|
1045
|
-
"
|
1046
|
-
'METADATA "gml_types" "auto"',
|
1043
|
+
'Due to MapServer weird behavior when using METADATA "gml_types" "auto"'
|
1044
|
+
"the type 'ms:Character' is returned as type '%sType' for feature '%s'.",
|
1047
1045
|
name,
|
1048
1046
|
name,
|
1049
1047
|
)
|
@@ -1057,12 +1055,14 @@ class Theme:
|
|
1057
1055
|
|
1058
1056
|
return attributes, namespace, all_errors
|
1059
1057
|
|
1060
|
-
|
1061
|
-
|
1062
|
-
|
1058
|
+
if cache:
|
1059
|
+
result = _get_features_attributes_cache.get(url_internal_wfs, ogc_server.name)
|
1060
|
+
if result != dogpile.cache.api.NO_VALUE:
|
1061
|
+
return result # type: ignore
|
1062
|
+
|
1063
|
+
feature_type, errors = await self._wfs_get_features_type(url_internal_wfs, ogc_server, False, cache)
|
1063
1064
|
|
1064
|
-
|
1065
|
-
return _get_features_attributes_cache(url_internal_wfs, ogc_server_name) # type: ignore
|
1065
|
+
return _get_features_attributes_cache.refresh(url_internal_wfs, ogc_server.name) # type: ignore
|
1066
1066
|
|
1067
1067
|
@view_config(route_name="themes", renderer="json") # type: ignore
|
1068
1068
|
def themes(self) -> Dict[str, Union[Dict[str, Dict[str, Any]], List[str]]]:
|
@@ -1083,11 +1083,12 @@ class Theme:
|
|
1083
1083
|
all_errors: Set[str] = set()
|
1084
1084
|
LOG.debug("Start preload")
|
1085
1085
|
start_time = time.time()
|
1086
|
-
await self.
|
1086
|
+
await self._preload(all_errors)
|
1087
1087
|
LOG.debug("End preload")
|
1088
1088
|
# Don't log if it looks to be already preloaded.
|
1089
1089
|
if (time.time() - start_time) > 1:
|
1090
1090
|
LOG.info("Do preload in %.3fs.", time.time() - start_time)
|
1091
|
+
LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
|
1091
1092
|
result["ogcServers"] = {}
|
1092
1093
|
for ogc_server in models.DBSession.query(main.OGCServer).all():
|
1093
1094
|
nb_layers = (
|
@@ -1096,9 +1097,11 @@ class Theme:
|
|
1096
1097
|
.one()
|
1097
1098
|
)
|
1098
1099
|
if nb_layers[0] == 0:
|
1099
|
-
# QGIS Server
|
1100
|
+
# QGIS Server landing page requires an OGC server that can't be used here.
|
1100
1101
|
continue
|
1101
1102
|
|
1103
|
+
LOG.debug("Process OGC server '%s'", ogc_server.name)
|
1104
|
+
|
1102
1105
|
url_internal_wfs, url, url_wfs = self.get_url_internal_wfs(ogc_server, all_errors)
|
1103
1106
|
|
1104
1107
|
attributes = None
|
@@ -1110,7 +1113,7 @@ class Theme:
|
|
1110
1113
|
)
|
1111
1114
|
if ogc_server.wfs_support and url_internal_wfs:
|
1112
1115
|
attributes, namespace, errors = await self._get_features_attributes(
|
1113
|
-
url_internal_wfs, ogc_server
|
1116
|
+
url_internal_wfs, ogc_server
|
1114
1117
|
)
|
1115
1118
|
# Create a local copy (don't modify the cache)
|
1116
1119
|
if attributes is not None:
|
@@ -1209,3 +1212,34 @@ class Theme:
|
|
1209
1212
|
f"{', '.join([i[0] for i in models.DBSession.query(main.LayerGroup.name).all()])}"
|
1210
1213
|
},
|
1211
1214
|
)
|
1215
|
+
|
1216
|
+
@view_config(route_name="ogc_server_clear_cache", renderer="json") # type: ignore
|
1217
|
+
def ogc_server_clear_cache_view(self) -> Dict[str, Any]:
|
1218
|
+
self._ogc_server_clear_cache(
|
1219
|
+
models.DBSession.query(main.OGCServer).filter_by(id=self.request.matchdict.get("id")).one()
|
1220
|
+
)
|
1221
|
+
came_from = self.request.params.get("came_from")
|
1222
|
+
if came_from:
|
1223
|
+
raise pyramid.httpexceptions.HTTPFound(location=came_from)
|
1224
|
+
return {"success": True}
|
1225
|
+
|
1226
|
+
def _ogc_server_clear_cache(self, ogc_server: main.OGCServer) -> None:
|
1227
|
+
errors: Set[str] = set()
|
1228
|
+
url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors)
|
1229
|
+
if errors:
|
1230
|
+
LOG.error(
|
1231
|
+
"Error while getting the URL of the OGC Server %s:\n%s", ogc_server.id, "\n".join(errors)
|
1232
|
+
)
|
1233
|
+
return
|
1234
|
+
if url_internal_wfs is None:
|
1235
|
+
return
|
1236
|
+
|
1237
|
+
asyncio.run(self._async_cache_invalidate_ogc_server_cb(ogc_server, url_internal_wfs))
|
1238
|
+
|
1239
|
+
async def _async_cache_invalidate_ogc_server_cb(
|
1240
|
+
self, ogc_server: main.OGCServer, url_internal_wfs: Url
|
1241
|
+
) -> None:
|
1242
|
+
# Fill the cache
|
1243
|
+
await self.preload_ogc_server(ogc_server, url_internal_wfs, False)
|
1244
|
+
|
1245
|
+
cache_invalidate_cb()
|