c2cgeoportal-geoportal 2.7.1.156__py2.py3-none-any.whl → 2.8.1.180__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. c2cgeoportal_geoportal/__init__.py +24 -14
  2. c2cgeoportal_geoportal/lib/authentication.py +10 -14
  3. c2cgeoportal_geoportal/lib/caching.py +8 -6
  4. c2cgeoportal_geoportal/lib/checker.py +10 -6
  5. c2cgeoportal_geoportal/lib/common_headers.py +5 -8
  6. c2cgeoportal_geoportal/lib/dbreflection.py +8 -8
  7. c2cgeoportal_geoportal/lib/filter_capabilities.py +5 -1
  8. c2cgeoportal_geoportal/lib/lingua_extractor.py +11 -12
  9. c2cgeoportal_geoportal/lib/loader.py +1 -1
  10. c2cgeoportal_geoportal/lib/oauth2.py +217 -100
  11. c2cgeoportal_geoportal/lib/wmstparsing.py +8 -12
  12. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/Dockerfile +9 -11
  13. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/development.ini +1 -1
  14. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/gunicorn.conf.py +0 -2
  15. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/requirements.txt +1 -1
  16. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/webpack.api.js +6 -4
  17. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/webpack.apps.js +1 -3
  18. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/webpack.commons.js +1 -0
  19. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/{{cookiecutter.package}}_geoportal/__init__.py +1 -6
  20. c2cgeoportal_geoportal/scaffolds/advance_update/{{cookiecutter.project}}/geoportal/CONST_Makefile +0 -20
  21. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/.github/workflows/main.yaml +20 -6
  22. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/.github/workflows/update_l10n.yaml +4 -3
  23. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/Dockerfile +22 -22
  24. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/Makefile +58 -2
  25. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/build +48 -24
  26. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/ci/config.yaml +2 -5
  27. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/ci/docker-compose-check +25 -0
  28. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/ci/requirements.txt +1 -1
  29. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose-db.yaml +26 -0
  30. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose-lib.yaml +53 -26
  31. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose-qgis.yaml +23 -0
  32. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose.override.sample.yaml +0 -1
  33. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/docker-compose.yaml +3 -3
  34. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/env.default +21 -2
  35. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/env.project +9 -0
  36. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/geoportal/vars.yaml +38 -14
  37. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/mapserver/data/Readme.txt +2 -2
  38. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/mapserver/mapserver.conf +15 -0
  39. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/mapserver/mapserver.map.tmpl +2 -3
  40. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A3_Landscape.jrxml +5 -0
  41. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A3_Portrait.jrxml +5 -0
  42. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A4_Landscape.jrxml +5 -0
  43. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/A4_Portrait.jrxml +5 -0
  44. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/print/print-apps/{{cookiecutter.package}}/config.yaml.tmpl +6 -0
  45. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/pyproject.toml +4 -0
  46. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/run_alembic.sh +3 -5
  47. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/scripts/db-backup +1 -1
  48. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/scripts/db-restore +1 -1
  49. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/spell-ignore-words.txt +2 -0
  50. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/tests/__init__.py +0 -0
  51. c2cgeoportal_geoportal/scaffolds/create/{{cookiecutter.project}}/tests/test_app.py +38 -0
  52. c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/.upgrade.yaml +2 -132
  53. c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_CHANGELOG.txt +210 -1097
  54. c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/CONST_create_template/tests/test_testapp.py +48 -0
  55. c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_config-schema.yaml +17 -15
  56. c2cgeoportal_geoportal/scaffolds/update/{{cookiecutter.project}}/geoportal/CONST_vars.yaml +46 -2
  57. c2cgeoportal_geoportal/scripts/c2cupgrade.py +1 -2
  58. c2cgeoportal_geoportal/scripts/pcreate.py +8 -10
  59. c2cgeoportal_geoportal/scripts/theme2fts.py +58 -3
  60. c2cgeoportal_geoportal/views/__init__.py +1 -3
  61. c2cgeoportal_geoportal/views/dynamic.py +1 -1
  62. c2cgeoportal_geoportal/views/entry.py +2 -10
  63. c2cgeoportal_geoportal/views/fulltextsearch.py +1 -1
  64. c2cgeoportal_geoportal/views/geometry_processing.py +3 -3
  65. c2cgeoportal_geoportal/views/layers.py +10 -11
  66. c2cgeoportal_geoportal/views/login.py +63 -8
  67. c2cgeoportal_geoportal/views/mapserverproxy.py +2 -3
  68. c2cgeoportal_geoportal/views/ogcproxy.py +6 -2
  69. c2cgeoportal_geoportal/views/pdfreport.py +4 -4
  70. c2cgeoportal_geoportal/views/printproxy.py +2 -2
  71. c2cgeoportal_geoportal/views/profile.py +1 -1
  72. c2cgeoportal_geoportal/views/proxy.py +2 -4
  73. c2cgeoportal_geoportal/views/raster.py +2 -2
  74. c2cgeoportal_geoportal/views/resourceproxy.py +1 -1
  75. c2cgeoportal_geoportal/views/shortener.py +1 -2
  76. c2cgeoportal_geoportal/views/theme.py +97 -63
  77. c2cgeoportal_geoportal/views/tinyowsproxy.py +3 -12
  78. c2cgeoportal_geoportal/views/vector_tiles.py +1 -1
  79. {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/METADATA +21 -15
  80. {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/RECORD +96 -90
  81. {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/entry_points.txt +1 -0
  82. tests/__init__.py +3 -2
  83. tests/test_cachebuster.py +3 -3
  84. tests/test_caching.py +7 -7
  85. tests/test_checker.py +1 -1
  86. tests/test_decimaljson.py +1 -1
  87. tests/test_headerstween.py +1 -1
  88. tests/test_i18n.py +1 -1
  89. tests/test_init.py +14 -15
  90. tests/test_locale_negociator.py +4 -4
  91. tests/test_mapserverproxy_route_predicate.py +1 -2
  92. tests/test_raster.py +15 -15
  93. tests/test_wmstparsing.py +10 -10
  94. tests/xmlstr.py +1 -3
  95. c2cgeoportal_geoportal/scaffolds/advance_create/{{cookiecutter.project}}/geoportal/tools/extract-messages.js +0 -41
  96. {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/WHEEL +0 -0
  97. {c2cgeoportal_geoportal-2.7.1.156.dist-info → c2cgeoportal_geoportal-2.8.1.180.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2011-2022, Camptocamp SA
1
+ # Copyright (c) 2011-2024, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -28,9 +28,9 @@
28
28
 
29
29
  import json
30
30
  import logging
31
- import secrets
32
31
  import sys
33
32
  import urllib.parse
33
+ from random import Random
34
34
  from typing import Any, Dict, List, Optional, Tuple, Union
35
35
 
36
36
  import pyotp
@@ -88,7 +88,7 @@ class Login:
88
88
  if not hasattr(self.request, "is_valid_referer"):
89
89
  self.request.is_valid_referer = is_valid_referrer(self.request)
90
90
  if not self.request.is_valid_referer:
91
- LOG.info("Invalid referer for %s: %s", self.request.path_qs, repr(self.request.referer))
91
+ LOG.info("Invalid referrer for %s: %s", self.request.path_qs, repr(self.request.referrer))
92
92
 
93
93
  @forbidden_view_config(renderer="login.html") # type: ignore
94
94
  def loginform403(self) -> Union[Dict[str, Any], pyramid.response.Response]:
@@ -341,11 +341,12 @@ class Login:
341
341
 
342
342
  @staticmethod
343
343
  def generate_password() -> str:
344
- all_chars = "123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
344
+ allchars = "123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
345
+ rand = Random() # nosec
345
346
 
346
347
  password = "" # nosec
347
348
  for _ in range(8):
348
- password += secrets.choice(all_chars)
349
+ password += rand.choice(allchars)
349
350
 
350
351
  return password
351
352
 
@@ -397,6 +398,36 @@ class Login:
397
398
 
398
399
  return {"success": True}
399
400
 
401
+ @view_config(route_name="oauth2introspect") # type: ignore
402
+ def oauth2introspect(self) -> pyramid.response.Response:
403
+ LOG.debug(
404
+ "Call OAuth create_introspect_response with:\nurl: %s\nmethod: %s\nbody:\n%s",
405
+ self.request.current_route_url(_query=self.request.GET),
406
+ self.request.method,
407
+ self.request.body,
408
+ )
409
+ headers, body, status = oauth2.get_oauth_client(
410
+ self.request.registry.settings
411
+ ).create_introspect_response(
412
+ self.request.current_route_url(_query=self.request.GET),
413
+ self.request.method,
414
+ self.request.body,
415
+ self.request.headers,
416
+ )
417
+ LOG.debug("OAuth create_introspect_response return status: %s", status)
418
+
419
+ # All requests to /token will return a json response, no redirection.
420
+ if status != 200:
421
+ if body:
422
+ raise exception_response(status, detail=body)
423
+ raise exception_response(status)
424
+ return set_common_headers(
425
+ self.request,
426
+ "login",
427
+ Cache.PRIVATE_NO,
428
+ response=Response(body, headers=headers.items()),
429
+ )
430
+
400
431
  @view_config(route_name="oauth2token") # type: ignore
401
432
  def oauth2token(self) -> pyramid.response.Response:
402
433
  LOG.debug(
@@ -414,9 +445,6 @@ class Login:
414
445
  )
415
446
  LOG.debug("OAuth create_token_response return status: %s", status)
416
447
 
417
- if hasattr(self.request, "tm"):
418
- self.request.tm.commit()
419
-
420
448
  # All requests to /token will return a json response, no redirection.
421
449
  if status != 200:
422
450
  if body:
@@ -429,6 +457,33 @@ class Login:
429
457
  response=Response(body, headers=headers.items()),
430
458
  )
431
459
 
460
+ @view_config(route_name="oauth2revoke_token") # type: ignore
461
+ def oauth2revoke_token(self) -> pyramid.response.Response:
462
+ LOG.debug(
463
+ "Call OAuth create_revocation_response with:\nurl: %s\nmethod: %s\nbody:\n%s",
464
+ self.request.create_revocation_response(_query=self.request.GET),
465
+ self.request.method,
466
+ self.request.body,
467
+ )
468
+ headers, body, status = oauth2.get_oauth_client(
469
+ self.request.registry.settings
470
+ ).create_authorize_response(
471
+ self.request.current_route_url(_query=self.request.GET),
472
+ self.request.method,
473
+ self.request.body,
474
+ self.request.headers,
475
+ )
476
+ if status != 200:
477
+ if body:
478
+ raise exception_response(status, detail=body)
479
+ raise exception_response(status)
480
+ return set_common_headers(
481
+ self.request,
482
+ "login",
483
+ Cache.PRIVATE_NO,
484
+ response=Response(body, headers=headers.items()),
485
+ )
486
+
432
487
  @view_config(route_name="oauth2loginform", renderer="login.html") # type: ignore
433
488
  def oauth2loginform(self) -> Dict[str, Any]:
434
489
  set_common_headers(self.request, "login", Cache.PUBLIC)
@@ -29,7 +29,7 @@
29
29
  import logging
30
30
  from typing import Any, Dict, Set
31
31
 
32
- from pyramid.httpexceptions import HTTPFound, HTTPInternalServerError, HTTPUnauthorized
32
+ from pyramid.httpexceptions import HTTPForbidden, HTTPFound, HTTPInternalServerError, HTTPUnauthorized
33
33
  from pyramid.request import Request
34
34
  from pyramid.response import Response
35
35
  from pyramid.view import view_config
@@ -67,7 +67,7 @@ class MapservProxy(OGCProxy):
67
67
  raise HTTPUnauthorized(
68
68
  headers={"WWW-Authenticate": 'Basic realm="Access to restricted layers"'}
69
69
  )
70
- raise HTTPUnauthorized(headers={"WWW-Authenticate": 'Bearer realm="Access to restricted layers"'})
70
+ raise HTTPForbidden("Basic auth is not enabled")
71
71
 
72
72
  # We have a user logged in. We need to set group_id and possible layer_name in the params. We set
73
73
  # layer_name when either QUERY_PARAMS or LAYERS is set in the WMS params, i.e. for GetMap and
@@ -108,7 +108,6 @@ class MapservProxy(OGCProxy):
108
108
  self.params = {}
109
109
  else:
110
110
  if self.ogc_server.type != main.OGCSERVER_TYPE_QGISSERVER or "user_id" not in self.params:
111
-
112
111
  use_cache = self.lower_params["request"] in ("getlegendgraphic",)
113
112
 
114
113
  # no user_id and role_id or cached queries
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2011-2022, Camptocamp SA
1
+ # Copyright (c) 2011-2023, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -60,6 +60,8 @@ class OGCProxy(Proxy):
60
60
  if "user_id" in self.params:
61
61
  del self.params["user_id"]
62
62
 
63
+ main_ogc_server = self.request.registry.settings.get("main_ogc_server")
64
+
63
65
  self.lower_params = self._get_lower_params(self.params)
64
66
 
65
67
  # We need original case for OGCSERVER parameter value
@@ -69,11 +71,13 @@ class OGCProxy(Proxy):
69
71
  self.ogc_server = self._get_ogcserver_byname(request.matchdict["ogcserver"])
70
72
  elif "ogcserver" in self.lower_key_params:
71
73
  self.ogc_server = self._get_ogcserver_byname(self.lower_key_params["ogcserver"])
74
+ elif main_ogc_server is not None:
75
+ self.ogc_server = self._get_ogcserver_byname(main_ogc_server)
72
76
  elif not has_default_ogc_server:
73
77
  raise HTTPBadRequest("The querystring argument 'ogcserver' is required")
74
78
 
75
79
  @CACHE_REGION.cache_on_arguments() # type: ignore
76
- def _get_ogcserver_byname(self, name: str) -> main.OGCServer: # pylint: disable=no-self-use
80
+ def _get_ogcserver_byname(self, name: str) -> main.OGCServer:
77
81
  try:
78
82
  result = DBSession.query(main.OGCServer).filter(main.OGCServer.name == name).one()
79
83
  DBSession.expunge(result)
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2011-2021, Camptocamp SA
1
+ # Copyright (c) 2011-2024, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -106,14 +106,14 @@ class PdfReport(OGCProxy):
106
106
  self.layername = self.request.matchdict["layername"]
107
107
  layer_config = self.config["layers"].get(self.layername)
108
108
 
109
+ if layer_config is None:
110
+ raise HTTPBadRequest("Layer not found: " + self.layername)
111
+
109
112
  multiple = layer_config.get("multiple", False)
110
113
  ids = self.request.matchdict["ids"]
111
114
  if multiple:
112
115
  ids = ids.split(",")
113
116
 
114
- if layer_config is None:
115
- raise HTTPBadRequest("Layer not found")
116
-
117
117
  features_ids = (
118
118
  [self.layername + "." + id_ for id_ in ids] if multiple else [self.layername + "." + ids]
119
119
  )
@@ -68,8 +68,8 @@ class PrintProxy(Proxy):
68
68
  )
69
69
 
70
70
  response = self._build_response(resp, content, Cache.PRIVATE, "print")
71
- # Mapfish print will check the referer header to return the capabilities.
72
- response.vary += ("Referer",)
71
+ # Mapfish print will check the referrer header to return the capabilities.
72
+ response.vary += ("Referrer", "Referer")
73
73
  return response
74
74
 
75
75
  @CACHE_REGION.cache_on_arguments() # type: ignore
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2012-2021, Camptocamp SA
1
+ # Copyright (c) 2012-2023, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,3 @@
1
- # -*- coding: utf-8 -*-
2
-
3
1
  # Copyright (c) 2011-2024, Camptocamp SA
4
2
  # All rights reserved.
5
3
 
@@ -47,7 +45,7 @@ CACHE_REGION = get_region("std")
47
45
 
48
46
 
49
47
  class Proxy:
50
- """Some methodes used by all the proxy."""
48
+ """Some methods used by all the proxy."""
51
49
 
52
50
  def __init__(self, request: pyramid.request.Request):
53
51
  self.request = request
@@ -252,7 +250,7 @@ class Proxy:
252
250
 
253
251
  @staticmethod
254
252
  def _get_lower_params(params: Dict[str, str]) -> Dict[str, str]:
255
- return dict((k.lower(), str(v).lower()) for k, v in params.items())
253
+ return {k.lower(): str(v).lower() for k, v in params.items()}
256
254
 
257
255
  def get_headers(self) -> Dict[str, str]:
258
256
  headers: Dict[str, str] = self.request.headers
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2012-2021, Camptocamp SA
1
+ # Copyright (c) 2012-2023, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -167,7 +167,7 @@ class Raster:
167
167
  result = None if result == layer.get("nodata", dataset.nodata) else result
168
168
  else:
169
169
  LOG.debug(
170
- "Out of index for layer: %s (%s), " "lon/lat: %dx%d, index: %dx%d, shape: %dx%d.",
170
+ "Out of index for layer: %s (%s), lon/lat: %dx%d, index: %dx%d, shape: %dx%d.",
171
171
  name,
172
172
  layer["file"],
173
173
  lon,
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2011-2021, Camptocamp SA
1
+ # Copyright (c) 2011-2023, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2013-2022, Camptocamp SA
1
+ # Copyright (c) 2013-2023, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -71,7 +71,6 @@ class Shortener:
71
71
 
72
72
  @view_config(route_name="shortener_create", renderer="json") # type: ignore
73
73
  def create(self) -> Dict[str, str]:
74
-
75
74
  if "url" not in self.request.params:
76
75
  raise HTTPBadRequest("The parameter url is required")
77
76
 
@@ -1,4 +1,4 @@
1
- # Copyright (c) 2011-2022, Camptocamp SA
1
+ # Copyright (c) 2011-2024, Camptocamp SA
2
2
  # All rights reserved.
3
3
 
4
4
  # Redistribution and use in source and binary forms, with or without
@@ -38,6 +38,7 @@ from math import sqrt
38
38
  from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
39
39
 
40
40
  import dogpile.cache.api
41
+ import pyramid.httpexceptions
41
42
  import pyramid.request
42
43
  import requests
43
44
  import sqlalchemy
@@ -52,7 +53,7 @@ from sqlalchemy.orm.exc import NoResultFound
52
53
 
53
54
  from c2cgeoportal_commons import models
54
55
  from c2cgeoportal_commons.lib.url import Url, get_url2
55
- from c2cgeoportal_commons.models import main
56
+ from c2cgeoportal_commons.models import cache_invalidate_cb, main
56
57
  from c2cgeoportal_geoportal.lib import get_roles_id, get_typed, get_types_map, is_intranet
57
58
  from c2cgeoportal_geoportal.lib.caching import get_region
58
59
  from c2cgeoportal_geoportal.lib.common_headers import Cache, set_common_headers
@@ -63,27 +64,31 @@ from c2cgeoportal_geoportal.lib.layers import (
63
64
  get_protected_layers_query,
64
65
  )
65
66
  from c2cgeoportal_geoportal.lib.wmstparsing import TimeInformation, parse_extent
66
- from c2cgeoportal_geoportal.views import restrict_headers
67
67
  from c2cgeoportal_geoportal.views.layers import get_layer_metadata
68
68
 
69
69
  LOG = logging.getLogger(__name__)
70
70
  CACHE_REGION = get_region("std")
71
+ CACHE_OGC_SERVER_REGION = get_region("ogc-server")
71
72
  TIMEOUT = int(os.environ.get("C2CGEOPORTAL_THEME_TIMEOUT", "300"))
72
73
 
73
74
  Metadata = Union[str, int, float, bool, List[Any], Dict[str, Any]]
74
75
 
75
76
 
76
- def get_http_cached(http_options: Dict[str, Any], url: str, headers: Dict[str, str]) -> Tuple[bytes, str]:
77
+ def get_http_cached(
78
+ http_options: Dict[str, Any], url: str, headers: Dict[str, str], cache: bool = True
79
+ ) -> Tuple[bytes, str]:
77
80
  """Get the content of the URL with a cash (dogpile)."""
78
81
 
79
- @CACHE_REGION.cache_on_arguments() # type: ignore
82
+ @CACHE_OGC_SERVER_REGION.cache_on_arguments() # type: ignore
80
83
  def do_get_http_cached(url: str) -> Tuple[bytes, str]:
81
84
  response = requests.get(url, headers=headers, timeout=TIMEOUT, **http_options)
82
85
  response.raise_for_status()
83
- LOG.info("Get url '%s' in %.1fs.", url, response.elapsed.total_seconds())
86
+ LOG.info("Get URL '%s' in %.1fs.", url, response.elapsed.total_seconds())
84
87
  return response.content, response.headers.get("Content-Type", "")
85
88
 
86
- return do_get_http_cached(url) # type: ignore
89
+ if cache:
90
+ return do_get_http_cached(url) # type: ignore
91
+ return do_get_http_cached.refresh(url) # type: ignore
87
92
 
88
93
 
89
94
  class DimensionInformation:
@@ -143,8 +148,6 @@ class Theme:
143
148
  self.request = request
144
149
  self.settings = request.registry.settings
145
150
  self.http_options = self.settings.get("http_options", {})
146
- self.headers_whitelist = self.settings.get("headers_whitelist", [])
147
- self.headers_blacklist = self.settings.get("headers_blacklist", [])
148
151
  self.metadata_type = get_types_map(
149
152
  self.settings.get("admin_interface", {}).get("available_metadata", [])
150
153
  )
@@ -179,14 +182,16 @@ class Theme:
179
182
  return metadatas
180
183
 
181
184
  async def _wms_getcap(
182
- self, ogc_server: main.OGCServer, preload: bool = False
185
+ self, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True
183
186
  ) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]:
184
- @CACHE_REGION.cache_on_arguments() # type: ignore
187
+ LOG.debug("Get the WMS Capabilities of '%s', preload: %s, cache: %s", ogc_server.name, preload, cache)
188
+
189
+ @CACHE_OGC_SERVER_REGION.cache_on_arguments() # type: ignore
185
190
  def build_web_map_service(ogc_server_id: int) -> Tuple[Optional[Dict[str, Dict[str, Any]]], Set[str]]:
186
191
  del ogc_server_id # Just for cache
187
192
 
188
193
  if url is None:
189
- raise RuntimeError("Url is None")
194
+ raise RuntimeError("URL is None")
190
195
 
191
196
  version = url.query.get("VERSION", "1.1.1")
192
197
  layers = {}
@@ -220,16 +225,16 @@ class Theme:
220
225
  }
221
226
 
222
227
  del wms
223
- LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
224
228
 
225
229
  return {"layers": layers}, set()
226
230
 
227
- result = build_web_map_service.get(ogc_server.id)
228
- if result != dogpile.cache.api.NO_VALUE:
229
- return result # type: ignore
231
+ if cache:
232
+ result = build_web_map_service.get(ogc_server.id)
233
+ if result != dogpile.cache.api.NO_VALUE:
234
+ return result # type: ignore
230
235
 
231
236
  try:
232
- url, content, errors = await self._wms_getcap_cached(ogc_server)
237
+ url, content, errors = await self._wms_getcap_cached(ogc_server, cache=cache)
233
238
  except requests.exceptions.RequestException as exception:
234
239
  error = (
235
240
  f"Unable to get the WMS Capabilities for OGC server '{ogc_server.name}', "
@@ -240,10 +245,10 @@ class Theme:
240
245
  if errors or preload:
241
246
  return None, errors
242
247
 
243
- return build_web_map_service(ogc_server.id) # type: ignore
248
+ return build_web_map_service.refresh(ogc_server.id) # type: ignore
244
249
 
245
250
  async def _wms_getcap_cached(
246
- self, ogc_server: main.OGCServer
251
+ self, ogc_server: main.OGCServer, cache: bool = True
247
252
  ) -> Tuple[Optional[Url], Optional[bytes], Set[str]]:
248
253
  errors: Set[str] = set()
249
254
  url = get_url2(f"The OGC server '{ogc_server.name}'", ogc_server.url, self.request, errors)
@@ -269,23 +274,15 @@ class Theme:
269
274
 
270
275
  LOG.debug("Get WMS GetCapabilities for URL: %s", url)
271
276
 
272
- # Forward request to target (without Host Header)
273
- headers = dict(self.request.headers)
277
+ headers = {}
274
278
 
275
279
  # Add headers for Geoserver
276
280
  if ogc_server.auth == main.OGCSERVER_AUTH_GEOSERVER:
277
281
  headers["sec-username"] = "root"
278
282
  headers["sec-roles"] = "root"
279
283
 
280
- if url.hostname != "localhost" and "Host" in headers:
281
- headers.pop("Host")
282
-
283
- headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist)
284
-
285
284
  try:
286
- content, content_type = await asyncio.get_event_loop().run_in_executor(
287
- None, get_http_cached, self.http_options, url, headers
288
- )
285
+ content, content_type = get_http_cached(self.http_options, url.url(), headers, cache=cache)
289
286
  except Exception:
290
287
  error = f"Unable to GetCapabilities from URL {url}"
291
288
  errors.add(error)
@@ -386,8 +383,6 @@ class Theme:
386
383
  layer_info = {"id": layer.id, "name": layer.name, "metadata": self._get_metadata_list(layer, errors)}
387
384
  if re.search("[/?#]", layer.name):
388
385
  errors.add(f"The layer has an unsupported name '{layer.name}'.")
389
- if isinstance(layer, main.LayerWMS) and re.search("[/?#]", layer.layer):
390
- errors.add(f"The layer has an unsupported layers '{layer.layer}'.")
391
386
  if layer.geo_table:
392
387
  errors |= self._fill_editable(layer_info, layer)
393
388
  if mixed:
@@ -858,7 +853,7 @@ class Theme:
858
853
  return None if self.request.user is None else {role.id for role in self.request.user.roles}
859
854
 
860
855
  async def _wfs_get_features_type(
861
- self, wfs_url: Url, ogc_server_name: str, preload: bool = False
856
+ self, wfs_url: Url, ogc_server: main.OGCServer, preload: bool = False, cache: bool = True
862
857
  ) -> Tuple[Optional[etree.Element], Set[str]]:
863
858
  errors = set()
864
859
 
@@ -872,23 +867,23 @@ class Theme:
872
867
  }
873
868
  )
874
869
 
875
- LOG.debug("WFS DescribeFeatureType for the URL: %s", wfs_url)
870
+ LOG.debug(
871
+ "Get the WFS DescribeFeatureType of '%s', preload: %s, cache: %s", ogc_server.name, preload, cache
872
+ )
876
873
 
877
- # forward request to target (without Host Header)
878
- headers = dict(self.request.headers)
879
- if wfs_url.hostname != "localhost" and "Host" in headers:
880
- headers.pop("Host")
874
+ headers = {}
881
875
 
882
- headers = restrict_headers(headers, self.headers_whitelist, self.headers_blacklist)
876
+ # Add headers for Geoserver
877
+ if ogc_server.auth == main.OGCSERVER_AUTH_GEOSERVER:
878
+ headers["sec-username"] = "root"
879
+ headers["sec-roles"] = "root"
883
880
 
884
881
  try:
885
- content, _ = await asyncio.get_event_loop().run_in_executor(
886
- None, get_http_cached, self.http_options, wfs_url, headers
887
- )
882
+ content, _ = get_http_cached(self.http_options, wfs_url.url(), headers, cache)
888
883
  except requests.exceptions.RequestException as exception:
889
884
  error = (
890
885
  f"Unable to get WFS DescribeFeatureType from the URL '{wfs_url.url()}' for "
891
- f"OGC server {ogc_server_name}, "
886
+ f"OGC server {ogc_server.name}, "
892
887
  + (
893
888
  f"return the error: {exception.response.status_code} {exception.response.reason}"
894
889
  if exception.response is not None
@@ -901,7 +896,7 @@ class Theme:
901
896
  except Exception:
902
897
  error = (
903
898
  f"Unable to get WFS DescribeFeatureType from the URL {wfs_url} for "
904
- f"OGC server {ogc_server_name}"
899
+ f"OGC server {ogc_server.name}"
905
900
  )
906
901
  errors.add(error)
907
902
  LOG.exception(error)
@@ -948,10 +943,10 @@ class Theme:
948
943
  url_internal_wfs = url_wfs
949
944
  return url_internal_wfs, url, url_wfs
950
945
 
951
- async def preload(self, errors: Set[str]) -> None:
946
+ async def _preload(self, errors: Set[str]) -> None:
952
947
  tasks = set()
953
948
  for ogc_server in models.DBSession.query(main.OGCServer).all():
954
- # Don't load unused OGC servers, required for Landigpage, because the related OGC server
949
+ # Don't load unused OGC servers, required for landing page, because the related OGC server
955
950
  # will be on error in those functions.
956
951
  nb_layers = (
957
952
  models.DBSession.query(sqlalchemy.func.count(main.LayerWMS.id))
@@ -963,26 +958,30 @@ class Theme:
963
958
  LOG.debug("Preload OGC server '%s'", ogc_server.name)
964
959
  url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors)
965
960
  if url_internal_wfs is not None:
966
- if ogc_server.wfs_support:
967
- tasks.add(self._wfs_get_features_type(url_internal_wfs, ogc_server.name, True))
968
- tasks.add(self._wms_getcap(ogc_server, True))
961
+ tasks.add(self.preload_ogc_server(ogc_server, url_internal_wfs))
969
962
 
970
963
  await asyncio.gather(*tasks)
971
964
 
965
+ async def preload_ogc_server(
966
+ self, ogc_server: main.OGCServer, url_internal_wfs: Url, cache: bool = True
967
+ ) -> None:
968
+ if ogc_server.wfs_support:
969
+ await self._get_features_attributes(url_internal_wfs, ogc_server, cache=cache)
970
+ await self._wms_getcap(ogc_server, False, cache=cache)
971
+
972
972
  async def _get_features_attributes(
973
- self, url_internal_wfs: Url, ogc_server_name: str
973
+ self, url_internal_wfs: Url, ogc_server: main.OGCServer, cache: bool = True
974
974
  ) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]:
975
- @CACHE_REGION.cache_on_arguments() # type: ignore
975
+ @CACHE_OGC_SERVER_REGION.cache_on_arguments() # type: ignore
976
976
  def _get_features_attributes_cache(
977
977
  url_internal_wfs: Url, ogc_server_name: str
978
978
  ) -> Tuple[Optional[Dict[str, Dict[Any, Dict[str, Any]]]], Optional[str], Set[str]]:
979
979
  del url_internal_wfs # Just for cache
980
980
  all_errors: Set[str] = set()
981
- LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
982
981
  if errors:
983
982
  all_errors |= errors
984
983
  return None, None, all_errors
985
- assert feature_type
984
+ assert feature_type is not None
986
985
  namespace: str = feature_type.attrib.get("targetNamespace")
987
986
  types: Dict[Any, Dict[str, Any]] = {}
988
987
  elements = {}
@@ -1041,9 +1040,8 @@ class Theme:
1041
1040
  attributes[name] = types[type_]
1042
1041
  elif (type_ == "Character") and (name + "Type") in types:
1043
1042
  LOG.debug(
1044
- "Due MapServer strange result the type 'ms:Character' is fallbacked to type '%sType'"
1045
- " for feature '%s', This is a strange comportement of MapServer when we use the "
1046
- 'METADATA "gml_types" "auto"',
1043
+ 'Due to MapServer weird behavior when using METADATA "gml_types" "auto"'
1044
+ "the type 'ms:Character' is returned as type '%sType' for feature '%s'.",
1047
1045
  name,
1048
1046
  name,
1049
1047
  )
@@ -1057,12 +1055,14 @@ class Theme:
1057
1055
 
1058
1056
  return attributes, namespace, all_errors
1059
1057
 
1060
- result = _get_features_attributes_cache.get(url_internal_wfs, ogc_server_name)
1061
- if result != dogpile.cache.api.NO_VALUE:
1062
- return result # type: ignore
1058
+ if cache:
1059
+ result = _get_features_attributes_cache.get(url_internal_wfs, ogc_server.name)
1060
+ if result != dogpile.cache.api.NO_VALUE:
1061
+ return result # type: ignore
1062
+
1063
+ feature_type, errors = await self._wfs_get_features_type(url_internal_wfs, ogc_server, False, cache)
1063
1064
 
1064
- feature_type, errors = await self._wfs_get_features_type(url_internal_wfs, ogc_server_name)
1065
- return _get_features_attributes_cache(url_internal_wfs, ogc_server_name) # type: ignore
1065
+ return _get_features_attributes_cache.refresh(url_internal_wfs, ogc_server.name) # type: ignore
1066
1066
 
1067
1067
  @view_config(route_name="themes", renderer="json") # type: ignore
1068
1068
  def themes(self) -> Dict[str, Union[Dict[str, Dict[str, Any]], List[str]]]:
@@ -1083,11 +1083,12 @@ class Theme:
1083
1083
  all_errors: Set[str] = set()
1084
1084
  LOG.debug("Start preload")
1085
1085
  start_time = time.time()
1086
- await self.preload(all_errors)
1086
+ await self._preload(all_errors)
1087
1087
  LOG.debug("End preload")
1088
1088
  # Don't log if it looks to be already preloaded.
1089
1089
  if (time.time() - start_time) > 1:
1090
1090
  LOG.info("Do preload in %.3fs.", time.time() - start_time)
1091
+ LOG.debug("Run garbage collection: %s", ", ".join([str(gc.collect(n)) for n in range(3)]))
1091
1092
  result["ogcServers"] = {}
1092
1093
  for ogc_server in models.DBSession.query(main.OGCServer).all():
1093
1094
  nb_layers = (
@@ -1096,9 +1097,11 @@ class Theme:
1096
1097
  .one()
1097
1098
  )
1098
1099
  if nb_layers[0] == 0:
1099
- # QGIS Server langing page requires an OGC server that can't be used here.
1100
+ # QGIS Server landing page requires an OGC server that can't be used here.
1100
1101
  continue
1101
1102
 
1103
+ LOG.debug("Process OGC server '%s'", ogc_server.name)
1104
+
1102
1105
  url_internal_wfs, url, url_wfs = self.get_url_internal_wfs(ogc_server, all_errors)
1103
1106
 
1104
1107
  attributes = None
@@ -1110,7 +1113,7 @@ class Theme:
1110
1113
  )
1111
1114
  if ogc_server.wfs_support and url_internal_wfs:
1112
1115
  attributes, namespace, errors = await self._get_features_attributes(
1113
- url_internal_wfs, ogc_server.name
1116
+ url_internal_wfs, ogc_server
1114
1117
  )
1115
1118
  # Create a local copy (don't modify the cache)
1116
1119
  if attributes is not None:
@@ -1209,3 +1212,34 @@ class Theme:
1209
1212
  f"{', '.join([i[0] for i in models.DBSession.query(main.LayerGroup.name).all()])}"
1210
1213
  },
1211
1214
  )
1215
+
1216
+ @view_config(route_name="ogc_server_clear_cache", renderer="json") # type: ignore
1217
+ def ogc_server_clear_cache_view(self) -> Dict[str, Any]:
1218
+ self._ogc_server_clear_cache(
1219
+ models.DBSession.query(main.OGCServer).filter_by(id=self.request.matchdict.get("id")).one()
1220
+ )
1221
+ came_from = self.request.params.get("came_from")
1222
+ if came_from:
1223
+ raise pyramid.httpexceptions.HTTPFound(location=came_from)
1224
+ return {"success": True}
1225
+
1226
+ def _ogc_server_clear_cache(self, ogc_server: main.OGCServer) -> None:
1227
+ errors: Set[str] = set()
1228
+ url_internal_wfs, _, _ = self.get_url_internal_wfs(ogc_server, errors)
1229
+ if errors:
1230
+ LOG.error(
1231
+ "Error while getting the URL of the OGC Server %s:\n%s", ogc_server.id, "\n".join(errors)
1232
+ )
1233
+ return
1234
+ if url_internal_wfs is None:
1235
+ return
1236
+
1237
+ asyncio.run(self._async_cache_invalidate_ogc_server_cb(ogc_server, url_internal_wfs))
1238
+
1239
+ async def _async_cache_invalidate_ogc_server_cb(
1240
+ self, ogc_server: main.OGCServer, url_internal_wfs: Url
1241
+ ) -> None:
1242
+ # Fill the cache
1243
+ await self.preload_ogc_server(ogc_server, url_internal_wfs, False)
1244
+
1245
+ cache_invalidate_cb()