spinta 0.2.dev27__py3-none-any.whl → 0.2.dev28__py3-none-any.whl

spinta/api/__init__.py

@@ -50,7 +50,7 @@ from spinta.exceptions import (
     error_response,
 )
 from spinta.formats.html.helpers import get_templates
-from spinta.middlewares import ContextMiddleware, StrictTransportSecurityMiddleware
+from spinta.middlewares import ContextMiddleware, PathNormalizationMiddleware, StrictTransportSecurityMiddleware
 from spinta.urlparams import Version, get_response_type
 
 log = logging.getLogger(__name__)
@@ -370,7 +370,7 @@ async def error(request, exc):
     elif isinstance(exc, BaseError):
         status_code = exc.status_code
         errors = [error_response(exc)]
-        headers = exc.headers
+        headers = dict(exc.headers)
     else:
         if isinstance(exc, HTTPException):
             status_code = exc.status_code
@@ -400,6 +400,11 @@ async def error(request, exc):
 
     response = {"errors": errors}
 
+    # Error responses can reflect request input (e.g. the requested path) and
+    # must never be stored by shared caches, otherwise they can be used for
+    # web cache poisoning.
+    headers["Cache-Control"] = "no-store"
+
     fmt = get_response_type(request.state.context, request)
     if fmt == "json" or fmt is None:
         return JSONResponse(
@@ -467,6 +472,7 @@ def init(context: Context):
             StrictTransportSecurityMiddleware,
             value=config.http_strict_transport_security,
         ),
+        Middleware(PathNormalizationMiddleware),
         Middleware(ContextMiddleware, context=context),
     ]
 

spinta/manifests/tabular/helpers.py

@@ -653,9 +653,20 @@ class PropertyReader(TabularReader):
     enums: Set[str]
 
     def read(self, row: Dict[str, str]) -> None:
-        complete_structure, parent_structure, prop_name = _extract_and_create_parent_data(self, row, row["property"])
+        # Parse the data type once, here at the boundary, and pass the parsed
+        # result inward. Downstream code must not re-parse `row["type"]` or
+        # compare against the raw string: modifiers (`required`, `unique`) and
+        # arguments would make a type such as `backref required` or
+        # `object required` unrecognisable. Parent and nesting resolution only
+        # needs the bare type, while `_handle_datatype` keeps the modifiers.
+        dtype = _resolve_dtype(self, row)
+        bare_row = {**row, "type": dtype["type"]}
+        complete_structure, parent_structure, prop_name = _extract_and_create_parent_data(
+            self, bare_row, bare_row["property"]
+        )
+
+        prop_data = _handle_datatype(self, row, dtype)
 
-        prop_data = _handle_datatype(self, row)
         prop_name = _combine_parent_with_prop(prop_name, prop_data, parent_structure, complete_structure)
 
         # Edge case where there is no nesting, need to couple `prop_data` with `complete_structure`
@@ -748,8 +759,11 @@ def _initial_text_property_schema(given_name: str, dtype: dict, row: dict):
     return result
 
 
-def _datatype_handler(reader: PropertyReader, row: dict, initial_data_loader: Callable[[str, dict, dict], dict]):
-    dtype: dict = _resolve_dtype(reader, row)
+def _datatype_handler(
+    reader: PropertyReader, row: dict, initial_data_loader: Callable[[str, dict, dict], dict], dtype: dict = None
+):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     given_name = row["property"]
     reader.name = _clean_up_prop_name(row["property"].split(".")[-1])
 
@@ -759,7 +773,7 @@ def _datatype_handler(reader: PropertyReader, row: dict, initial_data_loader: Ca
             f"Property {reader.name!r} must be defined in a model context. "
             f"Now it is defined in {context.name!r} {context.type} context."
         )
-    _check_if_property_already_set(reader, row, given_name)
+    _check_if_property_already_set(reader, dtype["type"], given_name)
 
     if reader.state.base and not dtype["type"]:
         dtype["type"] = "inherit"
@@ -804,8 +818,9 @@ def _datatype_handler(reader: PropertyReader, row: dict, initial_data_loader: Ca
     return new_data
 
 
-def _string_datatype_handler(reader: PropertyReader, row: dict):
-    dtype: dict = _resolve_dtype(reader, row)
+def _string_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     given_name = row["property"]
     reader.name = _clean_up_prop_name(row["property"].split(".")[-1])
 
@@ -815,7 +830,7 @@ def _string_datatype_handler(reader: PropertyReader, row: dict):
             f"Property {reader.name!r} must be defined in a model context. "
             f"Now it is defined in {context.name!r} {context.type} context."
         )
-    existing_data = _check_if_property_already_set(reader, row, given_name)
+    existing_data = _check_if_property_already_set(reader, dtype["type"], given_name)
     if dtype["type"] == DataTypeEnum.TEXT.value and existing_data:
         reader.error(
             f"Property {reader.name!r} with the same name is already "
@@ -835,8 +850,9 @@ def _string_datatype_handler(reader: PropertyReader, row: dict):
     return new_data
 
 
-def _text_datatype_handler(reader: PropertyReader, row: dict):
-    dtype: dict = _resolve_dtype(reader, row)
+def _text_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     given_name = row["property"]
     reader.name = _clean_up_prop_name(row["property"].split(".")[-1])
 
@@ -846,7 +862,7 @@ def _text_datatype_handler(reader: PropertyReader, row: dict):
             f"Property {reader.name!r} must be defined in a model context. "
             f"Now it is defined in {context.name!r} {context.type} context."
         )
-    result = _check_if_property_already_set(reader, row, given_name)
+    result = _check_if_property_already_set(reader, dtype["type"], given_name)
     if not (result and result["explicitly_given"] is False and result["type"] == DataTypeEnum.TEXT.value or not result):
         reader.error(
             f"Property {reader.name!r} with the same name is already "
@@ -893,22 +909,23 @@ def _text_datatype_handler(reader: PropertyReader, row: dict):
     return new_data
 
 
-def _default_datatype_handler(reader: PropertyReader, row: dict):
-    return _datatype_handler(reader, row, _initial_normal_property_schema)
+def _default_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    return _datatype_handler(reader, row, _initial_normal_property_schema, dtype)
 
 
-def _array_datatype_handler(reader: PropertyReader, row: dict):
-    return _datatype_handler(reader, row, _initial_array_property_schema)
+def _array_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    return _datatype_handler(reader, row, _initial_array_property_schema, dtype)
 
 
-def _partial_datatype_handler(reader: PropertyReader, row: dict):
-    return _datatype_handler(reader, row, _initial_partial_property_schema)
+def _partial_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    return _datatype_handler(reader, row, _initial_partial_property_schema, dtype)
 
 
-def _handle_datatype(reader: PropertyReader, row: dict):
-    dtype: dict = _resolve_dtype(reader, row)
+def _handle_datatype(reader: PropertyReader, row: dict, dtype: dict = None):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     handler = DATATYPE_HANDLERS.get(dtype["type"], DATATYPE_HANDLERS["_default"])
-    return handler(reader, row)
+    return handler(reader, row, dtype)
 
 
 DATATYPE_HANDLERS = {
@@ -1204,7 +1221,7 @@ def _extract_children_from_nested(base: dict, children_name: str) -> dict:
     return base
 
 
-def _check_if_property_already_set(reader: PropertyReader, given_row: dict, full_name: str):
+def _check_if_property_already_set(reader: PropertyReader, given_type: str, full_name: str):
     # Treat '@' as normal '.', since '_extract_children_from_nested' is able to extract based on type
     split = full_name.replace("@", ".").split(".")
     base = {}
@@ -1250,11 +1267,14 @@ def _check_if_property_already_set(reader: PropertyReader, given_row: dict, full
             f"Property {full_name!r} with the same name is already defined for this {reader.state.model.name!r} model."
         )
 
+    # `given_type` is the bare data type (modifiers and arguments already
+    # stripped by the caller), so a nesting type such as `object` is recognised
+    # even when it was declared as `object required`.
     if base and (
-        (base["type"] in ALLOWED_PARTIAL_TYPES and given_row["type"] not in ALLOWED_PARTIAL_TYPES)
-        or (base["type"] in ALLOWED_ARRAY_TYPES and given_row["type"] not in ALLOWED_ARRAY_TYPES)
+        (base["type"] in ALLOWED_PARTIAL_TYPES and given_type not in ALLOWED_PARTIAL_TYPES)
+        or (base["type"] in ALLOWED_ARRAY_TYPES and given_type not in ALLOWED_ARRAY_TYPES)
     ):
-        raise DataTypeCannotBeUsedForNesting(dtype=given_row["type"])
+        raise DataTypeCannotBeUsedForNesting(dtype=given_type)
     return base
 
 

spinta/middlewares.py

@@ -1,8 +1,66 @@
+import posixpath
+
+from starlette.responses import JSONResponse
 from starlette.types import ASGIApp, Message, Receive, Scope, Send
 
 from spinta.components import Context
 
 
+def _is_normalized_path(scope: Scope) -> bool:
+    raw_path = (scope.get("raw_path") or b"").lower()
+    # Percent-encoded slashes, backslashes and dots decode into extra path
+    # delimiters and dot segments that shared caches do not see when they
+    # build the cache key.
+    if b"%2f" in raw_path or b"%5c" in raw_path or b"%2e" in raw_path:
+        return False
+    path = scope["path"]
+    if "\\" in path:
+        return False
+    normalized = posixpath.normpath(path)
+    # posixpath.normpath preserves a leading "//" (POSIX leaves two leading
+    # slashes implementation-defined), but shared caches collapse it to "/",
+    # so treat such a path as non-normalized.
+    if normalized.startswith("//"):
+        normalized = normalized[1:]
+    if path.endswith("/") and normalized != "/":
+        normalized += "/"
+    return path == normalized
+
+
+class PathNormalizationMiddleware:
+    """Rejects requests whose path is not in normalized form.
+
+    Shared caches (proxies, CDNs) normalize URLs (resolve `.` and `..`
+    segments, decode percent-encoded slashes, collapse duplicate slashes)
+    when building the cache key, while the application receives the original
+    path and may reflect it in the response. This discrepancy enables web
+    cache poisoning: a response generated for a malicious non-normalized path
+    gets cached under the key of a legitimate URL. Rejecting such requests
+    with a non-cacheable response closes that gap.
+    """
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] == "http" and not _is_normalized_path(scope):
+            response = JSONResponse(
+                {
+                    "errors": [
+                        {
+                            "code": "InvalidRequestPath",
+                            "message": "Request path is not normalized.",
+                        }
+                    ]
+                },
+                status_code=404,
+                headers={"Cache-Control": "no-store"},
+            )
+            await response(scope, receive, send)
+            return
+        await self.app(scope, receive, send)
+
+
 class StrictTransportSecurityMiddleware:
     """Adds `Strict-Transport-Security` (HSTS) header to every HTTP response.
 

spinta/utils/response.py

@@ -387,6 +387,9 @@ def cache_control_response_headers(context: Context, model: Model, target_id: st
 
     cache_control = {
         "Cache-Control": config.cache_control,
+        # Response body depends on these request headers (content negotiation,
+        # auth scopes), so shared caches must include them in the cache key.
+        "Vary": "Accept, Accept-Language, Authorization",
         "Last-Modified": last_modified,
         "ETag": revision,
     }

{spinta-0.2.dev27.dist-info → spinta-0.2.dev28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev27
+Version: 0.2.dev28
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev27.dist-info → spinta-0.2.dev28.dist-info}/RECORD

@@ -7,7 +7,7 @@ spinta/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/adapters/rc/__init__.py,sha256=FhuhKn-tFIafQVGqSocMpw4bL_TobtgyNWTBZibKjXg,75
 spinta/adapters/rc/signature_adapter.py,sha256=jOWaISD2jZQdYEb_zDuPufJFPvaVyK2bU936Comt93w,4371
 spinta/adapters/soap_plugins.py,sha256=wPtoX83qBC7rr3JyCyFBAAsRgQGnocYHQnqsEvURSNw,4463
-spinta/api/__init__.py,sha256=64I1ecMrBhRWmi3MBKjqJe7XPHz63gZubmgRc4Hploo,15978
+spinta/api/__init__.py,sha256=vCCROIgtmqoI17GRMqCNZoOlMmGASxd33_xGEV0xOx4,16286
 spinta/api/inspect.py,sha256=zWry4uSiYGn7FHLaqrn9k_8OVKRHLs3EH2Ni7IwgSqY,6775
 spinta/api/schema.py,sha256=annABS0b7yFKxNoCyu8kbZ0eMMv6qwCoapMKnljj0Jc,7634
 spinta/api/validators.py,sha256=FpCFd2EA1-aPU5totb5-bNDioM-67OnQ_7gt6s3qH0U,666
@@ -590,7 +590,7 @@ spinta/manifests/tabular/components.py,sha256=_SUFrsFxEFetLcdXymDiWVKOCSLK7pylyv
 spinta/manifests/tabular/constants.py,sha256=fgFnIcPhozim_aQj5f8u4PGcloifDl497T589aJ8It4,720
 spinta/manifests/tabular/formats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/tabular/formats/gsheets.py,sha256=iu-rJrzr6Inz_8-B75xzIryR8KWTO60xg0TNKizSijk,937
-spinta/manifests/tabular/helpers.py,sha256=qt2SNNEwoCCzzheCA2E1aRA5_3fIovgOmTVILoayUIs,94874
+spinta/manifests/tabular/helpers.py,sha256=DrzxCP9mph6IlTcd734memdYqsE2BdI5Q-HnnPIWeQ4,95934
 spinta/manifests/xsd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/xsd/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/manifests/xsd/commands/configure.py,sha256=FgVOwTy8SosDjHbgeIRWzYocX1mAIW_jAymbzwxDDro,539
@@ -609,7 +609,7 @@ spinta/manifests/yaml/commands/sync.py,sha256=hj9RLApoZvhK8I2ftx-KU4yGqi1uxqZPfw
 spinta/manifests/yaml/components.py,sha256=HPK_rJya_F47p3E6tYsZWUROvfycfPLFnrgGeMS0nMA,317
 spinta/manifests/yaml/helpers.py,sha256=xjOY3ascGPhaLgdSjJJmacBM0If5pFjSuo82WnpbQ34,4236
 spinta/methods.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-spinta/middlewares.py,sha256=XjNRKtaPfxOfZwvVgU34CHkxSHzQ3VaclA7UiWdIwtY,2143
+spinta/middlewares.py,sha256=DyOG8tBB9674f4fzHQQbmWNNnrDx4ZUNbymMa3IJfZI,4421
 spinta/naming/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 spinta/naming/components.py,sha256=BPlv9F91v_KyS7-4rJi1UfG9b21qKgiD6vouXRRgHt4,181
 spinta/naming/helpers.py,sha256=H22uq33wgGh-c0Cdud7oz5MN33LVU50N_PVY5zQails,4462
@@ -742,7 +742,7 @@ spinta/utils/nin.py,sha256=eNkYDtph6dFF-4d1SbYdLzz9FZhdpSJcFAJX9OsEw14,2295
 spinta/utils/passwords.py,sha256=v6UPoJADbc4bRmd1CRWDvxtUWcT9ePxURbvnB-BLeR4,1424
 spinta/utils/path.py,sha256=fbEBizEzHPpDRKodL_9cQXuIkoLsSuRBv44MFuRQxIk,1289
 spinta/utils/refs.py,sha256=E2cF2KKJka7jmCCQxYhSckKTCIkkorrtWb6wRs8UY4Q,530
-spinta/utils/response.py,sha256=fECmQD_--g6XHE3iz0vqGSBMptaH22SWYwnb6YB2lk4,12921
+spinta/utils/response.py,sha256=LcSldFOGQvUfJO_Q8WFkZVoz3R_y7bEZVvuCcNgmcXw,13135
 spinta/utils/schema.py,sha256=K0eCDeiHlh1WchuXfJoS5M6xm1CK7Co8h13-QPNzSro,2737
 spinta/utils/scopes.py,sha256=ffxg15O63tifEVWmFY-aWcdM5-xUXkRxtbu8bTmVNks,796
 spinta/utils/sqlalchemy.py,sha256=jJGteC63szzQIJv6zXKMdig6lUjubb7z8mGTVsG9dnM,1623
@@ -753,8 +753,8 @@ spinta/utils/tree.py,sha256=14USJGfLvfErh15zihJsLunorMql2KXbRuB_Eqsys_w,590
 spinta/utils/types.py,sha256=lfYSxKGPuPeUsO14d2OYodtbRY3zsa-o-z8HveVH3t0,801
 spinta/utils/units.py,sha256=CFFLv1NHYsoSSzwiar3zXYmt4m3sccW5niUgkZQgo3k,747
 spinta/utils/url.py,sha256=SLnwEEpXfJZzjvHP8ZPPlb0QXzkeAzY2e72qD1edy-A,3470
-spinta-0.2.dev27.dist-info/METADATA,sha256=892ytUFXKAxzCFQDO1a15fBPCyECUpoH-Ka-pmEy7PI,10295
-spinta-0.2.dev27.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-spinta-0.2.dev27.dist-info/entry_points.txt,sha256=-jdsOQZcMu3rUOwgIJNS3gZS4rwWPACuXXy128F676w,46
-spinta-0.2.dev27.dist-info/licenses/LICENSE,sha256=JKmjfBLapeFWNI_qdVr5bXGlsuMPa6nRarKPK5davKw,1071
-spinta-0.2.dev27.dist-info/RECORD,,
+spinta-0.2.dev28.dist-info/METADATA,sha256=FjmBsVZsxTdtmlgYTmI0sjVZohRGGk-6GZ0sECkeitc,10295
+spinta-0.2.dev28.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+spinta-0.2.dev28.dist-info/entry_points.txt,sha256=-jdsOQZcMu3rUOwgIJNS3gZS4rwWPACuXXy128F676w,46
+spinta-0.2.dev28.dist-info/licenses/LICENSE,sha256=JKmjfBLapeFWNI_qdVr5bXGlsuMPa6nRarKPK5davKw,1071
+spinta-0.2.dev28.dist-info/RECORD,,