spinta 0.2.dev27__tar.gz → 0.2.dev28__tar.gz

{spinta-0.2.dev27 → spinta-0.2.dev28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev27
+Version: 0.2.dev28
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev27 → spinta-0.2.dev28}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev27"
+version = "0.2dev28"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

{spinta-0.2.dev27 → spinta-0.2.dev28}/spinta/api/__init__.py

@@ -50,7 +50,7 @@ from spinta.exceptions import (
     error_response,
 )
 from spinta.formats.html.helpers import get_templates
-from spinta.middlewares import ContextMiddleware, StrictTransportSecurityMiddleware
+from spinta.middlewares import ContextMiddleware, PathNormalizationMiddleware, StrictTransportSecurityMiddleware
 from spinta.urlparams import Version, get_response_type
 
 log = logging.getLogger(__name__)
@@ -370,7 +370,7 @@ async def error(request, exc):
     elif isinstance(exc, BaseError):
         status_code = exc.status_code
         errors = [error_response(exc)]
-        headers = exc.headers
+        headers = dict(exc.headers)
     else:
         if isinstance(exc, HTTPException):
             status_code = exc.status_code
@@ -400,6 +400,11 @@ async def error(request, exc):
 
     response = {"errors": errors}
 
+    # Error responses can reflect request input (e.g. the requested path) and
+    # must never be stored by shared caches, otherwise they can be used for
+    # web cache poisoning.
+    headers["Cache-Control"] = "no-store"
+
     fmt = get_response_type(request.state.context, request)
     if fmt == "json" or fmt is None:
         return JSONResponse(
@@ -467,6 +472,7 @@ def init(context: Context):
             StrictTransportSecurityMiddleware,
             value=config.http_strict_transport_security,
         ),
+        Middleware(PathNormalizationMiddleware),
         Middleware(ContextMiddleware, context=context),
     ]
 

{spinta-0.2.dev27 → spinta-0.2.dev28}/spinta/manifests/tabular/helpers.py

@@ -653,9 +653,20 @@ class PropertyReader(TabularReader):
     enums: Set[str]
 
     def read(self, row: Dict[str, str]) -> None:
-        complete_structure, parent_structure, prop_name = _extract_and_create_parent_data(self, row, row["property"])
+        # Parse the data type once, here at the boundary, and pass the parsed
+        # result inward. Downstream code must not re-parse `row["type"]` or
+        # compare against the raw string: modifiers (`required`, `unique`) and
+        # arguments would make a type such as `backref required` or
+        # `object required` unrecognisable. Parent and nesting resolution only
+        # needs the bare type, while `_handle_datatype` keeps the modifiers.
+        dtype = _resolve_dtype(self, row)
+        bare_row = {**row, "type": dtype["type"]}
+        complete_structure, parent_structure, prop_name = _extract_and_create_parent_data(
+            self, bare_row, bare_row["property"]
+        )
+
+        prop_data = _handle_datatype(self, row, dtype)
 
-        prop_data = _handle_datatype(self, row)
         prop_name = _combine_parent_with_prop(prop_name, prop_data, parent_structure, complete_structure)
 
         # Edge case where there is no nesting, need to couple `prop_data` with `complete_structure`
@@ -748,8 +759,11 @@ def _initial_text_property_schema(given_name: str, dtype: dict, row: dict):
     return result
 
 
-def _datatype_handler(reader: PropertyReader, row: dict, initial_data_loader: Callable[[str, dict, dict], dict]):
-    dtype: dict = _resolve_dtype(reader, row)
+def _datatype_handler(
+    reader: PropertyReader, row: dict, initial_data_loader: Callable[[str, dict, dict], dict], dtype: dict = None
+):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     given_name = row["property"]
     reader.name = _clean_up_prop_name(row["property"].split(".")[-1])
 
@@ -759,7 +773,7 @@ def _datatype_handler(reader: PropertyReader, row: dict, initial_data_loader: Ca
             f"Property {reader.name!r} must be defined in a model context. "
             f"Now it is defined in {context.name!r} {context.type} context."
         )
-    _check_if_property_already_set(reader, row, given_name)
+    _check_if_property_already_set(reader, dtype["type"], given_name)
 
     if reader.state.base and not dtype["type"]:
         dtype["type"] = "inherit"
@@ -804,8 +818,9 @@ def _datatype_handler(reader: PropertyReader, row: dict, initial_data_loader: Ca
     return new_data
 
 
-def _string_datatype_handler(reader: PropertyReader, row: dict):
-    dtype: dict = _resolve_dtype(reader, row)
+def _string_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     given_name = row["property"]
     reader.name = _clean_up_prop_name(row["property"].split(".")[-1])
 
@@ -815,7 +830,7 @@ def _string_datatype_handler(reader: PropertyReader, row: dict):
             f"Property {reader.name!r} must be defined in a model context. "
             f"Now it is defined in {context.name!r} {context.type} context."
         )
-    existing_data = _check_if_property_already_set(reader, row, given_name)
+    existing_data = _check_if_property_already_set(reader, dtype["type"], given_name)
     if dtype["type"] == DataTypeEnum.TEXT.value and existing_data:
         reader.error(
             f"Property {reader.name!r} with the same name is already "
@@ -835,8 +850,9 @@ def _string_datatype_handler(reader: PropertyReader, row: dict):
     return new_data
 
 
-def _text_datatype_handler(reader: PropertyReader, row: dict):
-    dtype: dict = _resolve_dtype(reader, row)
+def _text_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     given_name = row["property"]
     reader.name = _clean_up_prop_name(row["property"].split(".")[-1])
 
@@ -846,7 +862,7 @@ def _text_datatype_handler(reader: PropertyReader, row: dict):
             f"Property {reader.name!r} must be defined in a model context. "
             f"Now it is defined in {context.name!r} {context.type} context."
         )
-    result = _check_if_property_already_set(reader, row, given_name)
+    result = _check_if_property_already_set(reader, dtype["type"], given_name)
     if not (result and result["explicitly_given"] is False and result["type"] == DataTypeEnum.TEXT.value or not result):
         reader.error(
             f"Property {reader.name!r} with the same name is already "
@@ -893,22 +909,23 @@ def _text_datatype_handler(reader: PropertyReader, row: dict):
     return new_data
 
 
-def _default_datatype_handler(reader: PropertyReader, row: dict):
-    return _datatype_handler(reader, row, _initial_normal_property_schema)
+def _default_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    return _datatype_handler(reader, row, _initial_normal_property_schema, dtype)
 
 
-def _array_datatype_handler(reader: PropertyReader, row: dict):
-    return _datatype_handler(reader, row, _initial_array_property_schema)
+def _array_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    return _datatype_handler(reader, row, _initial_array_property_schema, dtype)
 
 
-def _partial_datatype_handler(reader: PropertyReader, row: dict):
-    return _datatype_handler(reader, row, _initial_partial_property_schema)
+def _partial_datatype_handler(reader: PropertyReader, row: dict, dtype: dict = None):
+    return _datatype_handler(reader, row, _initial_partial_property_schema, dtype)
 
 
-def _handle_datatype(reader: PropertyReader, row: dict):
-    dtype: dict = _resolve_dtype(reader, row)
+def _handle_datatype(reader: PropertyReader, row: dict, dtype: dict = None):
+    if dtype is None:
+        dtype = _resolve_dtype(reader, row)
     handler = DATATYPE_HANDLERS.get(dtype["type"], DATATYPE_HANDLERS["_default"])
-    return handler(reader, row)
+    return handler(reader, row, dtype)
 
 
 DATATYPE_HANDLERS = {
@@ -1204,7 +1221,7 @@ def _extract_children_from_nested(base: dict, children_name: str) -> dict:
     return base
 
 
-def _check_if_property_already_set(reader: PropertyReader, given_row: dict, full_name: str):
+def _check_if_property_already_set(reader: PropertyReader, given_type: str, full_name: str):
     # Treat '@' as normal '.', since '_extract_children_from_nested' is able to extract based on type
     split = full_name.replace("@", ".").split(".")
     base = {}
@@ -1250,11 +1267,14 @@ def _check_if_property_already_set(reader: PropertyReader, given_row: dict, full
             f"Property {full_name!r} with the same name is already defined for this {reader.state.model.name!r} model."
         )
 
+    # `given_type` is the bare data type (modifiers and arguments already
+    # stripped by the caller), so a nesting type such as `object` is recognised
+    # even when it was declared as `object required`.
     if base and (
-        (base["type"] in ALLOWED_PARTIAL_TYPES and given_row["type"] not in ALLOWED_PARTIAL_TYPES)
-        or (base["type"] in ALLOWED_ARRAY_TYPES and given_row["type"] not in ALLOWED_ARRAY_TYPES)
+        (base["type"] in ALLOWED_PARTIAL_TYPES and given_type not in ALLOWED_PARTIAL_TYPES)
+        or (base["type"] in ALLOWED_ARRAY_TYPES and given_type not in ALLOWED_ARRAY_TYPES)
     ):
-        raise DataTypeCannotBeUsedForNesting(dtype=given_row["type"])
+        raise DataTypeCannotBeUsedForNesting(dtype=given_type)
     return base
 
 

spinta-0.2.dev28/spinta/middlewares.py

@@ -0,0 +1,113 @@
+import posixpath
+
+from starlette.responses import JSONResponse
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from spinta.components import Context
+
+
+def _is_normalized_path(scope: Scope) -> bool:
+    raw_path = (scope.get("raw_path") or b"").lower()
+    # Percent-encoded slashes, backslashes and dots decode into extra path
+    # delimiters and dot segments that shared caches do not see when they
+    # build the cache key.
+    if b"%2f" in raw_path or b"%5c" in raw_path or b"%2e" in raw_path:
+        return False
+    path = scope["path"]
+    if "\\" in path:
+        return False
+    normalized = posixpath.normpath(path)
+    # posixpath.normpath preserves a leading "//" (POSIX leaves two leading
+    # slashes implementation-defined), but shared caches collapse it to "/",
+    # so treat such a path as non-normalized.
+    if normalized.startswith("//"):
+        normalized = normalized[1:]
+    if path.endswith("/") and normalized != "/":
+        normalized += "/"
+    return path == normalized
+
+
+class PathNormalizationMiddleware:
+    """Rejects requests whose path is not in normalized form.
+
+    Shared caches (proxies, CDNs) normalize URLs (resolve `.` and `..`
+    segments, decode percent-encoded slashes, collapse duplicate slashes)
+    when building the cache key, while the application receives the original
+    path and may reflect it in the response. This discrepancy enables web
+    cache poisoning: a response generated for a malicious non-normalized path
+    gets cached under the key of a legitimate URL. Rejecting such requests
+    with a non-cacheable response closes that gap.
+    """
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] == "http" and not _is_normalized_path(scope):
+            response = JSONResponse(
+                {
+                    "errors": [
+                        {
+                            "code": "InvalidRequestPath",
+                            "message": "Request path is not normalized.",
+                        }
+                    ]
+                },
+                status_code=404,
+                headers={"Cache-Control": "no-store"},
+            )
+            await response(scope, receive, send)
+            return
+        await self.app(scope, receive, send)
+
+
+class StrictTransportSecurityMiddleware:
+    """Adds `Strict-Transport-Security` (HSTS) header to every HTTP response.
+
+    HSTS instructs browsers to only communicate with the server via HTTPS. The
+    header value is configurable via the `http_strict_transport_security`
+    configuration option and defaults to a `max-age` of one year together with
+    the `includeSubDomains` directive.
+    """
+
+    def __init__(self, app: ASGIApp, *, value: str) -> None:
+        self.app = app
+        self.value = value
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http" or not self.value:
+            await self.app(scope, receive, send)
+            return
+
+        async def send_with_hsts(message: Message) -> None:
+            if message["type"] == "http.response.start":
+                headers = message.setdefault("headers", [])
+                headers.append((b"strict-transport-security", self.value.encode("latin-1")))
+            await send(message)
+
+        await self.app(scope, receive, send_with_hsts)
+
+
+class ContextMiddleware:
+    """Adds `request.state.context`.
+
+    There is a global `context`, where all heavy things are preloaded as
+    startup. This preloading happens on Starlette's 'startup' event.
+
+    `ContextMiddleware` creates a fork of global preloaded context for each
+    request and assigns it to `request.state.context`. Forked context can be
+    modified in each request without effecting global context.
+    """
+
+    def __init__(self, app: ASGIApp, *, context: Context) -> None:
+        self.app = app
+        self.context = context
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] in ["http", "websocket"]:
+            with self.context.fork("request") as context:
+                scope.setdefault("state", {})
+                scope["state"]["context"] = context
+                await self.app(scope, receive, send)
+        else:
+            await self.app(scope, receive, send)

{spinta-0.2.dev27 → spinta-0.2.dev28}/spinta/utils/response.py

@@ -387,6 +387,9 @@ def cache_control_response_headers(context: Context, model: Model, target_id: st
 
     cache_control = {
         "Cache-Control": config.cache_control,
+        # Response body depends on these request headers (content negotiation,
+        # auth scopes), so shared caches must include them in the cache key.
+        "Vary": "Accept, Accept-Language, Authorization",
         "Last-Modified": last_modified,
         "ETag": revision,
     }

spinta-0.2.dev27/spinta/middlewares.py

@@ -1,55 +0,0 @@
-from starlette.types import ASGIApp, Message, Receive, Scope, Send
-
-from spinta.components import Context
-
-
-class StrictTransportSecurityMiddleware:
-    """Adds `Strict-Transport-Security` (HSTS) header to every HTTP response.
-
-    HSTS instructs browsers to only communicate with the server via HTTPS. The
-    header value is configurable via the `http_strict_transport_security`
-    configuration option and defaults to a `max-age` of one year together with
-    the `includeSubDomains` directive.
-    """
-
-    def __init__(self, app: ASGIApp, *, value: str) -> None:
-        self.app = app
-        self.value = value
-
-    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        if scope["type"] != "http" or not self.value:
-            await self.app(scope, receive, send)
-            return
-
-        async def send_with_hsts(message: Message) -> None:
-            if message["type"] == "http.response.start":
-                headers = message.setdefault("headers", [])
-                headers.append((b"strict-transport-security", self.value.encode("latin-1")))
-            await send(message)
-
-        await self.app(scope, receive, send_with_hsts)
-
-
-class ContextMiddleware:
-    """Adds `request.state.context`.
-
-    There is a global `context`, where all heavy things are preloaded as
-    startup. This preloading happens on Starlette's 'startup' event.
-
-    `ContextMiddleware` creates a fork of global preloaded context for each
-    request and assigns it to `request.state.context`. Forked context can be
-    modified in each request without effecting global context.
-    """
-
-    def __init__(self, app: ASGIApp, *, context: Context) -> None:
-        self.app = app
-        self.context = context
-
-    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        if scope["type"] in ["http", "websocket"]:
-            with self.context.fork("request") as context:
-                scope.setdefault("state", {})
-                scope["state"]["context"] = context
-                await self.app(scope, receive, send)
-        else:
-            await self.app(scope, receive, send)