schemathesis 3.25.6__py3-none-any.whl → 4.0.0a1__py3-none-any.whl

schemathesis/specs/openapi/converter.py

@@ -1,13 +1,20 @@
 from __future__ import annotations
+
 from itertools import chain
 from typing import Any, Callable
 
-from ...internal.jsonschema import traverse_schema
-from ...internal.copy import fast_deepcopy
+from schemathesis.core.transforms import deepclone, transform
+
+from .patterns import update_quantifier
 
 
 def to_json_schema(
-    schema: dict[str, Any], *, nullable_name: str, copy: bool = True, is_response_schema: bool = False
+    schema: dict[str, Any],
+    *,
+    nullable_name: str,
+    copy: bool = True,
+    is_response_schema: bool = False,
+    update_quantifiers: bool = True,
 ) -> dict[str, Any]:
     """Convert Open API parameters to JSON Schema.
 
@@ -15,7 +22,7 @@ def to_json_schema(
     See a recursive version below.
     """
     if copy:
-        schema = fast_deepcopy(schema)
+        schema = deepclone(schema)
     if schema.get(nullable_name) is True:
         del schema[nullable_name]
         schema = {"anyOf": [schema, {"type": "null"}]}
@@ -23,6 +30,8 @@ def to_json_schema(
     if schema_type == "file":
         schema["type"] = "string"
         schema["format"] = "binary"
+    if update_quantifiers:
+        update_pattern_in_schema(schema)
     if schema_type == "object":
         if is_response_schema:
             # Write-only properties should not occur in responses
@@ -33,6 +42,18 @@ def to_json_schema(
     return schema
 
 
+def update_pattern_in_schema(schema: dict[str, Any]) -> None:
+    pattern = schema.get("pattern")
+    min_length = schema.get("minLength")
+    max_length = schema.get("maxLength")
+    if pattern and (min_length or max_length):
+        new_pattern = update_quantifier(pattern, min_length, max_length)
+        if new_pattern != pattern:
+            schema.pop("minLength", None)
+            schema.pop("maxLength", None)
+            schema["pattern"] = new_pattern
+
+
 def rewrite_properties(schema: dict[str, Any], predicate: Callable[[dict[str, Any]], bool]) -> None:
     required = schema.get("required", [])
     forbidden = []
@@ -71,6 +92,12 @@ def is_read_only(schema: dict[str, Any] | bool) -> bool:
 
 
 def to_json_schema_recursive(
-    schema: dict[str, Any], nullable_name: str, is_response_schema: bool = False
+    schema: dict[str, Any], nullable_name: str, is_response_schema: bool = False, update_quantifiers: bool = True
 ) -> dict[str, Any]:
-    return traverse_schema(schema, to_json_schema, nullable_name=nullable_name, is_response_schema=is_response_schema)
+    return transform(
+        schema,
+        to_json_schema,
+        nullable_name=nullable_name,
+        is_response_schema=is_response_schema,
+        update_quantifiers=update_quantifiers,
+    )

schemathesis/specs/openapi/definitions.py

@@ -1,8 +1,9 @@
 # These schemas are copied from https://github.com/OAI/OpenAPI-Specification/tree/master/schemas
 from __future__ import annotations
-from typing import Any, TYPE_CHECKING
 
-from ..._lazy_import import lazy_import
+from typing import TYPE_CHECKING, Any
+
+from schemathesis.core.lazy_import import lazy_import
 
 if TYPE_CHECKING:
     from jsonschema import Validator
@@ -1329,6 +1330,8 @@ OPENAPI_30 = {
         },
     },
 }
+# Generated from the updated schema.yaml from 0035208, which includes unpublished bugfixes
+# https://github.com/OAI/OpenAPI-Specification/blob/0035208611701b4f7f2c959eb99a8725cca41e6e/schemas/v3.1/schema.yaml
 OPENAPI_31 = {
     "$id": "https://spec.openapis.org/oas/3.1/schema/2022-10-07",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
@@ -1344,7 +1347,7 @@ OPENAPI_31 = {
         },
         "servers": {"type": "array", "items": {"$ref": "#/$defs/server"}, "default": [{"url": "/"}]},
         "paths": {"$ref": "#/$defs/paths"},
-        "webhooks": {"type": "object", "additionalProperties": {"$ref": "#/$defs/path-item-or-reference"}},
+        "webhooks": {"type": "object", "additionalProperties": {"$ref": "#/$defs/path-item"}},
         "components": {"$ref": "#/$defs/components"},
         "security": {"type": "array", "items": {"$ref": "#/$defs/security-requirement"}},
         "tags": {"type": "array", "items": {"$ref": "#/$defs/tag"}},
@@ -1399,7 +1402,7 @@ OPENAPI_31 = {
             "$comment": "https://spec.openapis.org/oas/v3.1.0#server-object",
             "type": "object",
             "properties": {
-                "url": {"type": "string", "format": "uri-reference"},
+                "url": {"type": "string"},
                 "description": {"type": "string"},
                 "variables": {"type": "object", "additionalProperties": {"$ref": "#/$defs/server-variable"}},
             },
@@ -1438,7 +1441,7 @@ OPENAPI_31 = {
                 },
                 "links": {"type": "object", "additionalProperties": {"$ref": "#/$defs/link-or-reference"}},
                 "callbacks": {"type": "object", "additionalProperties": {"$ref": "#/$defs/callbacks-or-reference"}},
-                "pathItems": {"type": "object", "additionalProperties": {"$ref": "#/$defs/path-item-or-reference"}},
+                "pathItems": {"type": "object", "additionalProperties": {"$ref": "#/$defs/path-item"}},
             },
             "patternProperties": {
                 "^(schemas|responses|parameters|examples|requestBodies|headers|securitySchemes|links|callbacks|pathItems)$": {
@@ -1460,6 +1463,7 @@ OPENAPI_31 = {
             "$comment": "https://spec.openapis.org/oas/v3.1.0#path-item-object",
             "type": "object",
             "properties": {
+                "$ref": {"type": "string", "format": "uri-reference"},
                 "summary": {"type": "string"},
                 "description": {"type": "string"},
                 "servers": {"type": "array", "items": {"$ref": "#/$defs/server"}},
@@ -1476,11 +1480,6 @@ OPENAPI_31 = {
             "$ref": "#/$defs/specification-extensions",
             "unevaluatedProperties": False,
         },
-        "path-item-or-reference": {
-            "if": {"type": "object", "required": ["$ref"]},
-            "then": {"$ref": "#/$defs/reference"},
-            "else": {"$ref": "#/$defs/path-item"},
-        },
         "operation": {
             "$comment": "https://spec.openapis.org/oas/v3.1.0#operation-object",
             "type": "object",
@@ -1541,7 +1540,6 @@ OPENAPI_31 = {
                             "if": {"properties": {"in": {"const": "path"}}, "required": ["in"]},
                             "then": {
                                 "properties": {
-                                    "name": {"pattern": "[^/#?]+$"},
                                     "style": {"default": "simple", "enum": ["matrix", "label", "simple"]},
                                     "required": {"const": True},
                                 },
@@ -1661,7 +1659,7 @@ OPENAPI_31 = {
             "$comment": "https://spec.openapis.org/oas/v3.1.0#callback-object",
             "type": "object",
             "$ref": "#/$defs/specification-extensions",
-            "additionalProperties": {"$ref": "#/$defs/path-item-or-reference"},
+            "additionalProperties": {"$ref": "#/$defs/path-item"},
         },
         "callbacks-or-reference": {
             "if": {"type": "object", "required": ["$ref"]},
@@ -1754,7 +1752,6 @@ OPENAPI_31 = {
                 "summary": {"type": "string"},
                 "description": {"type": "string"},
             },
-            "unevaluatedProperties": False,
         },
         "schema": {
             "$comment": "https://spec.openapis.org/oas/v3.1.0#schema-object",
@@ -1906,11 +1903,7 @@ _VALIDATORS = [
     "OPENAPI_31_VALIDATOR",
 ]
 
-__all__ = [
-    "SWAGGER_20",
-    "OPENAPI_30",
-    "OPENAPI_31",
-] + _VALIDATORS
+__all__ = ["SWAGGER_20", "OPENAPI_30", "OPENAPI_31", *_VALIDATORS]
 
 _imports = {
     "SWAGGER_20_VALIDATOR": lambda: make_validator(SWAGGER_20),

schemathesis/specs/openapi/examples.py

@@ -4,23 +4,26 @@ from contextlib import suppress
 from dataclasses import dataclass
 from functools import lru_cache
 from itertools import chain, cycle, islice
-from typing import TYPE_CHECKING, Any, Generator, Union, cast
+from typing import TYPE_CHECKING, Any, Generator, Iterator, Union, cast
 
 import requests
-from hypothesis.strategies import SearchStrategy
 from hypothesis_jsonschema import from_schema
 
-from ...constants import DEFAULT_RESPONSE_TIMEOUT
-from ...models import APIOperation, Case
-from ..._hypothesis import get_single_example
-from ._hypothesis import get_case_strategy, get_default_format_strategies
-from .formats import STRING_FORMATS
+from schemathesis.core.transforms import deepclone
+from schemathesis.core.transport import DEFAULT_RESPONSE_TIMEOUT
+from schemathesis.generation import GenerationConfig
+from schemathesis.generation.case import Case
+from schemathesis.generation.hypothesis import examples
+from schemathesis.generation.meta import TestPhase
+from schemathesis.schemas import APIOperation
+
+from ._hypothesis import get_default_format_strategies, openapi_cases
 from .constants import LOCATION_TO_CONTAINER
+from .formats import STRING_FORMATS
 from .parameters import OpenAPIBody, OpenAPIParameter
 
-
 if TYPE_CHECKING:
-    from ...generation import GenerationConfig
+    from hypothesis.strategies import SearchStrategy
 
 
 @dataclass
@@ -44,7 +47,7 @@ Example = Union[ParameterExample, BodyExample]
 
 
 def get_strategies_from_examples(
-    operation: APIOperation[OpenAPIParameter, Case], examples_field: str = "examples"
+    operation: APIOperation[OpenAPIParameter], **kwargs: Any
 ) -> list[SearchStrategy[Case]]:
     """Build a set of strategies that generate test cases based on explicit examples in the schema."""
     maps = {}
@@ -68,13 +71,16 @@ def get_strategies_from_examples(
     # Add examples from parameter's schemas
     examples.extend(extract_from_schemas(operation))
     return [
-        get_case_strategy(operation=operation, **parameters).map(serialize_components)
+        openapi_cases(operation=operation, **{**parameters, **kwargs, "phase": TestPhase.EXPLICIT}).map(
+            serialize_components
+        )
         for parameters in produce_combinations(examples)
     ]
 
 
-def extract_top_level(operation: APIOperation[OpenAPIParameter, Case]) -> Generator[Example, None, None]:
+def extract_top_level(operation: APIOperation[OpenAPIParameter]) -> Generator[Example, None, None]:
     """Extract top-level parameter examples from `examples` & `example` fields."""
+    responses = find_in_responses(operation)
     for parameter in operation.iter_parameters():
         if "schema" in parameter.definition:
             definitions = [parameter.definition, *_expand_subschemas(parameter.definition["schema"])]
@@ -104,6 +110,10 @@ def extract_top_level(operation: APIOperation[OpenAPIParameter, Case]) -> Genera
                         yield ParameterExample(
                             container=LOCATION_TO_CONTAINER[parameter.location], name=parameter.name, value=value
                         )
+        for value in find_matching_in_responses(responses, parameter.name):
+            yield ParameterExample(
+                container=LOCATION_TO_CONTAINER[parameter.location], name=parameter.name, value=value
+            )
     for alternative in operation.body:
         alternative = cast(OpenAPIBody, alternative)
         if "schema" in alternative.definition:
@@ -131,14 +141,30 @@ def extract_top_level(operation: APIOperation[OpenAPIParameter, Case]) -> Genera
 def _expand_subschemas(schema: dict[str, Any] | bool) -> Generator[dict[str, Any] | bool, None, None]:
     yield schema
     if isinstance(schema, dict):
-        for key in ("anyOf", "oneOf", "allOf"):
+        for key in ("anyOf", "oneOf"):
             if key in schema:
                 for subschema in schema[key]:
                     yield subschema
+        if "allOf" in schema:
+            subschema = deepclone(schema["allOf"][0])
+            for sub in schema["allOf"][1:]:
+                if isinstance(sub, dict):
+                    for key, value in sub.items():
+                        if key == "properties":
+                            subschema.setdefault("properties", {}).update(value)
+                        elif key == "required":
+                            subschema.setdefault("required", []).extend(value)
+                        elif key == "examples":
+                            subschema.setdefault("examples", []).extend(value)
+                        elif key == "example":
+                            subschema.setdefault("examples", []).append(value)
+                        else:
+                            subschema[key] = value
+            yield subschema
 
 
 def _find_parameter_examples_definition(
-    operation: APIOperation[OpenAPIParameter, Case], parameter_name: str, field_name: str
+    operation: APIOperation[OpenAPIParameter], parameter_name: str, field_name: str
 ) -> dict[str, Any]:
     """Find the original, unresolved `examples` definition of a parameter."""
     from .schemas import BaseOpenAPISchema
@@ -156,13 +182,13 @@ def _find_parameter_examples_definition(
 
 
 def _find_request_body_examples_definition(
-    operation: APIOperation[OpenAPIParameter, Case], alternative: OpenAPIBody
+    operation: APIOperation[OpenAPIParameter], alternative: OpenAPIBody
 ) -> dict[str, Any]:
     """Find the original, unresolved `examples` definition of a request body variant."""
     from .schemas import BaseOpenAPISchema
 
     schema = cast(BaseOpenAPISchema, operation.schema)
-    if schema.spec_version == "2.0":
+    if schema.specification.version == "2.0":
         raw_schema = schema.raw_schema
         path_data = raw_schema["paths"][operation.path]
         parameters = chain(path_data[operation.method].get("parameters", []), path_data.get("parameters", []))
@@ -183,7 +209,7 @@ def extract_inner_examples(
 ) -> Generator[Any, None, None]:
     """Extract exact examples values from the `examples` dictionary."""
     for name, example in examples.items():
-        if "$ref" in unresolved_definition[name]:
+        if "$ref" in unresolved_definition[name] and "value" not in example and "externalValue" not in example:
             # The example here is a resolved example and should be yielded as is
             yield example
         if isinstance(example, dict):
@@ -198,12 +224,12 @@ def extract_inner_examples(
 @lru_cache
 def load_external_example(url: str) -> bytes:
     """Load examples the `externalValue` keyword."""
-    response = requests.get(url, timeout=DEFAULT_RESPONSE_TIMEOUT / 1000)
+    response = requests.get(url, timeout=DEFAULT_RESPONSE_TIMEOUT)
     response.raise_for_status()
     return response.content
 
 
-def extract_from_schemas(operation: APIOperation[OpenAPIParameter, Case]) -> Generator[Example, None, None]:
+def extract_from_schemas(operation: APIOperation[OpenAPIParameter]) -> Generator[Example, None, None]:
     """Extract examples from parameters' schema definitions."""
     for parameter in operation.iter_parameters():
         schema = parameter.as_json_schema(operation)
@@ -214,12 +240,13 @@ def extract_from_schemas(operation: APIOperation[OpenAPIParameter, Case]) -> Gen
     for alternative in operation.body:
         alternative = cast(OpenAPIBody, alternative)
         schema = alternative.as_json_schema(operation)
-        for value in extract_from_schema(operation, schema, alternative.example_field, alternative.examples_field):
-            yield BodyExample(value=value, media_type=alternative.media_type)
+        for example_field, examples_field in (("example", "examples"), ("x-example", "x-examples")):
+            for value in extract_from_schema(operation, schema, example_field, examples_field):
+                yield BodyExample(value=value, media_type=alternative.media_type)
 
 
 def extract_from_schema(
-    operation: APIOperation[OpenAPIParameter, Case],
+    operation: APIOperation[OpenAPIParameter],
     schema: dict[str, Any],
     example_field_name: str,
     examples_field_name: str,
@@ -241,6 +268,8 @@ def extract_from_schema(
                 if examples_field_name in subsubschema and isinstance(subsubschema[examples_field_name], list):
                     # These are JSON Schema examples, which is an array of values
                     values.extend(subsubschema[examples_field_name])
+                # Check nested examples as well
+                values.extend(extract_from_schema(operation, subsubschema, example_field_name, examples_field_name))
                 if not values:
                     if name in required:
                         # Defer generation to only generate these variants if at least one property has examples
@@ -279,7 +308,7 @@ def _generate_single_example(
         allow_x00=generation_config.allow_x00,
         codec=generation_config.codec,
     )
-    return get_single_example(strategy)
+    return examples.generate_one(strategy)
 
 
 def produce_combinations(examples: list[Example]) -> Generator[dict[str, Any], None, None]:
@@ -328,3 +357,90 @@ def _produce_parameter_combinations(parameters: dict[str, dict[str, list]]) -> G
             }
             for container, variants in parameters.items()
         }
+
+
+def find_in_responses(operation: APIOperation) -> dict[str, list[dict[str, Any]]]:
+    """Find schema examples in responses."""
+    output: dict[str, list[dict[str, Any]]] = {}
+    for status_code, response in operation.definition.raw.get("responses", {}).items():
+        if not str(status_code).startswith("2"):
+            # Check only 2xx responses
+            continue
+        if isinstance(response, dict) and "$ref" in response:
+            _, response = operation.schema.resolver.resolve_in_scope(response, operation.definition.scope)  # type:ignore[attr-defined]
+        for media_type, definition in response.get("content", {}).items():
+            schema_ref = definition.get("schema", {}).get("$ref")
+            if schema_ref:
+                name = schema_ref.split("/")[-1]
+            else:
+                name = f"{status_code}/{media_type}"
+            for examples_field, example_field in (
+                ("examples", "example"),
+                ("x-examples", "x-example"),
+            ):
+                examples = definition.get(examples_field, {})
+                for example in examples.values():
+                    if "value" in example:
+                        output.setdefault(name, []).append(example["value"])
+                if example_field in definition:
+                    output.setdefault(name, []).append(definition[example_field])
+    return output
+
+
+NOT_FOUND = object()
+
+
+def find_matching_in_responses(examples: dict[str, list], param: str) -> Iterator[Any]:
+    """Find matching parameter examples."""
+    normalized = param.lower()
+    is_id_param = normalized.endswith("id")
+    # Extract values from response examples that match input parameters.
+    # E.g., for `GET /orders/{id}/`, use "id" or "orderId" from `Order` response
+    # as examples for the "id" path parameter.
+    for schema_name, schema_examples in examples.items():
+        for example in schema_examples:
+            if not isinstance(example, dict):
+                continue
+            # Unwrapping example from `{"item": [{...}]}`
+            if isinstance(example, dict) and len(example) == 1 and list(example)[0].lower() == schema_name.lower():
+                inner = list(example.values())[0]
+                if isinstance(inner, list):
+                    for sub_example in inner:
+                        found = _find_matching_in_responses(sub_example, schema_name, param, normalized, is_id_param)
+                        if found is not NOT_FOUND:
+                            yield found
+                    continue
+                if isinstance(inner, dict):
+                    example = inner
+            found = _find_matching_in_responses(example, schema_name, param, normalized, is_id_param)
+            if found is not NOT_FOUND:
+                yield found
+
+
+def _find_matching_in_responses(
+    example: dict[str, Any], schema_name: str, param: str, normalized: str, is_id_param: bool
+) -> Any:
+    # Check for exact match
+    if param in example:
+        return example[param]
+    if is_id_param and param[:-2] in example:
+        return example[param[:-2]]
+
+    # Check for case-insensitive match
+    for key in example:
+        if key.lower() == normalized:
+            return example[key]
+    else:
+        # If no match found and it's an ID parameter, try additional checks
+        if is_id_param:
+            # Check for 'id' if parameter is '{something}Id'
+            if "id" in example:
+                return example["id"]
+            # Check for '{schemaName}Id' or '{schemaName}_id'
+            if normalized == "id" or normalized.startswith(schema_name.lower()):
+                for key in (schema_name, schema_name.lower()):
+                    for suffix in ("_id", "Id"):
+                        with_suffix = f"{key}{suffix}"
+                        if with_suffix in example:
+                            return example[with_suffix]
+    return NOT_FOUND

schemathesis/specs/openapi/expressions/__init__.py

@@ -2,14 +2,27 @@
 
 https://swagger.io/docs/specification/links/#runtime-expressions
 """
+
+from __future__ import annotations
+
+import json
 from typing import Any
 
 from . import lexer, nodes, parser
 from .context import ExpressionContext
 
+__all__ = [
+    "lexer",
+    "nodes",
+    "parser",
+    "ExpressionContext",
+]
 
-def evaluate(expr: Any, context: ExpressionContext) -> str:
+
+def evaluate(expr: Any, context: ExpressionContext, evaluate_nested: bool = False) -> Any:
     """Evaluate runtime expression in context."""
+    if isinstance(expr, (dict, list)) and evaluate_nested:
+        return _evaluate_nested(expr, context)
     if not isinstance(expr, str):
         # Can be a non-string constant
         return expr
@@ -17,4 +30,26 @@ def evaluate(expr: Any, context: ExpressionContext) -> str:
     if len(parts) == 1:
         return parts[0]  # keep the return type the same as the internal value type
     # otherwise, concatenate into a string
-    return "".join(map(str, parts))
+    return "".join(str(part) for part in parts if part is not None)
+
+
+def _evaluate_nested(expr: dict[str, Any] | list, context: ExpressionContext) -> Any:
+    if isinstance(expr, dict):
+        return {
+            _evaluate_object_key(key, context): evaluate(value, context, evaluate_nested=True)
+            for key, value in expr.items()
+        }
+    return [evaluate(item, context, evaluate_nested=True) for item in expr]
+
+
+def _evaluate_object_key(key: str, context: ExpressionContext) -> Any:
+    evaluated = evaluate(key, context)
+    if isinstance(evaluated, str):
+        return evaluated
+    if isinstance(evaluated, bool):
+        return "true" if evaluated else "false"
+    if isinstance(evaluated, (int, float)):
+        return str(evaluated)
+    if evaluated is None:
+        return "null"
+    return json.dumps(evaluated)

schemathesis/specs/openapi/expressions/context.py

@@ -1,16 +1,14 @@
 from __future__ import annotations
-from dataclasses import dataclass
-from typing import TYPE_CHECKING
 
+from dataclasses import dataclass
 
-if TYPE_CHECKING:
-    from ....models import Case
-    from ....transports.responses import GenericResponse
+from schemathesis.core.transport import Response
+from schemathesis.generation.case import Case
 
 
 @dataclass
 class ExpressionContext:
     """Context in what an expression are evaluated."""
 
-    response: GenericResponse
+    response: Response
     case: Case

schemathesis/specs/openapi/expressions/extractors.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class Extractor:
+    def extract(self, value: str) -> str | None:
+        raise NotImplementedError
+
+
+@dataclass
+class RegexExtractor(Extractor):
+    """Extract value via a regex."""
+
+    value: re.Pattern
+
+    def extract(self, value: str) -> str | None:
+        match = self.value.search(value)
+        if match is None:
+            return None
+        return match.group(1)

schemathesis/specs/openapi/expressions/lexer.py

@@ -1,4 +1,5 @@
 """Lexical analysis of runtime expressions."""
+
 from dataclasses import dataclass
 from enum import Enum, unique
 from typing import Callable, Generator
@@ -19,33 +20,34 @@ class Token:
     """Lexical token that may occur in a runtime expression."""
 
     value: str
+    end: int
     type_: TokenType
 
     # Helpers for cleaner instantiation
 
     @classmethod
-    def variable(cls, value: str) -> "Token":
-        return cls(value, TokenType.VARIABLE)
+    def variable(cls, value: str, end: int) -> "Token":
+        return cls(value, end, TokenType.VARIABLE)
 
     @classmethod
-    def string(cls, value: str) -> "Token":
-        return cls(value, TokenType.STRING)
+    def string(cls, value: str, end: int) -> "Token":
+        return cls(value, end, TokenType.STRING)
 
     @classmethod
-    def pointer(cls, value: str) -> "Token":
-        return cls(value, TokenType.POINTER)
+    def pointer(cls, value: str, end: int) -> "Token":
+        return cls(value, end, TokenType.POINTER)
 
     @classmethod
-    def lbracket(cls) -> "Token":
-        return cls("{", TokenType.LBRACKET)
+    def lbracket(cls, end: int) -> "Token":
+        return cls("{", end, TokenType.LBRACKET)
 
     @classmethod
-    def rbracket(cls) -> "Token":
-        return cls("}", TokenType.RBRACKET)
+    def rbracket(cls, end: int) -> "Token":
+        return cls("}", end, TokenType.RBRACKET)
 
     @classmethod
-    def dot(cls) -> "Token":
-        return cls(".", TokenType.DOT)
+    def dot(cls, end: int) -> "Token":
+        return cls(".", end, TokenType.DOT)
 
     # Helpers for simpler type comparison
 
@@ -102,15 +104,15 @@ def tokenize(expression: str) -> TokenGenerator:
         if current_symbol() == "$":
             start = cursor
             move_until(lambda: is_eol() or current_symbol() in stop_symbols)
-            yield Token.variable(expression[start:cursor])
+            yield Token.variable(expression[start:cursor], cursor - 1)
         elif current_symbol() == ".":
-            yield Token.dot()
+            yield Token.dot(cursor)
             move()
         elif current_symbol() == "{":
-            yield Token.lbracket()
+            yield Token.lbracket(cursor)
             move()
         elif current_symbol() == "}":
-            yield Token.rbracket()
+            yield Token.rbracket(cursor)
             move()
         elif current_symbol() == "#":
             start = cursor
@@ -125,8 +127,8 @@ def tokenize(expression: str) -> TokenGenerator:
             # `ID_{$response.body#/foo}_{$response.body#/bar}`
             # Which is much easier if we treat `}` as a closing bracket of an embedded runtime expression
             move_until(lambda: is_eol() or current_symbol() == "}")
-            yield Token.pointer(expression[start:cursor])
+            yield Token.pointer(expression[start:cursor], cursor - 1)
         else:
             start = cursor
             move_until(lambda: is_eol() or current_symbol() in stop_symbols)
-            yield Token.string(expression[start:cursor])
+            yield Token.string(expression[start:cursor], cursor - 1)