arkindex-client 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

arkindex/pagination.py

@@ -51,9 +51,6 @@ class ResponsePaginator(Sized, Iterator):
         self.request_kwargs = request_kwargs
         """Keyword arguments to send to :meth:`arkindex.ArkindexClient.request` with each request."""
 
-        self.mode = None
-        """`page` for PageNumberPagination endpoints or `cursor` for CursorPagination endpoints."""
-
         self.count = None
         """Total results count."""
 
@@ -69,8 +66,26 @@ class ResponsePaginator(Sized, Iterator):
         ), "retries must be a positive integer"
         """Max number of retries per API request"""
 
-        # First page key is an empty string as we do not know yet the pagination type (e.g. page, cursor)
-        self.initial_page = ""
+        # Detect and store the pagination mode
+        self.mode = None
+        if any(
+            field.name == "cursor"
+            for field in self.client.lookup_operation(self.operation_id).fields
+        ):
+            self.mode = PaginationMode.Cursor
+        elif any(
+            field.name == "page"
+            for field in self.client.lookup_operation(self.operation_id).fields
+        ):
+            self.mode = PaginationMode.PageNumber
+        if not self.mode:
+            raise NotImplementedError(
+                "Pagination only implements page and cursor modes."
+            )
+
+        # First page key is an empty string by default (to stay coherent with page or cursor modes)
+        self.initial_page = request_kwargs.get(self.mode.value, "")
+
         # Store retrieved pages remaining retries
         self.pages = {self.initial_page: self.retries}
 
@@ -90,9 +105,8 @@ class ResponsePaginator(Sized, Iterator):
         Returns False in case the page returned an empty result
         Raises a StopIteration in case there are no pages left to iterate on
         """
-        # Filter out pages with no retries
         # Transform as a list of tuples for simpler output
-        remaining = sorted([(m, v) for m, v in self.pages.items() if v > 0])
+        remaining = [(m, v) for m, v in self.pages.items()]
 
         # No remaining pages, end of iteration
         if not remaining:
@@ -101,28 +115,53 @@ class ResponsePaginator(Sized, Iterator):
         # Get next page to load
         index, retry = remaining[0]
 
-        if self.mode:
+        if index:
             self.request_kwargs[self.mode.value] = index
 
         try:
             extra_kwargs = {}
             if not self.pages_loaded:
-                logger.info(
-                    f"Loading first page on try {self.retries - retry + 1}/{self.retries}"
-                )
+                if (
+                    self.mode == PaginationMode.PageNumber
+                    and self.initial_page
+                    and int(self.initial_page) > 1
+                ) or (self.mode == PaginationMode.Cursor and self.initial_page):
+                    logger.info(
+                        f"Loading page {self.initial_page} on try {self.retries - retry + 1}/{self.retries}"
+                    )
+                else:
+                    logger.info(
+                        f"Loading first page on try {self.retries - retry + 1}/{self.retries}"
+                    )
                 operation_fields = [
                     f.name
                     for f in self.client.lookup_operation(self.operation_id).fields
                 ]
-                # Ask to count results if the operation handle it as we do not know the pagination mode yet
+                # Ask to count results if the operation handle it (this is usually the case with cursors)
                 if "with_count" in operation_fields:
-                    extra_kwargs["with_count"] = "true"
+                    extra_kwargs = {
+                        "with_count": "true",
+                        **extra_kwargs,
+                    }
             else:
-                remaining_count = self.pages_count - self.pages_loaded
-                logger.info(
-                    f"Loading {self.mode.value} {index} on try {self.retries - retry + 1}/{self.retries}"
-                    f" - remains {remaining_count} page{'s' if remaining_count > 1 else ''} to load."
-                )
+                message = f"Loading {self.mode.value} {index} on try {self.retries - retry + 1}/{self.retries}"
+                if self.pages_count is not None:
+                    if self.mode is PaginationMode.Cursor and self.initial_page:
+                        # The number of remaining pages is unknown when an initial cursor is set
+                        max_pages = self.pages_count - self.pages_loaded
+                        message = message + (
+                            f" - remains a maximum of {max_pages} page{'s' if max_pages > 1 else ''} to load."
+                        )
+                    else:
+                        initial = int(self.initial_page) if self.initial_page else 1
+                        remaining_count = (
+                            self.pages_count - self.pages_loaded - (initial - 1)
+                        )
+                        message = message + (
+                            f" - remains {remaining_count} page{'s' if remaining_count > 1 else ''} to load."
+                        )
+
+                logger.info(message)
 
             # Fetch the next page
             self.data = self.client.single_request(
@@ -133,33 +172,32 @@ class ResponsePaginator(Sized, Iterator):
             )
             self.results = self.data.get("results", [])
 
-            if not self.mode and self.data:
-                # Autodetect if this endpoint uses page or cursor pagination
-                if self.data.get("number"):
-                    self.mode = PaginationMode.PageNumber
-                else:
-                    self.mode = PaginationMode.Cursor
-
+            # Retrieve information on the first page with results count
             if self.count is None and "count" in self.data:
-                # Retrieve information on first page with results count
                 self.count = self.data["count"]
                 if self.count == 0:
                     # Pagination has retrieved 0 results
                     self.pages = {}
                     return False
                 self.pages_count = math.ceil(self.count / len(self.results))
-                logger.info(
-                    f"Pagination will load a total of {self.pages_count} page{'s' if self.pages_count > 1 else ''}."
-                )
-                if self.mode == PaginationMode.PageNumber:
+                if self.mode == PaginationMode.Cursor:
+                    logger.info(
+                        f"Pagination will load a {'maximum' if self.initial_page else 'total'} "
+                        f"of {self.pages_count} page{'s' if self.pages_count > 1 else ''}"
+                    )
+                elif self.mode == PaginationMode.PageNumber:
+                    initial = int(self.initial_page) if self.initial_page else 1
+                    total = self.pages_count - initial + 1
+                    logger.info(
+                        f"Pagination will load a total of {total} page{'s' if total > 1 else ''}."
+                    )
                     # Initialize all pages once
-                    self.pages = {
-                        i: self.retries for i in range(2, self.pages_count + 1)
-                    }
-            elif self.mode == PaginationMode.PageNumber:
-                # Mark page as loaded on other pages
-                del self.pages[index]
-
+                    self.pages.update(
+                        {
+                            i: self.retries
+                            for i in range(initial + 1, self.pages_count + 1)
+                        }
+                    )
             if self.mode == PaginationMode.Cursor:
                 # Parse next URL to retrieve the cursor of the next page
                 query = urlsplit(self.data["next"]).query
@@ -168,6 +206,9 @@ class ResponsePaginator(Sized, Iterator):
                     self.pages = {}
                 else:
                     self.pages = {cursor_query[0]: self.retries}
+            elif self.mode == PaginationMode.PageNumber:
+                # Mark the current page as loaded
+                del self.pages[index]
 
             # Stop happy path here, we don't need to process errors
             self.pages_loaded += 1
@@ -201,6 +242,7 @@ class ResponsePaginator(Sized, Iterator):
             logger.warning(error_text)
             if self.allow_missing_data:
                 self.missing.add(index)
+                del self.pages[index]
             else:
                 raise Exception("Stopping pagination as data will be incomplete")
 

arkindex/schema/openapi.py

@@ -3,9 +3,9 @@ import re
 from urllib.parse import urljoin
 
 import typesystem
+from typesystem.json_schema import JSONSchema
 
-from arkindex.document import Document, Field, Link, Section
-from arkindex.schema.jsonschema import JSON_SCHEMA
+from arkindex.document import Document, Field, Link
 
 SCHEMA_REF = typesystem.Object(
     properties={"$ref": typesystem.String(pattern="^#/components/schemas/")}
@@ -13,92 +13,27 @@ SCHEMA_REF = typesystem.Object(
 REQUESTBODY_REF = typesystem.Object(
     properties={"$ref": typesystem.String(pattern="^#/components/requestBodies/")}
 )
-RESPONSE_REF = typesystem.Object(
-    properties={"$ref": typesystem.String(pattern="^#/components/responses/")}
-)
 
-definitions = typesystem.SchemaDefinitions()
+definitions = typesystem.Definitions()
 
-OPEN_API = typesystem.Object(
-    title="OpenAPI",
-    properties={
+OPEN_API = typesystem.Schema(
+    fields={
         "openapi": typesystem.String(),
-        "info": typesystem.Reference("Info", definitions=definitions),
         "servers": typesystem.Array(
-            items=typesystem.Reference("Server", definitions=definitions)
+            items=typesystem.Reference("Server", definitions=definitions),
+            default=[],
         ),
         "paths": typesystem.Reference("Paths", definitions=definitions),
-        "components": typesystem.Reference("Components", definitions=definitions),
-        "security": typesystem.Array(
-            items=typesystem.Reference("SecurityRequirement", definitions=definitions)
-        ),
-        "tags": typesystem.Array(
-            items=typesystem.Reference("Tag", definitions=definitions)
+        "components": typesystem.Reference(
+            "Components", default=None, definitions=definitions
         ),
-        "externalDocs": typesystem.Reference(
-            "ExternalDocumentation", definitions=definitions
-        ),
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["openapi", "info", "paths"],
-)
-
-definitions["Info"] = typesystem.Object(
-    properties={
-        "title": typesystem.String(allow_blank=True),
-        "description": typesystem.Text(allow_blank=True),
-        "termsOfService": typesystem.String(format="url"),
-        "contact": typesystem.Reference("Contact", definitions=definitions),
-        "license": typesystem.Reference("License", definitions=definitions),
-        "version": typesystem.String(allow_blank=True),
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["title", "version"],
-)
-
-definitions["Contact"] = typesystem.Object(
-    properties={
-        "name": typesystem.String(allow_blank=True),
-        "url": typesystem.String(format="url"),
-        "email": typesystem.String(format="email"),
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
 )
 
-definitions["License"] = typesystem.Object(
-    properties={"name": typesystem.String(), "url": typesystem.String(format="url")},
-    required=["name"],
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-)
-
-definitions["Server"] = typesystem.Object(
-    properties={
+definitions["Server"] = typesystem.Schema(
+    fields={
         "url": typesystem.String(),
-        "description": typesystem.Text(allow_blank=True),
-        "variables": typesystem.Object(
-            additional_properties=typesystem.Reference(
-                "ServerVariable", definitions=definitions
-            )
-        ),
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["url"],
-)
-
-definitions["ServerVariable"] = typesystem.Object(
-    properties={
-        "enum": typesystem.Array(items=typesystem.String()),
-        "default": typesystem.String(),
-        "description": typesystem.Text(allow_blank=True),
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["default"],
 )
 
 definitions["Paths"] = typesystem.Object(
@@ -112,7 +47,6 @@ definitions["Paths"] = typesystem.Object(
 definitions["Path"] = typesystem.Object(
     properties={
         "summary": typesystem.String(allow_blank=True),
-        "description": typesystem.Text(allow_blank=True),
         "get": typesystem.Reference("Operation", definitions=definitions),
         "put": typesystem.Reference("Operation", definitions=definitions),
         "post": typesystem.Reference("Operation", definitions=definitions),
@@ -129,18 +63,11 @@ definitions["Path"] = typesystem.Object(
         ),
         # TODO: | ReferenceObject
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
+    additional_properties=True,
 )
 
 definitions["Operation"] = typesystem.Object(
     properties={
-        "tags": typesystem.Array(items=typesystem.String()),
-        "summary": typesystem.String(allow_blank=True),
-        "description": typesystem.Text(allow_blank=True),
-        "externalDocs": typesystem.Reference(
-            "ExternalDocumentation", definitions=definitions
-        ),
         "operationId": typesystem.String(),
         "parameters": typesystem.Array(
             items=typesystem.Reference("Parameter", definitions=definitions)
@@ -149,61 +76,29 @@ definitions["Operation"] = typesystem.Object(
         | typesystem.Reference(
             "RequestBody", definitions=definitions
         ),  # TODO: RequestBody | ReferenceObject
-        "responses": typesystem.Reference("Responses", definitions=definitions),
-        # TODO: 'callbacks'
         "deprecated": typesystem.Boolean(),
-        "security": typesystem.Array(
-            typesystem.Reference("SecurityRequirement", definitions=definitions)
-        ),
         "servers": typesystem.Array(
             items=typesystem.Reference("Server", definitions=definitions)
         ),
+        # A custom property added by Arkindex to provide a hint to the API client
+        # that this endpoint is paginated, which changes the behavior of ArkindexClient.paginate.
+        "x-paginated": typesystem.Boolean(default=None),
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-)
-
-definitions["ExternalDocumentation"] = typesystem.Object(
-    properties={
-        "description": typesystem.Text(allow_blank=True),
-        "url": typesystem.String(format="url"),
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["url"],
+    additional_properties=True,
 )
 
-definitions["Parameter"] = typesystem.Object(
-    properties={
+definitions["Parameter"] = typesystem.Schema(
+    fields={
         "name": typesystem.String(),
         "in": typesystem.Choice(choices=["query", "header", "path", "cookie"]),
-        "description": typesystem.Text(allow_blank=True),
-        "required": typesystem.Boolean(),
-        "deprecated": typesystem.Boolean(),
-        "allowEmptyValue": typesystem.Boolean(),
-        "style": typesystem.Choice(
-            choices=[
-                "matrix",
-                "label",
-                "form",
-                "simple",
-                "spaceDelimited",
-                "pipeDelimited",
-                "deepObject",
-            ]
-        ),
-        "schema": JSON_SCHEMA | SCHEMA_REF,
-        "example": typesystem.Any(),
-        # TODO: Other fields
+        "required": typesystem.Boolean(default=False),
+        "deprecated": typesystem.Boolean(default=False),
+        "schema": JSONSchema | SCHEMA_REF,
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["name", "in"],
 )
 
 definitions["RequestBody"] = typesystem.Object(
     properties={
-        "description": typesystem.String(allow_blank=True),
         "content": typesystem.Object(
             additional_properties=typesystem.Reference(
                 "MediaType", definitions=definitions
@@ -211,88 +106,19 @@ definitions["RequestBody"] = typesystem.Object(
         ),
         "required": typesystem.Boolean(),
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-)
-
-definitions["Responses"] = typesystem.Object(
-    properties={
-        "default": typesystem.Reference("Response", definitions=definitions)
-        | RESPONSE_REF
-    },
-    pattern_properties={
-        "^([1-5][0-9][0-9]|[1-5]XX)$": typesystem.Reference(
-            "Response", definitions=definitions
-        )
-        | RESPONSE_REF,
-        "^x-": typesystem.Any(),
-    },
-    additional_properties=False,
-)
-
-definitions["Response"] = typesystem.Object(
-    properties={
-        "description": typesystem.String(allow_blank=True),
-        "content": typesystem.Object(
-            additional_properties=typesystem.Reference(
-                "MediaType", definitions=definitions
-            )
-        ),
-        "headers": typesystem.Object(
-            additional_properties=typesystem.Reference(
-                "Header", definitions=definitions
-            )
-        ),
-        # TODO: Header | ReferenceObject
-        # TODO: links
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
+    additional_properties=True,
 )
 
 definitions["MediaType"] = typesystem.Object(
     properties={
-        "schema": JSON_SCHEMA | SCHEMA_REF,
-        "example": typesystem.Any(),
-        # TODO 'examples', 'encoding'
+        "schema": JSONSchema | SCHEMA_REF,
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-)
-
-definitions["Header"] = typesystem.Object(
-    properties={
-        "description": typesystem.Text(),
-        "required": typesystem.Boolean(),
-        "deprecated": typesystem.Boolean(),
-        "allowEmptyValue": typesystem.Boolean(),
-        "style": typesystem.Choice(
-            choices=[
-                "matrix",
-                "label",
-                "form",
-                "simple",
-                "spaceDelimited",
-                "pipeDelimited",
-                "deepObject",
-            ]
-        ),
-        "schema": JSON_SCHEMA | SCHEMA_REF,
-        "example": typesystem.Any(),
-        # TODO: Other fields
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
+    additional_properties=True,
 )
 
 definitions["Components"] = typesystem.Object(
     properties={
-        "schemas": typesystem.Object(additional_properties=JSON_SCHEMA),
-        "responses": typesystem.Object(
-            additional_properties=typesystem.Reference(
-                "Response", definitions=definitions
-            )
-        ),
+        "schemas": typesystem.Object(additional_properties=JSONSchema),
         "parameters": typesystem.Object(
             additional_properties=typesystem.Reference(
                 "Parameter", definitions=definitions
@@ -303,53 +129,10 @@ definitions["Components"] = typesystem.Object(
                 "RequestBody", definitions=definitions
             )
         ),
-        "securitySchemes": typesystem.Object(
-            additional_properties=typesystem.Reference(
-                "SecurityScheme", definitions=definitions
-            )
-        ),
-        # TODO: Other fields
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-)
-
-definitions["Tag"] = typesystem.Object(
-    properties={
-        "name": typesystem.String(),
-        "description": typesystem.Text(allow_blank=True),
-        "externalDocs": typesystem.Reference(
-            "ExternalDocumentation", definitions=definitions
-        ),
-    },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["name"],
-)
-
-definitions["SecurityRequirement"] = typesystem.Object(
-    additional_properties=typesystem.Array(items=typesystem.String())
-)
-
-definitions["SecurityScheme"] = typesystem.Object(
-    properties={
-        "type": typesystem.Choice(
-            choices=["apiKey", "http", "oauth2", "openIdConnect"]
-        ),
-        "description": typesystem.Text(allow_blank=True),
-        "name": typesystem.String(),
-        "in": typesystem.Choice(choices=["query", "header", "cookie"]),
-        "scheme": typesystem.String(),
-        "bearerFormat": typesystem.String(),
-        "flows": typesystem.Any(),  # TODO: OAuthFlows
-        "openIdConnectUrl": typesystem.String(format="url"),
     },
-    pattern_properties={"^x-": typesystem.Any()},
-    additional_properties=False,
-    required=["type"],
+    additional_properties=True,
 )
 
-
 METHODS = ["get", "put", "post", "delete", "options", "head", "patch", "trace"]
 
 
@@ -373,23 +156,13 @@ def _simple_slugify(text):
 
 class OpenAPI:
     def load(self, data):
-        title = lookup(data, ["info", "title"])
-        description = lookup(data, ["info", "description"])
-        version = lookup(data, ["info", "version"])
-        base_url = lookup(data, ["servers", 0, "url"])
         schema_definitions = self.get_schema_definitions(data)
-        content = self.get_content(data, base_url, schema_definitions)
-
-        return Document(
-            title=title,
-            description=description,
-            version=version,
-            url=base_url,
-            content=content,
-        )
+        links = self.get_links(data, schema_definitions)
+
+        return Document(links=links)
 
     def get_schema_definitions(self, data):
-        definitions = typesystem.SchemaDefinitions()
+        definitions = typesystem.Definitions()
         schemas = lookup(data, ["components", "schemas"], {})
         for key, value in schemas.items():
             ref = f"#/components/schemas/{key}"
@@ -398,19 +171,16 @@ class OpenAPI:
             )
         return definitions
 
-    def get_content(self, data, base_url, schema_definitions):
+    def get_links(self, data, schema_definitions):
         """
-        Return all the links in the document, laid out by tag and operationId.
+        Return all the links in the document, laid out by operationId.
         """
-        links_by_tag = {}
         links = []
 
         for path, path_info in data.get("paths", {}).items():
             operations = {key: path_info[key] for key in path_info if key in METHODS}
             for operation, operation_info in operations.items():
-                tag = lookup(operation_info, ["tags", 0])
                 link = self.get_link(
-                    base_url,
                     path,
                     path_info,
                     operation,
@@ -420,36 +190,20 @@ class OpenAPI:
                 if link is None:
                     continue
 
-                if tag is None:
-                    links.append(link)
-                elif tag not in links_by_tag:
-                    links_by_tag[tag] = [link]
-                else:
-                    links_by_tag[tag].append(link)
+                links.append(link)
 
-        sections = [
-            Section(name=_simple_slugify(tag), title=tag.title(), content=links)
-            for tag, links in links_by_tag.items()
-        ]
-        return links + sections
+        return links
 
-    def get_link(
-        self, base_url, path, path_info, operation, operation_info, schema_definitions
-    ):
+    def get_link(self, path, path_info, operation, operation_info, schema_definitions):
         """
         Return a single link in the document.
         """
-        name = operation_info.get("operationId")
-        title = operation_info.get("summary")
-        description = operation_info.get("description")
-
-        if name is None:
-            name = _simple_slugify(title)
-            if not name:
-                return None
+        name = operation_info["operationId"]
+        deprecated = operation_info.get("deprecated", False)
+        paginated = operation_info.get("x-paginated")
 
-        # Allow path info and operation info to override the base url.
-        base_url = lookup(path_info, ["servers", 0, "url"], default=base_url)
+        # Allow path info and operation info to override the base URL, which is normally managed by the ArkindexClient.
+        base_url = lookup(path_info, ["servers", 0, "url"], default=None)
         base_url = lookup(operation_info, ["servers", 0, "url"], default=base_url)
 
         # Parameters are taken both from the path info, and from the operation.
@@ -487,10 +241,10 @@ class OpenAPI:
             name=name,
             url=urljoin(base_url, path),
             method=operation,
-            title=title,
-            description=description,
             fields=fields,
             encoding=encoding,
+            deprecated=deprecated,
+            paginated=paginated,
         )
 
     def get_field(self, parameter, schema_definitions):
@@ -499,10 +253,9 @@ class OpenAPI:
         """
         name = parameter.get("name")
         location = parameter.get("in")
-        description = parameter.get("description")
         required = parameter.get("required", False)
         schema = parameter.get("schema")
-        example = parameter.get("example")
+        deprecated = parameter.get("deprecated", False)
 
         if schema is not None:
             if "$ref" in schema:
@@ -516,8 +269,7 @@ class OpenAPI:
         return Field(
             name=name,
             location=location,
-            description=description,
             required=required,
             schema=schema,
-            example=example,
+            deprecated=deprecated,
         )