b24api 0.5.0__tar.gz → 0.6.1__tar.gz

{b24api-0.5.0/b24api.egg-info → b24api-0.6.1}/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: b24api
-Version: 0.5.0
+Version: 0.6.1
 Summary: Bitrix24 API
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aioitertools>=0.12.0
 Requires-Dist: fast-depends>=2.4.12
 Requires-Dist: httpx[http2]>=0.28.1
 Requires-Dist: pydantic>=2.11.7

b24api-0.6.1/b24api/async_api.py

@@ -0,0 +1,262 @@
+from collections.abc import AsyncGenerator, AsyncIterable, Iterable
+from itertools import batched, islice
+from typing import Any
+
+from aioitertools.itertools import chain as achain
+from httpx import AsyncClient
+from tenacity import AsyncRetrying
+
+from b24api.base_api import BaseBitrix24, BatchedNoCountHelper, ReferenceNoCountHelper
+from b24api.entity import ApiTypes, ListRequest, Request, Response
+
+
+class AsyncBitrix24(BaseBitrix24):
+    @staticmethod
+    def _http() -> type[AsyncClient]:
+        return AsyncClient
+
+    @staticmethod
+    def _retry() -> type[AsyncRetrying]:
+        return AsyncRetrying
+
+    async def _call(self, request: Request | dict) -> Response:
+        """Call any method and return full response."""
+        request = Request.model_validate(request)
+        self.logger.debug("Sending request: %s", request)
+
+        response = await self.http.post(
+            f"{self.settings.webhook_url}{request.method}",
+            headers={"Content-Type": "application/json"},
+            json=request.model_dump(mode="json")["parameters"],
+        )
+        response = self._validate_call_response(request, response)
+        self.logger.debug("Received response: %s", response)
+
+        return response
+
+    async def call(self, request: Request | dict) -> ApiTypes:
+        """Call any method (with retries) and return `result` from response."""
+        return (await self.retry(self._call, request)).result
+
+    async def _batch(self, requests: tuple[Request | dict, ...]) -> list[Response]:
+        """Call limited batch of methods and return full responses."""
+        commands, request = self._batch_requests(requests)
+
+        response = await self._call(request)
+
+        return self._batch_responses(commands, response)
+
+    async def batch(
+        self,
+        requests: Iterable[Request | dict | tuple[Request | dict, Any]],
+        *,
+        batch_size: int | None = None,
+        list_method: bool = False,
+        with_payload: bool = False,
+    ) -> AsyncGenerator[ApiTypes | tuple[ApiTypes, Any]]:
+        """Call unlimited sequence of methods within batches and return `result` from responses."""
+        batch_size = batch_size or self.settings.batch_size
+
+        for batched_requests_ in batched(requests, batch_size):
+            if with_payload:
+                batched_requests, batched_payloads = zip(*batched_requests_, strict=True)
+            else:
+                batched_requests, batched_payloads = batched_requests_, None
+
+            batched_responses = await self.retry(self._batch, batched_requests)
+
+            for i, response in enumerate(batched_responses):
+                result = response.list_result if list_method else response.result
+                if with_payload:
+                    yield result, batched_payloads[i]
+                else:
+                    yield result
+
+    async def list_sequential(
+        self,
+        request: Request | dict,
+        *,
+        list_size: int | None = None,
+    ) -> AsyncGenerator[ApiTypes]:
+        """Call `list` method and return full `result`.
+
+        Slow (sequential tail) list gathering for methods without `filter` parameter (e.g. `department.get`).
+        """
+        request = Request.model_validate(request)
+        list_size = list_size or self.settings.list_size
+
+        head_request = request.model_copy(deep=True)
+        head_request.parameters["start"] = 0
+
+        head_response = await self.retry(self._call, head_request)
+        for item in head_response.list_result:
+            yield item
+
+        tail_requests = self._list_tail_requests(head_request, head_response, list_size=list_size)
+        for tail_request in tail_requests:
+            tail_response = await self.retry(self._call, tail_request)
+
+            start = tail_request.parameters["start"]
+            if tail_response.next and tail_response.next != start + list_size:
+                raise ValueError(
+                    f"Expecting next list chunk to start at {start + list_size}. Got: {tail_response.next}",
+                )
+
+            for item in tail_response.list_result:
+                yield item
+
+    async def list_batched(
+        self,
+        request: Request | dict,
+        *,
+        list_size: int | None = None,
+        batch_size: int | None = None,
+    ) -> AsyncGenerator[ApiTypes]:
+        """Call `list` method and return full `result`.
+
+        Faster (batched tail) list gathering for methods without `filter` parameter (e.g. `department.get`).
+        """
+        request = Request.model_validate(request)
+        list_size = list_size or self.settings.list_size
+        batch_size = batch_size or self.settings.batch_size
+
+        head_request = request.model_copy(deep=True)
+        head_request.parameters["start"] = 0
+
+        head_response = await self.retry(self._call, head_request)
+        for item in head_response.list_result:
+            yield item
+
+        tail_requests = self._list_tail_requests(head_request, head_response, list_size=list_size)
+        tail_responses = self.batch(tail_requests, batch_size=batch_size, list_method=True)
+        async for tail_response in tail_responses:
+            for item in tail_response:
+                yield item
+
+    async def list_batched_no_count(
+        self,
+        request: ListRequest | dict,
+        *,
+        id_key: str = "ID",
+        list_size: int | None = None,
+        batch_size: int | None = None,
+    ) -> AsyncGenerator[ApiTypes]:
+        """Call `list` method and return full `result`.
+
+        Fastest (batched, no count) list gathering for methods with `filter` parameter (e.g. `crm.lead.list`).
+        """
+        list_size = list_size or self.settings.list_size
+        batch_size = batch_size or self.settings.batch_size
+
+        batched_helper = BatchedNoCountHelper(request, id_key, list_size, batch_size)
+
+        boundary_requests = batched_helper.head_request(), batched_helper.tail_request()
+        head_result, tail_result = await self.retry(self._batch, boundary_requests)
+        for item in head_result.list_result:
+            yield item
+
+        body_requests = batched_helper.body_requests(head_result, tail_result)
+        body_results = self.batch(body_requests, batch_size=batch_size, list_method=True)
+        async for item in achain.from_iterable(body_results):
+            yield item
+
+        for item in batched_helper.tail_results(head_result, tail_result):
+            yield item
+
+    async def reference_batched_no_count(  # noqa: PLR0913
+        self,
+        request: ListRequest | dict,
+        updates: Iterable[dict | tuple[dict, Any]] | AsyncIterable[dict | tuple[dict, Any]],
+        *,
+        id_key: str = "ID",
+        list_size: int | None = None,
+        batch_size: int | None = None,
+        with_payload: bool = False,
+    ) -> AsyncGenerator[ApiTypes | tuple[ApiTypes, Any]]:
+        """Call `list` method with reference `updates` for `filter` and return full `result`.
+
+        Fastest (batched, no count) list gathering for methods with `filter` parameter and required `reference`
+        (e.g. `crm.timeline.comment.list`).
+        """
+        list_size = list_size or self.settings.list_size
+        batch_size = batch_size or self.settings.batch_size
+
+        reference_helper = ReferenceNoCountHelper(request, updates, id_key, list_size, batch_size, with_payload)
+
+        if isinstance(updates, AsyncIterable):
+            async for item in self._reference_batched_no_count_async_updates(reference_helper):
+                yield item
+        else:
+            async for item in self._reference_batched_no_count_sync_updates(reference_helper):
+                yield item
+
+    async def _reference_batched_no_count_async_updates(
+        self,
+        reference_helper: ReferenceNoCountHelper,
+    ) -> AsyncGenerator[ApiTypes | tuple[ApiTypes, Any]]:
+        head_requests, body_requests = (), []
+        async for tail_request in reference_helper.atail_requests():
+            body_requests.append(tail_request)
+            if len(head_requests) + len(body_requests) < reference_helper.batch_size:
+                continue
+
+            body_results, head_requests = await self._reference_batched_no_count_async_updates_batch(
+                reference_helper,
+                body_requests,
+                head_requests,
+            )
+            body_requests = []
+            for item in reference_helper.body_results(body_results):
+                yield item
+
+        while head_requests or body_requests:
+            body_results, head_requests = await self._reference_batched_no_count_async_updates_batch(
+                reference_helper,
+                body_requests,
+                head_requests,
+            )
+            body_requests = []
+            for item in reference_helper.body_results(body_results):
+                yield item
+
+    async def _reference_batched_no_count_async_updates_batch(
+        self,
+        reference_helper: ReferenceNoCountHelper,
+        body_requests: list[ListRequest],
+        head_requests: tuple[ListRequest, ...],
+    ) -> tuple[list[ApiTypes], tuple[ListRequest, ...]]:
+        body_requests = head_requests + tuple(body_requests)
+        body_results = [
+            r
+            async for r in self.batch(
+                body_requests,
+                batch_size=reference_helper.batch_size,
+                list_method=True,
+                with_payload=reference_helper.with_payload,
+            )
+        ]
+        head_requests = reference_helper.head_requests(body_requests, body_results)
+
+        return body_results, head_requests
+
+    async def _reference_batched_no_count_sync_updates(
+        self,
+        reference_helper: ReferenceNoCountHelper,
+    ) -> AsyncGenerator[ApiTypes | tuple[ApiTypes, Any]]:
+        head_requests = ()
+        tail_requests = iter(reference_helper.tail_requests())
+        while body_requests := head_requests + tuple(
+            islice(tail_requests, reference_helper.batch_size - len(head_requests)),
+        ):
+            body_results = [
+                r
+                async for r in self.batch(
+                    body_requests,
+                    batch_size=reference_helper.batch_size,
+                    list_method=True,
+                    with_payload=reference_helper.with_payload,
+                )
+            ]
+            head_requests = reference_helper.head_requests(body_requests, body_results)
+            for item in reference_helper.body_results(body_results):
+                yield item

{b24api-0.5.0 → b24api-0.6.1}/b24api/base_api.py

@@ -1,7 +1,8 @@
 import contextlib
 import logging
 from abc import ABC, abstractmethod
-from collections.abc import Generator, Iterable
+from collections.abc import AsyncGenerator, AsyncIterable, Generator, Iterable
+from itertools import chain
 from operator import itemgetter
 from typing import Any
 
@@ -21,6 +22,7 @@ from tenacity import (
 from b24api.entity import BatchResult, ErrorResponse, ListRequest, Request, Response
 from b24api.error import RetryApiResponseError, RetryHTTPStatusError
 from b24api.settings import ApiSettings
+from b24api.type import ApiTypes
 
 
 class BaseBitrix24(ABC):
@@ -55,7 +57,7 @@ class BaseBitrix24(ABC):
     def _retry() -> type[Retrying | AsyncRetrying]:
         pass
 
-    def _estimate_call_response(self, request: Request, response: httpx.Response) -> Response:
+    def _validate_call_response(self, request: Request, response: httpx.Response) -> Response:
         # Checking more informative errors first (content may exist with 5xx status)
         with contextlib.suppress(httpx.ResponseNotRead, ValidationError):
             ErrorResponse.model_validate_json(response.content).raise_error(request, self.settings.retry_errors)
@@ -74,9 +76,10 @@ class BaseBitrix24(ABC):
         return Response.model_validate_json(response.content)
 
     @staticmethod
-    def _estimate_batch_requests(requests: Iterable[Request | dict]) -> tuple[dict[str, Request], Request]:
-        # Using string keys to simplify errors extraction
-        commands = {f"_{i}": Request.model_validate(request) for i, request in enumerate(requests)}
+    def _batch_requests(requests: tuple[Request | dict, ...]) -> tuple[dict[str, Request], Request]:
+        # Using string keys with equal length to keep requests order and simplify errors extraction
+        width = len(str(len(requests)))
+        commands = {f"_{i:0>{width}d}": Request.model_validate(request) for i, request in enumerate(requests)}
         request = Request(
             method="batch",
             parameters={
@@ -87,8 +90,12 @@ class BaseBitrix24(ABC):
 
         return commands, request
 
-    def _estimate_batch_responses(self, commands: dict[str, Request], result: Any) -> list[Response]:
-        result = BatchResult.model_validate(result)
+    def _batch_responses(
+        self,
+        commands: dict[str, Request],
+        response: Response,
+    ) -> list[Response]:
+        result = BatchResult.model_validate(response.result)
 
         responses = []
         for key, command in commands.items():
@@ -114,11 +121,10 @@ class BaseBitrix24(ABC):
                     next=result.result_next.get(key, None),
                 ),
             )
-
         return responses
 
     @staticmethod
-    def _estimate_list_tail_requests(
+    def _list_tail_requests(
         head_request: Request,
         head_response: Response,
         *,
@@ -133,12 +139,11 @@ class BaseBitrix24(ABC):
             tail_request.parameters["start"] = start
             yield tail_request
 
-    @staticmethod
-    def _estimate_list_batched_no_count_boundary_requests(
-        request: ListRequest | dict,
-        id_key: str,
-    ) -> tuple[ListRequest, ListRequest]:
+
+class BatchedNoCountHelper:
+    def __init__(self, request: ListRequest | dict, id_key: str, list_size: int, batch_size: int) -> None:
         request = ListRequest.model_validate(request)
+
         select_ = request.parameters.select
         if "*" not in select_ and id_key not in select_:
             request.select.append(id_key)
@@ -154,44 +159,56 @@ class BaseBitrix24(ABC):
         if request.parameters.order:
             raise ValueError("Ordering parameters are reserved in `list_batched_no_count`")
 
-        head_request = request.model_copy(deep=True)
-        head_request.parameters.start = -1
-        head_request.parameters.order = {"ID": "ASC"}
+        self.request = request
+        self.id_key = id_key
+        self.list_size = list_size
+        self.batch_size = batch_size
 
-        tail_request = request.model_copy(deep=True)
-        tail_request.parameters.start = -1
-        tail_request.parameters.order = {"ID": "DESC"}
+        self.get_id = itemgetter(id_key)
 
-        return head_request, tail_request
+        self.id_from = f">{self.id_key}"
+        self.id_to = f"<{self.id_key}"
 
-    @staticmethod
-    def _estimate_list_batched_no_count_body_requests(
-        head_request: ListRequest,
-        head_result: list,
-        tail_result: list,
-        id_key: str,
-        list_size: int,
-    ) -> Generator[ListRequest]:
-        get_id = itemgetter(id_key)
-        max_head_id = max(map(int, map(get_id, head_result)), default=None)
-        min_tail_id = min(map(int, map(get_id, tail_result)), default=None)
+    def head_request(self) -> ListRequest:
+        request = self.request.model_copy(deep=True)
+        request.parameters.start = -1
+        request.parameters.order = {"ID": "ASC"}
+        return request
 
-        if max_head_id and min_tail_id and max_head_id < min_tail_id:
-            id_from, id_to = f">{id_key}", f"<{id_key}"
+    def tail_request(self) -> ListRequest:
+        request = self.request.model_copy(deep=True)
+        request.parameters.start = -1
+        request.parameters.order = {"ID": "DESC"}
+        return request
 
-            for start in range(max_head_id, min_tail_id, list_size):
-                body_request = head_request.model_copy(deep=True)
-                body_request.parameters.filter[id_from] = start
-                body_request.parameters.filter[id_to] = min(start + list_size + 1, min_tail_id)
+    def body_requests(self, head_result: Response, tail_result: Response) -> Generator[ListRequest]:
+        max_head_id = max(map(int, map(self.get_id, head_result.list_result)), default=None)
+        min_tail_id = min(map(int, map(self.get_id, tail_result.list_result)), default=None)
+
+        if max_head_id and min_tail_id and max_head_id < min_tail_id:
+            for start in range(max_head_id, min_tail_id, self.list_size):
+                body_request = self.head_request()
+                body_request.parameters.filter[self.id_from] = start
+                body_request.parameters.filter[self.id_to] = min(start + self.list_size + 1, min_tail_id)
                 yield body_request
 
-    @staticmethod
-    def _estimate_reference_batched_no_count_tail_requests(
+    def tail_results(self, head_result: Response, tail_result: Response) -> Generator[ApiTypes]:
+        max_head_id = max(map(int, map(self.get_id, head_result.list_result)), default=None)
+        for item in reversed(tail_result.list_result):
+            if int(self.get_id(item)) > max_head_id:
+                yield item
+
+
+class ReferenceNoCountHelper:
+    def __init__(  # noqa: PLR0913
+        self,
         request: ListRequest | dict,
-        updates: Iterable[dict | tuple[dict, Any]],
+        updates: Iterable[dict | tuple[dict, Any]] | AsyncIterable[dict | tuple[dict, Any]],
         id_key: str,
-        with_payload: bool,
-    ) -> Generator[ListRequest | tuple[ListRequest, Any]]:
+        list_size: int,
+        batch_size: int,
+        with_payload: bool,  # noqa: FBT001
+    ) -> None:
         request = ListRequest.model_validate(request)
 
         select_ = request.parameters.select
@@ -208,84 +225,85 @@ class BaseBitrix24(ABC):
         if request.parameters.order:
             raise ValueError("Ordering parameters are reserved `order`in `reference_batched_no_count`")
 
-        for update in updates:
-            if with_payload:
-                update, payload = update
-            else:
-                payload = None
-
-            if id_from in update:
-                raise ValueError(
-                    f"Filter parameters `{id_from}` is reserved in `reference_batched_no_count`",
-                )
+        self.request = request
+        self.updates = updates
+        self.id_key = id_key
+        self.list_size = list_size
+        self.batch_size = batch_size
+        self.with_payload = with_payload
 
-            tail_request = request.model_copy(deep=True)
-            tail_request.parameters.filter |= update
-            tail_request.parameters.start = -1
-            tail_request.parameters.order = {"ID": "ASC"}
+        self.get_id = itemgetter(id_key)
 
-            if with_payload:
-                yield tail_request, payload
-            else:
-                yield tail_request
+        self.id_from = f">{self.id_key}"
+        self.id_to = f"<{self.id_key}"
 
-    @staticmethod
-    def _estimate_reference_batched_no_count_result_next(
-        body_request: ListRequest,
-        body_result: list,
-        body_payload: Any,
-        id_key: str,
-        list_size: int,
-        with_payload: bool,
-    ) -> tuple[[list[ListRequest | tuple[ListRequest, Any]]], Iterable[Any | tuple[Any, Any]]]:
-        id_from = f">{id_key}"
-        get_id = itemgetter(id_key)
+    def tail_requests(self) -> Generator[ListRequest | tuple[ListRequest, Any]]:
+        if isinstance(self.updates, AsyncIterable):
+            raise TypeError("Use `atail_requests` to get asynchronous tail requests")
 
-        head_requests = []
-        body_result = BaseBitrix24._fix_list_result(body_result)
+        for update in self.updates:
+            yield self._updated_request(update)
 
-        if len(body_result) == list_size:
-            max_id = max(map(int, map(get_id, body_result)), default=None)
-            head_request = body_request.model_copy(deep=True)
-            head_request.parameters.filter[id_from] = max_id
-            if with_payload:
-                head_requests.append((head_request, body_payload))
-            else:
-                head_requests.append(head_request)
+    async def atail_requests(self) -> AsyncGenerator[ListRequest | tuple[ListRequest, Any]]:
+        if isinstance(self.updates, Iterable):
+            raise TypeError("Use `atail_requests` to get asynchronous tail requests")
 
-        if with_payload:
-            body_payload = [body_payload] * len(body_result)
-            body_result = zip(body_result, body_payload, strict=True)
+        async for update in self.updates:
+            yield self._updated_request(update)
 
-        return head_requests, body_result
+    def _updated_request(self, update: dict | tuple[dict, Any]) -> ListRequest | tuple[ListRequest, Any]:
+        if self.with_payload:
+            update, payload = update
+        else:
+            payload = None
 
-    @staticmethod
-    def _fix_list_result(result: list | dict[str, list]) -> list:
-        """Fix `list` method result to `list of items` structure.
-
-        There are two kinds of what `list` method `result` may contain:
-        - a list of items (e.g. `department-get` and `disk.folder.getchildren`),
-        - a dictionary with single item that contains the desired list of items
-            (e.g. `tasks` in `tasks.task.list`).
-        """
-        if not isinstance(result, list | dict):
-            raise TypeError(f"Expecting `result` to be a `list` or a `dict`. Got: {result}")
-
-        if not result:
-            return []
-
-        if isinstance(result, list):
-            return result
-
-        if len(result) != 1:
-            raise TypeError(
-                f"If `result` is a `dict`, expecting single item. Got: {result}",
+        if self.id_from in update:
+            raise ValueError(
+                f"Filter parameters `{self.id_from}` is reserved in `reference_batched_no_count`",
             )
 
-        key = next(iter(result))
-        value = result[key]
-
-        if not isinstance(value, list):
-            raise TypeError(f"If `result` is a `dict`, expecting single item to be a `list`. Got: {result}")
-
-        return value
+        tail_request = self.request.model_copy(deep=True)
+        tail_request.parameters.filter |= update
+        tail_request.parameters.start = -1
+        tail_request.parameters.order = {"ID": "ASC"}
+
+        if self.with_payload:
+            return tail_request, payload
+
+        return tail_request
+
+    def head_requests(
+        self,
+        body_requests: tuple[ListRequest | tuple[ListRequest, Any], ...],
+        body_results: list[ApiTypes | tuple[ApiTypes, Any]],
+    ) -> tuple[ListRequest | tuple[ListRequest, Any], ...]:
+        requests = []
+        for body_request_, body_result_ in zip(body_requests, body_results, strict=True):
+            if self.with_payload:
+                body_request, body_payload = body_request_
+                body_result, _ = body_result_
+            else:
+                body_request, body_result, body_payload = body_request_, body_result_, None
+
+            id_from = f">{self.id_key}"
+
+            if len(body_result) == self.list_size:
+                max_id = max(map(int, map(self.get_id, body_result)))
+                head_request = body_request.model_copy(deep=True)
+                head_request.parameters.filter[id_from] = max_id
+                if self.with_payload:
+                    requests.append((head_request, body_payload))
+                else:
+                    requests.append(head_request)
+
+        return tuple(requests)
+
+    def body_results(
+        self,
+        results: list[ApiTypes | tuple[ApiTypes, Any]],
+    ) -> Generator[ApiTypes | tuple[ApiTypes, Any]]:
+        if self.with_payload:
+            for result, payload in results:
+                yield from zip(result, [payload] * len(result), strict=False)
+        else:
+            yield from chain.from_iterable(results)

{b24api-0.5.0 → b24api-0.6.1}/b24api/entity.py

@@ -1,7 +1,7 @@
 from datetime import datetime
-from typing import Annotated, Any
+from typing import Any
 
-from pydantic import BaseModel, BeforeValidator, field_validator
+from pydantic import BaseModel, field_validator
 
 from b24api.error import ApiResponseError, RetryApiResponseError
 from b24api.query import build_query
@@ -58,13 +58,8 @@ class ErrorResponse(BaseModel):
         return value
 
     def raise_error(self, request: Request, retry_errors: list[str]) -> None:
-        if self.error in retry_errors:
-            raise RetryApiResponseError(
-                code=self.error,
-                description=self.error_description,
-                request=str(request),
-            )
-        raise ApiResponseError(
+        error_cls = RetryApiResponseError if self.error in retry_errors else ApiResponseError
+        raise error_cls(
             code=self.error,
             description=self.error_description,
             request=str(request),
@@ -92,18 +87,50 @@ class Response(BaseModel):
     total: int | None = None
     next: int | None = None
 
+    @property
+    def list_result(self) -> list[ApiTypes]:
+        """Fix `list` methods result to `list of items` structure.
+
+        There are two kinds of what `list` method `result` may contain:
+        - a list of items (e.g. `department-get` and `disk.folder.getchildren`),
+        - a dictionary with single item that contains the desired list of items
+            (e.g. `tasks` in `tasks.task.list`).
+        """
+        if not isinstance(self.result, list | dict):
+            raise TypeError(f"Expecting `result` to be a `list` or a `dict`. Got: {self.result}")
+
+        if not self.result:
+            return []
+
+        if isinstance(self.result, list):
+            return self.result
+
+        if len(self.result) != 1:
+            raise TypeError(
+                f"If `result` is a `dict`, expecting single item. Got: {self.result}",
+            )
 
-def _php_dict(value: Any) -> Any:  # noqa: ANN401
-    if isinstance(value, list) and not value:
-        return {}
-    return value
+        key = next(iter(self.result))
+        value = self.result[key]
+
+        if not isinstance(value, list):
+            raise TypeError(f"If `result` is a `dict`, expecting single item to be a `list`. Got: {self.result}")
+
+        return value
 
 
 class BatchResult(BaseModel):
     """API response `result` structure for `batch` method."""
 
-    result: Annotated[dict[str, ApiTypes], BeforeValidator(_php_dict)]
-    result_time: Annotated[dict[str, ResponseTime], BeforeValidator(_php_dict)]
-    result_error: Annotated[dict[str, ErrorResponse], BeforeValidator(_php_dict)]
-    result_total: Annotated[dict[str, int], BeforeValidator(_php_dict)]
-    result_next: Annotated[dict[str, int], BeforeValidator(_php_dict)]
+    result: dict[str, ApiTypes]
+    result_time: dict[str, ResponseTime]
+    result_error: dict[str, ErrorResponse]
+    result_total: dict[str, int]
+    result_next: dict[str, int]
+
+    @field_validator("result", "result_time", "result_error", "result_total", "result_next", mode="before")
+    @classmethod
+    def php_dict(cls, value: Any) -> Any:  # noqa: ANN401
+        if isinstance(value, list) and not value:
+            return {}
+        return value

{b24api-0.5.0 → b24api-0.6.1}/b24api/sync_api.py

@@ -1,12 +1,11 @@
 from collections.abc import Generator, Iterable
 from itertools import batched, chain, islice
-from operator import itemgetter
 from typing import Any
 
 from httpx import Client
 from tenacity import Retrying
 
-from b24api.base_api import BaseBitrix24
+from b24api.base_api import BaseBitrix24, BatchedNoCountHelper, ReferenceNoCountHelper
 from b24api.entity import ApiTypes, ListRequest, Request, Response
 
 
@@ -19,10 +18,6 @@ class SyncBitrix24(BaseBitrix24):
     def _retry() -> type[Retrying]:
         return Retrying
 
-    def call(self, request: Request | dict) -> ApiTypes:
-        """Call any method (with retries) and return `result` from response."""
-        return self.retry(self._call, request).result
-
     def _call(self, request: Request | dict) -> Response:
         """Call any method and return full response."""
         request = Request.model_validate(request)
@@ -33,41 +28,48 @@ class SyncBitrix24(BaseBitrix24):
             headers={"Content-Type": "application/json"},
             json=request.model_dump(mode="json")["parameters"],
         )
-        response = self._estimate_call_response(request, response)
+        response = self._validate_call_response(request, response)
         self.logger.debug("Received response: %s", response)
 
         return response
 
+    def call(self, request: Request | dict) -> ApiTypes:
+        """Call any method (with retries) and return `result` from response."""
+        return self.retry(self._call, request).result
+
+    def _batch(self, requests: tuple[Request | dict, ...]) -> list[Response]:
+        """Call limited batch of methods and return full responses."""
+        commands, request = self._batch_requests(requests)
+
+        response = self._call(request)
+
+        return self._batch_responses(commands, response)
+
     def batch(
         self,
         requests: Iterable[Request | dict | tuple[Request | dict, Any]],
         *,
         batch_size: int | None = None,
+        list_method: bool = False,
         with_payload: bool = False,
     ) -> Generator[ApiTypes | tuple[ApiTypes, Any]]:
         """Call unlimited sequence of methods within batches and return `result` from responses."""
         batch_size = batch_size or self.settings.batch_size
 
-        for batched_requests in batched(requests, batch_size):
+        for batched_requests_ in batched(requests, batch_size):
             if with_payload:
-                batched_requests, batched_payloads = zip(*batched_requests, strict=True)
+                batched_requests, batched_payloads = zip(*batched_requests_, strict=True)
             else:
-                batched_payloads = None
+                batched_requests, batched_payloads = batched_requests_, None
 
             batched_responses = self.retry(self._batch, batched_requests)
+
             for i, response in enumerate(batched_responses):
+                result = response.list_result if list_method else response.result
                 if with_payload:
-                    yield response.result, batched_payloads[i]
+                    yield result, batched_payloads[i]
                 else:
-                    yield response.result
-
-    def _batch(self, requests: Iterable[Request | dict]) -> list[Response]:
-        """Call limited batch of methods and return full responses."""
-        commands, request = self._estimate_batch_requests(requests)
-
-        result = self._call(request).result
-
-        return self._estimate_batch_responses(commands, result)
+                    yield result
 
     def list_sequential(
         self,
@@ -86,9 +88,9 @@ class SyncBitrix24(BaseBitrix24):
         head_request.parameters["start"] = 0
 
         head_response = self.retry(self._call, head_request)
-        yield from self._fix_list_result(head_response.result)
+        yield from head_response.list_result
 
-        tail_requests = self._estimate_list_tail_requests(head_request, head_response, list_size=list_size)
+        tail_requests = self._list_tail_requests(head_request, head_response, list_size=list_size)
         for tail_request in tail_requests:
             tail_response = self.retry(self._call, tail_request)
 
@@ -98,7 +100,7 @@ class SyncBitrix24(BaseBitrix24):
                     f"Expecting next list chunk to start at {start + list_size}. Got: {tail_response.next}",
                 )
 
-            yield from self._fix_list_result(tail_response.result)
+            yield from tail_response.list_result
 
     def list_batched(
         self,
@@ -119,11 +121,10 @@ class SyncBitrix24(BaseBitrix24):
         head_request.parameters["start"] = 0
 
         head_response = self.retry(self._call, head_request)
-        yield from self._fix_list_result(head_response.result)
+        yield from head_response.list_result
 
-        tail_requests = self._estimate_list_tail_requests(head_request, head_response, list_size=list_size)
-        tail_responses = self.batch(tail_requests, batch_size=batch_size)
-        tail_responses = map(self._fix_list_result, tail_responses)
+        tail_requests = self._list_tail_requests(head_request, head_response, list_size=list_size)
+        tail_responses = self.batch(tail_requests, batch_size=batch_size, list_method=True)
         tail_responses = chain.from_iterable(tail_responses)
         yield from tail_responses
 
@@ -142,31 +143,19 @@ class SyncBitrix24(BaseBitrix24):
         list_size = list_size or self.settings.list_size
         batch_size = batch_size or self.settings.batch_size
 
-        head_request, tail_request = self._estimate_list_batched_no_count_boundary_requests(request, id_key)
+        batched_helper = BatchedNoCountHelper(request, id_key, list_size, batch_size)
 
-        boundary_results = self.batch([head_request, tail_request], batch_size=batch_size)
-        head_result, tail_result = tuple(map(self._fix_list_result, boundary_results))
-        yield from head_result
+        boundary_requests = batched_helper.head_request(), batched_helper.tail_request()
+        head_result, tail_result = self.retry(self._batch, boundary_requests)
+        yield from head_result.list_result
 
-        body_requests = self._estimate_list_batched_no_count_body_requests(
-            head_request,
-            head_result,
-            tail_result,
-            id_key,
-            list_size,
-        )
-        body_results = self.batch(body_requests, batch_size=batch_size)
-        body_results = map(self._fix_list_result, body_results)
-        body_results = chain.from_iterable(body_results)
-        yield from body_results
-
-        get_id = itemgetter(id_key)
-        max_head_id = max(map(int, map(get_id, head_result)), default=None)
-        for item in reversed(tail_result):
-            if int(get_id(item)) > max_head_id:
-                yield item
-
-    def reference_batched_no_count(
+        body_requests = batched_helper.body_requests(head_result, tail_result)
+        body_results = self.batch(body_requests, batch_size=batch_size, list_method=True)
+        yield from chain.from_iterable(body_results)
+
+        yield from batched_helper.tail_results(head_result, tail_result)
+
+    def reference_batched_no_count(  # noqa: PLR0913
         self,
         request: ListRequest | dict,
         updates: Iterable[dict | tuple[dict, Any]],
@@ -184,32 +173,12 @@ class SyncBitrix24(BaseBitrix24):
         list_size = list_size or self.settings.list_size
         batch_size = batch_size or self.settings.batch_size
 
-        head_requests = []
-        tail_requests = iter(
-            self._estimate_reference_batched_no_count_tail_requests(
-                request,
-                updates,
-                id_key,
-                with_payload,
-            ),
-        )
-        while body_requests := head_requests + list(islice(tail_requests, batch_size - len(head_requests))):
-            if with_payload:
-                body_requests, body_payloads = zip(*body_requests, strict=True)
-            else:
-                body_payloads = None
-
-            body_results = self.batch(body_requests, batch_size=batch_size)
-
-            head_requests = []
-            for i, (body_request, body_result) in enumerate(zip(body_requests, body_results, strict=True)):
-                body_payload = body_payloads[i] if with_payload else None
-                head_requests, body_result = self._estimate_reference_batched_no_count_result_next(
-                    body_request,
-                    body_result,
-                    body_payload,
-                    id_key,
-                    list_size,
-                    with_payload,
-                )
-                yield from body_result
+        reference_helper = ReferenceNoCountHelper(request, updates, id_key, list_size, batch_size, with_payload)
+
+        head_requests = ()
+        tail_requests = iter(reference_helper.tail_requests())
+        while body_requests := head_requests + tuple(islice(tail_requests, batch_size - len(head_requests))):
+            body_results = self.batch(body_requests, batch_size=batch_size, list_method=True, with_payload=with_payload)
+            body_results = list(body_results)
+            head_requests = reference_helper.head_requests(body_requests, body_results)
+            yield from reference_helper.body_results(body_results)

b24api-0.6.1/b24api/type.py

@@ -0,0 +1,4 @@
+from datetime import datetime
+
+# Types allowed in response and request
+type ApiTypes = bool | str | int | float | dict | list | datetime | None

{b24api-0.5.0 → b24api-0.6.1/b24api.egg-info}/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: b24api
-Version: 0.5.0
+Version: 0.6.1
 Summary: Bitrix24 API
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aioitertools>=0.12.0
 Requires-Dist: fast-depends>=2.4.12
 Requires-Dist: httpx[http2]>=0.28.1
 Requires-Dist: pydantic>=2.11.7

{b24api-0.5.0 → b24api-0.6.1}/b24api.egg-info/requires.txt

@@ -1,3 +1,4 @@
+aioitertools>=0.12.0
 fast-depends>=2.4.12
 httpx[http2]>=0.28.1
 pydantic>=2.11.7

{b24api-0.5.0 → b24api-0.6.1}/pyproject.toml

@@ -16,6 +16,7 @@ description = "Bitrix24 API"
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
+    "aioitertools>=0.12.0",
     "fast-depends>=2.4.12",
     "httpx[http2]>=0.28.1",
     "pydantic>=2.11.7",

b24api-0.5.0/b24api/async_api.py

@@ -1,221 +0,0 @@
-from collections.abc import AsyncGenerator, Iterable
-from itertools import batched, islice
-from operator import itemgetter
-from typing import Any
-
-from httpx import AsyncClient
-from tenacity import AsyncRetrying
-
-from b24api.base_api import BaseBitrix24
-from b24api.entity import ApiTypes, ListRequest, Request, Response
-
-
-class AsyncBitrix24(BaseBitrix24):
-    @staticmethod
-    def _http() -> type[AsyncClient]:
-        return AsyncClient
-
-    @staticmethod
-    def _retry() -> type[AsyncRetrying]:
-        return AsyncRetrying
-
-    async def call(self, request: Request | dict) -> ApiTypes:
-        """Call any method (with retries) and return `result` from response."""
-        return (await self.retry(self._call, request)).result
-
-    async def _call(self, request: Request | dict) -> Response:
-        """Call any method and return full response."""
-        request = Request.model_validate(request)
-        self.logger.debug("Sending request: %s", request)
-
-        response = await self.http.post(
-            f"{self.settings.webhook_url}{request.method}",
-            headers={"Content-Type": "application/json"},
-            json=request.model_dump(mode="json")["parameters"],
-        )
-        response = self._estimate_call_response(request, response)
-        self.logger.debug("Received response: %s", response)
-
-        return response
-
-    async def batch(
-        self,
-        requests: Iterable[Request | dict | tuple[Request | dict, Any]],
-        *,
-        batch_size: int | None = None,
-        with_payload: bool = False,
-    ) -> AsyncGenerator[ApiTypes | tuple[ApiTypes, Any]]:
-        """Call unlimited sequence of methods within batches and return `result` from responses."""
-        batch_size = batch_size or self.settings.batch_size
-
-        for batched_requests in batched(requests, batch_size):
-            if with_payload:
-                batched_requests, batched_payloads = zip(*batched_requests, strict=True)
-            else:
-                batched_payloads = None
-
-            batched_responses = await self.retry(self._batch, batched_requests)
-            for i, response in enumerate(batched_responses):
-                if with_payload:
-                    yield response.result, batched_payloads[i]
-                else:
-                    yield response.result
-
-    async def _batch(self, requests: Iterable[Request | dict]) -> list[Response]:
-        """Call limited batch of methods and return full responses."""
-        commands, request = self._estimate_batch_requests(requests)
-
-        result = (await self._call(request)).result
-
-        return self._estimate_batch_responses(commands, result)
-
-    async def list_sequential(
-        self,
-        request: Request | dict,
-        *,
-        list_size: int | None = None,
-    ) -> AsyncGenerator[ApiTypes]:
-        """Call `list` method and return full `result`.
-
-        Slow (sequential tail) list gathering for methods without `filter` parameter (e.g. `department.get`).
-        """
-        request = Request.model_validate(request)
-        list_size = list_size or self.settings.list_size
-
-        head_request = request.model_copy(deep=True)
-        head_request.parameters["start"] = 0
-
-        head_response = await self.retry(self._call, head_request)
-        for item in self._fix_list_result(head_response.result):
-            yield item
-
-        tail_requests = self._estimate_list_tail_requests(head_request, head_response, list_size=list_size)
-        for tail_request in tail_requests:
-            tail_response = await self.retry(self._call, tail_request)
-
-            start = tail_request.parameters["start"]
-            if tail_response.next and tail_response.next != start + list_size:
-                raise ValueError(
-                    f"Expecting next list chunk to start at {start + list_size}. Got: {tail_response.next}",
-                )
-
-            for item in self._fix_list_result(tail_response.result):
-                yield item
-
-    async def list_batched(
-        self,
-        request: Request | dict,
-        *,
-        list_size: int | None = None,
-        batch_size: int | None = None,
-    ) -> AsyncGenerator[ApiTypes]:
-        """Call `list` method and return full `result`.
-
-        Faster (batched tail) list gathering for methods without `filter` parameter (e.g. `department.get`).
-        """
-        request = Request.model_validate(request)
-        list_size = list_size or self.settings.list_size
-        batch_size = batch_size or self.settings.batch_size
-
-        head_request = request.model_copy(deep=True)
-        head_request.parameters["start"] = 0
-
-        head_response = await self.retry(self._call, head_request)
-        for item in self._fix_list_result(head_response.result):
-            yield item
-
-        tail_requests = self._estimate_list_tail_requests(head_request, head_response, list_size=list_size)
-        async for tail_response in self.batch(tail_requests, batch_size=batch_size):
-            for item in self._fix_list_result(tail_response):
-                yield item
-
-    async def list_batched_no_count(
-        self,
-        request: ListRequest | dict,
-        *,
-        id_key: str = "ID",
-        list_size: int | None = None,
-        batch_size: int | None = None,
-    ) -> AsyncGenerator[ApiTypes]:
-        """Call `list` method and return full `result`.
-
-        Fastest (batched, no count) list gathering for methods with `filter` parameter (e.g. `crm.lead.list`).
-        """
-        list_size = list_size or self.settings.list_size
-        batch_size = batch_size or self.settings.batch_size
-
-        head_request, tail_request = self._estimate_list_batched_no_count_boundary_requests(request, id_key)
-
-        boundary_results = [r async for r in self.batch([head_request, tail_request], batch_size=batch_size)]
-        head_result, tail_result = tuple(map(self._fix_list_result, boundary_results))
-        for item in head_result:
-            yield item
-
-        body_requests = self._estimate_list_batched_no_count_body_requests(
-            head_request,
-            head_result,
-            tail_result,
-            id_key,
-            list_size,
-        )
-        async for body_result in self.batch(body_requests, batch_size=batch_size):
-            body_result = self._fix_list_result(body_result)
-            for item in body_result:
-                yield item
-
-        get_id = itemgetter(id_key)
-        max_head_id = max(map(int, map(get_id, head_result)), default=None)
-        for item in reversed(tail_result):
-            if int(get_id(item)) > max_head_id:
-                yield item
-
-    async def reference_batched_no_count(
-        self,
-        request: ListRequest | dict,
-        updates: Iterable[dict | tuple[dict, Any]],
-        *,
-        id_key: str = "ID",
-        list_size: int | None = None,
-        batch_size: int | None = None,
-        with_payload: bool = False,
-    ) -> AsyncGenerator[ApiTypes | tuple[ApiTypes, Any]]:
-        """Call `list` method with reference `updates` for `filter` and return full `result`.
-
-        Fastest (batched, no count) list gathering for methods with `filter` parameter and required `reference`
-        (e.g. `crm.timeline.comment.list`).
-        """
-        list_size = list_size or self.settings.list_size
-        batch_size = batch_size or self.settings.batch_size
-
-        head_requests = []
-        tail_requests = iter(
-            self._estimate_reference_batched_no_count_tail_requests(
-                request,
-                updates,
-                id_key,
-                with_payload,
-            ),
-        )
-        while body_requests := head_requests + list(islice(tail_requests, batch_size - len(head_requests))):
-            if with_payload:
-                body_requests, body_payloads = zip(*body_requests, strict=True)
-            else:
-                body_payloads = None
-
-            head_requests = []
-
-            i = 0
-            async for body_result in self.batch(body_requests, batch_size=batch_size):
-                body_request = body_requests[i]
-                body_payload = body_payloads[i] if with_payload else None
-                head_requests, body_result = self._estimate_reference_batched_no_count_result_next(
-                    body_request,
-                    body_result,
-                    body_payload,
-                    id_key,
-                    list_size,
-                    with_payload,
-                )
-                for item in body_result:
-                    yield item
-                i += 1

b24api-0.5.0/b24api/type.py

@@ -1,5 +0,0 @@
-from datetime import datetime
-from typing import TypeAlias
-
-# Types allowed in response and request
-ApiTypes: TypeAlias = bool | str | int | float | dict | list | datetime | None