fluidattacks_zoho_sdk 1.0.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

fluidattacks_zoho_sdk/_http_client/_core.py

@@ -0,0 +1,151 @@
+from __future__ import (
+    annotations,
+)
+
+from collections.abc import Callable
+from dataclasses import (
+    dataclass,
+    field,
+)
+
+from fa_purity import Cmd, Coproduct, FrozenList, Result, ResultE, UnitType, unit
+from fa_purity.json import (
+    JsonObj,
+)
+from pure_requests.retry import (
+    MaxRetriesReached,
+)
+from requests import Response
+from requests.exceptions import (
+    ChunkedEncodingError as RawChunkedEncodingError,
+)
+from requests.exceptions import (
+    ConnectionError as RawConnectionError,
+)
+from requests.exceptions import (
+    HTTPError as RawHTTPError,
+)
+from requests.exceptions import (
+    JSONDecodeError as RawJSONDecodeError,
+)
+from requests.exceptions import (
+    RequestException as RawRequestException,
+)
+
+from fluidattacks_zoho_sdk.auth import Token
+
+
+@dataclass(frozen=True)
+class UnhandledErrors:
+    error: Coproduct[JSONDecodeError, Coproduct[HTTPError, RequestException]]
+
+
+@dataclass(frozen=True)
+class HandledErrors:
+    error: Coproduct[
+        HTTPError,
+        Coproduct[ChunkedEncodingError, RequestsConnectionError],
+    ]
+
+
+@dataclass(frozen=True)
+class _Private:
+    pass
+
+
+@dataclass(frozen=True)
+class HTTPError:
+    raw: RawHTTPError
+
+
+@dataclass(frozen=True)
+class JSONDecodeError:
+    raw: RawJSONDecodeError
+
+
+@dataclass(frozen=True)
+class ChunkedEncodingError:
+    _private: _Private = field(repr=False, hash=False, compare=False)
+    raw: RawChunkedEncodingError
+
+
+@dataclass(frozen=True)
+class RequestsConnectionError:
+    _private: _Private = field(repr=False, hash=False, compare=False)
+    raw: RawConnectionError
+
+
+@dataclass(frozen=True)
+class RequestException:
+    raw: RawRequestException
+
+    def to_chunk_error(self) -> ResultE[ChunkedEncodingError]:
+        if isinstance(self.raw, RawChunkedEncodingError):
+            return Result.success(ChunkedEncodingError(_Private(), self.raw))
+        return Result.failure(ValueError("Not a ChunkedEncodingError"))
+
+    def to_connection_error(self) -> ResultE[RequestsConnectionError]:
+        if isinstance(self.raw, RawConnectionError):
+            return Result.success(RequestsConnectionError(_Private(), self.raw))
+        return Result.failure(ValueError("Not a RequestsConnectionError"))
+
+
+@dataclass(frozen=True)
+class RelativeEndpoint:
+    paths: FrozenList[str]
+
+    @staticmethod
+    def new(*args: str) -> RelativeEndpoint:
+        return RelativeEndpoint(tuple(args))
+
+
+@dataclass(frozen=True)
+class HttpJsonClient:
+    get: Callable[
+        [RelativeEndpoint, JsonObj],  # Token
+        Cmd[
+            Result[
+                Coproduct[JsonObj, FrozenList[JsonObj]],
+                Coproduct[UnhandledErrors, MaxRetriesReached],
+            ]
+        ],
+    ]
+    post: Callable[
+        [RelativeEndpoint, JsonObj],
+        Cmd[
+            Result[
+                Coproduct[JsonObj, FrozenList[JsonObj]],
+                Coproduct[UnhandledErrors, MaxRetriesReached],
+            ]
+        ],
+    ]
+
+    get_response: Callable[
+        [RelativeEndpoint, JsonObj],
+        Cmd[Result[Response, Coproduct[UnhandledErrors, MaxRetriesReached]]],
+    ]
+
+
+@dataclass(frozen=True)
+class TokenManager:
+    @dataclass(frozen=True)
+    class _Private:
+        pass
+
+    _private: TokenManager._Private = field(repr=False, hash=False, compare=False)
+    _inner: dict[UnitType, Token]
+
+    @staticmethod
+    def new(token: Token) -> Cmd[TokenManager]:
+        return Cmd.wrap_impure(lambda: TokenManager(TokenManager._Private(), {unit: token}))
+
+    @property
+    def get(self) -> Cmd[Token]:
+        return Cmd.wrap_impure(lambda: self._inner[unit])
+
+    def update(self, token: Token) -> Cmd[UnitType]:
+        def _action() -> UnitType:
+            self._inner[unit] = token
+            return unit
+
+        return Cmd.wrap_impure(_action)

fluidattacks_zoho_sdk/_http_client/paginate.py

@@ -0,0 +1,35 @@
+from collections.abc import Callable
+from typing import NewType, TypeVar
+
+from fa_purity import Cmd, FrozenList, Maybe, Result, ResultE
+from fa_purity._core.utils import raise_exception
+
+from ._core import HttpJsonClient
+
+_T = TypeVar("_T")
+FromIndex = NewType("FromIndex", int)
+Limit = NewType("Limit", int)
+
+
+def validate_next_page(
+    from_index: FromIndex,
+    items: FrozenList[_T],
+    limit: Limit,
+) -> ResultE[tuple[FrozenList[_T], Maybe[FromIndex]]]:
+    return Result.success(
+        (items, Maybe.some(FromIndex(from_index + limit)) if len(items) > 0 else Maybe.empty()),
+    )
+
+
+def get_page(
+    client: HttpJsonClient,
+    from_index: Maybe[FromIndex],
+    limit: Limit,
+    get_endpoint: Callable[
+        [HttpJsonClient, Maybe[FromIndex], Limit],
+        Cmd[ResultE[tuple[FrozenList[_T], Maybe[FromIndex]]]],
+    ],
+) -> Cmd[tuple[FrozenList[_T], Maybe[FromIndex]]]:
+    return get_endpoint(client, from_index, limit).map(
+        lambda r: r.alt(raise_exception).to_union(),
+    )

fluidattacks_zoho_sdk/bulk_export/__init__.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from fluidattacks_zoho_sdk._http_client import ClientFactory, HttpJsonClient, TokenManager
+from fluidattacks_zoho_sdk.auth import Credentials
+from fluidattacks_zoho_sdk.ids import OrgId
+
+from . import _client
+from .core import BulkClient, BulkEndpoint, FileName, ModuleName
+
+
+def _from_client(client: HttpJsonClient) -> BulkClient:
+    return BulkClient(
+        lambda m, i: _client.create_bulk_export(client, m, i),
+        lambda obj: _client.get_status_bulk_export(client, obj),
+        lambda b, f: _client.download_bulk(client, b, f),
+        lambda m, e, f: _client.fetch_bulk(client, m, e, f),
+    )
+
+
+@dataclass(frozen=True)
+class BulkApiFactory:
+    @staticmethod
+    def new(creds: Credentials, org_id: OrgId, token: TokenManager) -> BulkClient:
+        return _from_client(ClientFactory.new(creds, org_id, token))
+
+
+__all__ = ["BulkApiFactory", "BulkClient", "BulkEndpoint", "FileName", "ModuleName"]

fluidattacks_zoho_sdk/bulk_export/_client.py

@@ -0,0 +1,195 @@
+import inspect
+import logging
+
+from fa_purity import Cmd, FrozenList, Result, ResultE, Unsafe, cast_exception
+from fa_purity.json import JsonObj, Primitive, UnfoldedFactory
+from fluidattacks_etl_utils.bug import Bug
+from fluidattacks_etl_utils.retry import retry_cmd
+from requests import Response
+
+from fluidattacks_zoho_sdk._decoders import assert_single
+from fluidattacks_zoho_sdk._http_client import HttpJsonClient, RelativeEndpoint
+
+from ._decode import decode_bulk_info, transform_columns
+from .core import (
+    BulkData,
+    BulkEndpoint,
+    BulkExportObj,
+    BulkStatus,
+    FileName,
+    ModuleName,
+    ShouldRetry,
+)
+from .utils import _handle_status, _wait_job, unzip_data
+
+LOG = logging.getLogger(__name__)
+
+
+def _retry_waiting_step(
+    retry: int,
+    result: ResultE[BulkStatus],
+    job: BulkExportObj,
+    client: HttpJsonClient,
+) -> Cmd[ResultE[BulkStatus]]:
+    error_or_ok = result.to_coproduct().map(lambda _s: False, lambda e: isinstance(e, ShouldRetry))
+    if error_or_ok is True:
+        return _wait_job(retry, job.id_bulk).bind(
+            lambda _: get_status_bulk_export(client, job).map(  # token
+                lambda r: r.bind(lambda tup: _handle_status(tup[1])),
+            ),
+        )
+
+    return Cmd.wrap_value(result)
+
+
+def create_bulk_export(
+    client: HttpJsonClient,
+    # access_token: Token,
+    module: ModuleName,
+    endpoint_bulk: BulkEndpoint,
+) -> Cmd[ResultE[tuple[BulkExportObj, BulkStatus]]]:
+    endpoint = RelativeEndpoint.new(endpoint_bulk.route)
+    params: dict[str, Primitive] = {"module": module.module}
+
+    return client.post(endpoint, UnfoldedFactory.from_dict(params)).map(
+        lambda result: (
+            result.alt(
+                lambda e: cast_exception(
+                    Bug.new("_create_bulk_export", inspect.currentframe(), e, ()),
+                ),
+            )
+            .bind(assert_single)
+            .bind(decode_bulk_info)
+        ),
+    )
+
+
+def get_status_retry(
+    client: HttpJsonClient,
+    bulk: BulkExportObj,
+    max_attempts: int,
+) -> Cmd[ResultE[BulkStatus]]:
+    def get_attempt(
+        obj: BulkExportObj,
+        client: HttpJsonClient,
+    ) -> Cmd[ResultE[BulkStatus]]:
+        return get_status_bulk_export(client, obj).map(
+            lambda result: (
+                result.alt(
+                    lambda e: cast_exception(
+                        Bug.new("_get_bulk_export_status", inspect.currentframe(), e, ()),
+                    ),
+                ).bind(lambda tup: _handle_status(tup[1]))
+            ),
+        )
+
+    log = Cmd.wrap_impure(lambda: LOG.info("[API]: Starting read status of bulk export"))
+    handled = log + get_attempt(bulk, client)  # token
+
+    return retry_cmd(
+        handled,
+        lambda retry, result: _retry_waiting_step(retry, result, bulk, client),  # token
+        max_attempts,
+    ).map(lambda r: r.alt(cast_exception))
+
+
+def get_status_bulk_export(
+    client: HttpJsonClient,
+    # access_token: Token,
+    bulk: BulkExportObj,
+) -> Cmd[ResultE[tuple[BulkExportObj, BulkStatus]]]:
+    endpoint = RelativeEndpoint.new("bulkExport", bulk.id_bulk.id_bulk)
+    params: dict[str, Primitive] = {}
+
+    return client.get(endpoint, UnfoldedFactory.from_dict(params)).map(  # access_token
+        lambda result: (
+            result.alt(
+                lambda e: cast_exception(
+                    Bug.new("_get_bulk_export_status", inspect.currentframe(), e, ()),
+                ),
+            )
+            .bind(assert_single)
+            .bind(decode_bulk_info)
+        ),
+    )
+
+
+def _unwrap_and_unzip(
+    result: Result[Response, Exception],
+    file_name: FileName,
+) -> Cmd[ResultE[BulkData]]:
+    return result.to_coproduct().map(
+        lambda inl: unzip_data(inl, file_name),
+        lambda ri: Cmd.wrap_value(Result.failure(ri)),
+    )
+
+
+def download_bulk(
+    client: HttpJsonClient,
+    # access_token: Token,
+    bulk: BulkExportObj,
+    file_name: FileName,
+) -> Cmd[ResultE[BulkData]]:
+    endpoint = RelativeEndpoint.new("downloadBulkExportFile")
+    params: dict[str, Primitive] = {"exportId": bulk.id_bulk.id_bulk}
+
+    return (
+        client.get_response(endpoint, UnfoldedFactory.from_dict(params))
+        .map(
+            lambda result: result.alt(
+                lambda e: cast_exception(
+                    Bug.new("_download_bulk_export", inspect.currentframe(), e, ()),
+                ),
+            ),
+        )
+        .bind(lambda v: _unwrap_and_unzip(v, file_name))
+    )
+
+
+def _wait_and_download(
+    bulk_obj: BulkExportObj,
+    client: HttpJsonClient,
+    file_name: FileName,
+) -> Cmd[ResultE[BulkData]]:
+    return (
+        get_status_retry(client, bulk_obj, 10)  # token
+        .map(lambda r: r.map(lambda _: bulk_obj))
+        .map(lambda r: r.alt(Unsafe.raise_exception).to_union())
+        .bind(lambda bulk: download_bulk(client, bulk, file_name))
+    )
+
+
+def transform_csv_phase(csv: ResultE[BulkData]) -> ResultE[FrozenList[JsonObj]]:
+    bulk_data = csv.alt(Unsafe.raise_exception).to_union()
+    return transform_columns(bulk_data)
+
+
+def fetch_bulk(
+    client: HttpJsonClient,
+    # access_token: Token,
+    module: ModuleName,
+    endpoint_bulk: BulkEndpoint,
+    file_name: FileName,
+) -> Cmd[ResultE[FrozenList[JsonObj]]]:
+    create_bulk: Cmd[ResultE[tuple[BulkExportObj, BulkStatus]]] = create_bulk_export(
+        client,
+        # access_token,
+        module,
+        endpoint_bulk,
+    )
+
+    def on_created(
+        result: ResultE[tuple[BulkExportObj, BulkStatus]],
+    ) -> Cmd[ResultE[FrozenList[JsonObj]]]:
+        return (
+            result.map(lambda pair: pair[0])
+            .to_coproduct()
+            .map(
+                lambda bulk_obj: _wait_and_download(bulk_obj, client, file_name).map(
+                    lambda data: transform_csv_phase(data),
+                ),
+                lambda err: Cmd.wrap_value(Result.failure(err)),
+            )
+        )
+
+    return create_bulk.bind(on_created)

fluidattacks_zoho_sdk/bulk_export/_decode.py

@@ -0,0 +1,41 @@
+import csv
+
+from fa_purity import FrozenList, ResultE
+from fa_purity.json import JsonObj, JsonUnfolder, JsonValueFactory, Unfolder
+from fluidattacks_etl_utils import smash
+from fluidattacks_etl_utils.decode import DecodeUtils
+
+from .core import BulkData, BulkExportId, BulkExportObj, BulkStatus, ViewId
+
+
+def decode_id_bulk(raw: JsonObj) -> ResultE[BulkExportId]:
+    return JsonUnfolder.require(raw, "exportId", DecodeUtils.to_str).map(lambda v: BulkExportId(v))
+
+
+def decode_id_view(raw: JsonObj) -> ResultE[ViewId]:
+    return JsonUnfolder.require(raw, "viewId", DecodeUtils.to_str).map(lambda v: ViewId(v))
+
+
+def decode_status_bulk(raw: JsonObj) -> ResultE[BulkStatus]:
+    return JsonUnfolder.require(raw, "status", DecodeUtils.to_str).bind(
+        lambda v: BulkStatus.from_raw(v),
+    )
+
+
+def decode_bulk_export(raw: JsonObj) -> ResultE[BulkExportObj]:
+    return smash.smash_result_2(
+        decode_id_bulk(raw),
+        JsonUnfolder.require(raw, "module", DecodeUtils.to_str),
+    ).map(lambda bulk: BulkExportObj(*bulk))
+
+
+def decode_bulk_info(raw: JsonObj) -> ResultE[tuple[BulkExportObj, BulkStatus]]:
+    return smash.smash_result_2(decode_bulk_export(raw), decode_status_bulk(raw))
+
+
+def transform_columns(bulk: BulkData) -> ResultE[FrozenList[JsonObj]]:
+    bulk.file.seek(0)
+    reader = csv.DictReader(bulk.file)
+    rows: list[dict[str, str | None]] = list(reader)
+
+    return JsonValueFactory.from_any(rows).bind(lambda v: Unfolder.to_list_of(v, Unfolder.to_json))

fluidattacks_zoho_sdk/bulk_export/core.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import Enum
+from typing import IO, Final, final
+
+from fa_purity import Cmd, FrozenList, ResultE
+from fa_purity.json import JsonObj
+from fluidattacks_etl_utils.handle_errors import handle_value_error
+
+
+@dataclass(frozen=True)
+class BulkExportId:
+    id_bulk: str
+
+
+@dataclass(frozen=True)
+class ViewId:
+    id_bulk: str
+
+
+@dataclass(frozen=True)
+class ModuleName:
+    module: str
+
+
+@final
+class BulkStatus(Enum):
+    QUEUED = "QUEUED"
+    INITIATED = "INITIATED"
+    IN_PROGRESS = "IN_PROGRESS"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+    EXPIRED = "EXPIRED"
+    CANCELLED = "CANCELLED"
+
+    @staticmethod
+    def from_raw(raw: str) -> ResultE[BulkStatus]:
+        return handle_value_error(lambda: BulkStatus(raw))
+
+
+NonTerminalStatus: Final[tuple[BulkStatus, ...]] = (
+    BulkStatus.QUEUED,
+    BulkStatus.IN_PROGRESS,
+    BulkStatus.INITIATED,
+)
+TerminalStatus: Final[tuple[BulkStatus, ...]] = (
+    BulkStatus.CANCELLED,
+    BulkStatus.FAILED,
+    BulkStatus.EXPIRED,
+)
+
+
+@dataclass(frozen=True)
+class BulkExportObj:
+    id_bulk: BulkExportId
+    module: str
+
+
+@dataclass(frozen=True)
+class BulkEndpoint:
+    route: str
+
+
+@dataclass(frozen=True)
+class ShouldRetry(Exception):
+    status: BulkStatus
+
+
+@dataclass(frozen=True)
+class MustRestart(Exception):
+    status: BulkStatus
+
+
+@dataclass(frozen=True)
+class BulkClient:
+    create_bulk: Callable[
+        [ModuleName, BulkEndpoint],
+        Cmd[ResultE[tuple[BulkExportObj, BulkStatus]]],
+    ]
+    get_status: Callable[[BulkExportObj], Cmd[ResultE[tuple[BulkExportObj, BulkStatus]]]]
+    download_bulk: Callable[[BulkExportObj, FileName], Cmd[ResultE[BulkData]]]
+    fetch_bulk: Callable[
+        [ModuleName, BulkEndpoint, FileName],
+        Cmd[ResultE[FrozenList[JsonObj]]],
+    ]
+
+
+@dataclass(frozen=True)
+class BulkData:
+    file: IO[str]
+
+
+@dataclass(frozen=True)
+class FileName:
+    name: str

fluidattacks_zoho_sdk/bulk_export/utils.py

@@ -0,0 +1,72 @@
+import logging
+import tempfile
+from typing import IO
+from zipfile import ZipFile
+
+from fa_purity import Cmd, Result, ResultE, cast_exception
+from pure_requests.retry import sleep_cmd
+from requests import Response
+
+from .core import (
+    BulkData,
+    BulkExportId,
+    BulkStatus,
+    FileName,
+    MustRestart,
+    NonTerminalStatus,
+    ShouldRetry,
+)
+
+LOG = logging.getLogger(__name__)
+
+
+def _top_truncation(value: float, limit: float) -> float:
+    if value > limit:
+        return limit
+    return value
+
+
+def _waiting_msg(job: BulkExportId, time: float) -> Cmd[None]:
+    return Cmd.wrap_impure(
+        lambda: LOG.info("Waiting bulk export %s to be ready (%s)", job.id_bulk, int(time)),
+    )
+
+
+def _wait_job(retry: int, job: BulkExportId) -> Cmd[None]:
+    wait_time = _top_truncation(60 * retry, 5 * 50)
+    return _waiting_msg(job, wait_time) + sleep_cmd(wait_time)
+
+
+def _handle_status(status_bulk: BulkStatus) -> ResultE[BulkStatus]:
+    if status_bulk == BulkStatus.COMPLETED:
+        return Result.success(status_bulk)
+    if status_bulk in NonTerminalStatus:
+        return Result.failure(ShouldRetry(status_bulk))
+    return Result.failure(MustRestart(status_bulk))
+
+
+def unzip_data(response: Response, file_name: FileName) -> Cmd[ResultE[BulkData]]:
+    def _action() -> ResultE[BulkData]:
+        # pylint: disable=consider-using-with
+        # need refac of BulkData for enabling the above check
+        name_file = file_name.name
+        tmp_zipdir = tempfile.mkdtemp()
+        file_zip: IO[bytes] = tempfile.NamedTemporaryFile(mode="wb+")  # noqa: SIM115
+        file_unzip: IO[str] = tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8")  # noqa: SIM115
+        file_zip.write(response.content)
+        file_zip.seek(0)
+        LOG.debug("Unzipping file")
+        with ZipFile(file_zip, "r") as zip_obj:
+            files = zip_obj.namelist()
+
+            if name_file not in files:
+                err = ValueError(f"Expected  {name_file} file. Decompressed {len(files)} files.")
+                return Result.failure(cast_exception(err))
+            zip_obj.extract(name_file, tmp_zipdir)
+        LOG.debug("Generating BulkData")
+        with open(tmp_zipdir + f"/{name_file}", encoding="UTF-8") as unzipped:  # noqa: PTH123
+            file_unzip.write(unzipped.read())
+        LOG.debug("Unzipped size: %s", file_unzip.tell())
+        return Result.success(BulkData(file_unzip))
+
+    return Cmd.wrap_impure(_action)

fluidattacks_zoho_sdk/ids.py

@@ -1,9 +1,13 @@
 from dataclasses import dataclass
 
-from fa_purity import Maybe
 from fluidattacks_etl_utils.natural import Natural
 
 
+@dataclass(frozen=True)
+class OrgId:
+    org_id: Natural
+
+
 @dataclass(frozen=True)
 class UserId:
     user_id: Natural
@@ -11,7 +15,7 @@ class UserId:
 
 @dataclass(frozen=True)
 class AccountId:
-    accound_id: Natural
+    account_id: Natural
 
 
 @dataclass(frozen=True)
@@ -21,7 +25,7 @@ class CrmId:
 
 @dataclass(frozen=True)
 class DeparmentId:
-    id_deparment: Maybe[Natural]
+    id_deparment: Natural
 
 
 @dataclass(frozen=True)
@@ -46,7 +50,7 @@ class RoleId:
 
 @dataclass(frozen=True)
 class ProductId:
-    id_product: Maybe[Natural]
+    id_product: Natural
 
 
 @dataclass(frozen=True)