fluidattacks_dynamo_etl 4.0.5__py3-none-any.whl

fluidattacks_dynamo_etl/__init__.py

@@ -0,0 +1,11 @@
+from fa_purity import (
+    Unsafe,
+)
+
+from ._logger import (
+    set_logger,
+)
+
+__version__ = "4.0.5"
+
+Unsafe.compute(set_logger(__name__, __version__))

fluidattacks_dynamo_etl/_core.py

@@ -0,0 +1,74 @@
+from __future__ import (
+    annotations,
+)
+
+from dataclasses import (
+    dataclass,
+    field,
+)
+from enum import (
+    Enum,
+)
+from typing import (
+    TYPE_CHECKING,
+    TypeVar,
+)
+
+from fa_purity import (
+    Maybe,
+    Result,
+    ResultE,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import (
+        Callable,
+    )
+
+    from fluidattacks_batch_client.core import Natural
+
+_T = TypeVar("_T")
+
+
+@dataclass(frozen=True)
+class _Private:
+    pass
+
+
+class SupportedEtl(Enum):
+    integrates = "integrates"
+    sifts = "sifts"
+
+
+class TargetTables(Enum):
+    INTEGRATES = "integrates_vms"
+    SIFTS = "sifts_state"
+
+
+@dataclass(frozen=True)
+class TotalSegments:
+    total: Natural
+
+
+@dataclass(frozen=True)
+class Segment:
+    _private: _Private = field(repr=False, hash=False, compare=False)
+    _raw: Maybe[int]
+
+    @staticmethod
+    def from_raw(raw: str) -> ResultE[Segment]:
+        try:
+            item = Segment(_Private(), Maybe.some(int(raw)))
+            return Result.success(item, Exception)
+        except ValueError as err:
+            if raw == "auto":
+                item = Segment(_Private(), Maybe.empty())
+                return Result.success(item, Exception)
+            return Result.failure(Exception(err))
+
+    def map(
+        self,
+        case_num: Callable[[int], _T],
+        case_auto: Callable[[], _T],
+    ) -> _T:
+        return self._raw.map(case_num).or_else_call(case_auto)

fluidattacks_dynamo_etl/_logger.py

@@ -0,0 +1,27 @@
+from fa_purity import (
+    Cmd,
+)
+from fluidattacks_utils_logger import (
+    set_main_log,
+)
+from fluidattacks_utils_logger.env import (
+    current_app_env,
+    observes_debug,
+)
+from fluidattacks_utils_logger.handlers import (
+    LoggingConf,
+)
+
+
+def set_logger(root_name: str, version: str) -> Cmd[None]:
+    app_env = current_app_env()
+    debug = observes_debug()
+    conf = app_env.map(
+        lambda env: LoggingConf(
+            app_name="observes",
+            app_type="etl",
+            app_version=version,
+            release_stage=env,
+        ),
+    )
+    return debug.bind(lambda d: conf.bind(lambda c: set_main_log(root_name, c, d)))

fluidattacks_dynamo_etl/_multi_file/classifier.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Generic, TypeVar
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from fa_purity import Cmd, UnitType
+    from fluidattacks_etl_utils.mutable import MutableMap
+
+_File = TypeVar("_File")
+
+
+@dataclass(frozen=True)
+class Record:
+    stream: str
+    line: str
+
+
+@dataclass(frozen=True)
+class StreamClassifier(Generic[_File]):
+    """
+    Handle writes of tagged records into various writable objects.
+
+    - A new writable "_File" object is created per stream/tag.
+    - each record is redirected/written into its stream file.
+    """
+
+    _new_file: Cmd[_File]
+    _write_line: Callable[[_File, str], Cmd[UnitType]]
+    streams: MutableMap[str, _File]
+
+    def write_record(self, record: Record) -> Cmd[UnitType]:
+        return self.streams.get_or_create(record.stream, self._new_file).bind(
+            lambda f: self._write_line(f, record.line),
+        )

fluidattacks_dynamo_etl/_multi_file/splitter.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from fa_purity import (
+    Cmd,
+    CmdSmash,
+    NewFrozenList,
+    PureIterFactory,
+    PureIterTransform,
+    ResultTransform,
+    UnitType,
+    unit,
+)
+from fa_purity.lock import ThreadLock
+from fluidattacks_etl_utils.typing import none_to_unit, unit_to_none
+
+from fluidattacks_dynamo_etl._utils.non_empty import MutableVar, NonEmptyList
+from fluidattacks_dynamo_etl._utils.pure_io import FileProperties, WritableIO
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+LOG = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class File:
+    props: FileProperties[str]
+    writer: WritableIO[str]
+    lock: ThreadLock
+
+
+@dataclass(frozen=True)
+class MultifileSplitter:
+    """Interface for handling lines writes over multiple files."""
+
+    write_line: Callable[[str], Cmd[UnitType]]
+    write_lines_batch: Callable[[NewFrozenList[str]], Cmd[UnitType]]
+    close: Cmd[UnitType]
+    paths: Cmd[NonEmptyList[Path]]
+
+
+@dataclass(frozen=True)
+class _WriteBuffer:
+    _data: dict[UnitType, list[str]]
+
+    @staticmethod
+    def new() -> Cmd[_WriteBuffer]:
+        return Cmd.wrap_impure(lambda: _WriteBuffer({unit: []}))
+
+    def append(self, line: str) -> Cmd[UnitType]:
+        def _action() -> UnitType:
+            self._data[unit].append(line)
+            return unit
+
+        return Cmd.wrap_impure(_action)
+
+    def append_many(self, lines: NewFrozenList[str]) -> Cmd[UnitType]:
+        def _action() -> UnitType:
+            self._data[unit].extend(lines)
+            return unit
+
+        return Cmd.wrap_impure(_action)
+
+    @property
+    def flush(self) -> Cmd[tuple[str, ...]]:
+        def _action() -> tuple[str, ...]:
+            result = tuple(self._data[unit])
+            self._data[unit].clear()
+            return result
+
+        return Cmd.wrap_impure(_action)
+
+    @property
+    def size(self) -> Cmd[int]:
+        return Cmd.wrap_impure(lambda: len(self._data[unit]))
+
+
+@dataclass(frozen=True)
+class _DefaultSplitter:
+    _new_file: Cmd[File]
+    _lines_limit: int
+    _buffer_size: int
+    _files: MutableVar[NonEmptyList[File]]
+    _line_counter: MutableVar[int]
+    _write_buffer: _WriteBuffer
+    _lock: ThreadLock
+
+    def _flush_lines(
+        self,
+        lines: tuple[str, ...],
+        counter: int,
+        files: NonEmptyList[File],
+    ) -> Cmd[UnitType]:
+        if not lines:
+            return Cmd.wrap_value(unit)
+        current = ResultTransform.get_index(files.next_items, -1).value_or(files.item)
+        remaining = self._lines_limit - counter
+        if len(lines) <= remaining:
+            return current.lock.execute_with_lock(
+                current.writer.write_lines(PureIterFactory.from_list(lines)),
+            ) + self._line_counter.update(counter + len(lines))
+        fitting = lines[:remaining]
+        overflow = lines[remaining:]
+        write_fitting = current.lock.execute_with_lock(
+            current.writer.write_lines(PureIterFactory.from_list(fitting)),
+        )
+
+        def _add_next_file(current_files: NonEmptyList[File]) -> Cmd[UnitType]:
+            return self._new_file.bind(
+                lambda new_f: Cmd.wrap_impure(
+                    lambda: LOG.info(
+                        "New file created: %s (file #%d)",
+                        new_f.props.path,
+                        len(current_files.next_items.items) + 2,
+                    ),
+                )
+                + self._files.update(
+                    NonEmptyList(
+                        current_files.item,
+                        NewFrozenList.new(*current_files.next_items, new_f),
+                    ),
+                ).bind(
+                    lambda _: self._flush_lines(
+                        overflow,
+                        0,
+                        NonEmptyList(new_f, NewFrozenList.new()),
+                    ),
+                ),
+            )
+
+        return write_fitting + self._files.get.bind(_add_next_file)
+
+    @property
+    def _flush_buffer(self) -> Cmd[UnitType]:
+        return self._write_buffer.size.bind(
+            lambda size: self._write_buffer.flush.bind(
+                lambda lines: self._line_counter.get.bind(
+                    lambda c: self._files.get.bind(lambda f: self._flush_lines(lines, c, f)),
+                ),
+            )
+            if size > 0
+            else Cmd.wrap_value(unit),
+        )
+
+    def write_line(self, line: str) -> Cmd[UnitType]:
+        def _maybe_flush(_: UnitType) -> Cmd[UnitType]:
+            return self._write_buffer.size.bind(
+                lambda size: self._flush_buffer
+                if size >= self._buffer_size
+                else Cmd.wrap_value(unit),
+            )
+
+        return self._lock.execute_with_lock(
+            self._write_buffer.append(line + "\n").bind(_maybe_flush),
+        )
+
+    def write_lines_batch(self, lines: NewFrozenList[str]) -> Cmd[UnitType]:
+        if not lines.items:
+            return Cmd.wrap_value(unit)
+        tagged = lines.map(lambda i: i + "\n")
+
+        def _maybe_flush(_: UnitType) -> Cmd[UnitType]:
+            return self._write_buffer.size.bind(
+                lambda size: self._flush_buffer
+                if size >= self._buffer_size
+                else Cmd.wrap_value(unit),
+            )
+
+        return self._lock.execute_with_lock(
+            self._write_buffer.append_many(tagged).bind(_maybe_flush),
+        )
+
+    @property
+    def close(self) -> Cmd[UnitType]:
+        close_files = self._files.get.bind(
+            lambda n: n.item.writer.close
+            + PureIterTransform.consume(
+                PureIterFactory.from_list(n.next_items.items).map(
+                    lambda f: f.writer.close.map(unit_to_none),
+                ),
+            ).map(none_to_unit),
+        )
+        return self._lock.execute_with_lock(self._flush_buffer) + close_files
+
+    @property
+    def paths(self) -> Cmd[NonEmptyList[Path]]:
+        return self._files.get.map(lambda n: n.map(lambda f: f.props.path))
+
+    def to_interface(self) -> MultifileSplitter:
+        return MultifileSplitter(
+            write_line=self.write_line,
+            write_lines_batch=self.write_lines_batch,
+            close=self.close,
+            paths=self.paths,
+        )
+
+
+def new_splitter(new_file: Cmd[File], lines_limit: int, buffer_size: int) -> Cmd[MultifileSplitter]:
+    return CmdSmash.smash_cmds_3(
+        new_file.bind(lambda f: MutableVar.new(NonEmptyList(f, NewFrozenList.new()))),
+        MutableVar.new(0),
+        ThreadLock.new(),
+    ).bind(
+        lambda t: _WriteBuffer.new().map(
+            lambda buf: _DefaultSplitter(
+                new_file,
+                lines_limit,
+                buffer_size,
+                t[0],
+                t[1],
+                buf,
+                t[2],
+            ).to_interface(),
+        ),
+    )

fluidattacks_dynamo_etl/_parallel.py

@@ -0,0 +1,124 @@
+from __future__ import (
+    annotations,
+)
+
+import logging
+import queue
+from dataclasses import (
+    dataclass,
+    field,
+)
+from typing import (
+    TYPE_CHECKING,
+    Generic,
+    TypeVar,
+)
+
+from fa_purity import (
+    Cmd,
+    Coproduct,
+    Maybe,
+    PureIter,
+    PureIterFactory,
+    Stream,
+    StreamFactory,
+    StreamTransform,
+    UnitType,
+    Unsafe,
+)
+from fluidattacks_etl_utils.typing import none_to_unit, unit_to_none
+
+if TYPE_CHECKING:
+    from fluidattacks_etl_utils.parallel import ThreadPool
+
+LOG = logging.getLogger(__name__)
+_T = TypeVar("_T")
+
+
+@dataclass(frozen=True)
+class _Private:
+    pass
+
+
+@dataclass(frozen=True)
+class _FinishMark:
+    pass
+
+
+@dataclass(frozen=True)
+class Queue(Generic[_T]):
+    _private: _Private = field(repr=False, hash=False, compare=False)
+    _inner: queue.Queue[_T]
+
+    @staticmethod
+    def new(maxsize: int = 0) -> Cmd[Queue[_T]]:
+        def _action() -> Queue[_T]:
+            inner: queue.Queue[_T] = queue.Queue(maxsize)
+            return Queue(_Private(), inner)
+
+        return Cmd.wrap_impure(_action)
+
+    def put(self, item: _T) -> Cmd[UnitType]:
+        def _action() -> None:
+            self._inner.put(item)
+            LOG.info("Queue.put qsize=%s", self._inner.qsize())
+
+        return Cmd.wrap_impure(_action).map(none_to_unit)
+
+    def get(self) -> Cmd[_T]:
+        return Cmd.wrap_impure(lambda: self._inner.get())
+
+    def empty(self) -> Cmd[bool]:
+        return Cmd.wrap_impure(lambda: self._inner.empty())
+
+    def qsize(self) -> Cmd[int]:
+        return Cmd.wrap_impure(lambda: self._inner.qsize())
+
+    def task_done(self) -> Cmd[UnitType]:
+        return Cmd.wrap_impure(lambda: self._inner.task_done()).map(none_to_unit)
+
+    def join(self) -> Cmd[UnitType]:
+        return Cmd.wrap_impure(lambda: self._inner.join()).map(none_to_unit)
+
+
+def squash_cmd_stream(cmd: Cmd[Stream[_T]]) -> Stream[_T]:
+    new_iter = cmd.bind(lambda s: Unsafe.stream_to_iter(s))
+    return Unsafe.stream_from_cmd(new_iter)
+
+
+def _combine_streams(
+    pool: ThreadPool,
+    queue: Queue[Coproduct[_T, _FinishMark]],
+    streams: PureIter[Stream[_T]],
+) -> Stream[_T]:
+    queue_emissions = streams.map(
+        lambda s: s.map(lambda i: queue.put(Coproduct.inl(i)))
+        .map(lambda c: c.map(unit_to_none))
+        .transform(StreamTransform.consume)
+        .map(none_to_unit),
+    )
+    signal_done = queue.put(Coproduct.inr(_FinishMark()))
+    emit_all = pool.in_background(pool.in_threads_unit(queue_emissions) + signal_done)
+
+    def _filter_finish(item: Coproduct[_T, _FinishMark]) -> Maybe[_T]:
+        return item.map(Maybe.some, lambda _: Maybe.empty())
+
+    stream_queue = (
+        PureIterFactory.infinite_range(0, 0)
+        .map(lambda _: queue.get().map(_filter_finish))
+        .transform(StreamFactory.from_commands)
+        .transform(StreamTransform.until_empty)
+    )
+    return squash_cmd_stream(emit_all.map(lambda _: stream_queue))
+
+
+def combine_streams(
+    pool: ThreadPool,
+    streams: PureIter[Stream[_T]],
+    max_queue_size: int,
+) -> Stream[_T]:
+    return squash_cmd_stream(
+        Queue[Coproduct[_T, _FinishMark]]
+        .new(max_queue_size)
+        .map(lambda q: _combine_streams(pool, q, streams)),
+    )

fluidattacks_dynamo_etl/_utils/__init__.py

@@ -0,0 +1,93 @@
+from __future__ import (
+    annotations,
+)
+
+import logging
+from typing import (
+    TYPE_CHECKING,
+    TypeVar,
+)
+
+from fa_purity import (
+    Cmd,
+    CmdUnwrapper,
+    Coproduct,
+    PureIter,
+    Result,
+    ResultE,
+    ResultFactory,
+    Stream,
+    Unsafe,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import (
+        Callable,
+        Iterable,
+    )
+
+LOG = logging.getLogger(__name__)
+_T = TypeVar("_T")
+_S = TypeVar("_S")
+_A = TypeVar("_A")
+_F = TypeVar("_F")
+
+
+def append_to_stream(stream: Stream[_T], item: _T) -> Stream[_T]:
+    def _iter(unwrapper: CmdUnwrapper) -> Iterable[_T]:
+        yield from unwrapper.act(Unsafe.stream_to_iter(stream))
+        yield item
+
+    new_iter = Cmd.new_cmd(_iter)
+    return Unsafe.stream_from_cmd(new_iter)
+
+
+def join_stream(stream_1: Stream[_T], stream_2: Stream[_T]) -> Stream[_T]:
+    def _iter(unwrapper: CmdUnwrapper) -> Iterable[_T]:
+        yield from unwrapper.act(Unsafe.stream_to_iter(stream_1))
+        yield from unwrapper.act(Unsafe.stream_to_iter(stream_2))
+
+    new_iter = Cmd.new_cmd(_iter)
+    return Unsafe.stream_from_cmd(new_iter)
+
+
+def chain_cmd_result(
+    cmd_1: Cmd[Result[_S, _F]],
+    cmd_2: Callable[[_S], Cmd[Result[_A, _F]]],
+) -> Cmd[Result[_A, _F]]:
+    factory: ResultFactory[_A, _F] = ResultFactory()
+    return cmd_1.bind(
+        lambda r: r.map(cmd_2).alt(lambda e: Cmd.wrap_value(factory.failure(e))).to_union(),
+    )
+
+
+def merge_result_exceptions(
+    result: Result[_T, Coproduct[Exception, Exception]],
+) -> ResultE[_T]:
+    return result.alt(lambda c: c.map(lambda x: x, lambda x: x))
+
+
+def consume_results(cmds: PureIter[Cmd[Result[None, _F]]]) -> Cmd[Result[None, _F]]:
+    def _action(unwrapper: CmdUnwrapper) -> Result[None, _F]:
+        for cmd in cmds:
+            result = unwrapper.act(cmd)
+            success = result.map(lambda _: True).value_or(False)
+            if not success:
+                return result
+        return Result.success(None)
+
+    return Cmd.new_cmd(_action)
+
+
+def consume_stream_until_error(
+    results: Stream[Result[_T, _F]],
+    success_result: _T,
+) -> Cmd[Result[_T, _F]]:
+    def _action(unwrapper: CmdUnwrapper) -> Result[_T, _F]:
+        for result in unwrapper.act(Unsafe.stream_to_iter(results)):
+            success = result.map(lambda _: True).value_or(False)
+            if not success:
+                return result
+        return Result.success(success_result)
+
+    return Cmd.new_cmd(_action)

fluidattacks_dynamo_etl/_utils/dict.py

@@ -0,0 +1,23 @@
+from typing import (
+    TypeVar,
+)
+
+from fa_purity import (
+    FrozenDict,
+    ResultE,
+    ResultFactory,
+)
+
+_K = TypeVar("_K")
+_V = TypeVar("_V")
+
+
+def not_empty_dict(items: FrozenDict[_K, _V]) -> ResultE[FrozenDict[_K, _V]]:
+    _factory: ResultFactory[FrozenDict[_K, _V], Exception] = ResultFactory()
+    if items:
+        return _factory.success(items)
+    return _factory.failure(ValueError("Empty FrozenDict"))
+
+
+def merge(items_1: FrozenDict[_K, _V], items_2: FrozenDict[_K, _V]) -> FrozenDict[_K, _V]:
+    return FrozenDict(dict(items_1) | dict(items_2))

fluidattacks_dynamo_etl/_utils/iter.py

@@ -0,0 +1,32 @@
+from collections.abc import Callable, Iterable
+from typing import (
+    TypeVar,
+)
+
+from fa_purity import (
+    Cmd,
+    NewFrozenList,
+    PureIter,
+    Result,
+    ResultTransform,
+    Unsafe,
+)
+
+_T = TypeVar("_T")
+_S = TypeVar("_S")
+_F = TypeVar("_F")
+
+
+def append(items_1: PureIter[_T], items_2: PureIter[_T]) -> PureIter[_T]:
+    def _new_iter() -> Iterable[_T]:
+        yield from items_1
+        yield from items_2
+
+    return Unsafe.pure_iter_from_cmd(Cmd.wrap_impure(_new_iter))
+
+
+def apply_and_handle_error(
+    function: Callable[[_T], Result[_S, _F]],
+    items: PureIter[_T],
+) -> Result[NewFrozenList[_S], _F]:
+    return ResultTransform.all_ok_2(NewFrozenList(tuple(items.map(function))))

fluidattacks_dynamo_etl/_utils/merge.py

@@ -0,0 +1,15 @@
+from typing import TypeVar
+
+from fa_purity import Coproduct, Result
+
+_T = TypeVar("_T")
+_A = TypeVar("_A")
+
+
+def merge_failures(item: Result[_A, Coproduct[_T, _T]]) -> Result[_A, _T]:
+    return item.alt(
+        lambda c: c.map(
+            lambda x: x,
+            lambda x: x,
+        ),
+    )