tesorotools-python 0.0.0__py2.py3-none-any.whl

tesorotools/main.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+import docx
+import pandas as pd
+
+from tesorotools.artists.table import generate_tables_from_flash
+from tesorotools.render.content.table import render_table
+from tesorotools.utils.config import read_config
+from tesorotools.utils.globals import DEBUG, EXAMPLES
+
+if __name__ == "__main__":
+    # test tables here
+    table_config_file: Path = EXAMPLES / "tables.yaml"
+    offsets_config_file: Path = EXAMPLES / "offsets.yaml"
+
+    config_dicts = read_config(table_config_file)
+    offsets_config = read_config(offsets_config_file)
+
+    flash = pd.read_feather("derivates.feather")
+    generate_tables_from_flash(flash, config_dicts)
+
+    document = docx.Document("template.docx")
+    for table_path in (DEBUG / "table").iterdir():
+        if table_path.stem.endswith(("color", "shade")):
+            continue
+        table_dict = config_dicts[table_path.stem]
+        table: pd.DataFrame = pd.read_feather(table_path)
+        color: pd.DataFrame = pd.read_feather(
+            table_path.parent / f"{table_path.stem}_color.feather"
+        )
+        shade: pd.DataFrame = pd.read_feather(
+            table_path.parent / f"{table_path.stem}_shade.feather"
+        )
+
+        render_table(table, color, shade, document, **table_dict)
+        document.add_paragraph()
+    document.save("test.docx")

tesorotools/offsets/offsets.py

@@ -0,0 +1,439 @@
+from collections.abc import Callable
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+# stats and stat functions
+type StatFunction = Callable[[pd.Series], np.float64]
+type StatRollingFunction = Callable[..., float]
+
+
+class Stat(Enum):
+    VALUE = "value"
+    ROLL_AVG = "roll_avg"
+    ROLL_STD = "roll_std"
+
+    @property
+    def stat_function(self) -> StatFunction:
+        match self:
+            case self.VALUE:
+                return lambda x: x.iloc[-1]
+            case self.ROLL_AVG:
+                return lambda x: np.mean(x)
+            case self.ROLL_STD:
+                return lambda x: np.std(x, ddof=0)
+
+    @property
+    def update_function(self) -> StatRollingFunction:
+        match self:
+            case self.VALUE:
+                return lambda **kwargs: kwargs["newest_point"]
+            case self.ROLL_AVG:
+                return lambda **kwargs: _update_rolling_avg(**kwargs)
+            case self.ROLL_STD:
+                return lambda **kwargs: _update_rolling_std(**kwargs)
+
+
+# offsets and offset functions
+type OffsetFunction = Callable[[pd.DatetimeIndex], pd.DatetimeIndex]
+
+
+def _fixed_date_offset_function(reference_date: datetime) -> OffsetFunction:
+    return lambda dates: pd.DatetimeIndex([reference_date] * len(dates))
+
+
+class FloatingOffset(Enum):
+    NO = "no"
+    BDAY = "bday"
+    FTD = "ftd"
+    MTD = "mtd"
+    YTD = "ytd"
+
+    @property
+    def offset_function(self) -> OffsetFunction:
+        match self:
+            case self.BDAY:
+                return lambda dates: dates - pd.tseries.offsets.BDay(1)
+            case self.FTD:
+                return lambda dates: dates - pd.tseries.offsets.Week(weekday=4)
+            case self.MTD:
+                return (
+                    lambda dates: dates
+                    - pd.offsets.MonthBegin()
+                    - pd.tseries.offsets.BDay(1)
+                )
+            case self.YTD:
+                return (
+                    lambda dates: dates
+                    - pd.offsets.YearBegin()
+                    - pd.tseries.offsets.BDay(1)
+                )
+
+
+# differences and difference functions
+type DifferenceFunction = Callable[[pd.Series, pd.Series], pd.Series]
+
+
+class Difference(Enum):
+    NO = "no"
+    ABS = "absolute"
+    REL = "relative"
+
+    @property
+    def difference_function(self) -> DifferenceFunction:
+        match self:
+            case self.ABS:
+                return lambda original, offset: original - offset
+            case self.REL:
+                return lambda original, offset: (original - offset) / offset
+
+
+def process_raw_data(raw_data: pd.DataFrame, **config) -> pd.DataFrame:
+    # preprocess index
+    dates: pd.DatetimeIndex = pd.to_datetime(raw_data.index)
+    raw_data.index = dates
+    raw_data = raw_data.sort_index()
+
+    # parse config and compute common values
+    window: int = min(config["window"], len(raw_data.index))
+    offsets_dict = _parse_offsets(config["offsets"], dates)
+    differences_dict = _parse_differences(config["differences"])
+    stats_dict = _parse_stats(config["stats"])
+
+    names = ["date", "offset", "difference_type"]
+    columns = stats_dict.keys()
+
+    offset_df, raw_df = _result_templates(
+        dates,
+        offsets_dict.keys(),
+        differences_dict.keys(),
+        names,
+        columns,
+        columns_name="stat",
+    )
+
+    # computing enriched data
+    enriched_data: pd.DataFrame = raw_data.apply(
+        lambda x: _process_raw_data(
+            x,
+            offsets_dict,
+            differences_dict,
+            stats_dict,
+            raw_df=raw_df,
+            offset_df=offset_df,
+            window=window,
+        )
+    )
+    return enriched_data
+
+
+def _parse_offsets(
+    offsets_cfg: list[str | datetime], dates: pd.DatetimeIndex
+) -> dict[str, pd.DatetimeIndex]:
+    offsets: dict[str | datetime, OffsetFunction] = {}
+    for offset_str in offsets_cfg:
+        if offset_str is FloatingOffset.NO:
+            continue
+        if offset_str in FloatingOffset:
+            offset_enum = FloatingOffset(offset_str)
+            offsets[offset_str] = offset_enum.offset_function
+        else:
+            offsets[str(offset_str)] = _fixed_date_offset_function(offset_str)
+    offsets_dict: dict[str, pd.DatetimeIndex] = {
+        offset_str: offset_function(dates)
+        for offset_str, offset_function in offsets.items()
+    }
+    return offsets_dict
+
+
+def _parse_differences(
+    differences_cfg: list[str],
+) -> dict[str, DifferenceFunction]:
+    differences: dict[str, DifferenceFunction] = {
+        diff_str: Difference(diff_str).difference_function
+        for diff_str in differences_cfg
+        if diff_str is not Difference.NO
+    }
+    return differences
+
+
+def _parse_stats(
+    stats_cfg: list[str], update: bool = False
+) -> dict[str, StatFunction]:
+    if update:
+        stats: dict[str, StatFunction] = {
+            stat_str: Stat(stat_str).update_function for stat_str in stats_cfg
+        }
+        stats[Stat.VALUE.value] = Stat.VALUE.update_function
+    else:
+        stats: dict[str, StatFunction] = {
+            stat_str: Stat(stat_str).stat_function for stat_str in stats_cfg
+        }
+        stats[Stat.VALUE.value] = Stat.VALUE.stat_function
+    return stats
+
+
+def _result_templates(
+    dates: pd.DatetimeIndex,
+    offsets: list[str],
+    differences: list[str],
+    names: list[str],
+    columns: list[str],
+    columns_name: str,
+) -> tuple[pd.DataFrame, pd.DataFrame]:
+
+    offset_idx: pd.MultiIndex = pd.MultiIndex.from_product(
+        [
+            dates,
+            offsets,
+            differences,
+        ],
+        names=names,
+    )
+    offset_df: pd.DataFrame = pd.DataFrame(
+        index=offset_idx,
+        columns=columns,
+    )
+    offset_df.columns.name = columns_name
+
+    raw_idx: pd.MultiIndex = pd.MultiIndex.from_product(
+        [
+            dates,
+            [FloatingOffset.NO.value],
+            [Difference.NO.value],
+        ],
+        names=names,
+    )
+
+    raw_df: pd.DataFrame = pd.DataFrame(
+        index=raw_idx,
+        columns=columns,
+    )
+    raw_df.columns.name = columns_name
+
+    return offset_df, raw_df
+
+
+def _process_raw_data(
+    column: pd.Series,
+    offsets_dict: dict[str, pd.DatetimeIndex],
+    differences_dict: dict[str, DifferenceFunction],
+    stats_dict: dict[str, StatFunction],
+    *,
+    raw_df: pd.DataFrame,
+    offset_df: pd.DataFrame,
+    window: int,
+) -> pd.Series:
+
+    stat_functions = list(stats_dict.values())
+
+    # for every non trivial offset and difference apply all the stats
+    for offset_str, offset_idx in offsets_dict.items():
+        for diff_str, diff_func in differences_dict.items():
+            offset_data = column.reindex(offset_idx)
+            offset_data.index = column.index
+            diff_data = diff_func(column, offset_data)
+            stat_data: pd.DataFrame = diff_data.rolling(
+                window=window, min_periods=0
+            ).agg(stat_functions)
+            offset_df.loc[(slice(None), offset_str, diff_str), :] = (
+                stat_data.values
+            )
+    offset_series: pd.Series = offset_df.stack()
+
+    # apply the stats to the original data
+    stat_data: pd.DataFrame = column.rolling(window=window, min_periods=0).agg(
+        stat_functions
+    )
+    raw_df.loc[(slice(None)), :] = stat_data.values
+    raw_series: pd.Series = raw_df.stack()
+
+    result = pd.concat([offset_series, raw_series])
+    result = result.apply(pd.to_numeric, errors="coerce")
+    return result
+
+
+def trim(full_data: pd.DataFrame | pd.Series) -> pd.DataFrame | pd.Series:
+    indexer = (
+        slice(None),
+        FloatingOffset.NO.value,
+        Difference.NO.value,
+        Stat.VALUE.value,
+    )
+    if isinstance(full_data, pd.DataFrame):
+        trimmed_data: pd.DataFrame = full_data.loc[indexer, :]
+    else:
+        trimmed_data: pd.Series = full_data.loc[indexer]
+    trimmed_data.index = trimmed_data.index.get_level_values(level=0)
+    return trimmed_data
+
+
+def add_rows(
+    old_full_df: pd.DataFrame,
+    new_trimmed_df: pd.DataFrame,
+    offsets_cfg: dict[str, Any],
+):
+    ## sanity check (raise if failed)
+    # same columns
+    # simple index
+    # datetime index
+    # sorted index
+    # dates not in the old df
+
+    dates: pd.DatetimeIndex = old_full_df.index.get_level_values(
+        level=0
+    ).unique()
+    window: int = min(
+        offsets_cfg["window"],
+        len(dates) + len(new_trimmed_df.index),
+    )
+    offsets_dict = _parse_offsets(offsets_cfg["offsets"], new_trimmed_df.index)
+    differences_dict = _parse_differences(offsets_cfg["differences"])
+    stats_dict = _parse_stats(offsets_cfg["stats"], update=True)
+    names = ["date", "offset", "difference_type"]
+    columns = stats_dict.keys()
+    offset_df, raw_df = _result_templates(
+        new_trimmed_df.index,
+        offsets_dict.keys(),
+        differences_dict.keys(),
+        names,
+        columns,
+        columns_name="stat",
+    )
+    updated_data: pd.DataFrame = new_trimmed_df.apply(
+        lambda x: _process_new_data(
+            x,
+            old_full_df.loc[:, x.name],
+            offsets_dict,
+            differences_dict,
+            stats_dict,
+            raw_df=raw_df,
+            offset_df=offset_df,
+            window=window,
+        )
+    )
+    result = pd.concat([old_full_df, updated_data])
+    return result
+
+
+def _process_new_data(
+    new_trimmed_column: pd.Series,
+    old_full_column: pd.Series,
+    offsets_dict: dict[str, pd.DatetimeIndex],
+    differences_dict: dict[str, DifferenceFunction],
+    stats_dict: dict[str, StatFunction],
+    *,
+    raw_df: pd.DataFrame,
+    offset_df: pd.DataFrame,
+    window: int,
+) -> pd.Series:
+
+    old_trimmed_column: pd.Series = trim(old_full_column)
+    fused_trimmed_column: pd.Series = pd.concat(
+        [old_trimmed_column, new_trimmed_column]
+    )
+
+    # for every non trivial offset and difference apply all the stats
+    for offset_str, offset_idx in offsets_dict.items():
+        for diff_str, diff_func in differences_dict.items():
+            offset_data = fused_trimmed_column.reindex(offset_idx)
+            offset_data.index = new_trimmed_column.index
+            diff_data = diff_func(new_trimmed_column, offset_data)
+            old_diff_stats: pd.Series = old_full_column.loc[
+                (slice(None), offset_str, diff_str)
+            ]
+            stat_data: pd.DataFrame = _rolling_update(
+                window, diff_data, old_diff_stats, stats_dict
+            )
+            offset_df.loc[(stat_data.index.values, offset_str, diff_str), :] = (
+                stat_data.values
+            )
+    offset_series: pd.Series = offset_df.stack()
+
+    # apply the stats to the original data
+    old_stats: pd.Series = old_full_column.loc[
+        (slice(None), FloatingOffset.NO.value, Difference.NO.value)
+    ]
+    stat_data: pd.DataFrame = _rolling_update(
+        window, new_trimmed_column, old_stats, stats_dict
+    )
+    raw_df.loc[(slice(None)), :] = stat_data.values
+    raw_series: pd.Series = raw_df.stack()
+
+    result = pd.concat([offset_series, raw_series])
+    result = result.apply(pd.to_numeric, errors="coerce")
+    return result
+
+
+def _rolling_update(
+    window: int,
+    new_trimmed_data: pd.Series,
+    old_full_data: pd.Series,
+    stats_dict: dict[str, StatRollingFunction],
+):
+    result_df: pd.DataFrame = old_full_data.unstack(level=-1)
+    fused_values = pd.concat(
+        [result_df.loc[:, Stat.VALUE.value], new_trimmed_data]
+    )
+    result_df = result_df.reindex(fused_values.index)
+    for index, value in new_trimmed_data.items():
+        current_position = fused_values.index.get_loc(index)
+        previous_position = max(current_position - window, 0)
+        previous_value = fused_values.iloc[previous_position]
+        actual_window = min(window, current_position + 1)
+        expanding = actual_window < window
+        for f_name, function in stats_dict.items():
+            result_df.loc[index, f_name] = function(
+                window=actual_window,
+                expanding=expanding,
+                old_stat=result_df.iloc[current_position - 1][f_name],
+                oldest_point=previous_value,
+                newest_point=value,
+                other_stats=result_df.iloc[current_position - 1],
+            )
+    return result_df.loc[new_trimmed_data.index, :]
+
+
+def _update_rolling_std(
+    window: int,
+    expanding: bool,
+    old_stat: float,
+    oldest_point: float,
+    newest_point: float,
+    other_stats: pd.Series,
+    **_,
+) -> float:
+    old_rolling_mean: float = other_stats[Stat.ROLL_AVG.value]
+    if expanding:
+        # https://math.stackexchange.com/questions/374881/recursive-formula-for-variance
+        new_rolling_var = (
+            old_stat**2
+            + (1 / (window + 1)) * (old_rolling_mean - newest_point) ** 2
+        ) * (window / (window + 1))
+    else:
+        # https://jonisalonen.com/2014/efficient-and-accurate-rolling-standard-deviation/
+
+        delta = newest_point - oldest_point
+        new_rolling_mean = old_rolling_mean + (delta / window)
+        new_rolling_var = old_stat**2 + (delta / window) * (
+            newest_point - new_rolling_mean + oldest_point - old_rolling_mean
+        )
+    new_rolling_var = max(new_rolling_var, 0)  # avoid complex numbers
+    return new_rolling_var**0.5
+
+
+def _update_rolling_avg(
+    window: int,
+    expanding: bool,
+    old_stat: float,
+    oldest_point: float,
+    newest_point: float,
+    **_,
+) -> float:
+    if expanding:
+        return ((window - 1) * old_stat + newest_point) / window
+    else:
+        return old_stat + (newest_point - oldest_point) / window

tesorotools/offsets/outliers.py

@@ -0,0 +1,15 @@
+import pandas as pd
+
+from .offsets import Stat
+
+
+def flag_outliers(data: pd.DataFrame) -> pd.Series:
+    return (data[Stat.VALUE.value] - data[Stat.ROLL_AVG.value]) / data[
+        Stat.ROLL_STD.value
+    ]
+
+
+def flag_outliers_with_limit(data: pd.DataFrame, limit: float) -> pd.Series:
+    return (data[Stat.VALUE.value] - data[Stat.ROLL_AVG.value]) / data[
+        Stat.ROLL_STD.value
+    ] > limit

tesorotools/render/__init__.py

@@ -0,0 +1,11 @@
+from tesorotools.render.content.images import Image, Images
+from tesorotools.render.content.section import Section
+from tesorotools.render.content.table import Table
+from tesorotools.render.report import Report
+from tesorotools.utils.template import TemplateLoader
+
+TemplateLoader.add_constructor("!report", Report.from_yaml)
+TemplateLoader.add_constructor("!section", Section.from_yaml)
+TemplateLoader.add_constructor("!image", Image.from_yaml)
+TemplateLoader.add_constructor("!images", Images.from_yaml)
+TemplateLoader.add_constructor("!table", Table.from_yaml)

tesorotools/render/content/content.py

@@ -0,0 +1,17 @@
+from typing import Protocol, Self
+
+from docx.document import Document
+from yaml import Loader, MappingNode
+
+
+class Content(Protocol):
+    def render(self, document: Document) -> Document: ...
+
+    @classmethod
+    def from_yaml(cls, loader: Loader, node: MappingNode) -> Self: ...
+
+    @property
+    def level(self) -> int: ...
+
+    @level.setter
+    def level(self, level: int) -> None: ...

tesorotools/render/content/images.py

@@ -0,0 +1,147 @@
+from pathlib import Path
+from typing import Any, Self
+
+from docx.document import Document
+from docx.enum.text import WD_ALIGN_PARAGRAPH
+from docx.shared import Inches
+from docx.table import Table, _Cell
+from docx.text.paragraph import Paragraph
+from docx.text.run import Run
+from yaml import MappingNode
+
+from tesorotools.utils.template import TemplateLoader
+
+CENTER = WD_ALIGN_PARAGRAPH.CENTER
+
+
+def _style_container_table(table: Table) -> None:
+    """Center and bold every paragraph in every cell"""
+    for row in table.rows:
+        for cell in row.cells:
+            for paragraph in cell.paragraphs:
+                paragraph.alignment = CENTER
+                for run in paragraph.runs:
+                    run.bold = True
+
+
+def _fill_titles(
+    cell: _Cell, title: str | None, subtitle: str | None, bold: bool = True
+):
+    """Fill the title cells of an image(s) container table"""
+    title_par: Paragraph = cell.paragraphs[0]
+    title_run: Run = title_par.add_run(title)
+    title_run.bold = bold
+    if subtitle is not None:
+        title_run.add_break()
+        subtitle_run: Run = title_par.add_run(subtitle)
+        subtitle_run.bold = False
+        subtitle_run.italic = True
+
+
+class Image:
+    """A single image with an optional title"""
+
+    def __init__(
+        self,
+        file: Path,
+        title: str | None = None,
+        subtitle: str | None = None,
+        width: int = 4,
+    ):
+        self._title: str | None = title
+        self._subtitle: str | None = subtitle
+        self._file: Path = file
+        self._width = width
+
+    @classmethod
+    def from_yaml(cls, loader: TemplateLoader, node: MappingNode) -> Self:
+        image_cfg: dict[str, Any] = loader.construct_mapping(node, deep=True)
+        return cls(
+            file=loader.imports["image"] / image_cfg.pop("id"), **image_cfg
+        )
+
+    def render(self, document: Document) -> Document:
+        # add container table
+        container_table: Table = document.add_table(2, 1)
+        container_table.alignment = CENTER
+        _style_container_table(container_table)
+
+        # set titles
+        title_cell: _Cell = container_table.cell(0, 0)
+        _fill_titles(title_cell, title=self._title, subtitle=self._subtitle)
+
+        # fill container table
+        content_cell: _Cell = container_table.cell(1, 0)
+        content_par: Paragraph = content_cell.paragraphs[0]
+        content_run: Run = content_par.add_run()
+        content_run.add_picture(str(self._file), width=Inches(self._width))
+
+        return document
+
+
+class Images:
+    """Multiple images side by side"""
+
+    def __init__(
+        self,
+        id: str,
+        images: list[Image],
+        title: str | None = None,
+        subtitle: str | None = None,
+        width: int = 3,
+    ) -> None:
+        self._id: str = id
+        self._images: list[Image] = images
+        self._title: str = title
+        self._subtitle: str = subtitle
+        self._width: int = width
+
+    @classmethod
+    def from_yaml(cls, loader: TemplateLoader, node: MappingNode) -> Self:
+        images_cfg: dict[str, Any] = loader.construct_mapping(node, deep=True)
+        images_dict: dict[str, Image] = {
+            id: image
+            for (id, image) in images_cfg.items()
+            if isinstance(image, Image)
+        }
+        other_dict: dict[str, Any] = {
+            k: v for (k, v) in images_cfg.items() if k not in images_dict
+        }
+        return cls(
+            id=other_dict.pop("id"),
+            images=list(images_dict.values()),
+            **other_dict
+        )
+
+    def render(self, document: Document) -> Document:
+        # add container table
+        columns: int = len(self._images)
+        rows: int = 2 if columns == 1 else 3
+        container_table: Table = document.add_table(rows, columns)
+        container_table.alignment = CENTER
+        _style_container_table(container_table)
+
+        # merge title cells if necessary
+        title_cell: _Cell = container_table.cell(0, 0)
+        for idx, _ in enumerate(container_table.columns):
+            title_cell.merge(container_table.cell(0, idx))
+
+        # fill container table titles
+        _fill_titles(title_cell, title=self._title, subtitle=self._subtitle)
+        # fill container table subtitles
+        for idx, subtitle_cell in enumerate(container_table.rows[1].cells):
+            _fill_titles(
+                subtitle_cell,
+                title=self._images[idx]._title,
+                subtitle=self._images[idx]._subtitle,
+                bold=False,
+            )
+
+        # fill container table plots
+        for idx, content_cell in enumerate(container_table.rows[2].cells):
+            content_par: Paragraph = content_cell.paragraphs[0]
+            content_run: Run = content_par.add_run()
+            content_run.add_picture(
+                str(self._images[idx]._file), width=Inches(self._width)
+            )
+        return document