tesorotools-python 0.0.18__py3-none-any.whl

tesorotools/artists/table.py

@@ -0,0 +1,200 @@
+from math import floor
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from tesorotools.dependencies.resolution import collect_series
+from tesorotools.offsets.outliers import flag_outliers
+from tesorotools.utils.globals import DEBUG
+from tesorotools.utils.matplotlib import format_annotation, is_zero
+
+# this file is by far the worst and most spaghettified, must be rewritten
+
+# to global config
+GOOD: str = "00c800"
+BAD: str = "c80000"
+THRESHOLD: float = 1
+SHADE_LEVELS = 2
+
+
+def _shade_intensity(
+    ratio: float, shade_levels: int = 2, continuous: bool = False
+) -> str:
+    # intensity may vary from 150 (highest) to 255 (lowest), a grand difference of 105
+    # there are SHADE_LEVELS levels, so increments will be of 105/SHADE_LEVELS
+    corrected_ratio: float = min(ratio, shade_levels)
+    corrected_ratio: float = (
+        floor(corrected_ratio) if not continuous else corrected_ratio
+    )
+    increment: float = (corrected_ratio - 1) * (105 / shade_levels)
+    intensity: float = 255 - increment
+    intensity_hex: str = f"{int(intensity):x}"
+    return intensity_hex
+
+
+def _generate_column(
+    column_data: pd.Series,
+    column_cfg: dict[str, Any],
+    outliers_flags: pd.Series | None = None,
+):
+    # TODO: factor out
+    # data
+    if column_cfg["show_units_in_title"]:
+        column_data.name = f"{column_cfg["name"]} ({column_cfg["unit"]})"
+    else:
+        column_data.name = column_cfg["name"]
+    column_cfg["formatted_name"] = column_data.name
+
+    unit = (
+        column_cfg["unit"]
+        if column_cfg["show_units_in_cell"] and column_cfg["unit"] is not None
+        else ""
+    )
+    scaled_data: pd.Series = column_data * column_cfg["scale"]
+    formatted_data = scaled_data.apply(
+        lambda x: format_annotation(
+            x, decimals=column_cfg["decimals"], units=unit
+        )
+    )
+    zeros: pd.Series = scaled_data.apply(
+        lambda x: is_zero(x, decimals=column_cfg["decimals"])
+    )
+    positives: pd.Series = scaled_data > 0
+    negatives: pd.Series = scaled_data < 0
+
+    # colors
+    colors_cfg: bool = column_cfg["colors"]
+    color_data: pd.Series = pd.Series(
+        index=formatted_data.index, name=column_data.name, dtype=str
+    )
+    if colors_cfg:
+        positive_good: bool = column_cfg["positive_good"]
+        color_data[positives] = GOOD if positive_good else BAD
+        color_data.loc[negatives] = BAD if positive_good else GOOD
+        color_data.loc[zeros.values] = pd.NA
+
+    # shades
+    shade_data: pd.Series = pd.Series(
+        index=formatted_data.index, name=column_data.name, dtype=str
+    )
+    if outliers_flags is not None:
+        thresholds: pd.Series = abs(outliers_flags / THRESHOLD)
+        intensities: pd.Series = thresholds.apply(
+            lambda x: _shade_intensity(x, SHADE_LEVELS)
+        )
+        shade_data[(thresholds >= 1) & (outliers_flags > 0)] = intensities[
+            (thresholds >= 1) & (outliers_flags > 0)
+        ].apply(lambda x: f"00{x}00" if positive_good else f"{x}0000")
+        shade_data[(thresholds >= 1) & (outliers_flags < 0)] = intensities[
+            (thresholds >= 1) & (outliers_flags < 0)
+        ].apply(lambda x: f"{x}0000" if positive_good else f"00{x}00")
+
+    return formatted_data, color_data, shade_data
+
+
+def _generate_block(block_data: pd.DataFrame, block_cfg: dict[str, Any]):
+    columns: dict[str, Any] = block_cfg["columns"]
+    formatted_columns: list[pd.Series] = []
+    color_columns: list[pd.Series] = []
+    shade_columns: list[pd.Series] = []
+    sort_idx: pd.Index | None = None
+    for column_name, column_cfg in columns.items():
+        last_date: pd.Timestamp = block_data.index.get_level_values(
+            level=0
+        ).max()
+        block_data = block_data.rename(columns=block_cfg["series"])
+        offset: str = column_cfg["offset"]
+        difference: str = column_cfg["difference"]
+        stat: str = column_cfg["stat"]
+        outliers: bool = column_cfg["outliers"]
+
+        stat_data = block_data.loc[
+            (last_date, offset, difference, slice(None)), :
+        ]
+        stat_data.index = stat_data.index.get_level_values(level=-1)
+
+        outliers_flags: pd.Series | None = None
+        if outliers:
+            outliers_flags = flag_outliers(stat_data.T)
+
+        column_data: pd.Series = stat_data.loc[stat, :]
+        # sort capability
+        sort: str = block_cfg.get("sort", None)
+        if sort is not None and column_name == sort:
+            column_data: pd.Series = column_data.sort_values(ascending=False)
+            sort_idx = column_data.index
+        formatted_column, color_column, shade_column = _generate_column(
+            column_data, column_cfg, outliers_flags
+        )
+        formatted_columns.append(formatted_column)
+        color_columns.append(color_column)
+        shade_columns.append(shade_column)
+
+    if sort_idx is not None:
+        formatted_columns = [s.reindex(sort_idx) for s in formatted_columns]
+        color_columns = [s.reindex(sort_idx) for s in color_columns]
+        shade_columns = [s.reindex(sort_idx) for s in shade_columns]
+
+    formatted_block: pd.DataFrame = pd.concat(formatted_columns, axis=1)
+    color_block: pd.DataFrame = pd.concat(color_columns, axis=1)
+    shade_block: pd.DataFrame = pd.concat(shade_columns, axis=1)
+
+    formatted_block.columns.name = block_cfg["title"]
+    color_block.columns.name = block_cfg["title"]
+    shade_block.columns.name = block_cfg["title"]
+    return formatted_block, color_block, shade_block
+
+
+def generate_table(table_data: pd.DataFrame, table_cfg: dict[str, Any]):
+    blocks: dict[str, dict] = table_cfg["blocks"]
+    formatted_blocks: list[pd.DataFrame] = []
+    color_blocks: list[pd.DataFrame] = []
+    shade_blocks: list[pd.DataFrame] = []
+    axis = 0 if table_cfg["axis"] == "vertical" else 1
+    # sorting capabilities
+    for block_name, block_cfg in blocks.items():
+        series_dict: dict[str, str] = block_cfg["series"]
+        block_data: pd.DataFrame = table_data.loc[:, series_dict.keys()]
+        formatted_block, color_block, shade_block = _generate_block(
+            block_data, block_cfg
+        )
+        formatted_blocks.append(formatted_block)
+        color_blocks.append(color_block)
+        shade_blocks.append(shade_block)
+    formatted_table = pd.concat(
+        formatted_blocks,
+        axis=axis,
+        keys=[
+            formatted_block.columns.name for formatted_block in formatted_blocks
+        ],
+    )
+    color_table = pd.concat(
+        color_blocks,
+        axis=axis,
+        keys=[
+            formatted_block.columns.name for formatted_block in formatted_blocks
+        ],
+    )
+    shade_table = pd.concat(
+        shade_blocks,
+        axis=axis,
+        keys=[
+            formatted_block.columns.name for formatted_block in formatted_blocks
+        ],
+    )
+    return formatted_table, color_table, shade_table
+
+
+def generate_tables_from_flash(
+    out_path: Path, flash: pd.DataFrame, config_dicts: dict[str, dict]
+):
+    for table_name, table_cfg in config_dicts.items():
+        series: list[str] = list(collect_series(table_cfg))
+        table_data: pd.DataFrame = flash.loc[:, series]
+        formatted_table, color_table, shade_table = generate_table(
+            table_data, table_cfg
+        )
+        formatted_table.to_feather(out_path / f"{table_name}.feather")
+        color_table.to_feather(out_path / f"{table_name}_color.feather")
+        shade_table.to_feather(out_path / f"{table_name}_shade.feather")

tesorotools/artists/type_curve.py

@@ -0,0 +1,218 @@
+# pending to assure stylesheet data and fonts are only loaded once
+
+from pathlib import Path
+from typing import Any
+
+import matplotlib.pyplot as plt
+import pandas as pd
+from matplotlib.ticker import FuncFormatter
+from pandas import Timestamp
+
+from tesorotools.utils.config import merge
+from tesorotools.utils.globals import DEBUG
+from tesorotools.utils.matplotlib import (
+    PLOT_CONFIG,
+    format_annotation,
+    load_fonts,
+)
+
+TYPE_CURVE_CONFIG: dict[str, Any] = PLOT_CONFIG["type_curve"]
+AX_CONFIG: dict[str, Any] = PLOT_CONFIG["ax"]
+FIG_CONFIG: dict[str, Any] = PLOT_CONFIG["figure"]
+
+load_fonts()
+
+
+def _style_spines(
+    ax: plt.Axes,
+    decimals: int,
+    units: str,
+    *,
+    color: str,
+    linewidth: str,
+):
+    ax.grid(visible=True, axis="y")
+    for spine in ax.spines.values():
+        spine.set_color(color)
+        spine.set_linewidth(linewidth)
+    ax.yaxis.tick_right()
+    ax.yaxis.set_major_formatter(
+        FuncFormatter(lambda y, _: format_annotation(y, decimals, units))
+    )
+    ax.tick_params(axis="both", which="major")
+    ax.set_xticks(
+        ax.get_xticks(), ax.get_xticklabels(), rotation=45, ha="right"
+    )
+    for tick in ax.get_xticklines():
+        tick.set_markeredgecolor(color)
+    for tick in ax.get_yticklines():
+        tick.set_markeredgecolor(color)
+
+
+def _style_baseline(ax: plt.Axes, **baseline_config):
+    color: str = baseline_config["color"]
+    bottom_lim, top_lim = ax.get_ylim()
+    ax.set_ylim(bottom=min(0, bottom_lim), top=max(0, top_lim))
+    bottom_lim, top_lim = ax.get_ylim()
+    if bottom_lim == 0:
+        ax.spines["bottom"].set_edgecolor(color)
+    elif top_lim == 0:
+        ax.spines["top"].set_edgecolor(color)
+    else:
+        ax.axhline(y=0, **baseline_config)
+
+
+def _format_data(data: pd.DataFrame) -> dict[str, Any]:
+    # metadata
+    date_index: pd.DatetimeIndex = data.index
+    current_date: Timestamp = date_index.max()
+    current_year: int = current_date.year
+    last_year: int = (current_date - pd.DateOffset(years=1)).year
+
+    # current data
+    current_data: pd.Series = data.loc[current_date, :]
+    current_data.name = "current_data"
+
+    # current year
+    current_year_data: pd.DataFrame = data.loc[
+        date_index.year == current_year, :
+    ]
+    current_year_max: pd.Series = current_year_data.max()
+    current_year_max.name = "current_year_max"
+    current_year_min: pd.Series = current_year_data.min()
+    current_year_min.name = "current_year_min"
+
+    # last year
+    last_year_data: pd.DataFrame = data.loc[date_index.year == last_year, :]
+    last_year_max: pd.Series = last_year_data.max()
+    last_year_max.name = "last_year_max"
+    last_year_min: pd.Series = last_year_data.min()
+    last_year_min.name = "last_year_min"
+
+    formatted_data: pd.DataFrame = pd.concat(
+        [
+            last_year_max,
+            last_year_min,
+            current_year_max,
+            current_year_min,
+            current_data,
+        ],
+        axis=1,
+    )
+
+    return {
+        "data": formatted_data,
+        "current_date": current_date.strftime("%d/%m/%Y"),
+        "current_year": current_year,
+        "last_year": last_year,
+    }
+
+
+def _plot_current_data(
+    ax: plt.Axes,
+    data: pd.DataFrame,
+    date_fmt: str,
+    *,
+    linewidth: float,
+    marker: str,
+    points_to_mark: list[str],
+    color: str,
+    decimals: int,
+    units: str,
+):
+    data.plot(
+        ax=ax,
+        color=color,
+        linewidth=linewidth,
+        label=date_fmt,
+    )
+    for point in points_to_mark:
+        value = data.loc[point]
+        ax.plot(
+            point,
+            value,
+            marker=marker,
+            color=color,
+        )
+        ax.annotate(
+            format_annotation(value, decimals, units),
+            (point, value),
+            textcoords="offset points",
+            xytext=(0, 10),
+            ha="center",
+        )
+
+
+def plot_type_curve(
+    data: pd.DataFrame,
+    out_file: Path,
+    **config,
+):
+    config: dict[str, Any] = merge(config, TYPE_CURVE_CONFIG)
+
+    if config["yaxis"]["units"] == "p.b.":
+        data = data * 100
+
+    formatted_assets: dict[str, Any] = _format_data(data)
+    formatted_data: pd.DataFrame = formatted_assets["data"]
+    due_index: pd.Index = formatted_data.index
+
+    fig = plt.figure(**FIG_CONFIG)
+    ax: plt.Axes = fig.add_subplot()
+
+    last_config: dict[str, Any] = config["last"]
+    ax.fill_between(
+        due_index,
+        formatted_data["last_year_min"],
+        formatted_data["last_year_max"],
+        alpha=last_config["alpha"],
+        color=last_config["color"],
+        edgecolor=None,
+        label=f"Rango {formatted_assets['last_year']}",
+    )
+
+    current_config: dict[str, Any] = config["current"]
+    ax.fill_between(
+        due_index,
+        formatted_data["current_year_min"],
+        formatted_data["current_year_max"],
+        alpha=current_config["alpha"],
+        color=current_config["color"],
+        edgecolor=None,
+        label=f"Rango {formatted_assets['current_year']}",
+    )
+
+    _plot_current_data(
+        ax,
+        formatted_data["current_data"],
+        formatted_assets["current_date"],
+        **config["line"],
+    )
+    _style_spines(ax, **config["yaxis"], **AX_CONFIG["spines"])
+    _style_baseline(ax, **AX_CONFIG["baseline"])
+    ax.legend(
+        loc="upper center",
+        bbox_to_anchor=(0.5, config["legend_sep"]),
+        ncol=3,
+    )
+    fig.savefig(out_file)
+
+
+# data is expected to be a simple time series data, columns are series and rows represents dates
+def plot_type_curves(
+    out_path: Path, data: pd.DataFrame, config_dicts: dict[str, Any]
+):
+    for name, config in config_dicts.items():
+        if not name.startswith("."):  # aux entries
+            series: dict[str, str] = config["series"]
+            if len(series) < 2:
+                raise ValueError(
+                    f"In plot {name}: A type curve must have at least two due periods. Given periods: {series.keys()}"
+                )
+            trimmed_data: pd.DataFrame = data.loc[:, series.keys()]
+            trimmed_data: pd.DataFrame = trimmed_data.rename(columns=series)
+            plot_type_curve(
+                data=trimmed_data,
+                out_file=out_path / f"{name}.png",
+                **config,
+            )

tesorotools/assets/README.md

@@ -0,0 +1,5 @@
+En esta carpeta se guardarán todos los *assets* del proyecto, como
+
+- [fonts](fonts/): Fuentes (archivos *.otf).
+- [plots.yaml](plots.yaml): Archivo de configuración general para los gráficos.
+- [tesoro.mplstyle](tesoro.mplstyle): Archivo de estilos *matplotlib* para los gráficos.

tesorotools/assets/fonts/README.md

@@ -0,0 +1 @@
+Archivos *.otf de fuentes. No los incluimos en el control de versiones.

tesorotools/assets/plots.yaml

@@ -0,0 +1,43 @@
+# general plot config file
+
+style:
+  font: Cabinet Grotesk
+
+figure:
+  dpi: 500
+  layout: constrained
+
+ax:
+  spines:
+    color: gray
+    linewidth: 1
+  baseline:
+    color: "#d9e1fc"
+    linestyle: solid
+    linewidth: 1
+    zorder: 1
+
+type_curve:
+  last:
+    alpha: 0.2
+    color: C1
+  current:
+    alpha: 0.3
+    color: C0
+  line:
+    linewidth: 2
+    marker: D
+    color: C0
+  legend_sep: -0.1
+
+barh:
+  highlight_factor: 0.4
+  padding: 5
+
+line:
+  legend_sep: -0.125
+  ncol: 5
+
+table:
+  style: Light Shading Accent 1
+  autofit: False

tesorotools/assets/tesoro.mplstyle

@@ -0,0 +1,21 @@
+# fonts
+font.size: 12.0
+xtick.labelsize: 12.0
+ytick.labelsize: 12.0
+legend.fontsize: 12.0
+
+# modified dashed_pattern
+lines.dashed_pattern: 10, 5
+
+# markers
+lines.markersize: 8
+
+# grids
+grid.color: d9e1fc
+grid.linestyle: dashed
+grid.linewidth: 0.5
+grid.alpha: 0.5
+
+# color palette
+axes.prop_cycle: cycler('color', ['001e93', 'ffbd4c', '6d6d6d', '000000', 'e6821e', '2957ff', 'fba737', '5da6ff', '00ebb2', '6d0000', '009200', 'df2871', 'd9e1fc', 'ffddab'])
+

tesorotools/convert.py

@@ -0,0 +1,99 @@
+# this file may migrate to the utils package
+
+from pathlib import Path
+
+import pandas as pd
+
+from tesorotools.artists.barh_plot import plot_barh_charts_from_flash
+from tesorotools.artists.line_plot import plot_line_charts
+from tesorotools.artists.table import generate_tables_from_flash
+from tesorotools.artists.type_curve import plot_type_curves
+from tesorotools.dependencies.resolution import (
+    compute_derivate_series,
+    concat_derivate_series,
+    resolve_series,
+)
+from tesorotools.offsets.offsets import trim
+from tesorotools.utils.config import read_config
+
+
+def index_replace(old: str, new: str, index: pd.MultiIndex) -> pd.MultiIndex:
+    return pd.MultiIndex.from_tuples(
+        [
+            tuple(
+                [
+                    x.replace(old, new) if isinstance(x, str) else x
+                    for x in tuple_index
+                ]
+            )
+            for tuple_index in index
+        ]
+    )
+
+
+def cheap_convert(old_file: Path) -> pd.DataFrame:
+    old = pd.read_feather(old_file)
+
+    trimmed = old.loc[(slice(None), "no", "absolute", "value"), :].copy()
+
+    old = old.loc[
+        (
+            slice(None),
+            ["day", "mtd", "week", "year", "tariff_crisis"],
+            slice(None),
+            slice(None),
+        ),
+        :,
+    ]
+    old.index = index_replace("day", "bday", old.index)
+    old.index = index_replace("week", "ftd", old.index)
+    old.index = index_replace("year", "ytd", old.index)
+    old.index = index_replace("roll_var", "roll_std", old.index)
+    old.index = index_replace("tariff_crisis", "2025-04-02", old.index)
+
+    trimmed.index = index_replace("absolute", "no", trimmed.index)
+
+    new = pd.concat([old, trimmed])
+    return new
+
+
+if __name__ == "__main__":
+    preprocess = True
+    barh_config_dicts = read_config(Path("examples") / "barh_plots.yaml")
+    line_config_dicts = read_config(Path("examples") / "line_plots.yaml")
+    type_config_dicts = read_config(Path("examples") / "type_curves.yaml")
+    table_config_dicts = read_config(Path("examples") / "tables.yaml")
+
+    if preprocess:
+        old_file: Path = Path("debug") / "flash.feather"
+        dependencies_cfg = read_config(Path("examples") / "dependencies.yaml")
+        flash: pd.DataFrame = cheap_convert(old_file)
+        resolved_dict = resolve_series(
+            [
+                barh_config_dicts,
+                line_config_dicts,
+                type_config_dicts,
+                table_config_dicts,
+            ],
+            dependencies_cfg,
+        )
+        independent_full_df = flash.loc[:, list(resolved_dict["independent"])]
+        independent_trimmed_df = trim(independent_full_df)
+        dependent_trimmed_df = compute_derivate_series(
+            resolved_dict["dependent"], independent_trimmed_df
+        )
+        offsets_config = read_config(Path("examples") / "offsets.yaml")
+        full_df = concat_derivate_series(
+            independent_full_df,
+            dependent_trimmed_df,
+            offsets_config,
+            # force_trim=True,
+        )
+        full_df.to_feather("derivates.feather")
+
+    full_df = pd.read_feather("derivates.feather")
+    trimmed_df = trim(full_df)
+    plot_barh_charts_from_flash(full_df, barh_config_dicts)
+    plot_line_charts(trimmed_df, line_config_dicts)
+    plot_type_curves(trimmed_df, type_config_dicts)
+    generate_tables_from_flash(full_df, table_config_dicts)

tesorotools/data_sources/README.md

@@ -0,0 +1,14 @@
+# LSEG Data Library for Python
+
+## Capa de acceso
+
+## Capa de contenido
+
+## Capa de reparto
+
+## Capa de sesión
+
+## Referencias
+
+- https://cdn.refinitiv.com/public/lseg-lib-python-doc/2.0.0.2/book/en/index.html
+- https://pypi.org/project/lseg-data/

tesorotools/data_sources/debug.py

@@ -0,0 +1,26 @@
+import numpy as np
+import pandas as pd
+
+from tesorotools.utils.config import read_config
+from tesorotools.utils.globals import DEBUG, EXAMPLES
+
+# just mocking
+CATALOG: dict[str, str] = read_config(EXAMPLES / "data.yaml")["debug"]
+
+
+def get_series(start: str, end: str, series: list[str]) -> pd.DataFrame:
+    # series is a list of agnostic ids
+    specific_ids = []
+    for agnostic_id in series:
+        specific_id: str | None = CATALOG.get(agnostic_id, None)
+        if specific_id is None:
+            raise KeyError(f"{agnostic_id} not found in debug configuration")
+        specific_ids.append(specific_id)
+
+    index: pd.DatetimeIndex = pd.date_range(start=start, end=end).sort_values()
+    df: pd.DataFrame = pd.DataFrame(
+        data=np.random.randn(len(index), len(series)),
+        index=index,
+        columns=series,
+    )
+    return df