tesorotools-python 0.0.0__py2.py3-none-any.whl

tesorotools/artists/type_curve.py

@@ -0,0 +1,216 @@
+# pending to assure stylesheet data and fonts are only loaded once
+
+from pathlib import Path
+from typing import Any
+
+import matplotlib.pyplot as plt
+import pandas as pd
+from matplotlib.ticker import FuncFormatter
+from pandas import Timestamp
+
+from tesorotools.utils.config import merge
+from tesorotools.utils.globals import DEBUG
+from tesorotools.utils.matplotlib import (
+    PLOT_CONFIG,
+    format_annotation,
+    load_fonts,
+)
+
+TYPE_CURVE_CONFIG: dict[str, Any] = PLOT_CONFIG["type_curve"]
+AX_CONFIG: dict[str, Any] = PLOT_CONFIG["ax"]
+FIG_CONFIG: dict[str, Any] = PLOT_CONFIG["figure"]
+
+load_fonts()
+
+
+def _style_spines(
+    ax: plt.Axes,
+    decimals: int,
+    units: str,
+    *,
+    color: str,
+    linewidth: str,
+):
+    ax.grid(visible=True, axis="y")
+    for spine in ax.spines.values():
+        spine.set_color(color)
+        spine.set_linewidth(linewidth)
+    ax.yaxis.tick_right()
+    ax.yaxis.set_major_formatter(
+        FuncFormatter(lambda y, _: format_annotation(y, decimals, units))
+    )
+    ax.tick_params(axis="both", which="major")
+    ax.set_xticks(
+        ax.get_xticks(), ax.get_xticklabels(), rotation=45, ha="right"
+    )
+    for tick in ax.get_xticklines():
+        tick.set_markeredgecolor(color)
+    for tick in ax.get_yticklines():
+        tick.set_markeredgecolor(color)
+
+
+def _style_baseline(ax: plt.Axes, **baseline_config):
+    color: str = baseline_config["color"]
+    bottom_lim, top_lim = ax.get_ylim()
+    ax.set_ylim(bottom=min(0, bottom_lim), top=max(0, top_lim))
+    bottom_lim, top_lim = ax.get_ylim()
+    if bottom_lim == 0:
+        ax.spines["bottom"].set_edgecolor(color)
+    elif top_lim == 0:
+        ax.spines["top"].set_edgecolor(color)
+    else:
+        ax.axhline(y=0, **baseline_config)
+
+
+def _format_data(data: pd.DataFrame) -> dict[str, Any]:
+    # metadata
+    date_index: pd.DatetimeIndex = data.index
+    current_date: Timestamp = date_index.max()
+    current_year: int = current_date.year
+    last_year: int = (current_date - pd.DateOffset(years=1)).year
+
+    # current data
+    current_data: pd.Series = data.loc[current_date, :]
+    current_data.name = "current_data"
+
+    # current year
+    current_year_data: pd.DataFrame = data.loc[
+        date_index.year == current_year, :
+    ]
+    current_year_max: pd.Series = current_year_data.max()
+    current_year_max.name = "current_year_max"
+    current_year_min: pd.Series = current_year_data.min()
+    current_year_min.name = "current_year_min"
+
+    # last year
+    last_year_data: pd.DataFrame = data.loc[date_index.year == last_year, :]
+    last_year_max: pd.Series = last_year_data.max()
+    last_year_max.name = "last_year_max"
+    last_year_min: pd.Series = last_year_data.min()
+    last_year_min.name = "last_year_min"
+
+    formatted_data: pd.DataFrame = pd.concat(
+        [
+            last_year_max,
+            last_year_min,
+            current_year_max,
+            current_year_min,
+            current_data,
+        ],
+        axis=1,
+    )
+
+    return {
+        "data": formatted_data,
+        "current_date": current_date.strftime("%d/%m/%Y"),
+        "current_year": current_year,
+        "last_year": last_year,
+    }
+
+
+def _plot_current_data(
+    ax: plt.Axes,
+    data: pd.DataFrame,
+    date_fmt: str,
+    *,
+    linewidth: float,
+    marker: str,
+    points_to_mark: list[str],
+    color: str,
+    decimals: int,
+    units: str,
+):
+    data.plot(
+        ax=ax,
+        color=color,
+        linewidth=linewidth,
+        label=date_fmt,
+    )
+    for point in points_to_mark:
+        value = data.loc[point]
+        ax.plot(
+            point,
+            value,
+            marker=marker,
+            color=color,
+        )
+        ax.annotate(
+            format_annotation(value, decimals, units),
+            (point, value),
+            textcoords="offset points",
+            xytext=(0, 10),
+            ha="center",
+        )
+
+
+def plot_type_curve(
+    data: pd.DataFrame,
+    out_file: Path,
+    **config,
+):
+    config: dict[str, Any] = merge(config, TYPE_CURVE_CONFIG)
+
+    if config["yaxis"]["units"] == "p.b.":
+        data = data * 100
+
+    formatted_assets: dict[str, Any] = _format_data(data)
+    formatted_data: pd.DataFrame = formatted_assets["data"]
+    due_index: pd.Index = formatted_data.index
+
+    fig = plt.figure(**FIG_CONFIG)
+    ax: plt.Axes = fig.add_subplot()
+
+    last_config: dict[str, Any] = config["last"]
+    ax.fill_between(
+        due_index,
+        formatted_data["last_year_min"],
+        formatted_data["last_year_max"],
+        alpha=last_config["alpha"],
+        color=last_config["color"],
+        edgecolor=None,
+        label=f"Rango {formatted_assets['last_year']}",
+    )
+
+    current_config: dict[str, Any] = config["current"]
+    ax.fill_between(
+        due_index,
+        formatted_data["current_year_min"],
+        formatted_data["current_year_max"],
+        alpha=current_config["alpha"],
+        color=current_config["color"],
+        edgecolor=None,
+        label=f"Rango {formatted_assets['current_year']}",
+    )
+
+    _plot_current_data(
+        ax,
+        formatted_data["current_data"],
+        formatted_assets["current_date"],
+        **config["line"],
+    )
+    _style_spines(ax, **config["yaxis"], **AX_CONFIG["spines"])
+    _style_baseline(ax, **AX_CONFIG["baseline"])
+    ax.legend(
+        loc="upper center",
+        bbox_to_anchor=(0.5, config["legend_sep"]),
+        ncol=3,
+    )
+    fig.savefig(out_file)
+
+
+# data is expected to be a simple time series data, columns are series and rows represents dates
+def plot_type_curves(data: pd.DataFrame, config_dicts: dict[str, Any]):
+    for name, config in config_dicts.items():
+        if not name.startswith("."):  # aux entries
+            series: dict[str, str] = config["series"]
+            if len(series) < 2:
+                raise ValueError(
+                    f"In plot {name}: A type curve must have at least two due periods. Given periods: {series.keys()}"
+                )
+            trimmed_data: pd.DataFrame = data.loc[:, series.keys()]
+            trimmed_data: pd.DataFrame = trimmed_data.rename(columns=series)
+            plot_type_curve(
+                data=trimmed_data,
+                out_file=DEBUG / "type_curve" / f"{name}.png",
+                **config,
+            )

tesorotools/convert.py

@@ -0,0 +1,93 @@
+# this file may migrate to the utils package
+
+from pathlib import Path
+
+import pandas as pd
+
+from tesorotools.artists.barh_plot import plot_barh_charts_from_flash
+from tesorotools.artists.line_plot import plot_line_charts
+from tesorotools.artists.table import generate_tables_from_flash
+from tesorotools.artists.type_curve import plot_type_curves
+from tesorotools.dependencies.resolution import (
+    compute_derivate_series,
+    concat_derivate_series,
+    resolve_series,
+)
+from tesorotools.offsets.offsets import trim
+from tesorotools.utils.config import read_config
+
+
+def index_replace(old: str, new: str, index: pd.MultiIndex) -> pd.MultiIndex:
+    return pd.MultiIndex.from_tuples(
+        [
+            tuple(
+                [
+                    x.replace(old, new) if isinstance(x, str) else x
+                    for x in tuple_index
+                ]
+            )
+            for tuple_index in index
+        ]
+    )
+
+
+def cheap_convert(old_file: Path) -> pd.DataFrame:
+    old = pd.read_feather(old_file)
+
+    trimmed = old.loc[(slice(None), "no", "absolute", "value"), :].copy()
+
+    old = old.loc[
+        (slice(None), ["day", "mtd", "week", "year"], slice(None), slice(None)),
+        :,
+    ]
+    old.index = index_replace("day", "bday", old.index)
+    old.index = index_replace("week", "ftd", old.index)
+    old.index = index_replace("year", "ytd", old.index)
+    old.index = index_replace("roll_var", "roll_std", old.index)
+
+    trimmed.index = index_replace("absolute", "no", trimmed.index)
+
+    new = pd.concat([old, trimmed])
+    return new
+
+
+if __name__ == "__main__":
+    preprocess = False
+    barh_config_dicts = read_config(Path("examples") / "barh_plots.yaml")
+    line_config_dicts = read_config(Path("examples") / "line_plots.yaml")
+    type_config_dicts = read_config(Path("examples") / "type_curves.yaml")
+    table_config_dicts = read_config(Path("examples") / "tables.yaml")
+
+    if preprocess:
+        old_file: Path = Path("debug") / "flash.feather"
+        dependencies_cfg = read_config(Path("examples") / "dependencies.yaml")
+        flash: pd.DataFrame = cheap_convert(old_file)
+        resolved_dict = resolve_series(
+            [
+                barh_config_dicts,
+                line_config_dicts,
+                type_config_dicts,
+                table_config_dicts,
+            ],
+            dependencies_cfg,
+        )
+        independent_full_df = flash.loc[:, list(resolved_dict["independent"])]
+        independent_trimmed_df = trim(independent_full_df)
+        dependent_trimmed_df = compute_derivate_series(
+            resolved_dict["dependent"], independent_trimmed_df
+        )
+        offsets_config = read_config(Path("examples") / "offsets.yaml")
+        full_df = concat_derivate_series(
+            independent_full_df,
+            dependent_trimmed_df,
+            offsets_config,
+            force_trim=True,
+        )
+        full_df.to_feather("derivates.feather")
+
+    full_df = pd.read_feather("derivates.feather")
+    trimmed_df = trim(full_df)
+    plot_barh_charts_from_flash(full_df, barh_config_dicts)
+    plot_line_charts(trimmed_df, line_config_dicts)
+    plot_type_curves(trimmed_df, type_config_dicts)
+    generate_tables_from_flash(full_df, table_config_dicts)

tesorotools/data_sources/debug.py

@@ -0,0 +1,26 @@
+import numpy as np
+import pandas as pd
+
+from tesorotools.utils.config import read_config
+from tesorotools.utils.globals import DEBUG, EXAMPLES
+
+# just mocking
+CATALOG: dict[str, str] = read_config(EXAMPLES / "data.yaml")["debug"]
+
+
+def get_series(start: str, end: str, series: list[str]) -> pd.DataFrame:
+    # series is a list of agnostic ids
+    specific_ids = []
+    for agnostic_id in series:
+        specific_id: str | None = CATALOG.get(agnostic_id, None)
+        if specific_id is None:
+            raise KeyError(f"{agnostic_id} not found in debug configuration")
+        specific_ids.append(specific_id)
+
+    index: pd.DatetimeIndex = pd.date_range(start=start, end=end).sort_values()
+    df: pd.DataFrame = pd.DataFrame(
+        data=np.random.randn(len(index), len(series)),
+        index=index,
+        columns=series,
+    )
+    return df

tesorotools/data_sources/eikon.py

@@ -0,0 +1,117 @@
+import time
+from pathlib import Path
+
+import eikon as ek
+import pandas as pd
+
+
+def get_series(
+    api_key: str,
+    series_id_list: list[str],
+    start_date: str,
+    end_date: str,
+    freq: str = "B",
+    fields: list[str] | None = None,
+    cooldown: int = 60,
+    datapoint_limit: int = 2_000,
+    cache_path: Path | None = None,
+) -> pd.DataFrame:
+    """Downloads data from eikon given that tou have a valid API key"""
+    ek.set_app_key(api_key)
+    fields = (
+        ["TIMESTAMP", "CLOSE", "CF_LAST", "CF_YIELD"]
+        if fields is None
+        else fields
+    )
+
+    dates_list: list[str] = list(
+        pd.date_range(start=start_date, end=end_date, freq=freq).astype("str")
+    )
+    partial_data: list[pd.DataFrame] = []
+    download_step: int = datapoint_limit // (
+        len(series_id_list) * (len(fields) - 1)
+    )
+    downloaded_dates: int = 0
+    while downloaded_dates < len(dates_list):
+        dates_to_download = dates_list[
+            downloaded_dates : downloaded_dates + download_step
+        ]
+        start = dates_to_download[0]
+        end = dates_to_download[-1]
+        cache_file_path: Path = (
+            cache_path / f"from_{start}_to_{end}.csv"
+            if cache_path is not None
+            else None
+        )
+        if (cache_file_path is None) or (not cache_file_path.exists()):
+            data: pd.DataFrame = block_download(
+                series_id_list,
+                start_date=start,
+                end_date=end,
+                freq=freq,
+                fields=fields,
+                cooldown=cooldown,
+                file_path=cache_file_path,
+                debug=True,
+            )
+            if cache_file_path is None:
+                partial_data.append(data)
+            if downloaded_dates + download_step < len(dates_list):
+                print(f"Waiting {cooldown} seconds for Eikon to cool down...")
+                time.sleep(cooldown)
+        downloaded_dates += download_step
+    data = concat_partial_data(cache_path, partial_data)
+    return data
+
+
+def block_download(
+    series_id_list: list[str],
+    start_date: str,
+    end_date: str,
+    freq: str = "B",
+    fields: list[str] | None = None,
+    cooldown: int = 60,
+    file_path: Path | None = None,
+    debug: bool = False,
+):
+    interval = "daily" if freq == "B" else freq
+
+    while True:
+        try:
+            data: pd.DataFrame | None = ek.get_timeseries(
+                rics=series_id_list,
+                start_date=start_date,
+                end_date=end_date,
+                fields=fields,
+                interval=interval,
+                debug=debug,
+            )
+            if data is None:
+                raise ek.EikonError(
+                    code=404, message="Service temporarily unavailable"
+                )
+            data = data.drop_duplicates()
+            data = data.sort_index()
+            if len(data.columns) == 1:
+                data.columns = series_id_list
+            if file_path is not None:
+                data.to_csv(file_path)
+            break
+        except ek.EikonError as e:
+            print(f"Eikon error: {e}")
+            print("This is probably not our fault")
+        print(f"Waiting {cooldown} seconds for Eikon to cool down...")
+        time.sleep(cooldown)
+    return data
+
+
+def concat_partial_data(
+    cache_path: Path, partial_data: list[pd.DataFrame]
+) -> pd.DataFrame:
+    dfs = partial_data
+    if cache_path is not None:
+        for chunk in cache_path.iterdir():
+            df = pd.read_csv(chunk, index_col="Date")
+            dfs.append(df)
+    full = pd.concat(dfs)
+    return full

tesorotools/database/push.py

@@ -0,0 +1,70 @@
+import json
+
+import pandas as pd
+from sqlalchemy import create_engine
+
+
+def get_connection_string(
+    engine: str,
+    driver: str,
+    username: str,
+    password: str,
+    host: str,
+    database: str,
+) -> str:
+    return f"{engine}+{driver}://{username}:{password}@{host}/{database}"
+
+
+def flash_to_database_format(data: pd.DataFrame) -> pd.DataFrame:
+    """Transform the data in *flash* format to data in *database* format
+
+    This operation takes a little while"""
+
+    data.columns.name = "name"
+    stacked_data = data.stack().to_frame(name="value")
+    database_data = stacked_data.reset_index()
+    database_data["dimensions"] = database_data.apply(
+        lambda row: json.dumps(
+            {
+                "offset": row["offset"],
+                "difference_type": row["difference_type"],
+                "stat": row["stat"],
+            }
+        ),
+        axis=1,
+    )
+    database_data = database_data.drop(
+        columns=["offset", "difference_type", "stat"]
+    )
+    database_data["value_meta"] = pd.NA
+    print(database_data)
+    return database_data
+
+
+def database_to_flash_format():
+    pass
+
+
+def push_to_database(data: pd.DataFrame, conn_string: str, table: str) -> None:
+    engine = create_engine(url=conn_string)
+    data.to_sql(
+        name=table, con=engine, if_exists="append", chunksize=1000, index=False
+    )
+
+
+# this may be just as a datasource (maybe a common protocol?)
+# check the api long video for further reference
+def pull_from_database(
+    conn_string: str, start: str, end: str, series: list[str]
+):
+    # just testing...
+    pd.read_sql_query(
+        """
+        SELECT date, value
+            FROM measurements
+            WHERE
+                date BETWEEN '2007-12-31' AND '2025-04-10' AND name='ES10YT=RR_DIFF' AND
+                dimensions @> '{"offset":"no", "stat":"value"}'
+            ORDER BY date ASC
+        """
+    )

tesorotools/dependencies/functions.py

@@ -0,0 +1,11 @@
+import pandas as pd
+
+
+def difference(
+    target: float | pd.Series, reference: float | pd.Series
+) -> float | pd.Series:
+    return target - reference
+
+
+def inverse(target: float | pd.Series) -> float | pd.Series:
+    return 1 / target

tesorotools/dependencies/node.py

@@ -0,0 +1,34 @@
+from collections.abc import Callable
+from typing import Self
+
+import pandas as pd
+
+from . import functions
+
+
+class Node:
+    def __init__(self, name: str) -> None:
+        self._name: str = name
+        self._edges: list[Self] = []
+
+    def add_edge(self, node: Self) -> None:
+        self._edges.append(node)
+
+    def build_edges(self, *, dependencies: list[str], function: str) -> None:
+        self._resolving_function: Callable[..., float | pd.Series] = getattr(
+            functions, function
+        )
+        for d in dependencies:
+            self.add_edge(Node(d))
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def edges(self) -> list[Self]:
+        return self._edges
+
+    @property
+    def resolving_function(self) -> Callable[..., float | pd.Series]:
+        return self._resolving_function

tesorotools/dependencies/resolution.py

@@ -0,0 +1,118 @@
+from typing import Any
+
+import pandas as pd
+
+from tesorotools.offsets.offsets import process_raw_data, trim
+
+from .node import Node
+
+
+def resolve(
+    start: Node,
+    resolved: list[Node],
+    unresolved: list[str],
+    independent: set[str],
+    dependencies_cfg: dict[str, Any],
+):
+    if start.name in dependencies_cfg:
+        is_independent: bool = False
+        config: dict[str, Any] = dependencies_cfg[start.name]
+        start.build_edges(**config)
+    else:
+        is_independent: bool = True
+        independent.add(start.name)
+
+    if not is_independent:
+        unresolved.append(start.name)
+        for node in start.edges:
+            if node.name in unresolved:
+                raise ValueError(
+                    f"circular dependency: {start.name} <-> {node.name}"
+                )
+            if node not in resolved:
+                resolve(
+                    node, resolved, unresolved, independent, dependencies_cfg
+                )
+        resolved.append(start)
+        unresolved.remove(start.name)
+
+
+def collect_document_series(
+    config_dicts: list[dict[str, Any]], find: str = "series"
+) -> list[str]:
+    series: set[str] = set()
+    for config_dict in config_dicts:
+        series = series | collect_series(config_dict, find)
+    return list(series)
+
+
+def resolve_series(
+    config_dicts: list[dict[str, Any]], dependencies_cfg: dict[str, Any]
+):
+    series: list[str] = collect_document_series(config_dicts)
+    nodes: list[Node] = [Node(name=name) for name in series]
+    independent_nodes: set[str] = set()
+    resolved: list[Node] = []
+    for node in nodes:
+        resolve(
+            start=node,
+            resolved=resolved,
+            unresolved=[],
+            independent=independent_nodes,
+            dependencies_cfg=dependencies_cfg,
+        )
+
+    return {
+        "independent": independent_nodes,
+        "dependent": resolved,
+    }
+
+
+def compute_derivate_series(
+    dependent_nodes: list[Node], trimmed_data: pd.DataFrame
+):
+    inferred_series: list[pd.Series] = []
+    for node in dependent_nodes:
+        dependencies_names: list[str] = [n.name for n in node.edges]
+        dependencies_df = trimmed_data.loc[:, dependencies_names]
+        dependencies_dict = dependencies_df.to_dict(orient="series").values()
+        inferred: pd.Series = node.resolving_function(*dependencies_dict)
+        inferred.name = node.name
+        inferred_series.append(inferred)
+    inferred_df: pd.DataFrame = pd.concat(inferred_series, axis=1)
+    return inferred_df
+
+
+def concat_derivate_series(
+    independent_full_df: pd.DataFrame,
+    derivate_trimmed_df: pd.DataFrame,
+    offsets_config: dict[str, Any],
+    force_trim: bool = False,
+) -> pd.DataFrame:
+
+    # useful when adding emergency fixed offsets
+    if force_trim:
+        independent_full_df: pd.DataFrame = process_raw_data(
+            trim(independent_full_df), **offsets_config
+        )
+
+    derivate_full_df: pd.DataFrame = process_raw_data(
+        derivate_trimmed_df, **offsets_config
+    )
+    full: pd.DataFrame = pd.concat(
+        [independent_full_df, derivate_full_df], axis=1
+    )
+    return full
+
+
+def collect_series(
+    config_dict: dict[str, Any], find: str = "series"
+) -> set[str]:
+    series: set[str] = set()
+    if find in config_dict:
+        config_series: dict[str, str] = config_dict[find]
+        series = series | set(config_series.keys())
+    for k, v in config_dict.items():
+        if k != find and isinstance(v, dict):
+            series = series | collect_series(v, find)
+    return series