financetoolkit 2.3.0__tar.gz → 2.3.1__tar.gz

{financetoolkit-2.3.0 → financetoolkit-2.3.1}/PKG-INFO

@@ -1,7 +1,11 @@
 Metadata-Version: 2.5
 Name: financetoolkit
-Version: 2.3.0
+Version: 2.3.1
 Summary: Transparent and Efficient Financial Analysis
+Project-URL: Homepage, https://www.jeroenbouma.com/projects/financetoolkit
+Project-URL: Documentation, https://www.jeroenbouma.com/projects/financetoolkit/docs
+Project-URL: Repository, https://github.com/JerBouma/FinanceToolkit
+Project-URL: Issues, https://github.com/JerBouma/FinanceToolkit/issues
 Author: Jeroen Bouma
 License: MIT
 License-File: LICENSE.txt
@@ -36,7 +40,6 @@ Requires-Dist: mcp[cli]<2,>=1.27.0; extra == 'mcp'
 Requires-Dist: python-dotenv>=1.0; extra == 'mcp'
 Requires-Dist: rich>=13.0; extra == 'mcp'
 Requires-Dist: statsmodels>=0.14; extra == 'mcp'
-Requires-Dist: tabulate>=0.9.0; extra == 'mcp'
 Description-Content-Type: text/markdown
 
 <!-- mcp-name: io.github.JerBouma/financetoolkit -->

{financetoolkit-2.3.0 → financetoolkit-2.3.1}/financetoolkit/cache/cache_controller.py

@@ -29,6 +29,13 @@ DATABASE_FILE_NAME = "financetoolkit_cache.db"
 # Compacted into merged intervals past this many rows, to bound table growth.
 COVERAGE_COMPACTION_THRESHOLD = 16
 
+# Point-in-time entries are kept at least this long after they were written, also past their
+# time-to-live, since an expired entry still answers when its source cannot be reached.
+STALE_ENTRY_RETENTION_SECONDS = 90 * policy_model.DAY
+
+# How often the entries nobody asks for any more are removed.
+EVICTION_INTERVAL_SECONDS = policy_model.DAY
+
 _CACHE_REGISTRY: dict[str, "Cache"] = {}
 _REGISTRY_LOCK = Lock()
 
@@ -103,7 +110,7 @@ def get_default_cache_location() -> Path:
     override = os.environ.get("FINANCE_TOOLKIT_CACHE_DB")
 
     if override:
-        return Path(override).expanduser()
+        return Path(override).expanduser().absolute()
 
     if platform.system() == "Windows":
         base = Path(
@@ -113,7 +120,7 @@ def get_default_cache_location() -> Path:
         xdg_config_home = os.environ.get("XDG_CONFIG_HOME")
         base = Path(xdg_config_home) if xdg_config_home else Path.home() / ".config"
 
-    return base / "financetoolkit" / DATABASE_FILE_NAME
+    return (base / "financetoolkit" / DATABASE_FILE_NAME).absolute()
 
 
 def resolve_cache_location(location: str | Path | None) -> Path:
@@ -133,7 +140,9 @@ def resolve_cache_location(location: str | Path | None) -> Path:
     if location is None:
         return get_default_cache_location()
 
-    path = Path(location).expanduser()
+    # Absolute, so a relative folder keeps pointing at the same database after a change of
+    # working directory and caches opened from different directories are told apart.
+    path = Path(location).expanduser().absolute()
 
     if path.suffix in (".db", ".sqlite", ".sqlite3"):
         return path
@@ -283,6 +292,38 @@ class Cache:
                 )
 
         self._warn_on_schema_mismatch()
+        self._remove_stale_entries()
+
+    def _remove_stale_entries(self) -> None:
+        """
+        Remove the point-in-time entries of every source that have not been refreshed for
+        longer than their time-to-live and the retention for unreachable sources, at most
+        once a day. The MCP server's responses are left to the server, which evicts them
+        on its own time-to-live.
+        """
+        try:
+            # A database of another schema version is left exactly as it is.
+            if (
+                self._backend is None
+                or self._backend.get_schema_version() != SCHEMA_VERSION
+            ):
+                return
+
+            evicted = self._backend.remove_stale_blobs(
+                lambda source, dataset: max(
+                    policy_model.get_policy(source, dataset).ttl_seconds,
+                    STALE_ENTRY_RETENTION_SECONDS,
+                ),
+                EVICTION_INTERVAL_SECONDS,
+                excluded_sources=frozenset({policy_model.MCP}),
+            )
+        except Exception as error:  # pylint: disable=broad-except
+            # Eviction is housekeeping, so a failure here is not surfaced.
+            logger.debug("Could not evict stale cache entries: %s", error)
+            return
+
+        if evicted:
+            logger.debug("Removed %s stale cache entries.", evicted)
 
     def _warn_on_schema_mismatch(self) -> None:
         """
@@ -388,9 +429,9 @@ class Cache:
             start (str | date | None): Inclusive start of the requested range.
             end (str | date | None): Inclusive end of the requested range.
             parameters (dict[str, Any] | None): Parameters that change the data.
-            date_axis (int): 0 when the index holds the dates, 1 when the columns do.
+            date_axis (int): 0 when the index holds the dates, 1 when the columns do. Defaults to 0.
             reference_date (date | None): The date the revision window is measured
-                back from when the requested range has no end. Defaults to today;
+                back from when the requested range has no end or ends after it. Defaults to today;
                 mainly useful for testing.
 
         Returns:
@@ -405,6 +446,10 @@ class Cache:
             coverage_model.normalize_date(start) if start else date(1900, 1, 1)
         )
         requested_end = coverage_model.normalize_date(end) if end else reference
+        # Nothing after today has been published yet, so a range that runs into the future is
+        # covered up to today and its revision window ends today, as an open range's does.
+        if requested_start <= reference < requested_end:
+            requested_end = reference
 
         if not self.serves(source):
             plan.missing = {
@@ -507,7 +552,7 @@ class Cache:
             start (str | date | None): Inclusive start of the range that was requested.
             end (str | date | None): Inclusive end of the range that was requested.
             parameters (dict[str, Any] | None): Parameters that change the data.
-            date_axis (int): 0 when the index holds the dates, 1 when the columns do.
+            date_axis (int): 0 when the index holds the dates, 1 when the columns do. Defaults to 0.
         """
         if not self.serves(source):
             return
@@ -520,8 +565,10 @@ class Cache:
             if start
             else (bounds[0] if bounds else None)
         )
+        # A request that runs into the future holds what was published up to today; claiming the
+        # future as well would never ask for those days once they have passed.
         coverage_end = (
-            coverage_model.normalize_date(end)
+            min(coverage_model.normalize_date(end), date.today())  # noqa: DTZ011
             if end
             else (bounds[1] if bounds else None)
         )
@@ -534,7 +581,8 @@ class Cache:
                 coverage_start = min(coverage_start, bounds[0])
                 coverage_end = max(coverage_end, bounds[1])
 
-            coverage = (coverage_start, coverage_end)
+            if coverage_start <= coverage_end:
+                coverage = (coverage_start, coverage_end)
 
         def merge_payload(existing_payload: bytes | None) -> bytes:
             """
@@ -813,7 +861,7 @@ def get_cache(
         location (str | Path | None): Directory or database file to use. None
             selects the shared default location.
         enabled (bool): When False a disabled cache is returned, which misses on
-            every read and ignores every write.
+            every read and ignores every write. Defaults to True.
         excluded_sources (frozenset[str] | set[str] | None): Sources this cache never
             reads or writes, see Cache. Defaults to None.
 
@@ -829,12 +877,14 @@ def get_cache(
     registry_key = f"{resolved}|{','.join(sorted(excluded))}"
 
     with _REGISTRY_LOCK:
-        if registry_key not in _CACHE_REGISTRY:
-            _CACHE_REGISTRY[registry_key] = Cache(
-                location=location, enabled=True, excluded_sources=excluded
-            )
+        cache = _CACHE_REGISTRY.get(registry_key)
+    if cache is not None:
+        return cache
 
-        return _CACHE_REGISTRY[registry_key]
+    # Opened outside the lock: opening evicts stale entries once a day, which can wait for another process's write lock, and that would hold up every other caller meanwhile. When two threads open the same location at once, the first instance registered is kept.  # noqa: E501
+    cache = Cache(location=location, enabled=True, excluded_sources=excluded)
+    with _REGISTRY_LOCK:
+        return _CACHE_REGISTRY.setdefault(registry_key, cache)
 
 
 def reset_cache_registry() -> None:

{financetoolkit-2.3.0 → financetoolkit-2.3.1}/financetoolkit/cache/policy_model.py

@@ -47,9 +47,9 @@ DE_NEDERLANDSCHE_BANK = "DeNederlandscheBank"
 EUROPEAN_SYSTEMIC_RISK_BOARD = "ESRB"
 DAMODARAN = "Damodaran"
 
-# The market risk premium is published per country, so it is one cache entry. Its name
-# changed when the premiums became decimals (v2.2.2), so an entry cached in percent by an
-# earlier version is never read back as decimals.
+# The market risk premium is published per country, so it is one cache entry. The name
+# marks the premiums as decimals, so an entry holding percentages under another name is
+# never read back as decimals.
 MARKET_RISK_PREMIUM_ENTITY = "global_decimals"
 
 

{financetoolkit-2.3.0 → financetoolkit-2.3.1}/financetoolkit/cache/sqlite_model.py

@@ -20,6 +20,10 @@ SCHEMA_VERSION = 1
 # How long a writer waits for another process to release the write lock.
 BUSY_TIMEOUT_SECONDS = 30
 
+# A database up to this size is compacted after an eviction freed most of it; a larger one
+# would hold every other process off the cache for too long while it is rewritten.
+VACUUM_MAXIMUM_BYTES = 256 * 1024 * 1024
+
 # Three tables; source and dataset stay plain columns so clearing can be scoped.
 SCHEMA_STATEMENTS = (
     """
@@ -130,10 +134,15 @@ class SQLiteBackend:
                 the block, so that every statement inside it commits or rolls back
                 as one unit. False opens a plain read connection.
 
+        Only writers take the instance lock, so the threads of one process queue for the
+        write lock here instead of each waiting out SQLite's busy timeout. Readers never
+        take it: under write-ahead logging they read alongside a writer, also while that
+        writer waits for another process to release the database.
+
         Yields:
-            sqlite3.Connection: An open connection guarded by the instance lock.
+            sqlite3.Connection: An open connection, guarded by the instance lock when writing.
         """
-        with self._lock:
+        with self._lock if write else contextlib.nullcontext():
             connection = sqlite3.connect(
                 self._database_location,
                 timeout=BUSY_TIMEOUT_SECONDS,
@@ -277,8 +286,8 @@ class SQLiteBackend:
 
         Reading the coverage and the payload over three separate connections lets
         a concurrent writer commit in between, so the planner could see coverage
-        that does not describe the payload it read. A single connection gives one
-        consistent view of both.
+        that does not describe the payload it read. One read transaction on a single
+        connection gives one consistent view of both.
 
         Args:
             key (str): The dataset key.
@@ -292,15 +301,20 @@ class SQLiteBackend:
                 stored payload (None when nothing is cached).
         """
         with self._connect() as connection:
-            ever_rows = connection.execute(
-                "SELECT start_date, end_date, fetched_at FROM coverage "
-                "WHERE key = ? AND entity = ?",
-                (key, entity),
-            ).fetchall()
-            payload_row = connection.execute(
-                "SELECT payload FROM series WHERE key = ? AND entity = ?",
-                (key, entity),
-            ).fetchone()
+            # One read transaction, so both SELECTs see the same snapshot; in autocommit mode each would take its own.  # noqa: E501
+            connection.execute("BEGIN")
+            try:
+                ever_rows = connection.execute(
+                    "SELECT start_date, end_date, fetched_at FROM coverage "
+                    "WHERE key = ? AND entity = ?",
+                    (key, entity),
+                ).fetchall()
+                payload_row = connection.execute(
+                    "SELECT payload FROM series WHERE key = ? AND entity = ?",
+                    (key, entity),
+                ).fetchone()
+            finally:
+                connection.execute("COMMIT")
 
         ever_covered = [
             (normalize_date(start), normalize_date(end)) for start, end, _ in ever_rows
@@ -671,6 +685,75 @@ class SQLiteBackend:
 
         return deleted
 
+    def remove_stale_blobs(
+        self,
+        maximum_age: Callable[[str, str], float],
+        interval: float,
+        excluded_sources: frozenset[str] = frozenset(),
+    ) -> int:
+        """
+        Delete the point-in-time entries of every source that have not been refreshed for
+        longer than their dataset allows, at most once per interval.
+
+        An entry that is read again is refreshed by its next fetch, so only entries nobody
+        asks for any more pile up. The time of the last run is kept in the database, so
+        every process sharing it runs this once per interval in total. When the deletes
+        free more than half of a small database, it is compacted as well.
+
+        Args:
+            maximum_age (Callable[[str, str], float]): The age in seconds after which an
+                entry of a source and dataset is removed.
+            interval (float): The minimum number of seconds between two runs.
+            excluded_sources (frozenset[str]): Sources that evict their own entries.
+
+        Returns:
+            int: The number of entries deleted, 0 when the interval has not passed.
+        """
+        now = time.time()
+        deleted = 0
+
+        with self._connect(write=True) as connection:
+            row = connection.execute(
+                "SELECT value FROM meta WHERE name = ?", ("last_blob_eviction",)
+            ).fetchone()
+
+            if row is not None and now - float(row[0]) < interval:
+                return 0
+
+            connection.execute(
+                "INSERT OR REPLACE INTO meta (name, value) VALUES (?, ?)",
+                ("last_blob_eviction", str(now)),
+            )
+
+            scopes = connection.execute(
+                "SELECT DISTINCT source, dataset FROM blobs"
+            ).fetchall()
+
+            for source, dataset in scopes:
+                if source in excluded_sources:
+                    continue
+
+                deleted += connection.execute(
+                    "DELETE FROM blobs WHERE source = ? AND dataset = ? AND updated_at < ?",
+                    (source, dataset, now - maximum_age(source, dataset)),
+                ).rowcount
+
+            page_count = connection.execute("PRAGMA page_count").fetchone()[0]
+            free_pages = connection.execute("PRAGMA freelist_count").fetchone()[0]
+
+        database_path = Path(self._database_location)
+        if (
+            deleted
+            and free_pages * 2 > page_count
+            and database_path.exists()
+            and database_path.stat().st_size <= VACUUM_MAXIMUM_BYTES
+        ):
+            # VACUUM cannot run inside a transaction, so it takes a plain connection.
+            with self._lock, self._connect() as connection:
+                connection.execute("VACUUM")
+
+        return deleted
+
     def clear_all(self) -> int:
         """
         Remove every cached entry regardless of age.

{financetoolkit-2.3.0 → financetoolkit-2.3.1}/financetoolkit/currencies_model.py

@@ -190,6 +190,12 @@ CURRENCY_CODES: frozenset[str] = frozenset(
 )  # codespell:ignore zar
 
 
+# How old the last published rate may be on a period end before the period has no rate.
+# Sparse or exotic pairs can go a few weeks without a quote, so only a rate older than a
+# month marks a series that ended before the period did.
+STALE_RATE_DAYS = pd.Timedelta(days=31)
+
+
 def is_currency_pair(ticker: str) -> bool:
     """
     Returns whether a ticker is a forex pair: six letters, optionally followed by "=X" as
@@ -474,11 +480,17 @@ def get_statement_exchange_rates(
             ]
         else:
             dates = end_dates if rate == "closing" else start_dates
-            # The last published rate on or before each date.
+            # The last published rate on or before each date, unless that rate is more than
+            # a month old: a period ending after the rates do has no rate yet.
             positions = currency_rates.index.searchsorted(dates, side="right") - 1
             values = [
-                currency_rates.iloc[position] if position >= 0 else float("nan")
-                for position in positions
+                (
+                    currency_rates.iloc[position]
+                    if position >= 0
+                    and date - currency_rates.index[position] <= STALE_RATE_DAYS
+                    else float("nan")
+                )
+                for position, date in zip(positions, dates, strict=True)
             ]
 
         rates[ticker] = pd.Series(values, index=end_dates.index, dtype="float64")

{financetoolkit-2.3.0 → financetoolkit-2.3.1}/financetoolkit/discovery/discovery_controller.py

@@ -214,14 +214,14 @@ class Discovery:
         self,
         market_cap_higher: int | None = None,
         market_cap_lower: int | None = None,
-        price_higher: int | None = None,
-        price_lower: int | None = None,
-        beta_higher: int | None = None,
-        beta_lower: int | None = None,
+        price_higher: float | None = None,
+        price_lower: float | None = None,
+        beta_higher: float | None = None,
+        beta_lower: float | None = None,
         volume_higher: int | None = None,
         volume_lower: int | None = None,
-        dividend_higher: int | None = None,
-        dividend_lower: int | None = None,
+        dividend_higher: float | None = None,
+        dividend_lower: float | None = None,
         sector: str | None = None,
         industry: str | None = None,
         country: str | None = None,
@@ -250,22 +250,32 @@ class Discovery:
 
         Args:
             market_cap_higher (int): The minimum market capitalization of the stock in US dollars
-                (not in millions), e.g. 1_000_000_000 for $1 billion.
+                (not in millions), e.g. 1_000_000_000 for $1 billion. Defaults to None, which applies no filter.
             market_cap_lower (int): The maximum market capitalization of the stock in US dollars.
-            price_higher (int): The minimum price of the stock.
-            price_lower (int): The maximum price of the stock.
-            beta_higher (int): The minimum beta of the stock.
-            beta_lower (int): The maximum beta of the stock.
-            volume_higher (int): The minimum volume of the stock, in shares traded.
-            volume_lower (int): The maximum volume of the stock.
-            dividend_higher (int): The minimum dividend of the stock, as an amount per share
-                over the last annual period rather than as a yield.
-            dividend_lower (int): The maximum dividend of the stock.
-            sector (str | None): The sector to restrict the screen to, e.g. "Energy".
-            industry (str | None): The industry to restrict the screen to, e.g. "Biotechnology".
-            country (str | None): The two-letter country code to restrict the screen to, e.g. "US".
-            exchange (str | None): The exchange code to restrict the screen to, e.g. "NASDAQ".
-            is_etf (bool | None): Whether to restrict the screen to ETFs or to exclude them.
+                The screener returns the largest companies first, so this bound is only sent to it
+                when the currency of the listings is known: with an exchange, or with a country and
+                primary_only. Otherwise it is applied to the companies returned. Defaults to None,
+                which applies no filter.
+            price_higher (float | None): The minimum price of the stock. Defaults to None, which applies no filter.
+            price_lower (float | None): The maximum price of the stock. Defaults to None, which applies no filter.
+            beta_higher (float | None): The minimum beta of the stock. Defaults to None, which applies no filter.
+            beta_lower (float | None): The maximum beta of the stock. Defaults to None, which applies no filter.
+            volume_higher (int): The minimum volume of the stock, in shares traded. Defaults to None, which applies no
+                filter.
+            volume_lower (int): The maximum volume of the stock. Defaults to None, which applies no filter.
+            dividend_higher (float | None): The minimum dividend of the stock, as an amount per share
+                over the last annual period rather than as a yield. Defaults to None, which applies no filter.
+            dividend_lower (float | None): The maximum dividend of the stock. Defaults to None, which applies no filter.
+            sector (str | None): The sector to restrict the screen to, e.g. "Energy". Defaults to None, which applies no
+                filter.
+            industry (str | None): The industry to restrict the screen to, e.g. "Biotechnology". Defaults to None, which
+                applies no filter.
+            country (str | None): The country to restrict the screen to, as a two-letter code ("US")
+                or a name ("United States"). Defaults to None, which applies no filter.
+            exchange (str | None): The exchange code to restrict the screen to, e.g. "NASDAQ". Defaults to None, which
+                applies no filter.
+            is_etf (bool | None): Whether to restrict the screen to ETFs or to exclude them. Defaults to None, which
+                applies no filter.
             limit (int): The maximum number of companies to return. Defaults to 1000.
             primary_only (bool): Whether to keep only the home listing of each company (on an
                 exchange in its own country, the most traded one among several) rather than
@@ -1666,8 +1676,8 @@ class Discovery:
         Also known as: sector performance snapshot, sector performance history, sector trend.
 
         Args:
-            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01".
-            sector (str, optional): The sector to retrieve the history for, e.g. "Energy".
+            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01". Defaults to None.
+            sector (str, optional): The sector to retrieve the history for, e.g. "Energy". Defaults to None.
             start_date (str, optional): The start of the history of one sector. Defaults to a year
                 ago.
             end_date (str, optional): The end of the history of one sector. Defaults to today.
@@ -1735,8 +1745,8 @@ class Discovery:
         Also known as: industry performance snapshot, industry performance history, industry trend.
 
         Args:
-            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01".
-            industry (str, optional): The industry to retrieve the history for, e.g. "Biotechnology".
+            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01". Defaults to None.
+            industry (str, optional): The industry to retrieve the history for, e.g. "Biotechnology". Defaults to None.
             start_date (str, optional): The start of the history of one industry. Defaults to a year
                 ago.
             end_date (str, optional): The end of the history of one industry. Defaults to today.
@@ -1803,8 +1813,8 @@ class Discovery:
         Also known as: sector P/E snapshot, sector P/E history, sector valuation trend.
 
         Args:
-            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01".
-            sector (str, optional): The sector to retrieve the history for, e.g. "Energy".
+            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01". Defaults to None.
+            sector (str, optional): The sector to retrieve the history for, e.g. "Energy". Defaults to None.
             start_date (str, optional): The start of the history of one sector. Defaults to a year
                 ago.
             end_date (str, optional): The end of the history of one sector. Defaults to today.
@@ -1872,8 +1882,8 @@ class Discovery:
         Also known as: industry P/E snapshot, industry P/E history, industry valuation trend.
 
         Args:
-            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01".
-            industry (str, optional): The industry to retrieve the history for, e.g. "Biotechnology".
+            date (str, optional): The date to retrieve a snapshot for, e.g. "2024-02-01". Defaults to None.
+            industry (str, optional): The industry to retrieve the history for, e.g. "Biotechnology". Defaults to None.
             start_date (str, optional): The start of the history of one industry. Defaults to a year
                 ago.
             end_date (str, optional): The end of the history of one industry. Defaults to today.