tickerinside 0.1.0__py3-none-any.whl

tickerinside/__init__.py

@@ -0,0 +1,83 @@
+"""tickerinside: Python client for the free TickerInside API.
+
+The full holdings of US ETFs, which funds hold a stock, the overlap between
+funds and the look-through of a portfolio, read from the static JSON files
+tickerinside.com serves. No key, no account, standard library only; pandas is
+used when it is installed.
+
+    import tickerinside as ti
+
+    ti.holdings("VOO").df            # every position, heaviest first
+    ti.holders("NVDA").df            # the funds that hold NVDA
+    ti.compare("VOO", "QQQ").data    # overlap and correlation
+    ti.look_through({"VOO": 6000, "QQQ": 3000, "SCHD": 1000}).df
+
+Every call returns a Result with .data, .df, .as_of, .holdings_as_of,
+.source, .cite, .url and .page_url. The module functions share one Client;
+create your own Client for another base URL, timeout or cache.
+
+The data is published by TickerInside under CC BY 4.0: credit TickerInside
+and link to https://tickerinside.com when you use it.
+"""
+from __future__ import annotations
+
+import functools
+import inspect
+from typing import Optional, Sequence
+
+from ._http import USER_AGENT
+from ._version import __version__
+from .client import Client, Positions
+from .errors import APIError, NotCovered, TickerInsideError
+from .result import OverlapMatrix, Result
+
+__all__ = [
+    "__version__", "USER_AGENT", "Client", "Result", "OverlapMatrix",
+    "TickerInsideError", "APIError", "NotCovered",
+    "funds", "fund", "holdings", "holders", "security", "compare", "overlap_matrix",
+    "look_through", "census", "configure", "clear_cache",
+]
+
+_default: Optional[Client] = None
+
+
+def _client() -> Client:
+    global _default
+    if _default is None:
+        _default = Client()
+    return _default
+
+
+def configure(base_url: Optional[str] = None, timeout: float = 20.0, cache_ttl: Optional[float] = 3600.0) -> Client:
+    """Replace the client the module functions use, and return it."""
+    global _default
+    _default = Client(base_url=base_url, timeout=timeout, cache_ttl=cache_ttl)
+    return _default
+
+
+def clear_cache() -> None:
+    """Forget every file the module functions have read."""
+    _client().clear_cache()
+
+
+def _bind(name):
+    method = getattr(Client, name)
+
+    @functools.wraps(method)
+    def call(*args, **kwargs):
+        return getattr(_client(), name)(*args, **kwargs)
+    sig = inspect.signature(method)
+    call.__signature__ = sig.replace(parameters=list(sig.parameters.values())[1:])  # type: ignore[attr-defined]
+    del call.__wrapped__
+    return call
+
+
+funds = _bind("funds")
+fund = _bind("fund")
+holdings = _bind("holdings")
+holders = _bind("holders")
+security = _bind("security")
+compare = _bind("compare")
+overlap_matrix = _bind("overlap_matrix")
+look_through = _bind("look_through")
+census = _bind("census")

tickerinside/_http.py

@@ -0,0 +1,177 @@
+"""Reading the API: one GET per file, standard library only.
+
+The API is a set of static files on a CDN, with no key and no quota, so the
+only things worth doing here are naming the program honestly in the
+User-Agent, not fetching the same file twice in a session, retrying once when
+the CDN has a bad moment, and telling a ticker we do not cover apart from a
+network that is down. Nothing else is sent anywhere.
+"""
+from __future__ import annotations
+
+import gzip
+import http.client
+import json
+import socket
+import threading
+import time
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict, List, Optional, Sequence, Tuple
+
+from ._version import __version__
+from .errors import APIError, NotCovered
+
+USER_AGENT = f"tickerinside-python/{__version__} (+https://tickerinside.com/api/)"
+DEFAULT_TIMEOUT = 20.0
+
+
+class Transport:
+    """GETs files under a base URL and keeps each one in memory, by URL,
+    for ``cache_ttl`` seconds (0 turns the cache off, None keeps files for
+    the life of the object)."""
+
+    def __init__(self, base_url: str, timeout: float = DEFAULT_TIMEOUT,
+                 cache_ttl: Optional[float] = 3600.0, retry_wait: float = 1.0, workers: int = 8):
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.cache_ttl = cache_ttl
+        self.retry_wait = retry_wait
+        self.workers = max(1, int(workers))
+        self._cache: Dict[str, Tuple[float, str]] = {}
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------ cache
+
+    def clear_cache(self) -> None:
+        with self._lock:
+            self._cache.clear()
+
+    def _cached(self, url: str) -> Optional[str]:
+        if self.cache_ttl == 0:
+            return None
+        with self._lock:
+            hit = self._cache.get(url)
+        if hit is None:
+            return None
+        if self.cache_ttl is not None and time.monotonic() - hit[0] > self.cache_ttl:
+            with self._lock:
+                self._cache.pop(url, None)
+            return None
+        return hit[1]
+
+    def _store(self, url: str, text: str) -> None:
+        if self.cache_ttl == 0:
+            return
+        with self._lock:
+            self._cache[url] = (time.monotonic(), text)
+
+    # ------------------------------------------------------------ reading
+
+    def url(self, path: str) -> str:
+        return self.base_url + path
+
+    def text(self, url: str, ticker: Optional[str] = None, accept: str = "application/json") -> str:
+        """The body of ``url`` as text. A 404 raises NotCovered when the
+        file belongs to a ticker (``ticker`` is given) and the body is the
+        API's own not-covered answer, APIError otherwise."""
+        hit = self._cached(url)
+        if hit is not None:
+            return hit
+        body = self._get(url, ticker, accept)
+        self._store(url, body)
+        return body
+
+    def json(self, path: str, ticker: Optional[str] = None) -> Tuple[Any, str]:
+        """(parsed file, url) for an API path such as "/fund/voo.json"."""
+        url = self.url(path)
+        body = self.text(url, ticker)
+        try:
+            return json.loads(body), url
+        except ValueError:
+            raise APIError(f"{url} did not answer with JSON.", url) from None
+
+    def many(self, items: Sequence[Tuple[str, Optional[str]]]) -> List[Any]:
+        """Several (path, ticker) reads at once. Each result is (file, url) or
+        the NotCovered raised for it; any other error is raised."""
+        def one(item):
+            try:
+                return self.json(item[0], item[1])
+            except NotCovered as e:
+                return e
+        if len(items) <= 1:
+            return [one(i) for i in items]
+        with ThreadPoolExecutor(max_workers=min(self.workers, len(items))) as pool:
+            return list(pool.map(one, items))
+
+    def _get(self, url: str, ticker: Optional[str], accept: str) -> str:
+        req = urllib.request.Request(url, headers={
+            "User-Agent": USER_AGENT,
+            "Accept": accept,
+            "Accept-Encoding": "gzip",
+        })
+        for attempt in (0, 1):
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as resp:
+                    return _decode(resp.read(), resp.headers.get("Content-Encoding"), url)
+            except urllib.error.HTTPError as e:
+                status = e.code
+                raw = _read_error_body(e)
+                if 500 <= status < 600 and attempt == 0:
+                    if self.retry_wait:
+                        time.sleep(self.retry_wait)
+                    continue
+                if status == 404:
+                    if ticker is not None:
+                        err = _not_covered(ticker, url, raw, e.headers.get("Content-Encoding") if e.headers else None)
+                        if err is not None:
+                            raise err
+                    raise APIError(f"{url} answered HTTP 404, so there is no file at that address; "
+                                   f"check the base URL.", url, status) from None
+                raise APIError(f"{url} answered HTTP {status}.", url, status) from None
+            except (urllib.error.URLError, socket.timeout, TimeoutError, ConnectionError,
+                    http.client.HTTPException, OSError) as e:
+                reason = getattr(e, "reason", None) or e
+                raise APIError(f"Could not read {url}: {reason}", url) from None
+        raise APIError(f"{url} could not be read.", url)  # pragma: no cover
+
+
+def _read_error_body(e: urllib.error.HTTPError) -> bytes:
+    try:
+        return e.read() or b""
+    except Exception:
+        return b""
+    finally:
+        try:
+            e.close()
+        except Exception:
+            pass
+
+
+def _decode(raw: bytes, encoding: Optional[str], url: str) -> str:
+    if (encoding or "").lower() == "gzip" or raw[:2] == b"\x1f\x8b":
+        try:
+            raw = gzip.decompress(raw)
+        except (OSError, EOFError):
+            raise APIError(f"{url} sent a body that could not be decompressed.", url) from None
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError:
+        raise APIError(f"{url} sent a body that is not UTF-8.", url) from None
+
+
+def _not_covered(ticker: str, url: str, raw: bytes, encoding: Optional[str]) -> Optional[NotCovered]:
+    """The NotCovered for a 404, or None when the 404 is not the API's own
+    answer about a ticker (a wrong base URL, a proxy's error page), which is
+    then reported as an APIError rather than as a fact about the ticker."""
+    try:
+        body = json.loads(_decode(raw, encoding, url))
+    except (APIError, ValueError):
+        return None
+    if not isinstance(body, dict) or body.get("error") != "not_covered":
+        return None
+    message = body.get("message") if isinstance(body.get("message"), str) else None
+    if not message:
+        message = (f"{ticker} is not covered yet. The funds covered today are listed at "
+                   f"https://tickerinside.com/api/v1/index.json.")
+    return NotCovered({ticker: message}, {ticker: url}, {ticker: body})

tickerinside/_math.py

@@ -0,0 +1,276 @@
+"""The arithmetic, ported from the MCP server's math.js and tools.js.
+
+Every figure this package computes (overlap, correlation, look-through) must
+equal the figure the TickerInside MCP server and website give for the same
+files, to the last digit. Three details of JavaScript are reproduced for that:
+
+- Rounding is Math.round, which sends a half up towards positive infinity,
+  where Python's round() sends it to the even neighbour.
+- An object's keys are walked in JavaScript's order: keys that are array
+  indices ("1", "7203") first, in ascending numeric order, then the others in
+  the order the file lists them. Sums of floating point numbers depend on
+  their order, and so does the order of ties after a stable sort.
+- A null inside a weekly series counts as 0 in arithmetic, as it does there.
+
+The tests compare every function here with values node computed on math.js
+and tools.js for the same files.
+"""
+from __future__ import annotations
+
+import math
+import re
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple
+
+_INDEX = re.compile(r"(?:0|[1-9][0-9]*)", re.ASCII)
+
+
+def js_round(x: Optional[float], d: int) -> Optional[float]:
+    """math.js r(x, d): Math.round(x * 10**d) / 10**d, null for null or NaN."""
+    if x is None:
+        return None
+    x = float(x)
+    if math.isnan(x):
+        return None
+    if math.isinf(x):
+        return x
+    m = math.pow(10, d)
+    y = x * m
+    if math.isinf(y):
+        return x
+    f = math.floor(y)
+    n = f + 1 if y - f >= 0.5 else f
+    return n / m
+
+
+def js_keys(obj: Mapping[str, Any]) -> List[str]:
+    """The keys of a parsed JSON object in the order JavaScript walks them."""
+    idx, rest = [], []
+    for k in obj:
+        if _INDEX.fullmatch(k) and int(k) < 4294967295:
+            idx.append(k)
+        else:
+            rest.append(k)
+    idx.sort(key=int)
+    return idx + rest
+
+
+def overlap(ha: Mapping[str, float], hb: Mapping[str, float],
+            skip: Optional[Set[str]] = None) -> Dict[str, Any]:
+    """math.js overlap(): by weight, and the share of each fund held in common.
+
+    by_weight is the sum of the smaller weight each shared company has in the
+    two funds, and it is symmetric; share_of_a and share_of_b are not.
+    skip lists tickers both funds hold for two different companies, which are
+    left out of every figure and listed apart.
+    """
+    by_weight = sa = sb = wa = wb = 0.0
+    shared, apart = [], []
+    ka = js_keys(ha)
+    for k in ka:
+        wa += ha[k]
+    for k in js_keys(hb):
+        wb += hb[k]
+    for k in ka:
+        if k not in hb:
+            continue
+        if skip and k in skip:
+            apart.append(k)
+            continue
+        m = min(ha[k], hb[k])
+        by_weight += m
+        sa += ha[k]
+        sb += hb[k]
+        shared.append({"ticker": k, "in_a": js_round(ha[k], 2), "in_b": js_round(hb[k], 2),
+                       "counted": js_round(m, 2)})
+    shared.sort(key=lambda x: -x["counted"])
+    out = {
+        "by_weight": js_round(by_weight, 1),
+        "share_of_a": js_round(sa / wa * 100, 1) if wa else None,
+        "share_of_b": js_round(sb / wb * 100, 1) if wb else None,
+        "companies_in_both": len(shared),
+        "positions_a": len(ha),
+        "positions_b": len(hb),
+        "largest_shared": shared[:15],
+    }
+    if apart:
+        out["same_ticker_two_companies"] = sorted(apart)
+    return out
+
+
+def _num(x: Any) -> float:
+    return 0.0 if x is None else float(x)
+
+
+def covariance(series: Sequence[Sequence[Optional[float]]]) -> Optional[Dict[str, Any]]:
+    """math.js covariance(): annualised covariance in percent squared, on the
+    window every series shares (their common tail). None under 52 weeks."""
+    n = len(series)
+    if not n:
+        return None
+    ln = min(len(s) for s in series)
+    if not ln or ln < 52:
+        return None
+    rs = [[_num(x) for x in s[len(s) - ln:]] for s in series]
+    mean = []
+    for s in rs:
+        acc = 0.0
+        for x in s:
+            acc += x
+        mean.append(acc / ln)
+    S = []
+    for i in range(n):
+        row = []
+        for j in range(n):
+            acc = 0.0
+            for k in range(ln):
+                acc += (rs[i][k] - mean[i]) * (rs[j][k] - mean[j])
+            row.append(acc / (ln - 1) * 52 * 10000)
+        S.append(row)
+    return {"S": S, "weeks": ln}
+
+
+def correlation(a: Optional[Sequence[Optional[float]]],
+                b: Optional[Sequence[Optional[float]]]) -> Tuple[Optional[float], Optional[float], Optional[int]]:
+    """(correlation, covariance, weeks) of two weekly series, as compare_funds
+    rounds them, or (None, None, None) when either is missing or short."""
+    cov = covariance([a or [], b or []])
+    if not cov:
+        return None, None, None
+    S = cov["S"]
+    den = math.sqrt(S[0][0] * S[1][1])
+    corr = S[0][1] / den if den else float("nan")
+    return js_round(corr, 2), js_round(S[0][1], 1), cov["weeks"]
+
+
+def look_through(rows: Sequence[Dict[str, Any]], by_ticker: Mapping[str, Mapping[str, Any]]) -> Dict[str, Any]:
+    """math.js lookThrough(): the companies under a list of funds, in weight
+    and, when amounts are given, in money, with the funds each comes through.
+
+    rows are {"ticker", "amount"} with amount None when not given; by_ticker
+    maps a fund ticker to its fund file (only "holdings" is read).
+    """
+    priced = any((p.get("amount") or 0) > 0 for p in rows)
+    total = 0
+    for p in rows:
+        total = total + (p.get("amount") or 0)
+    if priced:
+        w = [(p.get("amount") or 0) / (total or 1) for p in rows]
+    else:
+        w = [1 / len(rows) for _ in rows]
+    look: Dict[str, Dict[str, Any]] = {}
+    for i, p in enumerate(rows):
+        h = (by_ticker.get(p["ticker"]) or {}).get("holdings") or {}
+        for k in js_keys(h):
+            e = look.get(k)
+            if e is None:
+                e = look[k] = {"ticker": k, "weight_pct": 0.0, "through": []}
+            part = h[k] / 100 * w[i]
+            e["weight_pct"] += part * 100
+            e["through"].append({"fund": p["ticker"], "weight_pct": js_round(part * 100, 4)})
+    names = []
+    for k in js_keys(look):
+        x = look[k]
+        names.append({
+            "ticker": x["ticker"],
+            "weight_pct": js_round(x["weight_pct"], 3),
+            "through": sorted(x["through"], key=lambda t: -t["weight_pct"]),
+            "value": js_round(x["weight_pct"] / 100 * total, 2) if priced else None,
+        })
+    names.sort(key=lambda x: -x["weight_pct"])
+    return {"priced": priced, "total": total, "weights": w, "companies": names}
+
+
+# ------------------------------------------------ one ticker, two companies
+#
+# From tools.js. A fund from SEC filings lists under "abroad" the tickers it
+# holds for a foreign company's local code where the US list gives the code
+# to another company: Allianz is ALV in Frankfurt, ALV in New York is Autoliv.
+# Two funds holding such a ticker can hold two companies, so the ticker is not
+# counted as held in common, and a look-through lists it twice.
+
+US_ONLY = frozenset(["Large Blend", "Large Growth", "Large Value", "Mid-Cap Blend", "Mid-Cap Growth",
+                     "Mid-Cap Value", "Small Blend", "Small Growth", "Small Value"])
+
+TWO_COMPANIES = ("The same ticker stands for a foreign company's local code in one fund and for the US "
+                 "company of that ticker in another, so it is not counted as held in common.")
+
+SPLIT_NOTE = ("Each of these tickers is listed twice: once for the US company of that ticker, and once, "
+              "marked listed_abroad, for the foreign company whose local code it is in the other funds. "
+              "A line marked listed_abroad is never the US company of its ticker.")
+
+ABROAD_KEY = "\u0000abroad"
+
+
+def is_sec(f: Optional[Mapping[str, Any]]) -> bool:
+    return bool(f) and f.get("source_layer") == "sec_nport"
+
+
+def needs_categories(files: Mapping[str, Mapping[str, Any]]) -> bool:
+    """Whether the categories from funds.json are needed: only when a fund
+    from SEC filings lists a foreign code."""
+    return any(is_sec(f) and (f.get("abroad") or []) for f in files.values())
+
+
+def reading_of(f: Mapping[str, Any], cat: Optional[str], k: str) -> Optional[str]:
+    if is_sec(f):
+        return "abroad" if k in (f.get("abroad") or []) else "us"
+    return "us" if cat in US_ONLY else None
+
+
+def two_companies(a: str, b: str, files: Mapping[str, Mapping[str, Any]],
+                  cats: Mapping[str, Optional[str]]) -> Set[str]:
+    fa, fb = files[a], files[b]
+    codes = (list(fa.get("abroad") or []) if is_sec(fa) else []) + (list(fb.get("abroad") or []) if is_sec(fb) else [])
+    out: Set[str] = set()
+    ha, hb = fa.get("holdings") or {}, fb.get("holdings") or {}
+    for k in codes:
+        if k not in ha or k not in hb:
+            continue
+        ra, rb = reading_of(fa, cats.get(a), k), reading_of(fb, cats.get(b), k)
+        if ra and rb and ra != rb:
+            out.add(k)
+    return out
+
+
+def split_abroad(tickers: Sequence[str], files: Mapping[str, Mapping[str, Any]],
+                 cats: Mapping[str, Optional[str]]) -> Tuple[Dict[str, Mapping[str, Any]], List[str]]:
+    codes: List[str] = []
+    for t in tickers:
+        if is_sec(files[t]):
+            for k in files[t].get("abroad") or []:
+                if k not in codes:
+                    codes.append(k)
+    if not codes:
+        return dict(files), []
+    split = [k for k in codes
+             if any(k in (files[t].get("holdings") or {}) and reading_of(files[t], cats.get(t), k) == "us"
+                    for t in tickers)]
+    out = dict(files)
+    for t in tickers:
+        h = files[t].get("holdings") or {}
+        moved = [k for k in codes if k in h and reading_of(files[t], cats.get(t), k) != "us"]
+        if not moved:
+            continue
+        copy = dict(h)
+        for k in moved:
+            copy[k + ABROAD_KEY] = copy[k]
+            del copy[k]
+        f = dict(files[t])
+        f["holdings"] = copy
+        out[t] = f
+    return out, split
+
+
+def unkey(c: Dict[str, Any]) -> Dict[str, Any]:
+    if c["ticker"].endswith(ABROAD_KEY):
+        c = dict(c)
+        c["ticker"] = c["ticker"][:-len(ABROAD_KEY)]
+        c["listed_abroad"] = True
+    return c
+
+
+def top_sum(values: Iterable[float]) -> float:
+    acc = 0.0
+    for v in values:
+        acc += v
+    return acc

tickerinside/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"