tirramind 0.1.0__tar.gz

tirramind-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.egg-info/
+.venv/
+.cache/
+dist/
+.pytest_cache/

tirramind-0.1.0/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

tirramind-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.5
+Name: tirramind
+Version: 0.1.0
+Summary: Survivorship-bias-free US listing ledger with delisting causes and a Form 144 -> Form 4 insider ledger, reconciled daily from SEC public filings
+Project-URL: Repository, https://github.com/savabs/tirramind-universe
+Author: savabs
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: delistings,form-144,form-4,insider-trading,quant,sec,security-master,survivorship-bias
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Requires-Python: >=3.10
+Requires-Dist: pandas>=2.0
+Requires-Dist: pyarrow>=14
+Requires-Dist: requests>=2.31
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# tirramind
+
+A survivorship-bias-free ledger of every US listed security, with the
+**cause** of every delisting and the SEC filing that proves it — plus a
+Form 144 → Form 4 insider ledger nobody else publishes.
+
+Reconciled daily from SEC public filings only. Every row links to its
+source. Where the cause cannot be determined, the row says `UNKNOWN`
+rather than guessing.
+
+Code Apache-2.0. Data CC-BY-4.0, derived from SEC public filings.

tirramind-0.1.0/README.md

@@ -0,0 +1,11 @@
+# tirramind
+
+A survivorship-bias-free ledger of every US listed security, with the
+**cause** of every delisting and the SEC filing that proves it — plus a
+Form 144 → Form 4 insider ledger nobody else publishes.
+
+Reconciled daily from SEC public filings only. Every row links to its
+source. Where the cause cannot be determined, the row says `UNKNOWN`
+rather than guessing.
+
+Code Apache-2.0. Data CC-BY-4.0, derived from SEC public filings.

tirramind-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "tirramind"
+version = "0.1.0"
+description = "Survivorship-bias-free US listing ledger with delisting causes and a Form 144 -> Form 4 insider ledger, reconciled daily from SEC public filings"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+authors = [{ name = "savabs" }]
+keywords = ["sec", "delistings", "survivorship-bias", "security-master", "form-144", "form-4", "insider-trading", "quant"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Financial and Insurance Industry",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Topic :: Office/Business :: Financial :: Investment",
+]
+dependencies = ["requests>=2.31", "pandas>=2.0", "pyarrow>=14"]
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "ruff>=0.4"]
+
+[project.urls]
+Repository = "https://github.com/savabs/tirramind-universe"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/tirramind"]
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src/tirramind", "README.md", "LICENSE", "pyproject.toml"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100

tirramind-0.1.0/src/tirramind/__init__.py

@@ -0,0 +1,8 @@
+"""tirramind — survivorship-bias-free US listing ledger from SEC filings."""
+
+__version__ = "0.1.0"
+
+from .client import delistings, events, form144, load  # noqa: E402
+from .store import read_index, snapshot  # noqa: E402
+
+__all__ = ["load", "events", "delistings", "form144", "snapshot", "read_index", "__version__"]

tirramind-0.1.0/src/tirramind/accuracy.py

@@ -0,0 +1,100 @@
+"""C4: the accuracy table for the 144 -> 4 ledger, with honest uncertainty.
+
+Form 144 notices are not independent observations: one insider files
+several, several insiders at one issuer file in the same week, and a whole
+sector sells into the same window. So every rate here carries
+
+* ``n``          nominal count,
+* ``n_clusters`` distinct issuer-weeks,
+* ``n_eff``      design-effect effective sample size, n / (1 + (m-1)·ICC),
+* a cluster-bootstrap CI: resample issuer-weeks with replacement, never rows.
+
+The CI must widen when clusters are merged; the test asserts it.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+HORIZONS = (30, 60, 90)
+
+
+def cluster_key(df: pd.DataFrame) -> pd.Series:
+    wk = pd.to_datetime(df["notice_date"]).dt.to_period("W").astype(str)
+    return df["issuer_cik"].astype(str) + ":" + wk
+
+
+def icc_binary(y: np.ndarray, g: np.ndarray) -> float:
+    """ANOVA-style intraclass correlation for a binary outcome."""
+    df = pd.DataFrame({"y": y, "g": g})
+    grp = df.groupby("g")["y"]
+    k = grp.ngroups
+    n = len(df)
+    if k < 2 or n <= k:
+        return 0.0
+    m = grp.size().values.astype(float)
+    n0 = (n - (m ** 2).sum() / n) / (k - 1)
+    grand = df["y"].mean()
+    msb = (m * (grp.mean().values - grand) ** 2).sum() / (k - 1)
+    msw = ((df["y"] - grp.transform("mean")) ** 2).sum() / (n - k)
+    if msb + (n0 - 1) * msw <= 0:
+        return 0.0
+    return float(max(0.0, (msb - msw) / (msb + (n0 - 1) * msw)))
+
+
+def effective_n(y: np.ndarray, g: np.ndarray) -> tuple[int, int, float]:
+    n = len(y)
+    k = len(set(g))
+    if n == 0:
+        return 0, 0, 0.0
+    m_bar = n / k
+    rho = icc_binary(y, g)
+    deff = 1.0 + (m_bar - 1.0) * rho
+    return n, k, float(n / deff)
+
+
+def cluster_bootstrap(stat_fn, df: pd.DataFrame, g: pd.Series, *, n_boot: int = 1000,
+                      conf: float = 0.9, seed: int = 7) -> tuple[float, float, float]:
+    rng = np.random.default_rng(seed)
+    groups = {k: v.index.values for k, v in df.groupby(g)}
+    keys = list(groups)
+    point = float(stat_fn(df))
+    if len(keys) < 2:
+        return point, float("nan"), float("nan")
+    boots = np.empty(n_boot)
+    for b in range(n_boot):
+        pick = rng.choice(len(keys), size=len(keys), replace=True)
+        idx = np.concatenate([groups[keys[i]] for i in pick])
+        boots[b] = stat_fn(df.loc[idx])
+    a = (1 - conf) / 2
+    return point, float(np.quantile(boots, a)), float(np.quantile(boots, 1 - a))
+
+
+def accuracy_table(links: pd.DataFrame, *, n_boot: int = 1000, seed: int = 7) -> pd.DataFrame:
+    """Rows: (segment, horizon) -> P(any sale within horizon) with CI, plus
+    executed-fraction median, n, n_clusters, n_eff. Segments: all, by
+    relationship, by match_method."""
+    L = links.copy()
+    L = L[L["notice_date"].ne("")]
+    if "window_complete" in L:
+        L = L[L["window_complete"]]
+    L["g"] = cluster_key(L)
+    segments = [("all", L)]
+    for rel, sub in L.groupby("relationship"):
+        if len(sub) >= 20:
+            segments.append((f"relationship={rel}", sub))
+    for mm, sub in L.groupby("match_method"):
+        segments.append((f"match={mm}", sub))
+
+    rows = []
+    for name, sub in segments:
+        for h in HORIZONS:
+            hit = ((sub["days_to_first_sale"] >= 0) & (sub["days_to_first_sale"] <= h)).astype(float)
+            n, k, neff = effective_n(hit.values, sub["g"].values)
+            tmp = sub.assign(hit=hit)
+            p, lo, hi = cluster_bootstrap(lambda d: d["hit"].mean(), tmp, tmp["g"], n_boot=n_boot, seed=seed)
+            rows.append({"segment": name, "horizon_days": h, "p_executed": p, "ci_lo": lo, "ci_hi": hi,
+                         "n": n, "n_clusters": k, "n_eff": round(neff, 1),
+                         "median_executed_fraction": float(sub["executed_fraction"].median())})
+    return pd.DataFrame(rows)

tirramind-0.1.0/src/tirramind/build.py

@@ -0,0 +1,180 @@
+"""B6: regenerate the published parquet outputs from snapshots and ledgers.
+
+Everything here is derived. Delete ``data/*.parquet`` and rerun and you get
+byte-identical files; the snapshots and ledgers are the record.
+"""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+
+import pandas as pd
+
+from . import __version__
+from .accuracy import accuracy_table
+from .diff import annotate, events_over_history, fill_exchange, permanent_removals
+from .link import link
+from .page import render_weekly
+from .ledger import DEFAULT_ROOT as LEDGER_ROOT, read as read_ledger
+from .reconcile import CAUSES, NOT_DELISTINGS, reconcile, summary
+from .store import DEFAULT_ROOT as SNAP_ROOT, load_snapshot, snapshots_for
+
+OUT = os.path.join(os.getcwd(), "data")
+
+
+def build(*, snap_root: str = SNAP_ROOT, ledger_root: str = LEDGER_ROOT, out: str = OUT) -> dict:
+    os.makedirs(out, exist_ok=True)
+    snaps = snapshots_for("tickers", root=snap_root)
+    if not snaps:
+        raise RuntimeError("no ticker snapshots; run tickers.snapshot_current first")
+    latest = load_snapshot(snap_root, snaps[-1]["path"])
+    events = annotate(fill_exchange(events_over_history("tickers", root=snap_root), root=snap_root))
+    filings = read_ledger("filings", root=ledger_root)
+    if filings.empty:
+        raise RuntimeError("filings ledger is empty; run filings.backfill first")
+    delist = reconcile(permanent_removals(events), filings)
+
+    latest.to_parquet(os.path.join(out, "tickers_latest.parquet"), index=False)
+    events.to_parquet(os.path.join(out, "events.parquet"), index=False)
+    delist.to_parquet(os.path.join(out, "delistings.parquet"), index=False)
+    filings.drop(columns=[c for c in ("filed_ts",) if c in filings]).to_parquet(
+        os.path.join(out, "filings.parquet"), index=False)
+
+    # Insider ledger: optional until C2 has run at least once.
+    f144 = read_ledger("form144", root=ledger_root)
+    f4 = read_ledger("form4", root=ledger_root)
+    insider = {}
+    if not f144.empty and not f4.empty:
+        # dense coverage = bulk-dataset rows; EFTS rows after that are sparse until the fill completes
+        cov = f4[f4["source"].str.startswith("dataset")]["transaction_date"].max() if "source" in f4 else None
+        links = link(f144, f4, coverage_end=cov)
+        acc = accuracy_table(links)
+        links.to_parquet(os.path.join(out, "form144_links.parquet"), index=False)
+        acc.to_parquet(os.path.join(out, "accuracy.parquet"), index=False)
+        render_weekly(links, acc, f144, out=os.path.join(os.path.dirname(out), "docs"))
+        mm = links["match_method"].value_counts(normalize=True).to_dict()
+        insider = {"form144": int(len(f144)), "form4_rows": int(len(f4)), "form4_coverage_end": str(cov),
+                   "window_complete": int(links["window_complete"].sum()),
+                   "form4_sales": int((f4["code"] == "S").sum()),
+                   "links": int(len(links)), "match_method_shares": {k: round(float(v), 4) for k, v in mm.items()},
+                   "p_executed_90d_all": float(acc[(acc.segment == "all") & (acc.horizon_days == 90)]["p_executed"].iloc[0])}
+
+    shares = summary(delist)
+    stats = {
+        "built_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "version": __version__,
+        "snapshots": len(snaps),
+        "first_snapshot": snaps[0]["captured_at"],
+        "latest_snapshot": snaps[-1]["captured_at"],
+        "tickers_latest": int(len(latest)),
+        "events": int(len(events)),
+        "events_by_type": events["event"].value_counts().to_dict(),
+        "suspect_steps": sorted(set(events.loc[events.suspect, "to_captured_at"].str[:10])),
+        "raw_removals": int(events.event.eq("DELISTED_FROM_MAP").sum()),
+        "superseded": delist.loc[delist.cause.isin(NOT_DELISTINGS), "cause"].value_counts().to_dict(),
+        "true_delistings": int((~delist.cause.isin(NOT_DELISTINGS)).sum()),
+        "cause_by_exchange_2022": pd.crosstab(delist[(delist.to_captured_at >= "2022") & ~delist.cause.isin(NOT_DELISTINGS)].exchange.replace("", "(blank)"),
+                                              delist[(delist.to_captured_at >= "2022") & ~delist.cause.isin(NOT_DELISTINGS)].cause).to_dict(),
+        "relisted_share": round(float((events.event.eq("DELISTED_FROM_MAP") & events.relisted_at.ne("")).sum()
+                                      / max(1, events.event.eq("DELISTED_FROM_MAP").sum())), 4),
+        "delistings": int(len(delist)),
+        "cause_shares": {c: round(float(shares[c]), 4) for c in CAUSES},
+        "filings": int(len(filings)),
+        "filings_by_form": filings["form"].value_counts().to_dict(),
+        "insider": insider,
+    }
+    _write_readme(out, stats)
+    return stats
+
+
+def _write_readme(out: str, s: dict) -> None:
+    ev = "\n".join(f"| {k} | {v} |" for k, v in sorted(s["events_by_type"].items()))
+    causes = "\n".join(f"| {k} | {v:.1%} |" for k, v in s["cause_shares"].items())
+    forms = "\n".join(f"| {k} | {v} |" for k, v in sorted(s["filings_by_form"].items()))
+    bx = pd.DataFrame(s["cause_by_exchange_2022"]).fillna(0).astype(int)
+    if len(bx):
+        bx["total"] = bx.sum(axis=1)
+        bx["known"] = (1 - bx.get("UNKNOWN", 0) / bx["total"]).map(lambda v: f"{v:.0%}")
+        by_ex = "| exchange | " + " | ".join(bx.columns) + " |\n|---|" + "---|" * len(bx.columns) + "\n" + \
+            "\n".join(f"| {i} | " + " | ".join(str(v) for v in r) + " |" for i, r in bx.iterrows())
+    else:
+        by_ex = "(no rows)"
+    ins = s.get("insider") or {}
+    insider = "" if not ins else f"""
+## Form 144 → Form 4 ledger
+| | |
+|---|---|
+| Form 144 notices | {ins['form144']} |
+| Form 4 transaction rows | {ins['form4_rows']} (sales: {ins['form4_sales']}) |
+| linked notices | {ins['links']} |
+| match method | {', '.join(f'{k} {v:.0%}' for k, v in ins['match_method_shares'].items())} |
+| P(sale within 90d), all | {ins['p_executed_90d_all']:.1%} |
+
+Files: `form144_links.parquet` (one row per notice), `accuracy.parquet`
+(rates with cluster-bootstrap CIs and effective n). Weekly page: `docs/index.html`.
+"""
+    text = f"""# Data
+
+Built {s['built_at']} by tirramind {s['version']}. Derived from SEC public
+filings only (sec.gov: `company_tickers.json`, `company_tickers_exchange.json`,
+EDGAR full-text search). Licence: CC-BY-4.0, attribution "derived from SEC
+public filings". Nothing here is investment advice.
+
+Snapshots before {s['first_snapshot'][:10]}'s own capture come from the
+Wayback Machine (roughly monthly); daily resolution starts 2026-09-06.
+
+| file | rows | what |
+|---|---|---|
+| `tickers_latest.parquet` | {s['tickers_latest']} | current (cik, ticker, name, exchange) |
+| `events.parquet` | {s['events']} | every change between consecutive snapshots |
+| `delistings.parquet` | {s['delistings']} | permanent removals (not re-listed, not from a suspect capture) with a cause and evidence accessions |
+| `filings.parquet` | {s['filings']} | Form 25 / 15 and item-filtered 8-K filings, 2015→ |
+
+## Events ({s['snapshots']} snapshots, {s['first_snapshot'][:10]} → {s['latest_snapshot'][:10]})
+| event | count |
+|---|---|
+{ev}
+
+## What a "removal" is
+`events.parquet` has {s['raw_removals']} raw `DELISTED_FROM_MAP` rows. **{s['relisted_share']:.0%} of
+them re-appear later** (`relisted_at`) — the SEC file churns for housekeeping
+reasons. Steps that removed ≥1,500 tickers at once are partial or
+format-shifted captures and are flagged `suspect`: {', '.join(s['suspect_steps']) or 'none'}.
+Of the removals that are neither re-listed nor suspect ({s['delistings']}),
+those whose CIK gained a *different* ticker within ±60 days **and** have no
+stronger filing evidence are two-step symbol changes or exchange transfers,
+not delistings ({', '.join(f'{k} {v}' for k, v in s['superseded'].items()) or 'none'}).
+They stay in `delistings.parquet` with that cause; the shares below are
+over the remaining {s['true_delistings']} true delistings. The raw rows stay in `events.parquet`; nothing is deleted.
+
+## Delisting causes
+`UNKNOWN` means no qualifying filing was found on that CIK within
+−180/+30 days of the ticker's disappearance. It is reported, not guessed.
+A ticker can also leave the SEC map for housekeeping reasons that are not a
+delisting; that share is inside `UNKNOWN`.
+
+| cause | share |
+|---|---|
+{causes}
+
+### By exchange, removals since 2022
+{by_ex}
+
+Known blind spots: foreign private issuers file 6-K/20-F, not 8-K, so a
+going-private of an ADR shows as `EXCHANGE_DELISTING` (Form 25 only) or
+`UNKNOWN`. Dormant OTC registrants that simply stop filing are `UNKNOWN`
+until a last-filing-date check is added.
+
+## Filings collected
+| form | count |
+|---|---|
+{forms}
+{insider}"""
+    with open(os.path.join(out, "README.md"), "w", encoding="utf-8") as fh:
+        fh.write(text)
+
+
+if __name__ == "__main__":
+    import json
+    print(json.dumps(build(), indent=1))

tirramind-0.1.0/src/tirramind/client.py

@@ -0,0 +1,79 @@
+"""D2: read the published ledger without cloning the repo.
+
+    import tirramind
+    tirramind.load("delistings")           # DataFrame
+    tirramind.events(since="2025-01-01")
+    tirramind.form144(since="2025-06-01")
+
+Files are fetched from the repo's raw URL and cached locally by ETag-less
+content hash for ``ttl`` seconds. Point ``TIRRAMIND_DATA`` at a local
+``data/`` directory to read a checkout instead.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import time
+
+import pandas as pd
+
+RAW = "https://raw.githubusercontent.com/savabs/tirramind-universe/main/data/"
+TABLES = {"tickers": "tickers_latest.parquet", "events": "events.parquet",
+          "delistings": "delistings.parquet", "filings": "filings.parquet",
+          "form144": "form144_links.parquet", "accuracy": "accuracy.parquet"}
+CACHE = os.path.join(os.path.expanduser("~"), ".cache", "tirramind")
+
+
+def _local_dir() -> str | None:
+    return os.environ.get("TIRRAMIND_DATA") or None
+
+
+def _fetch(name: str, ttl: int) -> str:
+    fname = TABLES[name]
+    local = _local_dir()
+    if local:
+        return os.path.join(local, fname)
+    os.makedirs(CACHE, exist_ok=True)
+    path = os.path.join(CACHE, fname)
+    if os.path.exists(path) and time.time() - os.path.getmtime(path) < ttl:
+        return path
+    import requests
+    r = requests.get(RAW + fname, timeout=120)
+    r.raise_for_status()
+    tmp = path + f".{hashlib.md5(r.content).hexdigest()[:8]}.tmp"
+    with open(tmp, "wb") as fh:
+        fh.write(r.content)
+    os.replace(tmp, path)
+    return path
+
+
+def load(name: str, *, ttl: int = 3600) -> pd.DataFrame:
+    if name not in TABLES:
+        raise KeyError(f"unknown table {name!r}; one of {sorted(TABLES)}")
+    return pd.read_parquet(_fetch(name, ttl))
+
+
+def events(*, since: str | None = None, event: str | None = None, **kw) -> pd.DataFrame:
+    df = load("events", **kw)
+    if since:
+        df = df[df["to_captured_at"] >= since]
+    if event:
+        df = df[df["event"].eq(event)]
+    return df.reset_index(drop=True)
+
+
+def delistings(*, since: str | None = None, cause: str | None = None, **kw) -> pd.DataFrame:
+    df = load("delistings", **kw)
+    if since:
+        df = df[df["to_captured_at"] >= since]
+    if cause:
+        df = df[df["cause"].eq(cause)]
+    return df.reset_index(drop=True)
+
+
+def form144(*, since: str | None = None, **kw) -> pd.DataFrame:
+    df = load("form144", **kw)
+    if since:
+        df = df[df["notice_date"] >= since]
+    return df.reset_index(drop=True)