confoundsearch 0.1.0__tar.gz

confoundsearch-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 The ConfoundSearch authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

confoundsearch-0.1.0/MANIFEST.in

@@ -0,0 +1,11 @@
+# The source distribution carries the package, its description and its licence, not the repository around it.
+include README.pypi.md
+include LICENSE*
+prune tests
+prune docs
+prune examples
+prune elastic
+prune tools
+prune badges
+prune .github
+global-exclude *.py[cod] __pycache__ .DS_Store .env .env.*

confoundsearch-0.1.0/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: confoundsearch
+Version: 0.1.0
+Summary: ConfoundSearch: finding confounds as a Markov Decision Process
+License-Expression: MIT
+Keywords: causal inference,confounding,markov decision process,llm,openai
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-cov>=5; extra == "dev"
+Requires-Dist: hypothesis>=6; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Requires-Dist: pyyaml>=6; extra == "dev"
+Requires-Dist: rust-just>=1.17; extra == "dev"
+Requires-Dist: tomli>=2; python_version < "3.11" and extra == "dev"
+Provides-Extra: release
+Requires-Dist: build>=1; extra == "release"
+Requires-Dist: twine>=5; extra == "release"
+Requires-Dist: tomli>=2; python_version < "3.11" and extra == "release"
+Dynamic: license-file
+
+# ConfoundSearch
+
+Find candidate **confounds** for a study of whether X affects Y, together with the published passages that
+support each one and a verifiable audit trail. An AI agent searches a collection of papers and builds a
+diagram of confounds one small step at a time: it searches, adds a confound, links it to your two variables,
+and cites passages that support the link. It suggests; the researcher decides.
+
+Supporting code for our *Organization Science* proposal, *Using AI to find confounds*.
+
+```bash
+pip install confoundsearch        # Python 3.10 or newer; no dependencies
+```
+
+## From Python
+
+```python
+from confoundsearch import *
+
+corpus = InMemoryCorpus(
+    [
+        Passage("p1", "Larger firms invest more in information technology.", title="Firm size and IT"),
+        Passage(
+            "p2",
+            "Firm size is associated with firm performance and profitability.",
+            title="Firm size and profits",
+        ),
+    ]
+)
+client = AuditedClient(OpenAICompatibleClient("gpt-4.1"))  # reads OPENAI_API_KEY; logs every model call
+env = Environment(corpus, LLMJudge(client))
+policy = EpsilonGreedy(LookaheadQ(env.preview), epsilon=0.1, explore_stop=False)
+run = run_episode(
+    env, policy, Proposer(LLMProposer(client)), "IT investment", "firm performance", max_steps=20
+)
+
+print(run.final.graph.labels, run.score)  # the confounds found, and the score
+print(run.audit_trail().to_markdown())  # the evidence behind each, and where it came from
+```
+
+## From the command line
+
+```bash
+confoundsearch serve --model gpt-4.1 --corpus-file passages.jsonl    # the web interface, on your computer
+confoundsearch run --json --model gpt-4.1 --corpus-file passages.jsonl --iv "IT investment" --dv "firm performance"
+confoundsearch audit confoundsearch.audit.jsonl                       # check and read a run's audit trail
+confoundsearch install-skill                                          # add the Claude Code skill
+```
+
+## What you get
+
+- **Any papers.** Search our hosted Elasticsearch index (set `ES_URL` and `ES_API_KEY`), Semantic Scholar
+  (`--corpus semanticscholar`; a [free API key](https://www.semanticscholar.org/product/api) in `S2_KEY` makes it
+  much faster), OpenAlex (`--corpus openalex`), or a file of your own passages (`--corpus-file`).
+- **Any model.** OpenAI, [OpenRouter](https://openrouter.ai) (`--openrouter`, with your own `OPENROUTER_API_KEY`), or
+  any OpenAI-compatible server, including one on your own computer.
+- **Auditable.** Every run keeps a hash-chained log of each search, each passage retrieved, the judge's verdict on
+  each citation, and each model prompt and reply, so any edit to it is detectable.
+- **Four modular layers**, one directory each: `data`, `logic`, `model` and `interface`.
+- **Agent-ready.** `run --json` returns the result as JSON, and a Claude Code skill is included.
+
+Each run calls the AI model, which costs money and sends your text to its provider.
+
+Released under the MIT licence.

confoundsearch-0.1.0/README.md

@@ -0,0 +1,82 @@
+# ConfoundSearch
+
+Supporting code for our *Organization Science* proposal, *Using AI to find confounds*.
+
+![coverage](badges/coverage.svg)
+![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)
+![Coverage gate](https://img.shields.io/badge/coverage%20gate-100%25%20(branch)-brightgreen)
+![mypy](https://img.shields.io/badge/mypy-strict-blue)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+![Dependencies](https://img.shields.io/badge/runtime%20dependencies-0-brightgreen)
+![License](https://img.shields.io/badge/license-MIT-blue)
+
+![Example output: candidate confounds of IT investment and firm performance, and the passage behind one link](docs/img/example.png)
+
+*Example output on made-up demo passages, with a scripted stand-in for the AI model (no key needed): open
+[`examples/example_output.html`](examples/example_output.html). **Reviewing this artifact?** See the [reviewer guide](docs/artifact-evaluation.md).*
+
+## How do I run this?
+
+You need Python 3.10+ and an API key for OpenAI, [OpenRouter](https://openrouter.ai) (`--openrouter`) or a compatible service. Run `pip install .` in the unzipped
+repository, then use it in any of four ways. We plan to push updates to pip as we keep developing ConfoundSearch.
+
+**1. From Python.**
+
+```python
+from confoundsearch import *
+
+client = AuditedClient(OpenAICompatibleClient("gpt-4.1"))  # reads OPENAI_API_KEY; logs every model call
+env = Environment(InMemoryCorpus.from_jsonl("examples/passages.jsonl"), LLMJudge(client))
+policy = EpsilonGreedy(LookaheadQ(env.preview), epsilon=0.1, explore_stop=False)
+run = run_episode(
+    env, policy, Proposer(LLMProposer(client)), "IT investment", "firm performance", max_steps=40
+)
+
+print(run.final.graph.labels, run.score)  # the confounds found, and the score
+print(run.audit_trail().to_markdown())  # the evidence behind each, and where it came from
+save_export(run.final, "run.zip", audit=run.audit)  # export for the interface, with its audit trail
+```
+
+**2. In the web interface, on your computer.**
+`confoundsearch serve --model gpt-4.1 --api-key YOUR_KEY --corpus-file examples/passages.jsonl`,
+then open <http://127.0.0.1:7860>, type two variables, and click **Find confounds**.
+
+**3. In the hosted web app,** which serves the same interface with nothing to install (link withheld during blind review).
+
+**4. From Claude Code or another agent.** `confoundsearch install-skill` adds a skill, and `confoundsearch run --json`
+returns the results and the audit-trail path as JSON. More: [docs/claude-code.md](docs/claude-code.md).
+
+**Papers.** The commands above use the demo file. By default the agent searches our **hosted Elasticsearch index**
+(put `ES_URL` and `ES_API_KEY` in a `.env` file). Or add `--corpus semanticscholar` (a [free API key](https://www.semanticscholar.org/product/api),
+set as `S2_KEY`, makes it much faster), `--corpus openalex`, or `--corpus-file your.jsonl`.
+
+Each run calls the AI model, which costs money and sends your text to its provider. Details: [docs/running.md](docs/running.md).
+
+## What's going on under the hood?
+
+An AI agent builds a diagram of candidate confounds one small step at a time: it searches the papers,
+adds a confound, links it to your two variables, and cites passages that support the link. A score
+rewards confounds that are linked to both variables and well supported. It suggests; the researcher decides.
+
+**Auditable.** ConfoundSearch keeps a log of the evidence it collects and processes during search: every query,
+every passage retrieved, the judge's verdict on every citation, and every model prompt and reply, in order and
+hash-chained so any edit is detectable. Read it with `run.audit_trail()` or `confoundsearch audit FILE`, or open
+[`examples/example_audit.jsonl`](examples/example_audit.jsonl). More: [docs/auditability.md](docs/auditability.md).
+
+**Decoupled.** The package never draws anything: it exports a plain data file (`save_export`), and the interface only
+reads such files, never running the agent. So one output can be shown in the local interface, in the hosted app,
+or as a single HTML page (`confoundsearch run --out page.html`). More: [docs/how-it-works.md](docs/how-it-works.md).
+
+## What's in this code base?
+
+The application is divided into four layers, as described in the technical appendix. Each is one directory:
+
+- **Data** ([`confoundsearch/data/`](confoundsearch/data)): where papers come from (hosted Elasticsearch, Semantic Scholar, OpenAlex, or your file).
+- **Logic** ([`confoundsearch/logic/`](confoundsearch/logic)): the Markov decision process (graph, moves, reward, policy) and the audit log.
+- **Language model** ([`confoundsearch/model/`](confoundsearch/model)): any OpenAI-compatible model, including OpenRouter.
+- **Interface** ([`confoundsearch/interface/`](confoundsearch/interface)): the web app, command line, JSON output and agent skill.
+
+Also here: `examples/`, `tests/` (100% branch coverage, and they enforce the layer boundaries), and `elastic/`
+(loads the corpus into Elasticsearch Serverless: [how](elastic/README.md)).
+
+More: [paper-to-code map and how we know it works](docs/code-map.md) · [for programmers](docs/for-programmers.md)

confoundsearch-0.1.0/README.pypi.md

@@ -0,0 +1,63 @@
+# ConfoundSearch
+
+Find candidate **confounds** for a study of whether X affects Y, together with the published passages that
+support each one and a verifiable audit trail. An AI agent searches a collection of papers and builds a
+diagram of confounds one small step at a time: it searches, adds a confound, links it to your two variables,
+and cites passages that support the link. It suggests; the researcher decides.
+
+Supporting code for our *Organization Science* proposal, *Using AI to find confounds*.
+
+```bash
+pip install confoundsearch        # Python 3.10 or newer; no dependencies
+```
+
+## From Python
+
+```python
+from confoundsearch import *
+
+corpus = InMemoryCorpus(
+    [
+        Passage("p1", "Larger firms invest more in information technology.", title="Firm size and IT"),
+        Passage(
+            "p2",
+            "Firm size is associated with firm performance and profitability.",
+            title="Firm size and profits",
+        ),
+    ]
+)
+client = AuditedClient(OpenAICompatibleClient("gpt-4.1"))  # reads OPENAI_API_KEY; logs every model call
+env = Environment(corpus, LLMJudge(client))
+policy = EpsilonGreedy(LookaheadQ(env.preview), epsilon=0.1, explore_stop=False)
+run = run_episode(
+    env, policy, Proposer(LLMProposer(client)), "IT investment", "firm performance", max_steps=20
+)
+
+print(run.final.graph.labels, run.score)  # the confounds found, and the score
+print(run.audit_trail().to_markdown())  # the evidence behind each, and where it came from
+```
+
+## From the command line
+
+```bash
+confoundsearch serve --model gpt-4.1 --corpus-file passages.jsonl    # the web interface, on your computer
+confoundsearch run --json --model gpt-4.1 --corpus-file passages.jsonl --iv "IT investment" --dv "firm performance"
+confoundsearch audit confoundsearch.audit.jsonl                       # check and read a run's audit trail
+confoundsearch install-skill                                          # add the Claude Code skill
+```
+
+## What you get
+
+- **Any papers.** Search our hosted Elasticsearch index (set `ES_URL` and `ES_API_KEY`), Semantic Scholar
+  (`--corpus semanticscholar`; a [free API key](https://www.semanticscholar.org/product/api) in `S2_KEY` makes it
+  much faster), OpenAlex (`--corpus openalex`), or a file of your own passages (`--corpus-file`).
+- **Any model.** OpenAI, [OpenRouter](https://openrouter.ai) (`--openrouter`, with your own `OPENROUTER_API_KEY`), or
+  any OpenAI-compatible server, including one on your own computer.
+- **Auditable.** Every run keeps a hash-chained log of each search, each passage retrieved, the judge's verdict on
+  each citation, and each model prompt and reply, so any edit to it is detectable.
+- **Four modular layers**, one directory each: `data`, `logic`, `model` and `interface`.
+- **Agent-ready.** `run --json` returns the result as JSON, and a Claude Code skill is included.
+
+Each run calls the AI model, which costs money and sends your text to its provider.
+
+Released under the MIT licence.

confoundsearch-0.1.0/confoundsearch/__init__.py

@@ -0,0 +1,72 @@
+"""ConfoundSearch: an agent that builds a confound graph as a Markov Decision Process."""
+
+from confoundsearch.data.corpus import Corpus, InMemoryCorpus
+from confoundsearch.data.es import ElasticsearchCorpus
+from confoundsearch.data.sources import OpenAlexCorpus, SemanticScholarCorpus
+from confoundsearch.errors import CorpusError, InvalidAction, LLMError, RemoteError
+from confoundsearch.interface.export import export_zip, save_export, to_share_data
+from confoundsearch.logic.actions import (
+    Action,
+    AddConfound,
+    AddEdge,
+    AttachPassage,
+    Merge,
+    Retrieve,
+    Stop,
+    enumerate_actions,
+)
+from confoundsearch.logic.audit import AuditedClient, AuditLog, AuditTrail, verify
+from confoundsearch.logic.episode import Step, Trajectory, run_episode
+from confoundsearch.logic.graph import Graph, Passage, Vertex
+from confoundsearch.logic.policy import EpsilonGreedy, LookaheadQ, Softmax
+from confoundsearch.logic.reward import phi, reward
+from confoundsearch.logic.state import State
+from confoundsearch.logic.transition import Environment, Judge
+from confoundsearch.model.llm import LLMJudge, LLMProposer, OpenAICompatibleClient, Proposer
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Action",
+    "AddConfound",
+    "AddEdge",
+    "AttachPassage",
+    "AuditLog",
+    "AuditTrail",
+    "AuditedClient",
+    "Corpus",
+    "CorpusError",
+    "ElasticsearchCorpus",
+    "Environment",
+    "EpsilonGreedy",
+    "Graph",
+    "InMemoryCorpus",
+    "InvalidAction",
+    "Judge",
+    "LLMError",
+    "LLMJudge",
+    "LLMProposer",
+    "LookaheadQ",
+    "Merge",
+    "OpenAICompatibleClient",
+    "OpenAlexCorpus",
+    "Passage",
+    "Proposer",
+    "RemoteError",
+    "Retrieve",
+    "SemanticScholarCorpus",
+    "Softmax",
+    "State",
+    "Step",
+    "Stop",
+    "Trajectory",
+    "Vertex",
+    "enumerate_actions",
+    "export_zip",
+    "phi",
+    "reward",
+    "run_episode",
+    "save_export",
+    "to_share_data",
+    "verify",
+]

confoundsearch-0.1.0/confoundsearch/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from confoundsearch.interface.cli import main
+
+sys.exit(main())

confoundsearch-0.1.0/confoundsearch/data/__init__.py

@@ -0,0 +1,2 @@
+"""The data layer: where papers come from (a file, Elasticsearch, Semantic Scholar, OpenAlex), all behind the
+`Corpus` interface."""

confoundsearch-0.1.0/confoundsearch/data/corpus.py

@@ -0,0 +1,36 @@
+"""The corpus D, behind a one-method protocol."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Protocol
+
+from confoundsearch.logic.graph import Passage, normalize
+
+
+class Corpus(Protocol):
+    def search(self, query: str, k: int) -> list[Passage]:
+        """Return up to k passages relevant to the query."""
+        ...
+
+
+class InMemoryCorpus:
+    """Ranks passages by how many query words they contain. Deterministic."""
+
+    def __init__(self, passages: Iterable[Passage]) -> None:
+        self.passages = tuple(passages)
+
+    @classmethod
+    def from_jsonl(cls, path: str | Path) -> InMemoryCorpus:
+        """One JSON object per line with `id` and `text`, and optionally `title` and `url`."""
+        rows = [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()]
+        return cls(Passage(str(r["id"]), r["text"], r.get("title", ""), r.get("url", "")) for r in rows)
+
+    def search(self, query: str, k: int) -> list[Passage]:
+        words = set(normalize(query).split())
+        scored = [(len(words & set(normalize(p.text).split())), p) for p in self.passages]
+        hits = [(n, p) for n, p in scored if n > 0]
+        hits.sort(key=lambda hit: (-hit[0], hit[1].id))
+        return [p for _, p in hits[: max(k, 0)]]

confoundsearch-0.1.0/confoundsearch/data/es.py

@@ -0,0 +1,72 @@
+"""An Elasticsearch-backed corpus, over the plain REST API.
+
+Works with a local node or Elastic Cloud. Documents need a text field (default
+`abstract`) and may have `title`, `url` and `corpusid`, as in the Semantic
+Scholar index the paper's experiments use.
+
+    ElasticsearchCorpus("https://my-deployment.es.us-east-1.aws.elastic.cloud", api_key="...")
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from confoundsearch.errors import CorpusError, RemoteError
+from confoundsearch.http import Transport, urllib_transport
+from confoundsearch.logic.graph import Passage
+
+
+class ElasticsearchCorpus:
+    def __init__(
+        self,
+        url: str,
+        index: str = "papers",
+        api_key: str | None = None,
+        text_field: str = "abstract",
+        title_field: str = "title",
+        min_citations: int = 0,
+        timeout: float = 30.0,
+        transport: Transport = urllib_transport,
+    ) -> None:
+        self.url = url.rstrip("/")
+        self.index = index
+        self.api_key = api_key
+        self.text_field = text_field
+        self.title_field = title_field
+        self.min_citations = min_citations
+        self.timeout = timeout
+        self.transport = transport
+
+    def _query(self, query: str, k: int) -> dict[str, Any]:
+        match = {"multi_match": {"query": query, "fields": [f"{self.title_field}^2", self.text_field]}}
+        bool_query: dict[str, Any] = {"must": match, "filter": [{"exists": {"field": self.text_field}}]}
+        if self.min_citations:
+            bool_query["filter"].append({"range": {"citationcount": {"gte": self.min_citations}}})
+        return {"query": {"bool": bool_query}, "size": k}
+
+    def search(self, query: str, k: int) -> list[Passage]:
+        if k <= 0:
+            return []
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"ApiKey {self.api_key}"
+        body = json.dumps(self._query(query, k)).encode()
+        try:
+            raw = self.transport(f"{self.url}/{self.index}/_search", headers, body, self.timeout)
+        except RemoteError as err:
+            raise CorpusError(str(err)) from err
+        try:
+            hits = json.loads(raw)["hits"]["hits"]
+            return [self._passage(hit) for hit in hits]
+        except (ValueError, KeyError, TypeError) as err:
+            raise CorpusError(f"unexpected response: {raw[:200]!r}") from err
+
+    def _passage(self, hit: dict[str, Any]) -> Passage:
+        source = hit["_source"]
+        return Passage(
+            id=str(source.get("corpusid", hit["_id"])),
+            text=source[self.text_field],
+            title=source.get(self.title_field) or "",
+            url=source.get("url") or "",
+        )

confoundsearch-0.1.0/confoundsearch/data/sources.py

@@ -0,0 +1,148 @@
+"""Paper sources that need no corpus of your own: Semantic Scholar's API and OpenAlex's.
+
+Both implement the same one-method `Corpus` protocol as the file and Elasticsearch corpora, so the agent
+does not know or care which it is searching. Only papers that have an abstract are returned, because an
+abstract is the passage the agent reads and cites.
+
+    SemanticScholarCorpus(api_key="...")      # a free key raises the rate limit; request one at S2_KEY_URL
+    OpenAlexCorpus(mailto="you@example.org")  # an email (or an api_key) puts you in OpenAlex's polite pool
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import urlencode
+
+from confoundsearch.errors import CorpusError, RemoteError
+from confoundsearch.http import Transport, urllib_transport
+from confoundsearch.logic.graph import Passage
+
+MAX_PAGE = 100
+S2_KEY_URL = "https://www.semanticscholar.org/product/api"  # where to request a free Semantic Scholar API key
+
+
+def _get(transport: Transport, url: str, headers: Mapping[str, str], timeout: float, hint: str) -> Any:
+    try:
+        raw = transport(url, headers, None, timeout)
+    except RemoteError as err:
+        extra = f" {hint}" if "HTTP 429" in str(err) else ""
+        raise CorpusError(f"{err}{extra}") from err
+    try:
+        return json.loads(raw)
+    except ValueError as err:
+        raise CorpusError(f"unexpected response: {raw[:200]!r}") from err
+
+
+def _page_size(k: int) -> int:
+    """Ask for extra results, since some papers have no abstract and are dropped."""
+    return min(max(2 * k, k), MAX_PAGE)
+
+
+class SemanticScholarCorpus:
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str = "https://api.semanticscholar.org/graph/v1",
+        timeout: float = 30.0,
+        transport: Transport = urllib_transport,
+    ) -> None:
+        self.api_key = api_key
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.transport = transport
+
+    def search(self, query: str, k: int) -> list[Passage]:
+        if k <= 0 or not query.strip():
+            return []
+        params = urlencode(
+            {
+                "query": query,
+                "limit": _page_size(k),
+                "fields": "title,abstract,url,year,citationCount,corpusId",
+            }
+        )
+        headers = {"Accept": "application/json"}
+        if self.api_key:
+            headers["x-api-key"] = self.api_key
+        hint = (
+            f"Semantic Scholar is rate limiting you. Request a free API key at {S2_KEY_URL} "
+            "and set it as S2_KEY (or --s2-key); it raises the limit."
+        )
+        data = _get(self.transport, f"{self.base_url}/paper/search?{params}", headers, self.timeout, hint)
+        try:
+            papers = data.get("data") or []  # an empty result has no "data" at all
+            passages = [self._passage(p) for p in papers if p.get("abstract")]
+        except (AttributeError, KeyError, TypeError) as err:
+            raise CorpusError(f"unexpected response: {str(data)[:200]!r}") from err
+        return passages[:k]
+
+    @staticmethod
+    def _passage(paper: dict[str, Any]) -> Passage:
+        return Passage(
+            id=str(paper.get("corpusId") or paper["paperId"]),
+            text=paper["abstract"],
+            title=paper.get("title") or "",
+            url=paper.get("url") or "",
+        )
+
+
+def abstract_from_inverted_index(inverted: Mapping[str, list[int]]) -> str:
+    """OpenAlex stores an abstract as {word: [positions]}. Put the words back in order."""
+    placed = sorted((position, word) for word, positions in inverted.items() for position in positions)
+    return " ".join(word for _, word in placed)
+
+
+class OpenAlexCorpus:
+    def __init__(
+        self,
+        api_key: str | None = None,
+        mailto: str | None = None,
+        base_url: str = "https://api.openalex.org",
+        timeout: float = 30.0,
+        transport: Transport = urllib_transport,
+    ) -> None:
+        self.api_key = api_key
+        self.mailto = mailto
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+        self.transport = transport
+
+    def search(self, query: str, k: int) -> list[Passage]:
+        if k <= 0 or not query.strip():
+            return []
+        params = {
+            "search": query,
+            "per_page": _page_size(k),
+            "select": "id,title,abstract_inverted_index,doi,publication_year",
+        }
+        if self.api_key:
+            params["api_key"] = self.api_key
+        if self.mailto:
+            params["mailto"] = self.mailto
+        hint = (
+            "OpenAlex is rate limiting you; add an email (OPENALEX_EMAIL) or an API key (OPENALEX_API_KEY)."
+        )
+        data = _get(
+            self.transport,
+            f"{self.base_url}/works?{urlencode(params)}",
+            {"Accept": "application/json"},
+            self.timeout,
+            hint,
+        )
+        try:
+            works = data["results"]
+            passages = [self._passage(w) for w in works if w.get("abstract_inverted_index")]
+        except (KeyError, TypeError, AttributeError) as err:
+            raise CorpusError(f"unexpected response: {str(data)[:200]!r}") from err
+        return passages[:k]
+
+    @staticmethod
+    def _passage(work: dict[str, Any]) -> Passage:
+        return Passage(
+            id=work["id"].rsplit("/", 1)[-1],
+            text=abstract_from_inverted_index(work["abstract_inverted_index"]),
+            title=work.get("title") or "",
+            url=work.get("doi") or work["id"],
+        )

confoundsearch-0.1.0/confoundsearch/dotenv.py

@@ -0,0 +1,21 @@
+"""Read settings from a `.env` file, so API keys need not be typed into a terminal."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def load_dotenv(path: Path) -> None:
+    """Put KEY=VALUE lines from `path` into the environment. Never overrides a variable already set."""
+    if not path.is_file():
+        return
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip().removeprefix("export ")
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        os.environ.setdefault(key.strip(), value)

confoundsearch-0.1.0/confoundsearch/errors.py

@@ -0,0 +1,17 @@
+"""Exceptions shared across the package."""
+
+
+class InvalidAction(ValueError):
+    """An edit or action violates a graph invariant or the MDP's rules."""
+
+
+class RemoteError(RuntimeError):
+    """A remote HTTP service failed or returned something unusable."""
+
+
+class LLMError(RemoteError):
+    """An OpenAI-compatible endpoint failed or returned something unusable."""
+
+
+class CorpusError(RemoteError):
+    """The search backend failed or returned something unusable."""