newsrag 0.2.0__tar.gz

newsrag-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: Quality checks
+on:
+  push:
+  pull_request:
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: python -m pip install -U pip ruff mypy pytest build "pypdf>=5,<7"
+      - run: python -m pip install -e .
+      - run: ruff check src tests examples setup.py
+      - run: mypy src/newsrag
+      - run: pytest -q
+      - run: python -m build

newsrag-0.2.0/.github/workflows/release.yml

@@ -0,0 +1,42 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m pip install -U build twine pytest ruff mypy 'pypdf>=5,<7'
+      - run: python -m pip install -e .
+      - run: ruff check src tests examples setup.py
+      - run: mypy src/newsrag
+      - run: pytest -q
+      - run: python -m build
+      - run: python -m twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+          if-no-files-found: error
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

newsrag-0.2.0/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+## 0.2.0 - 2026-09-26
+
+- Added caller-supplied HTML and optional pypdf text extraction.
+- Added local JSON source snapshots with atomic writes and restricted POSIX permissions.
+- Expanded automated checks and source metadata documentation.
+
+## 0.1.0 - 2026-09-26
+
+- Initial provider-neutral in-memory retrieval with Arabic and English lexical matching.
+- Added source metadata, exact offsets, date filters, optional embeddings and generation adapters.
+- Added citation reference validation, abstention on no matches and review warnings.
+
+## Unreleased
+
+- Normalize common Arabic letter variants after Unicode NFKC normalization (including Persian keyboard variants).
+- Allow an optional caller-supplied reranker to reorder retrieved evidence without changing or injecting passages; citation IDs are reassigned after reranking.
+- Add tests for Arabic variants, reranking, citation IDs and rejected adapter output.
+- Add a small deterministic retrieval evaluation harness with source-level recall@k, hit rate@k, mean reciprocal rank and missed queries; document its limits.

newsrag-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,14 @@
+# Contributing
+
+Open an issue before proposing a large change. For a small fix, include a test that fails before the fix and passes afterward. Please do not commit private reporting notes, unpublished sources, credentials or API tokens.
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+python -m pip install -e . pytest ruff mypy 'pypdf>=5,<7'
+ruff check src tests examples setup.py
+mypy src/newsrag
+pytest -q
+```
+
+Keep adapters optional and avoid hidden network calls. API additions should preserve source IDs and exact offsets. Include tests with Arabic and English input, missing dates and malformed provider output where relevant. Document backward-incompatible changes in CHANGELOG.md.

newsrag-0.2.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

newsrag-0.2.0/MANIFEST.in

@@ -0,0 +1,5 @@
+include README.md LICENSE NOTICE CHANGELOG.md CONTRIBUTING.md SECURITY.md
+recursive-include docs *.md
+recursive-include examples *.py
+recursive-include tests *.py
+recursive-include .github *.yml

newsrag-0.2.0/NOTICE

@@ -0,0 +1,5 @@
+NewsRAG
+Copyright 2026 Shehata El-sayed
+
+This product is licensed under the Apache License, Version 2.0.
+See the LICENSE file for the complete terms.

newsrag-0.2.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: newsrag
+Version: 0.2.0
+Summary: Auditable, provider-neutral RAG for newsroom research
+Author: Shehata El-sayed
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Provides-Extra: pdf
+Requires-Dist: pypdf<7,>=5; extra == "pdf"
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-python
+Dynamic: summary
+
+# NewsRAG by Shehata El-sayed
+
+A small, auditable Python RAG toolkit for journalistic research. This v0.2 pre-release is a research prototype; check PyPI for publication status. Python 3.10+; standard library core. Licensed under Apache-2.0 (see LICENSE and NOTICE).
+
+## Example
+
+```python
+from datetime import datetime, timezone
+from newsrag import NewsroomRAG, Source
+
+rag = NewsroomRAG(recency_half_life_days=30).add(
+    Source(id="1", title="Official statement", text="A limited pilot began in September. No public launch was announced.",
+           url="https://example.org/statement", published_at=datetime(2026, 9, 10, tzinfo=timezone.utc))
+)
+print(rag.ask("Was there a public launch?").as_dict())
+```
+
+Save a source's actual URL and publication date. The default `ask` retrieves evidence without generating prose. Add provider functions when you want answer generation:
+
+```python
+rag = NewsroomRAG(
+    embed=lambda texts: embedding_provider(texts),   # list[str] -> list[list[float]]
+    generate=lambda prompt: llm_provider(prompt),    # str -> str
+).add(...)
+answer = rag.ask("ما الذي أُعلن؟")
+print(answer.text, answer.evidence, answer.warnings)
+```
+
+Install locally with `python -m pip install .`; run tests with `PYTHONPATH=src python -m unittest discover -s tests -v`, and run `PYTHONPATH=src python examples/quickstart.py` from this directory. No network calls occur unless your adapters make them.
+
+## Newsroom-specific controls
+
+- Every source carries an ID, title, original URL, publisher, type, timestamp with timezone, and original text. Search results retain exact offsets and excerpts for audit.
+- Arabic orthographic normalization and English token search work locally. Optional provider embeddings improve semantic search and cross-language matching, but an appropriate multilingual model is required for Arabic-English semantic retrieval.
+- Optional publication date range filters (`before`, `after`) and a modest recency ranking boost. Unknown publication dates are excluded when filtering by date; they remain eligible otherwise. The timestamp means publication time, not event time.
+- Generator prompt asks for bracketed evidence IDs (`[E1]`). IDs are checked against retrieved passages and unknown IDs flagged. This does **not** verify the claims, the source's truth, or whether cited passages support each sentence.
+- No-match answers abstain. Source text is marked untrusted, but prompt injection remains a risk with any model. Do not feed confidential material to a third-party provider without permission. Review original sources, licensing, quotes, dates, and contested claims before publishing.
+
+## Scope and limitations
+
+This v0.2 is an in-memory retrieval prototype with local JSON source snapshots, not a production crawler or fact-checker. It handles plain text and synchronous callable adapters; automatic fetching of URLs, OCR for scanned PDFs, database connectors, persistent vector indexes, source deduplication, named provider packages, automated entailment checking, and benchmark-based tuning are future work. Lexical Arabic tokenization is basic, not morphological. No claim that this works with every provider out of the box: providers must offer embeddings with stable vector dimensions and/or text generation and be wrapped in the two simple callables. The toy example.org URLs are not real news sources.
+
+
+## Quality checks
+
+Run `python -m pip install -e . pytest ruff mypy build "pypdf>=5,<7"`, then `ruff check src tests examples setup.py`, `mypy src/newsrag`, `pytest -q`, and `python -m build`. CI is configured to run these checks on Python 3.10–3.13; this local package was checked on Python 3.10 only. A passing suite reduces risk but cannot guarantee zero bugs or factual accuracy.
+
+## Local ingestion and snapshots
+
+```python
+from newsrag import source_from_html, source_from_pdf, save_sources, load_sources
+
+# You fetch the original page yourself. This library does not fetch URLs or bypass site terms.
+source = source_from_html(id="story-1", title="Original article", url="https://example.org/story",
+                          html="<article><p>An independently confirmed report.</p></article>")
+rag.add(source)
+save_sources(rag, "research.json")  # source content, mode 0600; no API keys or model code
+restored = load_sources("research.json")
+# PDF extraction, optional: pip install 'newsrag[pdf]'
+# document = source_from_pdf(id="file-1", title="Report", pdf_bytes=pdf_data)
+```
+
+For web ingestion, the caller is responsible for fetching, verifying the URL, handling redirects, authentication, robots/licensing rules and SSRF protection. Passing HTML does not prove its authorship. HTML extraction is simple and may include navigation text; review the output. PDF extraction is text-only and may require OCR for scans. Snapshots store unencrypted source text, so keep them in a secure location; the writer uses 0600 permissions on POSIX. There is no network fetcher or hosted persistence. The latest source content should be rechecked at the original publisher before publication.
+
+Further reading: [architecture and provider contract](docs/architecture.md), [release checklist](docs/release-checklist.md), [contributing](CONTRIBUTING.md), [security](SECURITY.md), and [changelog](CHANGELOG.md). These are preparation for a public release, not evidence it has been published or reviewed independently.
+
+An optional provider illustration is in [`examples/openai_adapter.py`](examples/openai_adapter.py). It requires a separately installed SDK and the user's own provider credentials; no credentials belong in this repository. Model names and SDK behavior can change, so validate an adapter against its provider's current documentation before using it.
+
+For release mechanics, see [PyPI Trusted Publishing setup](docs/publishing.md). No API token is needed in GitHub Actions. A GitHub release or a pending publisher alone is not proof that PyPI published the package; check the PyPI project and install before relying on it.
+
+### Optional evidence reranking
+
+Pass `rerank=lambda query, evidence: ...` when you have a trusted reranking function. The function receives the initially retrieved evidence and must return a permutation of those same objects; new or rewritten passages are rejected. Citation IDs are reassigned after the new order. Reranking cannot recover evidence absent from the initial `top_k`, and does not check whether an answer is true. Arabic normalization includes common hamza, ta marbuta, alef maqsura and Persian keyboard letter variants; it is still not Arabic stemming or morphological analysis.
+
+### Retrieval evaluation
+
+Use `RetrievalCase` and `evaluate_retrieval` with independently labeled relevant source IDs to measure recall@k, hit rate@k, and reciprocal rank. See [evaluation guide](docs/evaluation.md). These metrics do not validate generated claims or source credibility.

newsrag-0.2.0/README.md

@@ -0,0 +1,77 @@
+# NewsRAG by Shehata El-sayed
+
+A small, auditable Python RAG toolkit for journalistic research. This v0.2 pre-release is a research prototype; check PyPI for publication status. Python 3.10+; standard library core. Licensed under Apache-2.0 (see LICENSE and NOTICE).
+
+## Example
+
+```python
+from datetime import datetime, timezone
+from newsrag import NewsroomRAG, Source
+
+rag = NewsroomRAG(recency_half_life_days=30).add(
+    Source(id="1", title="Official statement", text="A limited pilot began in September. No public launch was announced.",
+           url="https://example.org/statement", published_at=datetime(2026, 9, 10, tzinfo=timezone.utc))
+)
+print(rag.ask("Was there a public launch?").as_dict())
+```
+
+Save a source's actual URL and publication date. The default `ask` retrieves evidence without generating prose. Add provider functions when you want answer generation:
+
+```python
+rag = NewsroomRAG(
+    embed=lambda texts: embedding_provider(texts),   # list[str] -> list[list[float]]
+    generate=lambda prompt: llm_provider(prompt),    # str -> str
+).add(...)
+answer = rag.ask("ما الذي أُعلن؟")
+print(answer.text, answer.evidence, answer.warnings)
+```
+
+Install locally with `python -m pip install .`; run tests with `PYTHONPATH=src python -m unittest discover -s tests -v`, and run `PYTHONPATH=src python examples/quickstart.py` from this directory. No network calls occur unless your adapters make them.
+
+## Newsroom-specific controls
+
+- Every source carries an ID, title, original URL, publisher, type, timestamp with timezone, and original text. Search results retain exact offsets and excerpts for audit.
+- Arabic orthographic normalization and English token search work locally. Optional provider embeddings improve semantic search and cross-language matching, but an appropriate multilingual model is required for Arabic-English semantic retrieval.
+- Optional publication date range filters (`before`, `after`) and a modest recency ranking boost. Unknown publication dates are excluded when filtering by date; they remain eligible otherwise. The timestamp means publication time, not event time.
+- Generator prompt asks for bracketed evidence IDs (`[E1]`). IDs are checked against retrieved passages and unknown IDs flagged. This does **not** verify the claims, the source's truth, or whether cited passages support each sentence.
+- No-match answers abstain. Source text is marked untrusted, but prompt injection remains a risk with any model. Do not feed confidential material to a third-party provider without permission. Review original sources, licensing, quotes, dates, and contested claims before publishing.
+
+## Scope and limitations
+
+This v0.2 is an in-memory retrieval prototype with local JSON source snapshots, not a production crawler or fact-checker. It handles plain text and synchronous callable adapters; automatic fetching of URLs, OCR for scanned PDFs, database connectors, persistent vector indexes, source deduplication, named provider packages, automated entailment checking, and benchmark-based tuning are future work. Lexical Arabic tokenization is basic, not morphological. No claim that this works with every provider out of the box: providers must offer embeddings with stable vector dimensions and/or text generation and be wrapped in the two simple callables. The toy example.org URLs are not real news sources.
+
+
+## Quality checks
+
+Run `python -m pip install -e . pytest ruff mypy build "pypdf>=5,<7"`, then `ruff check src tests examples setup.py`, `mypy src/newsrag`, `pytest -q`, and `python -m build`. CI is configured to run these checks on Python 3.10–3.13; this local package was checked on Python 3.10 only. A passing suite reduces risk but cannot guarantee zero bugs or factual accuracy.
+
+## Local ingestion and snapshots
+
+```python
+from newsrag import source_from_html, source_from_pdf, save_sources, load_sources
+
+# You fetch the original page yourself. This library does not fetch URLs or bypass site terms.
+source = source_from_html(id="story-1", title="Original article", url="https://example.org/story",
+                          html="<article><p>An independently confirmed report.</p></article>")
+rag.add(source)
+save_sources(rag, "research.json")  # source content, mode 0600; no API keys or model code
+restored = load_sources("research.json")
+# PDF extraction, optional: pip install 'newsrag[pdf]'
+# document = source_from_pdf(id="file-1", title="Report", pdf_bytes=pdf_data)
+```
+
+For web ingestion, the caller is responsible for fetching, verifying the URL, handling redirects, authentication, robots/licensing rules and SSRF protection. Passing HTML does not prove its authorship. HTML extraction is simple and may include navigation text; review the output. PDF extraction is text-only and may require OCR for scans. Snapshots store unencrypted source text, so keep them in a secure location; the writer uses 0600 permissions on POSIX. There is no network fetcher or hosted persistence. The latest source content should be rechecked at the original publisher before publication.
+
+Further reading: [architecture and provider contract](docs/architecture.md), [release checklist](docs/release-checklist.md), [contributing](CONTRIBUTING.md), [security](SECURITY.md), and [changelog](CHANGELOG.md). These are preparation for a public release, not evidence it has been published or reviewed independently.
+
+An optional provider illustration is in [`examples/openai_adapter.py`](examples/openai_adapter.py). It requires a separately installed SDK and the user's own provider credentials; no credentials belong in this repository. Model names and SDK behavior can change, so validate an adapter against its provider's current documentation before using it.
+
+For release mechanics, see [PyPI Trusted Publishing setup](docs/publishing.md). No API token is needed in GitHub Actions. A GitHub release or a pending publisher alone is not proof that PyPI published the package; check the PyPI project and install before relying on it.
+
+### Optional evidence reranking
+
+Pass `rerank=lambda query, evidence: ...` when you have a trusted reranking function. The function receives the initially retrieved evidence and must return a permutation of those same objects; new or rewritten passages are rejected. Citation IDs are reassigned after the new order. Reranking cannot recover evidence absent from the initial `top_k`, and does not check whether an answer is true. Arabic normalization includes common hamza, ta marbuta, alef maqsura and Persian keyboard letter variants; it is still not Arabic stemming or morphological analysis.
+
+### Retrieval evaluation
+
+Use `RetrievalCase` and `evaluate_retrieval` with independently labeled relevant source IDs to measure recall@k, hit rate@k, and reciprocal rank. See [evaluation guide](docs/evaluation.md). These metrics do not validate generated claims or source credibility.

newsrag-0.2.0/SECURITY.md

@@ -0,0 +1,5 @@
+# Security
+
+Do not submit secrets or unpublished material in a public issue. Contact the maintainer privately through their verified repository profile for a suspected vulnerability. No private security contact address has been supplied yet.
+
+Source text is untrusted. A generator may still obey instructions embedded in retrieved passages despite prompt wording; review outputs and avoid passing confidential data to external providers without permission. The library does not fetch URLs; applications that do must defend against server-side request forgery, unexpected redirects, oversized responses and licensing restrictions. JSON snapshots contain original text without encryption, so store them only on secured devices and back them up according to newsroom policy.

newsrag-0.2.0/docs/architecture.md

@@ -0,0 +1,26 @@
+# Architecture and contract
+
+`Source` is an immutable record with required ID, title and text. Optional metadata includes original URL, publisher, source type, language, timezone-aware publication and access timestamps. Journalists should keep the source document separately; `Source` metadata is a claim about provenance, not a signature proving authorship.
+
+`NewsroomRAG.add` chunks the text and keeps exact Python character offsets. It tokenizes basic Arabic and English, then scores matching chunks with a BM25-style lexical score. An optional caller-provided embedding function provides vectors for cosine ranking. A bounded, optional recency boost uses the publication date only; an unknown publication date is never filled in. `search` returns top passages with references (`E1`, `E2`) assigned per request. References are not durable IDs; persist the source ID, URL and offsets for an audit trail.
+
+`ask` retrieves first. With no generator it returns passages only. With a generator it sends excerpts and citation instructions to the caller's model and checks whether bracketed reference numbers in the returned text exist in the retrieved set. It does not decide whether claims follow from evidence. A reviewer must open the original source, check date and context, inspect the quote and get independent corroboration for consequential claims.
+
+`source_from_html` accepts already-fetched HTML and strips basic script/style content. It does not fetch or verify websites. `source_from_pdf` extracts text from caller-supplied bytes using optional pypdf; it is not OCR. `save_sources` stores original text and metadata as JSON, not embeddings. `load_sources` recomputes embeddings when given a configured rag instance. Storage is local and unencrypted.
+
+## Provider adapter shape
+
+```python
+from newsrag import NewsroomRAG
+
+rag = NewsroomRAG(
+    embed=lambda texts: embedding_api(texts),  # list[str] -> list[list[float]]
+    generate=lambda prompt: generation_api(prompt),  # str -> str
+)
+```
+
+A provider must return one vector per input and use stable vector dimensions. Avoid changing the embedding model for a partially indexed corpus; build a fresh index if the model changes. Calls are synchronous and exceptions from providers propagate. Do not send source text to a provider without a valid editorial and privacy basis.
+
+## Optional reranker
+
+Pass `rerank=lambda query, evidence: ...` to `NewsroomRAG` to reorder the retrieved top-k passages. It receives a tuple of `Evidence` and must return a permutation of those exact objects; it cannot add or rewrite evidence. References are reassigned to `E1`, `E2`, etc. after reordering. Reranking applies only to the initially retrieved candidates; increase `top_k` when a larger candidate pool is needed. A reranker may send passages to an outside provider, so treat that adapter as a separate privacy and trust decision. Reordering does not verify any claim.

newsrag-0.2.0/docs/evaluation.md

@@ -0,0 +1,17 @@
+# Evaluate retrieval on labeled questions
+
+This small harness measures whether manually labeled source IDs appear in the first `k` distinct sources. It reports recall@k, hit rate@k, mean reciprocal rank, and missed queries. Labels need to be made by a human independently of the model's ranking. Keep an evaluation set separate from tuning. These scores say nothing about whether generated prose is true, citations support claims, sources are credible, or the dataset represents a real newsroom.
+
+```python
+from newsrag import NewsroomRAG, Source, RetrievalCase, evaluate_retrieval
+
+rag = NewsroomRAG().add(
+    Source("original", "Original statement", "The ministry announced a pilot."),
+    Source("followup", "Follow-up", "The pilot was extended in October."),
+)
+cases = [RetrievalCase("What was announced?", frozenset({"original"})),
+         RetrievalCase("What happened in October?", frozenset({"followup"}))]
+print(evaluate_retrieval(rag, cases, top_k=2).as_dict())
+```
+
+Use source IDs that exist in the corpus; missing IDs count as misses. An embedding or reranking adapter configured on the RAG object is used during scoring and may make external calls, including transmitting queries and source text. Do not put confidential test material in an external provider without permission. Compare against an independent held-out set, track failures by topic and language, and inspect original sources before drawing editorial conclusions.

newsrag-0.2.0/docs/publishing.md

@@ -0,0 +1,15 @@
+# Publishing with PyPI Trusted Publishing
+
+This repository includes `.github/workflows/release.yml`. A version tag beginning with `v` triggers checks, builds a wheel and source distribution, then asks PyPI to publish through GitHub OIDC. No API token is stored in the repository.
+
+The owner must complete these first:
+
+1. Confirm the public repository under the correct GitHub account, its exact `owner/newsrag` path, and the desired license. The account/owner cannot be guessed from a similar name.
+2. In GitHub, create an environment named `pypi`. Protect it with required reviewers if appropriate, and restrict deployment to version tags if possible. Do not approve a release until the code and artifacts have been reviewed.
+3. Sign into the intended PyPI account. In the account sidebar, open **Publishing** and add a **pending GitHub publisher** for project `newsrag`: owner is the exact GitHub account or organization name, repository `newsrag`, workflow filename `release.yml`, environment `pypi`. A pending publisher does not reserve the name.
+4. Review the release, ensure the version in `setup.py` and `src/newsrag/__init__.py` matches the intended tag, then create and push an annotated tag such as `v0.2.0` on the reviewed commit. The release workflow will ask for the `pypi` environment approval and publish only after approval.
+5. Verify the workflow succeeded and check the actual project on PyPI. An accepted pending publisher or a green build alone is not proof of publication.
+
+Official PyPI instructions: https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/ and https://docs.pypi.org/trusted-publishers/using-a-publisher/ .
+
+If a token was created for a previous manual-upload plan, it is not used by this workflow. The owner can revoke it in PyPI account settings after this route works.

newsrag-0.2.0/docs/release-checklist.md

@@ -0,0 +1,11 @@
+# Release checklist
+
+- [ ] Confirm package name, repository visibility, maintainer identity and license with the owner.
+- [ ] Review code and dependencies for secrets and unintended private content.
+- [ ] Run Ruff, mypy, tests and build on all supported Python versions.
+- [ ] Verify source distribution and wheel install in clean environments.
+- [ ] Test real multilingual corpora, long files, citation failure cases and source-date edge cases.
+- [ ] Inspect dependency advisories, package metadata and README rendering.
+- [ ] Publish a signed/tagged release and verify the public repository and package index pages.
+
+The locally tested prototype is not a substitute for this release review. Do not claim automated fact checking or universal provider compatibility without tests for those claims.

newsrag-0.2.0/examples/html_and_snapshot.py

@@ -0,0 +1,9 @@
+from newsrag import NewsroomRAG, load_sources, save_sources, source_from_html
+
+source = source_from_html(id="example", title="Demo", url="https://example.org/demo",
+                          html="<article><p>The pilot has not launched publicly.</p></article>")
+rag = NewsroomRAG().add(source)
+print(rag.search("pilot"))
+save_sources(rag, "newsroom-sources.json")
+restored = load_sources("newsroom-sources.json")
+print(restored.ask("Was there a public launch?").as_dict())

newsrag-0.2.0/examples/openai_adapter.py

@@ -0,0 +1,25 @@
+"""Optional OpenAI SDK example. Install `openai` separately and set OPENAI_API_KEY.
+
+This is an example adapter, not a bundled dependency or an endorsement.
+Only use it with sources you may send to a third-party service.
+"""
+from newsrag import NewsroomRAG
+
+
+def make_rag(client):
+    def embed(texts: list[str]) -> list[list[float]]:
+        result = client.embeddings.create(model="text-embedding-3-small", input=texts)
+        return [item.embedding for item in sorted(result.data, key=lambda item: item.index)]
+
+    def generate(prompt: str) -> str:
+        result = client.responses.create(model="gpt-4.1-mini", input=prompt)
+        return result.output_text
+
+    return NewsroomRAG(embed=embed, generate=generate)
+
+
+if __name__ == "__main__":
+    from openai import OpenAI
+
+    rag = make_rag(OpenAI())
+    print("Adapter ready. Add your authorized Source records before calling ask().")

newsrag-0.2.0/examples/quickstart.py

@@ -0,0 +1,13 @@
+from datetime import datetime, timezone
+
+from newsrag import NewsroomRAG, Source
+
+rag = NewsroomRAG(recency_half_life_days=30).add(
+    Source(id="press-release", title="Official announcement", text="The company announced a pilot on 10 September. It did not announce a public launch.", url="https://example.org/official", publisher="Example Corp", published_at=datetime(2026, 9, 10, tzinfo=timezone.utc)),
+    Source(id="interview", title="Reporter notes", text="In an interview, the editor said the pilot still needs independent testing.", source_type="interview"),
+)
+print(rag.ask("Was there a public launch?").as_dict())
+# Plug in ANY synchronous provider with two callables:
+# rag = NewsroomRAG(embed=lambda texts: your_embedding_client(texts),
+#                   generate=lambda prompt: your_llm_client(prompt))
+# print(rag.add(...).ask("What changed? [in Arabic or English]").as_dict())

newsrag-0.2.0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"