ingestforge 0.4.0a6__tar.gz

ingestforge-0.4.0a6/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"

ingestforge-0.4.0a6/.github/workflows/ci.yml

@@ -0,0 +1,55 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  test:
+    name: py${{ matrix.python-version }} / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -e ".[dev]"
+      - run: python -m ruff check .
+      - run: python -m ruff format --check .
+      - run: python -m mypy src/ingestforge
+      - run: python -m compileall -q src tests
+      - run: python -m pytest -q
+      - run: ingestforge --help
+      - run: ingestforge version
+
+  package:
+    name: package hygiene
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+          cache: pip
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -e ".[dev]"
+      - run: python -m mypy src/ingestforge
+      - run: python scripts/clean_release_artifacts.py
+      - run: python scripts/release_hygiene_check.py
+      - run: python -m pip_audit --skip-editable
+      - run: python -m build
+      - run: python -m twine check dist/*
+      - run: python -m pip install --force-reinstall dist/*.whl
+      - run: python -c "import ingestforge; print(ingestforge.__version__)"
+      - run: ingestforge --help

ingestforge-0.4.0a6/.github/workflows/publish.yml

@@ -0,0 +1,41 @@
+name: Publish
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: pip
+      - run: python -m pip install --upgrade pip build twine
+      - run: python -m build --sdist --wheel
+      - run: python -m twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/*
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ingestforge
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          # For TestPyPI, configure repository-url in a separate workflow/environment.
+          skip-existing: true

ingestforge-0.4.0a6/CHANGELOG.md

@@ -0,0 +1,45 @@
+# Changelog
+
+## 0.4.0a6
+
+- Added optional Trafilatura-backed HTML extraction with `extraction.backend: auto|internal|trafilatura`.
+- Added `ingestforge[extraction]` optional dependency extra.
+- Recorded extraction backend diagnostics and source `extraction_method` in pipeline output.
+- Documented that the alpha claim gate remains exact-match based, not semantic verification.
+- Added config-driven `ai.source_language` and `ai.target_languages` with structural BCP 47-style validation and no provider-language allowlist.
+- Hardened `strict_live_template.yaml` by rejecting empty domain values such as unset `${env:INGESTFORGE_ALLOWED_DOMAIN}`.
+- Removed hard-coded live model defaults from the strict profile; offline defaults use mock/mock.
+- Added generic provider model policy: model IDs are opaque strings and live providers require explicit non-empty models when external AI calls are enabled.
+- Added provider doctor diagnostics for local payload contract validation.
+- Added current/legacy Gemini structured-output payload styles behind explicit config.
+- Made DeepSeek thinking control explicit and added OpenAI-SDK-compatible extra_body helper.
+- Fixed approximate token chunking to slice original text spans for multilingual source preservation.
+- Added provider contract matrix and release provider checklist.
+- Documented SSRF validate-mode DNS-rebinding/TOCTOU limitations honestly.
+- Added a real packaged prompt registry so `ai.prompt_version` resolves to `src/ingestforge/prompts/*.j2` at runtime and unknown prompt versions fail clearly.
+- Polished CLI boolean flags so release help does not expose confusing `--no-no-*` or `--no-offline` aliases.
+- Polished release metadata for PyPI and Zenodo: aligned `CITATION.cff` license with MIT, added ORCID/abstract/keywords, made README license/citation links absolute, and added a PyPI environment to Trusted Publishing.
+
+## 0.4.0a5
+
+- Hardened SSRF strict-allowlist semantics and documentation.
+- Added robust HTML byte decoding metadata for multilingual pages.
+- Replaced misleading token/word chunking with explicit tokenizer abstraction.
+- Fixed package hash ordering so dataset records are included before hashing.
+- Made Python API write semantics explicit through `write_dataset`.
+- Added conservative claim-gate statuses and content-policy extension points.
+- Added `.gitignore` and updated release hygiene expectations.
+
+## 0.4.0a4
+
+- Support target narrowed and documented as Python 3.11 through 3.14.
+- GitHub CI matrix expanded to Linux, Windows, and macOS across Python 3.11, 3.12, 3.13, and 3.14.
+- Added short public API helpers: `ingestforge.pipeline()` and `ingestforge.ingest_url()`.
+- Updated packaging metadata, Ruff target, MyPy target, docs, examples, release checks, and hygiene rules for public alpha distribution.
+- Removed generated caches and old build artifacts from source delivery.
+
+## 0.4.0a1
+
+- First clean public alpha under the IngestForge name.
+- Brand-neutral package/import/CLI layout.
+- Standard package contract, provenance ledger, evidence gate, reflection gate, data card generation, RAG export, and generic REST destination adapter.

ingestforge-0.4.0a6/CITATION.cff

@@ -0,0 +1,24 @@
+cff-version: 1.2.0
+message: "If you use IngestForge, please cite it."
+title: "IngestForge: Config-driven AI content ingestion and provenance-aware RAG dataset building"
+authors:
+  - family-names: "Jamei"
+    given-names: "Parvaz"
+    orcid: "https://orcid.org/0009-0002-9980-270X"
+abstract: "IngestForge is a lightweight, config-driven Python library for safe web ingestion, optional Trafilatura-backed extraction, evidence-gated AI article generation, provenance ledgers, RAG dataset chunks, data cards, and configurable multi-provider exports."
+keywords:
+  - "AI content ingestion"
+  - "RAG"
+  - "provenance"
+  - "web extraction"
+  - "structured output"
+  - "audit trail"
+  - "dataset generation"
+  - "OpenAI"
+  - "Gemini"
+  - "DeepSeek"
+version: "0.4.0a6"
+date-released: "2026-05-23"
+license: "MIT"
+repository-code: "https://github.com/Parvaz-Jamei/IngestForge"
+type: software

ingestforge-0.4.0a6/CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Code of Conduct
+
+Be respectful, evidence-focused, and constructive. This project is intended for safe and auditable ingestion workflows.

ingestforge-0.4.0a6/CONTRIBUTING.md

@@ -0,0 +1,7 @@
+# Contributing
+
+1. Create a virtual environment.
+2. Install with `pip install -e .[dev]`.
+3. Run `ruff check .`, `ruff format --check .`, `pytest -q`, and `compileall`.
+4. Keep public code brand-neutral and config-driven.
+5. Do not commit secrets or generated run artifacts.

ingestforge-0.4.0a6/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Parvaz Jamei
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ingestforge-0.4.0a6/MANIFEST.in

@@ -0,0 +1,8 @@
+include README.md LICENSE CHANGELOG.md SECURITY.md CONTRIBUTING.md CODE_OF_CONDUCT.md CITATION.cff pyproject.toml pytest.ini
+recursive-include src/ingestforge/prompts *.j2
+recursive-include src/ingestforge/profiles *.yaml
+include src/ingestforge/py.typed
+recursive-include docs *.md
+recursive-include examples *.md *.py
+recursive-include scripts *.py
+recursive-include .github *.yml