tabayyan 0.5.1__tar.gz

tabayyan-0.5.1/.dockerignore

@@ -0,0 +1,9 @@
+tests/
+benchmarks/
+docs/
+.github/
+.git/
+__pycache__/
+*.pyc
+.ruff_cache/
+.pytest_cache/

tabayyan-0.5.1/.github/workflows/docs.yml

@@ -0,0 +1,19 @@
+name: docs
+
+on:
+  push:
+    branches: [main]
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install mkdocs-material
+      - run: mkdocs gh-deploy --force

tabayyan-0.5.1/.github/workflows/release.yml

@@ -0,0 +1,36 @@
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build
+      - run: python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write   # OIDC trusted publishing — no API token needed
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

tabayyan-0.5.1/.github/workflows/tests.yml

@@ -0,0 +1,36 @@
+name: tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.11", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e ".[dev]"
+      - name: Lint
+        run: ruff check .
+      - name: Test
+        run: pytest -q
+
+  integration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install with Presidio extra
+        run: pip install -e ".[dev,presidio]"
+      - name: Run integration tests
+        run: pytest tests/test_presidio_integration.py -q

tabayyan-0.5.1/.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+.env
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+.idea/
+.vscode/
+.DS_Store

tabayyan-0.5.1/.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+  # Use Tabayyan itself to block accidental PII in commits:
+  - repo: local
+    hooks:
+      - id: tabayyan
+        name: tabayyan PII scan
+        entry: tabayyan scan --fail-on-find --min-confidence high
+        language: system
+        types: [text]
+        exclude: '^(tests/|benchmarks/|docs/)'

tabayyan-0.5.1/.pre-commit-hooks.yaml

@@ -0,0 +1,6 @@
+- id: tabayyan
+  name: tabayyan (Saudi PII scan)
+  description: Fail the commit if Saudi/PII entities are found in staged text.
+  entry: tabayyan scan --fail-on-find
+  language: python
+  types: [text]

tabayyan-0.5.1/CHANGELOG.md

@@ -0,0 +1,75 @@
+# Changelog
+
+## 0.5.1
+- **Fix:** Arabic comma (U+060C) and other Arabic punctuation no longer
+  corrupt Arabic-name tokenization (tightened the letter range).
+- **Fix:** tokenize/restore now reproduces the original span exactly,
+  including Arabic-Indic digits (vault stores the source span, not the
+  normalized value).
+- **Middleware hardening:** `protect_messages()` building block; handles
+  multimodal/list content, system/tool roles, and streaming (request
+  redacted, stream passed through). Honest 'reference adapter' disclaimer.
+- **Recall benchmark** (`benchmarks/recall.py`): recall under formatting
+  noise + an honest context-free section exposing heuristic limits.
+- **Arabic README**, before/after showcase, and an interactive notebook.
+- Broadened MRN trigger phrasing (recall 0.69 -> 1.0 with context).
+
+## 0.5.0
+- **Middleware + audit** (`Guard`, `AuditLog`): scan -> redact/block -> audit
+  before a prompt leaves for an LLM endpoint. Cross-border transfer flagging
+  (PDPL Art. 29), category-aware blocking, JSONL audit (values withheld by
+  default), and a duck-typed OpenAI/Azure wrapper with tokenize-restore.
+- **Presidio integration** (`tabayyan[presidio]`): validated Saudi/Arabic
+  recognizers (SA_NATIONAL_ID, SA_IQAMA, SA_IBAN, SA_CR, SA_PHONE_NUMBER,
+  MEDICAL_RECORD_NUMBER, PERSON, lookalike domains). Complements Presidio;
+  parity-tested against the standalone engine. Runtime core stays zero-dep.
+- Name detector: added field-label stopwords (الهوية، الآيبان، …) for tighter
+  boundaries in record-style text.
+
+## 0.4.2
+- **IBAN & Luhn cross-validation**: differential tests against
+  `python-stdnum` (dev-only oracle) — Luhn over 20k random samples, IBAN
+  over 12k generated Saudi IBANs + mutations.
+- **Golden vectors**: official card-network test PANs (Visa, Mastercard,
+  Amex, Discover, JCB, Diners) and canonical public example IBANs
+  (SA/GB/DE/FR). Runtime stays zero-dependency.
+
+## 0.4.1
+- **National ID cross-validation**: validator differentially tested against
+  the community reference alhazmy13/Saudi-ID-Validator (MIT; algorithm by
+  Abdul-Aziz Al-Oraij) — 100% agreement on a 50k+ random sample. Updated
+  REFERENCES.md and the checksum disclaimer accordingly.
+
+## 0.4.0
+- **Arabic name detection** (heuristic, context-gated, LOW confidence) — new
+  PERSON category; improves recall on health-sector PII.
+- **Streaming** large-file scan with overlap windows (`tabayyan scan --stream`,
+  `tabayyan.streaming.scan_file`).
+- **Reversible tokenize** redaction mode + `restore()` and a token vault.
+- **Config** (`--config`, `tabayyan.config.Config`): disable/add detectors,
+  custom regex detectors with labels, extend confusables, tune thresholds.
+- **Performance**: rewrote overlap resolution from O(n^2) to O(n log n)
+  (~110x faster on dense input). Added `benchmarks/perf.py`.
+- **Docs**: REFERENCES.md (algorithm provenance), FAQ, threat model, config.
+- Golden-vector tests; National ID disclaimer clarifying it is the
+  community algorithm, not an authoritative spec.
+
+## 0.3.0
+- **Homoglyph / lookalike-domain detection** (opt-in): IDN homograph
+  impersonation via confusable skeletons, mixed-script labels
+  (incl. Arabic+Latin), and edit-distance typosquats against a watchlist.
+  Punycode (`xn--`) labels decoded before analysis. New `tabayyan domains`
+  CLI command.
+- **Benchmark suite** (`benchmarks/run.py`): precision/recall/F1 on a
+  synthetic corpus with hard negatives, plus a naive-regex baseline that
+  quantifies the false positives checksum validation eliminates.
+- **Adoption**: Dockerfile, pre-commit hook, PyPI release workflow (OIDC
+  trusted publishing), MkDocs docs, Makefile.
+
+## 0.2.0
+- Redaction engine: mask / remove / hash / partial.
+- CLI: `scan` and `redact` with stdin/file/dir input, filters, JSON, exit codes.
+
+## 0.1.0
+- Detection core: Saudi (National ID, Iqama, IBAN, CR, mobile, MRN) +
+  generic (email, credit card, IP) detectors with checksum validation.

tabayyan-0.5.1/CODE_OF_CONDUCT.md

@@ -0,0 +1,6 @@
+# Code of Conduct
+
+This project follows the [Contributor Covenant](https://www.contributor-covenant.org/),
+version 2.1. Be respectful, assume good faith, and keep discussion focused on
+the work. Unacceptable behaviour may be reported to the maintainers via a
+private channel; maintainers will respond as promptly as circumstances allow.

tabayyan-0.5.1/CONTRIBUTING.md

@@ -0,0 +1,29 @@
+# Contributing
+
+Thanks for your interest in improving Tabayyan.
+
+## Ground rules
+
+1. **No real data, ever.** All fixtures must be synthetic and generated in
+   `tests/synthetic.py`. PRs containing real National IDs, IBANs, MRNs, or
+   any other real personal data will be rejected.
+2. **Keep the core offline.** The detection engine must not introduce network
+   calls or non-stdlib runtime dependencies. Optional enrichment belongs in a
+   separate, opt-in module.
+3. **Be honest about confidence.** A new detector must state HIGH / MEDIUM /
+   LOW honestly. HIGH is reserved for checksum-backed types.
+
+## Adding a detector
+
+1. Subclass `Detector` in `src/tabayyan/detectors/`.
+2. Return `Match` objects with correct `category` and `confidence`.
+3. Register it in `detectors/__init__.py`.
+4. Add round-trip tests (valid detected, invalid rejected) using synthetic data.
+
+## Dev setup
+
+```bash
+pip install -e ".[dev]"
+pytest -q
+ruff check .
+```

tabayyan-0.5.1/Dockerfile

@@ -0,0 +1,14 @@
+# Minimal image: no third-party runtime deps, CLI as entrypoint.
+FROM python:3.12-slim AS base
+
+WORKDIR /app
+COPY pyproject.toml README.md LICENSE ./
+COPY src ./src
+RUN pip install --no-cache-dir .
+
+# Run as non-root.
+RUN useradd --create-home --uid 10001 tabayyan
+USER tabayyan
+
+ENTRYPOINT ["tabayyan"]
+CMD ["--help"]

tabayyan-0.5.1/LICENSE

@@ -0,0 +1,143 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but not
+      limited to compiled object code, generated documentation, and
+      conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work.
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship.
+
+      "Contribution" shall mean any work of authorship, including the
+      original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work.
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have
+      made, use, offer to sell, sell, import, and otherwise transfer the
+      Work, where such license applies only to those patent claims
+      licensable by such Contributor that are necessarily infringed by
+      their Contribution(s) alone or by combination of their
+      Contribution(s) with the Work to which such Contribution(s) was
+      submitted. If You institute patent litigation against any entity
+      (including a cross-claim or counterclaim in a lawsuit) alleging
+      that the Work or a Contribution incorporated within the Work
+      constitutes direct or contributory patent infringement, then any
+      patent licenses granted to You under this License for that Work
+      shall terminate as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or Derivative
+          Works a copy of this License; and
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work; and
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute
+          must include a readable copy of the attribution notices
+          contained within such NOTICE file.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing
+      the origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or agreed
+      to in writing, Licensor provides the Work (and each Contributor
+      provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES
+      OR CONDITIONS OF ANY KIND, either express or implied, including,
+      without limitation, any warranties or conditions of TITLE,
+      NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR
+      PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2026 Tabayyan contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

tabayyan-0.5.1/Makefile

@@ -0,0 +1,23 @@
+.PHONY: install test lint bench build docker clean
+
+install:
+	pip install -e ".[dev]"
+
+test:
+	pytest -q
+
+lint:
+	ruff check .
+
+bench:
+	python benchmarks/run.py --write
+
+build:
+	python -m build
+
+docker:
+	docker build -t tabayyan:local .
+
+clean:
+	rm -rf build dist *.egg-info .pytest_cache .ruff_cache
+	find . -type d -name __pycache__ -exec rm -rf {} +