unmasker 0.1.0__tar.gz

unmasker-0.1.0/.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
+
+[*.{md,yml,yaml,json,toml,svg}]
+indent_size = 2
+
+# Specimens are artefacts under test. Nothing may normalise their bytes.
+[tests/specimens/**]
+insert_final_newline = false
+trim_trailing_whitespace = false

unmasker-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,94 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.12", "3.13"]
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: python -m pip install -e ".[dev]"
+      # One test shells out to poppler, and it is the one that proves this tool
+      # says something the obvious answer does not: `pdftotext` reports no
+      # annotations, so a comment carrying "Do not minute this" is invisible to
+      # the first thing anybody reaches for. It skips where poppler is absent,
+      # so installing it on one platform is the difference between that claim
+      # being checked somewhere and checked nowhere.
+      - name: Poppler (for the one test that needs it)
+        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get update && sudo apt-get install -y poppler-utils
+      # `ruff check`, not `ruff format --check`. This project lints and does
+      # not auto-format: the layout of the report, the specimens' XML and the
+      # docstrings are placed deliberately, and a formatter run at publication
+      # time would rewrite twelve files nobody asked it to. CI asserts the
+      # standard the project actually holds, which is the one in README.md.
+      - name: Ruff
+        run: ruff check .
+      # The specimens are the test suite here and they are committed, so this
+      # runs the real thing on every platform. `--ocr` is not exercised: it
+      # needs ghostscript and tesseract, and `tests/test_ocr.py` skips rather
+      # than failing over their absence. Every test that shells out is guarded
+      # the same way - the first CI run failed because one of them was not, and
+      # nothing on this machine could have told us.
+      - name: Tests
+        run: pytest
+
+  types:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: python -m pip install -e ".[dev]"
+      - name: Mypy
+        run: mypy
+
+  package:
+    runs-on: ubuntu-latest
+
+    # The tests above run from a checkout, so they prove nothing about the
+    # artifact people actually install. A wheel that cannot be built, or that
+    # is missing a file, fails only for them and only after publication.
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build
+        run: |
+          python -m pip install build twine
+          python -m build
+          twine check dist/*
+      - name: The typing marker has to be inside the wheel
+        run: |
+          python - <<'CHECK'
+          import glob, zipfile
+          names = zipfile.ZipFile(glob.glob("dist/*.whl")[0]).namelist()
+          assert "unmasker/py.typed" in names, names
+          CHECK
+      # Installing it and running it is the part that catches an import which
+      # type-checks and does not exist at runtime. Mypy passing is not the
+      # program running.
+      - name: Install what was built and run it
+        run: |
+          python -m pip install dist/*.whl
+          unmasker --version

unmasker-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,70 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+
+    # Trusted Publishing: PyPI verifies this workflow's identity over OIDC, so
+    # there is no API token to store, rotate or leak. `id-token: write` is what
+    # mints that identity and is the only permission this needs.
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # A tag is a claim about which version is being released, and nothing
+      # else checks it. Tagging v0.2.0 against a pyproject that still says
+      # 0.1.0 publishes 0.1.0 under a tag that says otherwise, and PyPI will
+      # not let that file name be reused once it is wrong.
+      - name: The tag has to be the version being built
+        run: |
+          python - <<'CHECK'
+          import os, pathlib, re, sys
+
+          tag = os.environ["GITHUB_REF_NAME"]
+          text = pathlib.Path("pyproject.toml").read_text(encoding="utf-8")
+          declared = re.search(r'^version = "([^"]+)"', text, re.MULTILINE).group(1)
+
+          if tag != f"v{declared}":
+              sys.exit(f"tag {tag} does not match pyproject version {declared}")
+          print(f"{tag} matches pyproject {declared}")
+          CHECK
+
+      - name: Build
+        run: |
+          python -m pip install build twine
+          python -m build
+          twine check dist/*
+
+      # The same assertion CI makes on every push. It is repeated here because
+      # this is the artifact that actually reaches people, and a wheel missing
+      # its typing marker is invisible until somebody imports the package.
+      - name: The typing marker has to be inside the wheel
+        run: |
+          python - <<'CHECK'
+          import glob, zipfile
+          names = zipfile.ZipFile(glob.glob("dist/*.whl")[0]).namelist()
+          assert "unmasker/py.typed" in names, names
+          CHECK
+
+      # Install the built wheel in a clean interpreter and run the console
+      # script. Mypy passing is not the program running, and neither is a green
+      # test suite executed from a checkout.
+      - name: Install what was built and run it
+        run: |
+          python -m pip install dist/*.whl
+          unmasker --version
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

unmasker-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+.venv/
+venv/
+.vscode/
+.idea/
+.DS_Store
+.coverage
+htmlcov/
+
+# Review archives built with `git archive`: a snapshot of the repository stored
+# inside that repository is redundant the day it is made and wrong after it.
+*.zip
+
+# Specimens are the test suite here and DO belong in git - see CLAUDE.md.
+# Only genuinely private material is excluded.
+private-samples/
+
+# Working notes, not documentation, and the only two files here that are not
+# published. HANDOFF.md is the running log of what was tried and what it cost,
+# and it names files in a sibling checkout that are private. CLAUDE.md is how
+# to work in this repository rather than what the tool does.
+#
+# CONTRIBUTING.md is tracked, deliberately: its rules are each named after the
+# failure that produced them, and that is worth publishing.
+HANDOFF.md
+CLAUDE.md

unmasker-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+# Lint only. This project does not auto-format: the report layout, the
+# specimens' XML and the docstrings are placed deliberately, and `ruff format`
+# would rewrite a dozen files nobody asked it to. CI runs the same check.
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+        args: [--fix]
+
+  # `if False:` is how a watched failure gets parked while the test that proves
+  # it is written, and ruff's selected rules (E, F, I, UP, B) do not flag it.
+  # One such guard reached the working tree in `jpeg.py` and disabled the
+  # start-of-scan check, so `dimensions()` read a frame header out of
+  # compressed data. The test caught it; nothing else would have.
+  - repo: local
+    hooks:
+      - id: no-parked-guards
+        name: no `if False:` left in src/
+        entry: 'if False:'
+        language: pygrep
+        files: ^src/.*\.py$

unmasker-0.1.0/CHANGELOG.md

@@ -0,0 +1,45 @@
+# Changelog
+
+Notable changes to `unmasker`, newest first. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the versions
+follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+`--json` carries its own `schema` field, which is versioned separately from the
+release: `unmasker.scan/1` for one file, `unmasker.survey/1` for a folder. A
+consumer should read that rather than the release number, because the release
+moves whenever anything does and the schema moves only when the shape changes.
+
+## [0.1.0] - 2026-09-05
+
+First release.
+
+### Added
+
+- 25 detectors across seven groups: covered and under-image text, invisible and
+  low-contrast text, off-page text; zero-width, bidi, tag and mixed-script
+  characters; hidden sheets, rows, columns, filtered rows and tracked cell
+  changes; hidden slides and speaker notes; stale EXIF thumbnails; tracked
+  deletions, comments, revision history and metadata leaks; and — with `--ocr` —
+  text that renders but does not extract, and text that extracts but does not
+  render.
+- Readers for PDF, DOCX, ODT, XLSX, ODS, PPTX, ODP, JPEG and plain text,
+  dispatched by content rather than by extension.
+- Folder surveys: point it at a directory and it reports which files to open
+  next, without ranking them against each other.
+- `--html` and `--md` reports, both escaping every value they quote, because
+  everything quoted came out of a document somebody else wrote.
+- `--json` for pipelines, with a `schema` field naming the shape and a
+  `searched` field distinguishing *nothing was found* from *there was nothing to
+  search*.
+- Three exit statuses: `0` clean, `1` findings, `2` unreadable — because a file
+  that could not be read is not a file that came back clean.
+- `py.typed`, so the annotations are visible to anything importing `unmasker`
+  as a library.
+
+### Fixed
+
+- `jpeg.dimensions()` walked past the start-of-scan marker and read a `0xFFC0`
+  out of the entropy-coded data, reporting a picture the size of noise. It now
+  stops at `SOS`/`EOI` the way `_segments()` already did.
+
+[0.1.0]: https://github.com/osint-shifu/unmasker/releases/tag/v0.1.0

unmasker-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,183 @@
+# Contributing
+
+Thank you for looking. This project has a small number of rules and every one
+of them was paid for by a bug. They are worth reading before writing code,
+because a change that breaks one of them will be asked to change back.
+
+## Setting up
+
+```bash
+python3 -m venv .venv
+.venv/bin/pip install -e ".[dev]"
+.venv/bin/pytest
+.venv/bin/ruff check .
+```
+
+Python 3.10 or later. One runtime dependency, `pypdf`.
+
+## The one thing this tool must never do
+
+**State something the evidence does not support.** No "this document was
+manipulated". Show what a human sees, show what a machine reads, name the gap,
+and stop.
+
+A reader who disagrees with a finding must be able to see why. A tool that
+concludes for its reader is a tool whose conclusion has to be trusted blindly,
+and nobody should trust this one blindly.
+
+Everything below follows from that.
+
+## A new detector needs a specimen a real producer wrote
+
+This is the rule that matters most, and it is not negotiable.
+
+A fixture built from a specification proves that your reader agrees with your
+reading of the spec. It does not prove it agrees with LibreOffice. The sibling
+project `filetrail` had a HEIC reader with a fully green test suite that
+decoded nothing at all on every real file, because every fixture had been built
+from the standard and no real encoder writes files that way.
+
+So:
+
+1. Write a **builder** in `tests/specimens/sources/` that drives a real
+   producer — `soffice`, `google-chrome`, `gs`, `tesseract`, `exiftool` — to
+   write the file.
+2. Commit the produced file. The specimens **are** the test suite here, which
+   is why they are in git rather than gitignored.
+3. Write a `.md` beside it recording what produced it, what a human sees, what
+   is actually in the file, and **what it does not carry**.
+
+Everything in a specimen must be invented. No real documents, no real people;
+the e-mail domain is `example.org`, which RFC 2606 reserves for the purpose.
+
+If the producer does something the specification does not describe — and it
+will — that is the most valuable thing in your pull request. Write it down.
+
+## Watch the test fail first
+
+Write the test, run it, read the failure, *then* implement. A test written
+afterwards passes against broken code, and this project has caught itself
+doing that twice.
+
+Check the failure is the one you expected. An `ImportError` is not evidence
+that your test works.
+
+## Break your own rule and see whether the suite notices
+
+Before opening a pull request, take each claim your docstrings make, break it
+in the source, and run the suite. Anything that stays green is a claim nothing
+is holding.
+
+This has found something every single time it has been run here — most
+recently a guard that could not fail because the fixture feeding it had a
+default that made the question moot. If you find one, the fix is a test, and
+that test is the interesting part of the change.
+
+## Report a line, never a show-operation
+
+If you are touching the PDF detectors: producers disagree wildly about how much
+text one drawing operation carries. Chrome writes one glyph per `Tj`; tesseract
+writes one operation per word. Group by **line on the page**, using
+`detectors._lines`, or a single hidden line becomes eighty-seven findings.
+
+That rule has been broken five times in this codebase.
+`test_no_detector_reports_per_show_operation` now asks the question directly.
+
+## Do not rank findings against each other
+
+A page can have a bar over its text *and* invisible characters *and* stale
+metadata. Those are three findings, not one winner. `filetrail` printed the
+winner once, and a geotagged photograph that had been downloaded reported its
+URL and no GPS at all.
+
+Nothing here sorts by strength, and `Finding` carries no score.
+
+## "Nothing found" has two meanings
+
+It can mean *searched, and it is not there*. It can mean *there was nothing to
+search*. A reader who confuses those has drawn a conclusion the tool never
+supported.
+
+Carry the difference in the data rather than reconstructing it later — a PDF
+with no content stream to parse is not a PDF with nothing hidden, and the
+report has to be able to say which it met.
+
+## The name of a field is evidence
+
+`LibreOffice/24.2.7.2$Linux_X86_64` contains a dotted quad. Pattern-matching
+alone reports an IP address — but the field is called `Producer`, so it is a
+version. Every metadata field's meaning is decided by its name **and its
+container**: a PDF's `/Creator` is an application, an OOXML `dc:creator` is a
+person.
+
+Context you already have beats a cleverer pattern.
+
+## Colour and words
+
+**Colour encodes how the tool knows, never how bad the finding is.** Three
+classes — `direct`, `circumstantial`, `self-reported` — and nothing else in
+the output may use colour.
+
+Prefer a word a reader can argue with over a number that implies a precision
+you do not have. `55` reads as a probability; it never was one.
+
+## Modes are commands, not flags
+
+If an option ignores most of the other options and is mutually exclusive with
+its neighbours, it is a subcommand. `--json` changes the shape of the output,
+not what the tool did, which is what keeps it a flag.
+
+## Output
+
+- **Nothing is truncated.** A value too long for the line wraps. An ellipsis
+  sends the reader to fetch the value another way.
+- **Nothing draws past the width it declared.**
+- **Size a column per section, not once per screen.** One global width makes a
+  layout look *almost* aligned, which reads worse than not aligning it.
+- A description that restates its flag teaches the reader to skip
+  descriptions, and once they skip one they skip the rest.
+- Every command the tool prints must run in the shell that printed it. A screen
+  disproved by the first thing a reader tries loses credibility for everything
+  else on it.
+
+The report's layout and palette are the sibling project `filetrail`'s, and its
+`DESIGN.md` is the design language for both — **read it before inventing a
+second one.** The same goes for anything it already solves: its embedded
+metadata sources read PDF, OOXML, ODF, legacy OLE2 and EXIF, and were read
+before the metadata reader here was written.
+
+What is deliberately *not* shared is the identity. `filetrail` is a terminal;
+`unmasker` is the document the terminal is looking at.
+
+## Commits
+
+Prose, and no trailers. Say what changed and **why** — the why is the part
+that cannot be recovered from the diff. If a change corrects an earlier
+assumption, say what the assumption was; those are the most useful messages in
+this repository's history.
+
+Stage deliberately. `git add -A` once swept a build archive into two commits of
+the sibling project and it reached a public repository.
+
+## What a good pull request looks like
+
+- a specimen a producer wrote, with its `.md`
+- a test you watched fail
+- the producer fact you measured, written down where the next person will hit it
+- the gap your change does *not* close, named rather than left implied
+
+## Reporting a bug
+
+The most useful report is a file. If it is not one you can share — and in this
+line of work it usually is not — a builder that produces something with the
+same shape is just as good, and better for everyone.
+
+If a finding is wrong, say what the tool printed and what the document actually
+shows. A false positive is a bug of the same size as a miss here: a report a
+reader learns to scroll past has stopped working.
+
+## Security
+
+See [`SECURITY.md`](SECURITY.md). In short: this tool parses hostile files by
+design, so a crash or a hang on a malformed document is a bug worth reporting
+privately first.

unmasker-0.1.0/LICENSE

@@ -0,0 +1,133 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the
+copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other
+entities that control, are controlled by, or are under common control with
+that entity. For the purposes of this definition, "control" means (i) the
+power, direct or indirect, to cause the direction or management of such
+entity, whether by contract or otherwise, or (ii) ownership of fifty percent
+(50%) or more of the outstanding shares, or (iii) beneficial ownership of
+such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation source,
+and configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation
+or translation of a Source form, including but not limited to compiled object
+code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form,
+made available under the License, as indicated by a copyright notice that is
+included in or attached to the work.
+
+"Derivative Works" shall mean any work, whether in Source or Object form,
+that is based on (or derived from) the Work and for which the editorial
+revisions, annotations, elaborations, or other modifications represent, as a
+whole, an original work of authorship. For the purposes of this License,
+Derivative Works shall not include works that remain separable from, or
+merely link (or bind by name) to the interfaces of, the Work and Derivative
+Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original
+version of the Work and any modifications or additions to that Work or
+Derivative Works thereof, that is intentionally submitted to Licensor for
+inclusion in the Work by the copyright owner or by an individual or Legal
+Entity authorized to submit on behalf of the copyright owner.
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on
+behalf of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this
+License, each Contributor hereby grants to You a perpetual, worldwide,
+non-exclusive, no-charge, royalty-free, irrevocable copyright license to
+reproduce, prepare Derivative Works of, publicly display, publicly perform,
+sublicense, and distribute the Work and such Derivative Works in Source or
+Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this
+License, each Contributor hereby grants to You a perpetual, worldwide,
+non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this
+section) patent license to make, have made, use, offer to sell, sell, import,
+and otherwise transfer the Work, where such license applies only to those
+patent claims licensable by such Contributor that are necessarily infringed
+by their Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted.
+
+If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this
+License for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or
+Derivative Works thereof in any medium, with or without modifications, and
+in Source or Object form, provided that You meet the following conditions:
+
+(a) You must give any other recipients of the Work or Derivative Works a
+copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices stating
+that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works that You
+distribute, all copyright, patent, trademark, and attribution notices from
+the Source form of the Work, excluding those notices that do not pertain to
+any part of the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its distribution,
+then any Derivative Works that You distribute must include a readable copy
+of the attribution notices contained within such NOTICE file, excluding those
+notices that do not pertain to any part of the Derivative Works, in at least
+one of the following places: within a NOTICE text file distributed as part of
+the Derivative Works; within the Source form or documentation, if provided
+along with the Derivative Works; or, within a display generated by the
+Derivative Works, if and wherever such third-party notices normally appear.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any
+Contribution intentionally submitted for inclusion in the Work by You to the
+Licensor shall be under the terms and conditions of this License, without
+any additional terms or conditions.
+
+6. Trademarks. This License does not grant permission to use the trade names,
+trademarks, service marks, or product names of the Licensor, except as
+required for reasonable and customary use in describing the origin of the
+Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in
+writing, Licensor provides the Work (and each Contributor provides its
+Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied, including, without limitation, any warranties
+or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in
+tort (including negligence), contract, or otherwise, unless required by
+applicable law, shall any Contributor be liable to You for damages, including
+any direct, indirect, special, incidental, or consequential damages of any
+character arising as a result of this License or out of the use or inability
+to use the Work.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work
+or Derivative Works thereof, You may choose to offer, and charge a fee for,
+acceptance of support, warranty, indemnity, or other liability obligations
+and/or rights consistent with this License. However, in accepting such
+obligations, You may act only on Your own behalf and on Your sole
+responsibility, not on behalf of any other Contributor.
+
+END OF TERMS AND CONDITIONS