watchdog-intel 0.1.0a1__tar.gz

watchdog_intel-0.1.0a1/.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package and dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest

watchdog_intel-0.1.0a1/.github/workflows/claude-review.yml

@@ -0,0 +1,39 @@
+name: Claude Code Review
+
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  review:
+    if: github.event.label.name == 'claude-review' && github.actor == 'tomcardoso'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code Review
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          label_trigger: claude-review
+          claude_args: |
+            --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)"
+          prompt: |
+            REPO: ${{ github.repository }}
+            PR NUMBER: ${{ github.event.pull_request.number }}
+
+            Please review this pull request with a focus on:
+            - Code quality and best practices
+            - Potential bugs or issues
+            - Security implications
+            - Performance considerations
+
+            Provide detailed feedback using inline comments for specific issues.

watchdog_intel-0.1.0a1/.github/workflows/publish.yml

@@ -0,0 +1,57 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install package and dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest
+
+  build:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build sdist and wheel
+        run: |
+          pip install hatch twine
+          hatch build
+          twine check --strict dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for OIDC trusted-publisher auth
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

watchdog_intel-0.1.0a1/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.pyc
+*.pyo
+.DS_Store
+.claude/
+dev/batch_results.json
+*.egg-info/
+dist/
+build/
+.venv/

watchdog_intel-0.1.0a1/CLAUDE.md

@@ -0,0 +1,104 @@
+# Watchdog — developer notes
+
+## Testing
+
+Write tests for new features and any non-trivial function. The suite lives in `tests/` and runs with:
+
+```
+pipx run pytest
+```
+
+Tests use `tmp_path` and `monkeypatch` to redirect `WATCHDOG_HOME`, `PROJECTS_FILE`, and `CONFIG_FILE` away from the real home directory — patch all three when testing anything that touches the registry or projects list. See the `wdg_home` and `configured` fixtures in `tests/test_cli.py` for the pattern.
+
+CI runs on every push and PR via `.github/workflows/ci.yml`.
+
+---
+
+## Releasing to PyPI
+
+The package publishes to PyPI automatically when a GitHub release is created. Publishing uses OIDC trusted-publisher auth — no API tokens or secrets.
+
+**Release steps:**
+
+1. Bump `version` in `pyproject.toml` (follows [PEP 440](https://peps.python.org/pep-0440/): `0.1.0a1`, `0.1.0b1`, `0.1.0`, `0.2.0`, etc.)
+2. Commit and push
+3. On GitHub: Releases → Draft a new release → create a tag matching the version (e.g. `v0.1.0`) → Publish release
+4. The `.github/workflows/publish.yml` workflow fires, builds the sdist + wheel with `hatch`, and uploads to PyPI
+
+The `pypi` GitHub environment and PyPI trusted-publisher entry for `watchdog-intel` are already configured — no further setup needed.
+
+---
+
+## Adding new record skills
+
+### Where skills live
+
+Skill files live in `src/watchdog/skills/records/`. They are plain markdown. No code changes are needed — skills are automatically picked up by the setup process when a new file is added to that directory.
+
+### Standard structure
+
+A blank template is at `src/watchdog/skills/records/_template.md` — copy it as the starting point for any new skill. Files starting with `_` are excluded from installation and will not be loaded by Claude.
+
+Every skill file should follow this structure in order:
+
+1. **Intro paragraph** — one or two sentences explaining when this skill is loaded by `/ingest`. Name the document types that trigger it.
+2. **Document types covered** — a bulleted list of the specific document types the skill applies to. This is the one section where it is acceptable to list jurisdiction-specific document names (since those are the literal names of the documents). Group by jurisdiction if there are many.
+3. **Always-present fields table** — a two-column table (`Field` | `What to look for`) listing the fields that appear in virtually every document of this type. Extract these even when not prominently displayed.
+4. **Red flags section** — the most important section. Use sub-headings to group related red flags. Each red flag should be a bolded label followed by a sentence or two explaining what to look for and why it matters. Write for pattern recognition, not just field extraction.
+5. **Terminology table(s)** — one or more two-column tables (`Term` | `Meaning`) covering jargon a journalist would encounter. If the terminology varies significantly by jurisdiction, use a three-column table (`Term` | `Jurisdiction` | `Meaning`) or separate tables per jurisdiction.
+6. **Relationships to extract** — a numbered list of entity relationships the skill should produce (e.g. `Person → Company: Director`). Use the `→` notation.
+7. **What investigators typically miss** — a numbered list of six to eight specific things that experienced journalists often overlook when reading this document type. Be concrete and specific.
+8. **Sources and further reading** — three subsections: **Official and regulatory** (government agencies, regulators, FATF, OECD, accounting standards bodies), **Practitioner and public interest** (law firm guides, NGO reports, public interest organizations), and **Journalism resources** (publicly accessible tipsheets, press freedom organizations). Omit a subsection entirely if there is nothing worth citing. End with a **Notes on unsourced claims** paragraph for any red flag claims that could not be traced to a specific source — these are flagged for editorial review, not silently included as fact. Every claim in the red flags section should be traceable to at least one source in this section.
+
+### Authoring principles
+
+- **Jurisdiction-agnostic by default.** Lead with principles and patterns that apply anywhere. Specific jurisdictions are examples, not the default frame. A journalist in Brazil or Germany should find the skill useful.
+- **Jurisdiction-specific terminology tables are valuable** — but position them clearly as jurisdiction guides, not as the primary content. The always-present fields and red flags sections must be universal.
+- **The red flags section is the most important.** This is where the skill earns its value. Think about what a twenty-year veteran investigative journalist would notice that a first-year reporter would miss.
+- **Write for a smart investigative journalist, not a specialist.** Assume the reader knows how journalism works but may not know the specific document type deeply. Explain jargon; don't assume it.
+- **Be specific.** "Look for unusual transactions" is useless. "A property transferred three or more times in 12 months may be involved in title fraud, mortgage fraud, or money laundering" is useful.
+
+### Before writing a new skill, ask the user
+
+1. What document type are you working with? (Get a sample if possible.)
+2. What jurisdiction(s) are most common for your work? (This shapes the terminology table.)
+3. Are there existing skills that overlap? (Check `src/watchdog/skills/records/` first — some document types are covered from a related angle by an existing skill.)
+
+If the new skill would overlap significantly with an existing one, consider extending the existing skill rather than creating a new file.
+
+---
+
+## CLI style guide
+
+All terminal output in `cli.py` follows a consistent visual language. The colour constants are defined at the top of the file — use them, never raw ANSI codes.
+
+### Colour semantics
+
+| Constant | Use for |
+|----------|---------|
+| `_BOLD` | Project names, important counts, section headers |
+| `_DIM` | Secondary metadata: dates, slugs, path labels, quiet prompts |
+| `_CYAN` | Actionable items: file paths, commands the user should type, directory names like `_INCOMING/` |
+| `_GREEN` | Success states (`Created:`) |
+| `_YELLOW` | Warnings (pending files, things that need attention) |
+| `_RESET` | Always close every coloured span |
+
+### Layout conventions
+
+- **Indent everything 2 spaces** — all output lines start with `"  "`. The banner and list headers set this pattern; every command should match it.
+- **Bold name, dim slug** — when showing a project, display its human name in bold and its slug in dim on the same line: `  **My Project**  [dim]my-project[/dim]`.
+- **Cyan for paths, never dim** — file system paths and `watchdog …` commands the user should run are always `_CYAN`, not `_DIM`. Dim is for decorative/secondary text only.
+- **Section headers: bold, no trailing colon** — e.g. `  **Documents by type**` not `Documents by type:`. The colon was dropped in the consistency pass.
+- **Dim labels, normal counts** — in type-breakdown tables, the label is `_DIM`, the count is unstyled (so it reads at normal brightness).
+- **No trailing colons on "Pending in" lines** — format is `Pending in _CYAN__INCOMING/_RESET  <label>`.
+
+### Adding a new command
+
+1. Print a blank line before the first content line and after the last, matching the spacing in `cmd_status`.
+2. Use `_find_project` for any command that takes a project name — it handles prefix matching and exits cleanly.
+3. Never call `print(f"Error: …")` and continue — use `sys.exit(f"Error: …")`.
+4. If the command produces a success confirmation, use `_GREEN` for the label and `_BOLD` for the key value.
+
+### Adding a new CLI alias
+
+Add the alias → canonical mapping to `_ALIASES` at the top of `cli.py`. Aliases are resolved before argparse sees `sys.argv`, so they are invisible to `--help`. Add a parametrized test case to the `test_aliases_remap_argv` test in `tests/test_cli.py`.

watchdog_intel-0.1.0a1/INSTALL.md

@@ -0,0 +1,298 @@
+# Installing Watchdog
+
+Watchdog is a tool for managing large collections of public records. Once installed, you drop documents into a folder and Watchdog extracts the names, addresses, companies, and connections — then lets you ask questions in plain language.
+
+This guide assumes you have never used a terminal before. Read through it once before starting.
+
+---
+
+## What you need
+
+| What | Why | Free? |
+|------|-----|-------|
+| A computer running macOS, Linux, or Windows | Watchdog runs on your computer, not in the cloud | n/a |
+| [Obsidian](https://obsidian.md) | The app where you'll read and explore your documents | Free |
+| [Claude Code](https://claude.ai/download) | The AI assistant that reads and connects your documents | Free to install |
+| A Claude.ai Pro or Max subscription | Powers the AI — required for document processing | Pro ~$20/month; Max from $100/month |
+
+**Obsidian** is a note-taking app that Watchdog uses to organize and display your research. You don't need to know how to use it before starting — Watchdog sets it up for you.
+
+**Claude Code** is the AI assistant that does the document processing. It's made by Anthropic, the same company that makes Claude. You install it once on your computer.
+
+**A subscription** is required because processing documents requires AI. A Pro subscription ($20/month) is enough for most journalism work. If you're ingesting hundreds of documents at a time, Max (from $100/month) gives you higher limits.
+
+---
+
+## Step 1: Install Obsidian
+
+1. Go to [obsidian.md](https://obsidian.md) and click **Download**
+2. Open the downloaded file and drag Obsidian to your Applications folder
+3. Open Obsidian — it will ask you to create or open a vault. Click **Create new vault** and give it any name for now (you'll create your real investigation vaults later)
+
+---
+
+## Step 2: Install Claude Code
+
+1. Go to [claude.ai/download](https://claude.ai/download) and download the app
+2. Open the downloaded file and follow the installation instructions
+3. Open Claude Code and sign in with your Claude.ai account
+
+If you don't have a Claude.ai account yet, create one at [claude.ai](https://claude.ai) and subscribe to Pro or Max before continuing.
+
+---
+
+## Step 3: Open Terminal
+
+Terminal is a built-in app that lets you type commands to your computer. You'll only need it for the next few steps.
+
+**macOS:** Press **Command + Space**, type **Terminal**, press Return.
+
+**Linux:** Press **Ctrl + Alt + T**, or search for Terminal in your application menu.
+
+**Windows:** Press **Windows + R**, type **cmd**, press Return. Or install [Windows Terminal](https://apps.microsoft.com/detail/9n0dx20hk701) for a better experience.
+
+---
+
+## Step 4: Install prerequisites
+
+Watchdog requires two tools for processing PDFs: **qpdf** and **ghostscript**. It also requires **pipx** to install Python tools.
+
+**macOS:**
+```
+brew install qpdf ghostscript pipx
+pipx ensurepath
+```
+Then close and reopen Terminal so the new `pipx` path takes effect.
+
+If you don't have Homebrew, install it first: [brew.sh](https://brew.sh)
+
+**Ubuntu / Debian Linux:**
+```
+sudo apt install qpdf ghostscript pipx tesseract-ocr libtesseract-dev
+```
+
+**Fedora / RHEL Linux:**
+```
+sudo dnf install qpdf ghostscript pipx tesseract tesseract-devel
+```
+
+**Windows:**
+- qpdf: [github.com/qpdf/qpdf/releases](https://github.com/qpdf/qpdf/releases) — download the installer
+- ghostscript: [ghostscript.com/releases/gsdnld.html](https://ghostscript.com/releases/gsdnld.html) — download the installer
+- pipx: open Terminal and run `python -m pip install pipx`, then `pipx ensurepath`
+
+---
+
+## Step 5: Install Watchdog
+
+```
+pipx install watchdog-intel
+```
+
+Wait for it to finish. You'll see a message saying the installation is complete.
+
+---
+
+## Step 6: Run setup
+
+```
+watchdog setup
+```
+
+This will:
+- Verify that qpdf and ghostscript are installed
+- Install the Watchdog skills into Claude Code
+- Ask where you want to store your investigation projects
+- Set up tab completion in your shell
+
+It will ask one question: where to store your projects. Press Return to accept the default, or type a different path.
+
+When it finishes, reload your shell as instructed (e.g. `source ~/.zshrc`), then:
+
+```
+watchdog new "My Investigation"
+```
+
+---
+
+## Creating your first investigation
+
+When you're ready to start a new investigation, type:
+
+```
+watchdog new "My Investigation Name"
+```
+
+Use a descriptive name — it will become the name of your Obsidian vault. For example:
+```
+watchdog new "Shell Company Investigation"
+```
+
+Watchdog will create a folder in your projects directory and print instructions.
+
+**To open the investigation in Obsidian:**
+1. Open Obsidian
+2. Click the vault icon in the bottom-left corner
+3. Click **Open folder as vault**
+4. Navigate to your investigation folder and click Open
+
+**To open the investigation in Claude Code:**
+1. Open Claude Code
+2. Click **Open project** or use File → Open
+3. Navigate to your investigation folder and click Open
+
+You're ready to start ingesting documents.
+
+---
+
+## How to ingest documents
+
+**Drop files into the Incoming folder:**
+
+In your file manager, navigate to your investigation folder. You'll see a folder called `_INCOMING`. Drag any documents you want to process into this folder.
+
+Supported file types: PDF, Word documents, Excel spreadsheets, images (JPG, PNG, TIFF), web pages (HTML), and plain text files.
+
+**Start a Claude Code session:**
+
+With Claude Code open and your investigation folder as the project, simply open a session. Claude will automatically check the Incoming folder for files at the start of every session and process them before anything else.
+
+You can also type `/watchdog-ingest` at any time to process files manually.
+
+**Watch for the briefing:**
+
+After processing, Claude will produce a briefing showing:
+- What documents were processed
+- What entities (people, companies, addresses) were found
+- Connections between entities that were already in your vault
+- Anything that looks unusual
+
+---
+
+## Asking questions
+
+Once documents are ingested, you can ask questions in plain English:
+
+- `/watchdog-query Who are the directors of Shell Co Ltd?`
+- `/watchdog-query What address does John Doe use?`
+- `/watchdog-query Which companies share the address 123 Main St?`
+
+Claude will answer using only the documents in your vault and will cite the specific page it's drawing from.
+
+---
+
+## Finding connections
+
+Type `/watchdog-surface` to run a full connection analysis across your entire vault. Claude will look for:
+
+- Addresses shared by companies that have no other apparent connection
+- People appearing in unusual roles
+- Entities mentioned in many documents but with no documented relationships
+
+---
+
+## Checking vault health
+
+Type `/watchdog-health` to check for any problems with your vault — missing files, broken links, or incomplete records.
+
+---
+
+## Tips
+
+**Ingesting web pages directly from your browser:**
+Install the [Obsidian Web Clipper](https://obsidian.md/clipper) browser extension. Point it at your investigation vault and set the destination folder to `_INCOMING`. You can then clip any web page — news articles, company profiles, government announcements — directly into the ingest pipeline with one click, without downloading anything manually.
+
+**Naming your documents before ingesting:**
+Watchdog uses the filename to organize documents. A filename like `shell-co-annual-report-2023.pdf` is much more useful than `scan0042.pdf`. Rename files before dropping them into Incoming when possible.
+
+**Adding context with sidecar files:**
+If you want to record where a document came from before Claude processes it, create a text file with the same name but `.yml` extension. For example, alongside `shell-co-annual-report-2023.pdf`, create `shell-co-annual-report-2023.yml` containing:
+
+```
+source: https://www.sedar.com/filing/xyz
+obtained: 2026-06-05
+notes: Check the director change on page 12.
+```
+
+This context is merged into the document record and preserved even if you re-ingest the document later.
+
+**Multiple investigations:**
+Each investigation is a separate vault. Create as many as you need:
+```
+watchdog new "City Hall Investigation"
+watchdog new "Contractor Investigation"
+```
+
+To switch between investigations, switch the open folder in both Obsidian and Claude Code.
+
+To list all your investigations:
+```
+watchdog list
+```
+
+To reopen an investigation in Claude Code:
+```
+watchdog open shell-company-investigation
+```
+
+---
+
+## Troubleshooting
+
+**`watchdog: command not found`**
+The install didn't add `watchdog` to your path. Try:
+```
+pipx ensurepath
+```
+Then close and reopen your terminal, and try again.
+
+**`Watchdog isn't set up yet`**
+Run:
+```
+watchdog setup
+```
+
+**`qpdf not found` or `ghostscript not found` during setup**
+Install the missing tool for your platform (see Step 4 above), then run `watchdog setup` again.
+
+**A document lands in `_FAILED/`**
+The document couldn't be processed. Common reasons:
+- Password-protected PDF — remove the password and try again
+- Corrupted file — try re-downloading the document
+- Unsupported format — check the supported file types list above
+
+To retry: move the file from `_INCOMING/_FAILED/` back to `_INCOMING/`, then run `/watchdog-ingest` in Claude Code.
+
+**Rate limit errors during a large ingest**
+If you're ingesting many documents at once and Claude hits a rate limit, it will stop and log where it paused. Files that were successfully processed will be in `morgue/`. Files that weren't processed will still be in `_INCOMING/`. Start a new Claude Code session and run `/watchdog-ingest` again — it will pick up where it left off, skipping any already-processed files.
+
+---
+
+## Audio and video transcription (optional)
+
+Watchdog can transcribe audio and video files if you install support for it. This requires **ffmpeg** and adds roughly 2 GB of dependencies.
+
+**macOS:** `brew install ffmpeg`
+**Ubuntu/Debian:** `sudo apt install ffmpeg`
+**Windows:** [ffmpeg.org/download.html](https://ffmpeg.org/download.html)
+
+Then reinstall Watchdog with transcription support:
+```
+pipx install watchdog-intel[asr] --force
+```
+
+---
+
+## Getting help
+
+If something isn't working, the best place to get help is the Watchdog GitHub repository:
+
+```
+https://github.com/tomcardoso/watchdog/issues
+```
+
+When reporting a problem, include:
+- What you typed or did
+- What you expected to happen
+- What actually happened (copy and paste any error messages)
+- Your operating system and version

watchdog_intel-0.1.0a1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tom Cardoso
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.