clipmd 0.1.0__tar.gz

clipmd-0.1.0/.claude/settings.json

@@ -0,0 +1,49 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(file:*)",
+      "Bash(gh pr:*)",
+      "Bash(git add:*)",
+      "Bash(git branch:*)",
+      "Bash(git checkout:*)",
+      "Bash(git commit:*)",
+      "Bash(git diff:*)",
+      "Bash(git fetch:*)",
+      "Bash(git log:*)",
+      "Bash(git mv:*)",
+      "Bash(git pull:*)",
+      "Bash(git push:*)",
+      "Bash(git reset:*)",
+      "Bash(git rm:*)",
+      "Bash(git show:*)",
+      "Bash(git stash:*)",
+      "Bash(git status:*)",
+      "Bash(git switch:*)",
+      "Bash(ls:*)",
+      "Bash(make:*)",
+      "Bash(mkdir:*)",
+      "Bash(pre-commit:*)",
+      "Bash(pwd:*)",
+      "Bash(python3:*)",
+      "Bash(rm -rf *.egg-info:*)",
+      "Bash(rm -rf .pytest_cache:*)",
+      "Bash(rm -rf .ruff_cache:*)",
+      "Bash(rm -rf __pycache__:*)",
+      "Bash(rm -rf build:*)",
+      "Bash(rm -rf dist:*)",
+      "Bash(rm -rf tests/fixtures/temp:*)",
+      "Bash(stat:*)",
+      "Bash(uv:*)",
+      "Bash(wc:*)",
+      "Bash(which:*)"
+    ],
+    "deny": [
+      "Bash(git push --force:*)",
+      "Bash(git push -f:*)",
+      "Bash(git reset --hard:*)",
+      "Bash(rm -rf /*:*)",
+      "Bash(rm -rf ~:*)",
+      "Bash(rm -rf .:*)"
+    ]
+  }
+}

clipmd-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.13", "3.14"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Run linter
+        run: uv run ruff check src/ tests/
+
+      - name: Run formatter check
+        run: uv run ruff format --check src/ tests/
+
+      - name: Run type checker
+        run: uv run ty check src/
+
+      - name: Run tests with coverage
+        run: uv run pytest --cov=clipmd --cov-report=term-missing --cov-fail-under=89

clipmd-0.1.0/.gitignore

@@ -0,0 +1,99 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy / type checkers
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.ty/
+
+# Ruff
+.ruff_cache/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+.clipmd/cache.json
+*.log
+
+# Test artifacts
+tests/fixtures/temp/

clipmd-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.13
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format

clipmd-0.1.0/CHANGELOG.md

@@ -0,0 +1,104 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2024-01-20
+
+### Added
+
+#### Core Commands
+- `init` command for initializing clipmd in a directory with config scaffolding
+- `validate` command for validating configuration and setup
+- `fetch` command for fetching web content and converting to markdown with frontmatter
+  - RSS/Atom feed support with `--rss` flag and `--rss-limit` option
+  - Duplicate detection with `--check-duplicates` (enabled by default)
+  - Readability mode for extracting main content
+  - Dry run mode with `--dry-run` flag
+  - Support for reading URLs from file with `--file` flag
+  - Async fetching with configurable concurrency
+  - Meta-refresh redirect handling
+  - Automatic tracking parameter cleaning
+  - Never overwrites existing files (appends suffix `-2.md`, `-3.md`, etc.)
+- `preprocess` command for cleaning and preparing articles
+  - URL cleaning (tracking parameters, redirect unwrapping)
+  - Filename sanitization with configurable replacements
+  - Date prefix addition (from frontmatter or content extraction)
+  - Frontmatter fixing (multi-line fields, wikilinks, YAML validation)
+  - Duplicate detection during preprocessing
+- `extract` command for generating LLM-optimized metadata
+  - Multiple output formats: markdown, json, yaml
+  - Configurable description/content preview length
+  - Optional word count and language detection with `--include-stats`
+  - Folder list inclusion with `--folders`
+- `move` command for executing categorization decisions
+  - Automatic folder creation
+  - System trash integration for TRASH category
+  - Cache updates after moves
+  - Dry run mode
+- `trash` command for moving files to system trash
+  - Glob pattern support
+  - Cache updates marking files as removed
+- `stats` command for folder statistics
+  - Configurable warning thresholds
+  - Multiple output formats: table, json, yaml
+  - Optional special folder inclusion
+- `duplicates` command for finding duplicate articles
+  - Detection by URL (default)
+  - Detection by content hash
+  - Detection by filename similarity
+  - Multiple output formats: markdown, json
+
+#### Core Features
+- XDG-compliant configuration file search (project root, .clipmd/, ~/.config/clipmd/)
+- Pydantic-based configuration validation
+- Flexible frontmatter field mapping (supports multiple field name variants)
+- Date parsing with multiple input formats
+- Date extraction from article content using regex patterns
+- URL cache for duplicate detection and history tracking
+- Content hashing for duplicate detection
+- Rich-formatted console output with progress bars and colored status
+- Shell completion support (bash, zsh, fish)
+- Global options: `--verbose`, `--quiet`, `--config`, `--no-color`
+- Custom exception hierarchy with exit codes (0=success, 1=error, 2=partial success)
+
+#### Development Infrastructure
+- Python 3.13+ support
+- UV package management
+- Ruff linting and formatting
+- Ty type checking
+- Pre-commit hooks
+- Pytest test suite with >89% coverage
+- Comprehensive unit, CLI, and integration tests
+- GitHub Actions CI/CD pipeline
+- Makefile for common development tasks
+
+#### Documentation
+- Complete specification in SPEC.md
+- Development guide with architecture patterns
+- Claude Code integration examples
+- LLM workflow examples
+
+### Dependencies
+- click>=8.1 (CLI framework)
+- pyyaml>=6.0 (config file parsing)
+- pydantic>=2.10 (config validation)
+- send2trash>=1.8 (system trash integration)
+- python-dateutil>=2.9 (date parsing)
+- python-frontmatter>=1.1.0 (frontmatter parsing)
+- rich>=13.9 (terminal formatting)
+- httpx>=0.28 (async HTTP client)
+- beautifulsoup4>=4.12 (HTML parsing)
+- trafilatura>=2.0 (content extraction)
+- markdownify>=0.14 (HTML to markdown conversion)
+- feedparser>=6.0 (RSS/Atom feed parsing)
+
+### Optional Dependencies
+- langdetect>=1.0 (language detection for `--include-stats`)
+
+[Unreleased]: https://github.com/jmlrt/clipmd/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/jmlrt/clipmd/releases/tag/v0.1.0

clipmd-0.1.0/CLAUDE.md

@@ -0,0 +1,229 @@
+# CLAUDE.md
+
+Project-specific guidance for the **clipmd** CLI tool.
+
+For general Python development patterns, see the **python-development** skill in Claude Code.
+
+---
+
+# clipmd
+
+CLI tool for saving, organizing, and managing markdown articles with YAML frontmatter.
+
+**See `SPEC.md` for full specification (source of truth).**
+
+## Quick Reference
+
+```bash
+# Development setup
+make dev            # Install all dependencies (including dev extras)
+make install        # Install dependencies without extras
+
+# Quality checks (must pass before commit)
+make check          # Run lint, typecheck, tests with coverage
+make lint           # Run ruff linter
+make format         # Format code with ruff
+make typecheck      # Run ty type checker
+
+# Testing
+uv run pytest tests/unit                # Run only unit tests
+uv run pytest tests/cli                 # Run only CLI tests
+uv run pytest tests/integration         # Run only integration tests
+uv run pytest tests/unit/test_config.py # Run specific test file
+uv run pytest -k test_function_name     # Run specific test by name
+uv run pytest --cov                     # Run with coverage report
+make test                               # Run all tests
+make test-cov                           # Run with coverage (89% minimum)
+
+# Running the CLI
+uv run clipmd --help           # Show help
+uv run clipmd init             # Initialize new vault
+uv run clipmd --config ./test-config.yaml extract  # Use specific config
+```
+
+## Implementation Approach
+
+- Work on feature branch
+- Phase-by-phase implementation (see spec for 9 phases)
+- Atomic commits: each commit must pass `make check`
+
+## Architecture Overview
+
+**Separation of Concerns:**
+- `cli.py` - Click CLI application entry point, global options, command registration
+- `context.py` - Context object holding config, verbosity, vault path (passed via Click context)
+- `config.py` - Pydantic-based configuration loading and validation (XDG-compliant paths)
+- `commands/` - Thin CLI wrappers (50-150 lines): parse args → call core → display output
+- `core/` - Pure business logic: no Click dependencies, returns dataclass results
+- `exceptions.py` - Custom exception hierarchy with exit codes
+
+**Key clipmd Decisions:**
+- **Config as parameter**: Core functions take `Config` as a parameter, not from global context
+- **Async fetching**: `core/fetcher.py` uses `httpx` async with semaphore for concurrent URL fetching (see [Async Fetching Architecture](#async-fetching-architecture) below)
+- **Result dataclasses**: Core functions return typed dataclasses, commands format them for display
+- **TYPE_CHECKING imports**: Core modules import `Config` under `TYPE_CHECKING` to avoid circular imports
+
+## Key Paths
+
+| Path | Purpose |
+|------|---------|
+| `SPEC.md` | Full specification (source of truth) |
+| `CHANGELOG.md` | Project changelog (Keep a Changelog format) |
+| `src/clipmd/cli.py` | CLI entry point, global options |
+| `src/clipmd/context.py` | Context object (config, verbosity) |
+| `src/clipmd/config.py` | Pydantic config models and loading |
+| `src/clipmd/commands/` | CLI command modules (thin wrappers) |
+| `src/clipmd/core/` | Business logic (pure functions) |
+| `src/clipmd/exceptions.py` | Custom exceptions with exit codes |
+| `tests/unit/` | Unit tests for core modules |
+| `tests/cli/` | CLI command tests |
+| `tests/integration/` | End-to-end workflow tests |
+| `tests/fixtures/sample-vault/` | Test data and config |
+
+## Configuration
+
+**Config Location (XDG-compliant search order):**
+1. `./config.yaml` (project root)
+2. `./.clipmd/config.yaml` (project .clipmd directory)
+3. `~/.config/clipmd/config.yaml` (user-wide config)
+4. `--config PATH` flag overrides all
+
+**Validation:**
+- Config uses Pydantic v2 models for validation (`config.py`)
+- Invalid config raises `ConfigError` with helpful messages
+- Missing config falls back to sensible defaults
+- All paths in config are resolved relative to vault root
+
+## Error Handling
+
+**Exception Hierarchy** (see `exceptions.py`):
+- `ClipmdError` - Base exception (exit code 1)
+  - `ConfigError` - Configuration errors
+  - `FetchError` - URL fetching errors
+  - `ParseError` - Frontmatter/content parsing errors
+  - `CacheError` - Cache read/write errors
+  - `ValidationError` - Input validation errors
+  - `PartialSuccessError` - Some operations succeeded, some failed (exit code 2)
+
+**Exit Codes:**
+- `0` - Success
+- `1` - Error (operation failed)
+- `2` - Partial success (some items succeeded, some failed)
+
+**Pattern**: Core functions return Result dataclasses with `success: bool` and optional `error: str | None`. Commands check results and raise `SystemExit(1)` on failure, or print Rich-formatted errors and exit.
+
+## Development Practices
+
+### When Fixing Bugs in Similar Commands
+
+**clipmd-specific practice**: When fixing a bug or inconsistency in one CLI command, proactively check ALL similar commands for the same issue before considering the task done. Do not wait for the user to ask twice.
+
+### Library Selection Principle
+
+Prefer well-maintained, actively-developed libraries from PyPI over custom implementations:
+
+- **Example**: Adopted `python-frontmatter` (20M+ downloads/month) to replace custom regex parsing
+- **Evaluation**: Does it handle our specific requirements? (normalization, truncation, etc.)
+- **Trade-off**: `python-slugify` NOT used because it doesn't support NFD normalization needed for sanitizer
+
+### Testing Strategy
+
+- `tests/unit/` - Test core business logic (frontmatter, config, sanitizer, etc.)
+- `tests/cli/` - Test CLI interfaces (argument parsing, output formatting)
+- `tests/integration/` - Test complete workflows (fetch → preprocess → extract → move)
+- Target coverage: ≥89%
+
+### Architecture Reference
+
+For generic architectural patterns, see the **python-development** skill in Claude Code:
+
+All clipmd commands follow these patterns with core logic in `core/` modules and thin CLI wrappers (50-150 lines) in `commands/`.
+
+## Git Workflow
+
+When staging and committing changes, ensure ONLY changes from the current task are included. Review staged files against the current session scope before committing.
+
+### Addressing PR Review Comments
+
+When asked to address PR review comments (from Copilot, human reviewers, etc.), follow this workflow to avoid excessive GitHub API calls:
+
+1. **Fetch all comments once**: Use `gh api` to retrieve all PR comments (including outdated/resolved ones) and save to a temporary file
+   ```bash
+   gh api repos/:owner/:repo/pulls/{PR_NUMBER}/comments --paginate > pr-comments-review.json
+   ```
+
+2. **Create human-readable summary**: Extract relevant fields into a readable format
+   ```bash
+   cat pr-comments-review.json | jq -r '.[] | select(.in_reply_to_id == null) | "---\nID: \(.id)\nFile: \(.path)\nLine: \(.line // .original_line // "N/A")\nUser: \(.user.login)\nCreated: \(.created_at)\n\n\(.body)\n"' > pr-comments-summary.txt
+   ```
+
+3. **Work from the file**: Review each comment systematically, checking current code state against the issues raised
+
+4. **Track progress**: Create an analysis file to track which issues are fixed, no longer applicable (due to refactoring), or still need work
+
+5. **Clean up**: Remove temporary files when all issues are addressed
+   ```bash
+   rm pr-comments-review.json pr-comments-summary.txt pr-comments-analysis.md
+   ```
+
+**Rationale**: This approach minimizes API calls, provides a stable reference while working, and creates a clear audit trail of what was addressed.
+
+### Changelog Maintenance
+
+**IMPORTANT**: Update `CHANGELOG.md` for all user-facing changes:
+
+**When to update:**
+- New features (commands, options, functionality)
+- Bug fixes that affect behavior
+- Breaking changes to CLI or config format
+- Dependency updates (major versions)
+- Deprecations or removals
+
+**When NOT to update:**
+- Internal refactoring (no behavior change)
+- Test-only changes
+- Documentation updates (unless major)
+- Code formatting/linting
+
+**How to update:**
+1. Add entry under `[Unreleased]` section
+2. Use Keep a Changelog categories: `Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, `Security`
+3. Write user-facing descriptions (not technical implementation details)
+4. Group related changes together
+
+**Example:**
+```markdown
+## [Unreleased]
+
+### Added
+- `extract` command now supports `--include-tags` option for tag filtering
+
+### Fixed
+- `fetch` command no longer crashes on malformed HTML meta tags
+```
+
+**On release:**
+- Move `[Unreleased]` entries to new version section with date
+- Update version links at bottom of file
+- Bump version in `pyproject.toml` following Semantic Versioning
+
+## Async Fetching Architecture
+
+The `core/fetcher.py` module uses async/await with `httpx` for concurrent URL fetching:
+
+**Key Functions:**
+- `fetch_url()` - Async function to fetch single URL with timeout and retries
+- `fetch_urls()` - Async orchestrator using `asyncio.Semaphore` to limit concurrency
+- `fetch_rss_feed()` - Async RSS/Atom feed parser
+- `orchestrate_fetch()` - Main entry point coordinating all fetch operations
+
+**Concurrency Control:**
+- `max_concurrent` setting controls semaphore limit (default: 5)
+- Uses `asyncio.gather()` for parallel execution
+- Each fetch operation is independent (failures don't block others)
+
+**Important Behaviors:**
+1. Meta-refresh redirects are handled automatically
+2. Tracking URL parameters are cleaned (utm_*, fbclid, etc.)
+3. Never overwrites existing files (appends suffix like `-2.md`, `-3.md`)
+4. Content extraction uses trafilatura for readability mode

clipmd-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Julien Mailleret
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

clipmd-0.1.0/Makefile

@@ -0,0 +1,38 @@
+.PHONY: install dev lint format typecheck test test-cov clean build publish check
+
+# Development
+install:
+	uv sync
+
+dev:
+	uv sync --all-extras
+
+# Quality
+lint:
+	uv run ruff check src tests
+
+format:
+	uv run ruff format src tests
+
+typecheck:
+	uv run ty check src
+
+# Testing
+test:
+	uv run pytest
+
+test-cov:
+	uv run pytest --cov=clipmd --cov-report=term-missing --cov-fail-under=89
+
+# Build & Publish
+clean:
+	rm -rf dist build *.egg-info
+
+build: clean
+	uv build
+
+publish: build
+	uv publish
+
+# All checks (used by CI and pre-commit)
+check: lint typecheck test-cov