fresh-docs 0.1.8__tar.gz

fresh_docs-0.1.8/.claude/settings.local.json

@@ -0,0 +1,18 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git add src/fresh/scraper/__init__.py)",
+      "Bash(git commit:*)",
+      "Bash(git add:*)",
+      "Bash(uv sync)",
+      "Bash(uv run python:*)",
+      "Bash(uv run ruff:*)",
+      "Bash(uv run mypy:*)",
+      "Bash(uv sync:*)",
+      "Bash(uv run pytest src/fresh/scraper/ -v)",
+      "Bash(gh issue edit:*)",
+      "Bash(gh issue view:*)",
+      "Bash(gh issue create:*)"
+    ]
+  }
+}

fresh_docs-0.1.8/.github/workflows/mypy.yml

@@ -0,0 +1,19 @@
+name: Mypy
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: pip install -e .
+      - run: pip install mypy
+      - run: mypy src

fresh_docs-0.1.8/.github/workflows/pr-review.yml

@@ -0,0 +1,96 @@
+name: PR Review
+
+on:
+  pull_request:
+    types: [opened, synchronize, ready_for_review, reopened]
+
+permissions:
+  contents: read
+  pull-requests: write
+  id-token: write
+
+jobs:
+  review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Generate Marty token
+        id: marty-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ secrets.MARTY_APP_ID }}
+          private-key: ${{ secrets.MARTY_APP_PRIVATE_KEY }}
+
+      - name: Run Marty PR Review
+        uses: nesalia-inc/marty-action@1.0.0
+        with:
+          github_token: ${{ steps.marty-token.outputs.token }}
+          prompt: |
+            REPO: ${{ github.repository }}
+            PR NUMBER: ${{ github.event.pull_request.number }}
+            AUTHOR: ${{ github.event.pull_request.user.login }}
+            TITLE: ${{ github.event.pull_request.title }}
+
+            Please review this pull request comprehensively:
+
+            ## Review Focus Areas
+
+            ### Code Quality
+            - Code follows best practices and design patterns
+            - Proper error handling and edge cases
+            - No code duplication (DRY principle)
+            - Clear and maintainable code structure
+
+            ### Security
+            - No hardcoded secrets or credentials
+            - Proper input validation and sanitization
+            - SQL injection and XSS vulnerabilities
+            - Authentication and authorization checks
+            - Sensitive data handling
+
+            ### Performance
+            - Potential performance bottlenecks
+            - Efficient database queries
+            - Proper caching strategies
+            - Resource cleanup and memory leaks
+
+            ### Testing
+            - Adequate test coverage for changes
+            - Edge cases covered
+            - Test quality and assertions
+
+            ### Documentation
+            - README updated if needed
+            - Inline comments for complex logic
+            - API documentation updated
+            - Breaking changes documented
+
+            ## Review Output Format
+
+            Use inline comments for specific code issues (highlight exact lines).
+            Use top-level PR comments for general observations and summary.
+
+            Structure your top-level comment as:
+            - **Summary**: Brief overview
+            - **Critical Issues**: Must-fix items
+            - **Recommendations**: Suggestions for improvement
+            - **Positive Notes**: Good practices observed
+
+            Note: The PR branch is already checked out in the current working directory.
+
+            Only post GitHub comments - don't submit review text as messages.
+
+          claude_args: |
+            --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)"
+            --max-turns 100
+
+        env:
+          ANTHROPIC_BASE_URL: https://api.minimax.io/anthropic
+          ANTHROPIC_AUTH_TOKEN: ${{ secrets.MINIMAX_API_KEY }}
+          ANTHROPIC_DEFAULT_SONNET_MODEL: MiniMax-M2.5
+          ANTHROPIC_DEFAULT_HAIKU_MODEL: MiniMax-M2.5
+          ANTHROPIC_DEFAULT_OPUS_MODEL: MiniMax-M2.5

fresh_docs-0.1.8/.github/workflows/pytest.yml

@@ -0,0 +1,18 @@
+name: Pytest
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: pip install -e .[dev]
+      - run: pytest

fresh_docs-0.1.8/.github/workflows/release.yml

@@ -0,0 +1,51 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3
+
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: pip install -e .[dev]
+      - run: mypy src/
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: pip install -e .[dev]
+      - run: pytest
+
+  pypi:
+    runs-on: ubuntu-latest
+    needs: [lint, typecheck, test]
+    environment: production
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: pip install build uv
+      - run: rm -rf dist/
+      - run: python -m build
+      - run: ls -la dist/
+      - run: |
+          uv publish \
+            --token "${{ secrets.PYPI_TOKEN }}"

fresh_docs-0.1.8/.github/workflows/ruff.yml

@@ -0,0 +1,14 @@
+name: Ruff
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3

fresh_docs-0.1.8/.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.14.1
+    hooks:
+      - id: mypy
+        pass_filenames: false
+        entry: uv
+        args: ['run', 'mypy', 'src']
+
+  - repo: local
+    hooks:
+      - id: pytest
+        name: pytest
+        entry: uv
+        args: ['run', 'pytest']
+        pass_filenames: false
+        language: system
+        stages: [pre-commit]

fresh_docs-0.1.8/CLAUDE.md

@@ -0,0 +1,35 @@
+## Language
+
+You MUST always respond in English, regardless of the language used by the user.
+
+## Git Commit Convention
+
+All commits MUST include both co-authors:
+- `martty-code <nesalia.inc@gmail.com>`
+- `Claude Sonnet <noreply@anthropic.com>`
+
+### Commit Message Format
+
+```bash
+git commit -m "$(cat <<'EOF'
+Your commit message here
+
+Co-Authored-By: martty-code <nesalia.inc@gmail.com>
+Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
+EOF
+)"
+```
+
+### Example
+
+```bash
+git commit -m "$(cat <<'EOF'
+feat: add employee search functionality
+
+Implement search bar with real-time filtering for employee list
+
+Co-Authored-By: martty-code <nesalia.inc@gmail.com>
+Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
+EOF
+)"
+```

fresh_docs-0.1.8/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Nesalia Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fresh_docs-0.1.8/PKG-INFO

@@ -0,0 +1,70 @@
+Metadata-Version: 2.4
+Name: fresh-docs
+Version: 0.1.8
+Summary: CLI to get the latest and freshest documentation from any website in Markdown format
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: beautifulsoup4>=4.14.3
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: markdownify>=0.12.0
+Requires-Dist: typer>=0.12.0
+Requires-Dist: typing-extensions>=4.0
+Provides-Extra: all
+Requires-Dist: pyyaml>=6.0; extra == 'all'
+Requires-Dist: rich>=13.0.0; extra == 'all'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.0.0; extra == 'dev'
+Requires-Dist: pre-commit>=4.0.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.4.0; extra == 'dev'
+Provides-Extra: rich
+Requires-Dist: rich>=13.0.0; extra == 'rich'
+Provides-Extra: yaml
+Requires-Dist: pyyaml>=6.0; extra == 'yaml'
+Description-Content-Type: text/markdown
+
+# fresh
+
+A Python CLI application created with @nesalia/create.
+
+## Requirements
+
+- Python 3.12+
+- uv (recommended) or pip
+
+## Installation
+
+```bash
+# Using uv (recommended)
+uv sync
+
+# Or using pip
+pip install -e .
+```
+
+## Usage
+
+```bash
+# Run the CLI
+fresh --help
+
+# Say hello
+fresh hello --name World
+
+# Run tests
+pytest
+```
+
+## Development
+
+```bash
+# Install dev dependencies
+uv sync --extra dev
+
+# Run linter
+ruff check .
+
+# Run tests
+pytest
+```

fresh_docs-0.1.8/README.md

@@ -0,0 +1,44 @@
+# fresh
+
+A Python CLI application created with @nesalia/create.
+
+## Requirements
+
+- Python 3.12+
+- uv (recommended) or pip
+
+## Installation
+
+```bash
+# Using uv (recommended)
+uv sync
+
+# Or using pip
+pip install -e .
+```
+
+## Usage
+
+```bash
+# Run the CLI
+fresh --help
+
+# Say hello
+fresh hello --name World
+
+# Run tests
+pytest
+```
+
+## Development
+
+```bash
+# Install dev dependencies
+uv sync --extra dev
+
+# Run linter
+ruff check .
+
+# Run tests
+pytest
+```

fresh_docs-0.1.8/docs/PROJECT.md

@@ -0,0 +1,52 @@
+# Fresh - Documentation Fetcher CLI
+
+Agent-first CLI to get the latest and freshest documentation from any website in Markdown format.
+
+## ⚠️ Work in Progress
+
+This project is currently under development. Features and API may change.
+
+## Purpose
+
+This CLI scrapes documentation websites and converts pages to Markdown, making it easy to keep documentation fresh and accessible for AI agents.
+
+## Commands
+
+### `fresh list <url>`
+
+Lists all documentation pages available on a given website.
+
+```bash
+fresh list https://nextjs.org
+# Output: /docs /api-reference /guides /tutorials ...
+```
+
+### `fresh get <url>`
+
+Fetches a specific documentation page and outputs it in Markdown format.
+
+```bash
+fresh get https://nextjs.org/docs/app/api-reference/file-conventions/page
+# Output: # Page API Reference\ncontent...
+```
+
+## Features
+
+- **Documentation discovery**: Automatically find all doc pages on a website
+- **Markdown output**: Convert HTML documentation pages to clean Markdown
+- **Public sources only**: Works with publicly accessible documentation sites
+- **Implicit cache**: Pages are cached automatically. Use `--no-cache` to bypass
+- **Flexible output**: Write to file or output to STDOUT
+
+## Use Cases
+
+- Keep local documentation updated for offline access
+- Feed documentation to AI agents for context-aware assistance
+- Quickly explore the structure of unfamiliar documentation sites
+
+## Technical Stack
+
+- **Language**: Python
+- **CLI Framework**: Typer
+- **Package Manager**: uv
+- **License**: MIT

fresh_docs-0.1.8/docs/features/FUTURE.md

@@ -0,0 +1,62 @@
+# Future Features
+
+This document outlines potential features for future releases.
+
+## Core Features
+
+### Sitemap Parsing
+Use website sitemaps for more reliable page discovery in `list` command, instead of relying solely on HTML scraping.
+
+### Retry & Rate Limiting
+Automatic retry with exponential backoff and rate limiting to avoid getting blocked by websites.
+
+### Config File
+Support for `~/.freshrc` configuration file to remember frequently accessed sites and default settings.
+
+### Custom Output Path
+`--output /path/to/save` flag to choose where to save downloaded documentation.
+
+### Format Choice
+`--format md` (default), but also support HTML or JSON output formats.
+
+### Diff Mode
+`--diff` flag to see what has changed since the last fetch.
+
+### Parallel Fetching
+Multi-threaded fetching to download multiple pages simultaneously for better performance.
+
+### Proxy Support
+`--proxy http://...` flag to route requests through a proxy for sites that block scraping.
+
+### Verbose Mode
+`-v` flag for detailed debug output when something doesn't work.
+
+## Nice to Have
+
+### Watch Mode
+`fresh watch <url>` command to monitor a documentation site and notify when changes occur.
+
+### Plugins
+Plugin system to add custom extractors for specific websites with unique structures.
+
+### CI Integration
+CI-friendly output to verify if documentation has changed between builds.
+
+### Version Pinning
+Ability to lock documentation to a specific date or version.
+
+## Implementation Priority
+
+1. **Sitemap parsing** - Improves reliability of `list` command
+2. **Retry + rate limiting** - Essential for production use
+3. **Config file** - Better UX for repeated use
+4. **Output path custom** - Common need
+5. **Format choice** - Depends on use case
+6. **Diff mode** - Very useful for change tracking
+7. **Parallel fetching** - Performance optimization
+8. **Proxy support** - Edge case but important when needed
+9. **Verbose mode** - Debugging tool
+10. **Watch mode** - Advanced feature
+11. **Plugins** - Only if needed for specific sites
+12. **CI integration** - Niche use case
+13. **Version pinning** - Low priority for MVP

fresh_docs-0.1.8/pyproject.toml

@@ -0,0 +1,57 @@
+[project]
+name = "fresh-docs"
+version = "0.1.8"
+description = "CLI to get the latest and freshest documentation from any website in Markdown format"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "typer>=0.12.0",
+    "typing-extensions>=4.0",
+    "httpx>=0.27.0",
+    "beautifulsoup4>=4.14.3",
+    "markdownify>=0.12.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.scripts]
+fresh = "fresh.__main__:main"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "ruff>=0.4.0",
+    "pytest-cov>=4.0",
+    "mypy>=1.0.0",
+    "pre-commit>=4.0.0",
+]
+rich = [
+    "rich>=13.0.0",
+]
+yaml = [
+    "pyyaml>=6.0",
+]
+all = [
+    "rich>=13.0.0",
+    "pyyaml>=6.0",
+]
+
+[tool.ruff]
+line-length = 100
+
+[tool.mypy]
+python_version = "3.12"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.hatch.version]
+path = "src/fresh/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/fresh"]

fresh_docs-0.1.8/src/fresh/__init__.py

@@ -0,0 +1,29 @@
+"""fresh - CLI application."""
+
+__version__ = "0.1.8"
+
+import typer
+
+from .commands.get import get
+from .commands.list import list_urls
+
+app = typer.Typer(help="fresh - A CLI application")
+
+# Register subcommands
+app.command(name="list")(list_urls)
+app.command(name="get")(get)
+
+
+@app.command()
+def hello(name: str = "World") -> None:
+    """Say hello to someone."""
+    print(f"Hello, {name}!")
+
+
+def main() -> None:
+    """Entry point for the CLI."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

fresh_docs-0.1.8/src/fresh/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for fresh CLI."""
+
+from . import main
+
+if __name__ == "__main__":
+    main()

fresh_docs-0.1.8/src/fresh/commands/__init__.py

@@ -0,0 +1,5 @@
+"""CLI commands for fresh."""
+
+from .list import list_urls
+
+__all__ = ["list_urls"]