aegis-stack-crawl4ai 0.1.0__tar.gz

aegis_stack_crawl4ai-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,102 @@
+name: Release to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*.*.*-rc*'  # Pre-release tags (v0.1.0-rc1, etc.) → TestPyPI
+      - 'v*.*.*'      # Production tags (v0.1.0, etc.) → PyPI
+
+jobs:
+  build:
+    name: Build Package
+    runs-on: ubuntu-latest
+    env:
+      UV_PYTHON: "3.11"  # The floor this plugin supports
+      UV_LINK_MODE: "copy"
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+      - run: uv python install 3.11
+      - run: uv sync --all-extras
+      - run: make check
+      - run: uv build
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: build
+    if: contains(github.ref, '-rc')
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # REQUIRED for Trusted Publishing
+      contents: write  # For creating GitHub pre-releases
+    environment:
+      name: release-test
+      url: https://test.pypi.org/project/aegis-stack-crawl4ai/
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          print-hash: true
+      - id: version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+      - uses: softprops/action-gh-release@v3
+        with:
+          name: "v${{ steps.version.outputs.VERSION }} (Release Candidate)"
+          draft: true
+          generate_release_notes: true
+          prerelease: true
+          files: dist/*
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: build
+    if: "!contains(github.ref, '-rc')"
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # REQUIRED for Trusted Publishing
+      contents: write  # For creating GitHub releases
+    environment:
+      name: release
+      url: https://pypi.org/project/aegis-stack-crawl4ai/
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          print-hash: true
+          attestations: true  # Sigstore attestations
+      - id: version
+        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+      - uses: softprops/action-gh-release@v3
+        with:
+          name: "v${{ steps.version.outputs.VERSION }}"
+          draft: true
+          generate_release_notes: true
+          body: |
+            ## Installation
+
+            Inside an Aegis Stack project:
+
+            ```bash
+            pip install aegis-stack-crawl4ai==${{ steps.version.outputs.VERSION }}
+            aegis add crawl4ai
+            ```
+
+            - [PyPI](https://pypi.org/project/aegis-stack-crawl4ai/${{ steps.version.outputs.VERSION }}/)
+            - [Aegis Stack](https://github.com/lbedner/aegis-stack)
+          files: dist/*
+          prerelease: false

aegis_stack_crawl4ai-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,16 @@
+name: test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install 3.12
+      - run: uv sync --all-extras
+      - run: make check

aegis_stack_crawl4ai-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+.venv/
+.pytest_cache/
+.ruff_cache/
+*.egg-info/
+dist/
+build/
+.coverage
+htmlcov/

aegis_stack_crawl4ai-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Match aegis-stack's own ``.pre-commit-config.yaml`` so plugin
+    # authors get the same lint surface as core devs.
+    rev: v0.12.7
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

aegis_stack_crawl4ai-0.1.0/LICENSE

@@ -0,0 +1,202 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

aegis_stack_crawl4ai-0.1.0/Makefile

@@ -0,0 +1,16 @@
+.PHONY: install test lint fix check
+
+install:
+	uv sync --all-extras
+
+test:
+	uv run pytest -v
+
+lint:
+	uv run ruff check .
+
+fix:
+	uv run ruff format .
+	uv run ruff check --fix .
+
+check: lint test

aegis_stack_crawl4ai-0.1.0/PKG-INFO

@@ -0,0 +1,317 @@
+Metadata-Version: 2.5
+Name: aegis-stack-crawl4ai
+Version: 0.1.0
+Summary: Web crawling and scraping for Aegis Stack projects, via Crawl4AI
+Project-URL: Aegis Plugin, https://aegis-stack.io/plugins/aegis-stack-crawl4ai
+Project-URL: Homepage, https://aegis-stack.io/plugins/aegis-stack-crawl4ai
+Project-URL: Crawl4AI, https://github.com/unclecode/crawl4ai
+Author: Aegis Stack
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: aegis-stack,aegis-stack-plugin,crawl4ai,web-crawling
+Requires-Python: >=3.11
+Requires-Dist: aegis-stack>=0.13.1
+Provides-Extra: dev
+Requires-Dist: pre-commit>=3.7.0; extra == 'dev'
+Requires-Dist: pytest>=9.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# aegis-stack-crawl4ai
+
+Web crawling and scraping for [Aegis Stack](https://github.com/lbedner/aegis-stack) projects.
+Wraps [crawl4ai](https://github.com/unclecode/crawl4ai) (Apache 2.0) and persists results
+into a generic, query-friendly `crawled_page` table.
+
+This is the reference third-party plugin: it exercises the plugin system end to end and
+rides on per-plugin schema isolation. On Postgres the plugin's tables live in their own
+`crawler` schema, apart from the rest of the stack; SQLite has no schemas, so there the
+table lands unqualified in the single database file.
+
+## Install
+
+Inside an Aegis Stack project:
+
+```bash
+pip install aegis-stack-crawl4ai
+aegis add crawl4ai
+```
+
+That will:
+
+- Add `crawl4ai>=0.8.6` and `alembic>=1.13` to your project's `pyproject.toml`
+- Render the service (`app/services/crawler/`), its router
+  (`app/components/backend/api/crawler/`), the dashboard card and modal, and a
+  `crawl` CLI
+- Generate an Alembic migration creating `crawler.crawled_page`
+- Mount the routes below under `/api/v1/crawler`
+- Add a `Crawler` row to the dashboard health table, with its own card and modal
+
+The install is HTTP-only: pages are fetched as the server sends them, with no browser.
+Client-rendered content is out of scope.
+
+## Why `>=0.8.6`?
+
+Crawl4AI 0.8.5 and earlier pull in a transitive `litellm` dependency that was
+[compromised in a PyPI supply-chain attack](https://github.com/unclecode/crawl4ai/releases).
+0.8.6 is the hotfix release that swaps in a clean fork. Always pin `>=0.8.6`.
+
+## What you get
+
+**Dashboard.** A Crawler card, and a modal with three tabs. *Overview* is the counts.
+*Pages* drills down: the sites you have crawled, then that site's pages, then one page's
+stored body with a copy button. *Crawl* starts a crawl and watches it run, with a row per
+page as it lands.
+
+**API.** Six routes under `/api/v1/crawler`:
+
+| route | does |
+|---|---|
+| `POST /crawl` | fetch one URL, store it, return the row |
+| `POST /crawl-batch` | start a background crawl, return `202` + a `job_id` |
+| `GET /sites` | crawled sites, with page and failure counts |
+| `GET /pages` | recent pages, newest first; `?site=` narrows, `?failed_only=` filters |
+| `GET /pages/{id}` | one stored page |
+| `GET /stats` | counts, distinct URLs, content-type split, last fetch |
+
+**CLI.** `<project> crawl stats | fetch | site | list | show | retry-failed`.
+
+**Progress.** A batch crawl runs as a framework job, so progress streams over
+`/api/v1/jobs/{job_id}/events` and the dashboard follows it without knowing anything about
+the crawler.
+
+## Crawl options
+
+Defaults reproduce a single-page fetch. `crawl4ai`'s `CrawlerRunConfig` takes about a
+hundred parameters; these are the handful that change what you actually get back, and the
+API, the CLI and the dashboard share them.
+
+| option | default | does |
+|---|---|---|
+| `depth` | `0` | how many links deep to follow (hard ceiling of 3) |
+| `max_pages` | `25` | ceiling on pages per seed URL when `depth > 0` |
+| `stay_on_domain` | `true` | follow links only within the seed's site |
+| `selector` | none | CSS selector: keep only this part of each page |
+| `min_words` | `0` | drop blocks shorter than this many words (nav, chrome) |
+| `fresh` | `false` | bypass the network cache and row-level dedup |
+
+```bash
+<project> crawl site example.com --depth 1 --max-pages 50 --selector "main article"
+```
+
+## Settings
+
+Composed into your project's `Settings`, so they are ordinary env vars:
+
+| setting | default | does |
+|---|---|---|
+| `CRAWLER_DEFAULT_TIMEOUT_S` | `30` | per-URL fetch timeout |
+| `CRAWLER_MAX_CONCURRENCY` | `5` | in-flight requests during a batch |
+| `CRAWLER_USER_AGENT` | `aegis-stack-crawl4ai/0.1` | `User-Agent` sent on every request |
+
+## The `crawled_page` table
+
+Generic by design: URL, content, content hash, JSONB metadata, status code, source kind.
+Future ingestion plugins (RSS, PDF, sitemap) can share the schema or follow the same shape.
+
+| column | type | notes |
+|----------------|-----------------|-------------------------------------------------------------|
+| `id` | int PK | autoincrement |
+| `source_url` | text | indexed |
+| `site` | varchar(255) | registrable host, `www.` stripped; indexed, and what the sites view groups by |
+| `content` | text | nullable (failures still produce a row) |
+| `content_hash` | varchar(64) | sha256, indexed (use for dedup queries) |
+| `content_type` | varchar(32) | `markdown` / `html` / `json` / `binary` |
+| `doc_metadata` | jsonb | caller-supplied tags, e.g. `{"ign_id": 5208, "cluster": 9}` |
+| `status_code` | int | HTTP status; `>= 400` is your retry queue |
+| `source_kind` | varchar(32) | producer label (`crawl4ai`, `rss`, `pdf_ingest`, ...) |
+| `fetched_at` | timestamptz | indexed |
+
+Re-fetching a URL writes a new row; dedup-by-hash is a query you run when you care, not a
+write-path branch.
+
+## Use from Python
+
+```python
+from app.core.db import AsyncSessionLocal
+from app.services.crawler.deps import get_crawler
+from app.services.crawler.options import CrawlOptions
+
+crawler = get_crawler()  # process-wide instance; the API and health check share it
+
+async with AsyncSessionLocal() as session:
+    doc = await crawler.fetch(
+        session,
+        url="https://example.com",
+        doc_metadata={"campaign": "march"},
+    )
+    print(doc.id, doc.status_code, len(doc.content or ""))
+
+    # Follow links, one level, staying on the site.
+    pages = await crawler.crawl(
+        session,
+        url="https://example.com",
+        options=CrawlOptions(depth=1, max_pages=50),
+    )
+```
+
+A short-lived script that owns its crawler should build its own and close it:
+
+```python
+from app.services.crawler.service import CrawlerService
+
+crawler = CrawlerService()
+try:
+    ...
+finally:
+    await crawler.aclose()
+```
+
+## Use from the API
+
+```bash
+curl -X POST http://localhost:8000/api/v1/crawler/crawl \
+  -H 'Content-Type: application/json' \
+  -d '{"url": "example.com", "doc_metadata": {"campaign": "march"}}'
+
+curl -X POST http://localhost:8000/api/v1/crawler/crawl-batch \
+  -H 'Content-Type: application/json' \
+  -d '{"urls": ["example.com"], "options": {"depth": 1, "max_pages": 50}}'
+
+curl http://localhost:8000/api/v1/crawler/sites
+curl "http://localhost:8000/api/v1/crawler/pages?site=example.com&limit=10"
+```
+
+A bare domain is fine anywhere a URL is taken: `example.com` becomes `https://example.com`
+on the way into the request model.
+
+## Bulk fetch example: NWVault recovery
+
+The motivating use case for the plugin: walk a local mirror of a deceased site, extract the
+identifiers each page references, fetch the corresponding archived snapshot from
+web.archive.org, and store both the body and the local-file pointer for later reconciliation.
+
+```python
+import asyncio
+from pathlib import Path
+from app.core.db import AsyncSessionLocal
+from app.services.crawler.service import CrawlerService
+
+NWVAULT_LOCAL = Path("/path/to/nwvault-legacy")
+
+
+def parse_ign_link(html_path: Path) -> tuple[int, int] | None:
+    """Pull the IGN cluster + id out of a local html page. Returns None if the
+    page doesn't reference an archived ratings link."""
+    # ... your parser ...
+    return None
+
+
+async def main() -> None:
+    crawler = CrawlerService()
+    try:
+        urls: list[str] = []
+        meta: dict[str, dict] = {}
+        for html in NWVAULT_LOCAL.rglob("*.html"):
+            parsed = parse_ign_link(html)
+            if not parsed:
+                continue
+            cluster, ign_id = parsed
+            url = (
+                "https://web.archive.org/web/2006/"
+                f"http://nwvault.ign.com/View.php?view=Ratings.Viewer"
+                f"&cluster={cluster}&id={ign_id}"
+            )
+            urls.append(url)
+            meta[url] = {
+                "local_file": str(html.relative_to(NWVAULT_LOCAL)),
+                "ign_id": ign_id,
+                "cluster": cluster,
+                "source": "ign_archive",
+            }
+
+        await crawler.batch_fetch(
+            AsyncSessionLocal,
+            urls=urls,
+            doc_metadata_for=meta,
+        )
+    finally:
+        await crawler.aclose()
+
+
+asyncio.run(main())
+```
+
+Then query by your domain identifiers:
+
+```sql
+SELECT content
+FROM crawler.crawled_page
+WHERE doc_metadata->>'ign_id' = '5208'
+ORDER BY fetched_at DESC
+LIMIT 1;
+```
+
+Failed fetches don't tank the batch; the retry queue is just:
+
+```sql
+SELECT source_url
+FROM crawler.crawled_page
+WHERE status_code IS NULL OR status_code >= 400;
+```
+
+or `<project> crawl retry-failed`.
+
+## Layout
+
+The plugin follows the host project's own conventions rather than inventing its
+own: business logic under `app/services/`, reads in a `queries` module, the HTTP
+layer with the other routers, display code with the other dashboard code.
+
+```
+aegis-stack-crawl4ai/
+├── pyproject.toml                        # entry point: aegis.plugins:crawl4ai
+├── src/aegis_stack_crawl4ai/
+│   ├── plugin.py                         # get_spec() returns the PluginSpec
+│   └── templates/{{ project_slug }}/
+│       ├── app/services/crawler/
+│       │   ├── service.py                # AsyncWebCrawler wrapper
+│       │   ├── queries.py                # every read, so an N+1 audit is one file
+│       │   ├── models.py                 # CrawledPage SQLModel
+│       │   ├── schemas.py                # Pydantic request / response
+│       │   ├── options.py                # CrawlOptions, the shared knobs
+│       │   ├── urls.py                   # bare domain -> fetchable URL
+│       │   ├── jobs.py                   # what a crawl job does
+│       │   ├── dispatch.py               # where it runs
+│       │   ├── deps.py                   # FastAPI DI, one crawler per process
+│       │   ├── settings.py               # settings mixin (leaf module)
+│       │   └── health.py                 # dashboard health check
+│       ├── app/components/backend/api/crawler/
+│       │   └── router.py                 # FastAPI router
+│       ├── app/components/frontend/dashboard/
+│       │   ├── crawler_ui.py             # names, routes and cells the surfaces share
+│       │   ├── cards/crawler_card.py
+│       │   └── modals/crawler_*.py       # modal, pages tab, crawl tab
+│       ├── app/cli/crawl.py              # crawl commands
+│       └── tests/                        # shipped into the project and run there
+└── tests/
+    └── test_plugin.py                    # spec contract tests
+```
+
+## Develop
+
+```bash
+make install     # uv sync
+make test        # pytest
+make check       # lint + test
+make fix         # ruff format + autofix
+```
+
+`tests/test_plugin.py` checks the spec contract here. The tests under
+`templates/{{ project_slug }}/tests/` ship into the generated project and run
+against the real service there, including a layout test that holds the plugin to
+the host project's structure.
+
+## License
+
+Apache 2.0, same as upstream crawl4ai.