crawlableseo 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlableseo-0.1.0/.github/workflows/ci.yml +48 -0
- crawlableseo-0.1.0/.github/workflows/release.yml +44 -0
- crawlableseo-0.1.0/.gitignore +13 -0
- crawlableseo-0.1.0/CHANGELOG.md +28 -0
- crawlableseo-0.1.0/LICENSE +21 -0
- crawlableseo-0.1.0/PKG-INFO +208 -0
- crawlableseo-0.1.0/README.md +153 -0
- crawlableseo-0.1.0/examples/fastapi_spa/app.py +134 -0
- crawlableseo-0.1.0/pyproject.toml +77 -0
- crawlableseo-0.1.0/src/crawlableseo/__init__.py +42 -0
- crawlableseo-0.1.0/src/crawlableseo/head.py +58 -0
- crawlableseo-0.1.0/src/crawlableseo/indexnow.py +111 -0
- crawlableseo-0.1.0/src/crawlableseo/integrations/__init__.py +0 -0
- crawlableseo-0.1.0/src/crawlableseo/integrations/fastapi.py +83 -0
- crawlableseo-0.1.0/src/crawlableseo/llmstxt.py +38 -0
- crawlableseo-0.1.0/src/crawlableseo/page.py +82 -0
- crawlableseo-0.1.0/src/crawlableseo/py.typed +0 -0
- crawlableseo-0.1.0/src/crawlableseo/robots.py +60 -0
- crawlableseo-0.1.0/src/crawlableseo/shell.py +134 -0
- crawlableseo-0.1.0/src/crawlableseo/shell_marker.py +7 -0
- crawlableseo-0.1.0/src/crawlableseo/site.py +270 -0
- crawlableseo-0.1.0/src/crawlableseo/sitemap.py +45 -0
- crawlableseo-0.1.0/src/crawlableseo/urls.py +36 -0
- crawlableseo-0.1.0/tests/test_fastapi_router.py +91 -0
- crawlableseo-0.1.0/tests/test_outputs.py +89 -0
- crawlableseo-0.1.0/tests/test_shell.py +134 -0
- crawlableseo-0.1.0/tests/test_site.py +110 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
- run: pip install -e ".[dev]"
|
|
24
|
+
- run: ruff check .
|
|
25
|
+
- run: mypy
|
|
26
|
+
- run: pytest -q
|
|
27
|
+
|
|
28
|
+
no_deps:
|
|
29
|
+
# The package must import and work with nothing else installed: the
|
|
30
|
+
# optional extras are optional.
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v4
|
|
34
|
+
- uses: actions/setup-python@v5
|
|
35
|
+
with:
|
|
36
|
+
python-version: "3.12"
|
|
37
|
+
- run: pip install .
|
|
38
|
+
- run: |
|
|
39
|
+
python - <<'PY'
|
|
40
|
+
from crawlableseo import Page, Site
|
|
41
|
+
site = Site("https://example.com", shell_html="<html><head><title>t</title></head><body><div id='root'></div></body></html>")
|
|
42
|
+
site.page("/", title="Home", description="A home page with a description.")
|
|
43
|
+
html, status = site.render("/")
|
|
44
|
+
assert status == 200 and "<title>Home</title>" in html
|
|
45
|
+
assert "Sitemap: https://example.com/sitemap.xml" in site.robots()
|
|
46
|
+
assert "<loc>https://example.com/</loc>" in site.sitemap()
|
|
47
|
+
print("ok")
|
|
48
|
+
PY
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Publishing is a push of a version tag, and nothing else. There is no token
|
|
4
|
+
# anywhere: PyPI trusts this workflow in this repository (trusted publishing),
|
|
5
|
+
# and the id-token permission below is what proves it.
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
tags: ["v*"]
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
build:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
- name: Refuse a tag that does not match the version in pyproject.toml
|
|
22
|
+
run: |
|
|
23
|
+
version=$(python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
24
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
25
|
+
test "$version" = "$tag" || { echo "tag $tag != version $version"; exit 1; }
|
|
26
|
+
- run: pip install build
|
|
27
|
+
- run: python -m build
|
|
28
|
+
- uses: actions/upload-artifact@v4
|
|
29
|
+
with:
|
|
30
|
+
name: dist
|
|
31
|
+
path: dist/
|
|
32
|
+
|
|
33
|
+
publish:
|
|
34
|
+
needs: build
|
|
35
|
+
runs-on: ubuntu-latest
|
|
36
|
+
environment: pypi
|
|
37
|
+
permissions:
|
|
38
|
+
id-token: write
|
|
39
|
+
steps:
|
|
40
|
+
- uses: actions/download-artifact@v4
|
|
41
|
+
with:
|
|
42
|
+
name: dist
|
|
43
|
+
path: dist/
|
|
44
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
|
|
5
|
+
[semantic versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0] - 2026-09-17
|
|
10
|
+
|
|
11
|
+
First release.
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- `Site`: one declaration of a site's crawlable surface, producing the per-URL
|
|
16
|
+
`<head>`, the crawlable body, `robots.txt`, `sitemap.xml` and `llms.txt`.
|
|
17
|
+
- `render_shell`, `head_block`, `robots_txt`, `sitemap_xml`, `llms_txt` and
|
|
18
|
+
`page_url` as plain functions, for use without `Site` or without FastAPI.
|
|
19
|
+
- `crawlableseo.integrations.fastapi.router`: the crawler-facing endpoints and
|
|
20
|
+
the SPA catch-all as an `APIRouter`.
|
|
21
|
+
- `IndexNow`: submit changed URLs, and serve the key file that authorises it.
|
|
22
|
+
- `Page.index` and `Page.follow` as separate flags, so `noindex, follow` is
|
|
23
|
+
expressible.
|
|
24
|
+
- `NotFound`, so a URL with no content answers 404 instead of a 200 with an
|
|
25
|
+
empty screen.
|
|
26
|
+
|
|
27
|
+
[Unreleased]: https://github.com/kulykivska/crawlableseo/compare/v0.1.0...HEAD
|
|
28
|
+
[0.1.0]: https://github.com/kulykivska/crawlableseo/releases/tag/v0.1.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yuliia Kulykivska
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: crawlableseo
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Make a client-rendered SPA crawlable from its own HTML shell: meta tags, JSON-LD, robots.txt, sitemap.xml and llms.txt from one declaration.
|
|
5
|
+
Project-URL: Homepage, https://github.com/kulykivska/crawlableseo
|
|
6
|
+
Project-URL: Issues, https://github.com/kulykivska/crawlableseo/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/kulykivska/crawlableseo/blob/main/CHANGELOG.md
|
|
8
|
+
Author: Yuliia Kulykivska
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2026 Yuliia Kulykivska
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: crawler,fastapi,indexnow,json-ld,llms-txt,robots,seo,sitemap,spa,starlette
|
|
32
|
+
Classifier: Development Status :: 4 - Beta
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
39
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
40
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
41
|
+
Classifier: Typing :: Typed
|
|
42
|
+
Requires-Python: >=3.10
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: fastapi>=0.100; extra == 'dev'
|
|
45
|
+
Requires-Dist: httpx>=0.24; extra == 'dev'
|
|
46
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
49
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
50
|
+
Provides-Extra: fastapi
|
|
51
|
+
Requires-Dist: fastapi>=0.100; extra == 'fastapi'
|
|
52
|
+
Provides-Extra: indexnow
|
|
53
|
+
Requires-Dist: httpx>=0.24; extra == 'indexnow'
|
|
54
|
+
Description-Content-Type: text/markdown
|
|
55
|
+
|
|
56
|
+
# crawlableseo
|
|
57
|
+
|
|
58
|
+
Make a client-rendered single-page app crawlable from its own HTML shell.
|
|
59
|
+
|
|
60
|
+
No headless browser, no SSR framework, no third-party prerendering service. Your
|
|
61
|
+
Python server fills in the shell's `<head>` and mount node for each URL, and the
|
|
62
|
+
same declaration produces `robots.txt`, `sitemap.xml` and `llms.txt`.
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install crawlableseo
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## The problem
|
|
69
|
+
|
|
70
|
+
A Vite or CRA build ships one `index.html` with an empty `<div id="root">`. Every
|
|
71
|
+
URL on the site returns the same document: the same title, the same description,
|
|
72
|
+
no content. Google renders JavaScript and will often cope; Bing is slower to; and
|
|
73
|
+
the crawlers behind AI answers — GPTBot, PerplexityBot, ClaudeBot — mostly read
|
|
74
|
+
the HTML they are given. What they are given is an empty div.
|
|
75
|
+
|
|
76
|
+
The usual answers are to adopt a meta-framework, run a headless browser per
|
|
77
|
+
request, or pay a prerendering service. This library takes the fourth option:
|
|
78
|
+
serve the same static shell, but write the real title, description, canonical,
|
|
79
|
+
Open Graph, JSON-LD and a block of readable HTML into it before it goes out.
|
|
80
|
+
|
|
81
|
+
## Quickstart
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from fastapi import FastAPI
|
|
85
|
+
from crawlableseo import DynamicUrl, NotFound, Page, Site
|
|
86
|
+
from crawlableseo.integrations.fastapi import router
|
|
87
|
+
|
|
88
|
+
site = Site(
|
|
89
|
+
"https://example.com",
|
|
90
|
+
shell="frontend/dist/index.html",
|
|
91
|
+
name="Example",
|
|
92
|
+
default_title="Example",
|
|
93
|
+
default_description="What this site is, in one sentence.",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
site.page(
|
|
97
|
+
"/pricing",
|
|
98
|
+
title="Pricing | Example",
|
|
99
|
+
description="Three plans, what each one includes, and what they cost.",
|
|
100
|
+
body="<h1>Pricing</h1><p>Free, Pro and Team...</p>",
|
|
101
|
+
priority=0.8,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
site.noindex("/admin", "/login", "/settings")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@site.dynamic("/product", urls=lambda: [DynamicUrl("/product", {"id": p.id}) for p in catalogue()])
|
|
108
|
+
def product(path: str, query: dict[str, str]) -> Page | NotFound | None:
|
|
109
|
+
item = lookup(query.get("id"))
|
|
110
|
+
if item is None:
|
|
111
|
+
return NotFound()
|
|
112
|
+
return Page(
|
|
113
|
+
title=f"{item.name} | Example",
|
|
114
|
+
description=item.summary,
|
|
115
|
+
params={"id": item.id},
|
|
116
|
+
body=f"<h1>{item.name}</h1><p>{item.summary}</p>",
|
|
117
|
+
jsonld=({"@context": "https://schema.org", "@type": "Product", "name": item.name},),
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
app = FastAPI()
|
|
122
|
+
# ... your API routes ...
|
|
123
|
+
app.include_router(router(site, disallow=["/api/admin"])) # mount last
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
That serves `/robots.txt`, `/sitemap.xml` and the SPA catch-all. Add `llms=` for
|
|
127
|
+
`/llms.txt` and `indexnow=` to publish an IndexNow key file.
|
|
128
|
+
|
|
129
|
+
Not using FastAPI? Everything underneath is a plain function:
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
from crawlableseo import Page, render_shell, robots_txt, sitemap_xml
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
## What it does
|
|
136
|
+
|
|
137
|
+
- **Per-URL `<head>`** — title, description, canonical, robots, Open Graph,
|
|
138
|
+
Twitter cards, and any number of JSON-LD objects.
|
|
139
|
+
- **Crawlable content** — HTML written into the mount node. Your framework
|
|
140
|
+
replaces the node's children when it mounts, so users never see it; a crawler
|
|
141
|
+
that does not run JavaScript reads it as the page.
|
|
142
|
+
- **`robots.txt`, `sitemap.xml`, `llms.txt`** — from the same declaration, so a
|
|
143
|
+
page cannot be in one and missing from another.
|
|
144
|
+
- **IndexNow** — submit changed URLs to Bing (and through it, ChatGPT search)
|
|
145
|
+
instead of waiting for the next crawl.
|
|
146
|
+
- **Honest status codes** — a URL with no content answers 404, not a 200 with an
|
|
147
|
+
empty screen.
|
|
148
|
+
|
|
149
|
+
## What it does not do
|
|
150
|
+
|
|
151
|
+
- It never writes your copy. You supply the text; the library places it.
|
|
152
|
+
- It does not render your JavaScript. If a page's content exists only after a
|
|
153
|
+
client-side fetch, give the resolver access to the same data on the server.
|
|
154
|
+
- It is not a meta-framework and will not become one.
|
|
155
|
+
|
|
156
|
+
## Bugs this prevents
|
|
157
|
+
|
|
158
|
+
Each of these was shipped to production on a live site before it was understood.
|
|
159
|
+
They are the reason the library exists, and every one has a test.
|
|
160
|
+
|
|
161
|
+
**A `<title>` mentioned in a build comment is not the title.** A shell carried a
|
|
162
|
+
comment explaining that the server rewrites `<title>`. A naive search-and-replace
|
|
163
|
+
matched that mention; the replacement ate the comment's closing `-->`, which
|
|
164
|
+
commented out the rest of `<head>` — stylesheet and bundle script included. The
|
|
165
|
+
site served a blank page with no console error and no failed request.
|
|
166
|
+
|
|
167
|
+
**Canonical and sitemap drift.** When the canonical tag and the sitemap entry are
|
|
168
|
+
built by two pieces of code, they disagree over a query parameter sooner or later,
|
|
169
|
+
and the two spellings become two pages with identical content. Here both come from
|
|
170
|
+
one function, and there is a test that fails if they ever differ.
|
|
171
|
+
|
|
172
|
+
**Blocking your own API starves the renderer.** `Disallow: /api/` in `robots.txt`
|
|
173
|
+
looks tidy. A crawler renders the page like a browser, so blocking the JSON the
|
|
174
|
+
app fetches leaves every route empty at render time. On one site that produced 22
|
|
175
|
+
soft 404s and 225 URLs stuck at "Discovered – currently not indexed". Disallow the
|
|
176
|
+
write and admin surfaces; leave the read-only data alone.
|
|
177
|
+
|
|
178
|
+
**`Content-Signal` costs more than it gives.** Lighthouse's `robots.txt` validator
|
|
179
|
+
does not know the directive, reports it as unknown, and takes points off the SEO
|
|
180
|
+
score of every page. Its absence already means no restriction, so this library has
|
|
181
|
+
no option to emit it.
|
|
182
|
+
|
|
183
|
+
**A `</` in your data closes the script tag.** One product name with a slash in it
|
|
184
|
+
and the JSON-LD block ends early, taking the rest of the document with it. Escaped
|
|
185
|
+
here, once.
|
|
186
|
+
|
|
187
|
+
**A backslash in a title raises a 500.** Titles built from query parameters can
|
|
188
|
+
contain anything; `re.sub` reads `\1` in a replacement string as a group
|
|
189
|
+
reference. An ordinary crafted URL becomes a server error.
|
|
190
|
+
|
|
191
|
+
**A 200 on an unknown path is a duplicate home page.** Every typo and stale link
|
|
192
|
+
becomes an indexable copy of your most important page. Unknown paths answer 404.
|
|
193
|
+
|
|
194
|
+
**`noindex` is not `nofollow`.** A page can be worth keeping out of the index while
|
|
195
|
+
its outgoing links are still worth crawling. They are separate flags.
|
|
196
|
+
|
|
197
|
+
**Do not detach the crawlable block early.** If you remove it before your framework
|
|
198
|
+
mounts, the page paints, empties, then repaints: on one site that measured a
|
|
199
|
+
cumulative layout shift of 0.28. Let the framework replace it.
|
|
200
|
+
|
|
201
|
+
## Compatibility
|
|
202
|
+
|
|
203
|
+
Python 3.10+. No required dependencies. `crawlableseo[indexnow]` adds `httpx`;
|
|
204
|
+
`crawlableseo[fastapi]` adds FastAPI for the router.
|
|
205
|
+
|
|
206
|
+
## License
|
|
207
|
+
|
|
208
|
+
MIT
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# crawlableseo
|
|
2
|
+
|
|
3
|
+
Make a client-rendered single-page app crawlable from its own HTML shell.
|
|
4
|
+
|
|
5
|
+
No headless browser, no SSR framework, no third-party prerendering service. Your
|
|
6
|
+
Python server fills in the shell's `<head>` and mount node for each URL, and the
|
|
7
|
+
same declaration produces `robots.txt`, `sitemap.xml` and `llms.txt`.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install crawlableseo
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## The problem
|
|
14
|
+
|
|
15
|
+
A Vite or CRA build ships one `index.html` with an empty `<div id="root">`. Every
|
|
16
|
+
URL on the site returns the same document: the same title, the same description,
|
|
17
|
+
no content. Google renders JavaScript and will often cope; Bing is slower to; and
|
|
18
|
+
the crawlers behind AI answers — GPTBot, PerplexityBot, ClaudeBot — mostly read
|
|
19
|
+
the HTML they are given. What they are given is an empty div.
|
|
20
|
+
|
|
21
|
+
The usual answers are to adopt a meta-framework, run a headless browser per
|
|
22
|
+
request, or pay a prerendering service. This library takes the fourth option:
|
|
23
|
+
serve the same static shell, but write the real title, description, canonical,
|
|
24
|
+
Open Graph, JSON-LD and a block of readable HTML into it before it goes out.
|
|
25
|
+
|
|
26
|
+
## Quickstart
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from fastapi import FastAPI
|
|
30
|
+
from crawlableseo import DynamicUrl, NotFound, Page, Site
|
|
31
|
+
from crawlableseo.integrations.fastapi import router
|
|
32
|
+
|
|
33
|
+
site = Site(
|
|
34
|
+
"https://example.com",
|
|
35
|
+
shell="frontend/dist/index.html",
|
|
36
|
+
name="Example",
|
|
37
|
+
default_title="Example",
|
|
38
|
+
default_description="What this site is, in one sentence.",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
site.page(
|
|
42
|
+
"/pricing",
|
|
43
|
+
title="Pricing | Example",
|
|
44
|
+
description="Three plans, what each one includes, and what they cost.",
|
|
45
|
+
body="<h1>Pricing</h1><p>Free, Pro and Team...</p>",
|
|
46
|
+
priority=0.8,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
site.noindex("/admin", "/login", "/settings")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@site.dynamic("/product", urls=lambda: [DynamicUrl("/product", {"id": p.id}) for p in catalogue()])
|
|
53
|
+
def product(path: str, query: dict[str, str]) -> Page | NotFound | None:
|
|
54
|
+
item = lookup(query.get("id"))
|
|
55
|
+
if item is None:
|
|
56
|
+
return NotFound()
|
|
57
|
+
return Page(
|
|
58
|
+
title=f"{item.name} | Example",
|
|
59
|
+
description=item.summary,
|
|
60
|
+
params={"id": item.id},
|
|
61
|
+
body=f"<h1>{item.name}</h1><p>{item.summary}</p>",
|
|
62
|
+
jsonld=({"@context": "https://schema.org", "@type": "Product", "name": item.name},),
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
app = FastAPI()
|
|
67
|
+
# ... your API routes ...
|
|
68
|
+
app.include_router(router(site, disallow=["/api/admin"])) # mount last
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
That serves `/robots.txt`, `/sitemap.xml` and the SPA catch-all. Add `llms=` for
|
|
72
|
+
`/llms.txt` and `indexnow=` to publish an IndexNow key file.
|
|
73
|
+
|
|
74
|
+
Not using FastAPI? Everything underneath is a plain function:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from crawlableseo import Page, render_shell, robots_txt, sitemap_xml
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## What it does
|
|
81
|
+
|
|
82
|
+
- **Per-URL `<head>`** — title, description, canonical, robots, Open Graph,
|
|
83
|
+
Twitter cards, and any number of JSON-LD objects.
|
|
84
|
+
- **Crawlable content** — HTML written into the mount node. Your framework
|
|
85
|
+
replaces the node's children when it mounts, so users never see it; a crawler
|
|
86
|
+
that does not run JavaScript reads it as the page.
|
|
87
|
+
- **`robots.txt`, `sitemap.xml`, `llms.txt`** — from the same declaration, so a
|
|
88
|
+
page cannot be in one and missing from another.
|
|
89
|
+
- **IndexNow** — submit changed URLs to Bing (and through it, ChatGPT search)
|
|
90
|
+
instead of waiting for the next crawl.
|
|
91
|
+
- **Honest status codes** — a URL with no content answers 404, not a 200 with an
|
|
92
|
+
empty screen.
|
|
93
|
+
|
|
94
|
+
## What it does not do
|
|
95
|
+
|
|
96
|
+
- It never writes your copy. You supply the text; the library places it.
|
|
97
|
+
- It does not render your JavaScript. If a page's content exists only after a
|
|
98
|
+
client-side fetch, give the resolver access to the same data on the server.
|
|
99
|
+
- It is not a meta-framework and will not become one.
|
|
100
|
+
|
|
101
|
+
## Bugs this prevents
|
|
102
|
+
|
|
103
|
+
Each of these was shipped to production on a live site before it was understood.
|
|
104
|
+
They are the reason the library exists, and every one has a test.
|
|
105
|
+
|
|
106
|
+
**A `<title>` mentioned in a build comment is not the title.** A shell carried a
|
|
107
|
+
comment explaining that the server rewrites `<title>`. A naive search-and-replace
|
|
108
|
+
matched that mention; the replacement ate the comment's closing `-->`, which
|
|
109
|
+
commented out the rest of `<head>` — stylesheet and bundle script included. The
|
|
110
|
+
site served a blank page with no console error and no failed request.
|
|
111
|
+
|
|
112
|
+
**Canonical and sitemap drift.** When the canonical tag and the sitemap entry are
|
|
113
|
+
built by two pieces of code, they disagree over a query parameter sooner or later,
|
|
114
|
+
and the two spellings become two pages with identical content. Here both come from
|
|
115
|
+
one function, and there is a test that fails if they ever differ.
|
|
116
|
+
|
|
117
|
+
**Blocking your own API starves the renderer.** `Disallow: /api/` in `robots.txt`
|
|
118
|
+
looks tidy. A crawler renders the page like a browser, so blocking the JSON the
|
|
119
|
+
app fetches leaves every route empty at render time. On one site that produced 22
|
|
120
|
+
soft 404s and 225 URLs stuck at "Discovered – currently not indexed". Disallow the
|
|
121
|
+
write and admin surfaces; leave the read-only data alone.
|
|
122
|
+
|
|
123
|
+
**`Content-Signal` costs more than it gives.** Lighthouse's `robots.txt` validator
|
|
124
|
+
does not know the directive, reports it as unknown, and takes points off the SEO
|
|
125
|
+
score of every page. Its absence already means no restriction, so this library has
|
|
126
|
+
no option to emit it.
|
|
127
|
+
|
|
128
|
+
**A `</` in your data closes the script tag.** One product name with a slash in it
|
|
129
|
+
and the JSON-LD block ends early, taking the rest of the document with it. Escaped
|
|
130
|
+
here, once.
|
|
131
|
+
|
|
132
|
+
**A backslash in a title raises a 500.** Titles built from query parameters can
|
|
133
|
+
contain anything; `re.sub` reads `\1` in a replacement string as a group
|
|
134
|
+
reference. An ordinary crafted URL becomes a server error.
|
|
135
|
+
|
|
136
|
+
**A 200 on an unknown path is a duplicate home page.** Every typo and stale link
|
|
137
|
+
becomes an indexable copy of your most important page. Unknown paths answer 404.
|
|
138
|
+
|
|
139
|
+
**`noindex` is not `nofollow`.** A page can be worth keeping out of the index while
|
|
140
|
+
its outgoing links are still worth crawling. They are separate flags.
|
|
141
|
+
|
|
142
|
+
**Do not detach the crawlable block early.** If you remove it before your framework
|
|
143
|
+
mounts, the page paints, empties, then repaints: on one site that measured a
|
|
144
|
+
cumulative layout shift of 0.28. Let the framework replace it.
|
|
145
|
+
|
|
146
|
+
## Compatibility
|
|
147
|
+
|
|
148
|
+
Python 3.10+. No required dependencies. `crawlableseo[indexnow]` adds `httpx`;
|
|
149
|
+
`crawlableseo[fastapi]` adds FastAPI for the router.
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
MIT
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""A runnable example: a tiny catalogue SPA made crawlable.
|
|
2
|
+
|
|
3
|
+
pip install "crawlableseo[fastapi]" uvicorn
|
|
4
|
+
uvicorn examples.fastapi_spa.app:app --reload
|
|
5
|
+
|
|
6
|
+
Then compare what a browser gets with what a crawler gets:
|
|
7
|
+
|
|
8
|
+
curl -s localhost:8000/product?id=2 | grep -E "<title>|canonical"
|
|
9
|
+
curl -s localhost:8000/sitemap.xml
|
|
10
|
+
curl -s localhost:8000/robots.txt
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
from fastapi import FastAPI
|
|
18
|
+
|
|
19
|
+
from crawlableseo import DynamicUrl, LlmsSection, NotFound, Page, Site
|
|
20
|
+
from crawlableseo.integrations.fastapi import router
|
|
21
|
+
|
|
22
|
+
BASE_URL = "http://localhost:8000"
|
|
23
|
+
|
|
24
|
+
# Whatever your build produces. Inlined here so the example runs with no
|
|
25
|
+
# frontend build step; a real app passes shell="frontend/dist/index.html".
|
|
26
|
+
SHELL = """<!doctype html>
|
|
27
|
+
<html lang="en">
|
|
28
|
+
<head>
|
|
29
|
+
<meta charset="UTF-8" />
|
|
30
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
31
|
+
<title>Catalogue</title>
|
|
32
|
+
<meta name="description" content="A catalogue." />
|
|
33
|
+
</head>
|
|
34
|
+
<body><div id="root"></div></body>
|
|
35
|
+
</html>
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Product:
|
|
41
|
+
id: str
|
|
42
|
+
name: str
|
|
43
|
+
summary: str
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
CATALOGUE = {
|
|
47
|
+
"1": Product(
|
|
48
|
+
"1", "Desk lamp", "A warm 2700K lamp with a weighted base and no visible cable."
|
|
49
|
+
),
|
|
50
|
+
"2": Product(
|
|
51
|
+
"2", "Wall clock", "A silent sweep movement in a birch case, 30 cm across."
|
|
52
|
+
),
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
site = Site(
|
|
56
|
+
BASE_URL,
|
|
57
|
+
shell_html=SHELL,
|
|
58
|
+
name="Catalogue",
|
|
59
|
+
default_title="Catalogue",
|
|
60
|
+
default_description="Two products, described properly, and crawlable without JavaScript.",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
site.page(
|
|
64
|
+
"/",
|
|
65
|
+
title="Catalogue",
|
|
66
|
+
description="Two products, described properly, and crawlable without JavaScript.",
|
|
67
|
+
body="<h1>Catalogue</h1><p>Everything we make, which is not much.</p>"
|
|
68
|
+
'<ul><li><a href="/product?id=1">Desk lamp</a></li>'
|
|
69
|
+
'<li><a href="/product?id=2">Wall clock</a></li></ul>',
|
|
70
|
+
priority=1.0,
|
|
71
|
+
changefreq="daily",
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
site.noindex("/admin")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@site.dynamic(
|
|
78
|
+
"/product",
|
|
79
|
+
urls=lambda: [
|
|
80
|
+
DynamicUrl("/product", {"id": p.id}, priority=0.8) for p in CATALOGUE.values()
|
|
81
|
+
],
|
|
82
|
+
)
|
|
83
|
+
def product(path: str, query: dict[str, str]) -> Page | NotFound | None:
|
|
84
|
+
item = CATALOGUE.get(query.get("id", ""))
|
|
85
|
+
if item is None:
|
|
86
|
+
# Not a 200 with an empty screen: that is what search engines file
|
|
87
|
+
# as a soft 404, and the crawl budget spent on it is gone.
|
|
88
|
+
return NotFound()
|
|
89
|
+
return Page(
|
|
90
|
+
title=f"{item.name} | Catalogue",
|
|
91
|
+
description=item.summary,
|
|
92
|
+
params={"id": item.id},
|
|
93
|
+
body=f"<h1>{item.name}</h1><p>{item.summary}</p>"
|
|
94
|
+
'<nav><a href="/">All products</a></nav>',
|
|
95
|
+
jsonld=(
|
|
96
|
+
{
|
|
97
|
+
"@context": "https://schema.org",
|
|
98
|
+
"@type": "Product",
|
|
99
|
+
"name": item.name,
|
|
100
|
+
"description": item.summary,
|
|
101
|
+
"url": f"{BASE_URL}/product?id={item.id}",
|
|
102
|
+
},
|
|
103
|
+
),
|
|
104
|
+
priority=0.8,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
app = FastAPI()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@app.get("/api/products")
|
|
112
|
+
def api_products() -> list[dict[str, str]]:
|
|
113
|
+
return [{"id": p.id, "name": p.name} for p in CATALOGUE.values()]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Mounted last: the catch-all answers every path the API did not claim.
|
|
117
|
+
app.include_router(
|
|
118
|
+
router(
|
|
119
|
+
site,
|
|
120
|
+
disallow=["/api/admin"],
|
|
121
|
+
llms=lambda: site.llms(
|
|
122
|
+
"A two-product catalogue, used as the example app for crawlableseo.",
|
|
123
|
+
[
|
|
124
|
+
LlmsSection(
|
|
125
|
+
"Products",
|
|
126
|
+
[
|
|
127
|
+
(p.name, f"{BASE_URL}/product?id={p.id}", p.summary)
|
|
128
|
+
for p in CATALOGUE.values()
|
|
129
|
+
],
|
|
130
|
+
)
|
|
131
|
+
],
|
|
132
|
+
),
|
|
133
|
+
)
|
|
134
|
+
)
|