sluicer 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sluicer-0.2.0/.claude-plugin/marketplace.json +15 -0
- sluicer-0.2.0/.claude-plugin/mcp.json +8 -0
- sluicer-0.2.0/.claude-plugin/plugin.json +22 -0
- sluicer-0.2.0/.dockerignore +10 -0
- sluicer-0.2.0/.github/ISSUE_TEMPLATE/bug_report.yml +44 -0
- sluicer-0.2.0/.github/ISSUE_TEMPLATE/config.yml +5 -0
- sluicer-0.2.0/.github/ISSUE_TEMPLATE/feature_request.yml +38 -0
- sluicer-0.2.0/.github/PULL_REQUEST_TEMPLATE.md +15 -0
- sluicer-0.2.0/.github/dependabot.yml +12 -0
- sluicer-0.2.0/.github/workflows/ci.yml +312 -0
- sluicer-0.2.0/.github/workflows/docs.yml +53 -0
- sluicer-0.2.0/.github/workflows/release.yml +86 -0
- sluicer-0.2.0/.gitignore +18 -0
- sluicer-0.2.0/AI_POLICY.md +28 -0
- sluicer-0.2.0/CHANGELOG.md +178 -0
- sluicer-0.2.0/CITATION.cff +25 -0
- sluicer-0.2.0/CODE_OF_CONDUCT.md +16 -0
- sluicer-0.2.0/CONTRIBUTING.md +79 -0
- sluicer-0.2.0/Dockerfile +43 -0
- sluicer-0.2.0/LICENSE +21 -0
- sluicer-0.2.0/PKG-INFO +340 -0
- sluicer-0.2.0/README.md +297 -0
- sluicer-0.2.0/ROADMAP.md +79 -0
- sluicer-0.2.0/SECURITY.md +63 -0
- sluicer-0.2.0/docs/assets/logo-dark.png +0 -0
- sluicer-0.2.0/docs/assets/logo.png +0 -0
- sluicer-0.2.0/docs/assets/mark-512.png +0 -0
- sluicer-0.2.0/docs/changelog.md +1 -0
- sluicer-0.2.0/docs/contributing.md +1 -0
- sluicer-0.2.0/docs/design-notes.md +88 -0
- sluicer-0.2.0/docs/extractors.md +126 -0
- sluicer-0.2.0/docs/field-survey.md +237 -0
- sluicer-0.2.0/docs/index.md +89 -0
- sluicer-0.2.0/docs/known-limits.md +241 -0
- sluicer-0.2.0/docs/roadmap.md +1 -0
- sluicer-0.2.0/docs/scoreboard.md +141 -0
- sluicer-0.2.0/docs/security.md +1 -0
- sluicer-0.2.0/examples/01_declared_fields.py +15 -0
- sluicer-0.2.0/examples/02_respecting_a_refusal.py +13 -0
- sluicer-0.2.0/examples/03_page_as_markdown.py +9 -0
- sluicer-0.2.0/examples/README.md +14 -0
- sluicer-0.2.0/mkdocs.yml +40 -0
- sluicer-0.2.0/pyproject.toml +144 -0
- sluicer-0.2.0/skills/sluicer/SKILL.md +135 -0
- sluicer-0.2.0/src/sluicer/__init__.py +22 -0
- sluicer-0.2.0/src/sluicer/api.py +128 -0
- sluicer-0.2.0/src/sluicer/cli.py +389 -0
- sluicer-0.2.0/src/sluicer/declared/__init__.py +1 -0
- sluicer-0.2.0/src/sluicer/declared/dublincore.py +37 -0
- sluicer-0.2.0/src/sluicer/declared/htmlmeta.py +44 -0
- sluicer-0.2.0/src/sluicer/declared/jsonld.py +200 -0
- sluicer-0.2.0/src/sluicer/declared/merge.py +241 -0
- sluicer-0.2.0/src/sluicer/declared/meta.py +93 -0
- sluicer-0.2.0/src/sluicer/declared/microdata.py +228 -0
- sluicer-0.2.0/src/sluicer/declared/microformats.py +143 -0
- sluicer-0.2.0/src/sluicer/declared/opengraph.py +53 -0
- sluicer-0.2.0/src/sluicer/declared/rdfa.py +220 -0
- sluicer-0.2.0/src/sluicer/declared/twitter.py +24 -0
- sluicer-0.2.0/src/sluicer/declared/types.py +24 -0
- sluicer-0.2.0/src/sluicer/document.py +249 -0
- sluicer-0.2.0/src/sluicer/extractor.py +802 -0
- sluicer-0.2.0/src/sluicer/extras.py +77 -0
- sluicer-0.2.0/src/sluicer/fetch/__init__.py +19 -0
- sluicer-0.2.0/src/sluicer/fetch/address.py +120 -0
- sluicer-0.2.0/src/sluicer/fetch/identity.py +142 -0
- sluicer-0.2.0/src/sluicer/fetch/ladder.py +254 -0
- sluicer-0.2.0/src/sluicer/fetch/result.py +40 -0
- sluicer-0.2.0/src/sluicer/fetch/rules.py +73 -0
- sluicer-0.2.0/src/sluicer/fetch/scrapling_rungs.py +131 -0
- sluicer-0.2.0/src/sluicer/markdown.py +54 -0
- sluicer-0.2.0/src/sluicer/mcp_server.py +314 -0
- sluicer-0.2.0/src/sluicer/py.typed +0 -0
- sluicer-0.2.0/src/sluicer/structure/__init__.py +36 -0
- sluicer-0.2.0/src/sluicer/structure/groups.py +97 -0
- sluicer-0.2.0/src/sluicer/structure/records.py +213 -0
- sluicer-0.2.0/src/sluicer/structure/shape.py +69 -0
- sluicer-0.2.0/src/sluicer/summary.py +570 -0
- sluicer-0.2.0/tests/conftest.py +82 -0
- sluicer-0.2.0/tests/fixtures/drift/product.html +1 -0
- sluicer-0.2.0/tests/fixtures/drift/product_without_jsonld.html +1 -0
- sluicer-0.2.0/tests/fixtures/drift/shop_empty.html +2 -0
- sluicer-0.2.0/tests/fixtures/drift/shop_price_is_a_button.html +8 -0
- sluicer-0.2.0/tests/fixtures/drift/shop_prices_gone.html +8 -0
- sluicer-0.2.0/tests/fixtures/drift/shop_redesigned.html +8 -0
- sluicer-0.2.0/tests/fixtures/drift/shop_v1.html +8 -0
- sluicer-0.2.0/tests/fixtures/drift/shop_v1_page2.html +7 -0
- sluicer-0.2.0/tests/fixtures/entry_microformats.html +14 -0
- sluicer-0.2.0/tests/fixtures/listing_between_sidebar_and_footer.html +47 -0
- sluicer-0.2.0/tests/fixtures/listing_no_declared_data.html +14 -0
- sluicer-0.2.0/tests/fixtures/listing_under_a_crowded_head.html +41 -0
- sluicer-0.2.0/tests/fixtures/plain.html +3 -0
- sluicer-0.2.0/tests/fixtures/product_all_three.html +19 -0
- sluicer-0.2.0/tests/fixtures/product_jsonld.html +9 -0
- sluicer-0.2.0/tests/fixtures/product_jsonld_and_a_listing.html +22 -0
- sluicer-0.2.0/tests/fixtures/product_latin1_declared.html +14 -0
- sluicer-0.2.0/tests/fixtures/product_microdata.html +7 -0
- sluicer-0.2.0/tests/fixtures/product_xhtml.html +14 -0
- sluicer-0.2.0/tests/fixtures/site_opengraph_and_a_story_list.html +36 -0
- sluicer-0.2.0/tests/test_api.py +292 -0
- sluicer-0.2.0/tests/test_cli.py +515 -0
- sluicer-0.2.0/tests/test_document.py +223 -0
- sluicer-0.2.0/tests/test_dublincore.py +69 -0
- sluicer-0.2.0/tests/test_edges.py +634 -0
- sluicer-0.2.0/tests/test_extractor.py +409 -0
- sluicer-0.2.0/tests/test_extractor_drift.py +457 -0
- sluicer-0.2.0/tests/test_extras.py +136 -0
- sluicer-0.2.0/tests/test_fetch_ladder.py +482 -0
- sluicer-0.2.0/tests/test_fetch_result.py +21 -0
- sluicer-0.2.0/tests/test_fetch_rules.py +121 -0
- sluicer-0.2.0/tests/test_groups.py +252 -0
- sluicer-0.2.0/tests/test_htmlmeta.py +89 -0
- sluicer-0.2.0/tests/test_identity.py +150 -0
- sluicer-0.2.0/tests/test_induce.py +274 -0
- sluicer-0.2.0/tests/test_jsonld.py +124 -0
- sluicer-0.2.0/tests/test_markdown.py +97 -0
- sluicer-0.2.0/tests/test_mcp_server.py +664 -0
- sluicer-0.2.0/tests/test_merge.py +517 -0
- sluicer-0.2.0/tests/test_microdata.py +129 -0
- sluicer-0.2.0/tests/test_microformats.py +386 -0
- sluicer-0.2.0/tests/test_nested.py +284 -0
- sluicer-0.2.0/tests/test_no_model_in_the_path.py +157 -0
- sluicer-0.2.0/tests/test_opengraph.py +103 -0
- sluicer-0.2.0/tests/test_package.py +153 -0
- sluicer-0.2.0/tests/test_rdfa.py +276 -0
- sluicer-0.2.0/tests/test_records.py +285 -0
- sluicer-0.2.0/tests/test_scrapling_rungs.py +311 -0
- sluicer-0.2.0/tests/test_shape.py +121 -0
- sluicer-0.2.0/tests/test_summary.py +361 -0
- sluicer-0.2.0/tests/test_twitter.py +63 -0
- sluicer-0.2.0/uv.lock +2602 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
|
|
3
|
+
"name": "sluicer",
|
|
4
|
+
"description": "Deterministic web extraction for agents: the data a page declares, with provenance, and no model in the loop.",
|
|
5
|
+
"owner": { "name": "Nichita Briculschi", "url": "https://github.com/Gi0tto" },
|
|
6
|
+
"plugins": [
|
|
7
|
+
{
|
|
8
|
+
"name": "sluicer",
|
|
9
|
+
"description": "Read what a page declares about itself, in eight vocabularies at once, and get back records where every field names the vocabulary it came from. Fetches only as expensively as it must, announces itself by name, and obeys robots.txt. Needs uv on the PATH: the MCP server runs from the plugin's own copy through uvx.",
|
|
10
|
+
"source": "./",
|
|
11
|
+
"category": "productivity",
|
|
12
|
+
"homepage": "https://github.com/Gi0tto/sluicer"
|
|
13
|
+
}
|
|
14
|
+
]
|
|
15
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "sluicer",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Read the structured data a web page already declares, with the provenance of every field, and no model in the loop. Fetches at the lowest price that works, announces itself, and obeys robots.txt. Learns extractors that fail loudly when a site changes layout, and heals them. Ships an MCP server with six tools and a skill that says when to use them.",
|
|
5
|
+
"author": { "name": "Nichita Briculschi", "url": "https://github.com/Gi0tto" },
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"homepage": "https://github.com/Gi0tto/sluicer",
|
|
8
|
+
"repository": "https://github.com/Gi0tto/sluicer",
|
|
9
|
+
"keywords": [
|
|
10
|
+
"scraping",
|
|
11
|
+
"structured-data",
|
|
12
|
+
"json-ld",
|
|
13
|
+
"microdata",
|
|
14
|
+
"opengraph",
|
|
15
|
+
"schema-org",
|
|
16
|
+
"extraction",
|
|
17
|
+
"markdown",
|
|
18
|
+
"robots-txt",
|
|
19
|
+
"deterministic"
|
|
20
|
+
],
|
|
21
|
+
"mcpServers": "./.claude-plugin/mcp.json"
|
|
22
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Everything is left out, and only what the image is built from comes back in.
|
|
2
|
+
# A list of exclusions forgets the next cache directory; a list of inclusions
|
|
3
|
+
# cannot send a .venv, a .git or a site/ it never named.
|
|
4
|
+
*
|
|
5
|
+
!pyproject.toml
|
|
6
|
+
!README.md
|
|
7
|
+
!LICENSE
|
|
8
|
+
!src/
|
|
9
|
+
**/__pycache__
|
|
10
|
+
**/*.pyc
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
name: Something read wrong
|
|
2
|
+
description: A page that Sluicer got wrong, or would not read at all
|
|
3
|
+
labels: [bug]
|
|
4
|
+
body:
|
|
5
|
+
- type: input
|
|
6
|
+
id: page
|
|
7
|
+
attributes:
|
|
8
|
+
label: The page
|
|
9
|
+
description: A URL. If it is not public, attach the HTML below instead. Without the page this is a guess.
|
|
10
|
+
placeholder: https://example.com/product
|
|
11
|
+
- type: textarea
|
|
12
|
+
id: returned
|
|
13
|
+
attributes:
|
|
14
|
+
label: What Sluicer returned
|
|
15
|
+
description: The output of `sluicer extract <page>`.
|
|
16
|
+
render: json
|
|
17
|
+
validations:
|
|
18
|
+
required: true
|
|
19
|
+
- type: textarea
|
|
20
|
+
id: declared
|
|
21
|
+
attributes:
|
|
22
|
+
label: What the page actually declares
|
|
23
|
+
description: The field, the value, and where in the source it lives.
|
|
24
|
+
validations:
|
|
25
|
+
required: true
|
|
26
|
+
- type: dropdown
|
|
27
|
+
id: kind
|
|
28
|
+
attributes:
|
|
29
|
+
label: Which is it?
|
|
30
|
+
description: A missing value is a known cost we accept in places; a wrong one is the kind of bug this project takes most seriously.
|
|
31
|
+
options:
|
|
32
|
+
- A value came back wrong
|
|
33
|
+
- A value did not come back at all
|
|
34
|
+
- The call failed or crashed
|
|
35
|
+
validations:
|
|
36
|
+
required: true
|
|
37
|
+
- type: textarea
|
|
38
|
+
id: versions
|
|
39
|
+
attributes:
|
|
40
|
+
label: Versions
|
|
41
|
+
description: "`sluicer --version`, your Python version, and which extras are installed."
|
|
42
|
+
render: text
|
|
43
|
+
validations:
|
|
44
|
+
required: true
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: Something missing
|
|
2
|
+
description: A capability Sluicer should have
|
|
3
|
+
labels: [enhancement]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: task
|
|
7
|
+
attributes:
|
|
8
|
+
label: What you were trying to do
|
|
9
|
+
description: The task, not the feature. The feature is your solution, and there may be a cheaper one.
|
|
10
|
+
validations:
|
|
11
|
+
required: true
|
|
12
|
+
- type: textarea
|
|
13
|
+
id: instead
|
|
14
|
+
attributes:
|
|
15
|
+
label: What you did instead
|
|
16
|
+
- type: dropdown
|
|
17
|
+
id: model
|
|
18
|
+
attributes:
|
|
19
|
+
label: Does it need an LLM or a paid API?
|
|
20
|
+
description: If it does, this project will say no, and it is better to know now. The constraint is deliberate and documented in CONTRIBUTING.md.
|
|
21
|
+
options:
|
|
22
|
+
- "No"
|
|
23
|
+
- "Yes"
|
|
24
|
+
- Not sure
|
|
25
|
+
validations:
|
|
26
|
+
required: true
|
|
27
|
+
- type: checkboxes
|
|
28
|
+
id: limits
|
|
29
|
+
attributes:
|
|
30
|
+
label: Known limits
|
|
31
|
+
description: docs/known-limits.md lists what we know we do not do yet. If your case is there, say below why it matters to you; that is how things get prioritised.
|
|
32
|
+
options:
|
|
33
|
+
- label: I checked docs/known-limits.md
|
|
34
|
+
required: true
|
|
35
|
+
- type: textarea
|
|
36
|
+
id: why
|
|
37
|
+
attributes:
|
|
38
|
+
label: Why it matters to you
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
**What changes, in one sentence.**
|
|
2
|
+
|
|
3
|
+
**The test that proves it.** Name it. It should fail without your change; say
|
|
4
|
+
that you watched it fail, and for which reason.
|
|
5
|
+
|
|
6
|
+
**Measured, not assumed.** If you claim it is faster, more accurate or reads
|
|
7
|
+
more pages, say by how much and against what.
|
|
8
|
+
|
|
9
|
+
**Checklist**
|
|
10
|
+
|
|
11
|
+
- [ ] `uv run pytest` passes from the repository root
|
|
12
|
+
- [ ] No network access in any test
|
|
13
|
+
- [ ] No LLM call and no paid API anywhere in the path
|
|
14
|
+
- [ ] Annotations are honest: nothing typed `object` that has a known type
|
|
15
|
+
- [ ] English in code, comments, tests and commit messages
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
# "uv" reads uv.lock; "pip" does not, and would leave the lock behind.
|
|
4
|
+
- package-ecosystem: "uv"
|
|
5
|
+
directory: "/"
|
|
6
|
+
schedule:
|
|
7
|
+
interval: "monthly"
|
|
8
|
+
open-pull-requests-limit: 5
|
|
9
|
+
- package-ecosystem: "github-actions"
|
|
10
|
+
directory: "/"
|
|
11
|
+
schedule:
|
|
12
|
+
interval: "monthly"
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
schedule:
|
|
6
|
+
# Once a week against whatever the extras resolve to today. A >=2 pin does not
|
|
7
|
+
# stop a 3.0 landing with another rename, and that would otherwise be found by
|
|
8
|
+
# a user rather than by us.
|
|
9
|
+
- cron: "0 6 * * 1"
|
|
10
|
+
push:
|
|
11
|
+
branches: [main]
|
|
12
|
+
paths-ignore: ["**/*.md", "docs/**"]
|
|
13
|
+
pull_request:
|
|
14
|
+
branches: [main]
|
|
15
|
+
paths-ignore: ["**/*.md", "docs/**"]
|
|
16
|
+
|
|
17
|
+
permissions:
|
|
18
|
+
contents: read
|
|
19
|
+
|
|
20
|
+
concurrency:
|
|
21
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
22
|
+
cancel-in-progress: true
|
|
23
|
+
|
|
24
|
+
jobs:
|
|
25
|
+
tests:
|
|
26
|
+
name: Tests on Python ${{ matrix.python }}
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
strategy:
|
|
29
|
+
fail-fast: false
|
|
30
|
+
matrix:
|
|
31
|
+
python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v7
|
|
34
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
35
|
+
with:
|
|
36
|
+
enable-cache: true
|
|
37
|
+
- name: Run the suite
|
|
38
|
+
run: uv run --python ${{ matrix.python }} pytest -q
|
|
39
|
+
|
|
40
|
+
quality:
|
|
41
|
+
# Separate from the matrix above on purpose: a red cross here names which
|
|
42
|
+
# gate refused the commit, and each step names which of the three it was.
|
|
43
|
+
# Four human review passes were the only thing standing between a defect
|
|
44
|
+
# and this repository, and a wrong type annotation still got past one.
|
|
45
|
+
name: Lint, types and coverage
|
|
46
|
+
runs-on: ubuntu-latest
|
|
47
|
+
steps:
|
|
48
|
+
- uses: actions/checkout@v7
|
|
49
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
50
|
+
with:
|
|
51
|
+
enable-cache: true
|
|
52
|
+
- name: Lint
|
|
53
|
+
run: uv run --python 3.13 ruff check src tests
|
|
54
|
+
- name: Types
|
|
55
|
+
# Reads its own settings from pyproject: strict, against src, at the
|
|
56
|
+
# 3.10 floor this package declares whatever interpreter is running it.
|
|
57
|
+
run: uv run --python 3.13 mypy
|
|
58
|
+
- name: Coverage
|
|
59
|
+
# One interpreter, so the floor is measured against the same thing
|
|
60
|
+
# every time. The matrix above is what proves the suite passes on all
|
|
61
|
+
# four versions.
|
|
62
|
+
run: uv run --python 3.13 pytest -q --cov --cov-report=term-missing
|
|
63
|
+
|
|
64
|
+
offline:
|
|
65
|
+
name: The suite opens no socket
|
|
66
|
+
runs-on: ubuntu-latest
|
|
67
|
+
steps:
|
|
68
|
+
- uses: actions/checkout@v7
|
|
69
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
70
|
+
- name: Run the suite with the network taken away
|
|
71
|
+
run: |
|
|
72
|
+
cat > /tmp/no_network.py <<'PY'
|
|
73
|
+
"""Fail the build if any test tries to reach the network.
|
|
74
|
+
|
|
75
|
+
The project's first promise is that a run costs CPU and nothing else,
|
|
76
|
+
and a promise nothing enforces is a wish.
|
|
77
|
+
|
|
78
|
+
This blocks connecting, not the socket type itself. Replacing
|
|
79
|
+
socket.socket wholesale looks stricter and is simply wrong: the stdlib
|
|
80
|
+
ssl module does `class SSLSocket(socket)`, so anything importing it
|
|
81
|
+
dies with a TypeError that has nothing to do with the network. Ask
|
|
82
|
+
what a test is forbidden to DO, not what it may name.
|
|
83
|
+
"""
|
|
84
|
+
import socket
|
|
85
|
+
import sys
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Blocked(RuntimeError):
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def refuse(*args, **kwargs):
|
|
93
|
+
raise Blocked(
|
|
94
|
+
"a test tried to reach the network; the suite must never do that"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
socket.socket.connect = refuse
|
|
99
|
+
socket.socket.connect_ex = refuse
|
|
100
|
+
socket.create_connection = refuse
|
|
101
|
+
socket.getaddrinfo = refuse
|
|
102
|
+
|
|
103
|
+
import pytest
|
|
104
|
+
|
|
105
|
+
sys.exit(pytest.main(["-q"]))
|
|
106
|
+
PY
|
|
107
|
+
uv run python /tmp/no_network.py
|
|
108
|
+
|
|
109
|
+
build:
|
|
110
|
+
name: The package builds and its metadata parses
|
|
111
|
+
runs-on: ubuntu-latest
|
|
112
|
+
steps:
|
|
113
|
+
- uses: actions/checkout@v7
|
|
114
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
115
|
+
- name: Build the wheel and the sdist
|
|
116
|
+
run: uv build
|
|
117
|
+
- name: Check the metadata
|
|
118
|
+
run: uvx twine check dist/*
|
|
119
|
+
|
|
120
|
+
import-without-extras:
|
|
121
|
+
name: Importing works without any extra
|
|
122
|
+
runs-on: ubuntu-latest
|
|
123
|
+
steps:
|
|
124
|
+
- uses: actions/checkout@v7
|
|
125
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
126
|
+
- name: Install the base package only, then import it
|
|
127
|
+
run: |
|
|
128
|
+
uv venv
|
|
129
|
+
uv pip install .
|
|
130
|
+
# `.venv/bin/python`, not `uv run`, in this job and the two below:
|
|
131
|
+
# `uv run` re-syncs the environment to uv.lock and installs the source
|
|
132
|
+
# tree in its place, so the check would run against the dev
|
|
133
|
+
# environment rather than the install it names -- the trap the
|
|
134
|
+
# floors job documents.
|
|
135
|
+
.venv/bin/python -c "
|
|
136
|
+
import importlib.util, sluicer
|
|
137
|
+
for extra in ('scrapling', 'trafilatura', 'mcp'):
|
|
138
|
+
assert importlib.util.find_spec(extra) is None, extra + ' leaked into the base install'
|
|
139
|
+
print('base install imports cleanly, version', sluicer.__version__)
|
|
140
|
+
"
|
|
141
|
+
|
|
142
|
+
with-extras:
|
|
143
|
+
name: The extras install and the code can actually use them
|
|
144
|
+
runs-on: ubuntu-latest
|
|
145
|
+
steps:
|
|
146
|
+
- uses: actions/checkout@v7
|
|
147
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
148
|
+
- name: Install every extra
|
|
149
|
+
run: |
|
|
150
|
+
uv venv
|
|
151
|
+
uv pip install '.[fetch,markdown,mcp]'
|
|
152
|
+
- name: Use each one against the package that was really installed
|
|
153
|
+
run: |
|
|
154
|
+
.venv/bin/python -c "
|
|
155
|
+
# The suite fakes these libraries, and a fake can only confirm the shape
|
|
156
|
+
# you already believe in. This job is the only place that finds out what
|
|
157
|
+
# the published package actually ships. It caught the mcp 2.x rename.
|
|
158
|
+
from sluicer.markdown import to_markdown
|
|
159
|
+
page = '<html><head><meta charset=\"windows-1252\"></head><body><article><h1>Caf\xe9</h1><p>' + ('Real sentences. ' * 30) + '</p></article></body></html>'
|
|
160
|
+
out = to_markdown(page.encode('windows-1252'))
|
|
161
|
+
assert 'Caf\xe9' in out, 'the markdown path lost the accent: ' + out[:80]
|
|
162
|
+
print('markdown: accent survived')
|
|
163
|
+
|
|
164
|
+
# protego parses robots.txt and arrives with scrapling[fetchers]. The unit
|
|
165
|
+
# tests fake it, and a fake agrees with whatever you wrote it to agree with,
|
|
166
|
+
# so the real semantics are only ever checked here.
|
|
167
|
+
from sluicer.fetch.identity import USER_AGENT, robots_allows
|
|
168
|
+
assert 'Sluicer' in USER_AGENT and 'github.com' in USER_AGENT, USER_AGENT
|
|
169
|
+
refuse = 'User-agent: Sluicer\nDisallow: /private/\n\nUser-agent: *\nAllow: /\n'
|
|
170
|
+
assert robots_allows('https://example.com/private/p', read=lambda u: refuse, cache={}) is False
|
|
171
|
+
assert robots_allows('https://example.com/public', read=lambda u: refuse, cache={}) is True
|
|
172
|
+
assert robots_allows('https://example.com/p', read=lambda u: None, cache={}) is True
|
|
173
|
+
print('robots: the real parser agrees with what the suite believes')
|
|
174
|
+
|
|
175
|
+
from sluicer.fetch.scrapling_rungs import default_rungs
|
|
176
|
+
names = [name for name, _ in default_rungs()]
|
|
177
|
+
assert names[:2] == ['http', 'browser'], names
|
|
178
|
+
print('fetch: three rungs built against the real scrapling')
|
|
179
|
+
|
|
180
|
+
import asyncio
|
|
181
|
+
from sluicer.mcp_server import build_server
|
|
182
|
+
server = build_server()
|
|
183
|
+
tools = sorted(t.name for t in asyncio.run(server.list_tools()))
|
|
184
|
+
assert tools == ['compile_extractor', 'extract_declared', 'fetch_page', 'heal_extractor', 'page_markdown', 'run_extractor'], tools
|
|
185
|
+
|
|
186
|
+
# Registration is not use. The SDK's dispatch is where a future release
|
|
187
|
+
# can break us while every import still resolves, so call a tool for real.
|
|
188
|
+
page = '<html><head><script type=\"application/ld+json\">{\"@type\":\"Product\",\"name\":\"Brake pad set\"}</script></head><body></body></html>'
|
|
189
|
+
answer = asyncio.run(server.call_tool('extract_declared', {'html_or_url': page}))
|
|
190
|
+
text = str(answer)
|
|
191
|
+
assert 'Brake pad set' in text, 'the SDK dispatch lost the record: ' + text[:200]
|
|
192
|
+
assert 'jsonld' in text, 'the provenance did not survive dispatch: ' + text[:200]
|
|
193
|
+
|
|
194
|
+
markdown_answer = asyncio.run(server.call_tool('page_markdown', {'html_or_url': '<html><body><article><h1>Title</h1><p>' + ('Real sentences. ' * 30) + '</p></article></body></html>'}))
|
|
195
|
+
assert 'Title' in str(markdown_answer), str(markdown_answer)[:200]
|
|
196
|
+
print('mcp: three tools listed, and two of them answered through the SDK dispatch')
|
|
197
|
+
"
|
|
198
|
+
|
|
199
|
+
floors:
|
|
200
|
+
name: The declared minimum versions actually work
|
|
201
|
+
runs-on: ubuntu-latest
|
|
202
|
+
steps:
|
|
203
|
+
- uses: actions/checkout@v7
|
|
204
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
205
|
+
# A dependency floor nobody installs is a wish, not a constraint. Both
|
|
206
|
+
# base floors were wrong when this job was written: lxml 5.0 ships no
|
|
207
|
+
# wheel for cp313 and none at all for macOS arm64, and click 8.1.8 fails
|
|
208
|
+
# fourteen tests because its CliRunner does not capture stderr
|
|
209
|
+
# separately. Both had been declared for months and never executed.
|
|
210
|
+
#
|
|
211
|
+
# Pinned to the minimum Python this project claims: on a newer
|
|
212
|
+
# interpreter the resolver climbs above the floor, because the floor
|
|
213
|
+
# version has no wheel for it, and the job would pass without ever
|
|
214
|
+
# installing what it exists to test.
|
|
215
|
+
#
|
|
216
|
+
# The base is resolved on its own. Installing the extras alongside it
|
|
217
|
+
# lifts lxml and click above their floors, because the extras depend on
|
|
218
|
+
# them too.
|
|
219
|
+
#
|
|
220
|
+
# `.venv/bin/python` throughout, never `uv run`: `uv run` re-syncs the
|
|
221
|
+
# environment from pyproject.toml and reinstalls lxml at its newest,
|
|
222
|
+
# which silently undoes the pin. The assertion below caught exactly that
|
|
223
|
+
# on this job's first run, which is why it is an assertion and not a
|
|
224
|
+
# printed list somebody skims.
|
|
225
|
+
- name: Install the base at its declared floors
|
|
226
|
+
run: |
|
|
227
|
+
uv venv --python 3.10
|
|
228
|
+
uv pip install --resolution lowest-direct .
|
|
229
|
+
uv pip install pytest
|
|
230
|
+
- name: Check those really are the floors, not something above them
|
|
231
|
+
run: |
|
|
232
|
+
.venv/bin/python -c "
|
|
233
|
+
import importlib.metadata as md
|
|
234
|
+
for name, floor in {'lxml': '5.3', 'click': '8.2'}.items():
|
|
235
|
+
got = md.version(name)
|
|
236
|
+
assert got.startswith(floor), (
|
|
237
|
+
name + ' resolved to ' + got + ', not the declared floor '
|
|
238
|
+
+ floor + ': this job is not testing what it claims to test'
|
|
239
|
+
)
|
|
240
|
+
print(name, got, '-- the declared floor')
|
|
241
|
+
"
|
|
242
|
+
- name: Run the suite against them
|
|
243
|
+
run: .venv/bin/python -m pytest -q
|
|
244
|
+
- name: And again with every extra at its own lowest version
|
|
245
|
+
run: |
|
|
246
|
+
uv pip install --resolution lowest-direct '.[fetch,markdown,mcp,microformats]'
|
|
247
|
+
uv pip list
|
|
248
|
+
.venv/bin/python -m pytest -q
|
|
249
|
+
|
|
250
|
+
latest-extras:
|
|
251
|
+
name: The extras still work at their newest versions
|
|
252
|
+
runs-on: ubuntu-latest
|
|
253
|
+
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
254
|
+
steps:
|
|
255
|
+
- uses: actions/checkout@v7
|
|
256
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
257
|
+
- name: Install the extras unpinned and use them
|
|
258
|
+
run: |
|
|
259
|
+
uv venv
|
|
260
|
+
uv pip install '.[fetch,markdown,mcp]' --upgrade
|
|
261
|
+
.venv/bin/python -c "
|
|
262
|
+
import asyncio, importlib.metadata as md
|
|
263
|
+
for name in ('scrapling', 'trafilatura', 'mcp'):
|
|
264
|
+
print(name, md.version(name))
|
|
265
|
+
from sluicer.mcp_server import build_server
|
|
266
|
+
server = build_server()
|
|
267
|
+
tools = sorted(t.name for t in asyncio.run(server.list_tools()))
|
|
268
|
+
assert tools == ['compile_extractor', 'extract_declared', 'fetch_page', 'heal_extractor', 'page_markdown', 'run_extractor'], tools
|
|
269
|
+
print('still building against the newest published extras')
|
|
270
|
+
"
|
|
271
|
+
|
|
272
|
+
docker:
|
|
273
|
+
name: The image builds, runs unprivileged, and reads a page
|
|
274
|
+
runs-on: ubuntu-latest
|
|
275
|
+
steps:
|
|
276
|
+
- uses: actions/checkout@v7
|
|
277
|
+
- name: Build the image
|
|
278
|
+
run: docker build -t sluicer:ci .
|
|
279
|
+
- name: Use it
|
|
280
|
+
# A page handed to it needs no network, so this proves the install in
|
|
281
|
+
# the image without depending on any site being up.
|
|
282
|
+
run: |
|
|
283
|
+
uid=$(docker run --rm sluicer:ci id -u)
|
|
284
|
+
test "$uid" != 0 || { echo "the image runs as root"; exit 1; }
|
|
285
|
+
printf '%s' '<script type="application/ld+json">{"@type":"Product","name":"Pad"}</script>' > page.html
|
|
286
|
+
docker run --rm -v "$PWD/page.html:/tmp/page.html:ro" sluicer:ci \
|
|
287
|
+
sluicer extract /tmp/page.html | tee out.json
|
|
288
|
+
grep -q '"Pad"' out.json
|
|
289
|
+
|
|
290
|
+
docker-browser:
|
|
291
|
+
# The browser variant is most of a gigabyte and minutes to build, so it is
|
|
292
|
+
# proven weekly rather than on every push -- but proven, because a build
|
|
293
|
+
# argument nobody builds is a claim in a comment.
|
|
294
|
+
name: The image builds with its browser
|
|
295
|
+
runs-on: ubuntu-latest
|
|
296
|
+
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
|
297
|
+
steps:
|
|
298
|
+
- uses: actions/checkout@v7
|
|
299
|
+
- name: Build the image with Chromium
|
|
300
|
+
run: docker build --build-arg WITH_BROWSER=1 -t sluicer:browser .
|
|
301
|
+
- name: The unprivileged user can launch the browser
|
|
302
|
+
run: |
|
|
303
|
+
docker run --rm sluicer:browser python -c "
|
|
304
|
+
from playwright.sync_api import sync_playwright
|
|
305
|
+
with sync_playwright() as p:
|
|
306
|
+
browser = p.chromium.launch()
|
|
307
|
+
page = browser.new_page()
|
|
308
|
+
page.set_content('<p>ok</p>')
|
|
309
|
+
assert page.inner_text('p') == 'ok'
|
|
310
|
+
browser.close()
|
|
311
|
+
print('chromium launches as', __import__('os').getuid())
|
|
312
|
+
"
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: Docs
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
paths: ["docs/**", "mkdocs.yml", "README.md", "ROADMAP.md"]
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
pages: write
|
|
12
|
+
id-token: write
|
|
13
|
+
|
|
14
|
+
concurrency:
|
|
15
|
+
group: pages
|
|
16
|
+
cancel-in-progress: false
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
# Two jobs rather than one, because they answer different questions. Building
|
|
20
|
+
# is a test: --strict fails on a broken link or a page nothing references, and
|
|
21
|
+
# it has to run on every push whatever the repository's visibility. Deploying
|
|
22
|
+
# is a publication, and GitHub Pages is unavailable on a private repository
|
|
23
|
+
# outside a paid plan -- so gating the build on that would mean a broken doc
|
|
24
|
+
# could land unnoticed while the repository is private, which is exactly when
|
|
25
|
+
# the docs are being written.
|
|
26
|
+
build:
|
|
27
|
+
name: The documentation builds
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@v7
|
|
31
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
32
|
+
- name: Build the site
|
|
33
|
+
# Pinned below 2 on both: MkDocs 2.0 is announced as a breaking release
|
|
34
|
+
# and mkdocs-material already warns about it, so an unpinned uvx would
|
|
35
|
+
# change what --strict means on the day it lands, not on a day we chose.
|
|
36
|
+
run: uvx --from 'mkdocs>=1.6,<2' --with 'mkdocs-material>=9.7,<10' mkdocs build --strict
|
|
37
|
+
- uses: actions/upload-pages-artifact@v5
|
|
38
|
+
with:
|
|
39
|
+
path: site
|
|
40
|
+
|
|
41
|
+
deploy:
|
|
42
|
+
name: The documentation is published
|
|
43
|
+
needs: build
|
|
44
|
+
# Pages refuses a private repository on the free plan with a 404 that reads
|
|
45
|
+
# like a missing artifact. Skipping is the honest report: nothing is wrong.
|
|
46
|
+
if: github.event.repository.visibility == 'public'
|
|
47
|
+
runs-on: ubuntu-latest
|
|
48
|
+
environment:
|
|
49
|
+
name: github-pages
|
|
50
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
51
|
+
steps:
|
|
52
|
+
- id: deployment
|
|
53
|
+
uses: actions/deploy-pages@v5
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
# Nothing here writes to the repository. The one elevated permission is the
|
|
9
|
+
# OIDC token trusted publishing needs, and only the job that publishes gets it.
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
build:
|
|
15
|
+
name: Build and check the distributions
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v7
|
|
19
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
20
|
+
- name: The tag and the version must agree
|
|
21
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
22
|
+
run: |
|
|
23
|
+
uv run python -c "
|
|
24
|
+
import os, tomllib, pathlib
|
|
25
|
+
declared = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version']
|
|
26
|
+
tagged = os.environ['GITHUB_REF_NAME'].removeprefix('v')
|
|
27
|
+
assert declared == tagged, f'pyproject says {declared}, the tag says {tagged}'
|
|
28
|
+
print('version agrees:', declared)
|
|
29
|
+
"
|
|
30
|
+
- name: The suite must pass before anything is published
|
|
31
|
+
run: uv run --python 3.13 pytest -q
|
|
32
|
+
- run: uv build
|
|
33
|
+
- name: Check the metadata the way PyPI will
|
|
34
|
+
run: uvx twine check dist/*
|
|
35
|
+
- uses: actions/upload-artifact@v7
|
|
36
|
+
with:
|
|
37
|
+
name: distributions
|
|
38
|
+
path: dist/
|
|
39
|
+
|
|
40
|
+
installs:
|
|
41
|
+
name: The built wheel installs and works on its own
|
|
42
|
+
needs: build
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
steps:
|
|
45
|
+
# No checkout: the wheel has to stand on its own, and a source tree in the
|
|
46
|
+
# working directory is what an import would quietly find instead of it.
|
|
47
|
+
- uses: actions/download-artifact@v8
|
|
48
|
+
with:
|
|
49
|
+
name: distributions
|
|
50
|
+
path: dist/
|
|
51
|
+
- uses: astral-sh/setup-uv@v10.2.0
|
|
52
|
+
- name: Install the wheel in a clean environment and use it
|
|
53
|
+
run: |
|
|
54
|
+
uv venv --python 3.13
|
|
55
|
+
uv pip install dist/*.whl
|
|
56
|
+
.venv/bin/python -c "
|
|
57
|
+
import sluicer
|
|
58
|
+
page = '<script type=\"application/ld+json\">{\"@type\":\"Product\",\"name\":\"Pad\"}</script>'
|
|
59
|
+
result = sluicer.extract(page)
|
|
60
|
+
assert result.records and result.records[0].type == 'Product', result
|
|
61
|
+
assert sluicer.__file__.startswith('$PWD/.venv/'), sluicer.__file__
|
|
62
|
+
print('wheel', sluicer.__version__, 'installs, imports and extracts')
|
|
63
|
+
"
|
|
64
|
+
.venv/bin/sluicer --version
|
|
65
|
+
|
|
66
|
+
publish:
|
|
67
|
+
name: Publish to PyPI
|
|
68
|
+
needs: [build, installs]
|
|
69
|
+
runs-on: ubuntu-latest
|
|
70
|
+
# Off until the PyPI side exists: a trusted publisher for this repository
|
|
71
|
+
# and workflow, registered at pypi.org, and the repository variable
|
|
72
|
+
# PUBLISH_TO_PYPI set to true. Until then a tag builds and checks the
|
|
73
|
+
# distributions and stops here, instead of failing on a missing publisher.
|
|
74
|
+
if: startsWith(github.ref, 'refs/tags/v') && vars.PUBLISH_TO_PYPI == 'true'
|
|
75
|
+
environment: pypi
|
|
76
|
+
permissions:
|
|
77
|
+
id-token: write
|
|
78
|
+
steps:
|
|
79
|
+
- uses: actions/download-artifact@v8
|
|
80
|
+
with:
|
|
81
|
+
name: distributions
|
|
82
|
+
path: dist/
|
|
83
|
+
# Trusted publishing: PyPI verifies this workflow's identity, so no API
|
|
84
|
+
# token is stored anywhere. Configure it once at
|
|
85
|
+
# https://pypi.org/manage/project/sluicer/settings/publishing/
|
|
86
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
sluicer-0.2.0/.gitignore
ADDED