sluicer 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. sluicer-0.2.0/.claude-plugin/marketplace.json +15 -0
  2. sluicer-0.2.0/.claude-plugin/mcp.json +8 -0
  3. sluicer-0.2.0/.claude-plugin/plugin.json +22 -0
  4. sluicer-0.2.0/.dockerignore +10 -0
  5. sluicer-0.2.0/.github/ISSUE_TEMPLATE/bug_report.yml +44 -0
  6. sluicer-0.2.0/.github/ISSUE_TEMPLATE/config.yml +5 -0
  7. sluicer-0.2.0/.github/ISSUE_TEMPLATE/feature_request.yml +38 -0
  8. sluicer-0.2.0/.github/PULL_REQUEST_TEMPLATE.md +15 -0
  9. sluicer-0.2.0/.github/dependabot.yml +12 -0
  10. sluicer-0.2.0/.github/workflows/ci.yml +312 -0
  11. sluicer-0.2.0/.github/workflows/docs.yml +53 -0
  12. sluicer-0.2.0/.github/workflows/release.yml +86 -0
  13. sluicer-0.2.0/.gitignore +18 -0
  14. sluicer-0.2.0/AI_POLICY.md +28 -0
  15. sluicer-0.2.0/CHANGELOG.md +178 -0
  16. sluicer-0.2.0/CITATION.cff +25 -0
  17. sluicer-0.2.0/CODE_OF_CONDUCT.md +16 -0
  18. sluicer-0.2.0/CONTRIBUTING.md +79 -0
  19. sluicer-0.2.0/Dockerfile +43 -0
  20. sluicer-0.2.0/LICENSE +21 -0
  21. sluicer-0.2.0/PKG-INFO +340 -0
  22. sluicer-0.2.0/README.md +297 -0
  23. sluicer-0.2.0/ROADMAP.md +79 -0
  24. sluicer-0.2.0/SECURITY.md +63 -0
  25. sluicer-0.2.0/docs/assets/logo-dark.png +0 -0
  26. sluicer-0.2.0/docs/assets/logo.png +0 -0
  27. sluicer-0.2.0/docs/assets/mark-512.png +0 -0
  28. sluicer-0.2.0/docs/changelog.md +1 -0
  29. sluicer-0.2.0/docs/contributing.md +1 -0
  30. sluicer-0.2.0/docs/design-notes.md +88 -0
  31. sluicer-0.2.0/docs/extractors.md +126 -0
  32. sluicer-0.2.0/docs/field-survey.md +237 -0
  33. sluicer-0.2.0/docs/index.md +89 -0
  34. sluicer-0.2.0/docs/known-limits.md +241 -0
  35. sluicer-0.2.0/docs/roadmap.md +1 -0
  36. sluicer-0.2.0/docs/scoreboard.md +141 -0
  37. sluicer-0.2.0/docs/security.md +1 -0
  38. sluicer-0.2.0/examples/01_declared_fields.py +15 -0
  39. sluicer-0.2.0/examples/02_respecting_a_refusal.py +13 -0
  40. sluicer-0.2.0/examples/03_page_as_markdown.py +9 -0
  41. sluicer-0.2.0/examples/README.md +14 -0
  42. sluicer-0.2.0/mkdocs.yml +40 -0
  43. sluicer-0.2.0/pyproject.toml +144 -0
  44. sluicer-0.2.0/skills/sluicer/SKILL.md +135 -0
  45. sluicer-0.2.0/src/sluicer/__init__.py +22 -0
  46. sluicer-0.2.0/src/sluicer/api.py +128 -0
  47. sluicer-0.2.0/src/sluicer/cli.py +389 -0
  48. sluicer-0.2.0/src/sluicer/declared/__init__.py +1 -0
  49. sluicer-0.2.0/src/sluicer/declared/dublincore.py +37 -0
  50. sluicer-0.2.0/src/sluicer/declared/htmlmeta.py +44 -0
  51. sluicer-0.2.0/src/sluicer/declared/jsonld.py +200 -0
  52. sluicer-0.2.0/src/sluicer/declared/merge.py +241 -0
  53. sluicer-0.2.0/src/sluicer/declared/meta.py +93 -0
  54. sluicer-0.2.0/src/sluicer/declared/microdata.py +228 -0
  55. sluicer-0.2.0/src/sluicer/declared/microformats.py +143 -0
  56. sluicer-0.2.0/src/sluicer/declared/opengraph.py +53 -0
  57. sluicer-0.2.0/src/sluicer/declared/rdfa.py +220 -0
  58. sluicer-0.2.0/src/sluicer/declared/twitter.py +24 -0
  59. sluicer-0.2.0/src/sluicer/declared/types.py +24 -0
  60. sluicer-0.2.0/src/sluicer/document.py +249 -0
  61. sluicer-0.2.0/src/sluicer/extractor.py +802 -0
  62. sluicer-0.2.0/src/sluicer/extras.py +77 -0
  63. sluicer-0.2.0/src/sluicer/fetch/__init__.py +19 -0
  64. sluicer-0.2.0/src/sluicer/fetch/address.py +120 -0
  65. sluicer-0.2.0/src/sluicer/fetch/identity.py +142 -0
  66. sluicer-0.2.0/src/sluicer/fetch/ladder.py +254 -0
  67. sluicer-0.2.0/src/sluicer/fetch/result.py +40 -0
  68. sluicer-0.2.0/src/sluicer/fetch/rules.py +73 -0
  69. sluicer-0.2.0/src/sluicer/fetch/scrapling_rungs.py +131 -0
  70. sluicer-0.2.0/src/sluicer/markdown.py +54 -0
  71. sluicer-0.2.0/src/sluicer/mcp_server.py +314 -0
  72. sluicer-0.2.0/src/sluicer/py.typed +0 -0
  73. sluicer-0.2.0/src/sluicer/structure/__init__.py +36 -0
  74. sluicer-0.2.0/src/sluicer/structure/groups.py +97 -0
  75. sluicer-0.2.0/src/sluicer/structure/records.py +213 -0
  76. sluicer-0.2.0/src/sluicer/structure/shape.py +69 -0
  77. sluicer-0.2.0/src/sluicer/summary.py +570 -0
  78. sluicer-0.2.0/tests/conftest.py +82 -0
  79. sluicer-0.2.0/tests/fixtures/drift/product.html +1 -0
  80. sluicer-0.2.0/tests/fixtures/drift/product_without_jsonld.html +1 -0
  81. sluicer-0.2.0/tests/fixtures/drift/shop_empty.html +2 -0
  82. sluicer-0.2.0/tests/fixtures/drift/shop_price_is_a_button.html +8 -0
  83. sluicer-0.2.0/tests/fixtures/drift/shop_prices_gone.html +8 -0
  84. sluicer-0.2.0/tests/fixtures/drift/shop_redesigned.html +8 -0
  85. sluicer-0.2.0/tests/fixtures/drift/shop_v1.html +8 -0
  86. sluicer-0.2.0/tests/fixtures/drift/shop_v1_page2.html +7 -0
  87. sluicer-0.2.0/tests/fixtures/entry_microformats.html +14 -0
  88. sluicer-0.2.0/tests/fixtures/listing_between_sidebar_and_footer.html +47 -0
  89. sluicer-0.2.0/tests/fixtures/listing_no_declared_data.html +14 -0
  90. sluicer-0.2.0/tests/fixtures/listing_under_a_crowded_head.html +41 -0
  91. sluicer-0.2.0/tests/fixtures/plain.html +3 -0
  92. sluicer-0.2.0/tests/fixtures/product_all_three.html +19 -0
  93. sluicer-0.2.0/tests/fixtures/product_jsonld.html +9 -0
  94. sluicer-0.2.0/tests/fixtures/product_jsonld_and_a_listing.html +22 -0
  95. sluicer-0.2.0/tests/fixtures/product_latin1_declared.html +14 -0
  96. sluicer-0.2.0/tests/fixtures/product_microdata.html +7 -0
  97. sluicer-0.2.0/tests/fixtures/product_xhtml.html +14 -0
  98. sluicer-0.2.0/tests/fixtures/site_opengraph_and_a_story_list.html +36 -0
  99. sluicer-0.2.0/tests/test_api.py +292 -0
  100. sluicer-0.2.0/tests/test_cli.py +515 -0
  101. sluicer-0.2.0/tests/test_document.py +223 -0
  102. sluicer-0.2.0/tests/test_dublincore.py +69 -0
  103. sluicer-0.2.0/tests/test_edges.py +634 -0
  104. sluicer-0.2.0/tests/test_extractor.py +409 -0
  105. sluicer-0.2.0/tests/test_extractor_drift.py +457 -0
  106. sluicer-0.2.0/tests/test_extras.py +136 -0
  107. sluicer-0.2.0/tests/test_fetch_ladder.py +482 -0
  108. sluicer-0.2.0/tests/test_fetch_result.py +21 -0
  109. sluicer-0.2.0/tests/test_fetch_rules.py +121 -0
  110. sluicer-0.2.0/tests/test_groups.py +252 -0
  111. sluicer-0.2.0/tests/test_htmlmeta.py +89 -0
  112. sluicer-0.2.0/tests/test_identity.py +150 -0
  113. sluicer-0.2.0/tests/test_induce.py +274 -0
  114. sluicer-0.2.0/tests/test_jsonld.py +124 -0
  115. sluicer-0.2.0/tests/test_markdown.py +97 -0
  116. sluicer-0.2.0/tests/test_mcp_server.py +664 -0
  117. sluicer-0.2.0/tests/test_merge.py +517 -0
  118. sluicer-0.2.0/tests/test_microdata.py +129 -0
  119. sluicer-0.2.0/tests/test_microformats.py +386 -0
  120. sluicer-0.2.0/tests/test_nested.py +284 -0
  121. sluicer-0.2.0/tests/test_no_model_in_the_path.py +157 -0
  122. sluicer-0.2.0/tests/test_opengraph.py +103 -0
  123. sluicer-0.2.0/tests/test_package.py +153 -0
  124. sluicer-0.2.0/tests/test_rdfa.py +276 -0
  125. sluicer-0.2.0/tests/test_records.py +285 -0
  126. sluicer-0.2.0/tests/test_scrapling_rungs.py +311 -0
  127. sluicer-0.2.0/tests/test_shape.py +121 -0
  128. sluicer-0.2.0/tests/test_summary.py +361 -0
  129. sluicer-0.2.0/tests/test_twitter.py +63 -0
  130. sluicer-0.2.0/uv.lock +2602 -0
@@ -0,0 +1,15 @@
1
+ {
2
+ "$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
3
+ "name": "sluicer",
4
+ "description": "Deterministic web extraction for agents: the data a page declares, with provenance, and no model in the loop.",
5
+ "owner": { "name": "Nichita Briculschi", "url": "https://github.com/Gi0tto" },
6
+ "plugins": [
7
+ {
8
+ "name": "sluicer",
9
+ "description": "Read what a page declares about itself, in eight vocabularies at once, and get back records where every field names the vocabulary it came from. Fetches only as expensively as it must, announces itself by name, and obeys robots.txt. Needs uv on the PATH: the MCP server runs from the plugin's own copy through uvx.",
10
+ "source": "./",
11
+ "category": "productivity",
12
+ "homepage": "https://github.com/Gi0tto/sluicer"
13
+ }
14
+ ]
15
+ }
@@ -0,0 +1,8 @@
1
+ {
2
+ "mcpServers": {
3
+ "sluicer": {
4
+ "command": "uvx",
5
+ "args": ["--from", "${CLAUDE_PLUGIN_ROOT}[mcp]", "sluicer-mcp"]
6
+ }
7
+ }
8
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "name": "sluicer",
3
+ "version": "0.2.0",
4
+ "description": "Read the structured data a web page already declares, with the provenance of every field, and no model in the loop. Fetches at the lowest price that works, announces itself, and obeys robots.txt. Learns extractors that fail loudly when a site changes layout, and heals them. Ships an MCP server with six tools and a skill that says when to use them.",
5
+ "author": { "name": "Nichita Briculschi", "url": "https://github.com/Gi0tto" },
6
+ "license": "MIT",
7
+ "homepage": "https://github.com/Gi0tto/sluicer",
8
+ "repository": "https://github.com/Gi0tto/sluicer",
9
+ "keywords": [
10
+ "scraping",
11
+ "structured-data",
12
+ "json-ld",
13
+ "microdata",
14
+ "opengraph",
15
+ "schema-org",
16
+ "extraction",
17
+ "markdown",
18
+ "robots-txt",
19
+ "deterministic"
20
+ ],
21
+ "mcpServers": "./.claude-plugin/mcp.json"
22
+ }
@@ -0,0 +1,10 @@
1
+ # Everything is left out, and only what the image is built from comes back in.
2
+ # A list of exclusions forgets the next cache directory; a list of inclusions
3
+ # cannot send a .venv, a .git or a site/ it never named.
4
+ *
5
+ !pyproject.toml
6
+ !README.md
7
+ !LICENSE
8
+ !src/
9
+ **/__pycache__
10
+ **/*.pyc
@@ -0,0 +1,44 @@
1
+ name: Something read wrong
2
+ description: A page that Sluicer got wrong, or would not read at all
3
+ labels: [bug]
4
+ body:
5
+ - type: input
6
+ id: page
7
+ attributes:
8
+ label: The page
9
+ description: A URL. If it is not public, attach the HTML below instead. Without the page this is a guess.
10
+ placeholder: https://example.com/product
11
+ - type: textarea
12
+ id: returned
13
+ attributes:
14
+ label: What Sluicer returned
15
+ description: The output of `sluicer extract <page>`.
16
+ render: json
17
+ validations:
18
+ required: true
19
+ - type: textarea
20
+ id: declared
21
+ attributes:
22
+ label: What the page actually declares
23
+ description: The field, the value, and where in the source it lives.
24
+ validations:
25
+ required: true
26
+ - type: dropdown
27
+ id: kind
28
+ attributes:
29
+ label: Which is it?
30
+ description: A missing value is a known cost we accept in places; a wrong one is the kind of bug this project takes most seriously.
31
+ options:
32
+ - A value came back wrong
33
+ - A value did not come back at all
34
+ - The call failed or crashed
35
+ validations:
36
+ required: true
37
+ - type: textarea
38
+ id: versions
39
+ attributes:
40
+ label: Versions
41
+ description: "`sluicer --version`, your Python version, and which extras are installed."
42
+ render: text
43
+ validations:
44
+ required: true
@@ -0,0 +1,5 @@
1
+ blank_issues_enabled: false
2
+ contact_links:
3
+ - name: A security vulnerability
4
+ url: https://github.com/Gi0tto/sluicer/security/advisories/new
5
+ about: Report it privately, never in a public issue. SECURITY.md says what happens next.
@@ -0,0 +1,38 @@
1
+ name: Something missing
2
+ description: A capability Sluicer should have
3
+ labels: [enhancement]
4
+ body:
5
+ - type: textarea
6
+ id: task
7
+ attributes:
8
+ label: What you were trying to do
9
+ description: The task, not the feature. The feature is your solution, and there may be a cheaper one.
10
+ validations:
11
+ required: true
12
+ - type: textarea
13
+ id: instead
14
+ attributes:
15
+ label: What you did instead
16
+ - type: dropdown
17
+ id: model
18
+ attributes:
19
+ label: Does it need an LLM or a paid API?
20
+ description: If it does, this project will say no, and it is better to know now. The constraint is deliberate and documented in CONTRIBUTING.md.
21
+ options:
22
+ - "No"
23
+ - "Yes"
24
+ - Not sure
25
+ validations:
26
+ required: true
27
+ - type: checkboxes
28
+ id: limits
29
+ attributes:
30
+ label: Known limits
31
+ description: docs/known-limits.md lists what we know we do not do yet. If your case is there, say below why it matters to you; that is how things get prioritised.
32
+ options:
33
+ - label: I checked docs/known-limits.md
34
+ required: true
35
+ - type: textarea
36
+ id: why
37
+ attributes:
38
+ label: Why it matters to you
@@ -0,0 +1,15 @@
1
+ **What changes, in one sentence.**
2
+
3
+ **The test that proves it.** Name it. It should fail without your change; say
4
+ that you watched it fail, and for which reason.
5
+
6
+ **Measured, not assumed.** If you claim it is faster, more accurate or reads
7
+ more pages, say by how much and against what.
8
+
9
+ **Checklist**
10
+
11
+ - [ ] `uv run pytest` passes from the repository root
12
+ - [ ] No network access in any test
13
+ - [ ] No LLM call and no paid API anywhere in the path
14
+ - [ ] Annotations are honest: nothing typed `object` that has a known type
15
+ - [ ] English in code, comments, tests and commit messages
@@ -0,0 +1,12 @@
1
+ version: 2
2
+ updates:
3
+ # "uv" reads uv.lock; "pip" does not, and would leave the lock behind.
4
+ - package-ecosystem: "uv"
5
+ directory: "/"
6
+ schedule:
7
+ interval: "monthly"
8
+ open-pull-requests-limit: 5
9
+ - package-ecosystem: "github-actions"
10
+ directory: "/"
11
+ schedule:
12
+ interval: "monthly"
@@ -0,0 +1,312 @@
1
+ name: CI
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ schedule:
6
+ # Once a week against whatever the extras resolve to today. A >=2 pin does not
7
+ # stop a 3.0 landing with another rename, and that would otherwise be found by
8
+ # a user rather than by us.
9
+ - cron: "0 6 * * 1"
10
+ push:
11
+ branches: [main]
12
+ paths-ignore: ["**/*.md", "docs/**"]
13
+ pull_request:
14
+ branches: [main]
15
+ paths-ignore: ["**/*.md", "docs/**"]
16
+
17
+ permissions:
18
+ contents: read
19
+
20
+ concurrency:
21
+ group: ${{ github.workflow }}-${{ github.ref }}
22
+ cancel-in-progress: true
23
+
24
+ jobs:
25
+ tests:
26
+ name: Tests on Python ${{ matrix.python }}
27
+ runs-on: ubuntu-latest
28
+ strategy:
29
+ fail-fast: false
30
+ matrix:
31
+ python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
32
+ steps:
33
+ - uses: actions/checkout@v7
34
+ - uses: astral-sh/setup-uv@v10.2.0
35
+ with:
36
+ enable-cache: true
37
+ - name: Run the suite
38
+ run: uv run --python ${{ matrix.python }} pytest -q
39
+
40
+ quality:
41
+ # Separate from the matrix above on purpose: a red cross here names which
42
+ # gate refused the commit, and each step names which of the three it was.
43
+ # Four human review passes were the only thing standing between a defect
44
+ # and this repository, and a wrong type annotation still got past one.
45
+ name: Lint, types and coverage
46
+ runs-on: ubuntu-latest
47
+ steps:
48
+ - uses: actions/checkout@v7
49
+ - uses: astral-sh/setup-uv@v10.2.0
50
+ with:
51
+ enable-cache: true
52
+ - name: Lint
53
+ run: uv run --python 3.13 ruff check src tests
54
+ - name: Types
55
+ # Reads its own settings from pyproject: strict, against src, at the
56
+ # 3.10 floor this package declares whatever interpreter is running it.
57
+ run: uv run --python 3.13 mypy
58
+ - name: Coverage
59
+ # One interpreter, so the floor is measured against the same thing
60
+ # every time. The matrix above is what proves the suite passes on all
61
+ # four versions.
62
+ run: uv run --python 3.13 pytest -q --cov --cov-report=term-missing
63
+
64
+ offline:
65
+ name: The suite opens no socket
66
+ runs-on: ubuntu-latest
67
+ steps:
68
+ - uses: actions/checkout@v7
69
+ - uses: astral-sh/setup-uv@v10.2.0
70
+ - name: Run the suite with the network taken away
71
+ run: |
72
+ cat > /tmp/no_network.py <<'PY'
73
+ """Fail the build if any test tries to reach the network.
74
+
75
+ The project's first promise is that a run costs CPU and nothing else,
76
+ and a promise nothing enforces is a wish.
77
+
78
+ This blocks connecting, not the socket type itself. Replacing
79
+ socket.socket wholesale looks stricter and is simply wrong: the stdlib
80
+ ssl module does `class SSLSocket(socket)`, so anything importing it
81
+ dies with a TypeError that has nothing to do with the network. Ask
82
+ what a test is forbidden to DO, not what it may name.
83
+ """
84
+ import socket
85
+ import sys
86
+
87
+
88
+ class Blocked(RuntimeError):
89
+ pass
90
+
91
+
92
+ def refuse(*args, **kwargs):
93
+ raise Blocked(
94
+ "a test tried to reach the network; the suite must never do that"
95
+ )
96
+
97
+
98
+ socket.socket.connect = refuse
99
+ socket.socket.connect_ex = refuse
100
+ socket.create_connection = refuse
101
+ socket.getaddrinfo = refuse
102
+
103
+ import pytest
104
+
105
+ sys.exit(pytest.main(["-q"]))
106
+ PY
107
+ uv run python /tmp/no_network.py
108
+
109
+ build:
110
+ name: The package builds and its metadata parses
111
+ runs-on: ubuntu-latest
112
+ steps:
113
+ - uses: actions/checkout@v7
114
+ - uses: astral-sh/setup-uv@v10.2.0
115
+ - name: Build the wheel and the sdist
116
+ run: uv build
117
+ - name: Check the metadata
118
+ run: uvx twine check dist/*
119
+
120
+ import-without-extras:
121
+ name: Importing works without any extra
122
+ runs-on: ubuntu-latest
123
+ steps:
124
+ - uses: actions/checkout@v7
125
+ - uses: astral-sh/setup-uv@v10.2.0
126
+ - name: Install the base package only, then import it
127
+ run: |
128
+ uv venv
129
+ uv pip install .
130
+ # `.venv/bin/python`, not `uv run`, in this job and the two below:
131
+ # `uv run` re-syncs the environment to uv.lock and installs the source
132
+ # tree in its place, so the check would run against the dev
133
+ # environment rather than the install it names -- the trap the
134
+ # floors job documents.
135
+ .venv/bin/python -c "
136
+ import importlib.util, sluicer
137
+ for extra in ('scrapling', 'trafilatura', 'mcp'):
138
+ assert importlib.util.find_spec(extra) is None, extra + ' leaked into the base install'
139
+ print('base install imports cleanly, version', sluicer.__version__)
140
+ "
141
+
142
+ with-extras:
143
+ name: The extras install and the code can actually use them
144
+ runs-on: ubuntu-latest
145
+ steps:
146
+ - uses: actions/checkout@v7
147
+ - uses: astral-sh/setup-uv@v10.2.0
148
+ - name: Install every extra
149
+ run: |
150
+ uv venv
151
+ uv pip install '.[fetch,markdown,mcp]'
152
+ - name: Use each one against the package that was really installed
153
+ run: |
154
+ .venv/bin/python -c "
155
+ # The suite fakes these libraries, and a fake can only confirm the shape
156
+ # you already believe in. This job is the only place that finds out what
157
+ # the published package actually ships. It caught the mcp 2.x rename.
158
+ from sluicer.markdown import to_markdown
159
+ page = '<html><head><meta charset=\"windows-1252\"></head><body><article><h1>Caf\xe9</h1><p>' + ('Real sentences. ' * 30) + '</p></article></body></html>'
160
+ out = to_markdown(page.encode('windows-1252'))
161
+ assert 'Caf\xe9' in out, 'the markdown path lost the accent: ' + out[:80]
162
+ print('markdown: accent survived')
163
+
164
+ # protego parses robots.txt and arrives with scrapling[fetchers]. The unit
165
+ # tests fake it, and a fake agrees with whatever you wrote it to agree with,
166
+ # so the real semantics are only ever checked here.
167
+ from sluicer.fetch.identity import USER_AGENT, robots_allows
168
+ assert 'Sluicer' in USER_AGENT and 'github.com' in USER_AGENT, USER_AGENT
169
+ refuse = 'User-agent: Sluicer\nDisallow: /private/\n\nUser-agent: *\nAllow: /\n'
170
+ assert robots_allows('https://example.com/private/p', read=lambda u: refuse, cache={}) is False
171
+ assert robots_allows('https://example.com/public', read=lambda u: refuse, cache={}) is True
172
+ assert robots_allows('https://example.com/p', read=lambda u: None, cache={}) is True
173
+ print('robots: the real parser agrees with what the suite believes')
174
+
175
+ from sluicer.fetch.scrapling_rungs import default_rungs
176
+ names = [name for name, _ in default_rungs()]
177
+ assert names[:2] == ['http', 'browser'], names
178
+ print('fetch: three rungs built against the real scrapling')
179
+
180
+ import asyncio
181
+ from sluicer.mcp_server import build_server
182
+ server = build_server()
183
+ tools = sorted(t.name for t in asyncio.run(server.list_tools()))
184
+ assert tools == ['compile_extractor', 'extract_declared', 'fetch_page', 'heal_extractor', 'page_markdown', 'run_extractor'], tools
185
+
186
+ # Registration is not use. The SDK's dispatch is where a future release
187
+ # can break us while every import still resolves, so call a tool for real.
188
+ page = '<html><head><script type=\"application/ld+json\">{\"@type\":\"Product\",\"name\":\"Brake pad set\"}</script></head><body></body></html>'
189
+ answer = asyncio.run(server.call_tool('extract_declared', {'html_or_url': page}))
190
+ text = str(answer)
191
+ assert 'Brake pad set' in text, 'the SDK dispatch lost the record: ' + text[:200]
192
+ assert 'jsonld' in text, 'the provenance did not survive dispatch: ' + text[:200]
193
+
194
+ markdown_answer = asyncio.run(server.call_tool('page_markdown', {'html_or_url': '<html><body><article><h1>Title</h1><p>' + ('Real sentences. ' * 30) + '</p></article></body></html>'}))
195
+ assert 'Title' in str(markdown_answer), str(markdown_answer)[:200]
196
+ print('mcp: three tools listed, and two of them answered through the SDK dispatch')
197
+ "
198
+
199
+ floors:
200
+ name: The declared minimum versions actually work
201
+ runs-on: ubuntu-latest
202
+ steps:
203
+ - uses: actions/checkout@v7
204
+ - uses: astral-sh/setup-uv@v10.2.0
205
+ # A dependency floor nobody installs is a wish, not a constraint. Both
206
+ # base floors were wrong when this job was written: lxml 5.0 ships no
207
+ # wheel for cp313 and none at all for macOS arm64, and click 8.1.8 fails
208
+ # fourteen tests because its CliRunner does not capture stderr
209
+ # separately. Both had been declared for months and never executed.
210
+ #
211
+ # Pinned to the minimum Python this project claims: on a newer
212
+ # interpreter the resolver climbs above the floor, because the floor
213
+ # version has no wheel for it, and the job would pass without ever
214
+ # installing what it exists to test.
215
+ #
216
+ # The base is resolved on its own. Installing the extras alongside it
217
+ # lifts lxml and click above their floors, because the extras depend on
218
+ # them too.
219
+ #
220
+ # `.venv/bin/python` throughout, never `uv run`: `uv run` re-syncs the
221
+ # environment from pyproject.toml and reinstalls lxml at its newest,
222
+ # which silently undoes the pin. The assertion below caught exactly that
223
+ # on this job's first run, which is why it is an assertion and not a
224
+ # printed list somebody skims.
225
+ - name: Install the base at its declared floors
226
+ run: |
227
+ uv venv --python 3.10
228
+ uv pip install --resolution lowest-direct .
229
+ uv pip install pytest
230
+ - name: Check those really are the floors, not something above them
231
+ run: |
232
+ .venv/bin/python -c "
233
+ import importlib.metadata as md
234
+ for name, floor in {'lxml': '5.3', 'click': '8.2'}.items():
235
+ got = md.version(name)
236
+ assert got.startswith(floor), (
237
+ name + ' resolved to ' + got + ', not the declared floor '
238
+ + floor + ': this job is not testing what it claims to test'
239
+ )
240
+ print(name, got, '-- the declared floor')
241
+ "
242
+ - name: Run the suite against them
243
+ run: .venv/bin/python -m pytest -q
244
+ - name: And again with every extra at its own lowest version
245
+ run: |
246
+ uv pip install --resolution lowest-direct '.[fetch,markdown,mcp,microformats]'
247
+ uv pip list
248
+ .venv/bin/python -m pytest -q
249
+
250
+ latest-extras:
251
+ name: The extras still work at their newest versions
252
+ runs-on: ubuntu-latest
253
+ if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
254
+ steps:
255
+ - uses: actions/checkout@v7
256
+ - uses: astral-sh/setup-uv@v10.2.0
257
+ - name: Install the extras unpinned and use them
258
+ run: |
259
+ uv venv
260
+ uv pip install '.[fetch,markdown,mcp]' --upgrade
261
+ .venv/bin/python -c "
262
+ import asyncio, importlib.metadata as md
263
+ for name in ('scrapling', 'trafilatura', 'mcp'):
264
+ print(name, md.version(name))
265
+ from sluicer.mcp_server import build_server
266
+ server = build_server()
267
+ tools = sorted(t.name for t in asyncio.run(server.list_tools()))
268
+ assert tools == ['compile_extractor', 'extract_declared', 'fetch_page', 'heal_extractor', 'page_markdown', 'run_extractor'], tools
269
+ print('still building against the newest published extras')
270
+ "
271
+
272
+ docker:
273
+ name: The image builds, runs unprivileged, and reads a page
274
+ runs-on: ubuntu-latest
275
+ steps:
276
+ - uses: actions/checkout@v7
277
+ - name: Build the image
278
+ run: docker build -t sluicer:ci .
279
+ - name: Use it
280
+ # A page handed to it needs no network, so this proves the install in
281
+ # the image without depending on any site being up.
282
+ run: |
283
+ uid=$(docker run --rm sluicer:ci id -u)
284
+ test "$uid" != 0 || { echo "the image runs as root"; exit 1; }
285
+ printf '%s' '<script type="application/ld+json">{"@type":"Product","name":"Pad"}</script>' > page.html
286
+ docker run --rm -v "$PWD/page.html:/tmp/page.html:ro" sluicer:ci \
287
+ sluicer extract /tmp/page.html | tee out.json
288
+ grep -q '"Pad"' out.json
289
+
290
+ docker-browser:
291
+ # The browser variant is most of a gigabyte and minutes to build, so it is
292
+ # proven weekly rather than on every push -- but proven, because a build
293
+ # argument nobody builds is a claim in a comment.
294
+ name: The image builds with its browser
295
+ runs-on: ubuntu-latest
296
+ if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
297
+ steps:
298
+ - uses: actions/checkout@v7
299
+ - name: Build the image with Chromium
300
+ run: docker build --build-arg WITH_BROWSER=1 -t sluicer:browser .
301
+ - name: The unprivileged user can launch the browser
302
+ run: |
303
+ docker run --rm sluicer:browser python -c "
304
+ from playwright.sync_api import sync_playwright
305
+ with sync_playwright() as p:
306
+ browser = p.chromium.launch()
307
+ page = browser.new_page()
308
+ page.set_content('<p>ok</p>')
309
+ assert page.inner_text('p') == 'ok'
310
+ browser.close()
311
+ print('chromium launches as', __import__('os').getuid())
312
+ "
@@ -0,0 +1,53 @@
1
+ name: Docs
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ paths: ["docs/**", "mkdocs.yml", "README.md", "ROADMAP.md"]
7
+ workflow_dispatch:
8
+
9
+ permissions:
10
+ contents: read
11
+ pages: write
12
+ id-token: write
13
+
14
+ concurrency:
15
+ group: pages
16
+ cancel-in-progress: false
17
+
18
+ jobs:
19
+ # Two jobs rather than one, because they answer different questions. Building
20
+ # is a test: --strict fails on a broken link or a page nothing references, and
21
+ # it has to run on every push whatever the repository's visibility. Deploying
22
+ # is a publication, and GitHub Pages is unavailable on a private repository
23
+ # outside a paid plan -- so gating the build on that would mean a broken doc
24
+ # could land unnoticed while the repository is private, which is exactly when
25
+ # the docs are being written.
26
+ build:
27
+ name: The documentation builds
28
+ runs-on: ubuntu-latest
29
+ steps:
30
+ - uses: actions/checkout@v7
31
+ - uses: astral-sh/setup-uv@v10.2.0
32
+ - name: Build the site
33
+ # Pinned below 2 on both: MkDocs 2.0 is announced as a breaking release
34
+ # and mkdocs-material already warns about it, so an unpinned uvx would
35
+ # change what --strict means on the day it lands, not on a day we chose.
36
+ run: uvx --from 'mkdocs>=1.6,<2' --with 'mkdocs-material>=9.7,<10' mkdocs build --strict
37
+ - uses: actions/upload-pages-artifact@v5
38
+ with:
39
+ path: site
40
+
41
+ deploy:
42
+ name: The documentation is published
43
+ needs: build
44
+ # Pages refuses a private repository on the free plan with a 404 that reads
45
+ # like a missing artifact. Skipping is the honest report: nothing is wrong.
46
+ if: github.event.repository.visibility == 'public'
47
+ runs-on: ubuntu-latest
48
+ environment:
49
+ name: github-pages
50
+ url: ${{ steps.deployment.outputs.page_url }}
51
+ steps:
52
+ - id: deployment
53
+ uses: actions/deploy-pages@v5
@@ -0,0 +1,86 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+ workflow_dispatch:
7
+
8
+ # Nothing here writes to the repository. The one elevated permission is the
9
+ # OIDC token trusted publishing needs, and only the job that publishes gets it.
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ build:
15
+ name: Build and check the distributions
16
+ runs-on: ubuntu-latest
17
+ steps:
18
+ - uses: actions/checkout@v7
19
+ - uses: astral-sh/setup-uv@v10.2.0
20
+ - name: The tag and the version must agree
21
+ if: startsWith(github.ref, 'refs/tags/v')
22
+ run: |
23
+ uv run python -c "
24
+ import os, tomllib, pathlib
25
+ declared = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version']
26
+ tagged = os.environ['GITHUB_REF_NAME'].removeprefix('v')
27
+ assert declared == tagged, f'pyproject says {declared}, the tag says {tagged}'
28
+ print('version agrees:', declared)
29
+ "
30
+ - name: The suite must pass before anything is published
31
+ run: uv run --python 3.13 pytest -q
32
+ - run: uv build
33
+ - name: Check the metadata the way PyPI will
34
+ run: uvx twine check dist/*
35
+ - uses: actions/upload-artifact@v7
36
+ with:
37
+ name: distributions
38
+ path: dist/
39
+
40
+ installs:
41
+ name: The built wheel installs and works on its own
42
+ needs: build
43
+ runs-on: ubuntu-latest
44
+ steps:
45
+ # No checkout: the wheel has to stand on its own, and a source tree in the
46
+ # working directory is what an import would quietly find instead of it.
47
+ - uses: actions/download-artifact@v8
48
+ with:
49
+ name: distributions
50
+ path: dist/
51
+ - uses: astral-sh/setup-uv@v10.2.0
52
+ - name: Install the wheel in a clean environment and use it
53
+ run: |
54
+ uv venv --python 3.13
55
+ uv pip install dist/*.whl
56
+ .venv/bin/python -c "
57
+ import sluicer
58
+ page = '<script type=\"application/ld+json\">{\"@type\":\"Product\",\"name\":\"Pad\"}</script>'
59
+ result = sluicer.extract(page)
60
+ assert result.records and result.records[0].type == 'Product', result
61
+ assert sluicer.__file__.startswith('$PWD/.venv/'), sluicer.__file__
62
+ print('wheel', sluicer.__version__, 'installs, imports and extracts')
63
+ "
64
+ .venv/bin/sluicer --version
65
+
66
+ publish:
67
+ name: Publish to PyPI
68
+ needs: [build, installs]
69
+ runs-on: ubuntu-latest
70
+ # Off until the PyPI side exists: a trusted publisher for this repository
71
+ # and workflow, registered at pypi.org, and the repository variable
72
+ # PUBLISH_TO_PYPI set to true. Until then a tag builds and checks the
73
+ # distributions and stops here, instead of failing on a missing publisher.
74
+ if: startsWith(github.ref, 'refs/tags/v') && vars.PUBLISH_TO_PYPI == 'true'
75
+ environment: pypi
76
+ permissions:
77
+ id-token: write
78
+ steps:
79
+ - uses: actions/download-artifact@v8
80
+ with:
81
+ name: distributions
82
+ path: dist/
83
+ # Trusted publishing: PyPI verifies this workflow's identity, so no API
84
+ # token is stored anywhere. Configure it once at
85
+ # https://pypi.org/manage/project/sluicer/settings/publishing/
86
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,18 @@
1
+ __pycache__/
2
+ *.pyc
3
+ .venv/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ .mypy_cache/
10
+ .coverage
11
+ htmlcov/
12
+ bench/cache/
13
+ .DS_Store
14
+ .superpowers/
15
+ stdout.txt
16
+ stderr.txt
17
+ exitcode.txt
18
+ site/