newsclip 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (284) hide show
  1. newsclip-0.7.0/.dockerignore +13 -0
  2. newsclip-0.7.0/.github/workflows/ci.yml +32 -0
  3. newsclip-0.7.0/.github/workflows/release.yml +145 -0
  4. newsclip-0.7.0/.gitignore +19 -0
  5. newsclip-0.7.0/AGENTS.md +34 -0
  6. newsclip-0.7.0/ARCHITECTURE.md +70 -0
  7. newsclip-0.7.0/CONTRIBUTING.md +25 -0
  8. newsclip-0.7.0/Dockerfile +65 -0
  9. newsclip-0.7.0/Dockerfile.dockerignore +39 -0
  10. newsclip-0.7.0/LICENSE +21 -0
  11. newsclip-0.7.0/Makefile +21 -0
  12. newsclip-0.7.0/NOTICE +10 -0
  13. newsclip-0.7.0/PKG-INFO +157 -0
  14. newsclip-0.7.0/README.md +126 -0
  15. newsclip-0.7.0/ROADMAP.md +18 -0
  16. newsclip-0.7.0/SECURITY.md +26 -0
  17. newsclip-0.7.0/compose.yaml +52 -0
  18. newsclip-0.7.0/docs/API.md +55 -0
  19. newsclip-0.7.0/docs/AUTHENTICATION.md +48 -0
  20. newsclip-0.7.0/docs/CAPTCHA_ARCHITECTURE.md +10 -0
  21. newsclip-0.7.0/docs/CLI.md +54 -0
  22. newsclip-0.7.0/docs/DEPLOYMENT.md +66 -0
  23. newsclip-0.7.0/docs/DEVELOPMENT.md +34 -0
  24. newsclip-0.7.0/docs/LIVE_DEBUGGING.md +12 -0
  25. newsclip-0.7.0/docs/PROVIDERS.md +43 -0
  26. newsclip-0.7.0/pyproject.toml +90 -0
  27. newsclip-0.7.0/scripts/check_browser_lifecycle.py +114 -0
  28. newsclip-0.7.0/scripts/explain_live_report.py +604 -0
  29. newsclip-0.7.0/scripts/print_source_fingerprint.py +14 -0
  30. newsclip-0.7.0/scripts/promote_image.sh +25 -0
  31. newsclip-0.7.0/scripts/setup.py +59 -0
  32. newsclip-0.7.0/scripts/smoke_standalone.py +99 -0
  33. newsclip-0.7.0/scripts/standalone_e2e.py +455 -0
  34. newsclip-0.7.0/scripts/synthetic_route_check.py +1369 -0
  35. newsclip-0.7.0/src/newsclip/__init__.py +13 -0
  36. newsclip-0.7.0/src/newsclip/__main__.py +9 -0
  37. newsclip-0.7.0/src/newsclip/account_keys.py +26 -0
  38. newsclip-0.7.0/src/newsclip/acquisition.py +220 -0
  39. newsclip-0.7.0/src/newsclip/api_keys.py +17 -0
  40. newsclip-0.7.0/src/newsclip/archive.py +1598 -0
  41. newsclip-0.7.0/src/newsclip/articles.py +98 -0
  42. newsclip-0.7.0/src/newsclip/assessment.py +189 -0
  43. newsclip-0.7.0/src/newsclip/auth_ui.py +337 -0
  44. newsclip-0.7.0/src/newsclip/browser_profiles.py +65 -0
  45. newsclip-0.7.0/src/newsclip/browser_readiness.py +234 -0
  46. newsclip-0.7.0/src/newsclip/browser_retrieval.py +1307 -0
  47. newsclip-0.7.0/src/newsclip/browser_runtime/__init__.py +161 -0
  48. newsclip-0.7.0/src/newsclip/browser_runtime/events.py +1486 -0
  49. newsclip-0.7.0/src/newsclip/browser_runtime/leases.py +210 -0
  50. newsclip-0.7.0/src/newsclip/browser_runtime/profile_ownership.py +219 -0
  51. newsclip-0.7.0/src/newsclip/browser_runtime/runtime.py +1979 -0
  52. newsclip-0.7.0/src/newsclip/browser_runtime/surface.py +666 -0
  53. newsclip-0.7.0/src/newsclip/browser_runtime/x11_pointer.py +227 -0
  54. newsclip-0.7.0/src/newsclip/browser_verification.py +276 -0
  55. newsclip-0.7.0/src/newsclip/challenge_catalog.py +118 -0
  56. newsclip-0.7.0/src/newsclip/challenge_detection.py +379 -0
  57. newsclip-0.7.0/src/newsclip/challenge_resolver.py +337 -0
  58. newsclip-0.7.0/src/newsclip/challenge_transport.py +139 -0
  59. newsclip-0.7.0/src/newsclip/challenges.py +126 -0
  60. newsclip-0.7.0/src/newsclip/cli.py +2517 -0
  61. newsclip-0.7.0/src/newsclip/client.py +250 -0
  62. newsclip-0.7.0/src/newsclip/cloudflare_interstitial.py +173 -0
  63. newsclip-0.7.0/src/newsclip/composition.py +1090 -0
  64. newsclip-0.7.0/src/newsclip/configured_provider.py +1016 -0
  65. newsclip-0.7.0/src/newsclip/contracts.py +336 -0
  66. newsclip-0.7.0/src/newsclip/datadome.py +183 -0
  67. newsclip-0.7.0/src/newsclip/defense.py +362 -0
  68. newsclip-0.7.0/src/newsclip/drift.py +293 -0
  69. newsclip-0.7.0/src/newsclip/errors.py +208 -0
  70. newsclip-0.7.0/src/newsclip/extensions.py +17 -0
  71. newsclip-0.7.0/src/newsclip/extraction.py +219 -0
  72. newsclip-0.7.0/src/newsclip/health.py +422 -0
  73. newsclip-0.7.0/src/newsclip/http.py +649 -0
  74. newsclip-0.7.0/src/newsclip/http_challenges.py +173 -0
  75. newsclip-0.7.0/src/newsclip/incidents.py +1018 -0
  76. newsclip-0.7.0/src/newsclip/keeper_outcomes.py +512 -0
  77. newsclip-0.7.0/src/newsclip/live_diagnostics.py +257 -0
  78. newsclip-0.7.0/src/newsclip/live_reporting.py +1744 -0
  79. newsclip-0.7.0/src/newsclip/mcp_server.py +844 -0
  80. newsclip-0.7.0/src/newsclip/models.py +755 -0
  81. newsclip-0.7.0/src/newsclip/network.py +204 -0
  82. newsclip-0.7.0/src/newsclip/perimeterx.py +576 -0
  83. newsclip-0.7.0/src/newsclip/private_state.py +142 -0
  84. newsclip-0.7.0/src/newsclip/process_signals.py +63 -0
  85. newsclip-0.7.0/src/newsclip/provenance.py +297 -0
  86. newsclip-0.7.0/src/newsclip/provider_check.py +103 -0
  87. newsclip-0.7.0/src/newsclip/provider_config.py +1018 -0
  88. newsclip-0.7.0/src/newsclip/provider_definitions/ap.toml +53 -0
  89. newsclip-0.7.0/src/newsclip/provider_definitions/axios.toml +57 -0
  90. newsclip-0.7.0/src/newsclip/provider_definitions/bloomberg.toml +278 -0
  91. newsclip-0.7.0/src/newsclip/provider_definitions/cnbc.toml +44 -0
  92. newsclip-0.7.0/src/newsclip/provider_definitions/economist.toml +132 -0
  93. newsclip-0.7.0/src/newsclip/provider_definitions/ft.toml +166 -0
  94. newsclip-0.7.0/src/newsclip/provider_definitions/guardian.toml +44 -0
  95. newsclip-0.7.0/src/newsclip/provider_definitions/nyt.toml +121 -0
  96. newsclip-0.7.0/src/newsclip/provider_definitions/reuters.toml +46 -0
  97. newsclip-0.7.0/src/newsclip/provider_definitions/wsj.toml +132 -0
  98. newsclip-0.7.0/src/newsclip/provider_guard.py +1113 -0
  99. newsclip-0.7.0/src/newsclip/provider_keys.py +14 -0
  100. newsclip-0.7.0/src/newsclip/provider_readiness.py +54 -0
  101. newsclip-0.7.0/src/newsclip/providers/__init__.py +67 -0
  102. newsclip-0.7.0/src/newsclip/providers/ap.py +251 -0
  103. newsclip-0.7.0/src/newsclip/providers/axios.py +309 -0
  104. newsclip-0.7.0/src/newsclip/providers/axios_story.py +161 -0
  105. newsclip-0.7.0/src/newsclip/providers/base.py +158 -0
  106. newsclip-0.7.0/src/newsclip/providers/bloomberg.py +588 -0
  107. newsclip-0.7.0/src/newsclip/providers/bloomberg_story.py +308 -0
  108. newsclip-0.7.0/src/newsclip/providers/builtins.py +85 -0
  109. newsclip-0.7.0/src/newsclip/providers/cnbc.py +215 -0
  110. newsclip-0.7.0/src/newsclip/providers/economist.py +64 -0
  111. newsclip-0.7.0/src/newsclip/providers/ft.py +144 -0
  112. newsclip-0.7.0/src/newsclip/providers/guardian.py +295 -0
  113. newsclip-0.7.0/src/newsclip/providers/nyt.py +167 -0
  114. newsclip-0.7.0/src/newsclip/providers/registration.py +77 -0
  115. newsclip-0.7.0/src/newsclip/providers/registry.py +43 -0
  116. newsclip-0.7.0/src/newsclip/providers/reuters.py +429 -0
  117. newsclip-0.7.0/src/newsclip/providers/wsj.py +91 -0
  118. newsclip-0.7.0/src/newsclip/recovery.py +693 -0
  119. newsclip-0.7.0/src/newsclip/recovery_watch.py +179 -0
  120. newsclip-0.7.0/src/newsclip/renewal_observation.py +59 -0
  121. newsclip-0.7.0/src/newsclip/results.py +341 -0
  122. newsclip-0.7.0/src/newsclip/runtime_projection.py +701 -0
  123. newsclip-0.7.0/src/newsclip/service.py +3165 -0
  124. newsclip-0.7.0/src/newsclip/session_cookies.py +233 -0
  125. newsclip-0.7.0/src/newsclip/session_keeper.py +1884 -0
  126. newsclip-0.7.0/src/newsclip/session_proofs.py +88 -0
  127. newsclip-0.7.0/src/newsclip/sessions.py +1483 -0
  128. newsclip-0.7.0/src/newsclip/signin.py +308 -0
  129. newsclip-0.7.0/src/newsclip/standalone.py +462 -0
  130. newsclip-0.7.0/src/newsclip/tenancy.py +138 -0
  131. newsclip-0.7.0/src/newsclip/timing.py +1187 -0
  132. newsclip-0.7.0/src/newsclip/trace_retention.py +107 -0
  133. newsclip-0.7.0/src/newsclip/transport_pins.py +662 -0
  134. newsclip-0.7.0/src/newsclip/visual_plans.py +113 -0
  135. newsclip-0.7.0/tests/conftest.py +16 -0
  136. newsclip-0.7.0/tests/fixtures/ap/cloudflare-interstitial.html +5 -0
  137. newsclip-0.7.0/tests/fixtures/bloomberg/auth.html +11 -0
  138. newsclip-0.7.0/tests/fixtures/bloomberg/bulleted-wrap.html +64 -0
  139. newsclip-0.7.0/tests/fixtures/bloomberg/full.html +29 -0
  140. newsclip-0.7.0/tests/fixtures/bloomberg/google-account-chooser.html +27 -0
  141. newsclip-0.7.0/tests/fixtures/bloomberg/google-email.html +12 -0
  142. newsclip-0.7.0/tests/fixtures/bloomberg/http_error.html +2 -0
  143. newsclip-0.7.0/tests/fixtures/bloomberg/incremental-four-paragraphs.html +24 -0
  144. newsclip-0.7.0/tests/fixtures/bloomberg/interstitial.html +10 -0
  145. newsclip-0.7.0/tests/fixtures/bloomberg/long_nonarticle.html +15 -0
  146. newsclip-0.7.0/tests/fixtures/bloomberg/malformed_metadata.html +18 -0
  147. newsclip-0.7.0/tests/fixtures/bloomberg/padded_teaser.html +25 -0
  148. newsclip-0.7.0/tests/fixtures/bloomberg/paywall.html +17 -0
  149. newsclip-0.7.0/tests/fixtures/bloomberg/search_results.html +14 -0
  150. newsclip-0.7.0/tests/fixtures/bloomberg/short-wire-story.html +109 -0
  151. newsclip-0.7.0/tests/fixtures/bloomberg/teaser.html +19 -0
  152. newsclip-0.7.0/tests/fixtures/bloomberg/thin-stub-payload.html +89 -0
  153. newsclip-0.7.0/tests/fixtures/bloomberg/wrong_canonical.html +20 -0
  154. newsclip-0.7.0/tests/fixtures/economist/auth.html +21 -0
  155. newsclip-0.7.0/tests/fixtures/economist/cloudflare-interstitial.html +20 -0
  156. newsclip-0.7.0/tests/fixtures/economist/datadome-interstitial.html +6 -0
  157. newsclip-0.7.0/tests/fixtures/ft/account_security_reset.html +7 -0
  158. newsclip-0.7.0/tests/fixtures/ft/auth/passkey.html +12 -0
  159. newsclip-0.7.0/tests/fixtures/ft/auth/password.html +14 -0
  160. newsclip-0.7.0/tests/fixtures/ft/auth/username.html +12 -0
  161. newsclip-0.7.0/tests/fixtures/ft/auth.html +12 -0
  162. newsclip-0.7.0/tests/fixtures/ft/brief_full.html +18 -0
  163. newsclip-0.7.0/tests/fixtures/ft/challenge_http.html +9 -0
  164. newsclip-0.7.0/tests/fixtures/ft/full.html +29 -0
  165. newsclip-0.7.0/tests/fixtures/ft/http_error.html +2 -0
  166. newsclip-0.7.0/tests/fixtures/ft/interstitial.html +9 -0
  167. newsclip-0.7.0/tests/fixtures/ft/long_nonarticle.html +15 -0
  168. newsclip-0.7.0/tests/fixtures/ft/malformed_metadata.html +12 -0
  169. newsclip-0.7.0/tests/fixtures/ft/search_results.html +18 -0
  170. newsclip-0.7.0/tests/fixtures/ft/search_results_missing_date.html +14 -0
  171. newsclip-0.7.0/tests/fixtures/ft/signed_out.html +21 -0
  172. newsclip-0.7.0/tests/fixtures/ft/teaser.html +17 -0
  173. newsclip-0.7.0/tests/fixtures/ft/wrong_canonical.html +14 -0
  174. newsclip-0.7.0/tests/fixtures/nyt/auth/email_step.html +26 -0
  175. newsclip-0.7.0/tests/fixtures/nyt/auth/password_step.html +30 -0
  176. newsclip-0.7.0/tests/fixtures/nyt/auth.html +15 -0
  177. newsclip-0.7.0/tests/fixtures/nyt/full.html +42 -0
  178. newsclip-0.7.0/tests/fixtures/nyt/http_error.html +2 -0
  179. newsclip-0.7.0/tests/fixtures/nyt/interstitial.html +1 -0
  180. newsclip-0.7.0/tests/fixtures/nyt/malformed_metadata.html +21 -0
  181. newsclip-0.7.0/tests/fixtures/nyt/metered_full_no_session.html +42 -0
  182. newsclip-0.7.0/tests/fixtures/nyt/search_results.html +32 -0
  183. newsclip-0.7.0/tests/fixtures/nyt/search_server_error.html +13 -0
  184. newsclip-0.7.0/tests/fixtures/nyt/teaser.html +34 -0
  185. newsclip-0.7.0/tests/fixtures/nyt/verification_full.html +42 -0
  186. newsclip-0.7.0/tests/fixtures/nyt/wrong_canonical.html +21 -0
  187. newsclip-0.7.0/tests/fixtures/reuters/challenge-datadome-device-check.html +12 -0
  188. newsclip-0.7.0/tests/fixtures/reuters/challenge-datadome.html +13 -0
  189. newsclip-0.7.0/tests/fixtures/wsj/auth.html +12 -0
  190. newsclip-0.7.0/tests/fixtures/wsj/brief_full.html +18 -0
  191. newsclip-0.7.0/tests/fixtures/wsj/datadome.html +18 -0
  192. newsclip-0.7.0/tests/fixtures/wsj/full.html +34 -0
  193. newsclip-0.7.0/tests/fixtures/wsj/http_error.html +2 -0
  194. newsclip-0.7.0/tests/fixtures/wsj/initial_excerpt.html +26 -0
  195. newsclip-0.7.0/tests/fixtures/wsj/interstitial.html +10 -0
  196. newsclip-0.7.0/tests/fixtures/wsj/long_nonarticle.html +15 -0
  197. newsclip-0.7.0/tests/fixtures/wsj/malformed_metadata.html +12 -0
  198. newsclip-0.7.0/tests/fixtures/wsj/search_results.html +24 -0
  199. newsclip-0.7.0/tests/fixtures/wsj/teaser.html +17 -0
  200. newsclip-0.7.0/tests/fixtures/wsj/verification_full.html +37 -0
  201. newsclip-0.7.0/tests/fixtures/wsj/wrong_canonical.html +14 -0
  202. newsclip-0.7.0/tests/mcp_wire.py +28 -0
  203. newsclip-0.7.0/tests/test_account_keys.py +60 -0
  204. newsclip-0.7.0/tests/test_acquisition.py +227 -0
  205. newsclip-0.7.0/tests/test_ap_provider.py +445 -0
  206. newsclip-0.7.0/tests/test_archive.py +1541 -0
  207. newsclip-0.7.0/tests/test_auth_ui.py +169 -0
  208. newsclip-0.7.0/tests/test_autonomous_recovery.py +396 -0
  209. newsclip-0.7.0/tests/test_axios_provider.py +457 -0
  210. newsclip-0.7.0/tests/test_bloomberg_provider.py +1215 -0
  211. newsclip-0.7.0/tests/test_bloomberg_story.py +219 -0
  212. newsclip-0.7.0/tests/test_bloomberg_structured_retrieval.py +518 -0
  213. newsclip-0.7.0/tests/test_bloomberg_verification_readiness.py +174 -0
  214. newsclip-0.7.0/tests/test_browser_diagnostics.py +200 -0
  215. newsclip-0.7.0/tests/test_browser_operation_bounds.py +353 -0
  216. newsclip-0.7.0/tests/test_browser_plane.py +2521 -0
  217. newsclip-0.7.0/tests/test_browser_profiles.py +76 -0
  218. newsclip-0.7.0/tests/test_browser_response_handoff.py +145 -0
  219. newsclip-0.7.0/tests/test_browser_verification.py +254 -0
  220. newsclip-0.7.0/tests/test_challenge_catalog.py +138 -0
  221. newsclip-0.7.0/tests/test_challenge_detection_browser.py +638 -0
  222. newsclip-0.7.0/tests/test_challenge_resolver.py +510 -0
  223. newsclip-0.7.0/tests/test_challenge_transport.py +328 -0
  224. newsclip-0.7.0/tests/test_cli_workflow.py +1141 -0
  225. newsclip-0.7.0/tests/test_cloudflare_browser.py +158 -0
  226. newsclip-0.7.0/tests/test_cloudflare_interstitial.py +169 -0
  227. newsclip-0.7.0/tests/test_cnbc_provider.py +261 -0
  228. newsclip-0.7.0/tests/test_defense.py +337 -0
  229. newsclip-0.7.0/tests/test_defense_service.py +663 -0
  230. newsclip-0.7.0/tests/test_drift.py +393 -0
  231. newsclip-0.7.0/tests/test_economist_provider.py +323 -0
  232. newsclip-0.7.0/tests/test_extraction.py +90 -0
  233. newsclip-0.7.0/tests/test_federated_search.py +1145 -0
  234. newsclip-0.7.0/tests/test_foundation.py +518 -0
  235. newsclip-0.7.0/tests/test_ft_account_action.py +161 -0
  236. newsclip-0.7.0/tests/test_ft_provider.py +763 -0
  237. newsclip-0.7.0/tests/test_guardian_provider.py +288 -0
  238. newsclip-0.7.0/tests/test_http.py +803 -0
  239. newsclip-0.7.0/tests/test_http_challenges.py +174 -0
  240. newsclip-0.7.0/tests/test_http_impersonation.py +66 -0
  241. newsclip-0.7.0/tests/test_image_promotion.py +69 -0
  242. newsclip-0.7.0/tests/test_incident_reporting.py +346 -0
  243. newsclip-0.7.0/tests/test_incidents.py +851 -0
  244. newsclip-0.7.0/tests/test_keeper_outcomes.py +373 -0
  245. newsclip-0.7.0/tests/test_keeper_repair_hint.py +144 -0
  246. newsclip-0.7.0/tests/test_mcp.py +986 -0
  247. newsclip-0.7.0/tests/test_mcp_http.py +227 -0
  248. newsclip-0.7.0/tests/test_mint_evidence.py +555 -0
  249. newsclip-0.7.0/tests/test_network.py +435 -0
  250. newsclip-0.7.0/tests/test_not_entitled.py +265 -0
  251. newsclip-0.7.0/tests/test_nyt_provider.py +530 -0
  252. newsclip-0.7.0/tests/test_probe_inconclusive_escalation.py +308 -0
  253. newsclip-0.7.0/tests/test_process_signals.py +137 -0
  254. newsclip-0.7.0/tests/test_profile_ownership.py +211 -0
  255. newsclip-0.7.0/tests/test_provenance.py +78 -0
  256. newsclip-0.7.0/tests/test_provider_check.py +1209 -0
  257. newsclip-0.7.0/tests/test_provider_config.py +728 -0
  258. newsclip-0.7.0/tests/test_provider_guard.py +1604 -0
  259. newsclip-0.7.0/tests/test_provider_search.py +385 -0
  260. newsclip-0.7.0/tests/test_public_error_selfdescription.py +129 -0
  261. newsclip-0.7.0/tests/test_recovery.py +278 -0
  262. newsclip-0.7.0/tests/test_recovery_watch.py +177 -0
  263. newsclip-0.7.0/tests/test_recovery_workflow.py +346 -0
  264. newsclip-0.7.0/tests/test_remote_cli.py +140 -0
  265. newsclip-0.7.0/tests/test_renewal_consumers.py +204 -0
  266. newsclip-0.7.0/tests/test_renewal_ordering.py +238 -0
  267. newsclip-0.7.0/tests/test_reuters_provider.py +1344 -0
  268. newsclip-0.7.0/tests/test_runtime_projection.py +237 -0
  269. newsclip-0.7.0/tests/test_service_cli.py +2144 -0
  270. newsclip-0.7.0/tests/test_serving_authority.py +203 -0
  271. newsclip-0.7.0/tests/test_serving_recovery_workflow.py +201 -0
  272. newsclip-0.7.0/tests/test_session_authority.py +551 -0
  273. newsclip-0.7.0/tests/test_session_cookies.py +211 -0
  274. newsclip-0.7.0/tests/test_session_evidence_holds.py +366 -0
  275. newsclip-0.7.0/tests/test_session_keeper.py +1937 -0
  276. newsclip-0.7.0/tests/test_session_verification.py +749 -0
  277. newsclip-0.7.0/tests/test_sessions.py +2365 -0
  278. newsclip-0.7.0/tests/test_standalone.py +310 -0
  279. newsclip-0.7.0/tests/test_synthetic_route_check.py +1321 -0
  280. newsclip-0.7.0/tests/test_timing.py +839 -0
  281. newsclip-0.7.0/tests/test_trace_retention.py +86 -0
  282. newsclip-0.7.0/tests/test_transport_pins.py +455 -0
  283. newsclip-0.7.0/tests/test_wsj_provider.py +790 -0
  284. newsclip-0.7.0/uv.lock +2266 -0
@@ -0,0 +1,13 @@
1
+ .git
2
+ .github
3
+ .artifacts
4
+ .mypy_cache
5
+ .pytest_cache
6
+ .ruff_cache
7
+ .venv
8
+ bench
9
+ docs
10
+ tests
11
+ *.sqlite3
12
+ *.jsonl
13
+ __pycache__
@@ -0,0 +1,32 @@
1
+ name: CI
2
+ on:
3
+ pull_request:
4
+ push:
5
+ branches: [main]
6
+ workflow_dispatch:
7
+ workflow_call:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ concurrency:
13
+ group: ci-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
14
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
15
+
16
+ jobs:
17
+ test:
18
+ runs-on: ubuntu-latest
19
+ timeout-minutes: 20
20
+ strategy:
21
+ fail-fast: false
22
+ matrix:
23
+ python: ['3.12', '3.13']
24
+ steps:
25
+ - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
26
+ - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990
27
+ with:
28
+ python-version: ${{ matrix.python }}
29
+ - run: uv sync --locked --extra mcp --extra signin
30
+ - run: uv run --locked patchright install --with-deps chromium
31
+ - run: make check
32
+ - run: make check-mcp check-standalone
@@ -0,0 +1,145 @@
1
+ name: Release
2
+ on:
3
+ push:
4
+ tags: ['v*.*.*']
5
+ workflow_dispatch:
6
+
7
+ permissions: {}
8
+
9
+ concurrency:
10
+ group: release-${{ github.ref }}
11
+ cancel-in-progress: false
12
+
13
+ jobs:
14
+ released:
15
+ runs-on: ubuntu-latest
16
+ timeout-minutes: 5
17
+ permissions:
18
+ contents: read
19
+ outputs:
20
+ done: ${{ steps.check.outputs.done }}
21
+ steps:
22
+ - id: check
23
+ env:
24
+ GH_TOKEN: ${{ github.token }}
25
+ TAG: ${{ github.ref_name }}
26
+ run: |
27
+ done=false
28
+ if [ "$GITHUB_REF_TYPE" = tag ] && gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
29
+ done=true
30
+ fi
31
+ echo "done=$done" >> "$GITHUB_OUTPUT"
32
+
33
+ ci:
34
+ needs: released
35
+ if: needs.released.outputs.done != 'true'
36
+ uses: ./.github/workflows/ci.yml
37
+ permissions:
38
+ contents: read
39
+
40
+ images:
41
+ needs: ci
42
+ runs-on: ubuntu-latest
43
+ timeout-minutes: 45
44
+ permissions:
45
+ contents: read
46
+ packages: write
47
+ steps:
48
+ - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
49
+ - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990
50
+ - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0
51
+ with:
52
+ registry: ghcr.io
53
+ username: ${{ github.actor }}
54
+ password: ${{ secrets.GITHUB_TOKEN }}
55
+ - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
56
+ - name: Identify immutable candidates
57
+ id: version
58
+ run: |
59
+ version="$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
60
+ if [ "$GITHUB_REF_TYPE" = tag ]; then
61
+ test "$GITHUB_REF_NAME" = "v$version"
62
+ fi
63
+ echo "version=$version" >> "$GITHUB_OUTPUT"
64
+ echo "candidate=candidate-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT"
65
+ - name: Build standalone server candidate
66
+ id: server
67
+ uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8
68
+ with:
69
+ context: .
70
+ target: server
71
+ platforms: linux/amd64
72
+ push: true
73
+ tags: ghcr.io/${{ github.repository_owner }}/newsclip-mcp:${{ steps.version.outputs.candidate }}
74
+ build-args: |
75
+ VCS_REF=${{ github.sha }}
76
+ IMAGE_VERSION=${{ steps.version.outputs.version }}
77
+ provenance: mode=max
78
+ sbom: true
79
+ cache-from: type=gha,scope=newsclip
80
+ cache-to: type=gha,mode=max,scope=newsclip
81
+ - name: Build sign-in browser candidate
82
+ id: login
83
+ uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8
84
+ with:
85
+ context: .
86
+ target: login
87
+ platforms: linux/amd64
88
+ push: true
89
+ tags: ghcr.io/${{ github.repository_owner }}/newsclip-login-browser:${{ steps.version.outputs.candidate }}
90
+ build-args: |
91
+ VCS_REF=${{ github.sha }}
92
+ IMAGE_VERSION=${{ steps.version.outputs.version }}
93
+ provenance: mode=max
94
+ sbom: true
95
+ cache-from: type=gha,scope=newsclip
96
+ cache-to: type=gha,mode=max,scope=newsclip
97
+ - name: Exercise shipped images without external network access
98
+ env:
99
+ SERVER_IMAGE: ghcr.io/${{ github.repository_owner }}/newsclip-mcp@${{ steps.server.outputs.digest }}
100
+ LOGIN_IMAGE: ghcr.io/${{ github.repository_owner }}/newsclip-login-browser@${{ steps.login.outputs.digest }}
101
+ run: |
102
+ docker pull "$SERVER_IMAGE"
103
+ docker pull "$LOGIN_IMAGE"
104
+ docker run --rm --init --network none --read-only --tmpfs /tmp:rw,exec,mode=1777,size=1g \
105
+ -v "$PWD/scripts/smoke_standalone.py:/acceptance.py:ro" \
106
+ --entrypoint python "$SERVER_IMAGE" /acceptance.py
107
+ timeout 300 docker run --rm --init --network none --read-only --tmpfs /tmp:rw,exec,mode=1777,size=1g --shm-size=256m \
108
+ -v "$PWD/scripts/standalone_e2e.py:/acceptance.py:ro" \
109
+ --entrypoint python "$LOGIN_IMAGE" /acceptance.py
110
+ - name: Build source and wheel distributions
111
+ run: uv build --no-sources
112
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
113
+ with:
114
+ name: distributions
115
+ path: dist/*
116
+ retention-days: 7
117
+ - name: Promote tested digests
118
+ if: github.ref_type == 'tag'
119
+ env:
120
+ REGISTRY: ghcr.io/${{ github.repository_owner }}
121
+ VERSION: ${{ steps.version.outputs.version }}
122
+ SERVER_DIGEST: ${{ steps.server.outputs.digest }}
123
+ LOGIN_DIGEST: ${{ steps.login.outputs.digest }}
124
+ run: |
125
+ bash scripts/promote_image.sh "$REGISTRY/newsclip-mcp" "$SERVER_DIGEST" "$VERSION"
126
+ bash scripts/promote_image.sh "$REGISTRY/newsclip-login-browser" "$LOGIN_DIGEST" "$VERSION"
127
+ docker buildx imagetools create --prefer-index=false --tag "$REGISTRY/newsclip-mcp:latest" "$REGISTRY/newsclip-mcp@$SERVER_DIGEST"
128
+ docker buildx imagetools create --prefer-index=false --tag "$REGISTRY/newsclip-login-browser:latest" "$REGISTRY/newsclip-login-browser@$LOGIN_DIGEST"
129
+
130
+ github-release:
131
+ needs: images
132
+ if: github.event_name == 'push' && github.ref_type == 'tag'
133
+ runs-on: ubuntu-latest
134
+ timeout-minutes: 5
135
+ permissions:
136
+ contents: write
137
+ steps:
138
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
139
+ with:
140
+ name: distributions
141
+ path: dist
142
+ - env:
143
+ GH_TOKEN: ${{ github.token }}
144
+ TAG: ${{ github.ref_name }}
145
+ run: gh release create "$TAG" dist/* --repo "$GITHUB_REPOSITORY" --verify-tag --title "$TAG" --generate-notes
@@ -0,0 +1,19 @@
1
+ __pycache__/
2
+ *.pyc
3
+ .venv/
4
+ bench/results/
5
+ bench/output/
6
+ tmp/
7
+ *.egg-info/
8
+ dist/
9
+ .newsclip/
10
+ *.sqlite3
11
+ *.sqlite3-wal
12
+ *.sqlite3-shm
13
+ *-cookies.json
14
+ cookies.json
15
+ .claude/
16
+ .artifacts/
17
+ local/
18
+ .env
19
+ *.token
@@ -0,0 +1,34 @@
1
+ # Working on Newsclip
2
+
3
+ Newsclip searches, retrieves and privately archives news through CLI, Python and
4
+ MCP interfaces. Only identity-matched, independently assessed FULL content may
5
+ be returned as an article or enter the SQLite archive.
6
+
7
+ Read README.md, ARCHITECTURE.md and docs/DEVELOPMENT.md. AGENTS.md is the only
8
+ agent instruction file; do not add CLAUDE.md or GEMINI.md.
9
+
10
+ ## Boundaries
11
+
12
+ - This source is published. Never write personal, account, household, host or
13
+ network details into public code, comments, docs, fixtures or artifacts.
14
+ - Use synthetic fixtures and example domains. Never copy real article bodies,
15
+ cookies, screenshots, incidents or profiles into a test or issue.
16
+ - Preserve complete user isolation across authorization, sessions, profiles,
17
+ archive, exports, diagnostics and asynchronous callbacks.
18
+ - Sign-in is driven by the person through signin-window. Reader tools cannot
19
+ collect credentials or automate credential submission.
20
+ - A successful sign-in is not proof of entitlement to an arbitrary article.
21
+ Preserve provider assessment and atomic verified-session publication.
22
+ - Keep provider drift local. Shared fixes require a shared reproducer.
23
+ - SQLite is archive authority. Use consistent backup/restore operations.
24
+
25
+ ## Verify and contribute
26
+
27
+ Use `uv sync --locked --extra mcp --extra signin`, install the Patchright
28
+ Chromium, then run `make check` and `make check-mcp`. Add synthetic browser and
29
+ user-isolation acceptance for changes to those paths. Live checks are explicit
30
+ private operator actions; never put credentials or live content in public CI.
31
+
32
+ Follow CONTRIBUTING.md: changes land in the maintained source and are exported
33
+ as releases. Never rewrite a public release tag or merge directly into a release
34
+ mirror. Preserve the MIT license and third-party notices.
@@ -0,0 +1,70 @@
1
+ # Architecture
2
+
3
+ All interfaces use one synchronous `NewsclipService`. Providers own publisher
4
+ URL identity, discovery, assessment and extraction details. Shared code owns
5
+ network budgets, browser leases, session storage and the SQLite archive.
6
+
7
+ ```text
8
+ CLI / Python client / MCP
9
+ |
10
+ authenticated user application
11
+ |
12
+ NewsclipService
13
+ | | |
14
+ providers | SQLite archive
15
+ verified session + retained browser profile
16
+ ```
17
+
18
+ ## User boundary
19
+
20
+ The standalone server loads a bounded user registry containing token hashes.
21
+ Each registered user has an independent mounted application at `/users/USER`.
22
+ Authentication checks that application's user token; a path or request field
23
+ cannot select another user's authority. Token rotation is checked on subsequent
24
+ requests. New users require a restart to create their mounted application.
25
+
26
+ Composition resolves every user's data and state roots independently, without
27
+ changing process-wide environment variables. Sessions, profiles, leases,
28
+ incidents, search cursors, archives and server exports use those roots.
29
+ Callbacks preserve the construction user's context when invoked from a worker.
30
+ The host administrator remains trusted.
31
+
32
+ ## Human sign-in
33
+
34
+ The authenticated `connect` operation creates a short-lived relay link for one
35
+ provider and user. A separate private browser service validates the target and
36
+ opens a restricted signin-window. The person drives the publisher's page.
37
+ Top-level navigation is limited to the provider and required identity hosts;
38
+ resource requests and DNS resolution have explicit allowlists. Browser policy
39
+ blocks local files, downloads, password storage and general browser controls.
40
+
41
+ After the person signs in, Newsclip visits the configured verification article
42
+ and applies the provider's existing full-content checks. It closes Chromium
43
+ before publishing the new profile-bound session. Publication is atomic and
44
+ compares the generation present when the window opened, so an obsolete window
45
+ cannot overwrite a newer session. Failed candidates are discarded without
46
+ replacing existing verified authority.
47
+
48
+ Retained browser state and portable session authority are distinct. A profile
49
+ alone is not permission to read an article. Browser readers use the profile
50
+ associated with the active verified session and serialize access with leases.
51
+
52
+ ## Retrieval and persistence
53
+
54
+ An owned URL is fetched with the applicable user's session, extracted, and
55
+ assessed. Requested, final, canonical and extracted article identity must agree.
56
+ Negative authentication, challenge and paywall signals take precedence over
57
+ plausible prose. Only independently accepted FULL content becomes an Article.
58
+ A verified session does not establish entitlement to every article.
59
+
60
+ `get` returns accepted content transiently. `save` commits immutable raw-fetch
61
+ evidence and an extraction revision in SQLite; Markdown is a derived export.
62
+ Re-extraction reads stored bytes, and archive search does not contact providers.
63
+ Backups use SQLite's consistent backup path rather than copying live WAL files.
64
+
65
+ ## Installation-specific integrations
66
+
67
+ The core has a small optional integration boundary for installations maintaining
68
+ additional operator capabilities. The standalone distribution contains no such
69
+ integration, deployment accounts or credential automation. Provider session
70
+ verification remains usable without an automated acquisition adapter.
@@ -0,0 +1,25 @@
1
+ # Contributing
2
+
3
+ Newsclip welcomes focused bug reports, synthetic regression cases, provider
4
+ improvements and documentation fixes. Read [the architecture](ARCHITECTURE.md)
5
+ and [development guide](docs/DEVELOPMENT.md) before changing provider behavior.
6
+
7
+ Never include credentials, cookies, browser profiles, account details, real
8
+ article bodies, screenshots of signed-in pages, or deployment information in an
9
+ issue, pull request, test fixture or log attachment. Use invented data and
10
+ example domains. Share security reports privately; see [SECURITY.md](SECURITY.md).
11
+
12
+ ## How changes land
13
+
14
+ This repository is a release mirror of the maintained source. Pull requests are
15
+ reviewed here and re-applied to the maintained source; they are not merged into
16
+ the mirror. Accepted changes arrive in a subsequent release, with contributor
17
+ credit in its release notes. This keeps every public release reproducible from
18
+ the same source without maintaining a separate implementation.
19
+
20
+ Explain the problem, expected behavior and a minimal reproducer. Run `make check`
21
+ and, for protocol changes, `make check-mcp`. Public tests use synthetic data and
22
+ must work without publisher accounts or private services. Provider-specific
23
+ selectors and access checks belong in that provider's adapter or definition.
24
+
25
+ Contributions are accepted under the MIT license. Preserve third-party notices.
@@ -0,0 +1,65 @@
1
+ # syntax=docker/dockerfile:1.7
2
+ FROM ghcr.io/astral-sh/uv:0.13.5@sha256:32a7bb39b496babbd997b0a39a4d66c126bdaf5661b302bf6f06abda8deca6ff AS uv
3
+ FROM node:24.22.1-bookworm-slim@sha256:55fb5f5a527ea2c5c4a95f963e48ef90aa38c7b346054fdc8ee9957a2b55d205 AS browser-node
4
+ FROM python:3.12.14-slim-trixie@sha256:f77ac9e44ae96ef2c90b8053ea08c31f8be030f824196b0ae4db6d462c84e51f AS base
5
+ COPY --from=browser-node /usr/local/bin/node /usr/local/bin/browser-node
6
+ ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 UV_LINK_MODE=copy \
7
+ PATH=/app/.venv/bin:$PATH PLAYWRIGHT_BROWSERS_PATH=/opt/chromium \
8
+ HOME=/tmp/newsclip-home NEWSCLIP_STATE_HOME=/state NEWSCLIP_DATA_HOME=/data \
9
+ PLAYWRIGHT_NODEJS_PATH=/usr/local/bin/browser-node
10
+ COPY --from=uv /uv /uvx /usr/local/bin/
11
+ RUN apt-get update && apt-get upgrade -y --no-install-recommends \
12
+ && apt-get install -y --no-install-recommends ca-certificates curl dumb-init libxtst6 xvfb \
13
+ && rm -rf /var/lib/apt/lists/*
14
+ WORKDIR /app
15
+ COPY pyproject.toml uv.lock README.md LICENSE ./
16
+ RUN uv sync --locked --no-dev --extra mcp --extra signin --no-install-project \
17
+ && patchright install --with-deps chromium \
18
+ && /app/.venv/bin/python -c "from pathlib import Path; import patchright; p = Path(patchright.__file__).parent / 'driver' / 'node'; p.unlink(); p.symlink_to('/usr/local/bin/browser-node')" \
19
+ && rm -rf /var/lib/apt/lists/* \
20
+ && groupadd --gid 1000 newsclip \
21
+ && useradd --uid 1000 --gid 1000 --no-create-home --shell /usr/sbin/nologin newsclip
22
+ COPY src ./src
23
+ RUN uv sync --locked --no-dev --extra mcp --extra signin
24
+ ARG VCS_REF=unknown
25
+ ARG IMAGE_VERSION=development
26
+ LABEL org.opencontainers.image.title="Newsclip" \
27
+ org.opencontainers.image.description="Private news search, reading and archiving with human sign-in" \
28
+ org.opencontainers.image.source="https://github.com/benjamin-di/newsclip-mcp" \
29
+ org.opencontainers.image.revision="${VCS_REF}" \
30
+ org.opencontainers.image.version="${IMAGE_VERSION}" \
31
+ org.opencontainers.image.licenses="MIT"
32
+ ENTRYPOINT ["dumb-init", "--"]
33
+
34
+ FROM base AS login
35
+ ENV NOVNC_DIR=/opt/novnc
36
+ ARG NOVNC_VERSION=1.6.0
37
+ ARG NOVNC_SHA256=5066103959ef4e9b10f37e5a148627360dd8414e4cf8a7db92bdbd022e728aaa
38
+ RUN apt-get update \
39
+ && apt-get install -y --no-install-recommends x11vnc fonts-noto-cjk \
40
+ && curl --fail --silent --show-error --location \
41
+ "https://github.com/novnc/noVNC/archive/refs/tags/v${NOVNC_VERSION}.tar.gz" -o /tmp/novnc.tar.gz \
42
+ && printf '%s /tmp/novnc.tar.gz\n' "$NOVNC_SHA256" | sha256sum --check --status \
43
+ && mkdir -p /opt/novnc \
44
+ && tar -xzf /tmp/novnc.tar.gz -C /opt/novnc --strip-components=1 \
45
+ "noVNC-${NOVNC_VERSION}/app" "noVNC-${NOVNC_VERSION}/core" \
46
+ "noVNC-${NOVNC_VERSION}/vendor" "noVNC-${NOVNC_VERSION}/vnc.html" \
47
+ "noVNC-${NOVNC_VERSION}/LICENSE.txt" "noVNC-${NOVNC_VERSION}/AUTHORS" \
48
+ "noVNC-${NOVNC_VERSION}/docs/LICENSE.MPL-2.0" \
49
+ "noVNC-${NOVNC_VERSION}/docs/LICENSE.OFL-1.1" \
50
+ "noVNC-${NOVNC_VERSION}/docs/LICENSE.BSD-3-Clause" \
51
+ "noVNC-${NOVNC_VERSION}/docs/LICENSE.BSD-2-Clause" \
52
+ && python -m signin_window.browser install-policy \
53
+ && rm -rf /tmp/novnc.tar.gz /var/lib/apt/lists/*
54
+ USER 1000:1000
55
+ EXPOSE 8080
56
+ HEALTHCHECK --interval=30s --timeout=5s --start-period=2m --start-interval=2s --retries=3 \
57
+ CMD ["curl", "--fail", "--silent", "http://127.0.0.1:8080/health"]
58
+ CMD ["newsclip", "signin-service", "--host", "0.0.0.0"]
59
+
60
+ FROM base AS server
61
+ USER 1000:1000
62
+ EXPOSE 8000
63
+ HEALTHCHECK --interval=30s --timeout=5s --start-period=2m --start-interval=2s --retries=3 \
64
+ CMD ["curl", "--fail", "--silent", "http://127.0.0.1:8000/health"]
65
+ CMD ["newsclip", "serve", "--host", "0.0.0.0"]
@@ -0,0 +1,39 @@
1
+ .git
2
+ .venv
3
+ __pycache__
4
+ *.pyc
5
+ local
6
+ .env
7
+ .artifacts
8
+ .work
9
+ tmp
10
+ dist
11
+ .github/
12
+ publish/
13
+ .gitattributes
14
+ private/
15
+ deploy/
16
+ bench/
17
+ src/newsclip/local_extensions.py
18
+ src/newsclip/identity*
19
+ src/newsclip/identity/
20
+ src/newsclip/identity_targets/
21
+ src/newsclip/identity_definitions/
22
+ src/newsclip/providers/*_auth.py
23
+ src/newsclip/login.py
24
+ src/newsclip/secret_reference.py
25
+ src/newsclip/credential_readiness.py
26
+ src/newsclip/agent_run_vision.py
27
+ src/newsclip/vision_cli.py
28
+ src/newsclip/vision_planning.py
29
+ src/newsclip/hcaptcha.py
30
+ src/newsclip/ops_server.py
31
+ src/newsclip/repair_policy.py
32
+ scripts/live_check.py
33
+ scripts/prepare_auth_repair.py
34
+ scripts/promote_image.sh
35
+ scripts/smoke_image.sh
36
+ scripts/synthetic_route_check.py
37
+ scripts/explain_live_report.py
38
+ tests/fixtures/diagnostics/
39
+ tests/fixtures/identity_snapshot/
newsclip-0.7.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 benjamin-di
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,21 @@
1
+ .PHONY: check check-mcp check-standalone check-runtime-lock
2
+
3
+ # Keep local and CI verification identical. Live providers are always explicit.
4
+ check: check-runtime-lock
5
+ uv run --locked ruff check .
6
+ uv run --locked ruff format --check .
7
+ uv run --locked mypy src/newsclip
8
+ uv run --locked pytest -q -m "not live"
9
+ git diff --check
10
+
11
+ # Keep image dependencies independent of the application's release version.
12
+ check-runtime-lock:
13
+ @test ! -f deploy/requirements.runtime.txt || { expected=$$(mktemp); trap 'rm -f "$$expected"' EXIT; \
14
+ uv export --locked --no-dev --extra mcp --no-emit-project --no-header --output-file "$$expected" >/dev/null && \
15
+ diff -u deploy/requirements.runtime.txt "$$expected"; }
16
+
17
+ check-mcp:
18
+ uv run --locked --extra mcp pytest -q -m "not live" tests/test_mcp.py tests/test_mcp_http.py tests/test_drift.py tests/test_serving_recovery_workflow.py
19
+
20
+ check-standalone:
21
+ uv run --locked --extra mcp --extra signin pytest -q tests/test_standalone.py tests/test_remote_cli.py
newsclip-0.7.0/NOTICE ADDED
@@ -0,0 +1,10 @@
1
+ Newsclip's original code is licensed under the MIT license in LICENSE.
2
+
3
+ Newsclip depends on separately licensed Python packages and system components.
4
+ Their licenses continue to apply. The login browser image bundles noVNC,
5
+ Chromium, Xvfb and x11vnc; it preserves their upstream license files and package
6
+ copyright notices. The MIT license for Newsclip does not relicense these works.
7
+
8
+ Publisher names identify compatible services. Newsclip is not affiliated with
9
+ or endorsed by those publishers. Publisher content remains the property of its
10
+ respective rights holders; the software license grants no rights to that content.
@@ -0,0 +1,157 @@
1
+ Metadata-Version: 2.5
2
+ Name: newsclip
3
+ Version: 0.7.0
4
+ Summary: News search, verified subscription reading and private archiving
5
+ Project-URL: Homepage, https://github.com/benjamin-di/newsclip-mcp
6
+ Project-URL: Documentation, https://github.com/benjamin-di/newsclip-mcp#readme
7
+ Project-URL: Issues, https://github.com/benjamin-di/newsclip-mcp/issues
8
+ Author: benjamin-di
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Requires-Python: >=3.12
12
+ Requires-Dist: click<9,>=8.4
13
+ Requires-Dist: curl-cffi<1,>=0.16.3
14
+ Requires-Dist: httpx<1,>=0.28
15
+ Requires-Dist: lxml<7,>=5
16
+ Requires-Dist: patchright<2,>=1.52
17
+ Requires-Dist: pydantic<3,>=2.11
18
+ Requires-Dist: rich<15,>=14
19
+ Requires-Dist: trafilatura<3,>=2
20
+ Requires-Dist: typer<1,>=0.16
21
+ Provides-Extra: login
22
+ Provides-Extra: mcp
23
+ Requires-Dist: anyio<5,>=4.8; extra == 'mcp'
24
+ Requires-Dist: fastmcp<5,>=4.0.10; extra == 'mcp'
25
+ Requires-Dist: starlette<2,>=1.0.1; extra == 'mcp'
26
+ Requires-Dist: uvicorn<1,>=0.34; extra == 'mcp'
27
+ Provides-Extra: signin
28
+ Requires-Dist: signin-window<0.4,>=0.3.0; extra == 'signin'
29
+ Requires-Dist: uvicorn<1,>=0.34; extra == 'signin'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # Newsclip
33
+
34
+ Search, read and privately archive news through a command-line interface,
35
+ Python API, or MCP server. Newsclip uses your existing subscriptions and public
36
+ publisher endpoints. It returns an article only when its identity and full
37
+ content pass the provider's verification checks.
38
+
39
+ Subscription sign-in uses [signin-window](https://github.com/benjamin-di/signin-window):
40
+ open a short-lived link and sign in on the publisher's page yourself. Newsclip
41
+ keeps the verified session afterwards. It does not need your password in a
42
+ configuration file or an assistant conversation.
43
+
44
+ - Ten built-in providers: Financial Times, WSJ, Bloomberg, CNBC, Reuters,
45
+ The Economist, The New York Times, The Guardian, AP News and Axios.
46
+ - Fair federated search with source status, date filters and coverage limits.
47
+ - Separate user tokens, sessions, retained browser profiles and SQLite archives.
48
+ - Transient reading with `get`; durable storage and Markdown export with `save`.
49
+ - Local CLI/Python access and authenticated HTTP/MCP access to the same core.
50
+ - MIT-licensed original code, with synthetic tests and no bundled account data.
51
+
52
+ ## Quick start
53
+
54
+ The reference server runs on Linux AMD64 with Docker Compose. Python 3.12+ and
55
+ [uv](https://docs.astral.sh/uv/) are used for setup and the CLI. The sign-in
56
+ viewer works in a browser or phone; the browser service itself runs on Linux.
57
+
58
+ ```sh
59
+ git clone https://github.com/benjamin-di/newsclip-mcp.git
60
+ cd newsclip-mcp
61
+ uv sync --locked --extra mcp --extra signin
62
+ uv run python scripts/setup.py --user alice --user bob
63
+ docker compose up -d
64
+ ```
65
+
66
+ Setup writes private token files and configuration without printing secrets.
67
+ Each person uses their own URL and token file:
68
+
69
+ ```sh
70
+ export NEWSCLIP_URL=http://127.0.0.1:8000/users/alice
71
+ export NEWSCLIP_TOKEN_FILE="$PWD/local/tokens/alice.token"
72
+
73
+ uv run newsclip search "energy markets" --source ap --json
74
+ uv run newsclip auth connect ft
75
+ # Open the returned link, complete sign-in, then check the session.
76
+ uv run newsclip auth status ft --json
77
+ uv run newsclip get 'https://www.ft.com/content/<article-id>'
78
+ uv run newsclip save 'https://www.ft.com/content/<article-id>' --out article.md
79
+ uv run newsclip archive-search "energy" --json
80
+ ```
81
+
82
+ `get` does not archive. `save` commits the verified article and can additionally
83
+ write a Markdown file on the CLI client's machine. Failed, partial and
84
+ unauthorized pages never become archived articles. Use `archive-get ARTICLE_ID`
85
+ to read a saved article without contacting its publisher.
86
+
87
+ The default deployment is reachable only on the server itself. For phone or
88
+ remote-client access, configure an HTTPS public origin and reverse proxy as
89
+ explained in [deployment](docs/DEPLOYMENT.md).
90
+
91
+ ## Interfaces
92
+
93
+ For MCP, connect your client to
94
+ `http://127.0.0.1:8000/users/alice/mcp` with Alice's bearer token. The server
95
+ exposes `search`, `get`, `save`, `archive_search`, `archive_get`, `reextract`,
96
+ `session_status`, and `connect`. `connect` returns a human sign-in link; read
97
+ operations never submit credentials.
98
+
99
+ For Python:
100
+
101
+ ```python
102
+ from pathlib import Path
103
+ from newsclip.client import NewsclipClient
104
+
105
+ with NewsclipClient(
106
+ "http://127.0.0.1:8000/users/alice",
107
+ Path("local/tokens/alice.token").read_text(),
108
+ ) as client:
109
+ response = client.search_many("energy markets", provider_keys=["ap"])
110
+ for result in response.results:
111
+ print(result.headline, result.url)
112
+ ```
113
+
114
+ The local CLI and synchronous `NewsclipService` are also available without
115
+ running an HTTP server. `newsclip --user alice ...` selects isolated local
116
+ storage; it is an administrator/local-process option, not remote authorization.
117
+ `newsclip mcp` serves local stdio MCP with the same local user's access.
118
+
119
+ ## Providers and limits
120
+
121
+ See the [provider matrix](docs/PROVIDERS.md) for authentication, search coverage
122
+ and retrieval behavior. A supported adapter cannot guarantee that a publisher
123
+ will accept every network, account or browser. Articles outside a subscription
124
+ are reported as `not_entitled`. Provider failures remain visible beside healthy
125
+ search results.
126
+
127
+ Newsclip is for content you are authorized to access. You supply the necessary
128
+ subscriptions and API permissions. It does not purchase access, redistribute
129
+ publisher content, or grant rights to that content through its software license.
130
+
131
+ ## Documentation
132
+
133
+ - [CLI reference](docs/CLI.md) and [Python/HTTP/MCP API](docs/API.md)
134
+ - [Sign-in and user isolation](docs/AUTHENTICATION.md)
135
+ - [Deployment, configuration, backups and upgrades](docs/DEPLOYMENT.md)
136
+ - [Provider support and extension](docs/PROVIDERS.md)
137
+ - [Architecture](ARCHITECTURE.md), [development](docs/DEVELOPMENT.md) and [roadmap](ROADMAP.md)
138
+ - [Contributing](CONTRIBUTING.md), [security reporting](SECURITY.md) and [third-party notices](NOTICE)
139
+
140
+ ## Development
141
+
142
+ ```sh
143
+ uv sync --locked --extra mcp --extra signin
144
+ uv run patchright install chromium
145
+ make check
146
+ make check-mcp
147
+ ```
148
+
149
+ Offline tests use synthetic content and require no publisher credentials. Live
150
+ provider checks are explicit private operator actions and are never run on a
151
+ contributor's pull request. Public images and release archives are built from
152
+ the public release source.
153
+
154
+ ## License
155
+
156
+ [MIT](LICENSE) for original code. Dependencies retain their own licenses; see
157
+ [NOTICE](NOTICE). Newsclip is not affiliated with the supported publishers.