papersearch 0.0.0-stage → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +208 -0
- package/LICENSE +21 -0
- package/README.md +213 -2
- package/README.zh.md +214 -0
- package/carsi/LICENSE +21 -0
- package/carsi/NOTICE +5 -0
- package/carsi/README.md +75 -0
- package/carsi/paper_search_carsi/__init__.py +6 -0
- package/carsi/paper_search_carsi/_core/__init__.py +26 -0
- package/carsi/paper_search_carsi/_core/racing.pyx +280 -0
- package/carsi/paper_search_carsi/adapters/__init__.py +26 -0
- package/carsi/paper_search_carsi/adapters/acm/__init__.py +13 -0
- package/carsi/paper_search_carsi/adapters/acs/__init__.py +47 -0
- package/carsi/paper_search_carsi/adapters/aha_journals/__init__.py +13 -0
- package/carsi/paper_search_carsi/adapters/ams/__init__.py +19 -0
- package/carsi/paper_search_carsi/adapters/annual_reviews/__init__.py +154 -0
- package/carsi/paper_search_carsi/adapters/aps/__init__.py +127 -0
- package/carsi/paper_search_carsi/adapters/asce/__init__.py +13 -0
- package/carsi/paper_search_carsi/adapters/base.py +320 -0
- package/carsi/paper_search_carsi/adapters/bmj/__init__.py +32 -0
- package/carsi/paper_search_carsi/adapters/copernicus/__init__.py +20 -0
- package/carsi/paper_search_carsi/adapters/dovepress/__init__.py +43 -0
- package/carsi/paper_search_carsi/adapters/elsevier/__init__.py +302 -0
- package/carsi/paper_search_carsi/adapters/elsevier/journal_site.py +237 -0
- package/carsi/paper_search_carsi/adapters/elsevier/sciencedirect.py +138 -0
- package/carsi/paper_search_carsi/adapters/ieee/__init__.py +184 -0
- package/carsi/paper_search_carsi/adapters/iop/__init__.py +41 -0
- package/carsi/paper_search_carsi/adapters/jama/__init__.py +54 -0
- package/carsi/paper_search_carsi/adapters/karger/__init__.py +75 -0
- package/carsi/paper_search_carsi/adapters/lww/__init__.py +103 -0
- package/carsi/paper_search_carsi/adapters/lww/journals_lww.py +36 -0
- package/carsi/paper_search_carsi/adapters/lww/ovid.py +72 -0
- package/carsi/paper_search_carsi/adapters/mdpi/__init__.py +20 -0
- package/carsi/paper_search_carsi/adapters/nejm/__init__.py +18 -0
- package/carsi/paper_search_carsi/adapters/neurology/__init__.py +13 -0
- package/carsi/paper_search_carsi/adapters/optica/__init__.py +33 -0
- package/carsi/paper_search_carsi/adapters/oup/__init__.py +366 -0
- package/carsi/paper_search_carsi/adapters/platforms/__init__.py +5 -0
- package/carsi/paper_search_carsi/adapters/platforms/atypon/__init__.py +149 -0
- package/carsi/paper_search_carsi/adapters/platforms/silverchair/__init__.py +83 -0
- package/carsi/paper_search_carsi/adapters/plos/__init__.py +20 -0
- package/carsi/paper_search_carsi/adapters/pnas/__init__.py +13 -0
- package/carsi/paper_search_carsi/adapters/rsc/__init__.py +74 -0
- package/carsi/paper_search_carsi/adapters/sage/__init__.py +154 -0
- package/carsi/paper_search_carsi/adapters/science/__init__.py +13 -0
- package/carsi/paper_search_carsi/adapters/springer/__init__.py +62 -0
- package/carsi/paper_search_carsi/adapters/taylor_francis/__init__.py +46 -0
- package/carsi/paper_search_carsi/adapters/thieme/__init__.py +164 -0
- package/carsi/paper_search_carsi/adapters/wiley/__init__.py +145 -0
- package/carsi/paper_search_carsi/adapters/world_scientific/__init__.py +27 -0
- package/carsi/paper_search_carsi/assets/icon-32.png +0 -0
- package/carsi/paper_search_carsi/assets/icon-512.png +0 -0
- package/carsi/paper_search_carsi/assets/icon-64.png +0 -0
- package/carsi/paper_search_carsi/assets/icon.svg +43 -0
- package/carsi/paper_search_carsi/blocked_journals.py +63 -0
- package/carsi/paper_search_carsi/browser/__init__.py +1 -0
- package/carsi/paper_search_carsi/browser/backend.py +823 -0
- package/carsi/paper_search_carsi/browser/cookies.py +386 -0
- package/carsi/paper_search_carsi/browser/ego_relay.py +152 -0
- package/carsi/paper_search_carsi/browser/engine.py +927 -0
- package/carsi/paper_search_carsi/browser/login.py +336 -0
- package/carsi/paper_search_carsi/browser/preview.py +379 -0
- package/carsi/paper_search_carsi/browser/watchdog.py +193 -0
- package/carsi/paper_search_carsi/cache.py +62 -0
- package/carsi/paper_search_carsi/carsi_schools.py +109 -0
- package/carsi/paper_search_carsi/cli.py +479 -0
- package/carsi/paper_search_carsi/cold_start.py +251 -0
- package/carsi/paper_search_carsi/config.py +301 -0
- package/carsi/paper_search_carsi/data/institutional_identity_policy.json +165 -0
- package/carsi/paper_search_carsi/data/publisher_access_catalog.json +969 -0
- package/carsi/paper_search_carsi/data/publisher_browser_verification_matrix.json +625 -0
- package/carsi/paper_search_carsi/download_diagnostics.py +228 -0
- package/carsi/paper_search_carsi/engine/__init__.py +1 -0
- package/carsi/paper_search_carsi/engine/access_rules.py +345 -0
- package/carsi/paper_search_carsi/engine/batch.py +702 -0
- package/carsi/paper_search_carsi/engine/batch_access.py +483 -0
- package/carsi/paper_search_carsi/engine/batch_capture.py +1531 -0
- package/carsi/paper_search_carsi/engine/batch_common.py +427 -0
- package/carsi/paper_search_carsi/engine/batch_diagnostics.py +307 -0
- package/carsi/paper_search_carsi/engine/batch_institution.py +981 -0
- package/carsi/paper_search_carsi/engine/batch_login.py +1605 -0
- package/carsi/paper_search_carsi/engine/batch_pdf_urls.py +274 -0
- package/carsi/paper_search_carsi/engine/batch_runs.py +711 -0
- package/carsi/paper_search_carsi/engine/batch_user.py +173 -0
- package/carsi/paper_search_carsi/engine/batch_waits.py +441 -0
- package/carsi/paper_search_carsi/engine/bridge.py +666 -0
- package/carsi/paper_search_carsi/engine/pdf_guard.py +310 -0
- package/carsi/paper_search_carsi/engine/router.py +514 -0
- package/carsi/paper_search_carsi/engine/supplements.py +148 -0
- package/carsi/paper_search_carsi/entitlements.py +233 -0
- package/carsi/paper_search_carsi/extractors/__init__.py +0 -0
- package/carsi/paper_search_carsi/extractors/pdf_extractor.py +149 -0
- package/carsi/paper_search_carsi/identifiers.py +121 -0
- package/carsi/paper_search_carsi/issues.py +227 -0
- package/carsi/paper_search_carsi/log.py +84 -0
- package/carsi/paper_search_carsi/mcp_progress.py +82 -0
- package/carsi/paper_search_carsi/network.py +359 -0
- package/carsi/paper_search_carsi/orchestrator.py +130 -0
- package/carsi/paper_search_carsi/pdf_utils.py +279 -0
- package/carsi/paper_search_carsi/progress_bar.py +778 -0
- package/carsi/paper_search_carsi/progress_reporter.py +530 -0
- package/carsi/paper_search_carsi/progress_status.py +212 -0
- package/carsi/paper_search_carsi/publishers/__init__.py +10 -0
- package/carsi/paper_search_carsi/publishers/builtin/acm.json +65 -0
- package/carsi/paper_search_carsi/publishers/builtin/acs.json +73 -0
- package/carsi/paper_search_carsi/publishers/builtin/aha-journals.json +47 -0
- package/carsi/paper_search_carsi/publishers/builtin/aip.json +66 -0
- package/carsi/paper_search_carsi/publishers/builtin/ams.json +66 -0
- package/carsi/paper_search_carsi/publishers/builtin/annual-reviews.json +64 -0
- package/carsi/paper_search_carsi/publishers/builtin/aps.json +70 -0
- package/carsi/paper_search_carsi/publishers/builtin/asce.json +46 -0
- package/carsi/paper_search_carsi/publishers/builtin/bmj.json +46 -0
- package/carsi/paper_search_carsi/publishers/builtin/copernicus.json +45 -0
- package/carsi/paper_search_carsi/publishers/builtin/dovepress.json +35 -0
- package/carsi/paper_search_carsi/publishers/builtin/elsevier.json +77 -0
- package/carsi/paper_search_carsi/publishers/builtin/frontiers.json +62 -0
- package/carsi/paper_search_carsi/publishers/builtin/ieee.json +68 -0
- package/carsi/paper_search_carsi/publishers/builtin/iop.json +69 -0
- package/carsi/paper_search_carsi/publishers/builtin/jama.json +48 -0
- package/carsi/paper_search_carsi/publishers/builtin/karger.json +48 -0
- package/carsi/paper_search_carsi/publishers/builtin/lww.json +57 -0
- package/carsi/paper_search_carsi/publishers/builtin/mdpi.json +46 -0
- package/carsi/paper_search_carsi/publishers/builtin/nejm.json +46 -0
- package/carsi/paper_search_carsi/publishers/builtin/neurology.json +45 -0
- package/carsi/paper_search_carsi/publishers/builtin/optica.json +77 -0
- package/carsi/paper_search_carsi/publishers/builtin/oxfordacademic.json +64 -0
- package/carsi/paper_search_carsi/publishers/builtin/plos.json +46 -0
- package/carsi/paper_search_carsi/publishers/builtin/pnas.json +67 -0
- package/carsi/paper_search_carsi/publishers/builtin/royalsocietypublishing.json +62 -0
- package/carsi/paper_search_carsi/publishers/builtin/rsc.json +71 -0
- package/carsi/paper_search_carsi/publishers/builtin/sage.json +48 -0
- package/carsi/paper_search_carsi/publishers/builtin/science.json +68 -0
- package/carsi/paper_search_carsi/publishers/builtin/springer.json +92 -0
- package/carsi/paper_search_carsi/publishers/builtin/taylor-francis.json +52 -0
- package/carsi/paper_search_carsi/publishers/builtin/thieme.json +50 -0
- package/carsi/paper_search_carsi/publishers/builtin/wiley.json +77 -0
- package/carsi/paper_search_carsi/publishers/builtin/world-scientific.json +68 -0
- package/carsi/paper_search_carsi/publishers/extensions.py +241 -0
- package/carsi/paper_search_carsi/publishers/loader.py +93 -0
- package/carsi/paper_search_carsi/publishers/model.py +90 -0
- package/carsi/paper_search_carsi/publishers/profiles.py +127 -0
- package/carsi/paper_search_carsi/publishers/schema.json +129 -0
- package/carsi/paper_search_carsi/publishers/site.py +86 -0
- package/carsi/paper_search_carsi/rate_limit.py +51 -0
- package/carsi/paper_search_carsi/server.py +197 -0
- package/carsi/paper_search_carsi/session_broker.py +163 -0
- package/carsi/paper_search_carsi/tor.py +9 -0
- package/carsi/paper_search_carsi/user_settings.py +55 -0
- package/carsi/pyproject.toml +41 -0
- package/dist/capabilities/body-snippet-search/handler.d.ts +9 -0
- package/dist/capabilities/body-snippet-search/handler.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/handler.js +17 -0
- package/dist/capabilities/body-snippet-search/handler.js.map +1 -0
- package/dist/capabilities/body-snippet-search/index.d.ts +7 -0
- package/dist/capabilities/body-snippet-search/index.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/index.js +6 -0
- package/dist/capabilities/body-snippet-search/index.js.map +1 -0
- package/dist/capabilities/body-snippet-search/schemas.d.ts +36 -0
- package/dist/capabilities/body-snippet-search/schemas.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/schemas.js +16 -0
- package/dist/capabilities/body-snippet-search/schemas.js.map +1 -0
- package/dist/capabilities/body-snippet-search/service.d.ts +4 -0
- package/dist/capabilities/body-snippet-search/service.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/service.js +4 -0
- package/dist/capabilities/body-snippet-search/service.js.map +1 -0
- package/dist/capabilities/body-snippet-search/tools.d.ts +187 -0
- package/dist/capabilities/body-snippet-search/tools.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/tools.js +49 -0
- package/dist/capabilities/body-snippet-search/tools.js.map +1 -0
- package/dist/capabilities/body-snippet-search/types.d.ts +14 -0
- package/dist/capabilities/body-snippet-search/types.d.ts.map +1 -0
- package/dist/capabilities/body-snippet-search/types.js +2 -0
- package/dist/capabilities/body-snippet-search/types.js.map +1 -0
- package/dist/capabilities/citation-expansion/CitationService.d.ts +47 -0
- package/dist/capabilities/citation-expansion/CitationService.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/CitationService.js +243 -0
- package/dist/capabilities/citation-expansion/CitationService.js.map +1 -0
- package/dist/capabilities/citation-expansion/handler.d.ts +36 -0
- package/dist/capabilities/citation-expansion/handler.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/handler.js +44 -0
- package/dist/capabilities/citation-expansion/handler.js.map +1 -0
- package/dist/capabilities/citation-expansion/index.d.ts +6 -0
- package/dist/capabilities/citation-expansion/index.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/index.js +5 -0
- package/dist/capabilities/citation-expansion/index.js.map +1 -0
- package/dist/capabilities/citation-expansion/schemas.d.ts +28 -0
- package/dist/capabilities/citation-expansion/schemas.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/schemas.js +13 -0
- package/dist/capabilities/citation-expansion/schemas.js.map +1 -0
- package/dist/capabilities/citation-expansion/tools.d.ts +82 -0
- package/dist/capabilities/citation-expansion/tools.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/tools.js +48 -0
- package/dist/capabilities/citation-expansion/tools.js.map +1 -0
- package/dist/capabilities/citation-expansion/types.d.ts +22 -0
- package/dist/capabilities/citation-expansion/types.d.ts.map +1 -0
- package/dist/capabilities/citation-expansion/types.js +2 -0
- package/dist/capabilities/citation-expansion/types.js.map +1 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.d.ts +3 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.js +142 -0
- package/dist/capabilities/journal-metrics/JournalMetricsService.js.map +1 -0
- package/dist/capabilities/journal-metrics/handler.d.ts +12 -0
- package/dist/capabilities/journal-metrics/handler.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/handler.js +38 -0
- package/dist/capabilities/journal-metrics/handler.js.map +1 -0
- package/dist/capabilities/journal-metrics/index.d.ts +6 -0
- package/dist/capabilities/journal-metrics/index.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/index.js +5 -0
- package/dist/capabilities/journal-metrics/index.js.map +1 -0
- package/dist/capabilities/journal-metrics/schemas.d.ts +28 -0
- package/dist/capabilities/journal-metrics/schemas.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/schemas.js +13 -0
- package/dist/capabilities/journal-metrics/schemas.js.map +1 -0
- package/dist/capabilities/journal-metrics/tools.d.ts +67 -0
- package/dist/capabilities/journal-metrics/tools.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/tools.js +27 -0
- package/dist/capabilities/journal-metrics/tools.js.map +1 -0
- package/dist/capabilities/journal-metrics/types.d.ts +36 -0
- package/dist/capabilities/journal-metrics/types.d.ts.map +1 -0
- package/dist/capabilities/journal-metrics/types.js +2 -0
- package/dist/capabilities/journal-metrics/types.js.map +1 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.d.ts +9 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.js +81 -0
- package/dist/capabilities/metadata-search/MultiSourceSearchService.js.map +1 -0
- package/dist/capabilities/metadata-search/handler.d.ts +23 -0
- package/dist/capabilities/metadata-search/handler.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/handler.js +153 -0
- package/dist/capabilities/metadata-search/handler.js.map +1 -0
- package/dist/capabilities/metadata-search/index.d.ts +6 -0
- package/dist/capabilities/metadata-search/index.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/index.js +5 -0
- package/dist/capabilities/metadata-search/index.js.map +1 -0
- package/dist/capabilities/metadata-search/schemas.d.ts +83 -0
- package/dist/capabilities/metadata-search/schemas.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/schemas.js +80 -0
- package/dist/capabilities/metadata-search/schemas.js.map +1 -0
- package/dist/capabilities/metadata-search/tools.d.ts +178 -0
- package/dist/capabilities/metadata-search/tools.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/tools.js +75 -0
- package/dist/capabilities/metadata-search/tools.js.map +1 -0
- package/dist/capabilities/metadata-search/types.d.ts +13 -0
- package/dist/capabilities/metadata-search/types.d.ts.map +1 -0
- package/dist/capabilities/metadata-search/types.js +2 -0
- package/dist/capabilities/metadata-search/types.js.map +1 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.d.ts +43 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.js +2 -0
- package/dist/capabilities/pdf-discovery/DownloadTier.js.map +1 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.d.ts +16 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.js +202 -0
- package/dist/capabilities/pdf-discovery/OpenAccessFallbackService.js.map +1 -0
- package/dist/capabilities/pdf-discovery/batch.d.ts +74 -0
- package/dist/capabilities/pdf-discovery/batch.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/batch.js +370 -0
- package/dist/capabilities/pdf-discovery/batch.js.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiAttempt.d.ts +7 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiAttempt.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiAttempt.js +46 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiAttempt.js.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiMcpClient.d.ts +35 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiMcpClient.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiMcpClient.js +689 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiMcpClient.js.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiProgress.d.ts +23 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiProgress.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiProgress.js +30 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiProgress.js.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiReadiness.d.ts +29 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiReadiness.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiReadiness.js +176 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiReadiness.js.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiResultNormalizer.d.ts +8 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiResultNormalizer.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiResultNormalizer.js +225 -0
- package/dist/capabilities/pdf-discovery/carsi/CarsiResultNormalizer.js.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/index.d.ts +8 -0
- package/dist/capabilities/pdf-discovery/carsi/index.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/index.js +11 -0
- package/dist/capabilities/pdf-discovery/carsi/index.js.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/types.d.ts +118 -0
- package/dist/capabilities/pdf-discovery/carsi/types.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/carsi/types.js +2 -0
- package/dist/capabilities/pdf-discovery/carsi/types.js.map +1 -0
- package/dist/capabilities/pdf-discovery/handler.d.ts +28 -0
- package/dist/capabilities/pdf-discovery/handler.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/handler.js +72 -0
- package/dist/capabilities/pdf-discovery/handler.js.map +1 -0
- package/dist/capabilities/pdf-discovery/identity.d.ts +8 -0
- package/dist/capabilities/pdf-discovery/identity.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/identity.js +24 -0
- package/dist/capabilities/pdf-discovery/identity.js.map +1 -0
- package/dist/capabilities/pdf-discovery/index.d.ts +8 -0
- package/dist/capabilities/pdf-discovery/index.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/index.js +6 -0
- package/dist/capabilities/pdf-discovery/index.js.map +1 -0
- package/dist/capabilities/pdf-discovery/oaHosts.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/oaHosts.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/oaHosts.js +37 -0
- package/dist/capabilities/pdf-discovery/oaHosts.js.map +1 -0
- package/dist/capabilities/pdf-discovery/pmcCloud.d.ts +6 -0
- package/dist/capabilities/pdf-discovery/pmcCloud.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/pmcCloud.js +95 -0
- package/dist/capabilities/pdf-discovery/pmcCloud.js.map +1 -0
- package/dist/capabilities/pdf-discovery/publisherApi/elsevier.d.ts +4 -0
- package/dist/capabilities/pdf-discovery/publisherApi/elsevier.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/publisherApi/elsevier.js +122 -0
- package/dist/capabilities/pdf-discovery/publisherApi/elsevier.js.map +1 -0
- package/dist/capabilities/pdf-discovery/publisherApi/index.d.ts +31 -0
- package/dist/capabilities/pdf-discovery/publisherApi/index.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/publisherApi/index.js +20 -0
- package/dist/capabilities/pdf-discovery/publisherApi/index.js.map +1 -0
- package/dist/capabilities/pdf-discovery/publisherApi/springer.d.ts +4 -0
- package/dist/capabilities/pdf-discovery/publisherApi/springer.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/publisherApi/springer.js +114 -0
- package/dist/capabilities/pdf-discovery/publisherApi/springer.js.map +1 -0
- package/dist/capabilities/pdf-discovery/schemas.d.ts +176 -0
- package/dist/capabilities/pdf-discovery/schemas.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/schemas.js +72 -0
- package/dist/capabilities/pdf-discovery/schemas.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.d.ts +5 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.js +40 -0
- package/dist/capabilities/pdf-discovery/tiers/directPdfUrl.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/entitledAccess.d.ts +13 -0
- package/dist/capabilities/pdf-discovery/tiers/entitledAccess.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/entitledAccess.js +123 -0
- package/dist/capabilities/pdf-discovery/tiers/entitledAccess.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.js +21 -0
- package/dist/capabilities/pdf-discovery/tiers/primary.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/publisherApi.d.ts +12 -0
- package/dist/capabilities/pdf-discovery/tiers/publisherApi.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/publisherApi.js +109 -0
- package/dist/capabilities/pdf-discovery/tiers/publisherApi.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.d.ts +10 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.js +37 -0
- package/dist/capabilities/pdf-discovery/tiers/repositories.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.js +21 -0
- package/dist/capabilities/pdf-discovery/tiers/scihub.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.d.ts +3 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.js +76 -0
- package/dist/capabilities/pdf-discovery/tiers/unpaywall.js.map +1 -0
- package/dist/capabilities/pdf-discovery/tools.d.ts +337 -0
- package/dist/capabilities/pdf-discovery/tools.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/tools.js +144 -0
- package/dist/capabilities/pdf-discovery/tools.js.map +1 -0
- package/dist/capabilities/pdf-discovery/types.d.ts +86 -0
- package/dist/capabilities/pdf-discovery/types.d.ts.map +1 -0
- package/dist/capabilities/pdf-discovery/types.js +2 -0
- package/dist/capabilities/pdf-discovery/types.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +1196 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/ConfigService.d.ts +28 -0
- package/dist/config/ConfigService.d.ts.map +1 -0
- package/dist/config/ConfigService.js +204 -0
- package/dist/config/ConfigService.js.map +1 -0
- package/dist/config/ResultCaps.d.ts +4 -0
- package/dist/config/ResultCaps.d.ts.map +1 -0
- package/dist/config/ResultCaps.js +10 -0
- package/dist/config/ResultCaps.js.map +1 -0
- package/dist/config/constants.d.ts +158 -0
- package/dist/config/constants.d.ts.map +1 -0
- package/dist/config/constants.js +102 -0
- package/dist/config/constants.js.map +1 -0
- package/dist/core/capabilityProfile.d.ts +2 -0
- package/dist/core/capabilityProfile.d.ts.map +1 -0
- package/dist/core/capabilityProfile.js +2 -0
- package/dist/core/capabilityProfile.js.map +1 -0
- package/dist/core/diagnostics.d.ts +2 -0
- package/dist/core/diagnostics.d.ts.map +1 -0
- package/dist/core/diagnostics.js +2 -0
- package/dist/core/diagnostics.js.map +1 -0
- package/dist/core/handleToolCall.d.ts +13 -0
- package/dist/core/handleToolCall.d.ts.map +1 -0
- package/dist/core/handleToolCall.js +304 -0
- package/dist/core/handleToolCall.js.map +1 -0
- package/dist/core/httpPolicies.d.ts +2 -0
- package/dist/core/httpPolicies.d.ts.map +1 -0
- package/dist/core/httpPolicies.js +2 -0
- package/dist/core/httpPolicies.js.map +1 -0
- package/dist/core/liveSmoke.d.ts +2 -0
- package/dist/core/liveSmoke.d.ts.map +1 -0
- package/dist/core/liveSmoke.js +2 -0
- package/dist/core/liveSmoke.js.map +1 -0
- package/dist/core/platformFactories.d.ts +2 -0
- package/dist/core/platformFactories.d.ts.map +1 -0
- package/dist/core/platformFactories.js +2 -0
- package/dist/core/platformFactories.js.map +1 -0
- package/dist/core/platformMetadata.d.ts +2 -0
- package/dist/core/platformMetadata.d.ts.map +1 -0
- package/dist/core/platformMetadata.js +2 -0
- package/dist/core/platformMetadata.js.map +1 -0
- package/dist/core/schemas.d.ts +365 -0
- package/dist/core/schemas.d.ts.map +1 -0
- package/dist/core/schemas.js +256 -0
- package/dist/core/schemas.js.map +1 -0
- package/dist/core/searchers.d.ts +41 -0
- package/dist/core/searchers.d.ts.map +1 -0
- package/dist/core/searchers.js +24 -0
- package/dist/core/searchers.js.map +1 -0
- package/dist/core/textReports.d.ts +2 -0
- package/dist/core/textReports.d.ts.map +1 -0
- package/dist/core/textReports.js +2 -0
- package/dist/core/textReports.js.map +1 -0
- package/dist/core/tools.d.ts +7 -0
- package/dist/core/tools.d.ts.map +1 -0
- package/dist/core/tools.js +489 -0
- package/dist/core/tools.js.map +1 -0
- package/dist/infrastructure/cache/RequestCache.d.ts +26 -0
- package/dist/infrastructure/cache/RequestCache.d.ts.map +1 -0
- package/dist/infrastructure/cache/RequestCache.js +66 -0
- package/dist/infrastructure/cache/RequestCache.js.map +1 -0
- package/dist/infrastructure/http/ErrorHandler.d.ts +101 -0
- package/dist/infrastructure/http/ErrorHandler.d.ts.map +1 -0
- package/dist/infrastructure/http/ErrorHandler.js +278 -0
- package/dist/infrastructure/http/ErrorHandler.js.map +1 -0
- package/dist/infrastructure/http/HttpClient.d.ts +31 -0
- package/dist/infrastructure/http/HttpClient.d.ts.map +1 -0
- package/dist/infrastructure/http/HttpClient.js +52 -0
- package/dist/infrastructure/http/HttpClient.js.map +1 -0
- package/dist/infrastructure/pdf/PDFExtractor.d.ts +34 -0
- package/dist/infrastructure/pdf/PDFExtractor.d.ts.map +1 -0
- package/dist/infrastructure/pdf/PDFExtractor.js +130 -0
- package/dist/infrastructure/pdf/PDFExtractor.js.map +1 -0
- package/dist/infrastructure/pdf/PdfDownload.d.ts +16 -0
- package/dist/infrastructure/pdf/PdfDownload.d.ts.map +1 -0
- package/dist/infrastructure/pdf/PdfDownload.js +242 -0
- package/dist/infrastructure/pdf/PdfDownload.js.map +1 -0
- package/dist/infrastructure/pdf/PdfFirstPage.cjs +70 -0
- package/dist/infrastructure/pdf/PdfFirstPage.cjs.map +1 -0
- package/dist/infrastructure/pdf/PdfFirstPage.d.cts +16 -0
- package/dist/infrastructure/pdf/PdfFirstPage.d.cts.map +1 -0
- package/dist/infrastructure/pdf/PdfIdentity.d.ts +29 -0
- package/dist/infrastructure/pdf/PdfIdentity.d.ts.map +1 -0
- package/dist/infrastructure/pdf/PdfIdentity.js +283 -0
- package/dist/infrastructure/pdf/PdfIdentity.js.map +1 -0
- package/dist/infrastructure/rate-limit/RateLimiter.d.ts +50 -0
- package/dist/infrastructure/rate-limit/RateLimiter.d.ts.map +1 -0
- package/dist/infrastructure/rate-limit/RateLimiter.js +128 -0
- package/dist/infrastructure/rate-limit/RateLimiter.js.map +1 -0
- package/dist/infrastructure/runlog/RunLedger.d.ts +61 -0
- package/dist/infrastructure/runlog/RunLedger.d.ts.map +1 -0
- package/dist/infrastructure/runlog/RunLedger.js +284 -0
- package/dist/infrastructure/runlog/RunLedger.js.map +1 -0
- package/dist/infrastructure/runlog/failureClass.d.ts +26 -0
- package/dist/infrastructure/runlog/failureClass.d.ts.map +1 -0
- package/dist/infrastructure/runlog/failureClass.js +125 -0
- package/dist/infrastructure/runlog/failureClass.js.map +1 -0
- package/dist/infrastructure/security/SecurityUtils.d.ts +80 -0
- package/dist/infrastructure/security/SecurityUtils.d.ts.map +1 -0
- package/dist/infrastructure/security/SecurityUtils.js +357 -0
- package/dist/infrastructure/security/SecurityUtils.js.map +1 -0
- package/dist/management/capability-profile/carsiReadiness.d.ts +10 -0
- package/dist/management/capability-profile/carsiReadiness.d.ts.map +1 -0
- package/dist/management/capability-profile/carsiReadiness.js +13 -0
- package/dist/management/capability-profile/carsiReadiness.js.map +1 -0
- package/dist/management/capability-profile/index.d.ts +26 -0
- package/dist/management/capability-profile/index.d.ts.map +1 -0
- package/dist/management/capability-profile/index.js +183 -0
- package/dist/management/capability-profile/index.js.map +1 -0
- package/dist/management/carsi/host.d.ts +4 -0
- package/dist/management/carsi/host.d.ts.map +1 -0
- package/dist/management/carsi/host.js +65 -0
- package/dist/management/carsi/host.js.map +1 -0
- package/dist/management/carsi/installer.d.ts +146 -0
- package/dist/management/carsi/installer.d.ts.map +1 -0
- package/dist/management/carsi/installer.js +527 -0
- package/dist/management/carsi/installer.js.map +1 -0
- package/dist/management/carsi/runtime.d.ts +31 -0
- package/dist/management/carsi/runtime.d.ts.map +1 -0
- package/dist/management/carsi/runtime.js +67 -0
- package/dist/management/carsi/runtime.js.map +1 -0
- package/dist/management/config/index.d.ts +2 -0
- package/dist/management/config/index.d.ts.map +1 -0
- package/dist/management/config/index.js +2 -0
- package/dist/management/config/index.js.map +1 -0
- package/dist/management/diagnostics/index.d.ts +43 -0
- package/dist/management/diagnostics/index.d.ts.map +1 -0
- package/dist/management/diagnostics/index.js +610 -0
- package/dist/management/diagnostics/index.js.map +1 -0
- package/dist/management/failures/index.d.ts +47 -0
- package/dist/management/failures/index.d.ts.map +1 -0
- package/dist/management/failures/index.js +134 -0
- package/dist/management/failures/index.js.map +1 -0
- package/dist/management/reports/index.d.ts +32 -0
- package/dist/management/reports/index.d.ts.map +1 -0
- package/dist/management/reports/index.js +123 -0
- package/dist/management/reports/index.js.map +1 -0
- package/dist/management/skills/autoSync.d.ts +41 -0
- package/dist/management/skills/autoSync.d.ts.map +1 -0
- package/dist/management/skills/autoSync.js +142 -0
- package/dist/management/skills/autoSync.js.map +1 -0
- package/dist/management/skills/index.d.ts +3 -0
- package/dist/management/skills/index.d.ts.map +1 -0
- package/dist/management/skills/index.js +3 -0
- package/dist/management/skills/index.js.map +1 -0
- package/dist/management/smoke/index.d.ts +44 -0
- package/dist/management/smoke/index.d.ts.map +1 -0
- package/dist/management/smoke/index.js +254 -0
- package/dist/management/smoke/index.js.map +1 -0
- package/dist/models/Paper.d.ts +64 -0
- package/dist/models/Paper.d.ts.map +1 -0
- package/dist/models/Paper.js +70 -0
- package/dist/models/Paper.js.map +1 -0
- package/dist/platforms/ACMSearcher.d.ts +16 -0
- package/dist/platforms/ACMSearcher.d.ts.map +1 -0
- package/dist/platforms/ACMSearcher.js +125 -0
- package/dist/platforms/ACMSearcher.js.map +1 -0
- package/dist/platforms/BioRxivSearcher.d.ts +47 -0
- package/dist/platforms/BioRxivSearcher.d.ts.map +1 -0
- package/dist/platforms/BioRxivSearcher.js +215 -0
- package/dist/platforms/BioRxivSearcher.js.map +1 -0
- package/dist/platforms/CORESearcher.d.ts +17 -0
- package/dist/platforms/CORESearcher.d.ts.map +1 -0
- package/dist/platforms/CORESearcher.js +185 -0
- package/dist/platforms/CORESearcher.js.map +1 -0
- package/dist/platforms/CrossrefSearcher.d.ts +34 -0
- package/dist/platforms/CrossrefSearcher.d.ts.map +1 -0
- package/dist/platforms/CrossrefSearcher.js +339 -0
- package/dist/platforms/CrossrefSearcher.js.map +1 -0
- package/dist/platforms/DBLPSearcher.d.ts +16 -0
- package/dist/platforms/DBLPSearcher.d.ts.map +1 -0
- package/dist/platforms/DBLPSearcher.js +116 -0
- package/dist/platforms/DBLPSearcher.js.map +1 -0
- package/dist/platforms/EuropePMCSearcher.d.ts +20 -0
- package/dist/platforms/EuropePMCSearcher.d.ts.map +1 -0
- package/dist/platforms/EuropePMCSearcher.js +173 -0
- package/dist/platforms/EuropePMCSearcher.js.map +1 -0
- package/dist/platforms/GoogleScholarSearcher.d.ts +142 -0
- package/dist/platforms/GoogleScholarSearcher.d.ts.map +1 -0
- package/dist/platforms/GoogleScholarSearcher.js +648 -0
- package/dist/platforms/GoogleScholarSearcher.js.map +1 -0
- package/dist/platforms/IACRSearcher.d.ts +51 -0
- package/dist/platforms/IACRSearcher.d.ts.map +1 -0
- package/dist/platforms/IACRSearcher.js +339 -0
- package/dist/platforms/IACRSearcher.js.map +1 -0
- package/dist/platforms/IEEESearcher.d.ts +15 -0
- package/dist/platforms/IEEESearcher.d.ts.map +1 -0
- package/dist/platforms/IEEESearcher.js +123 -0
- package/dist/platforms/IEEESearcher.js.map +1 -0
- package/dist/platforms/OpenAIRESearcher.d.ts +22 -0
- package/dist/platforms/OpenAIRESearcher.d.ts.map +1 -0
- package/dist/platforms/OpenAIRESearcher.js +235 -0
- package/dist/platforms/OpenAIRESearcher.js.map +1 -0
- package/dist/platforms/OpenAlexSearcher.d.ts +14 -0
- package/dist/platforms/OpenAlexSearcher.d.ts.map +1 -0
- package/dist/platforms/OpenAlexSearcher.js +114 -0
- package/dist/platforms/OpenAlexSearcher.js.map +1 -0
- package/dist/platforms/OpenReviewSearcher.d.ts +19 -0
- package/dist/platforms/OpenReviewSearcher.d.ts.map +1 -0
- package/dist/platforms/OpenReviewSearcher.js +141 -0
- package/dist/platforms/OpenReviewSearcher.js.map +1 -0
- package/dist/platforms/PMCSearcher.d.ts +20 -0
- package/dist/platforms/PMCSearcher.d.ts.map +1 -0
- package/dist/platforms/PMCSearcher.js +177 -0
- package/dist/platforms/PMCSearcher.js.map +1 -0
- package/dist/platforms/PaperSource.d.ts +149 -0
- package/dist/platforms/PaperSource.d.ts.map +1 -0
- package/dist/platforms/PaperSource.js +125 -0
- package/dist/platforms/PaperSource.js.map +1 -0
- package/dist/platforms/PubMedSearcher.d.ts +104 -0
- package/dist/platforms/PubMedSearcher.d.ts.map +1 -0
- package/dist/platforms/PubMedSearcher.js +430 -0
- package/dist/platforms/PubMedSearcher.js.map +1 -0
- package/dist/platforms/SciHubSearcher.d.ts +66 -0
- package/dist/platforms/SciHubSearcher.d.ts.map +1 -0
- package/dist/platforms/SciHubSearcher.js +398 -0
- package/dist/platforms/SciHubSearcher.js.map +1 -0
- package/dist/platforms/ScienceDirectSearcher.d.ts +42 -0
- package/dist/platforms/ScienceDirectSearcher.d.ts.map +1 -0
- package/dist/platforms/ScienceDirectSearcher.js +334 -0
- package/dist/platforms/ScienceDirectSearcher.js.map +1 -0
- package/dist/platforms/ScopusSearcher.d.ts +43 -0
- package/dist/platforms/ScopusSearcher.d.ts.map +1 -0
- package/dist/platforms/ScopusSearcher.js +377 -0
- package/dist/platforms/ScopusSearcher.js.map +1 -0
- package/dist/platforms/SemanticScholarSearcher.d.ts +96 -0
- package/dist/platforms/SemanticScholarSearcher.d.ts.map +1 -0
- package/dist/platforms/SemanticScholarSearcher.js +419 -0
- package/dist/platforms/SemanticScholarSearcher.js.map +1 -0
- package/dist/platforms/SpringerSearcher.d.ts +54 -0
- package/dist/platforms/SpringerSearcher.d.ts.map +1 -0
- package/dist/platforms/SpringerSearcher.js +387 -0
- package/dist/platforms/SpringerSearcher.js.map +1 -0
- package/dist/platforms/USENIXSearcher.d.ts +14 -0
- package/dist/platforms/USENIXSearcher.d.ts.map +1 -0
- package/dist/platforms/USENIXSearcher.js +75 -0
- package/dist/platforms/USENIXSearcher.js.map +1 -0
- package/dist/platforms/UnpaywallSearcher.d.ts +29 -0
- package/dist/platforms/UnpaywallSearcher.d.ts.map +1 -0
- package/dist/platforms/UnpaywallSearcher.js +135 -0
- package/dist/platforms/UnpaywallSearcher.js.map +1 -0
- package/dist/platforms/WebOfScienceSearcher.d.ts +111 -0
- package/dist/platforms/WebOfScienceSearcher.d.ts.map +1 -0
- package/dist/platforms/WebOfScienceSearcher.js +500 -0
- package/dist/platforms/WebOfScienceSearcher.js.map +1 -0
- package/dist/registry/aliases.d.ts +2 -0
- package/dist/registry/aliases.d.ts.map +1 -0
- package/dist/registry/aliases.js +2 -0
- package/dist/registry/aliases.js.map +1 -0
- package/dist/registry/httpPolicies.d.ts +3 -0
- package/dist/registry/httpPolicies.d.ts.map +1 -0
- package/dist/registry/httpPolicies.js +13 -0
- package/dist/registry/httpPolicies.js.map +1 -0
- package/dist/registry/platformFactories.d.ts +8 -0
- package/dist/registry/platformFactories.d.ts.map +1 -0
- package/dist/registry/platformFactories.js +51 -0
- package/dist/registry/platformFactories.js.map +1 -0
- package/dist/registry/platformMetadata.d.ts +55 -0
- package/dist/registry/platformMetadata.d.ts.map +1 -0
- package/dist/registry/platformMetadata.js +438 -0
- package/dist/registry/platformMetadata.js.map +1 -0
- package/dist/services/CitationService.d.ts +3 -0
- package/dist/services/CitationService.d.ts.map +1 -0
- package/dist/services/CitationService.js +2 -0
- package/dist/services/CitationService.js.map +1 -0
- package/dist/services/JournalMetricsService.d.ts +3 -0
- package/dist/services/JournalMetricsService.d.ts.map +1 -0
- package/dist/services/JournalMetricsService.js +2 -0
- package/dist/services/JournalMetricsService.js.map +1 -0
- package/dist/services/MultiSourceSearchService.d.ts +3 -0
- package/dist/services/MultiSourceSearchService.d.ts.map +1 -0
- package/dist/services/MultiSourceSearchService.js +2 -0
- package/dist/services/MultiSourceSearchService.js.map +1 -0
- package/dist/services/OpenAccessFallbackService.d.ts +4 -0
- package/dist/services/OpenAccessFallbackService.d.ts.map +1 -0
- package/dist/services/OpenAccessFallbackService.js +2 -0
- package/dist/services/OpenAccessFallbackService.js.map +1 -0
- package/dist/skills/SkillInstaller.d.ts +110 -0
- package/dist/skills/SkillInstaller.d.ts.map +1 -0
- package/dist/skills/SkillInstaller.js +393 -0
- package/dist/skills/SkillInstaller.js.map +1 -0
- package/dist/utils/ErrorHandler.d.ts +3 -0
- package/dist/utils/ErrorHandler.d.ts.map +1 -0
- package/dist/utils/ErrorHandler.js +3 -0
- package/dist/utils/ErrorHandler.js.map +1 -0
- package/dist/utils/HttpClient.d.ts +2 -0
- package/dist/utils/HttpClient.d.ts.map +1 -0
- package/dist/utils/HttpClient.js +2 -0
- package/dist/utils/HttpClient.js.map +1 -0
- package/dist/utils/Logger.d.ts +6 -0
- package/dist/utils/Logger.d.ts.map +1 -0
- package/dist/utils/Logger.js +26 -0
- package/dist/utils/Logger.js.map +1 -0
- package/dist/utils/PDFExtractor.d.ts +3 -0
- package/dist/utils/PDFExtractor.d.ts.map +1 -0
- package/dist/utils/PDFExtractor.js +3 -0
- package/dist/utils/PDFExtractor.js.map +1 -0
- package/dist/utils/PdfDownload.d.ts +2 -0
- package/dist/utils/PdfDownload.d.ts.map +1 -0
- package/dist/utils/PdfDownload.js +2 -0
- package/dist/utils/PdfDownload.js.map +1 -0
- package/dist/utils/QuotaManager.d.ts +32 -0
- package/dist/utils/QuotaManager.d.ts.map +1 -0
- package/dist/utils/QuotaManager.js +95 -0
- package/dist/utils/QuotaManager.js.map +1 -0
- package/dist/utils/RateLimiter.d.ts +2 -0
- package/dist/utils/RateLimiter.d.ts.map +1 -0
- package/dist/utils/RateLimiter.js +2 -0
- package/dist/utils/RateLimiter.js.map +1 -0
- package/dist/utils/RequestCache.d.ts +3 -0
- package/dist/utils/RequestCache.d.ts.map +1 -0
- package/dist/utils/RequestCache.js +3 -0
- package/dist/utils/RequestCache.js.map +1 -0
- package/dist/utils/SecurityUtils.d.ts +3 -0
- package/dist/utils/SecurityUtils.d.ts.map +1 -0
- package/dist/utils/SecurityUtils.js +3 -0
- package/dist/utils/SecurityUtils.js.map +1 -0
- package/package.json +118 -4
- package/skills/paper-search/SKILL.md +151 -0
- package/skills/paper-search/references/browser-download.md +101 -0
- package/skills/paper-search/references/capability-routing.md +174 -0
- package/skills/paper-search/references/carsi-setup.md +69 -0
- package/skills/paper-search/references/cli-contract.md +200 -0
- package/skills/paper-search/references/management-layer.md +149 -0
- package/skills/paper-search/references/publisher-support.md +72 -0
- package/skills/paper-search/references/troubleshooting.md +15 -0
|
@@ -0,0 +1,514 @@
|
|
|
1
|
+
"""Publisher-owned PDF routing helpers.
|
|
2
|
+
|
|
3
|
+
This module keeps DOI/source-page PDF routes out of individual download
|
|
4
|
+
callers. It follows the same shape as provider catalogs: profile templates
|
|
5
|
+
first, then source URL derivation, then page-discovered links.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import html
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from urllib.parse import parse_qs, quote, unquote, urljoin, urlparse, urlunparse
|
|
14
|
+
|
|
15
|
+
from bs4 import BeautifulSoup
|
|
16
|
+
|
|
17
|
+
from ..publishers.profiles import PublisherProfile
|
|
18
|
+
|
|
19
|
+
PDF_LINK_TEXT_MARKERS = ("pdf", "download pdf", "full text pdf", "view pdf")
|
|
20
|
+
PDF_URL_MARKERS = (".pdf", "/pdf", "/epdf", "/pdfdirect", "/pdfft", "download=true")
|
|
21
|
+
PDF_JS_DEFAULT_URL_RE = re.compile(
|
|
22
|
+
r"PDFViewerApplicationOptions\.set\(\s*['\"]defaultUrl['\"]\s*,\s*['\"]([^'\"]+)['\"]",
|
|
23
|
+
flags=re.IGNORECASE,
|
|
24
|
+
)
|
|
25
|
+
PLOS_JOURNAL_PATHS = {
|
|
26
|
+
"pbio": "plosbiology",
|
|
27
|
+
"pcbi": "ploscompbiol",
|
|
28
|
+
"pclm": "climate",
|
|
29
|
+
"pdig": "digitalhealth",
|
|
30
|
+
"pgen": "plosgenetics",
|
|
31
|
+
"pgph": "globalpublichealth",
|
|
32
|
+
"pmed": "plosmedicine",
|
|
33
|
+
"pntd": "plosntds",
|
|
34
|
+
"pone": "plosone",
|
|
35
|
+
"ppat": "plospathogens",
|
|
36
|
+
"pstr": "sustainabilitytransformation",
|
|
37
|
+
"pwat": "water",
|
|
38
|
+
}
|
|
39
|
+
PLOS_DOI_JOURNAL_RE = re.compile(r"^10\.1371/journal\.([a-z0-9]+)\.", flags=re.IGNORECASE)
|
|
40
|
+
COPERNICUS_DOI_RE = re.compile(
|
|
41
|
+
r"^10\.5194/(?P<journal>[a-z0-9]+)-(?P<volume>\d+)-(?P<page>.+)-(?P<year>\d{4})$",
|
|
42
|
+
flags=re.IGNORECASE,
|
|
43
|
+
)
|
|
44
|
+
MDPI_ISSN_JOURNAL_CODES = {
|
|
45
|
+
"1424-8220": "s",
|
|
46
|
+
"1660-4601": "ijerph",
|
|
47
|
+
"1996-1073": "en",
|
|
48
|
+
"2071-1050": "su",
|
|
49
|
+
"2072-4292": "rs",
|
|
50
|
+
"2073-4441": "w",
|
|
51
|
+
"2077-0375": "membranes",
|
|
52
|
+
"2227-7390": "math",
|
|
53
|
+
"2304-8158": "foods",
|
|
54
|
+
}
|
|
55
|
+
MDPI_JOURNAL_CODE_ISSNS = {
|
|
56
|
+
journal_code: issn
|
|
57
|
+
for issn, journal_code in MDPI_ISSN_JOURNAL_CODES.items()
|
|
58
|
+
}
|
|
59
|
+
MDPI_NUMERIC_PATH_RE = re.compile(r"/[0-9]{4}-[0-9Xx]{3,4}/[0-9]+/[0-9]+/[0-9]+$")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_ELSEVIER_FORMATTED_PII = re.compile(r"(S\d{4}-?\d{3}[\dX])(?:\(|%28)(\d{2})(?:\)|%29)(\d{5})-?([\dX])", re.I)
|
|
63
|
+
_PII = r"S\d{4}-?\d{3}[\dX]\(\d{2}\)\d{5}-?[\dX]"
|
|
64
|
+
_ELSEVIER_JOURNAL_ARTICLE = re.compile(
|
|
65
|
+
rf"^/(?:journals/[^/]+/|(?:[^/]+/){{1,2}})?(?:article/(?:PII)?(?P<a>{_PII})(?:/(?P<av>fulltext|abstract|pdf|ppt))?"
|
|
66
|
+
rf"|(?P<cv>fulltext|abstract|pdf)/(?P<c>{_PII})(?:\.pdf)?)/?$", re.I)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def elsevier_journal_article(url: str) -> tuple[str, str] | None:
|
|
70
|
+
"""``(formatted PII, view)`` for an article page on an Elsevier journal's own site.
|
|
71
|
+
|
|
72
|
+
Cell Press, Gastroenterology, Osteoarthritis and Cartilage, The Lancet and
|
|
73
|
+
other Elsevier journals are also served from their own hosts, e.g.
|
|
74
|
+
``https://www.gastrojournal.org/article/S0016-5085(24)00291-9/fulltext``,
|
|
75
|
+
``https://www.cell.com/cell-metabolism/fulltext/S1550-4131(25)00486-3``,
|
|
76
|
+
``https://www.cell.com/molecular-therapy-family/molecular-therapy/fulltext/S1525-0016(26)00767-7`` or
|
|
77
|
+
``https://www.thelancet.com/journals/lanrhe/article/PIIS2665-9913(25)00091-8/fulltext``.
|
|
78
|
+
"""
|
|
79
|
+
parsed = urlparse(url or "")
|
|
80
|
+
host = (parsed.hostname or "").lower()
|
|
81
|
+
if parsed.scheme != "https" or host == "sciencedirect.com" or host.endswith(".sciencedirect.com"):
|
|
82
|
+
return None
|
|
83
|
+
match = _ELSEVIER_JOURNAL_ARTICLE.match(unquote(parsed.path))
|
|
84
|
+
if not match:
|
|
85
|
+
return None
|
|
86
|
+
return match["a"] or match["c"], (match["av"] or match["cv"] or "").lower()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def extract_elsevier_pii(url: str) -> str:
|
|
90
|
+
"""Extract an Elsevier PII from article, retrieve, query, or signed asset URLs.
|
|
91
|
+
|
|
92
|
+
The formatted form used by journal sites (``S1063-4584(22)00770-1``) is
|
|
93
|
+
returned in ScienceDirect's compact form (``S1063458422007701``).
|
|
94
|
+
"""
|
|
95
|
+
formatted = _ELSEVIER_FORMATTED_PII.search(url or "")
|
|
96
|
+
if formatted:
|
|
97
|
+
return "".join(formatted.groups()).replace("-", "").upper()
|
|
98
|
+
for pattern in (
|
|
99
|
+
r"/pii/([A-Z0-9]+)",
|
|
100
|
+
r"/retrieve/pii/([A-Z0-9]+)",
|
|
101
|
+
r"[?&]pii=([A-Z0-9]+)",
|
|
102
|
+
r"1-s2\.0-([A-Z0-9]+)",
|
|
103
|
+
):
|
|
104
|
+
match = re.search(pattern, url, flags=re.IGNORECASE)
|
|
105
|
+
if match:
|
|
106
|
+
return match.group(1)
|
|
107
|
+
return ""
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def extract_ieee_article_number(url: str) -> str:
|
|
111
|
+
"""Extract an IEEE Xplore article number from document or query URLs."""
|
|
112
|
+
for pattern in (
|
|
113
|
+
r"/document/(\d+)",
|
|
114
|
+
r"[?&]arnumber=(\d+)",
|
|
115
|
+
):
|
|
116
|
+
match = re.search(pattern, url, flags=re.IGNORECASE)
|
|
117
|
+
if match:
|
|
118
|
+
return match.group(1)
|
|
119
|
+
return ""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def discover_pdf_candidates_from_html(html_text: str, source_url: str) -> list[str]:
|
|
123
|
+
"""Extract likely PDF URLs from publisher HTML without committing to a provider."""
|
|
124
|
+
candidates: list[str] = []
|
|
125
|
+
soup = BeautifulSoup(html_text, "lxml")
|
|
126
|
+
|
|
127
|
+
for meta in soup.find_all("meta"):
|
|
128
|
+
key = _clean(meta.get("name") or meta.get("property") or meta.get("itemprop")).lower()
|
|
129
|
+
content = _clean(meta.get("content"))
|
|
130
|
+
if content and ("citation_pdf_url" in key or key.endswith("pdf_url") or key == "pdf_url"):
|
|
131
|
+
_append_candidate(candidates, content, source_url=source_url)
|
|
132
|
+
_append_query_pdf_candidates(candidates, content, source_url=source_url)
|
|
133
|
+
|
|
134
|
+
for node in soup.find_all(["a", "link", "iframe", "embed", "object"]):
|
|
135
|
+
target = _clean(node.get("href") or node.get("src") or node.get("data"))
|
|
136
|
+
if not target:
|
|
137
|
+
continue
|
|
138
|
+
label = _clean(
|
|
139
|
+
" ".join(
|
|
140
|
+
filter(
|
|
141
|
+
None,
|
|
142
|
+
[node.get_text(" ", strip=True), node.get("title"), node.get("aria-label")],
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
).lower()
|
|
146
|
+
content_type = _clean(node.get("type")).lower()
|
|
147
|
+
lower_target = target.lower()
|
|
148
|
+
if (
|
|
149
|
+
any(marker in lower_target for marker in PDF_URL_MARKERS)
|
|
150
|
+
or any(marker in label for marker in PDF_LINK_TEXT_MARKERS)
|
|
151
|
+
or "pdf" in content_type
|
|
152
|
+
):
|
|
153
|
+
_append_candidate(candidates, target, source_url=source_url)
|
|
154
|
+
_append_query_pdf_candidates(candidates, target, source_url=source_url)
|
|
155
|
+
|
|
156
|
+
for script in soup.find_all("script"):
|
|
157
|
+
text = script.string if script.string is not None else script.get_text(" ", strip=False)
|
|
158
|
+
for match in PDF_JS_DEFAULT_URL_RE.finditer(str(text or "")):
|
|
159
|
+
target = html.unescape(match.group(1))
|
|
160
|
+
if any(marker in target.lower() for marker in PDF_URL_MARKERS):
|
|
161
|
+
_append_candidate(candidates, target, source_url=source_url)
|
|
162
|
+
_append_query_pdf_candidates(candidates, target, source_url=source_url)
|
|
163
|
+
|
|
164
|
+
return candidates
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def build_pdf_candidates(
|
|
168
|
+
profile: PublisherProfile,
|
|
169
|
+
doi: str,
|
|
170
|
+
*,
|
|
171
|
+
source_url: str = "",
|
|
172
|
+
discovered_urls: Sequence[str] = (),
|
|
173
|
+
) -> list[str]:
|
|
174
|
+
"""Build ordered, de-duplicated official PDF candidates for a publisher profile."""
|
|
175
|
+
adapter = _adapter_class(profile)
|
|
176
|
+
candidates: list[str] = []
|
|
177
|
+
for url in adapter.profile_pdf_candidates(doi, source_url):
|
|
178
|
+
_append_candidate(candidates, url)
|
|
179
|
+
|
|
180
|
+
_append_source_path_candidates(candidates, profile, doi, source_url)
|
|
181
|
+
|
|
182
|
+
for url in profile.pdf_urls(doi):
|
|
183
|
+
_append_candidate(candidates, url)
|
|
184
|
+
|
|
185
|
+
for url in discovered_urls:
|
|
186
|
+
if not _clean(url):
|
|
187
|
+
continue
|
|
188
|
+
if is_supplementary_url(profile, url):
|
|
189
|
+
continue
|
|
190
|
+
if not is_pdf_candidate_url(profile, url):
|
|
191
|
+
continue
|
|
192
|
+
if not belongs_to_current_article(profile, url, doi=doi, source_url=source_url):
|
|
193
|
+
continue
|
|
194
|
+
_append_candidate(candidates, url, source_url=source_url)
|
|
195
|
+
|
|
196
|
+
return [url for url in candidates if adapter.keeps_pdf_candidate(profile, url, doi)]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _adapter_class(profile: PublisherProfile):
|
|
200
|
+
"""The publisher's adapter class, for its profile-level rules. Imported here:
|
|
201
|
+
adapter modules import this one."""
|
|
202
|
+
from ..adapters.base import adapter_class_for
|
|
203
|
+
|
|
204
|
+
return adapter_class_for(profile)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def is_pdf_candidate_url(profile: PublisherProfile, url: str) -> bool:
|
|
208
|
+
verdict = _adapter_class(profile).profile_pdf_candidate_url(url)
|
|
209
|
+
if verdict is not None:
|
|
210
|
+
return verdict
|
|
211
|
+
lower = url.lower()
|
|
212
|
+
return lower.endswith(".pdf") or any(marker.lower() in lower for marker in profile.pdf_url_markers)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def is_thieme_article_pdf_url(url: str, doi: str = "") -> bool:
|
|
216
|
+
"""Exclude account/reset URLs that merely carry a PDF return address."""
|
|
217
|
+
parsed = urlparse(url.strip())
|
|
218
|
+
match = re.fullmatch(r"/products/ejournals/pdf/(10\.1055/[^/]+)\.pdf",
|
|
219
|
+
unquote(parsed.path).lower())
|
|
220
|
+
return (parsed.scheme == "https" and not parsed.username and not parsed.password
|
|
221
|
+
and parsed.netloc.lower() in {"www.thieme-connect.de", "thieme-connect.de",
|
|
222
|
+
"www.thieme-connect.com", "thieme-connect.com"}
|
|
223
|
+
and match is not None and (not doi or match[1] == doi.strip().lower()))
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
# Supplementary-file paths used the same way across publishers (ACS, SAGE,
|
|
227
|
+
# T&F, JAMA, LWW "/suppl/", Wiley "/supinfo/" and "downloadSupplement"),
|
|
228
|
+
# plus site CMS uploads (user guides, brochures).
|
|
229
|
+
_UNIVERSAL_SUPPLEMENT_MARKERS = ("/suppl/", "/supinfo/", "downloadsupplement", "/wp-content/")
|
|
230
|
+
# A supplement issue in the path ("/article-pdf/28/Supplement_11/suag119.023/...",
|
|
231
|
+
# a journal's supplement issue on Oxford Academic, 2026-11-01) names the issue,
|
|
232
|
+
# not supplementary material: it is left out before the publisher's markers.
|
|
233
|
+
_ISSUE_SUPPLEMENT = re.compile(r"/supplement[_-]?\d+/")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def is_supplementary_url(profile: PublisherProfile, url: str) -> bool:
|
|
237
|
+
lower = url.lower()
|
|
238
|
+
if any(marker in unquote(lower) for marker in _UNIVERSAL_SUPPLEMENT_MARKERS):
|
|
239
|
+
return True
|
|
240
|
+
adapter = _adapter_class(profile)
|
|
241
|
+
if adapter.unquotes_supplement_urls:
|
|
242
|
+
lower = unquote(lower)
|
|
243
|
+
if adapter.names_supplement(lower):
|
|
244
|
+
return True
|
|
245
|
+
lower = _ISSUE_SUPPLEMENT.sub("/", lower)
|
|
246
|
+
return any(marker.lower() in lower for marker in profile.supplementary_url_markers)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def is_wiley_article_pdf_url(url: str, doi: str = "") -> bool:
|
|
250
|
+
"""Only Wiley's article PDF endpoints, optionally bound to the exact DOI."""
|
|
251
|
+
parsed = urlparse(url)
|
|
252
|
+
host = (parsed.hostname or "").lower()
|
|
253
|
+
domain = "onlinelibrary.wiley.com"
|
|
254
|
+
if parsed.scheme not in {"http", "https"} or not (host == domain or host.endswith("." + domain)):
|
|
255
|
+
return False
|
|
256
|
+
path = unquote(parsed.path).lower().rstrip("/")
|
|
257
|
+
match = re.fullmatch(r"/doi/(?:pdfdirect|pdf|epdf)/(10\.\d{4,9}/.+)", path)
|
|
258
|
+
return bool(match and (not doi or match.group(1) == doi.strip().lower().rstrip("/")))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def is_springer_article_pdf_url(url: str, doi: str = "") -> bool:
|
|
262
|
+
"""Match Springer/Nature article endpoints, excluding their media attachments."""
|
|
263
|
+
parsed = urlparse(url)
|
|
264
|
+
host = (parsed.hostname or "").lower()
|
|
265
|
+
path = unquote(parsed.path).lower()
|
|
266
|
+
expected = doi.strip().lower()
|
|
267
|
+
if parsed.scheme not in {"http", "https"}:
|
|
268
|
+
return False
|
|
269
|
+
# Open-access articles offer "<doi>_reference.pdf" (data-article-pdf="true")
|
|
270
|
+
# as their Download PDF; the plain "<doi>.pdf" is refused there.
|
|
271
|
+
if host == "link.springer.com":
|
|
272
|
+
match = re.fullmatch(r"/content/pdf/(10\.\d{4,9}/.+?)(?:_reference)?\.pdf", path)
|
|
273
|
+
return bool(match and (not expected or match.group(1) == expected))
|
|
274
|
+
if host == "nature.com" or host.endswith(".nature.com"):
|
|
275
|
+
match = re.fullmatch(r"/articles/([^/]+?)(?:_reference)?\.pdf", path)
|
|
276
|
+
return bool(match and (not expected or (expected.startswith("10.1038/")
|
|
277
|
+
and match.group(1) == expected.split("/", 1)[1])))
|
|
278
|
+
return False
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def belongs_to_current_article(profile: PublisherProfile, url: str, *, doi: str, source_url: str = "") -> bool:
|
|
282
|
+
verdict = _adapter_class(profile).belongs_to_article(url, doi, source_url)
|
|
283
|
+
return True if verdict is None else verdict
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _append_source_path_candidates(
|
|
287
|
+
candidates: list[str],
|
|
288
|
+
profile: PublisherProfile,
|
|
289
|
+
doi: str,
|
|
290
|
+
source_url: str,
|
|
291
|
+
) -> None:
|
|
292
|
+
normalized = _clean(source_url)
|
|
293
|
+
if not normalized:
|
|
294
|
+
return
|
|
295
|
+
parsed = urlparse(normalized)
|
|
296
|
+
host = (parsed.hostname or "").lower()
|
|
297
|
+
source_path = parsed.path.rstrip("/")
|
|
298
|
+
if parsed.scheme not in {"http", "https"} or not host or not source_path:
|
|
299
|
+
return
|
|
300
|
+
for template in profile.pdf_source_path_templates:
|
|
301
|
+
domain = template.domain.lower()
|
|
302
|
+
if host != domain and not host.endswith(f".{domain}"):
|
|
303
|
+
continue
|
|
304
|
+
path_prefix = _clean(template.path_prefix)
|
|
305
|
+
if path_prefix and not source_path.startswith(path_prefix):
|
|
306
|
+
continue
|
|
307
|
+
candidate_path = _format_template(
|
|
308
|
+
template.path_template,
|
|
309
|
+
doi=doi,
|
|
310
|
+
source_path=source_path,
|
|
311
|
+
source_path_after_prefix=source_path[len(path_prefix):].lstrip("/") if path_prefix else source_path.lstrip("/"),
|
|
312
|
+
)
|
|
313
|
+
if not candidate_path:
|
|
314
|
+
continue
|
|
315
|
+
parsed_candidate = urlparse(candidate_path)
|
|
316
|
+
if parsed_candidate.scheme in {"http", "https"}:
|
|
317
|
+
_append_candidate(candidates, candidate_path)
|
|
318
|
+
continue
|
|
319
|
+
if not candidate_path.startswith("/"):
|
|
320
|
+
candidate_path = f"/{candidate_path}"
|
|
321
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, candidate_path, "", "", "")))
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _append_aps_candidates(candidates: list[str], source_url: str) -> None:
|
|
325
|
+
normalized = _clean(source_url)
|
|
326
|
+
if not normalized:
|
|
327
|
+
return
|
|
328
|
+
parsed = urlparse(normalized)
|
|
329
|
+
host = (parsed.hostname or "").lower()
|
|
330
|
+
source_path = parsed.path.rstrip("/")
|
|
331
|
+
if parsed.scheme not in {"http", "https"} or not host or not source_path:
|
|
332
|
+
return
|
|
333
|
+
if host.endswith("link.aps.org") and source_path.startswith("/doi/"):
|
|
334
|
+
pdf_path = source_path.replace("/doi/", "/pdf/", 1)
|
|
335
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
336
|
+
return
|
|
337
|
+
if not host.endswith("journals.aps.org"):
|
|
338
|
+
return
|
|
339
|
+
if "/abstract/" in source_path:
|
|
340
|
+
pdf_path = source_path.replace("/abstract/", "/pdf/", 1)
|
|
341
|
+
elif source_path.endswith("/pdf"):
|
|
342
|
+
pdf_path = source_path
|
|
343
|
+
else:
|
|
344
|
+
return
|
|
345
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _append_ams_candidates(candidates: list[str], source_url: str) -> None:
|
|
349
|
+
normalized = _clean(source_url)
|
|
350
|
+
if not normalized:
|
|
351
|
+
return
|
|
352
|
+
parsed = urlparse(normalized)
|
|
353
|
+
host = (parsed.hostname or "").lower()
|
|
354
|
+
source_path = parsed.path.rstrip("/")
|
|
355
|
+
if parsed.scheme not in {"http", "https"} or not host.endswith("journals.ametsoc.org"):
|
|
356
|
+
return
|
|
357
|
+
if not source_path.startswith("/view/") or not source_path.lower().endswith(".xml"):
|
|
358
|
+
return
|
|
359
|
+
pdf_path = f"/downloadpdf{source_path[:-4]}.pdf"
|
|
360
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _append_plos_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
364
|
+
journal_path = _plos_journal_path_from_source(source_url) or _plos_journal_path_from_doi(doi)
|
|
365
|
+
if not journal_path:
|
|
366
|
+
return
|
|
367
|
+
_append_candidate(
|
|
368
|
+
candidates,
|
|
369
|
+
f"https://journals.plos.org/{journal_path}/article/file?id={doi}&type=printable",
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _plos_journal_path_from_source(source_url: str) -> str:
|
|
374
|
+
parsed = urlparse(_clean(source_url))
|
|
375
|
+
host = (parsed.hostname or "").lower()
|
|
376
|
+
if host != "journals.plos.org":
|
|
377
|
+
return ""
|
|
378
|
+
parts = [part for part in parsed.path.split("/") if part]
|
|
379
|
+
if not parts or parts[0] == "article":
|
|
380
|
+
return ""
|
|
381
|
+
return parts[0]
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _plos_journal_path_from_doi(doi: str) -> str:
|
|
385
|
+
match = PLOS_DOI_JOURNAL_RE.match(doi.strip())
|
|
386
|
+
if not match:
|
|
387
|
+
return ""
|
|
388
|
+
return PLOS_JOURNAL_PATHS.get(match.group(1).lower(), "")
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _append_copernicus_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
392
|
+
match = COPERNICUS_DOI_RE.match(doi.strip().lower())
|
|
393
|
+
if not match:
|
|
394
|
+
return
|
|
395
|
+
parsed = urlparse(_clean(source_url))
|
|
396
|
+
host = (parsed.hostname or "").lower()
|
|
397
|
+
if not host.endswith("copernicus.org"):
|
|
398
|
+
host = f"{match.group('journal').lower()}.copernicus.org"
|
|
399
|
+
suffix = doi.split("/", 1)[-1].lower()
|
|
400
|
+
path = f"/articles/{match.group('volume')}/{match.group('page')}/{match.group('year')}/{suffix}.pdf"
|
|
401
|
+
_append_candidate(candidates, f"https://{host}{path}")
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _append_mdpi_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
405
|
+
source_pdf = _mdpi_pdf_url_from_landing_url(source_url)
|
|
406
|
+
if source_pdf:
|
|
407
|
+
_append_candidate(candidates, source_pdf)
|
|
408
|
+
derived_landing = _mdpi_landing_url_from_doi(doi)
|
|
409
|
+
derived_pdf = _mdpi_pdf_url_from_landing_url(derived_landing)
|
|
410
|
+
if derived_pdf:
|
|
411
|
+
_append_candidate(candidates, derived_pdf)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _mdpi_pdf_url_from_landing_url(url: str | None) -> str:
|
|
415
|
+
candidate = _clean(url)
|
|
416
|
+
if not candidate:
|
|
417
|
+
return ""
|
|
418
|
+
parsed = urlparse(candidate)
|
|
419
|
+
host = (parsed.hostname or "").lower()
|
|
420
|
+
if host not in {"www.mdpi.com", "mdpi.com"}:
|
|
421
|
+
return ""
|
|
422
|
+
path = parsed.path.rstrip("/")
|
|
423
|
+
if not path:
|
|
424
|
+
return ""
|
|
425
|
+
if path.endswith("/pdf") or "/pdf/" in path:
|
|
426
|
+
return urlunparse(parsed._replace(path=path, query=""))
|
|
427
|
+
if not MDPI_NUMERIC_PATH_RE.fullmatch(path):
|
|
428
|
+
return ""
|
|
429
|
+
return urlunparse(parsed._replace(path=f"{path}/pdf", query=""))
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _mdpi_landing_url_from_doi(doi: str) -> str:
|
|
433
|
+
normalized = doi.strip().lower()
|
|
434
|
+
if not normalized.startswith("10.3390/") or "/" not in normalized:
|
|
435
|
+
return ""
|
|
436
|
+
suffix = normalized.split("/", 1)[1]
|
|
437
|
+
for journal_code in sorted(MDPI_JOURNAL_CODE_ISSNS, key=len, reverse=True):
|
|
438
|
+
if not suffix.startswith(journal_code):
|
|
439
|
+
continue
|
|
440
|
+
encoded = suffix[len(journal_code):]
|
|
441
|
+
if len(encoded) < 7 or not encoded.isdigit():
|
|
442
|
+
continue
|
|
443
|
+
volume = encoded[:-6]
|
|
444
|
+
issue = encoded[-6:-4]
|
|
445
|
+
article_number = encoded[-4:]
|
|
446
|
+
if not volume:
|
|
447
|
+
continue
|
|
448
|
+
return (
|
|
449
|
+
f"https://www.mdpi.com/{MDPI_JOURNAL_CODE_ISSNS[journal_code]}/"
|
|
450
|
+
f"{int(volume)}/{int(issue)}/{int(article_number)}"
|
|
451
|
+
)
|
|
452
|
+
return ""
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _append_oxford_candidates(candidates: list[str], doi: str, source_url: str) -> None:
|
|
456
|
+
parsed = urlparse(_clean(source_url))
|
|
457
|
+
host = (parsed.hostname or "").lower()
|
|
458
|
+
source_path = parsed.path.rstrip("/")
|
|
459
|
+
if parsed.scheme not in {"http", "https"} or not host.endswith("academic.oup.com"):
|
|
460
|
+
return
|
|
461
|
+
match = re.match(r"^/(?P<journal>[^/]+)/article/(?P<rest>.+)$", source_path, flags=re.IGNORECASE)
|
|
462
|
+
if not match:
|
|
463
|
+
return
|
|
464
|
+
suffix = doi.rsplit("/", 1)[-1] if "/" in doi else doi
|
|
465
|
+
pdf_path = f"/{match.group('journal')}/article-pdf/{match.group('rest')}/{suffix}.pdf"
|
|
466
|
+
_append_candidate(candidates, urlunparse((parsed.scheme, parsed.netloc, pdf_path, "", "", "")))
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _format_template(
|
|
470
|
+
template: str,
|
|
471
|
+
*,
|
|
472
|
+
doi: str,
|
|
473
|
+
source_path: str = "",
|
|
474
|
+
source_path_after_prefix: str = "",
|
|
475
|
+
) -> str:
|
|
476
|
+
doi_suffix = doi.split("/", 1)[-1] if "/" in doi else doi
|
|
477
|
+
return template.format(
|
|
478
|
+
doi=doi,
|
|
479
|
+
doi_quoted=quote(doi, safe=""),
|
|
480
|
+
doi_suffix=doi_suffix,
|
|
481
|
+
doi_suffix_quoted=quote(doi_suffix, safe=""),
|
|
482
|
+
source_path=source_path,
|
|
483
|
+
source_path_quoted=quote(source_path, safe="/"),
|
|
484
|
+
source_path_after_prefix=source_path_after_prefix,
|
|
485
|
+
source_path_after_prefix_quoted=quote(source_path_after_prefix, safe="/"),
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _append_candidate(candidates: list[str], candidate: str | None, *, source_url: str = "") -> None:
|
|
490
|
+
normalized = _clean(candidate)
|
|
491
|
+
if not normalized:
|
|
492
|
+
return
|
|
493
|
+
if source_url:
|
|
494
|
+
normalized = urljoin(source_url, normalized)
|
|
495
|
+
parsed = urlparse(normalized)
|
|
496
|
+
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
497
|
+
return
|
|
498
|
+
if normalized not in candidates:
|
|
499
|
+
candidates.append(normalized)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _append_query_pdf_candidates(candidates: list[str], candidate: str, *, source_url: str) -> None:
|
|
503
|
+
absolute = urljoin(source_url, _clean(candidate))
|
|
504
|
+
parsed = urlparse(absolute)
|
|
505
|
+
for key, values in parse_qs(parsed.query, keep_blank_values=True).items():
|
|
506
|
+
if _clean(key).lower() not in {"file", "pdf", "src", "url"}:
|
|
507
|
+
continue
|
|
508
|
+
for value in values:
|
|
509
|
+
if any(marker in value.lower() for marker in PDF_URL_MARKERS):
|
|
510
|
+
_append_candidate(candidates, value, source_url=absolute)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _clean(value: object) -> str:
|
|
514
|
+
return re.sub(r"\s+", " ", str(value or "")).strip()
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Download an article's supplementary files with the browser session.
|
|
2
|
+
|
|
3
|
+
The adapter lists the links (``supplement_links``); this module fetches them
|
|
4
|
+
through the page's own request context, so the institution login and
|
|
5
|
+
Cloudflare clearance apply, and saves them next to the main text:
|
|
6
|
+
|
|
7
|
+
<output dir>/<DOI>_supplements/<original file name>
|
|
8
|
+
<output dir>/<DOI>_supplements/manifest.json
|
|
9
|
+
|
|
10
|
+
Every file is recorded, including the ones that failed or were skipped, with
|
|
11
|
+
the reason. Limits keep one article from stalling a batch.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import mimetypes
|
|
17
|
+
import re
|
|
18
|
+
import time
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, Callable, Iterable
|
|
21
|
+
from urllib.parse import unquote, urlparse
|
|
22
|
+
|
|
23
|
+
from ..adapters.base import SupplementLink
|
|
24
|
+
|
|
25
|
+
MAX_FILES = 50
|
|
26
|
+
MAX_FILE_BYTES = 200 * 1024 * 1024
|
|
27
|
+
TIME_BUDGET_SEC = 180
|
|
28
|
+
REQUEST_TIMEOUT_MS = 60_000
|
|
29
|
+
# The connection dropped before any answer: worth one more request.
|
|
30
|
+
_DROPPED_CONNECTION = ("socket disconnected", "socket hang up", "ECONNRESET", "EPIPE")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def supplements_dir(output_dir: Path, doi: str) -> Path:
|
|
34
|
+
return Path(output_dir) / (re.sub(r"[^A-Za-z0-9._-]+", "_", doi) + "_supplements")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _file_name(url: str, disposition: str, taken: set[str], label: str = "", content_type: str = "") -> str:
|
|
38
|
+
match = re.search(r"filename\*=(?:UTF-8'')?([^;]+)|filename=\"?([^\";]+)\"?", disposition or "", re.I)
|
|
39
|
+
raw = unquote((match.group(1) or match.group(2)).strip()) if match else ""
|
|
40
|
+
if not raw:
|
|
41
|
+
parsed = urlparse(url)
|
|
42
|
+
file_param = re.search(r"(?:^|&)file=([^&]+)", parsed.query)
|
|
43
|
+
raw = unquote(file_param.group(1)) if file_param else unquote(parsed.path.rsplit("/", 1)[-1])
|
|
44
|
+
if not re.search(r"\.[A-Za-z0-9]{1,5}$", raw):
|
|
45
|
+
# A short link without the file's name (links.lww.com/CORR/B539): the
|
|
46
|
+
# name in the link's text ("..._SDC1.docx (46.34 KB)"), else the type's extension.
|
|
47
|
+
named = re.search(r"[\w\-()+.]+\.[A-Za-z0-9]{2,5}(?=\s|$|\()", label or "")
|
|
48
|
+
extension = mimetypes.guess_extension((content_type or "").split(";")[0].strip()) or ""
|
|
49
|
+
raw = named.group(0) if named else f"{raw or 'supplement'}{extension}"
|
|
50
|
+
name = re.sub(r"[^\w.\-()+ ]+", "_", raw).strip(" ._") or "supplement"
|
|
51
|
+
stem, dot, ext = name.rpartition(".")
|
|
52
|
+
candidate, n = name, 2
|
|
53
|
+
while candidate.lower() in taken:
|
|
54
|
+
candidate = f"{stem or name}_{n}{dot}{ext}" if dot else f"{name}_{n}"
|
|
55
|
+
n += 1
|
|
56
|
+
taken.add(candidate.lower())
|
|
57
|
+
return candidate
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class _Downloaded:
|
|
61
|
+
"""A file the browser downloaded itself, shaped like Playwright's APIResponse."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, body: bytes, name: str):
|
|
64
|
+
self._body = body
|
|
65
|
+
self.status = 200
|
|
66
|
+
self.ok = True
|
|
67
|
+
self.headers = {"content-type": mimetypes.guess_type(name)[0] or "application/octet-stream",
|
|
68
|
+
"content-disposition": f'attachment; filename="{name}"',
|
|
69
|
+
"content-length": str(len(body))}
|
|
70
|
+
|
|
71
|
+
def body(self) -> bytes:
|
|
72
|
+
return self._body
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _get(request: Any, url: str, entry: dict[str, Any], sleep) -> Any:
|
|
76
|
+
"""GET the file; once more after the connection dropped before any answer
|
|
77
|
+
(a CDN's TLS handshake cut off midway, Oxford Academic 2026-11-01)."""
|
|
78
|
+
try:
|
|
79
|
+
return request.get(url, timeout=REQUEST_TIMEOUT_MS)
|
|
80
|
+
except Exception as exc:
|
|
81
|
+
if not any(marker in str(exc) for marker in _DROPPED_CONNECTION):
|
|
82
|
+
raise
|
|
83
|
+
entry["retried"] = str(exc).splitlines()[0][:120]
|
|
84
|
+
sleep(2)
|
|
85
|
+
return request.get(url, timeout=REQUEST_TIMEOUT_MS)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def collect(request: Any, links: Iterable[SupplementLink], output_dir: Path, doi: str, *,
|
|
89
|
+
clock=time.monotonic, sleep=time.sleep,
|
|
90
|
+
download: Callable[[str], tuple[bytes, str]] | None = None,
|
|
91
|
+
progress: Callable[[int], None] | None = None) -> list[dict[str, Any]]:
|
|
92
|
+
"""Fetch ``links`` with ``request`` (a Playwright APIRequestContext). A file
|
|
93
|
+
refused with 403 is taken through ``download`` when given: the browser
|
|
94
|
+
opens it and downloads it itself (Taylor & Francis behind Cloudflare
|
|
95
|
+
refused every scripted request with ego, the page's own fetch included,
|
|
96
|
+
and let the browser's own navigation through, 2026-11-01). ``progress`` is
|
|
97
|
+
told how many files are saved after each one (the progress window)."""
|
|
98
|
+
unique: list[SupplementLink] = []
|
|
99
|
+
for link in links:
|
|
100
|
+
if link.url and link.url not in {u.url for u in unique}:
|
|
101
|
+
unique.append(link)
|
|
102
|
+
if not unique:
|
|
103
|
+
return []
|
|
104
|
+
target = supplements_dir(output_dir, doi)
|
|
105
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
started = clock()
|
|
107
|
+
taken: set[str] = set()
|
|
108
|
+
files: list[dict[str, Any]] = []
|
|
109
|
+
for index, link in enumerate(unique):
|
|
110
|
+
entry: dict[str, Any] = {"url": link.url, "label": link.label}
|
|
111
|
+
if index >= MAX_FILES:
|
|
112
|
+
entry.update(status="skipped", reason="file_limit")
|
|
113
|
+
elif clock() - started > TIME_BUDGET_SEC:
|
|
114
|
+
entry.update(status="skipped", reason="time_budget")
|
|
115
|
+
else:
|
|
116
|
+
try:
|
|
117
|
+
response = _get(request, link.url, entry, sleep)
|
|
118
|
+
if response.status == 403 and download is not None:
|
|
119
|
+
response = _Downloaded(*download(link.url))
|
|
120
|
+
entry["via"] = "browser_download"
|
|
121
|
+
content_type = str(response.headers.get("content-type", "") or "")
|
|
122
|
+
length = int(response.headers.get("content-length") or 0)
|
|
123
|
+
if not response.ok:
|
|
124
|
+
entry.update(status="failed", reason=f"http_{response.status}")
|
|
125
|
+
elif length > MAX_FILE_BYTES:
|
|
126
|
+
entry.update(status="skipped", reason="too_large", size=length)
|
|
127
|
+
elif "text/html" in content_type.lower():
|
|
128
|
+
# A login wall or landing page instead of the file.
|
|
129
|
+
entry.update(status="failed", reason="html_page")
|
|
130
|
+
else:
|
|
131
|
+
body = response.body()
|
|
132
|
+
if len(body) > MAX_FILE_BYTES:
|
|
133
|
+
entry.update(status="skipped", reason="too_large", size=len(body))
|
|
134
|
+
else:
|
|
135
|
+
name = _file_name(link.url, str(response.headers.get("content-disposition", "") or ""), taken,
|
|
136
|
+
link.label, content_type)
|
|
137
|
+
path = target / name
|
|
138
|
+
path.write_bytes(body)
|
|
139
|
+
entry.update(status="saved", file=str(path), size=len(body),
|
|
140
|
+
content_type=content_type.split(";")[0].strip())
|
|
141
|
+
except Exception as exc:
|
|
142
|
+
entry.update(status="failed", reason=f"{type(exc).__name__}: {str(exc)[:160]}")
|
|
143
|
+
files.append(entry)
|
|
144
|
+
if progress is not None:
|
|
145
|
+
progress(sum(1 for f in files if f.get("status") == "saved"))
|
|
146
|
+
(target / "manifest.json").write_text(
|
|
147
|
+
json.dumps({"doi": doi, "files": files}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
148
|
+
return files
|