unique_web_search 2026.26.0.dev7__tar.gz → 2026.26.0.dev9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/PKG-INFO +2 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/pyproject.toml +2 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/client_settings.py +0 -27
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/__init__.py +1 -16
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/base.py +24 -9
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/basic.py +19 -1
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/crawl4ai.py +6 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/firecrawl.py +22 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/jina.py +29 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/tavily.py +22 -1
- unique_web_search-2026.26.0.dev9/src/unique_web_search/services/crawlers/url_safety/__init__.py +21 -0
- unique_web_search-2026.26.0.dev9/src/unique_web_search/services/proxy/__init__.py +21 -0
- unique_web_search-2026.26.0.dev9/src/unique_web_search/services/proxy/bridge.py +30 -0
- unique_web_search-2026.26.0.dev9/src/unique_web_search/services/proxy/mappers.py +73 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/base.py +18 -7
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/bing.py +30 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/brave.py +22 -3
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/custom_api.py +7 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/firecrawl.py +7 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/google.py +70 -46
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/jina.py +19 -18
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/perplexity.py +32 -3
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/tavily.py +7 -2
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/vertexai.py +32 -3
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/settings.py +1 -4
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/basic_proxy.py +0 -79
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/__init__.py +0 -15
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/dns.py +0 -53
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/models.py +0 -77
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/netloc.py +0 -62
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/policy.py +0 -52
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/redirect.py +0 -81
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/resolver.py +0 -103
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/service.py +0 -64
- unique_web_search-2026.26.0.dev7/src/unique_web_search/services/crawlers/url_safety/settings.py +0 -60
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/README.md +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/metrics.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/prompts.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/service.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/exceptions.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/prompts/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/prompts/guidelines.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/prompts/rejection_response.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/prompts/system_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/prompts/user_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/argument_screening/service.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/client/proxy_config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/cleaning/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/cleaning/character_sanitize.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/cleaning/clean.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/cleaning/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/base.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/llm_guard_judge.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/llm_keyword_redact.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/llm_process.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/judge_and_sanitize_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/judge_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/keyword_extract_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/page_context.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/sanitize_rules.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/system_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/prompts/user_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/settings.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/processing_strategies/truncate.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/content_processing/service.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/README.md +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/crawlers/utils.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/README.md +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/base_config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/base_executor.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/context.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/executor.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/prompts/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/prompts/refine_query_system_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/prompts/restrict_date_description.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/prompts/tool_description.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/prompts/tool_description_for_system_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v1/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/executor.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/prompts/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/prompts/step_query_description.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/prompts/tool_description.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/prompts/tool_description_for_system_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v2/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v3/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v3/config.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v3/executor.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v3/prompts/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v3/prompts/tool_description.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v3/prompts/tool_description_for_system_prompt.j2 +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/executors/v3/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/helpers.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/message_log.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/query_elicitation.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/README.md +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/google/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/bing/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/bing/client.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/bing/models.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/bing/runner.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/models.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/response_parsing.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/vertexai/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/vertexai/client.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/vertexai/exceptions.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/vertexai/gemini.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/search_engine/utils/grounding/vertexai/response_handler.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/snippet_judge/__init__.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/snippet_judge/prompts.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/snippet_judge/schema.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/snippet_judge/service.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/services/structured_llm.py +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/skills/web-search-v3.md +0 -0
- {unique_web_search-2026.26.0.dev7 → unique_web_search-2026.26.0.dev9}/src/unique_web_search/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: unique-web-search
|
|
3
|
-
Version: 2026.26.0.
|
|
3
|
+
Version: 2026.26.0.dev9
|
|
4
4
|
Summary:
|
|
5
5
|
Author: Andreas Hauri, Gustav Hartz, Rami Azouz
|
|
6
6
|
Author-email: Andreas Hauri <andreas@unique.ch>, Gustav Hartz <gustav.hartz.ext@unique.ch>, Rami Azouz <rami.ext@unique.ch>
|
|
@@ -28,7 +28,7 @@ Requires-Dist: certifi>=2025.11.12
|
|
|
28
28
|
Requires-Dist: jinja2>=3.1.6
|
|
29
29
|
Requires-Dist: playwright>=1.58.0
|
|
30
30
|
Requires-Dist: perplexityai>=0.34.0
|
|
31
|
-
Requires-Dist: unique-search-proxy-core>=2026.26.0.
|
|
31
|
+
Requires-Dist: unique-search-proxy-core>=2026.26.0.dev7,<2026.26.0rc0
|
|
32
32
|
Requires-Dist: unique-search-proxy-sdk>=2026.26.0.dev5,<2026.26.0rc0
|
|
33
33
|
Requires-Python: >=3.12, <4.0
|
|
34
34
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "unique_web_search"
|
|
3
|
-
version = "2026.26.0.
|
|
3
|
+
version = "2026.26.0.dev9"
|
|
4
4
|
description = ""
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12,<4.0"
|
|
@@ -34,7 +34,7 @@ dependencies = [
|
|
|
34
34
|
"jinja2>=3.1.6",
|
|
35
35
|
"playwright>=1.58.0",
|
|
36
36
|
"perplexityai>=0.34.0",
|
|
37
|
-
"unique-search-proxy-core>=2026.26.0.
|
|
37
|
+
"unique-search-proxy-core>=2026.26.0.dev7,<2026.26.0rc0",
|
|
38
38
|
"unique-search-proxy-sdk>=2026.26.0.dev5,<2026.26.0rc0",
|
|
39
39
|
]
|
|
40
40
|
|
|
@@ -235,30 +235,3 @@ def get_perplexity_search_settings() -> PerplexitySearchSettings:
|
|
|
235
235
|
if _perplexity_search_settings is None:
|
|
236
236
|
_perplexity_search_settings = PerplexitySearchSettings.from_env_settings()
|
|
237
237
|
return _perplexity_search_settings
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
class SearchProxySettings(BaseModel):
|
|
241
|
-
base_url: str | None = None
|
|
242
|
-
|
|
243
|
-
@property
|
|
244
|
-
def is_configured(self) -> bool:
|
|
245
|
-
return self.base_url is not None
|
|
246
|
-
|
|
247
|
-
@classmethod
|
|
248
|
-
def from_env_settings(cls):
|
|
249
|
-
if env_settings.search_proxy_base_url is None:
|
|
250
|
-
_LOGGER.warning("Unique Search Proxy missing required setting: Base URL")
|
|
251
|
-
else:
|
|
252
|
-
_LOGGER.info("Unique Search Proxy is properly configured")
|
|
253
|
-
|
|
254
|
-
return cls(base_url=env_settings.search_proxy_base_url)
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
_search_proxy_settings: SearchProxySettings | None = None
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
def get_search_proxy_settings() -> SearchProxySettings:
|
|
261
|
-
global _search_proxy_settings
|
|
262
|
-
if _search_proxy_settings is None:
|
|
263
|
-
_search_proxy_settings = SearchProxySettings.from_env_settings()
|
|
264
|
-
return _search_proxy_settings
|
|
@@ -9,10 +9,6 @@ from unique_web_search.services.crawlers.basic import (
|
|
|
9
9
|
BasicCrawler,
|
|
10
10
|
BasicCrawlerConfig,
|
|
11
11
|
)
|
|
12
|
-
from unique_web_search.services.crawlers.basic_proxy import (
|
|
13
|
-
BasicCrawlerProxyConfig,
|
|
14
|
-
BasicProxyCrawler,
|
|
15
|
-
)
|
|
16
12
|
from unique_web_search.services.crawlers.crawl4ai import (
|
|
17
13
|
Crawl4AiCrawler,
|
|
18
14
|
Crawl4AiCrawlerConfig,
|
|
@@ -31,17 +27,11 @@ from unique_web_search.services.crawlers.tavily import (
|
|
|
31
27
|
)
|
|
32
28
|
|
|
33
29
|
CrawlerTypes = (
|
|
34
|
-
BasicCrawler
|
|
35
|
-
| BasicProxyCrawler
|
|
36
|
-
| Crawl4AiCrawler
|
|
37
|
-
| FirecrawlCrawler
|
|
38
|
-
| JinaCrawler
|
|
39
|
-
| TavilyCrawler
|
|
30
|
+
BasicCrawler | Crawl4AiCrawler | FirecrawlCrawler | JinaCrawler | TavilyCrawler
|
|
40
31
|
)
|
|
41
32
|
|
|
42
33
|
CrawlerConfigTypes = (
|
|
43
34
|
BasicCrawlerConfig
|
|
44
|
-
| BasicCrawlerProxyConfig
|
|
45
35
|
| Crawl4AiCrawlerConfig
|
|
46
36
|
| FirecrawlCrawlerConfig
|
|
47
37
|
| JinaCrawlerConfig
|
|
@@ -54,7 +44,6 @@ CRAWLER_NAME_TO_CONFIG = {
|
|
|
54
44
|
"firecrawl": FirecrawlCrawlerConfig,
|
|
55
45
|
"jina": JinaCrawlerConfig,
|
|
56
46
|
"tavily": TavilyCrawlerConfig,
|
|
57
|
-
"basic_proxy": BasicCrawlerProxyConfig,
|
|
58
47
|
}
|
|
59
48
|
|
|
60
49
|
|
|
@@ -72,8 +61,6 @@ def get_crawler_service(
|
|
|
72
61
|
return FirecrawlCrawler(crawler_config)
|
|
73
62
|
case CrawlerType.JINA:
|
|
74
63
|
return JinaCrawler(crawler_config)
|
|
75
|
-
case CrawlerType.BASIC_PROXY:
|
|
76
|
-
return BasicProxyCrawler(crawler_config)
|
|
77
64
|
|
|
78
65
|
|
|
79
66
|
def get_crawler_config_types_from_names(crawler_names: list[str]) -> TypeAlias:
|
|
@@ -103,8 +90,6 @@ def get_default_crawler_config(
|
|
|
103
90
|
__all__ = [
|
|
104
91
|
"BasicCrawler",
|
|
105
92
|
"BasicCrawlerConfig",
|
|
106
|
-
"BasicProxyCrawler",
|
|
107
|
-
"BasicCrawlerProxyConfig",
|
|
108
93
|
"Crawl4AiCrawler",
|
|
109
94
|
"Crawl4AiCrawlerConfig",
|
|
110
95
|
"FirecrawlCrawler",
|
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from enum import StrEnum
|
|
3
|
-
from typing import Generic, TypeVar
|
|
3
|
+
from typing import ClassVar, Generic, TypeVar
|
|
4
4
|
|
|
5
5
|
from pydantic import BaseModel
|
|
6
|
+
from unique_search_proxy_core.url_safety import (
|
|
7
|
+
CrawlTargetValidationError,
|
|
8
|
+
ResolvedCrawlTarget,
|
|
9
|
+
UrlSafetyService,
|
|
10
|
+
)
|
|
6
11
|
from unique_toolkit.agentic.tools.config import (
|
|
7
12
|
get_configuration_dict,
|
|
8
13
|
)
|
|
9
14
|
|
|
10
|
-
from unique_web_search.
|
|
11
|
-
ResolvedCrawlTarget,
|
|
12
|
-
UrlSafetyService,
|
|
13
|
-
)
|
|
15
|
+
from unique_web_search.metrics import crawl_blocked
|
|
14
16
|
from unique_web_search.services.helpers import (
|
|
15
17
|
clean_model_title_generator,
|
|
16
18
|
experimental_model_title_generator,
|
|
17
19
|
)
|
|
20
|
+
from unique_web_search.services.proxy.bridge import search_proxy_client_enabled
|
|
18
21
|
|
|
19
22
|
|
|
20
23
|
class CrawlerType(StrEnum):
|
|
21
24
|
CRAWL4AI = "Crawl4AiCrawler"
|
|
22
25
|
BASIC = "BasicCrawler"
|
|
23
|
-
BASIC_PROXY = "BasicProxyCrawler"
|
|
24
26
|
NO_CRAWLER = "NoCrawler"
|
|
25
27
|
TAVILY = "TavilyCrawler"
|
|
26
28
|
FIRECRAWL = "FirecrawlCrawler"
|
|
@@ -51,12 +53,25 @@ CrawlerConfig = TypeVar(
|
|
|
51
53
|
|
|
52
54
|
|
|
53
55
|
class BaseCrawler(ABC, Generic[CrawlerConfig]):
|
|
56
|
+
supports_proxy_crawl: ClassVar[bool] = False
|
|
57
|
+
|
|
54
58
|
def __init__(self, config: CrawlerConfig):
|
|
55
59
|
self.config = config
|
|
56
60
|
|
|
57
61
|
async def crawl(self, urls: list[str]) -> list[str]:
|
|
58
|
-
|
|
59
|
-
|
|
62
|
+
if search_proxy_client_enabled and self.supports_proxy_crawl:
|
|
63
|
+
return await self._proxy_crawl(urls)
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
targets = await UrlSafetyService.validate_batch_urls(urls)
|
|
67
|
+
except CrawlTargetValidationError as exc:
|
|
68
|
+
for target in exc.blocked_targets:
|
|
69
|
+
crawl_blocked.labels(reason_category=target.category).inc()
|
|
70
|
+
raise
|
|
71
|
+
return await self._legacy_crawl(targets)
|
|
72
|
+
|
|
73
|
+
@abstractmethod
|
|
74
|
+
async def _proxy_crawl(self, urls: list[str]) -> list[str]: ...
|
|
60
75
|
|
|
61
76
|
@abstractmethod
|
|
62
|
-
async def
|
|
77
|
+
async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]: ...
|
|
@@ -20,6 +20,10 @@ from unique_web_search.services.crawlers.url_safety import (
|
|
|
20
20
|
ResolvedCrawlTarget,
|
|
21
21
|
)
|
|
22
22
|
from unique_web_search.services.crawlers.utils import get_random_user_agent
|
|
23
|
+
from unique_web_search.services.proxy.bridge import (
|
|
24
|
+
open_search_proxy_client,
|
|
25
|
+
)
|
|
26
|
+
from unique_web_search.services.proxy.mappers import map_crawl_response
|
|
23
27
|
|
|
24
28
|
_LOGGER = logging.getLogger(__name__)
|
|
25
29
|
|
|
@@ -51,6 +55,8 @@ class BasicCrawlerConfig(BaseCrawlerConfig[CrawlerType.BASIC]):
|
|
|
51
55
|
|
|
52
56
|
|
|
53
57
|
class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
|
|
58
|
+
supports_proxy_crawl = True
|
|
59
|
+
|
|
54
60
|
def __init__(self, config: BasicCrawlerConfig):
|
|
55
61
|
super().__init__(config)
|
|
56
62
|
|
|
@@ -63,7 +69,19 @@ class BasicCrawler(BaseCrawler[BasicCrawlerConfig]):
|
|
|
63
69
|
# tags=["basic", "scrape"],
|
|
64
70
|
# )
|
|
65
71
|
@override
|
|
66
|
-
async def
|
|
72
|
+
async def _proxy_crawl(self, urls: list[str]) -> list[str]:
|
|
73
|
+
async with open_search_proxy_client(
|
|
74
|
+
timeout=float(self.config.timeout),
|
|
75
|
+
) as client:
|
|
76
|
+
response = await client.crawl.basic(
|
|
77
|
+
urls=urls,
|
|
78
|
+
timeout=int(self.config.timeout),
|
|
79
|
+
max_concurrent_requests=self.config.max_concurrent_requests,
|
|
80
|
+
)
|
|
81
|
+
return map_crawl_response(response, urls)
|
|
82
|
+
|
|
83
|
+
@override
|
|
84
|
+
async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
|
|
67
85
|
async with async_client(timeout=Timeout(self.config.timeout)) as client:
|
|
68
86
|
tasks = [self._crawl_url_with_client(client, target) for target in targets]
|
|
69
87
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from enum import StrEnum
|
|
3
|
-
from typing import Any, Literal
|
|
3
|
+
from typing import Any, Literal, override
|
|
4
4
|
|
|
5
5
|
from pydantic import BaseModel, Field
|
|
6
6
|
from unique_toolkit.agentic.tools.config import get_configuration_dict
|
|
@@ -169,6 +169,9 @@ class Crawl4AiCrawler(BaseCrawler[Crawl4AiCrawlerConfig]):
|
|
|
169
169
|
# @track(
|
|
170
170
|
# tags=["crawl4ai", "scrape"],
|
|
171
171
|
# )
|
|
172
|
+
@override
|
|
173
|
+
async def _proxy_crawl(self, urls: list[str]) -> list[str]:
|
|
174
|
+
raise NotImplementedError("Crawl4AiCrawler does not support proxy crawl")
|
|
172
175
|
|
|
173
176
|
@staticmethod
|
|
174
177
|
def _get_ssrf_guard_hook():
|
|
@@ -209,7 +212,8 @@ class Crawl4AiCrawler(BaseCrawler[Crawl4AiCrawlerConfig]):
|
|
|
209
212
|
|
|
210
213
|
return _ssrf_guard_hook
|
|
211
214
|
|
|
212
|
-
|
|
215
|
+
@override
|
|
216
|
+
async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
|
|
213
217
|
urls = [target.normalized_url for target in targets]
|
|
214
218
|
# Lazy import of crawl4ai - only import when actually needed
|
|
215
219
|
from crawl4ai import (
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Literal
|
|
1
|
+
from typing import Literal, override
|
|
2
2
|
|
|
3
3
|
from firecrawl import AsyncFirecrawl
|
|
4
4
|
|
|
@@ -11,6 +11,10 @@ from unique_web_search.services.crawlers.base import (
|
|
|
11
11
|
CrawlerType,
|
|
12
12
|
)
|
|
13
13
|
from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
|
|
14
|
+
from unique_web_search.services.proxy.bridge import (
|
|
15
|
+
open_search_proxy_client,
|
|
16
|
+
)
|
|
17
|
+
from unique_web_search.services.proxy.mappers import map_crawl_response
|
|
14
18
|
|
|
15
19
|
|
|
16
20
|
class FirecrawlCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.FIRECRAWL]):
|
|
@@ -18,6 +22,8 @@ class FirecrawlCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.FIRECRAWL
|
|
|
18
22
|
|
|
19
23
|
|
|
20
24
|
class FirecrawlCrawler(BaseCrawler[FirecrawlCrawlerConfig]):
|
|
25
|
+
supports_proxy_crawl = True
|
|
26
|
+
|
|
21
27
|
def __init__(self, config: FirecrawlCrawlerConfig):
|
|
22
28
|
super().__init__(config)
|
|
23
29
|
|
|
@@ -25,8 +31,22 @@ class FirecrawlCrawler(BaseCrawler[FirecrawlCrawlerConfig]):
|
|
|
25
31
|
# @track(
|
|
26
32
|
# tags=["firecrawl", "scrape"],
|
|
27
33
|
# )
|
|
28
|
-
|
|
34
|
+
@override
|
|
35
|
+
async def _proxy_crawl(self, urls: list[str]) -> list[str]:
|
|
36
|
+
async with open_search_proxy_client(
|
|
37
|
+
timeout=float(self.config.timeout),
|
|
38
|
+
) as client:
|
|
39
|
+
response = await client.crawl.firecrawl(
|
|
40
|
+
urls=urls,
|
|
41
|
+
timeout=int(self.config.timeout),
|
|
42
|
+
only_main_content=True,
|
|
43
|
+
)
|
|
44
|
+
return map_crawl_response(response, urls)
|
|
45
|
+
|
|
46
|
+
@override
|
|
47
|
+
async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
|
|
29
48
|
urls = [target.normalized_url for target in targets]
|
|
49
|
+
|
|
30
50
|
api_key = get_firecrawl_search_settings().api_key
|
|
31
51
|
assert api_key is not None, "Firecrawl API key is not configured"
|
|
32
52
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
from typing import Any, Literal
|
|
2
|
+
from typing import Any, Literal, cast, override
|
|
3
3
|
|
|
4
4
|
from httpx import AsyncClient
|
|
5
5
|
from pydantic import BaseModel, Field, HttpUrl
|
|
6
|
+
from unique_search_proxy_core.crawlers.jina.schema import JinaEngine, JinaReturnFormat
|
|
6
7
|
|
|
7
8
|
from unique_web_search.client_settings import get_jina_search_settings
|
|
8
9
|
from unique_web_search.services.crawlers.base import (
|
|
@@ -11,6 +12,10 @@ from unique_web_search.services.crawlers.base import (
|
|
|
11
12
|
CrawlerType,
|
|
12
13
|
)
|
|
13
14
|
from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
|
|
15
|
+
from unique_web_search.services.proxy.bridge import (
|
|
16
|
+
open_search_proxy_client,
|
|
17
|
+
)
|
|
18
|
+
from unique_web_search.services.proxy.mappers import map_crawl_response
|
|
14
19
|
|
|
15
20
|
|
|
16
21
|
class ReaderBody(BaseModel):
|
|
@@ -46,6 +51,8 @@ class JinaCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.JINA]):
|
|
|
46
51
|
|
|
47
52
|
|
|
48
53
|
class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
|
|
54
|
+
supports_proxy_crawl = True
|
|
55
|
+
|
|
49
56
|
def __init__(self, config: JinaCrawlerConfig):
|
|
50
57
|
super().__init__(config)
|
|
51
58
|
|
|
@@ -53,8 +60,28 @@ class JinaCrawler(BaseCrawler[JinaCrawlerConfig]):
|
|
|
53
60
|
# @track(
|
|
54
61
|
# tags=["jina", "scrape"],
|
|
55
62
|
# )
|
|
56
|
-
|
|
63
|
+
@override
|
|
64
|
+
async def _proxy_crawl(self, urls: list[str]) -> list[str]:
|
|
65
|
+
headers = self.config.headers
|
|
66
|
+
async with open_search_proxy_client(
|
|
67
|
+
timeout=float(self.config.timeout),
|
|
68
|
+
) as client:
|
|
69
|
+
response = await client.crawl.jina(
|
|
70
|
+
urls=urls,
|
|
71
|
+
timeout=int(self.config.timeout),
|
|
72
|
+
return_format=cast(
|
|
73
|
+
JinaReturnFormat,
|
|
74
|
+
headers.get("X-Return-Format", "markdown"),
|
|
75
|
+
),
|
|
76
|
+
engine=cast(JinaEngine, headers.get("X-Engine", "browser")),
|
|
77
|
+
do_not_track=headers.get("DNT") == "1",
|
|
78
|
+
)
|
|
79
|
+
return map_crawl_response(response, urls)
|
|
80
|
+
|
|
81
|
+
@override
|
|
82
|
+
async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
|
|
57
83
|
urls = [target.normalized_url for target in targets]
|
|
84
|
+
|
|
58
85
|
jina_settings = get_jina_search_settings()
|
|
59
86
|
api_key = jina_settings.api_key
|
|
60
87
|
assert api_key is not None, "Jina API key is not configured"
|
|
@@ -10,6 +10,10 @@ from unique_web_search.services.crawlers.base import (
|
|
|
10
10
|
CrawlerType,
|
|
11
11
|
)
|
|
12
12
|
from unique_web_search.services.crawlers.url_safety import ResolvedCrawlTarget
|
|
13
|
+
from unique_web_search.services.proxy.bridge import (
|
|
14
|
+
open_search_proxy_client,
|
|
15
|
+
)
|
|
16
|
+
from unique_web_search.services.proxy.mappers import map_crawl_response
|
|
13
17
|
|
|
14
18
|
|
|
15
19
|
class TavilyCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.TAVILY]):
|
|
@@ -18,6 +22,8 @@ class TavilyCrawlerConfig(BaseCrawlerConfigExperimental[CrawlerType.TAVILY]):
|
|
|
18
22
|
|
|
19
23
|
|
|
20
24
|
class TavilyCrawler(BaseCrawler[TavilyCrawlerConfig]):
|
|
25
|
+
supports_proxy_crawl = True
|
|
26
|
+
|
|
21
27
|
def __init__(self, config: TavilyCrawlerConfig):
|
|
22
28
|
super().__init__(config)
|
|
23
29
|
|
|
@@ -25,8 +31,23 @@ class TavilyCrawler(BaseCrawler[TavilyCrawlerConfig]):
|
|
|
25
31
|
# @track(
|
|
26
32
|
# tags=["tavily", "scrape"],
|
|
27
33
|
# )
|
|
28
|
-
async def
|
|
34
|
+
async def _proxy_crawl(self, urls: list[str]) -> list[str]:
|
|
35
|
+
async with open_search_proxy_client(
|
|
36
|
+
timeout=float(self.config.timeout),
|
|
37
|
+
) as client:
|
|
38
|
+
response = await client.crawl.tavily(
|
|
39
|
+
urls=urls,
|
|
40
|
+
timeout=int(self.config.timeout),
|
|
41
|
+
extract_depth=self.config.depth,
|
|
42
|
+
format="markdown",
|
|
43
|
+
include_images=False,
|
|
44
|
+
include_favicon=False,
|
|
45
|
+
)
|
|
46
|
+
return map_crawl_response(response, urls)
|
|
47
|
+
|
|
48
|
+
async def _legacy_crawl(self, targets: list[ResolvedCrawlTarget]) -> list[str]:
|
|
29
49
|
urls = [target.normalized_url for target in targets]
|
|
50
|
+
|
|
30
51
|
api_key = get_tavily_search_settings().api_key
|
|
31
52
|
assert api_key is not None, "Tavily API key is not configured"
|
|
32
53
|
|
unique_web_search-2026.26.0.dev9/src/unique_web_search/services/crawlers/url_safety/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Deprecated re-exports — use ``unique_search_proxy_core.url_safety`` directly."""
|
|
2
|
+
|
|
3
|
+
from unique_search_proxy_core.url_safety import (
|
|
4
|
+
BlockedCrawlTarget,
|
|
5
|
+
CrawlTargetValidationError,
|
|
6
|
+
ResolvedCrawlTarget,
|
|
7
|
+
UrlSafetyOutcome,
|
|
8
|
+
UrlSafetyService,
|
|
9
|
+
UrlSafetySettings,
|
|
10
|
+
url_safety_settings,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"BlockedCrawlTarget",
|
|
15
|
+
"CrawlTargetValidationError",
|
|
16
|
+
"ResolvedCrawlTarget",
|
|
17
|
+
"UrlSafetyOutcome",
|
|
18
|
+
"UrlSafetyService",
|
|
19
|
+
"UrlSafetySettings",
|
|
20
|
+
"url_safety_settings",
|
|
21
|
+
]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from unique_web_search.services.proxy.bridge import (
|
|
2
|
+
open_search_proxy_client,
|
|
3
|
+
search_proxy_client_enabled,
|
|
4
|
+
)
|
|
5
|
+
from unique_web_search.services.proxy.mappers import (
|
|
6
|
+
agent_answer_text,
|
|
7
|
+
map_agent_answer,
|
|
8
|
+
map_crawl_response,
|
|
9
|
+
map_search_response,
|
|
10
|
+
result_to_markdown,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"agent_answer_text",
|
|
15
|
+
"map_agent_answer",
|
|
16
|
+
"map_crawl_response",
|
|
17
|
+
"map_search_response",
|
|
18
|
+
"open_search_proxy_client",
|
|
19
|
+
"result_to_markdown",
|
|
20
|
+
"search_proxy_client_enabled",
|
|
21
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncIterator
|
|
4
|
+
from contextlib import asynccontextmanager
|
|
5
|
+
|
|
6
|
+
from unique_search_proxy_sdk import UniqueSearchProxyClient
|
|
7
|
+
|
|
8
|
+
from unique_web_search.settings import env_settings
|
|
9
|
+
|
|
10
|
+
search_proxy_client_enabled = env_settings.search_proxy_base_url is not None
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@asynccontextmanager
|
|
14
|
+
async def open_search_proxy_client(
|
|
15
|
+
timeout: float,
|
|
16
|
+
) -> AsyncIterator[UniqueSearchProxyClient]:
|
|
17
|
+
base_url = env_settings.search_proxy_base_url
|
|
18
|
+
assert base_url is not None, "Unique Search Proxy base URL is not configured"
|
|
19
|
+
|
|
20
|
+
async with UniqueSearchProxyClient(
|
|
21
|
+
base_url=base_url,
|
|
22
|
+
timeout=timeout,
|
|
23
|
+
) as client:
|
|
24
|
+
yield client
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"open_search_proxy_client",
|
|
29
|
+
"search_proxy_client_enabled",
|
|
30
|
+
]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from unique_search_proxy_sdk._generated.models.agent_search_response import (
|
|
7
|
+
AgentSearchResponse,
|
|
8
|
+
)
|
|
9
|
+
from unique_search_proxy_sdk._generated.models.crawl_response import CrawlResponse
|
|
10
|
+
from unique_search_proxy_sdk._generated.models.crawl_url_result import CrawlUrlResult
|
|
11
|
+
from unique_search_proxy_sdk._generated.models.per_url_error import PerUrlError
|
|
12
|
+
from unique_search_proxy_sdk._generated.models.search_response import SearchResponse
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from unique_web_search.services.search_engine.schema import WebSearchResult
|
|
16
|
+
from unique_web_search.services.search_engine.utils.grounding.response_parsing import (
|
|
17
|
+
ResponseParser,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def map_search_response(response: SearchResponse) -> list[WebSearchResult]:
|
|
22
|
+
from unique_web_search.services.search_engine.schema import WebSearchResult
|
|
23
|
+
|
|
24
|
+
return [
|
|
25
|
+
WebSearchResult(
|
|
26
|
+
url=result.url,
|
|
27
|
+
title=result.title,
|
|
28
|
+
snippet=result.snippet,
|
|
29
|
+
content=result.content if isinstance(result.content, str) else "",
|
|
30
|
+
)
|
|
31
|
+
for result in response.curated
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
_CRAWL_RESULT_TEMPLATE = "URL: {url}\n\n{body}"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _crawl_result_body(result: CrawlUrlResult) -> str:
|
|
39
|
+
if isinstance(result.content, str):
|
|
40
|
+
return result.content
|
|
41
|
+
|
|
42
|
+
if isinstance(result.error, PerUrlError):
|
|
43
|
+
return f"Error: {result.error.message}"
|
|
44
|
+
|
|
45
|
+
return "Error: No content returned by search proxy"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def result_to_markdown(result: CrawlUrlResult) -> str:
|
|
49
|
+
return _CRAWL_RESULT_TEMPLATE.format(
|
|
50
|
+
url=result.url,
|
|
51
|
+
body=_crawl_result_body(result),
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def map_crawl_response(response: CrawlResponse, urls: list[str]) -> list[str]:
|
|
56
|
+
return [result_to_markdown(result) for result in response.results]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def map_agent_answer(
|
|
60
|
+
answer: str,
|
|
61
|
+
parsers: Sequence[ResponseParser],
|
|
62
|
+
) -> list[WebSearchResult]:
|
|
63
|
+
from unique_web_search.services.search_engine.utils.grounding.response_parsing import (
|
|
64
|
+
convert_response_to_search_results,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
return await convert_response_to_search_results(answer, list(parsers))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def agent_answer_text(response: AgentSearchResponse) -> str:
|
|
71
|
+
if isinstance(response.answer, str):
|
|
72
|
+
return response.answer
|
|
73
|
+
return ""
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from enum import StrEnum
|
|
3
|
-
from typing import Generic, TypeVar
|
|
3
|
+
from typing import ClassVar, Generic, TypeVar
|
|
4
4
|
|
|
5
5
|
from pydantic import BaseModel, ConfigDict, Field
|
|
6
6
|
from unique_toolkit.agentic.tools.config import get_configuration_dict
|
|
7
7
|
|
|
8
|
+
from unique_web_search.services.proxy.bridge import search_proxy_client_enabled
|
|
8
9
|
from unique_web_search.services.search_engine.schema import (
|
|
9
10
|
WebSearchResult,
|
|
10
11
|
)
|
|
@@ -104,16 +105,16 @@ class SearchEngine(ABC, Generic[SearchEngineConfig]):
|
|
|
104
105
|
"""Base class for the search engine. It provides the common methods to search the web. This allows to use different search engines easily.
|
|
105
106
|
|
|
106
107
|
Abstract Methods:
|
|
107
|
-
|
|
108
|
-
_get_request_params: Get the request parameters.
|
|
108
|
+
_legacy_search: Direct search implementation (no proxy).
|
|
109
109
|
requires_scraping: Whether the search engine requires scraping.
|
|
110
110
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
chat_manager: The chat manager.
|
|
111
|
+
Subclasses with proxy support set ``supports_proxy_search = True`` and
|
|
112
|
+
implement ``_proxy_search``.
|
|
114
113
|
|
|
115
114
|
"""
|
|
116
115
|
|
|
116
|
+
supports_proxy_search: ClassVar[bool] = False
|
|
117
|
+
|
|
117
118
|
def __init__(self, config: SearchEngineConfig):
|
|
118
119
|
self.config = config
|
|
119
120
|
|
|
@@ -122,6 +123,16 @@ class SearchEngine(ABC, Generic[SearchEngineConfig]):
|
|
|
122
123
|
def requires_scraping(self) -> bool:
|
|
123
124
|
"""Whether the search engine requires scraping."""
|
|
124
125
|
|
|
125
|
-
@abstractmethod
|
|
126
126
|
async def search(self, query: str, **kwargs) -> list[WebSearchResult]:
|
|
127
127
|
"""Search the web for the given query using the search engine."""
|
|
128
|
+
if search_proxy_client_enabled and self.supports_proxy_search:
|
|
129
|
+
return await self._proxy_search(query=query, **kwargs)
|
|
130
|
+
return await self._legacy_search(query=query, **kwargs)
|
|
131
|
+
|
|
132
|
+
@abstractmethod
|
|
133
|
+
async def _proxy_search(self, query: str, **kwargs) -> list[WebSearchResult]:
|
|
134
|
+
"""Search the web via the search proxy."""
|
|
135
|
+
|
|
136
|
+
@abstractmethod
|
|
137
|
+
async def _legacy_search(self, query: str, **kwargs) -> list[WebSearchResult]:
|
|
138
|
+
"""Search the web directly without the search proxy."""
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
from typing import Annotated, Literal
|
|
2
|
+
from typing import Annotated, Literal, override
|
|
3
3
|
|
|
4
4
|
from pydantic import BaseModel, Field
|
|
5
5
|
from unique_toolkit._common.default_language_model import DEFAULT_GPT_4o
|
|
@@ -8,6 +8,14 @@ from unique_toolkit._common.validators import LMI, get_LMI_default_field
|
|
|
8
8
|
from unique_toolkit.agentic.tools.config import get_configuration_dict
|
|
9
9
|
from unique_toolkit.language_model import LanguageModelService
|
|
10
10
|
|
|
11
|
+
from unique_web_search.services.proxy.bridge import (
|
|
12
|
+
open_search_proxy_client,
|
|
13
|
+
search_proxy_client_enabled,
|
|
14
|
+
)
|
|
15
|
+
from unique_web_search.services.proxy.mappers import (
|
|
16
|
+
agent_answer_text,
|
|
17
|
+
map_agent_answer,
|
|
18
|
+
)
|
|
11
19
|
from unique_web_search.services.search_engine.base import (
|
|
12
20
|
BaseSearchEngineConfig,
|
|
13
21
|
SearchEngine,
|
|
@@ -75,6 +83,8 @@ class BingSearchConfig(
|
|
|
75
83
|
|
|
76
84
|
|
|
77
85
|
class BingSearch(SearchEngine[BingSearchConfig]):
|
|
86
|
+
supports_proxy_search = True
|
|
87
|
+
|
|
78
88
|
def __init__(
|
|
79
89
|
self,
|
|
80
90
|
config: BingSearchConfig,
|
|
@@ -93,6 +103,9 @@ class BingSearch(SearchEngine[BingSearchConfig]):
|
|
|
93
103
|
|
|
94
104
|
@property
|
|
95
105
|
def is_configured(self) -> bool:
|
|
106
|
+
if search_proxy_client_enabled:
|
|
107
|
+
return True
|
|
108
|
+
|
|
96
109
|
async def _is_configured() -> bool:
|
|
97
110
|
return await credentials_are_valid(self.credentials)
|
|
98
111
|
|
|
@@ -102,7 +115,22 @@ class BingSearch(SearchEngine[BingSearchConfig]):
|
|
|
102
115
|
def requires_scraping(self) -> bool:
|
|
103
116
|
return self.config.requires_scraping
|
|
104
117
|
|
|
105
|
-
|
|
118
|
+
@override
|
|
119
|
+
async def _proxy_search(self, query: str, **kwargs) -> list[WebSearchResult]:
|
|
120
|
+
async with open_search_proxy_client(timeout=120.0) as client:
|
|
121
|
+
response = await client.agent_search.bing(
|
|
122
|
+
query=query,
|
|
123
|
+
fetch_size=self.config.fetch_size,
|
|
124
|
+
agent_id=self.config.agent_id or None,
|
|
125
|
+
generation_instructions=self.config.generation_instructions,
|
|
126
|
+
)
|
|
127
|
+
return await map_agent_answer(
|
|
128
|
+
agent_answer_text(response),
|
|
129
|
+
self.response_parsers,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
@override
|
|
133
|
+
async def _legacy_search(self, query: str, **kwargs) -> list[WebSearchResult]:
|
|
106
134
|
agent_client = get_project_client(self.credentials, self.config.endpoint)
|
|
107
135
|
|
|
108
136
|
search_results = await create_and_process_run(
|