academia-mcp 1.10.3__py3-none-any.whl → 1.10.5__py3-none-any.whl

academia_mcp/tools/visit_webpage.py

@@ -1,6 +1,6 @@
 import re
 import json
-from typing import Optional
+from typing import Optional, Dict, Any, cast
 
 from markdownify import markdownify  # type: ignore
 
@@ -11,9 +11,10 @@ from academia_mcp.utils import sanitize_output
 EXA_CONTENTS_URL = "https://api.exa.ai/contents"
 TAVILY_EXTRACT_URL = "https://api.tavily.com/extract"
 AVAILABLE_PROVIDERS = ("basic", "exa", "tavily")
+ERROR_MESSAGE = "Failed to get content from the page. Try to use another provider."
 
 
-def _exa_visit_webpage(url: str) -> str:
+def _exa_visit_webpage(url: str) -> Dict[str, Any]:
     key = settings.EXA_API_KEY or ""
     assert key, "Error: EXA_API_KEY is not set and no api_key was provided"
     payload = {
@@ -21,24 +22,48 @@ def _exa_visit_webpage(url: str) -> str:
         "text": True,
     }
     response = post_with_retries(EXA_CONTENTS_URL, payload=payload, api_key=key)
-    return sanitize_output(json.dumps(response.json()["results"][0]))
+    results = response.json()["results"]
+    if not results:
+        return {"error": ERROR_MESSAGE}
+    return cast(Dict[str, Any], results[0])
 
 
-def _tavily_visit_webpage(url: str) -> str:
+def _tavily_visit_webpage(url: str) -> Dict[str, Any]:
     key = settings.TAVILY_API_KEY or ""
     assert key, "Error: TAVILY_API_KEY is not set and no api_key was provided"
     payload = {
         "urls": [url],
     }
     response = post_with_retries(TAVILY_EXTRACT_URL, payload=payload, api_key=key)
-    return sanitize_output(json.dumps(response.json()["results"][0]["raw_content"]))
+    results = response.json()["results"]
+    if not results:
+        return {"error": ERROR_MESSAGE}
+    return {"text": results[0]["raw_content"]}
+
+
+def _basic_visit_webpage(url: str) -> Dict[str, Any]:
+    try:
+        response = get_with_retries(url)
+        content_type = response.headers.get("content-type", "").lower()
+        if not content_type or (
+            not content_type.startswith("text/") and "html" not in content_type
+        ):
+            if settings.EXA_API_KEY:
+                return _exa_visit_webpage(url)
+            return {"error": f"Unsupported content-type: {content_type or 'unknown'}"}
+        markdown_content = markdownify(response.text).strip()
+        markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
+        return {"text": markdown_content}
+    except Exception as e:
+        return {"error": str(e) + "\n" + ERROR_MESSAGE}
 
 
 def visit_webpage(url: str, provider: Optional[str] = "basic") -> str:
     """
     Visit a webpage and return the content.
 
-    Returns a JSON object serialized to a string. The structure is: {"url": "...", "text": "..."}
+    Returns a JSON object serialized to a string. The structure is: {"id": "...", "text": "..."}.
+    If there are errors, the structure is: {"id": "...", "error": "..."}.
     Use `json.loads` to deserialize the result if you want to get specific fields.
     Try to use both "tavily" and "basic" providers. They might work differently for the same URL.
 
@@ -51,21 +76,12 @@ def visit_webpage(url: str, provider: Optional[str] = "basic") -> str:
     ), f"Invalid provider: {provider}. Available providers: {AVAILABLE_PROVIDERS}"
 
     if provider == "exa" and settings.EXA_API_KEY:
-        return _exa_visit_webpage(url)
+        result = _exa_visit_webpage(url)
     elif provider == "tavily" and settings.TAVILY_API_KEY:
-        return _tavily_visit_webpage(url)
+        result = _tavily_visit_webpage(url)
     else:
-        provider = "basic"
+        result = _basic_visit_webpage(url)
 
-    assert provider == "basic"
-    response = get_with_retries(url)
-    content_type = response.headers.get("content-type", "").lower()
-    if not content_type or (not content_type.startswith("text/") and "html" not in content_type):
-        if settings.EXA_API_KEY:
-            return _exa_visit_webpage(url)
-        return json.dumps(
-            {"id": url, "error": f"Unsupported content-type: {content_type or 'unknown'}"}
-        )
-    markdown_content = markdownify(response.text).strip()
-    markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
-    return sanitize_output(json.dumps({"id": url, "text": markdown_content}))
+    result["id"] = url
+    result["provider"] = provider
+    return sanitize_output(json.dumps(result, ensure_ascii=False))

{academia_mcp-1.10.3.dist-info → academia_mcp-1.10.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: academia-mcp
-Version: 1.10.3
+Version: 1.10.5
 Summary: MCP server that provides different tools to search for scientific publications
 Author-email: Ilya Gusev <phoenixilya@gmail.com>
 Project-URL: Homepage, https://github.com/IlyaGusev/academia_mcp

{academia_mcp-1.10.3.dist-info → academia_mcp-1.10.5.dist-info}/RECORD

@@ -22,11 +22,11 @@ academia_mcp/tools/review.py,sha256=Va0lFJJKuk-NvWhKS3UZ-Dnuk7CyuDQ4S1nd70D-ffE,
 academia_mcp/tools/s2.py,sha256=QX7-pbetab3Xt_1tvVPU6o5D_NAe9y6jcTGRBK1vwtY,6200
 academia_mcp/tools/show_image.py,sha256=jiJlQ53dbZ0T61OBhCT3IKVvBl9NHc6jHgWLfg5BxiE,3856
 academia_mcp/tools/speech_to_text.py,sha256=YZzMqdvunzXkpcadP_mYhm6cs4qH1Y_42SfY-7eX4O4,1601
-academia_mcp/tools/visit_webpage.py,sha256=K4L8V5KB5b4mHZBWee3OBp1C9ZFI_ge-tZ3HZXxLZwA,2742
+academia_mcp/tools/visit_webpage.py,sha256=swlFwWRzWc7-AHP2ouRZJScSTA4dHZ32fuJnA2V0lUc,3311
 academia_mcp/tools/web_search.py,sha256=0gKE3gtLBhdQ6G1eSgYLs1LIuo__PHwsYx5I5mTn254,6408
-academia_mcp-1.10.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-academia_mcp-1.10.3.dist-info/METADATA,sha256=-MGPktQMI9OjYLCGnMvGN9kZUQN8qGeGBFA1L8dWeZ4,6311
-academia_mcp-1.10.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-academia_mcp-1.10.3.dist-info/entry_points.txt,sha256=gxkiKJ74w2FwJpSECpjA3XtCfI5ZfrM6N8cqnwsq4yY,51
-academia_mcp-1.10.3.dist-info/top_level.txt,sha256=CzGpRFsRRJRqWEb1e3SUlcfGqRzOxevZGaJWrtGF8W0,13
-academia_mcp-1.10.3.dist-info/RECORD,,
+academia_mcp-1.10.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+academia_mcp-1.10.5.dist-info/METADATA,sha256=UeUZu8wGM23bbwue80r60CSinNd2lcCWoIPHXdmr0Sc,6311
+academia_mcp-1.10.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+academia_mcp-1.10.5.dist-info/entry_points.txt,sha256=gxkiKJ74w2FwJpSECpjA3XtCfI5ZfrM6N8cqnwsq4yY,51
+academia_mcp-1.10.5.dist-info/top_level.txt,sha256=CzGpRFsRRJRqWEb1e3SUlcfGqRzOxevZGaJWrtGF8W0,13
+academia_mcp-1.10.5.dist-info/RECORD,,