langroid 0.50.6__py3-none-any.whl → 0.50.7__py3-none-any.whl

langroid/parsing/web_search.py

@@ -55,8 +55,7 @@ class WebSearchResult:
         try:
             # First check headers only to get content length and type
             head_response: Response = requests.head(self.link, timeout=5)
-            if head_response.status_code != 200:
-                return f"Error: HTTP {head_response.status_code} for {self.link}"
+            content_type = head_response.headers.get("content-type", "").lower()
 
             # Skip large files
             content_length = int(head_response.headers.get("content-length", 0))
@@ -64,8 +63,16 @@ class WebSearchResult:
                 return (
                     f"Error: Content too large ({content_length} bytes) for {self.link}"
                 )
+            # Skip non-HTML content types
+            if content_type and not any(
+                html_type in content_type
+                for html_type in ["text/html", "application/xhtml", "text/plain"]
+            ):
+                return f"Skipping Content type '{content_type}' " f"in {self.link}"
 
             response: Response = requests.get(self.link, timeout=10)
+            if response.status_code != 200:
+                return f"Error: HTTP {response.status_code} for {self.link}"
 
             import warnings
 

{langroid-0.50.6.dist-info → langroid-0.50.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.50.6
+Version: 0.50.7
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT

{langroid-0.50.6.dist-info → langroid-0.50.7.dist-info}/RECORD

@@ -96,7 +96,7 @@ langroid/parsing/table_loader.py,sha256=qNM4obT_0Y4tjrxNBCNUYjKQ9oETCZ7FbolKBTcz
 langroid/parsing/url_loader.py,sha256=NQuCxa-hTOuxLZDq4xKLvPfGVB4IWFzh2ItqWq297DI,15675
 langroid/parsing/urls.py,sha256=Tjzr64YsCusiYkY0LEGB5-rSuX8T2P_4DVoOFKAeKuI,8081
 langroid/parsing/utils.py,sha256=WwqzOhbQRlorbVvddDIZKv9b1KqZCBDm955lgIHDXRw,12828
-langroid/parsing/web_search.py,sha256=sARV1Tku4wiInhuCz0kRaMHcoF6Ok6CLu7vapLS8hjs,8222
+langroid/parsing/web_search.py,sha256=atk8wIpOfiGTvW8yL_26TvjvyY2zD24xIHIi0QjEklI,8599
 langroid/prompts/__init__.py,sha256=RW11vK6jiLPuaUh4GpeFvstti73gkm8_rDMtrbo2YsU,142
 langroid/prompts/dialog.py,sha256=SpfiSyofSgy2pwD1YboHR_yHO3LEEMbv6j2sm874jKo,331
 langroid/prompts/prompts_config.py,sha256=p_lp9nbMuQwhhMwAZsOxveRw9C0ZFZvql7pdIfgVZYo,143
@@ -129,7 +129,7 @@ langroid/vector_store/pineconedb.py,sha256=otxXZNaBKb9f_H75HTaU3lMHiaR2NUp5MqwLZ
 langroid/vector_store/postgres.py,sha256=wHPtIi2qM4fhO4pMQr95pz1ZCe7dTb2hxl4VYspGZoA,16104
 langroid/vector_store/qdrantdb.py,sha256=O6dSBoDZ0jzfeVBd7LLvsXu083xs2fxXtPa9gGX3JX4,18443
 langroid/vector_store/weaviatedb.py,sha256=Yn8pg139gOy3zkaPfoTbMXEEBCiLiYa1MU5d_3UA1K4,11847
-langroid-0.50.6.dist-info/METADATA,sha256=JsIZE0WSSPm0jpB-5UX2Bfdvk6-txTtUo4J49ryyQIw,63641
-langroid-0.50.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.50.6.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.50.6.dist-info/RECORD,,
+langroid-0.50.7.dist-info/METADATA,sha256=Am8GortMmvBBtL6aMTS88xmzYCa3ywMFEgraPvNYsTQ,63641
+langroid-0.50.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.50.7.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.50.7.dist-info/RECORD,,