langroid 0.48.0__tar.gz → 0.48.2__tar.gz

{langroid-0.48.0 → langroid-0.48.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.48.0
+Version: 0.48.2
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT

{langroid-0.48.0 → langroid-0.48.2}/langroid/mytypes.py

@@ -3,7 +3,7 @@ from textwrap import dedent
 from typing import Any, Callable, Dict, List, Union
 from uuid import uuid4
 
-from langroid.pydantic_v1 import BaseModel, Extra, Field
+from langroid.pydantic_v1 import BaseModel, Extra, Field, validator
 
 Number = Union[int, float]
 Embedding = List[Number]
@@ -45,10 +45,19 @@ class DocMetaData(BaseModel):
 
     source: str = "context"  # just reference
     source_content: str = "context"  # reference and content
+    title: str = "Unknown Title"
+    published_date: str = "Unknown Date"
     is_chunk: bool = False  # if it is a chunk, don't split
     id: str = Field(default_factory=lambda: str(uuid4()))
     window_ids: List[str] = []  # for RAG: ids of chunks around this one
 
+    @validator("source", "source_content", "id", "title", "published_date")
+    def ensure_not_empty(cls, v: str) -> str:
+        """Ensure required string fields are not empty."""
+        if not v:
+            raise ValueError("Field cannot be empty")
+        return v
+
     def dict_bool_int(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
         """
         Special dict method to convert bool fields to int, to appease some

{langroid-0.48.0 → langroid-0.48.2}/langroid/parsing/url_loader.py

@@ -258,7 +258,13 @@ class FirecrawlCrawler(BaseCrawler):
                     with open(filename, "w") as f:
                         f.write(content)
                     docs.append(
-                        Document(content=content, metadata=DocMetaData(source=url))
+                        Document(
+                            content=content,
+                            metadata=DocMetaData(
+                                source=url,
+                                title=page["metadata"].get("title", "Unknown Title"),
+                            ),
+                        )
                     )
                     processed_urls.add(url)
                     new_pages += 1
@@ -300,7 +306,10 @@ class FirecrawlCrawler(BaseCrawler):
                         docs.append(
                             Document(
                                 content=result["markdown"],
-                                metadata=DocMetaData(source=url),
+                                metadata=DocMetaData(
+                                    source=url,
+                                    title=metadata.get("title", "Unknown Title"),
+                                ),
                             )
                         )
                 except Exception as e:
@@ -336,7 +345,7 @@ class ExaCrawler(BaseCrawler):
 
     @property
     def needs_parser(self) -> bool:
-        return False
+        return True
 
     def crawl(self, urls: List[str]) -> List[Document]:
         """Crawl the given URLs using Exa SDK.
@@ -363,12 +372,29 @@ class ExaCrawler(BaseCrawler):
         docs = []
 
         try:
-            results = exa.get_contents(urls, text=True)
-
-            for result in results.results:
-                if result.text:
-                    metadata = DocMetaData(source=result.url)
-                    docs.append(Document(content=result.text, metadata=metadata))
+            for url in urls:
+                parsed_doc_chunks = self._process_document(url)
+                if parsed_doc_chunks:
+                    docs.extend(parsed_doc_chunks)
+                    continue
+                else:
+                    results = exa.get_contents([url], livecrawl="always", text=True)
+                    result = results.results[0]
+                    if result.text:
+                        # append a NON-chunked document
+                        # (metadata.is_chunk = False, so will be chunked downstream)
+                        docs.append(
+                            Document(
+                                content=result.text,
+                                metadata=DocMetaData(
+                                    source=url,
+                                    title=getattr(result, "title", "Unknown Title"),
+                                    published_date=getattr(
+                                        result, "published_date", "Unknown Date"
+                                    ),
+                                ),
+                            )
+                        )
 
         except Exception as e:
             logging.error(f"Error retrieving content from Exa API: {e}")
@@ -399,6 +425,8 @@ class URLLoader:
             crawler_config = TrafilaturaConfig(parser=Parser(parsing_config))
 
         self.crawler = CrawlerFactory.create_crawler(crawler_config)
+        if self.crawler.needs_parser:
+            self.crawler.parser = Parser(parsing_config)
 
     def load(self) -> List[Document]:
         """Load the URLs using the specified crawler."""

{langroid-0.48.0 → langroid-0.48.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langroid"
-version = "0.48.0"
+version = "0.48.2"
 authors = [
     {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
 ]