nosible 0.1.8__py3-none-any.whl → 0.2.1__py3-none-any.whl

nosible/classes/snippet_set.py

@@ -1,9 +1,13 @@
-from typing import Iterator
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass, field
 
 from nosible.classes.snippet import Snippet
 from nosible.utils.json_tools import json_dumps
 
 
+@dataclass()
 class SnippetSet(Iterator[Snippet]):
     """
     An iterator and container for a collection of Snippet objects.
@@ -19,64 +23,50 @@ class SnippetSet(Iterator[Snippet]):
     Examples
     --------
     >>> snippets_data = {
-    ...     "hash1": {"content": "Example snippet", "snippet_hash": "hash1"}
+    ...     "hash1": {"content": "Example snippet", "snippet_hash": "hash1"},
+    ...     "hash2": {"content": "Another snippet", "snippet_hash": "hash2"},
     ... }
-    >>> snippets = SnippetSet(snippets_data)
+    >>> snippets = SnippetSet().from_dict(snippets_data)
     >>> for snippet in snippets:
     ...     print(snippet.content)
     Example snippet
+    Another snippet
     """
 
-    def __init__(self, snippets: dict) -> None:
-        self._snippets = []
-
-        for key, value in snippets.items():
-            self._snippets.append(
-                Snippet(
-                    companies=value.get("companies", []),
-                    content=value.get("content", ""),
-                    images=value.get("images", []),
-                    language=value.get("language", ""),
-                    next_snippet_hash=value.get("next_snippet_hash", ""),
-                    prev_snippet_hash=value.get("prev_snippet_hash", ""),
-                    snippet_hash=key,
-                    statistics=value.get("statistics", {}),
-                    url_hash=value.get("url_hash", ""),
-                    words=value.get("words", ""),
-                )
-            )
-
-        self._index = 0
-
-    def __iter__(self):
-        """
-        Initialize the iterator.
+    snippets: list[Snippet] = field(default_factory=list)
+    """ List of `Snippet` objects contained in this ResultSet."""
+    _index: int = field(default=0, init=False, repr=False, compare=False)
+    """ Internal index for iteration over snippets."""
+
+    def __iter__(self) -> SnippetSet:
+        """
+        Reset iteration and return self.
+
         Returns
         -------
-        SnippetSet
-            The iterator itself.
+        ResultSet
+            Iterator over the ResultSet instance.
         """
-        self._index = 0
+        object.__setattr__(self, "_index", 0)
         return self
 
     def __next__(self) -> Snippet:
         """
-        Returns the next Snippet object from the collection.
+        Returns the next Result in the sequence.
 
         Returns
         -------
-        Snippet
-            The next snippet in the sequence.
-
+        Result
+            The next Result object in the sequence.
         Raises
         ------
         StopIteration
-            If there are no more snippets to return.
+            If the end of the sequence is reached.
         """
-        if self._index < len(self._snippets):
-            snippet = self._snippets[self._index]
-            self._index += 1
-            return snippet
+        if self._index < len(self.snippets):
+            item = self.snippets[self._index]
+            object.__setattr__(self, "_index", self._index + 1)
+            return item
         raise StopIteration
 
     def __len__(self) -> int:
@@ -88,7 +78,7 @@ class SnippetSet(Iterator[Snippet]):
         int
             The number of snippets.
         """
-        return len(self._snippets)
+        return len(self.snippets)
 
     def __getitem__(self, index: int) -> Snippet:
         """
@@ -109,9 +99,9 @@ class SnippetSet(Iterator[Snippet]):
         IndexError
             If the index is out of range.
         """
-        if index < 0 or index >= len(self._snippets):
-            raise IndexError("Index out of range.")
-        return self._snippets[index]
+        if 0 <= index < len(self.snippets):
+            return self.snippets[index]
+        raise IndexError(f"Index {index} out of range for SnippetSet of length {len(self.snippets)}.")
 
     def __str__(self):
         """
@@ -122,17 +112,6 @@ class SnippetSet(Iterator[Snippet]):
         """
         return "\n".join(str(s) for s in self)
 
-    def __repr__(self):
-        """
-        Returns a string representation of the SnippetSet object.
-
-        Returns
-        -------
-        str
-            A string representation of the SnippetSet.
-        """
-        return f"SnippetSet(snippets={len(self._snippets)})"
-
     def to_dict(self) -> dict:
         """
         Convert the SnippetSet to a dictionary representation.
@@ -144,15 +123,13 @@ class SnippetSet(Iterator[Snippet]):
 
         Examples
         --------
-        >>> snippets_data = {
-        ...     "hash1": {"content": "Example snippet", "snippet_hash": "hash1"}
-        ... }
-        >>> snippets = SnippetSet(snippets_data)
+        >>> snippets_data = {"hash1": {"content": "Example snippet", "snippet_hash": "hash1"}}
+        >>> snippets = SnippetSet().from_dict(snippets_data)
         >>> snippets_dict = snippets.to_dict()
         >>> isinstance(snippets_dict, dict)
         True
         """
-        return {s.snippet_hash: s.to_dict() for s in self._snippets} if self._snippets else {}
+        return {s.snippet_hash: s.to_dict() for s in self.snippets} if self.snippets else {}
 
     def to_json(self) -> str:
         """
@@ -165,12 +142,34 @@ class SnippetSet(Iterator[Snippet]):
 
         Examples
         --------
-        >>> snippets_data = {
-        ...     "hash1": {"content": "Example snippet", "snippet_hash": "hash1"}
-        ... }
-        >>> snippets = SnippetSet(snippets_data)
+        >>> snippets_data = {"hash1": {"content": "Example snippet", "snippet_hash": "hash1"}}
+        >>> snippets = SnippetSet().from_dict(snippets_data)
         >>> json_str = snippets.to_json()
         >>> isinstance(json_str, str)
         True
         """
         return json_dumps(self.to_dict())
+
+    @classmethod
+    def from_dict(cls, data: dict) -> SnippetSet:
+        """
+        Create a SnippetSet instance from a dictionary.
+
+        Parameters
+        ----------
+        data : dict
+            Dictionary containing snippet data.
+
+        Returns
+        -------
+        SnippetSet
+            An instance of SnippetSet populated with the provided data.
+
+        Examples
+        --------
+        >>> snippets_data = {"hash1": {"content": "Example snippet", "snippet_hash": "hash1"}}
+        >>> snippets = SnippetSet.from_dict(snippets_data)
+        >>> isinstance(snippets, SnippetSet)
+        True
+        """
+        return cls([Snippet.from_dict(s) for s in data.values()])

nosible/classes/web_page.py

@@ -1,7 +1,10 @@
+from dataclasses import asdict, dataclass, field
+
 from nosible.classes.snippet_set import SnippetSet
 from nosible.utils.json_tools import json_dumps, json_loads
 
 
+@dataclass(init=True, repr=True, eq=True, frozen=True)
 class WebPageData:
     """
     A data container for all extracted and processed information about a web page.
@@ -36,64 +39,26 @@ class WebPageData:
     {'description': 'Example'}
     """
 
-    def __init__(
-        self,
-        *,
-        companies: list = None,
-        full_text: str = None,
-        languages: dict = None,
-        metadata: dict = None,
-        page: dict = None,
-        request: dict = None,
-        snippets: dict = None,
-        statistics: dict = None,
-        structured: list = None,
-        url_tree: dict = None,
-    ):
-        """
-        Initialize a WebPageData instance.
-
-        Parameters
-        ----------
-        companies : list, optional
-            A list of companies mentioned in the webpage, if applicable. (GKIDS)
-        full_text : str, optional
-            The full text content of the webpage.
-        languages : dict, optional
-            Detected languages and their probabilities or counts.
-        metadata : dict, optional
-            Metadata extracted from the webpage (e.g., description, keywords).
-        page : dict, optional
-            Page-specific details such as title, canonical URL, etc.
-        request : dict, optional
-            Information about the HTTP request/response.
-        snippets : list, optional
-            Extracted text snippets or highlights from the page.
-        statistics : dict, optional
-            Statistical information about the page (e.g., word count).
-        structured : list, optional
-            Structured data (e.g., schema.org, OpenGraph).
-        url_tree : dict, optional
-            Hierarchical representation of the URL structure.
-
-        Examples
-        --------
-        >>> data = WebPageData(full_text="Example Domain", languages={"en": 1})
-        >>> data.languages
-        {'en': 1}
-        """
-        self.companies = companies or []
-        if snippets is None:
-            snippets = {}
-        self.full_text = full_text
-        self.languages = languages or {}
-        self.metadata = metadata or {}
-        self.page = page or {}
-        self.request = request or {}
-        self.snippets = SnippetSet(snippets)
-        self.statistics = statistics or {}
-        self.structured = structured or []
-        self.url_tree = url_tree or {}
+    companies: list = None
+    """A list of companies mentioned in the webpage, if applicable. (GKIDS)"""
+    full_text: str = None
+    """The full text content of the webpage."""
+    languages: dict = None
+    """Detected languages and their probabilities or counts."""
+    metadata: dict = None
+    """Metadata extracted from the webpage (e.g., description, keywords)."""
+    page: dict = None
+    """Page-specific details such as title, canonical URL, etc."""
+    request: dict = None
+    """Information about the HTTP request/response."""
+    snippets: SnippetSet = field(init=True, default_factory=SnippetSet)
+    """Extracted text snippets or highlights from the page."""
+    statistics: dict = None
+    """Statistical information about the page (e.g., word count)."""
+    structured: list = None
+    """Structured data (e.g., schema.org, OpenGraph)."""
+    url_tree: dict = None
+    """Hierarchical representation of the URL structure."""
 
     def __str__(self):
         """Return a string representation of the WebPageData.
@@ -109,24 +74,6 @@ class WebPageData:
             f"statistics={self.statistics}, structured={self.structured}, url_tree={self.url_tree})"
         )
 
-    def __repr__(self):
-        """
-        Return a JSON-formatted string representation of the WebPageData instance.
-
-        Returns
-        -------
-        str
-            JSON string representing the WebPageData for easy readability and debugging.
-
-        Examples
-        --------
-        >>> data = WebPageData(languages={"en": 1}, metadata={"description": "Example"})
-        >>> repr_str = repr(data)
-        >>> isinstance(repr_str, str)
-        True
-        """
-        return json_dumps(self.to_dict())
-
     def to_dict(self) -> dict:
         """
         Convert the WebPageData instance to a dictionary.
@@ -145,18 +92,10 @@ class WebPageData:
         >>> d["languages"] == {"en": 1}
         True
         """
-        return {
-            "companies": self.companies,
-            "full_text": self.full_text,
-            "languages": self.languages,
-            "metadata": self.metadata,
-            "page": self.page,
-            "request": self.request,
-            "snippets": self.snippets.to_dict(),
-            "statistics": self.statistics,
-            "structured": self.structured,
-            "url_tree": self.url_tree,
-        }
+        data = asdict(self)
+        # snippets is still a SnippetSet instance, so convert it:
+        data["snippets"] = self.snippets.to_dict()
+        return data
 
     def to_json(self) -> str:
         """
@@ -225,19 +164,12 @@ class WebPageData:
         >>> webpage_data.languages
         {'en': 1}
         """
-        parsed_data = json_loads(data)
-        return cls(
-            companies=parsed_data.get("companies", []),
-            full_text=parsed_data.get("full_text"),
-            languages=parsed_data.get("languages"),
-            metadata=parsed_data.get("metadata"),
-            page=parsed_data.get("page"),
-            request=parsed_data.get("request"),
-            snippets=parsed_data.get("snippets", {}),
-            statistics=parsed_data.get("statistics"),
-            structured=parsed_data.get("structured"),
-            url_tree=parsed_data.get("url_tree"),
-        )
+        data_dict = json_loads(data)
+        # Handle snippets separately to avoid passing it twice
+        snippets_data = data_dict.pop("snippets", None)
+        if snippets_data is not None:
+            data_dict["snippets"] = SnippetSet.from_dict(snippets_data)
+        return cls(**data_dict)
 
     @classmethod
     def load(cls, path: str) -> "WebPageData":
@@ -265,5 +197,9 @@ class WebPageData:
         {'en': 1}
         """
         with open(path, encoding="utf-8") as f:
-            data = f.read()
-        return cls.from_json(data)
+            data = json_loads(f.read())
+        # Handle snippets separately to avoid passing it twice
+        snippets_data = data.pop("snippets", None)
+        if snippets_data is not None:
+            data["snippets"] = SnippetSet.from_dict(snippets_data)
+        return cls(**data)