scrapegoat-core 1.2.0__py3-none-any.whl

scrapegoat_core/classes/node.py

@@ -0,0 +1,247 @@
+"""
+"""
+
+import uuid
+
+
+class HTMLNode:
+    """
+    """
+    VOID_TAGS = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
+
+    def __init__(self, raw: str, tag_type: str, has_data: bool = False, html_attributes: dict[str, any] = None, body: str = "", parent=None):
+        """
+        """
+        self.id = str(uuid.uuid4())
+        self.raw = raw
+        self.tag_type = tag_type
+        self.has_data = has_data
+        self.html_attributes = {"@"+k: v for k, v in (html_attributes or {}).items()}
+        self.body = body
+        self.children = []
+        self.retrieval_instructions = ""
+        self.parent = parent
+        self.extract_fields = None
+        self.extract_flags = {"ignore_children": False, "ignore_grandchildren": False}
+    
+    def to_dict(self, ignore_children=False) -> str:
+        """
+        """
+        ignore_children = self.extract_flags["ignore_children"] or ignore_children
+        for child in self.children:
+            child.set_extract_instructions(fields=self.extract_fields, ignore_children=self.extract_flags["ignore_grandchildren"])
+        dict_representation = {}
+        if self.extract_fields:
+            for field in self.extract_fields:
+                if field[0] == "@":
+                    dict_representation[field] = self.html_attributes.get(field, None)
+                else:
+                    if field == "id":
+                        dict_representation["id"] = self.id
+                    elif field == "tag_type":
+                        dict_representation["tag_type"] = self.tag_type
+                    elif field == "has_data":
+                        dict_representation["has_data"] = self.has_data
+                    elif field == "html_attributes":
+                        dict_representation["html_attributes"] = self.html_attributes
+                    elif field == "body":
+                        dict_representation["body"] = self.body
+                    elif field == "children" and not ignore_children:
+                        dict_representation["children"] = [child.to_dict() for child in self.children]
+                    elif field == "retrieval_instructions":
+                        dict_representation["retrieval_instructions"] = self.retrieval_instructions
+                    elif field == "parent":
+                        dict_representation["parent"] = self.parent.id if self.parent else None
+                    elif field == "extract_fields":
+                        dict_representation["extract_fields"] = self.extract_fields
+                    elif field == "extract_flags":
+                        dict_representation["extract_flags"] = self.extract_flags
+            return dict_representation
+        if ignore_children:
+            return {
+                "id": self.id,
+                "raw": self.raw,
+                "tag_type": self.tag_type,
+                "has_data": self.has_data,
+                "html_attributes": self.html_attributes,
+                "body": self.body,
+                "retrieval_instructions": self.retrieval_instructions,
+                "parent": self.parent.id if self.parent else None,
+                "extract_fields": self.extract_fields,
+                "extract_flags": self.extract_flags,
+            }
+        return {
+            "id": self.id,
+            "raw": self.raw,
+            "tag_type": self.tag_type,
+            "has_data": self.has_data,
+            "html_attributes": self.html_attributes,
+            "body": self.body,
+            "children": [child.to_dict() for child in self.children],
+            "retrieval_instructions": self.retrieval_instructions,
+            "parent": self.parent.id if self.parent else None,
+            "extract_fields": self.extract_fields,
+            "extract_flags": self.extract_flags,
+        }    
+
+    def to_string(self) -> str:
+        """
+        """
+        return str(self.to_dict())
+    
+    def __repr__(self):
+        """
+        """
+        return self.to_string()
+
+    def to_html(self, indent=0) -> str:
+        """
+        """
+        html_attribute_string = " ".join(f'{k}="{v}"' for k, v in self.html_attributes.items())
+        if html_attribute_string:
+            opening = f"<{self.tag_type} {html_attribute_string}"
+        else:
+            opening = f"<{self.tag_type}"
+
+        if self.tag_type in self.VOID_TAGS:
+            opening += " />"
+        else:
+            opening += ">"
+
+        text = f" {self.body}" if self.has_data else ""
+
+        pad = "  " * indent
+        result = f"{pad}{opening}{text}\n"
+
+        for child in self.children:
+            result += child.to_html(indent + 1)
+
+        if self.tag_type not in self.VOID_TAGS:
+            result += f"{pad}</{self.tag_type}>\n"
+        return result
+
+    def __str__(self):
+        return self.to_string()
+    
+    def get_parent(self):
+        """
+        """
+        return self.parent
+    
+    def get_children(self):
+        """
+        """
+        return self.children
+    
+    def get_ancestors(self):
+        """
+        """
+        ancestors = []
+        current = self.parent
+        while current:
+            ancestors.append(current)
+            current = current.parent
+        return ancestors
+    
+    def get_descendants(self, tag_type: str = None, **html_attributes) -> list:
+        """
+        """
+        descendants = []
+        for child in self.children:
+            if (tag_type is None or child.tag_type == tag_type) and all(child.html_attributes.get(k) == v for k, v in html_attributes.items()):
+                descendants.append(child)
+            descendants.extend(child.get_descendants(tag_type, **html_attributes))
+        return descendants
+    
+    def preorder_traversal(self):
+        """
+        """
+        yield self
+        for child in self.children:
+            yield from child.preorder_traversal()
+
+    def has_html_attribute(self, key, value=None) -> bool:
+        """
+        """
+        if value is None:
+            return key in self.html_attributes
+        if self.html_attributes.get(key) is None:
+            return False
+        return value in self.html_attributes.get(key)
+    
+    def has_attribute(self, key, value=None) -> bool:
+        """
+        """
+        if key == "tag_type":
+            if value is None:
+                return self.tag_type is not None
+            return self.tag_type == value
+        if key == "id":
+            if value is None:
+                return self.id is not None
+            return str(self.id) == value
+        if key == "has_data":
+            if value is None:
+                return self.has_data
+            return self.has_data == value
+        if key == "body":
+            if value is None:
+                return self.body is not None
+            return self.body == value
+        if key == "retrieval_instructions":
+            if value is None:
+                return self.retrieval_instructions is not None
+            return self.retrieval_instructions == value
+        if key == "extract_fields":
+            if value is None:
+                return self.extract_fields is not None
+            return self.extract_fields == value
+        if key == "extract_flags":
+            if value is None:
+                return self.extract_flags is not None
+            return self.extract_flags == value
+        if key == "parent":
+            if value is None:
+                return self.parent is not None
+            return self.parent and str(self.parent.id) == value
+        if key == "children":
+            if value is None:
+                return len(self.children) > 0
+            return any(str(child.id) == value for child in self.children)
+        if key == "raw":
+            if value is None:
+                return self.raw is not None
+            return self.raw == value
+        return False
+    
+    def is_descendant_of(self, tag_type) -> bool:
+        """
+        """
+        return any(ancestor.tag_type == tag_type for ancestor in self.get_ancestors())
+    
+    def set_retrieval_instructions(self, instruction: str):
+        """
+        """
+        self.retrieval_instructions = instruction
+
+    def set_extract_instructions(self, fields: list=None, ignore_children=False, ignore_grandchildren=False):
+        """
+        """
+        self.extract_fields = fields or None
+        self.extract_flags = {"ignore_children": ignore_children, "ignore_grandchildren": ignore_grandchildren}
+
+    def clear_extract_instructions(self):
+        """
+        """
+        self.extract_fields = None
+        self.extract_flags = None
+
+
+def main():
+    """
+    """
+    pass
+
+
+if __name__ == "__main__":
+    main()

scrapegoat_core/classes/sheepdog.py

@@ -0,0 +1,81 @@
+"""
+"""
+
+from typing import Union
+import requests
+
+from .command import FetchCommand
+
+
+class Sheepdog:
+    """
+    """
+    DEFAULT_HEADERS = {
+        "User-Agent": "Mozilla/5.0 (Scrapegoat)",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Accept-Encoding": "gzip, deflate, br",
+        "Connection": "keep-alive",
+        "Accept": "*/*",
+        "DNT": "1",
+        "Upgrade-Insecure-Requests": "1",
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+    }
+
+    def __init__(self, getter=None):
+        """
+        """
+        self.getter = getter or self.getter
+
+    def fetch(self, fetch_command: Union[str, FetchCommand]) -> str:
+        """
+        """
+        if not isinstance(fetch_command, FetchCommand):
+            fetch_command = FetchCommand(fetch_command)
+        fetch_command.set_getter(self.getter)
+        return fetch_command.execute()
+    
+    def getter(self, url: str, **kwargs) -> str:
+        """
+        """
+        headers = kwargs.pop('headers', self.DEFAULT_HEADERS)
+        response = requests.get(url, headers=headers, **kwargs)
+        response.raise_for_status()
+        return response.text
+
+
+class HeadlessSheepdog(Sheepdog):
+    """
+    """
+    def __init__(self, getter=None):
+        """
+        """
+        super().__init__(getter)
+
+    def getter(self, url: str, **kwargs):
+        """
+        """
+        try:
+            from playwright.sync_api import sync_playwright
+        except ImportError:
+            raise RuntimeError("Playwright is not installed. Please install it with 'pip install playwright'")
+
+        try:
+            with sync_playwright() as p:
+                browser = p.chromium.launch(headless=True)
+                page = browser.new_page()
+                page.goto(url, wait_until="domcontentloaded")
+                return page.content()
+        except Exception as e:
+            if "Executable doesn't exist" in str(e):
+                raise RuntimeError("Playwright browser executables are not installed. Please run 'playwright install' to install them.")
+
+
+def main():
+    """
+    """
+    pass
+
+
+if __name__ == "__main__":
+    main()

scrapegoat_core/classes/shepherd.py

@@ -0,0 +1,108 @@
+"""
+"""
+
+import os
+
+from .gardener import Gardener
+from .goat import Goat
+from .interpreter import Interpeter
+from .milkmaid import Milkmaid
+from .milkman import Milkman
+from .sheepdog import Sheepdog
+
+
+class Shepherd:
+    """
+    """
+    def __init__(self, gardener=None, sheepdog=None, goat=None, milkmaid=None, milkman=None):
+        """
+        """
+        self.gardener = gardener if gardener else Gardener()
+        self.interpreter = Interpeter()
+        self.sheepdog = sheepdog if sheepdog else Sheepdog()
+        self.goat = goat if goat else Goat()
+        self.milkmaid = milkmaid if milkmaid else Milkmaid()
+        self.milkman = milkman if milkman else Milkman()
+    
+    def herd(self, query: str) -> list:
+        """
+        """
+        goatspeak = self._convert_query_to_goatspeak(query)
+
+        results = []
+
+        for block in goatspeak:
+            html = self.sheepdog.fetch(block.fetch_command)
+            root = self.gardener.grow_tree(html)
+            self._query_list_handler(block.query_list, root, results)
+
+        return list(dict.fromkeys(results))
+    
+    def _convert_query_to_goatspeak(self, query: str) -> None:
+        """
+        """
+        if os.path.isfile(query):
+            try:
+                return self.interpreter.interpret(self.milkman.receive(query))
+            except Exception as e:
+                raise e
+        try:
+            return self.interpreter.interpret(query)
+        except Exception as e:
+            raise e
+            
+            
+
+    def _query_list_handler(self, query_list: str, root, results) -> list:
+        """
+        """
+        for query in query_list:
+            query_results = (self.goat.feast(root, query.graze_commands))
+            if query.churn_command:
+                self.milkmaid.churn(query_results, query.churn_command)
+
+            results.extend(query_results)
+
+            if query.deliver_command:
+                self.milkman.deliver(results, query.deliver_command)
+                results.clear()
+        return
+        
+    def _local_herd(self, query: str, root) -> list:
+        """
+        """
+        goatspeak = self._convert_query_to_goatspeak(query)
+
+        results = []
+
+        for block in goatspeak:
+            self._query_list_handler(block.query_list, root, results)
+                
+        return list(dict.fromkeys(results))
+    
+    def herd_from_node(self, query: str, root) -> list:
+        """
+        """
+        return self._local_herd(query, root=root)
+    
+    def herd_from_html(self, query: str, html: str) -> list:
+        """
+        """
+        root = self.gardener.grow_tree(html)
+        return self._local_herd(query, root=root)
+    
+    def herd_from_url(self, query: str, url: str) -> list:
+        """
+        """
+        html = self.sheepdog.fetch(url)
+        root = self.gardener.grow_tree(html)
+        return self._local_herd(query, root=root)
+
+def main():
+    """
+    """
+    pass
+
+
+if __name__ == "__main__":
+    main()

scrapegoat_core/cli.py

@@ -0,0 +1,38 @@
+import argparse
+from scrapegoat_core import Shepherd, HeadlessSheepdog
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Scrapegoat language executor")
+
+    # Positional file or query arg
+    parser.add_argument(
+        "file_or_query",
+        nargs="?",
+        help="Path to a .goat file or a raw query as a string",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Prints the results of the query to the console",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-j",
+        "--javascript",
+        help="Uses a headless browser to support javascript rendered pages",
+        action="store_true",
+    )
+
+    args = parser.parse_args()
+
+    if args.javascript:
+        shepherd = Shepherd(sheepdog=HeadlessSheepdog())
+    else:
+        shepherd = Shepherd()
+
+    nodes = shepherd.herd(args.file_or_query)
+
+    if args.verbose:
+        for node in nodes:
+            print(node)

scrapegoat_core-1.2.0.dist-info/METADATA

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: scrapegoat-core
+Version: 1.2.0
+Summary: A toolkit of functions, classes, and utilities for creating web scrapers.
+Author-email: Arman Chinai <chinaiarman@gmail.com>, Lucas Angelozzi <lucasangelozzi32@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/ChinaiArman/scrapegoat
+Project-URL: Documentation, https://github.com/ChinaiArman/scrapegoat/blob/main/README.md
+Project-URL: Source, https://github.com/ChinaiArman/scrapegoat
+Project-URL: Issues, https://github.com/ChinaiArman/scrapegoat/issues
+Keywords: scraping,webscraping,automation,html,parsing
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Provides-Extra: loom
+Requires-Dist: scrapegoat-loom>=1.2.0; extra == "loom"
+Provides-Extra: js
+Requires-Dist: playwright>=1.56.0; extra == "js"
+Dynamic: license-file
+
+# Scrapegoat SDK

scrapegoat_core-1.2.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+scrapegoat_core/__init__.py,sha256=rdVu3-8fAdVq_kxRqx-XRU4EQgLljk6zw00YksCSn2s,761
+scrapegoat_core/cli.py,sha256=MQtNOlN5waUgUlGLaDp1gG5lxeCvmIa_v7ydIrLSyjQ,986
+scrapegoat_core/main.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scrapegoat_core/classes/__init__.py,sha256=r6-e-57CW9DwSxsMzZnE4ujpv9xymQqwad_0_tnTG7A,982
+scrapegoat_core/classes/block.py,sha256=DAtYQSp3_rg6_PVDZHxxsq7dNiadaPZzwS1P2VUnn2U,1131
+scrapegoat_core/classes/command.py,sha256=7EnybkN5SxDV98J9KDZ93DNmC7Kz8PkxjNSqvpXaLwM,5726
+scrapegoat_core/classes/conditions.py,sha256=YxPqHsUhyk9vdM1nv95ErWbq0FhHbfomQC64F4fkVUU,2541
+scrapegoat_core/classes/gardener.py,sha256=jSOI2Mt8s_tVsRG8qE694v2qdwSqqtUQOlcPnd0tXZY,4373
+scrapegoat_core/classes/goat.py,sha256=ERBfSR7mK9IFYR4i0m73gubHU8nlj54wg2FzC8K3HOc,778
+scrapegoat_core/classes/interpreter.py,sha256=jWIL_ZG2ssc6oLfOlYQJO3V6GKHQbdmXfJtY85wp1gU,15161
+scrapegoat_core/classes/milkmaid.py,sha256=idjDLTicQ5XDfXjR-D8qVCEDDdsn-_XWalKsNYgLVA8,376
+scrapegoat_core/classes/milkman.py,sha256=dmC0zBXYr6LKtXZszVjWOGlWFUMqdoZp6Ab_xOSyXdA,510
+scrapegoat_core/classes/node.py,sha256=PzkneKBAuRlQ8Mm7xkLtw7sryhIv-6ng0BnP4BE9NRY,8919
+scrapegoat_core/classes/sheepdog.py,sha256=ni6Tdsf4dQbNWe69r03tlxCmUzLWHnRWoXZ6rcQuoCM,2219
+scrapegoat_core/classes/shepherd.py,sha256=jEEj5kGSAKvwfeU9mmr3LRSiK6GGBSm_RIwsI1AzBXk,3084
+scrapegoat_core/exceptions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scrapegoat_core-1.2.0.dist-info/licenses/LICENSE,sha256=vXZ_divzwGQpeaVHa8qH-DADn9CG-XKN_j5sBUio1B0,1090
+scrapegoat_core-1.2.0.dist-info/METADATA,sha256=zruVc6uG60UsC-ylbKiYaU6SFXyO6S4SmztjJcHcrk8,919
+scrapegoat_core-1.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scrapegoat_core-1.2.0.dist-info/entry_points.txt,sha256=2iISE8Mxs-d-RBU6pn8DjQs-vJ_l1OmfkvQM2VMw4fk,56
+scrapegoat_core-1.2.0.dist-info/top_level.txt,sha256=qP4DpWo4p3IWss3TcRlM10DCTMsHtN--W_KZqyg8TEM,16
+scrapegoat_core-1.2.0.dist-info/RECORD,,

scrapegoat_core-1.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scrapegoat_core-1.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+scrapegoat = scrapegoat_core.cli:main

scrapegoat_core-1.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Arman Chinai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scrapegoat_core-1.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scrapegoat_core