PyPI - osbot-utils - Versions diffs - 2.76.0__tar.gz → 2.77.0__tar.gz - Mend

osbot-utils 2.76.0tar.gz → 2.77.0tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (479) hide show

{osbot_utils-2.76.0 → osbot_utils-2.77.0}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: osbot_utils
-Version: 2.76.0
+Version: 2.77.0
 Summary: OWASP Security Bot - Utils
 License: MIT
 Author: Dinis Cruz
@@ -21,7 +21,7 @@ Description-Content-Type: text/markdown
 # OSBot-Utils
-![Current Release](https://img.shields.io/badge/release-v2.76.0-blue)
+![Current Release](https://img.shields.io/badge/release-v2.77.0-blue)
 ![Python](https://img.shields.io/badge/python-3.8+-green)
 ![Type-Safe](https://img.shields.io/badge/Type--Safe-✓-brightgreen)
 ![Caching](https://img.shields.io/badge/Caching-Built--In-orange)
@@ -93,7 +93,7 @@ fetcher.fetch(10)  # Returns cached result
 ### HTML Parsing
 ```python
-from osbot_utils.helpers.html.Html__To__Html_Dict import html_to_dict
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict import html_to_dict
 html_code = "<html><body><h1>Hello</h1></body></html>"
 print(html_to_dict(html_code))

{osbot_utils-2.76.0 → osbot_utils-2.77.0}/README.md RENAMED Viewed

@@ -1,6 +1,6 @@
 # OSBot-Utils
-![Current Release](https://img.shields.io/badge/release-v2.76.0-blue)
+![Current Release](https://img.shields.io/badge/release-v2.77.0-blue)
 ![Python](https://img.shields.io/badge/python-3.8+-green)
 ![Type-Safe](https://img.shields.io/badge/Type--Safe-✓-brightgreen)
 ![Caching](https://img.shields.io/badge/Caching-Built--In-orange)
@@ -72,7 +72,7 @@ fetcher.fetch(10)  # Returns cached result
 ### HTML Parsing
 ```python
-from osbot_utils.helpers.html.Html__To__Html_Dict import html_to_dict
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict import html_to_dict
 html_code = "<html><body><h1>Hello</h1></body></html>"
 print(html_to_dict(html_code))

{osbot_utils-2.76.0 → osbot_utils-2.77.0}/osbot_utils/helpers/ast/Ast_Load.py RENAMED Viewed

@@ -1,11 +1,11 @@
 import ast
-from osbot_utils.helpers.Type_Registry import type_registry
-from osbot_utils.utils.Files import file_contents, file_exists
-from osbot_utils.utils.Functions import python_file
-from osbot_utils.helpers.ast.Ast_Node import Ast_Node
+from osbot_utils.helpers.Type_Registry  import type_registry
+from osbot_utils.utils.Files            import file_contents, file_exists
+from osbot_utils.utils.Functions        import python_file
+from osbot_utils.helpers.ast.Ast_Node   import Ast_Node
-#todo: check if this is stll needed since it Ast_Visit does it similar
+#todo: check if this is still needed since it Ast_Visit does it similar
 class Ast_Load(ast.NodeVisitor):
     def __init__(self):

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/tags}/Tag__Base.py RENAMED Viewed

@@ -1,6 +1,6 @@
 from collections                                import defaultdict
-from osbot_utils.helpers.html.Html__To__Html_Dict import STRING__SCHEMA_TEXT
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict import STRING__SCHEMA_TEXT
 from osbot_utils.type_safe.Type_Safe            import Type_Safe
 from osbot_utils.utils.Files                    import file_create

osbot_utils-2.77.0/osbot_utils/helpers/html/tags/Tag__Body.py ADDED Viewed

@@ -0,0 +1,5 @@
+from osbot_utils.helpers.html.tags.Tag__Base import Tag__Base
+class Tag__Body(Tag__Base):
+    tag_name = 'body'

osbot_utils-2.77.0/osbot_utils/helpers/html/tags/Tag__Div.py ADDED Viewed

@@ -0,0 +1,5 @@
+from osbot_utils.helpers.html.tags.Tag__Base import Tag__Base
+class Tag__Div(Tag__Base):
+    tag_name    = 'div'

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/tags}/Tag__H.py RENAMED Viewed

@@ -1,4 +1,4 @@
-from osbot_utils.helpers.html.Tag__Base import Tag__Base
+from osbot_utils.helpers.html.tags.Tag__Base import Tag__Base
 class Tag__H(Tag__Base):

osbot_utils-2.77.0/osbot_utils/helpers/html/tags/Tag__HR.py ADDED Viewed

@@ -0,0 +1,5 @@
+from osbot_utils.helpers.html.tags.Tag__Base import Tag__Base
+class Tag__HR(Tag__Base):
+    tag_name    = 'hr'

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/tags}/Tag__Head.py RENAMED Viewed

@@ -1,8 +1,8 @@
 from typing import List
-from osbot_utils.helpers.html.Tag__Base import Tag__Base
-from osbot_utils.helpers.html.Tag__Link import Tag__Link
-from osbot_utils.helpers.html.Tag__Style import Tag__Style
+from osbot_utils.helpers.html.tags.Tag__Base import Tag__Base
+from osbot_utils.helpers.html.tags.Tag__Link import Tag__Link
+from osbot_utils.helpers.html.tags.Tag__Style import Tag__Style
 from osbot_utils.utils.Dev import pprint

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/tags}/Tag__Html.py RENAMED Viewed

@@ -1,6 +1,6 @@
-from osbot_utils.helpers.html.Tag__Base import Tag__Base
-from osbot_utils.helpers.html.Tag__Body import Tag__Body
-from osbot_utils.helpers.html.Tag__Head import Tag__Head
+from osbot_utils.helpers.html.tags.Tag__Base import Tag__Base
+from osbot_utils.helpers.html.tags.Tag__Body import Tag__Body
+from osbot_utils.helpers.html.tags.Tag__Head import Tag__Head
 from osbot_utils.utils.Misc             import str_to_bytes
 ATTRIBUTES_NAMES__LINK = ['lang']

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/tags}/Tag__Link.py RENAMED Viewed

@@ -1,4 +1,4 @@
-from osbot_utils.helpers.html.Tag__Base import Tag__Base
+from osbot_utils.helpers.html.tags.Tag__Base import Tag__Base
 ATTRIBUTES_NAMES__LINK = ['href', 'rel', 'integrity',  'crossorigin']

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/tags}/Tag__Style.py RENAMED Viewed

@@ -1,5 +1,5 @@
-from osbot_utils.helpers.html.CSS_Dict__To__Css import CSS_Dict__To__Css
-from osbot_utils.helpers.html.Tag__Base         import Tag__Base, INDENT_SIZE
+from osbot_utils.helpers.html.transformers.CSS_Dict__To__Css import CSS_Dict__To__Css
+from osbot_utils.helpers.html.tags.Tag__Base         import Tag__Base, INDENT_SIZE
 class Tag__Style(Tag__Base):

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/tags}/Tag__Text.py RENAMED Viewed

@@ -1,4 +1,4 @@
-from osbot_utils.helpers.html.Html__To__Html_Dict import STRING__SCHEMA_TEXT
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict import STRING__SCHEMA_TEXT
 class Tag__Text:

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/transformers}/Html_Dict__To__Html.py RENAMED Viewed

@@ -1,4 +1,4 @@
-from osbot_utils.helpers.html.Html__To__Html_Dict import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES
 HTML_SELF_CLOSING_TAGS = {'area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img', 'input', 'link', 'meta',
                           'param', 'source', 'track', 'wbr'}

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/transformers}/Html_Dict__To__Html_Document.py RENAMED Viewed

@@ -1,5 +1,5 @@
 from typing                                                         import Dict, Any
-from osbot_utils.helpers.html.Html__To__Html_Dict                   import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict      import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES
 from osbot_utils.helpers.html.schemas.Schema__Html_Document         import Schema__Html_Document
 from osbot_utils.helpers.html.schemas.Schema__Html_Node             import Schema__Html_Node
 from osbot_utils.helpers.html.schemas.Schema__Html_Node__Data       import Schema__Html_Node__Data

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/transformers}/Html_Dict__To__Html_Tags.py RENAMED Viewed

@@ -1,11 +1,11 @@
-from osbot_utils.helpers.html.Html_Dict__To__Html import HTML_SELF_CLOSING_TAGS
-from osbot_utils.helpers.html.Html__To__Html_Dict import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES
-from osbot_utils.helpers.html.Tag__Base           import Tag__Base
-from osbot_utils.helpers.html.Tag__Body           import Tag__Body
-from osbot_utils.helpers.html.Tag__Head           import Tag__Head
-from osbot_utils.helpers.html.Tag__Html           import Tag__Html
-from osbot_utils.helpers.html.Tag__Link           import Tag__Link
-from osbot_utils.helpers.html.Tag__Text           import Tag__Text
+from osbot_utils.helpers.html.transformers.Html_Dict__To__Html import HTML_SELF_CLOSING_TAGS
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES
+from osbot_utils.helpers.html.tags.Tag__Base                   import Tag__Base
+from osbot_utils.helpers.html.tags.Tag__Body                   import Tag__Body
+from osbot_utils.helpers.html.tags.Tag__Head                   import Tag__Head
+from osbot_utils.helpers.html.tags.Tag__Html                   import Tag__Html
+from osbot_utils.helpers.html.tags.Tag__Link                   import Tag__Link
+from osbot_utils.helpers.html.tags.Tag__Text                   import Tag__Text
 class Html_Dict__To__Html_Tags:

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/transformers}/Html_Document__To__Html_Dict.py RENAMED Viewed

@@ -1,5 +1,5 @@
 from typing                                                     import Dict, Any, List, Union
-from osbot_utils.helpers.html.Html__To__Html_Dict               import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES, STRING__SCHEMA_TAG, STRING__SCHEMA_ATTRS
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict  import STRING__SCHEMA_TEXT, STRING__SCHEMA_NODES, STRING__SCHEMA_TAG, STRING__SCHEMA_ATTRS
 from osbot_utils.helpers.html.schemas.Schema__Html_Document     import Schema__Html_Document
 from osbot_utils.helpers.html.schemas.Schema__Html_Node         import Schema__Html_Node
 from osbot_utils.type_safe.Type_Safe                            import Type_Safe

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/transformers}/Html__To__Html_Document.py RENAMED Viewed

@@ -1,7 +1,7 @@
-from osbot_utils.helpers.html.Html_Dict__To__Html_Document  import Html_Dict__To__Html_Document
-from osbot_utils.helpers.html.Html__To__Html_Dict           import Html__To__Html_Dict
-from osbot_utils.helpers.html.schemas.Schema__Html_Document import Schema__Html_Document
-from osbot_utils.type_safe.Type_Safe                        import Type_Safe
+from osbot_utils.helpers.html.transformers.Html_Dict__To__Html_Document  import Html_Dict__To__Html_Document
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict           import Html__To__Html_Dict
+from osbot_utils.helpers.html.schemas.Schema__Html_Document              import Schema__Html_Document
+from osbot_utils.type_safe.Type_Safe                                     import Type_Safe
 class Html__To__Html_Document(Type_Safe):
     html: str

{osbot_utils-2.76.0/osbot_utils/helpers/html → osbot_utils-2.77.0/osbot_utils/helpers/html/transformers}/Html__To__Html_Tag.py RENAMED Viewed

@@ -1,5 +1,5 @@
-from osbot_utils.helpers.html.Html_Dict__To__Html_Tags import Html_Dict__To__Html_Tags
-from osbot_utils.helpers.html.Html__To__Html_Dict      import Html__To__Html_Dict
+from osbot_utils.helpers.html.transformers.Html_Dict__To__Html_Tags import Html_Dict__To__Html_Tags
+from osbot_utils.helpers.html.transformers.Html__To__Html_Dict      import Html__To__Html_Dict
 class Html__To__Html_Tag:

osbot_utils-2.77.0/osbot_utils/helpers/html/utils/Html__Query.py ADDED Viewed

@@ -0,0 +1,226 @@
+from typing                                                         import List, Optional, Dict, Any
+from osbot_utils.helpers.html.transformers.Html__To__Html_Document  import Html__To__Html_Document
+from osbot_utils.helpers.html.schemas.Schema__Html_Document         import Schema__Html_Document
+from osbot_utils.helpers.html.schemas.Schema__Html_Node             import Schema__Html_Node
+from osbot_utils.type_safe.Type_Safe                                import Type_Safe
+class Html__Query(Type_Safe):
+    html     : str
+    document : Schema__Html_Document
+    def __enter__(self):
+        self.document = Html__To__Html_Document(html=self.html).convert()
+        return self
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+    @property
+    def root(self) -> Schema__Html_Node:                                          # Get the root node
+        return self.document.root_node
+    @property
+    def head(self) -> Optional[Schema__Html_Node]:                                # Get the head node
+        return self.find_child_by_tag(self.root, 'head')
+    @property
+    def body(self) -> Optional[Schema__Html_Node]:                                # Get the body node
+        return self.find_child_by_tag(self.root, 'body')
+    @property
+    def title(self) -> Optional[str]:                                             # Get the page title text
+        if not self.head:
+            return None
+        title_node = self.find_child_by_tag(self.head, 'title')
+        if title_node:
+            return self.get_text_content(title_node).strip()
+        return None
+    @property
+    def links(self) -> List[Dict[str, str]]:                                      # Get all link tags as list of attribute dicts
+        if not self.head:
+            return []
+        return [node.attrs for node in self.head.child_nodes if node.tag == 'link']
+    @property
+    def link_hrefs(self) -> List[str]:                                            # Get all link hrefs
+        return [link.get('href', '') for link in self.links if 'href' in link]
+    @property
+    def css_links(self) -> List[str]:                                             # Get all stylesheet link hrefs
+        result = []
+        for link in self.links:
+            if link.get('rel') == 'stylesheet' and 'href' in link:
+                result.append(link['href'])
+        return result
+    @property
+    def script_sources(self) -> List[str]:                                        # Get all external script src URLs
+        scripts = self.find_all_by_tag('script')
+        return [s.attrs.get('src') for s in scripts if 'src' in s.attrs]
+    @property
+    def inline_scripts(self) -> List[str]:                                        # Get all inline script contents
+        scripts = self.find_all_by_tag('script')
+        result  = []
+        for script in scripts:
+            if 'src' not in script.attrs:                                         # Only inline scripts
+                content = self.get_text_content(script)
+                if content:
+                    result.append(content)
+        return result
+    @property
+    def meta_tags(self) -> List[Dict[str, str]]:                                  # Get all meta tags as list of attribute dicts
+        if not self.head:
+            return []
+        return [node.attrs for node in self.head.child_nodes if node.tag == 'meta']
+    @property
+    def favicon(self) -> Optional[str]:                                           # Get favicon URL if present
+        return (self.find_link_by_rel('shortcut icon') or
+                self.find_link_by_rel('icon'))
+    @property
+    def stylesheets(self) -> List[str]:                                           # Alias for css_links
+        return self.css_links
+    # Query methods
+    def has_link(self, href : str  = None ,                                       # Check if link exists with attributes
+                       rel  : str  = None ,
+                       **attrs
+                 ) -> bool:
+        for link in self.links:
+            if href and link.get('href') != href:
+                continue
+            if rel and link.get('rel') != rel:
+                continue
+            if all(link.get(k) == v for k, v in attrs.items()):                  # Check any additional attributes
+                return True
+        return False
+    def has_script(self, src      : str = None ,                                  # Check if script exists with src or text
+                         contains : str = None
+                   ) -> bool:
+        scripts = self.find_all_by_tag('script')
+        for script in scripts:
+            if src and script.attrs.get('src') == src:
+                return True
+            if contains:
+                content = self.get_text_content(script)
+                if contains in content:
+                    return True
+        return False
+    def has_meta(self, name    : str = None ,                                     # Check if meta tag exists with attributes
+                       content : str = None ,
+                       **attrs
+                 ) -> bool:
+        for meta in self.meta_tags:
+            if name and meta.get('name') != name:
+                continue
+            if content and meta.get('content') != content:
+                continue
+            if all(meta.get(k) == v for k, v in attrs.items()):
+                return True
+        return False
+    def find_by_id(self, element_id : str                                         # Find element by id attribute
+                   ) -> Optional[Schema__Html_Node]:
+        return self.find_by_attribute('id', element_id)
+    def find_by_class(self, class_name : str                                      # Find all elements with a specific class
+                      ) -> List[Schema__Html_Node]:
+        result = []
+        for node in self.find_all():
+            classes = node.attrs.get('class', '').split()
+            if class_name in classes:
+                result.append(node)
+        return result
+    def find_by_tag(self, tag_name : str                                          # Find all elements with a specific tag
+                    ) -> List[Schema__Html_Node]:
+        return self.find_all_by_tag(tag_name)
+    def get_attribute(self, element   : Schema__Html_Node ,                       # Get attribute value from an element
+                            attr_name : str
+                      ) -> Optional[str]:
+        return element.attrs.get(attr_name)
+    def get_text(self, element : Optional[Schema__Html_Node] = None               # Get text content of element or document
+                 ) -> str:
+        if element is None:
+            element = self.root
+        return self.get_text_content(element)
+    # other helper methods
+    def find_child_by_tag(self, parent : Schema__Html_Node,                      # Find first direct child with tag
+                                tag    : str
+                           ) -> Optional[Schema__Html_Node]:
+        for child in parent.child_nodes:
+            if child.tag == tag:
+                return child
+        return None
+    def find_all_by_tag(self, tag  : str,                                       # Recursively find all nodes with tag
+                              node : Optional[Schema__Html_Node] = None
+                         ) -> List[Schema__Html_Node]:
+        if node is None:
+            node = self.root
+        result = []
+        if node.tag == tag:
+            result.append(node)
+        for child in node.child_nodes:
+            result.extend(self.find_all_by_tag(tag, child))
+        return result
+    def find_by_attribute(self, attr_name  : str                                ,  # Find first node with attribute value
+                                attr_value : str                                ,
+                                node       : Optional[Schema__Html_Node] = None
+                           ) -> Optional[Schema__Html_Node]:
+        if node is None:
+            node = self.root
+        if node.attrs.get(attr_name) == attr_value:
+            return node
+        for child in node.child_nodes:
+            result = self.find_by_attribute(attr_name, attr_value, child)
+            if result:
+                return result
+        return None
+    def find_all(self, node : Optional[Schema__Html_Node] = None                 # Get all nodes in the tree
+                  ) -> List[Schema__Html_Node]:
+        if node is None:
+            node = self.root
+        result = [node]
+        for child in node.child_nodes:
+            result.extend(self.find_all(child))
+        return result
+    def find_link_by_rel(self, rel_value: str) -> Optional[str]:
+        for link in self.links:
+            if link.get('rel') == rel_value:
+                return link.get('href')
+        return None
+    def get_text_content(self, node : Schema__Html_Node                           # Get all text content from node
+                         ) -> str:
+        texts = []
+        for text_node in node.text_nodes:                                         # Get text from this node's text_nodes
+            texts.append(text_node.data)
+        for child in node.child_nodes:                                            # Recursively get text from children
+            texts.append(self.get_text_content(child))
+        return ''.join(texts)

osbot_utils-2.77.0/osbot_utils/testing/test_data/const__test__data__html.py ADDED Viewed

@@ -0,0 +1,45 @@
+TEST__DATA__HTML__SIMPLE =  """
+            <html>
+                <head>
+                    <title>Test Page</title>
+                    <link rel="stylesheet" href="/css/main.css">
+                    <link rel="shortcut icon" href="/favicon.ico">
+                    <meta name="description" content="Test description">
+                </head>
+                <body>
+                    <div id="main">Content</div>
+                    <script src="/js/app.js"></script>
+                    <script>console.log('inline');</script>
+                </body>
+            </html>
+        """
+TEST__DATA__HTML__SWAGGER = """
+            <!DOCTYPE html>
+            <html>
+                <head>
+                    <link type="text/css" rel="stylesheet" href="/static/swagger-ui/swagger-ui.css">
+                    <link rel="shortcut icon" href="/static/swagger-ui/favicon.png">
+                    <title>Fast_API - Swagger UI</title>
+                </head>
+                <body>
+                    <div id="swagger-ui"></div>
+                    <script src="/static/swagger-ui/swagger-ui-bundle.js"></script>
+                    <script>
+                        const ui = SwaggerUIBundle({
+                            url: '/openapi.json',
+                            "dom_id": "#swagger-ui"
+                        })
+                    </script>
+                </body>
+            </html>
+        """
+TEST__DATA__HTML__MIXED_CONTENT = """
+            <div class="container">
+                Text before
+                <p class="paragraph">Paragraph content</p>
+                Text middle
+                <span class="highlight">Highlighted</span>
+                Text after
+            </div>
+        """

osbot_utils-2.77.0/osbot_utils/type_safe/type_safe_core/shared/__init__.py ADDED Viewed

File without changes

osbot_utils-2.77.0/osbot_utils/type_safe/type_safe_core/steps/__init__.py ADDED Viewed

File without changes

osbot_utils-2.77.0/osbot_utils/type_safe/validators/__init__.py ADDED Viewed

File without changes

osbot_utils-2.77.0/osbot_utils/utils/__init__.py ADDED Viewed

File without changes

osbot_utils-2.77.0/osbot_utils/version ADDED Viewed

	@@ -0,0 +1 @@
1	+ v2.77.0

{osbot_utils-2.76.0 → osbot_utils-2.77.0}/pyproject.toml RENAMED Viewed

@@ -1,6 +1,6 @@
 [tool.poetry]
 name        = "osbot_utils"
-version     = "v2.76.0"
+version     = "v2.77.0"
 description = "OWASP Security Bot - Utils"
 authors     = ["Dinis Cruz <dinis.cruz@owasp.org>"]
 license     = "MIT"