webscout 2025.10.14.1__py3-none-any.whl → 2025.10.16__py3-none-any.whl

webscout/scout/element.py

@@ -267,7 +267,14 @@ class Tag:
     def select(self, selector: str) -> List['Tag']:
         """
         Select elements using CSS selector.
-        Enhanced to support more complex selectors.
+        Enhanced to support more complex selectors including:
+        - Tag selectors: 'p', 'div'
+        - Class selectors: '.class', 'p.class'
+        - ID selectors: '#id', 'div#id'
+        - Attribute selectors: '[attr]', '[attr=value]'
+        - Descendant selectors: 'div p'
+        - Child selectors: 'div > p'
+        - Multiple classes: '.class1.class2'
 
         Args:
             selector (str): CSS selector string
@@ -275,54 +282,248 @@ class Tag:
         Returns:
             List[Tag]: List of matching elements
         """
-        # More advanced CSS selector parsing
-        # This is a simplified implementation and might need more robust parsing
-        parts = re.split(r'\s+', selector.strip())
         results = []
-
-        def _match_selector(tag, selector_part):
-            # Support more complex selectors
-            if selector_part.startswith('.'):
-                # Class selector
-                return selector_part[1:] in tag.get('class', [])
-            elif selector_part.startswith('#'):
-                # ID selector
-                return tag.get('id') == selector_part[1:]
-            elif '[' in selector_part and ']' in selector_part:
-                # Attribute selector
-                attr_match = re.match(r'(\w+)\[([^=]+)(?:=(.+))?\]', selector_part)
-                if attr_match:
-                    tag_name, attr, value = attr_match.groups()
-                    if tag_name and tag.name != tag_name:
+        
+        def _parse_simple_selector(simple_sel: str) -> dict:
+            """Parse a simple selector like 'p.class#id[attr=value]' into components."""
+            components = {
+                'tag': None,
+                'id': None,
+                'classes': [],
+                'attrs': {}
+            }
+            
+            # Extract tag name (at the start)
+            tag_match = re.match(r'^([a-zA-Z][\w-]*)', simple_sel)
+            if tag_match:
+                components['tag'] = tag_match.group(1)
+                simple_sel = simple_sel[len(tag_match.group(1)):]
+            
+            # Extract ID
+            id_matches = re.findall(r'#([\w-]+)', simple_sel)
+            if id_matches:
+                components['id'] = id_matches[0]
+            
+            # Extract classes
+            class_matches = re.findall(r'\.([\w-]+)', simple_sel)
+            components['classes'] = class_matches
+            
+            # Extract attributes
+            attr_matches = re.findall(r'\[([^\]]+)\]', simple_sel)
+            for attr_expr in attr_matches:
+                if '=' in attr_expr:
+                    attr_name, attr_value = attr_expr.split('=', 1)
+                    components['attrs'][attr_name.strip()] = attr_value.strip('\'"')
+                else:
+                    components['attrs'][attr_expr.strip()] = None
+            
+            return components
+        
+        def _match_simple_selector(tag: 'Tag', components: dict) -> bool:
+            """Check if a tag matches the parsed selector components."""
+            # Check tag name
+            if components['tag'] and tag.name != components['tag']:
+                return False
+            
+            # Check ID
+            if components['id'] and tag.get('id') != components['id']:
+                return False
+            
+            # Check classes
+            tag_classes = tag.get('class', '')
+            if isinstance(tag_classes, str):
+                tag_classes = tag_classes.split()
+            elif not isinstance(tag_classes, list):
+                tag_classes = [str(tag_classes)] if tag_classes else []
+            
+            for cls in components['classes']:
+                if cls not in tag_classes:
+                    return False
+            
+            # Check attributes
+            for attr_name, attr_value in components['attrs'].items():
+                if attr_value is None:
+                    # Just check attribute exists
+                    if attr_name not in tag.attrs:
                         return False
-                    if value:
-                        return tag.get(attr) == value.strip("'\"")
-                    return attr in tag.attrs
-            else:
-                # Tag selector
-                return tag.name == selector_part
-
-        def _recursive_select(element, selector_parts):
-            if not selector_parts:
-                results.append(element)
-                return
-
-            current_selector = selector_parts[0]
-            remaining_selectors = selector_parts[1:]
-
-            if _match_selector(element, current_selector):
-                if not remaining_selectors:
-                    results.append(element)
                 else:
-                    for child in element.contents:
-                        if isinstance(child, Tag):
-                            _recursive_select(child, remaining_selectors)
-
-        for child in self.contents:
-            if isinstance(child, Tag):
-                _recursive_select(child, parts)
-
+                    # Check attribute value
+                    if tag.get(attr_name) != attr_value:
+                        return False
+            
+            return True
+        
+        def _find_all_matching(element: 'Tag', components: dict) -> List['Tag']:
+            """Recursively find all elements matching the selector components."""
+            matches = []
+            
+            # Check current element
+            if _match_simple_selector(element, components):
+                matches.append(element)
+            
+            # Check children recursively
+            for child in element.contents:
+                if isinstance(child, Tag):
+                    matches.extend(_find_all_matching(child, components))
+            
+            return matches
+        
+        # Handle combinators (descendant ' ' and child '>')
+        if ' > ' in selector:
+            # Child combinator
+            parts = [p.strip() for p in selector.split(' > ')]
+            return self._select_with_child_combinator(parts)
+        elif ' ' in selector.strip():
+            # Descendant combinator
+            parts = [p.strip() for p in selector.split()]
+            return self._select_with_descendant_combinator(parts)
+        else:
+            # Simple selector
+            components = _parse_simple_selector(selector)
+            return _find_all_matching(self, components)
+    
+    def _select_with_descendant_combinator(self, parts: List[str]) -> List['Tag']:
+        """Handle descendant combinator (space)."""
+        if not parts:
+            return []
+        
+        if len(parts) == 1:
+            components = self._parse_selector_components(parts[0])
+            return self._find_all_matching_in_tree(self, components)
+        
+        # Find elements matching the first part
+        first_components = self._parse_selector_components(parts[0])
+        first_matches = self._find_all_matching_in_tree(self, first_components)
+        
+        # For each match, find descendants matching remaining parts
+        results = []
+        remaining_selector = ' '.join(parts[1:])
+        for match in first_matches:
+            descendants = match.select(remaining_selector)
+            results.extend(descendants)
+        
         return results
+    
+    def _select_with_child_combinator(self, parts: List[str]) -> List['Tag']:
+        """Handle child combinator (>)."""
+        if not parts:
+            return []
+        
+        if len(parts) == 1:
+            components = self._parse_selector_components(parts[0])
+            return self._find_all_matching_in_tree(self, components)
+        
+        # Find elements matching the first part
+        first_components = self._parse_selector_components(parts[0])
+        first_matches = self._find_all_matching_in_tree(self, first_components)
+        
+        # For each match, find direct children matching the next part
+        if len(parts) == 2:
+            # Last part, just check direct children
+            next_components = self._parse_selector_components(parts[1])
+            results = []
+            for match in first_matches:
+                for child in match.contents:
+                    if isinstance(child, Tag) and self._match_selector_components(child, next_components):
+                        results.append(child)
+            return results
+        else:
+            # More parts, need to continue recursively
+            results = []
+            next_components = self._parse_selector_components(parts[1])
+            remaining_parts = parts[2:]
+            for match in first_matches:
+                for child in match.contents:
+                    if isinstance(child, Tag) and self._match_selector_components(child, next_components):
+                        # Continue with remaining parts
+                        remaining_selector = ' > '.join(remaining_parts)
+                        descendants = child.select(remaining_selector)
+                        results.extend(descendants)
+            return results
+    
+    def _parse_selector_components(self, simple_sel: str) -> dict:
+        """Parse a simple selector like 'p.class#id[attr=value]' into components."""
+        components = {
+            'tag': None,
+            'id': None,
+            'classes': [],
+            'attrs': {}
+        }
+        
+        # Extract tag name (at the start)
+        tag_match = re.match(r'^([a-zA-Z][\w-]*)', simple_sel)
+        if tag_match:
+            components['tag'] = tag_match.group(1)
+            simple_sel = simple_sel[len(tag_match.group(1)):]
+        
+        # Extract ID
+        id_matches = re.findall(r'#([\w-]+)', simple_sel)
+        if id_matches:
+            components['id'] = id_matches[0]
+        
+        # Extract classes
+        class_matches = re.findall(r'\.([\w-]+)', simple_sel)
+        components['classes'] = class_matches
+        
+        # Extract attributes
+        attr_matches = re.findall(r'\[([^\]]+)\]', simple_sel)
+        for attr_expr in attr_matches:
+            if '=' in attr_expr:
+                attr_name, attr_value = attr_expr.split('=', 1)
+                components['attrs'][attr_name.strip()] = attr_value.strip('\'"')
+            else:
+                components['attrs'][attr_expr.strip()] = None
+        
+        return components
+    
+    def _match_selector_components(self, tag: 'Tag', components: dict) -> bool:
+        """Check if a tag matches the parsed selector components."""
+        # Check tag name
+        if components['tag'] and tag.name != components['tag']:
+            return False
+        
+        # Check ID
+        if components['id'] and tag.get('id') != components['id']:
+            return False
+        
+        # Check classes
+        tag_classes = tag.get('class', '')
+        if isinstance(tag_classes, str):
+            tag_classes = tag_classes.split()
+        elif not isinstance(tag_classes, list):
+            tag_classes = [str(tag_classes)] if tag_classes else []
+        
+        for cls in components['classes']:
+            if cls not in tag_classes:
+                return False
+        
+        # Check attributes
+        for attr_name, attr_value in components['attrs'].items():
+            if attr_value is None:
+                # Just check attribute exists
+                if attr_name not in tag.attrs:
+                    return False
+            else:
+                # Check attribute value
+                if tag.get(attr_name) != attr_value:
+                    return False
+        
+        return True
+    
+    def _find_all_matching_in_tree(self, element: 'Tag', components: dict) -> List['Tag']:
+        """Recursively find all elements matching the selector components."""
+        matches = []
+        
+        # Check current element
+        if self._match_selector_components(element, components):
+            matches.append(element)
+        
+        # Check children recursively
+        for child in element.contents:
+            if isinstance(child, Tag):
+                matches.extend(self._find_all_matching_in_tree(child, components))
+        
+        return matches
 
     def select_one(self, selector: str) -> Optional['Tag']:
         """
@@ -462,6 +663,11 @@ class Tag:
             new_child.parent = self
         self.contents.append(new_child)
 
+    def extend(self, new_children: List[Union['Tag', NavigableString, str]]) -> None:
+        """Extend the contents of this tag with a list of new children."""
+        for child in new_children:
+            self.append(child)
+
     def insert(self, index: int, new_child: Union['Tag', NavigableString, str]) -> None:
         """Insert a new child at the given index with error handling."""
         if isinstance(new_child, str):

webscout/search/__init__.py

@@ -3,16 +3,15 @@
 from .base import BaseSearch, BaseSearchEngine
 from .duckduckgo_main import DuckDuckGoSearch
 from .yep_main import YepSearch
+from .bing_main import BingSearch
+from .yahoo_main import YahooSearch
 
 # Import new search engines
-from .engines.bing import Bing
 from .engines.brave import Brave
 from .engines.mojeek import Mojeek
-from .engines.yahoo import Yahoo
+
 from .engines.yandex import Yandex
 from .engines.wikipedia import Wikipedia
-from .engines.bing_news import BingNews
-from .engines.yahoo_news import YahooNews
 
 # Import result models
 from .results import (
@@ -31,16 +30,14 @@ __all__ = [
     # Main search interfaces
     "DuckDuckGoSearch",
     "YepSearch",
+    "BingSearch",
+    "YahooSearch",
     
     # Individual engines
-    "Bing",
     "Brave",
     "Mojeek",
-    "Yahoo",
     "Yandex",
     "Wikipedia",
-    "BingNews",
-    "YahooNews",
     
     # Result models
     "TextResult",

webscout/search/bing_main.py

@@ -0,0 +1,42 @@
+"""Bing unified search interface."""
+
+from __future__ import annotations
+from typing import Dict, List, Optional
+from .base import BaseSearch
+from .engines.bing.text import BingTextSearch
+from .engines.bing.images import BingImagesSearch
+from .engines.bing.news import BingNewsSearch
+from .engines.bing.suggestions import BingSuggestionsSearch
+
+
+class BingSearch(BaseSearch):
+    """Unified Bing search interface."""
+
+    def text(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None, unique: bool = True) -> List[Dict[str, str]]:
+        search = BingTextSearch()
+        return search.run(keywords, region, safesearch, max_results, unique=unique)
+
+    def images(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = BingImagesSearch()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def news(self, keywords: str, region: str = "us", safesearch: str = "moderate", max_results: Optional[int] = None) -> List[Dict[str, str]]:
+        search = BingNewsSearch()
+        return search.run(keywords, region, safesearch, max_results)
+
+    def suggestions(self, query: str, region: str = "en-US") -> List[Dict[str, str]]:
+        search = BingSuggestionsSearch()
+        result = search.run(query, region)
+        return [{'suggestion': s} for s in result]
+
+    def answers(self, keywords: str) -> List[Dict[str, str]]:
+        raise NotImplementedError("Answers not implemented for Bing")
+
+    def maps(self, *args, **kwargs) -> List[Dict[str, str]]:
+        raise NotImplementedError("Maps not implemented for Bing")
+
+    def translate(self, keywords: str, from_lang: Optional[str] = None, to_lang: str = "en") -> List[Dict[str, str]]:
+        raise NotImplementedError("Translate not implemented for Bing")
+
+    def videos(self, *args, **kwargs) -> List[Dict[str, str]]:
+        raise NotImplementedError("Videos not implemented for Bing")

webscout/search/engines/bing/__init__.py

@@ -0,0 +1 @@
+"""Bing search engines."""

webscout/search/engines/bing/base.py

@@ -0,0 +1,33 @@
+"""Base class for Bing search implementations."""
+
+from __future__ import annotations
+
+from ....litagent import LitAgent
+from curl_cffi.requests import Session
+
+
+class BingBase:
+    """Base class for Bing search engines."""
+
+    def __init__(
+        self,
+        timeout: int = 10,
+        proxies: dict[str, str] | None = None,
+        verify: bool = True,
+        lang: str = "en-US",
+        sleep_interval: float = 0.0,
+        impersonate: str = "chrome110",
+    ):
+        self.timeout = timeout
+        self.proxies = proxies
+        self.verify = verify
+        self.lang = lang
+        self.sleep_interval = sleep_interval
+        self.base_url = "https://www.bing.com"
+        self.session = Session(
+            proxies=proxies,
+            verify=verify,
+            timeout=timeout,
+            impersonate=impersonate,
+        )
+        self.session.headers.update(LitAgent().generate_fingerprint())

webscout/search/engines/bing/images.py

@@ -0,0 +1,108 @@
+"""Bing images search."""
+
+from __future__ import annotations
+
+from typing import Dict, List
+from urllib.parse import urlencode
+from time import sleep
+
+from .base import BingBase
+from webscout.scout import Scout
+
+
+class BingImagesSearch(BingBase):
+    def run(self, *args, **kwargs) -> List[Dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safe_map = {
+            "on": "Strict",
+            "moderate": "Moderate",
+            "off": "Off"
+        }
+        safe = safe_map.get(safesearch.lower(), "Moderate")
+
+        # Bing images URL
+        url = f"{self.base_url}/images/async"
+        params = {
+            'q': keywords,
+            'first': '1',
+            'count': '35',  # Fetch more to get max_results
+            'cw': '1177',
+            'ch': '759',
+            'tsc': 'ImageHoverTitle',
+            'layout': 'RowBased_Landscape',
+            't': '0',
+            'IG': '',
+            'SFX': '0',
+            'iid': 'images.1'
+        }
+
+        results = []
+        first = 1
+        sfx = 0
+
+        while len(results) < max_results:
+            params['first'] = str(first)
+            params['SFX'] = str(sfx)
+            full_url = f"{url}?{urlencode(params)}"
+
+            try:
+                response = self.session.get(full_url, timeout=self.timeout)
+                response.raise_for_status()
+                html = response.text
+            except Exception as e:
+                raise Exception(f"Failed to fetch images: {str(e)}")
+
+            soup = Scout(html)
+            img_tags = soup.select('a.iusc img')
+
+            for img in img_tags:
+                if len(results) >= max_results:
+                    break
+
+                title = img.get('alt', '')
+                src = img.get('src', '')
+                m_attr = img.parent.get('m', '') if img.parent else ''
+
+                # Parse m attribute for full image URL
+                image_url = src
+                thumbnail = src
+                if m_attr:
+                    try:
+                        import json
+                        m_data = json.loads(m_attr)
+                        image_url = m_data.get('murl', src)
+                        thumbnail = m_data.get('turl', src)
+                    except:
+                        pass
+
+                source = ''
+                if img.parent and img.parent.parent:
+                    source_tag = img.parent.parent.select_one('.iusc .lnk')
+                    if source_tag:
+                        source = source_tag.get_text(strip=True)
+
+                results.append({
+                    'title': title,
+                    'image': image_url,
+                    'thumbnail': thumbnail,
+                    'url': image_url,  # For compatibility
+                    'source': source
+                })
+
+            first += 35
+            sfx += 1
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return results[:max_results]

webscout/search/engines/bing/news.py

@@ -0,0 +1,91 @@
+"""Bing news search."""
+
+from __future__ import annotations
+
+from typing import Dict, List
+from urllib.parse import urlencode
+from time import sleep
+
+from .base import BingBase
+from webscout.scout import Scout
+
+
+class BingNewsSearch(BingBase):
+    def run(self, *args, **kwargs) -> List[Dict[str, str]]:
+        keywords = args[0] if args else kwargs.get("keywords")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "us")
+        safesearch = args[2] if len(args) > 2 else kwargs.get("safesearch", "moderate")
+        max_results = args[3] if len(args) > 3 else kwargs.get("max_results", 10)
+
+        if max_results is None:
+            max_results = 10
+
+        if not keywords:
+            raise ValueError("Keywords are mandatory")
+
+        safe_map = {
+            "on": "Strict",
+            "moderate": "Moderate",
+            "off": "Off"
+        }
+        safe = safe_map.get(safesearch.lower(), "Moderate")
+
+        # Bing news URL
+        url = f"{self.base_url}/news/infinitescrollajax"
+        params = {
+            'q': keywords,
+            'InfiniteScroll': '1',
+            'first': '1',
+            'SFX': '0',
+            'cc': region.lower(),
+            'setlang': self.lang.split('-')[0]
+        }
+
+        results = []
+        first = 1
+        sfx = 0
+
+        while len(results) < max_results:
+            params['first'] = str(first)
+            params['SFX'] = str(sfx)
+            full_url = f"{url}?{urlencode(params)}"
+
+            try:
+                response = self.session.get(full_url, timeout=self.timeout)
+                response.raise_for_status()
+                html = response.text
+            except Exception as e:
+                raise Exception(f"Failed to fetch news: {str(e)}")
+
+            if not html:
+                break
+
+            soup = Scout(html)
+            news_items = soup.select('div.newsitem')
+
+            for item in news_items:
+                if len(results) >= max_results:
+                    break
+
+                title = item.select_one('a.title')
+                snippet = item.select_one('div.snippet')
+                source = item.select_one('div.source')
+                date = item.select_one('span.date')
+
+                if title:
+                    news_result = {
+                        'title': title.get_text(strip=True),
+                        'url': title.get('href', ''),
+                        'body': snippet.get_text(strip=True) if snippet else '',
+                        'source': source.get_text(strip=True) if source else '',
+                        'date': date.get_text(strip=True) if date else ''
+                    }
+                    results.append(news_result)
+
+            first += 10
+            sfx += 1
+
+            if self.sleep_interval:
+                sleep(self.sleep_interval)
+
+        return results[:max_results]

webscout/search/engines/bing/suggestions.py

@@ -0,0 +1,34 @@
+"""Bing suggestions search."""
+
+from __future__ import annotations
+
+from typing import List
+from urllib.parse import urlencode
+
+from .base import BingBase
+
+
+class BingSuggestionsSearch(BingBase):
+    def run(self, *args, **kwargs) -> List[str]:
+        query = args[0] if args else kwargs.get("query")
+        region = args[1] if len(args) > 1 else kwargs.get("region", "en-US")
+
+        if not query:
+            raise ValueError("Query is mandatory")
+
+        params = {
+            "query": query,
+            "mkt": region
+        }
+        url = f"https://api.bing.com/osjson.aspx?{urlencode(params)}"
+
+        try:
+            response = self.session.get(url, timeout=self.timeout)
+            response.raise_for_status()
+            data = response.json()
+            # Bing suggestions API returns [query, [suggestions]]
+            if len(data) > 1 and isinstance(data[1], list):
+                return data[1]
+            return []
+        except Exception as e:
+            raise Exception(f"Failed to fetch suggestions: {str(e)}")