universal-mcp-applications 0.1.30rc1__py3-none-any.whl → 0.1.36rc1__py3-none-any.whl

universal_mcp/applications/scraper/README.md

@@ -9,7 +9,10 @@ This is automatically generated from OpenAPI schema for the ScraperApp API.
 
 | Tool | Description |
 |------|-------------|
-| `linkedin_post_search` | Performs a general LinkedIn search for posts using keywords and filters like date and content type. It supports pagination and can utilize either the 'classic' or 'sales_navigator' API, searching broadly across the platform rather than fetching posts from a specific user's profile. |
-| `linkedin_list_profile_posts` | Fetches a paginated list of all LinkedIn posts from a specific user or company profile using their unique identifier. This function retrieves content directly from a profile, unlike `linkedin_post_search` which finds posts across LinkedIn based on keywords and other filters. |
-| `linkedin_retrieve_profile` | Retrieves a specific LinkedIn user's profile by their unique identifier, which can be an internal provider ID or a public username. This function simplifies data access by delegating the actual profile retrieval request to the integrated Unipile application, distinct from functions that list posts or comments. |
-| `linkedin_list_post_comments` | Fetches comments for a specified LinkedIn post. If a `comment_id` is provided, it retrieves replies to that comment instead of top-level comments. This function supports pagination and specifically targets comments, unlike others in the class that search for or list entire posts. |
+| `linkedin_list_profile_posts` | Fetches a paginated list of posts from a specific user or company profile using its provider ID. The `is_company` flag must specify the entity type. Unlike `linkedin_search_posts`, this function directly retrieves content from a known profile's feed instead of performing a global keyword search. |
+| `linkedin_retrieve_profile` | Fetches a specific LinkedIn user's profile using their public or internal ID. Unlike `linkedin_search_people`, which discovers multiple users via keywords, this function targets and retrieves detailed data for a single, known individual based on a direct identifier. |
+| `linkedin_list_post_comments` | Fetches a paginated list of comments for a specified LinkedIn post. It can retrieve either top-level comments or threaded replies if an optional `comment_id` is provided. This is a read-only operation, distinct from functions that search for posts or list user-specific content. |
+| `linkedin_search_people` | Performs a paginated search for people on LinkedIn, distinct from searches for companies or jobs. It filters results using keywords, location, industry, and company, internally converting filter names like 'United States' into their required API IDs before making the request. |
+| `linkedin_search_companies` | Executes a paginated LinkedIn search for companies, filtering by optional keywords, location, and industry. Unlike `linkedin_search_people` or `linkedin_search_jobs`, this function specifically sets the API search category to 'companies' to ensure that only company profiles are returned in the search results. |
+| `linkedin_search_posts` | Performs a keyword-based search for LinkedIn posts, allowing results to be filtered by date and sorted by relevance. This function specifically queries the 'posts' category, distinguishing it from other search methods in the class that target people, companies, or jobs, and returns relevant content. |
+| `linkedin_search_jobs` | Executes a LinkedIn search specifically for job listings using keywords and filters like region, industry, and minimum salary. Unlike other search functions targeting people or companies, this is specialized for job listings and converts friendly filter names (e.g., "United States") into their required API IDs. |

universal_mcp/applications/scraper/app.py

@@ -1,15 +1,12 @@
+import os
 from dotenv import load_dotenv
 
 load_dotenv()
-
 from typing import Any, Literal
-
 from loguru import logger
 from universal_mcp.applications.application import APIApplication
 from universal_mcp.integrations import Integration
 
-from universal_mcp.applications.linkedin import LinkedinApp
-
 
 class ScraperApp(APIApplication):
     """
@@ -22,143 +19,330 @@ class ScraperApp(APIApplication):
         if self.integration:
             credentials = self.integration.get_credentials()
             self.account_id = credentials.get("account_id")
-            self._unipile_app = LinkedinApp(integration=self.integration)
         else:
             logger.warning("Integration not found")
             self.account_id = None
-            self._unipile_app = None
 
-    def linkedin_search(
+    @property
+    def base_url(self) -> str:
+        if not self._base_url:
+            unipile_dsn = os.getenv("UNIPILE_DSN")
+            if not unipile_dsn:
+                logger.error("UnipileApp: UNIPILE_DSN environment variable is not set.")
+                raise ValueError("UnipileApp: UNIPILE_DSN environment variable is required.")
+            self._base_url = f"https://{unipile_dsn}"
+        return self._base_url
+
+    @base_url.setter
+    def base_url(self, base_url: str) -> None:
+        self._base_url = base_url
+        logger.info(f"UnipileApp: Base URL set to {self._base_url}")
+
+    def _get_headers(self) -> dict[str, str]:
+        """
+        Get the headers for Unipile API requests.
+        Overrides the base class method to use X-Api-Key.
+        """
+        if not self.integration:
+            logger.warning("UnipileApp: No integration configured, returning empty headers.")
+            return {}
+        api_key = os.getenv("UNIPILE_API_KEY")
+        if not api_key:
+            logger.error("UnipileApp: API key not found in integration credentials for Unipile.")
+            return {"Content-Type": "application/json", "Cache-Control": "no-cache"}
+        logger.debug("UnipileApp: Using X-Api-Key for authentication.")
+        return {"x-api-key": api_key, "Content-Type": "application/json", "Cache-Control": "no-cache"}
+
+    def _get_search_parameter_id(self, param_type: str, keywords: str) -> str:
+        """
+        Retrieves the ID for a given LinkedIn search parameter by its name.
+
+        Args:
+            param_type: The type of parameter to search for (e.g., "LOCATION", "COMPANY").
+            keywords: The name of the parameter to find (e.g., "United States").
+
+        Returns:
+            The corresponding ID for the search parameter.
+
+        Raises:
+            ValueError: If no exact match for the keywords is found.
+            httpx.HTTPError: If the API request fails.
+        """
+        url = f"{self.base_url}/api/v1/linkedin/search/parameters"
+        params = {"account_id": self.account_id, "keywords": keywords, "type": param_type}
+        response = self._get(url, params=params)
+        results = self._handle_response(response)
+        items = results.get("items", [])
+        if items:
+            return items[0]["id"]
+        raise ValueError(f'Could not find a matching ID for {param_type}: "{keywords}"')
+
+    async def linkedin_list_profile_posts(
+        self, identifier: str, cursor: str | None = None, limit: int | None = None, is_company: bool | None = None
+    ) -> dict[str, Any]:
+        """
+        Fetches a paginated list of posts from a specific user or company profile using its provider ID. The `is_company` flag must specify the entity type. Unlike `linkedin_search_posts`, this function directly retrieves content from a known profile's feed instead of performing a global keyword search.
+
+        Args:
+            identifier: The entity's provider internal ID (LinkedIn ID).
+            cursor: Pagination cursor.
+            limit: Number of items to return (1-100, as per Unipile example, though spec allows up to 250).
+            is_company: Boolean indicating if the identifier is for a company.
+
+        Returns:
+            A dictionary containing a list of post objects and pagination details.
+
+        Raises:
+            httpx.HTTPError: If the API request fails.
+
+        Tags:
+            linkedin, post, list, user_posts, company_posts, content, api, important
+        """
+        url = f"{self.base_url}/api/v1/users/{identifier}/posts"
+        params: dict[str, Any] = {"account_id": self.account_id}
+        if cursor:
+            params["cursor"] = cursor
+        if limit:
+            params["limit"] = limit
+        if is_company is not None:
+            params["is_company"] = is_company
+        response = self._get(url, params=params)
+        return response.json()
+
+    async def linkedin_retrieve_profile(self, identifier: str) -> dict[str, Any]:
+        """
+        Fetches a specific LinkedIn user's profile using their public or internal ID. Unlike `linkedin_search_people`, which discovers multiple users via keywords, this function targets and retrieves detailed data for a single, known individual based on a direct identifier.
+
+        Args:
+            identifier: Can be the provider's internal id OR the provider's public id of the requested user.For example, for https://www.linkedin.com/in/manojbajaj95/, the identifier is "manojbajaj95".
+
+        Returns:
+            A dictionary containing the user's profile details.
+
+        Raises:
+            httpx.HTTPError: If the API request fails.
+
+        Tags:
+            linkedin, user, profile, retrieve, get, api, important
+        """
+        url = f"{self.base_url}/api/v1/users/{identifier}"
+        params: dict[str, Any] = {"account_id": self.account_id}
+        response = self._get(url, params=params)
+        return self._handle_response(response)
+
+    async def linkedin_list_post_comments(
+        self, post_id: str, comment_id: str | None = None, cursor: str | None = None, limit: int | None = None
+    ) -> dict[str, Any]:
+        """
+        Fetches a paginated list of comments for a specified LinkedIn post. It can retrieve either top-level comments or threaded replies if an optional `comment_id` is provided. This is a read-only operation, distinct from functions that search for posts or list user-specific content.
+
+        Args:
+            post_id: The social ID of the post.
+            comment_id: If provided, retrieves replies to this comment ID instead of top-level comments.
+            cursor: Pagination cursor.
+            limit: Number of comments to return. (OpenAPI spec shows type string, passed as string if provided).
+
+        Returns:
+            A dictionary containing a list of comment objects and pagination details.
+
+        Raises:
+            httpx.HTTPError: If the API request fails.
+
+        Tags:
+            linkedin, post, comment, list, content, api, important
+        """
+        url = f"{self.base_url}/api/v1/posts/{post_id}/comments"
+        params: dict[str, Any] = {"account_id": self.account_id}
+        if cursor:
+            params["cursor"] = cursor
+        if limit is not None:
+            params["limit"] = str(limit)
+        if comment_id:
+            params["comment_id"] = comment_id
+        response = self._get(url, params=params)
+        return response.json()
+
+    async def linkedin_search_people(
         self,
-        category: Literal["people", "companies", "posts", "jobs"],
         cursor: str | None = None,
         limit: int | None = None,
         keywords: str | None = None,
-        date_posted: str | None = None,
-        sort_by: Literal["relevance", "date"] = "relevance",
-        minimum_salary_value: int = 40,
+        location: str | None = None,
+        industry: str | None = None,
+        company: str | None = None,
     ) -> dict[str, Any]:
         """
-        Performs a general LinkedIn search for posts, people, companies, or jobs using keywords and various filters. This function provides broad, keyword-based discovery across the platform, distinct from other methods that retrieve content from a specific user or company profile. It supports pagination and filtering criteria.
-        
+        Performs a paginated search for people on LinkedIn, distinct from searches for companies or jobs. It filters results using keywords, location, industry, and company, internally converting filter names like 'United States' into their required API IDs before making the request.
+
         Args:
-            category: Type of search to perform. Valid values are "people", "companies", "posts", or "jobs".
             cursor: Pagination cursor for the next page of entries.
             limit: Number of items to return (up to 50 for Classic search).
             keywords: Keywords to search for.
-            date_posted: Filter by when the post was posted (posts only). Valid values are "past_day", "past_week", or "past_month".
-            sort_by: How to sort the results (for posts and jobs). Valid values are "relevance" or "date".
-            minimum_salary_value: The minimum salary to filter for (jobs only).
-        
+            location: The geographical location to filter people by (e.g., "United States").
+            industry: The industry to filter people by.(e.g., "Software Development".)
+            company: The company to filter people by.(e.g., "Google".)
+
         Returns:
             A dictionary containing search results and pagination details.
-        
+
         Raises:
             httpx.HTTPError: If the API request fails.
-        
-        Tags:
-            linkedin, search, posts, people, companies, api, scrapper, important
-        """
-        return self._unipile_app.search(
-            account_id=self.account_id,
-            category=category,
-            cursor=cursor,
-            limit=limit,
-            keywords=keywords,
-            date_posted=date_posted,
-            sort_by=sort_by,
-            minimum_salary_value=minimum_salary_value,
-        )
-
-    def linkedin_list_profile_posts(
+        """
+        url = f"{self.base_url}/api/v1/linkedin/search"
+        params: dict[str, Any] = {"account_id": self.account_id}
+        if cursor:
+            params["cursor"] = cursor
+        if limit is not None:
+            params["limit"] = limit
+        payload: dict[str, Any] = {"api": "classic", "category": "people"}
+        if keywords:
+            payload["keywords"] = keywords
+        if location:
+            location_id = self._get_search_parameter_id("LOCATION", location)
+            payload["location"] = [location_id]
+        if industry:
+            industry_id = self._get_search_parameter_id("INDUSTRY", industry)
+            payload["industry"] = [industry_id]
+        if company:
+            company_id = self._get_search_parameter_id("COMPANY", company)
+            payload["company"] = [company_id]
+        response = self._post(url, params=params, data=payload)
+        return self._handle_response(response)
+
+    async def linkedin_search_companies(
         self,
-        identifier: str,
         cursor: str | None = None,
         limit: int | None = None,
+        keywords: str | None = None,
+        location: str | None = None,
+        industry: str | None = None,
     ) -> dict[str, Any]:
         """
-        Fetches a paginated list of all posts from a specific user or company profile using their unique identifier. This function retrieves content directly from a single profile, unlike the broader, keyword-based `linkedin_search` which searches across the entire LinkedIn platform.
-        
+        Executes a paginated LinkedIn search for companies, filtering by optional keywords, location, and industry. Unlike `linkedin_search_people` or `linkedin_search_jobs`, this function specifically sets the API search category to 'companies' to ensure that only company profiles are returned in the search results.
+
         Args:
-            identifier: The entity's provider internal ID (LinkedIn ID).starts with ACo for users, while for companies it's a series of numbers. You can get it in the results of linkedin_search.
             cursor: Pagination cursor for the next page of entries.
-            limit: Number of items to return (1-100, though spec allows up to 250).
-        
+            limit: Number of items to return (up to 50 for Classic search).
+            keywords: Keywords to search for.
+            location: The geographical location to filter companies by (e.g., "United States").
+            industry: The industry to filter companies by.(e.g., "Software Development".)
+
         Returns:
-            A dictionary containing a list of post objects and pagination details.
-        
+            A dictionary containing search results and pagination details.
+
         Raises:
             httpx.HTTPError: If the API request fails.
-        
-        Tags:
-            linkedin, post, list, user_posts, company_posts, content, api, important
         """
+        url = f"{self.base_url}/api/v1/linkedin/search"
+        params: dict[str, Any] = {"account_id": self.account_id}
+        if cursor:
+            params["cursor"] = cursor
+        if limit is not None:
+            params["limit"] = limit
+        payload: dict[str, Any] = {"api": "classic", "category": "companies"}
+        if keywords:
+            payload["keywords"] = keywords
+        if location:
+            location_id = self._get_search_parameter_id("LOCATION", location)
+            payload["location"] = [location_id]
+        if industry:
+            industry_id = self._get_search_parameter_id("INDUSTRY", industry)
+            payload["industry"] = [industry_id]
+        response = self._post(url, params=params, data=payload)
+        return self._handle_response(response)
 
-        return self._unipile_app.list_profile_posts(
-            identifier=identifier,
-            account_id=self.account_id,
-            cursor=cursor,
-            limit=limit,
-        )
-
-    def linkedin_retrieve_profile(
+    async def linkedin_search_posts(
         self,
-        identifier: str,
+        cursor: str | None = None,
+        limit: int | None = None,
+        keywords: str | None = None,
+        date_posted: Literal["past_day", "past_week", "past_month"] | None = None,
+        sort_by: Literal["relevance", "date"] = "relevance",
     ) -> dict[str, Any]:
         """
-        Retrieves a specific LinkedIn user's profile using their unique identifier (e.g., public username). Unlike other methods that list posts or comments, this function focuses on fetching the user's core profile data by delegating the request to the integrated Unipile application.
-        
+        Performs a keyword-based search for LinkedIn posts, allowing results to be filtered by date and sorted by relevance. This function specifically queries the 'posts' category, distinguishing it from other search methods in the class that target people, companies, or jobs, and returns relevant content.
+
         Args:
-            identifier: Use the id from linkedin_search results. It starts with ACo for users and is a series of numbers for companies.
-        
+            cursor: Pagination cursor for the next page of entries.
+            limit: Number of items to return (up to 50 for Classic search).
+            keywords: Keywords to search for.
+            date_posted: Filter by when the post was posted.
+            sort_by: How to sort the results.
+
         Returns:
-            A dictionary containing the user's profile details.
-        
+            A dictionary containing search results and pagination details.
+
         Raises:
             httpx.HTTPError: If the API request fails.
-        
-        Tags:
-            linkedin, user, profile, retrieve, get, api, important
         """
+        url = f"{self.base_url}/api/v1/linkedin/search"
+        params: dict[str, Any] = {"account_id": self.account_id}
+        if cursor:
+            params["cursor"] = cursor
+        if limit is not None:
+            params["limit"] = limit
+        payload: dict[str, Any] = {"api": "classic", "category": "posts"}
+        if keywords:
+            payload["keywords"] = keywords
+        if date_posted:
+            payload["date_posted"] = date_posted
+        if sort_by:
+            payload["sort_by"] = sort_by
+        response = self._post(url, params=params, data=payload)
+        return self._handle_response(response)
 
-        return self._unipile_app.retrieve_user_profile(
-            identifier=identifier,
-            account_id=self.account_id,
-        )
-
-    def linkedin_list_post_comments(
+    async def linkedin_search_jobs(
         self,
-        post_id: str,
-        comment_id: str | None = None,
         cursor: str | None = None,
         limit: int | None = None,
+        keywords: str | None = None,
+        region: str | None = None,
+        sort_by: Literal["relevance", "date"] = "relevance",
+        minimum_salary_value: int = 40,
+        industry: str | None = None,
     ) -> dict[str, Any]:
         """
-        Fetches a paginated list of comments for a specified LinkedIn post. Providing a `comment_id` retrieves replies to that specific comment instead of top-level ones. This function exclusively targets post interactions, differentiating it from others that list posts or retrieve entire profiles.
-        
+        Executes a LinkedIn search specifically for job listings using keywords and filters like region, industry, and minimum salary. Unlike other search functions targeting people or companies, this is specialized for job listings and converts friendly filter names (e.g., "United States") into their required API IDs.
+
         Args:
-            post_id: The social ID of the post. Example urn:li:ugcPost:7386500271624896512
-            comment_id: If provided, retrieves replies to this comment ID instead of top-level comments.
-            cursor: Pagination cursor.
-            limit: Number of comments to return.
-        
+            cursor: Pagination cursor for the next page of entries.
+            limit: Number of items to return (up to 50 for Classic search).
+            keywords: Keywords to search for.
+            region: The geographical region to filter jobs by (e.g., "United States").
+            sort_by: How to sort the results.(e.g., "relevance" or "date".)
+            minimum_salary_value: The minimum salary to filter for.
+            industry: The industry to filter jobs by. (e.g., "Software Development".)
+
         Returns:
-            A dictionary containing a list of comment objects and pagination details.
-        
+            A dictionary containing search results and pagination details.
+
         Raises:
             httpx.HTTPError: If the API request fails.
-        
-        Tags:
-            linkedin, post, comment, list, content, api, important
+            ValueError: If the specified location is not found.
         """
-
-        return self._unipile_app.list_post_comments(
-            post_id=post_id,
-            account_id=self.account_id,
-            comment_id=comment_id,
-            cursor=cursor,
-            limit=limit,
-        )
+        url = f"{self.base_url}/api/v1/linkedin/search"
+        params: dict[str, Any] = {"account_id": self.account_id}
+        if cursor:
+            params["cursor"] = cursor
+        if limit is not None:
+            params["limit"] = limit
+        payload: dict[str, Any] = {
+            "api": "classic",
+            "category": "jobs",
+            "minimum_salary": {"currency": "USD", "value": minimum_salary_value},
+        }
+        if keywords:
+            payload["keywords"] = keywords
+        if sort_by:
+            payload["sort_by"] = sort_by
+        if region:
+            location_id = self._get_search_parameter_id("LOCATION", region)
+            payload["region"] = location_id
+        if industry:
+            industry_id = self._get_search_parameter_id("INDUSTRY", industry)
+            payload["industry"] = [industry_id]
+        response = self._post(url, params=params, data=payload)
+        return self._handle_response(response)
 
     def list_tools(self):
         """
@@ -168,8 +352,11 @@ class ScraperApp(APIApplication):
             A list of functions that can be used as tools.
         """
         return [
-            self.linkedin_search,
             self.linkedin_list_profile_posts,
             self.linkedin_retrieve_profile,
             self.linkedin_list_post_comments,
+            self.linkedin_search_people,
+            self.linkedin_search_companies,
+            self.linkedin_search_posts,
+            self.linkedin_search_jobs,
         ]