dhisana 0.0.1.dev243__py3-none-any.whl

dhisana/utils/instantly_tools.py

@@ -0,0 +1,149 @@
+import os
+import aiohttp
+import logging
+from typing import List, Dict, Any
+from dhisana.utils.assistant_tool_tag import assistant_tool
+
+logging.basicConfig(level=logging.INFO)
+base_url = 'https://api.instantly.ai/v1'
+
+# Manage instantly lists for campaigns
+
+def get_api_key_and_headers() -> Dict[str, str]:
+    api_key = os.environ.get('INSTANTLY_API_KEY')
+    if not api_key:
+        raise ValueError(
+            "Instantly integration is not configured. Please configure the connection to Instantly in Integrations."
+        )
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+    return headers
+
+async def _handle_response(response: aiohttp.ClientResponse) -> Any:
+    if response.status == 200:
+        return await response.json()
+    elif response.status == 429:
+        raise aiohttp.ClientResponseError(
+            request_info=response.request_info,
+            history=response.history,
+            status=response.status,
+            message="Rate limit exceeded",
+            headers=response.headers
+        )
+    else:
+        error_message = await response.text()
+        logging.error(f"Error {response.status}: {error_message}")
+        response.raise_for_status()
+
+@assistant_tool
+async def add_leads_to_campaign(campaign_id: str, leads: List[Dict[str, str]]) -> Any:
+    """
+    Add leads to a campaign.
+
+    Args:
+        campaign_id (str): The ID of the campaign.
+        leads (List[Dict[str, str]]): A list of leads to add, where each lead is represented as a dictionary.
+
+    Returns:
+        Any: The response from the API.
+    """
+    url = f"{base_url}/lead/add"
+    payload = {
+        "campaign_id": campaign_id,
+        "leads": leads
+    }
+    headers = get_api_key_and_headers()
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, json=payload, headers=headers) as response:
+            return await _handle_response(response)
+
+@assistant_tool
+async def delete_leads_from_campaign(campaign_id: str, lead_emails: List[str]) -> Any:
+    """
+    Delete leads from a campaign.
+
+    Args:
+        campaign_id (str): The ID of the campaign.
+        lead_emails (List[str]): A list of lead emails to delete.
+
+    Returns:
+        Any: The response from the API.
+    """
+    url = f"{base_url}/lead/delete"
+    payload = {
+        "campaign_id": campaign_id,
+        "leads": lead_emails
+    }
+    headers = get_api_key_and_headers()
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, json=payload, headers=headers) as response:
+            return await _handle_response(response)
+
+@assistant_tool
+async def update_lead_variables(lead_email: str, variables: Dict[str, str]) -> Any:
+    """
+    Update variables for a lead.
+
+    Args:
+        lead_email (str): The email of the lead.
+        variables (Dict[str, str]): A dictionary of variables to update.
+
+    Returns:
+        Any: The response from the API.
+    """
+    url = f"{base_url}/lead/variable/update"
+    payload = {
+        "lead": lead_email,
+        "variables": variables
+    }
+    headers = get_api_key_and_headers()
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, json=payload, headers=headers) as response:
+            return await _handle_response(response)
+
+@assistant_tool
+async def set_lead_variables(lead_email: str, variables: Dict[str, str]) -> Any:
+    """
+    Set variables for a lead.
+
+    Args:
+        lead_email (str): The email of the lead.
+        variables (Dict[str, str]): A dictionary of variables to set.
+
+    Returns:
+        Any: The response from the API.
+    """
+    url = f"{base_url}/lead/variable/set"
+    payload = {
+        "lead": lead_email,
+        "variables": variables
+    }
+    headers = get_api_key_and_headers()
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, json=payload, headers=headers) as response:
+            return await _handle_response(response)
+
+@assistant_tool
+async def is_lead_in_campaign(campaign_id: str, lead_email: str) -> bool:
+    """
+    Check if a lead is in a campaign.
+
+    Args:
+        campaign_id (str): The ID of the campaign.
+        lead_email (str): The email of the lead.
+
+    Returns:
+        bool: True if the lead is in the campaign, False otherwise.
+    """
+    url = f"{base_url}/campaign/leads"
+    params = {
+        "campaign_id": campaign_id
+    }
+    headers = get_api_key_and_headers()
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, headers=headers, params=params) as response:
+            data = await _handle_response(response)
+            leads = data.get("leads", [])
+            return any(lead["email"] == lead_email for lead in leads)

dhisana/utils/linkedin_crawler.py

@@ -0,0 +1,168 @@
+# Login and Crawl linked in for relevant information in background.
+import asyncio
+import os
+import sys
+import logging
+from typing import List, Optional
+from pydantic import BaseModel
+from playwright.async_api import async_playwright
+import pandas as pd
+
+from dhisana.utils.dataframe_tools import get_structured_output
+from dhisana.utils.web_download_parse_tools import parse_html_content_as_text
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+# Pydantic models for structured data
+class LinkedInUserProfile(BaseModel):
+    first_name: Optional[str] = None
+    last_name: Optional[str] = None
+    headline: Optional[str] = None
+    location: Optional[str] = None
+    num_of_connections: Optional[int] = None
+    num_of_followers: Optional[int] = None
+    summary: Optional[str] = None
+    experience: Optional[List[str]] = None
+    education: Optional[List[str]] = None
+    skills: Optional[List[str]] = None
+    recommendations: Optional[List[str]] = None
+    accomplishments: Optional[List[str]] = None
+    interests: Optional[List[str]] = None
+    profile_url: Optional[str] = None
+
+class SalesNavigatorInsights(BaseModel):
+    sales_navigator_insight: Optional[str] = None
+    key_signals: Optional[str] = None
+    common_connection_paths: Optional[List[str]] = None
+
+
+async def get_html_content_from_url_internal(page, url):
+    """
+    Navigate to a URL using Playwright and retrieve the page content.
+    """
+    logging.info(f"Requesting {url}")
+    try:
+        await page.goto(url, timeout=10000)
+        html_content = await page.content()
+        return parse_html_content_as_text(html_content)
+    except Exception as e:
+        logging.info(f"Failed to fetch {url}: {e}")
+        return ""
+
+
+
+
+async def login_to_linkedin(page, email, password, headless):
+    """
+    Log into LinkedIn using the provided email and password.
+    """
+    await page.goto("https://www.linkedin.com/uas/login")
+    await page.wait_for_load_state('load')
+
+    await page.get_by_label("Email or Phone").click()
+    await page.get_by_label("Email or Phone").fill(email)
+    await page.get_by_label("Password").click()
+    await page.get_by_label("Password").fill(password)
+    await page.locator("#organic-div form").get_by_role("button", name="Sign in", exact=True).click()
+    await page.wait_for_load_state('load')
+
+    if "checkpoint/challenge" in page.url:
+        if not headless:
+            logger.warning("Captcha page encountered! Human intervention is needed.")
+            max_iterations = 25
+            for attempt in range(max_iterations):
+                await asyncio.sleep(3)  # Wait for 3 seconds before checking again
+                await page.wait_for_load_state('load')  # Ensure the page is loaded
+                if "checkpoint/challenge" not in page.url:
+                    logger.info("Captcha solved. Continuing with the process.")
+                    break
+            else:
+                logger.error(f"Captcha not solved after {max_iterations} attempts. Exiting.")
+                sys.exit(1)
+            await asyncio.sleep(3)
+        else:
+            logger.error("Captcha page encountered! Aborting due to headless mode.")
+            sys.exit(1)
+
+async def extract_from_page(page, url, response_type):
+    """
+    Extract structured data from a web page using OpenAI's API.
+    """
+    # Get page HTML content
+    content_text = await get_html_content_from_url_internal(page, url)
+    if not content_text:
+        return None, 'FAIL'
+
+    # Get structured content using OpenAI's API
+    extract_content, status = await get_structured_output(content_text, response_type)
+    return extract_content, status
+
+async def extract_user_content_from_linkedin(linkedin_id:str, output_csv_path:str):
+    """
+    Main function to orchestrate scraping and data extraction.
+    """
+    email = os.environ.get("LINKEDIN_EMAIL")
+    password = os.environ.get("LINKEDIN_PASSWORD")
+
+    if not email or not password:
+        logger.error("LinkedIn credentials not found in environment variables.")
+        return {"status": "FAIL", "message": "LinkedIn credentials not found in environment variables."}
+
+    # Start the browser using Playwright
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=False)
+        context = await browser.new_context()
+        page = await context.new_page()
+
+        # Login to LinkedIn
+        await login_to_linkedin(page, email, password, False)
+
+        # List of LinkedIn profiles to scrape
+        job_profiles = [
+            f"https://www.linkedin.com/in/{linkedin_id}/",
+            # Add more profile URLs as needed
+        ]
+
+        # Extract data from profiles
+        outputs = []
+        for profile in job_profiles:
+            output, status = await extract_from_page(page, profile, LinkedInUserProfile)
+            if status == 'SUCCESS':
+                outputs.append(output)
+            else:
+                outputs.append({})
+                logger.error(f"Failed to extract data from {profile}")
+
+        # Create a DataFrame from the outputs and save to CSV
+        df = pd.DataFrame(outputs)
+        csv_file_path = '/tmp/profile_data.csv'
+        df.to_csv(csv_file_path, index=False)
+        logger.info(f"Saved profile data to {csv_file_path}")
+
+        # List of Sales Navigator insights URLs to scrape
+        sales_navigator_insights = [
+            f"https://www.linkedin.com/in/{linkedin_id}/details/sales-lead-insights-details/",
+            # Add more URLs as needed
+        ]
+
+        # Extract data from Sales Navigator insights
+        insights_outputs = []
+        for profile in sales_navigator_insights:
+            output, status = await extract_from_page(page, profile, SalesNavigatorInsights)
+            if status == 'SUCCESS':
+                insights_outputs.append(output)
+            else:
+                insights_outputs.append({})
+                logger.error(f"Failed to extract data from {profile}")
+
+        # Create a DataFrame from the outputs and save to CSV
+        df_insights = pd.DataFrame(insights_outputs)
+        insights_csv_file_path = output_csv_path
+        df_insights.to_csv(insights_csv_file_path, index=False)
+        logger.info(f"Saved Sales Navigator insights to {insights_csv_file_path}")
+
+        # Close the browser
+        await browser.close()
+        return {"status": "SUCCESS", "message": f"Data extraction completed successfully to {insights_csv_file_path}."}

dhisana/utils/lusha_tools.py

@@ -0,0 +1,333 @@
+import os
+import json
+import logging
+from typing import Dict, List, Optional
+
+import aiohttp
+import backoff
+
+from dhisana.utils.cache_output_tools import cache_output, retrieve_output
+from dhisana.utils.assistant_tool_tag import assistant_tool
+
+
+def get_lusha_credentials_from_config(
+    tool_config: Optional[List[Dict]] = None
+) -> Optional[str]:
+    """
+    Retrieve Lusha API key from the tool_config (looking for 'name' == 'lusha'),
+    or fall back to environment variables if not found.
+
+    Args:
+        tool_config (List[Dict], optional): 
+            Configuration list that may contain Lusha credentials.
+
+    Returns:
+        str: Lusha API key from tool_config or environment variables
+    """
+    lusha_api_key = None
+
+    if tool_config:
+        lusha_config = next(
+            (item for item in tool_config if item.get("name") == "lusha"),
+            None
+        )
+        if lusha_config:
+            # Convert the list of dicts under 'configuration' to a map {name: value}
+            config_map = {
+                cfg["name"]: cfg["value"]
+                for cfg in lusha_config.get("configuration", [])
+                if cfg
+            }
+            lusha_api_key = config_map.get("apiKey")
+            config_map.get("apiSecret")
+
+    # Fallback to environment variables if not found in tool_config
+    lusha_api_key = lusha_api_key or os.environ.get("LUSHA_API_KEY")
+    if not lusha_api_key:
+        raise ValueError(
+            "Lusha integration is not configured. Please configure the connection to Lusha in Integrations."
+        )
+    return lusha_api_key
+
+
+@assistant_tool
+@backoff.on_exception(
+    backoff.expo,
+    (aiohttp.ClientResponseError, Exception),
+    max_tries=3,
+    giveup=lambda e: not (isinstance(e, aiohttp.ClientResponseError) and e.status == 429),
+    factor=2,
+)
+async def enrich_person_info_from_lusha(
+    linkedin_url: Optional[str] = None,
+    email: Optional[str] = None,
+    phone: Optional[str] = None,
+    tool_config: Optional[List[Dict]] = None
+) -> dict:
+    """
+    Fetch a person's details from Lusha using LinkedIn URL, email, or phone number.
+
+    Args:
+        linkedin_url (str, optional): LinkedIn profile URL of the person.
+        email (str, optional): Email address of the person.
+        phone (str, optional): Phone number of the person.
+        tool_config (List[Dict], optional): Configuration list that may contain Lusha credentials.
+
+    Returns:
+        dict: JSON response containing person information, or an error message.
+    """
+    try:
+        access_token = get_lusha_credentials_from_config(tool_config)
+    except ValueError as e:
+        return {"error": str(e)}
+
+    if not linkedin_url and not email and not phone:
+        return {"error": "At least one of linkedin_url, email, or phone must be provided"}
+
+    # Adjust these details according to Lusha’s actual enrichment endpoint and request format
+    url = "https://api.lusha.com/enrich/v1/person"
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json"
+    }
+
+    data: Dict[str, str] = {}
+    cache_key_value = None
+
+    if linkedin_url:
+        data["linkedin_url"] = linkedin_url
+        cache_key_value = linkedin_url
+    if email:
+        data["email"] = email
+    if phone:
+        data["phone"] = phone
+
+    if cache_key_value:
+        cached_response = retrieve_output("enrich_person_info_from_lusha", cache_key_value)
+        if cached_response is not None:
+            return cached_response
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, headers=headers, json=data) as response:
+            if response.status == 200:
+                json_result = await response.json()
+                if cache_key_value:
+                    cache_output("enrich_person_info_from_lusha", cache_key_value, json_result)
+                return json_result
+            elif response.status == 429:
+                logging.warning("enrich_person_info_from_lusha rate limit hit")
+                raise aiohttp.ClientResponseError(
+                    request_info=response.request_info,
+                    history=response.history,
+                    status=response.status,
+                    message="Rate limit exceeded",
+                    headers=response.headers
+                )
+            else:
+                error_result = await response.json()
+                logging.warning(
+                    f"enrich_person_info_from_lusha failed with status {response.status}: {error_result}"
+                )
+                return {"error": error_result}
+
+
+@assistant_tool
+@backoff.on_exception(
+    backoff.expo,
+    (aiohttp.ClientResponseError, Exception),
+    max_tries=3,
+    giveup=lambda e: not (isinstance(e, aiohttp.ClientResponseError) and e.status == 429),
+    factor=2,
+)
+async def enrich_organization_info_from_lusha(
+    organization_domain: Optional[str] = None,
+    tool_config: Optional[List[Dict]] = None
+) -> dict:
+    """
+    Fetch an organization's details from Lusha using the organization domain.
+
+    Args:
+        organization_domain (str, optional): Domain of the organization.
+        tool_config (List[Dict], optional): Configuration list that may contain Lusha credentials.
+
+    Returns:
+        dict: JSON response containing organization information, or an error message.
+    """
+    access_token = get_lusha_credentials_from_config(tool_config)
+    if not access_token:
+        return {"error": "Failed to obtain Lusha access token"}
+
+    if not organization_domain:
+        return {"error": "Organization domain must be provided"}
+
+    # Adjust these details according to Lusha’s actual company enrichment endpoint
+    url = "https://api.lusha.com/enrich/v1/company"
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json"
+    }
+
+    cached_response = retrieve_output("enrich_organization_info_from_lusha", organization_domain)
+    if cached_response is not None:
+        return cached_response
+
+    data = {"domain": organization_domain}
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, headers=headers, json=data) as response:
+            if response.status == 200:
+                json_result = await response.json()
+                cache_output("enrich_organization_info_from_lusha", organization_domain, json_result)
+                return json_result
+            elif response.status == 429:
+                logging.warning("enrich_organization_info_from_lusha rate limit hit")
+                raise aiohttp.ClientResponseError(
+                    request_info=response.request_info,
+                    history=response.history,
+                    status=response.status,
+                    message="Rate limit exceeded",
+                    headers=response.headers
+                )
+            else:
+                error_result = await response.json()
+                logging.warning(
+                    f"enrich_organization_info_from_lusha failed with status {response.status}: {error_result}"
+                )
+                return {"error": error_result}
+
+
+async def enrich_user_info_with_lusha(
+    input_user_properties: dict,
+    tool_config: Optional[List[Dict]] = None
+) -> dict:
+    """
+    Update user info using Lusha data. Checks LinkedIn URL, fetches data, and updates
+    the user's properties accordingly.
+
+    Args:
+        input_user_properties (dict): Existing properties about the user.
+        tool_config (List[Dict], optional): Configuration list that may contain Lusha credentials.
+
+    Returns:
+        dict: Updated user properties dictionary with Lusha data.
+    """
+    linkedin_url = input_user_properties.get("user_linkedin_url", "")
+    if not linkedin_url:
+        input_user_properties["linkedin_url_match"] = False
+        return input_user_properties
+
+    # Fetch person data from Lusha
+    lusha_data = await enrich_person_info_from_lusha(
+        linkedin_url=linkedin_url,
+        tool_config=tool_config
+    )
+    if not lusha_data:
+        input_user_properties["linkedin_url_match"] = False
+        return input_user_properties
+
+    person_data = lusha_data.get("person", {})
+    additional_props = input_user_properties.get("additional_properties") or {}
+    additional_props["lusha_person_data"] = json.dumps(person_data)
+    input_user_properties["additional_properties"] = additional_props
+
+    # Fill missing contact info
+    if not input_user_properties.get("email"):
+        input_user_properties["email"] = person_data.get("email", "")
+    if not input_user_properties.get("phone"):
+        input_user_properties["phone"] = person_data.get("phone", "")
+
+    # Map some fields
+    if person_data.get("name"):
+        input_user_properties["full_name"] = person_data["name"]
+    if person_data.get("first_name"):
+        input_user_properties["first_name"] = person_data["first_name"]
+    if person_data.get("last_name"):
+        input_user_properties["last_name"] = person_data["last_name"]
+    if person_data.get("linkedin_url"):
+        input_user_properties["user_linkedin_url"] = person_data["linkedin_url"]
+    if person_data.get("company") and person_data["company"].get("domain"):
+        input_user_properties["primary_domain_of_organization"] = person_data["company"]["domain"]
+    if person_data.get("title"):
+        input_user_properties["job_title"] = person_data["title"]
+    if person_data.get("headline"):
+        input_user_properties["headline"] = person_data["headline"]
+    if person_data.get("company") and person_data["company"].get("name"):
+        input_user_properties["organization_name"] = person_data["company"]["name"]
+    if person_data.get("company") and person_data["company"].get("website"):
+        input_user_properties["organization_website"] = person_data["company"]["website"]
+    if person_data.get("headline") and not input_user_properties.get("summary_about_lead"):
+        input_user_properties["summary_about_lead"] = person_data["headline"]
+
+    # Example: If Lusha provides a list of "keywords" in the company object
+    if person_data.get("company") and person_data["company"].get("keywords"):
+        input_user_properties["keywords"] = ", ".join(person_data["company"]["keywords"])
+
+    # Derive location
+    if person_data.get("city") or person_data.get("state"):
+        input_user_properties["lead_location"] = (
+            f"{person_data.get('city', '')}, {person_data.get('state', '')}".strip(", ")
+        )
+
+    # Check for a match
+    first_matched = bool(
+        input_user_properties.get("first_name")
+        and person_data.get("first_name") == input_user_properties["first_name"]
+    )
+    last_matched = bool(
+        input_user_properties.get("last_name")
+        and person_data.get("last_name") == input_user_properties["last_name"]
+    )
+    if first_matched and last_matched:
+        input_user_properties["linkedin_url_match"] = True
+
+    return input_user_properties
+
+
+@assistant_tool
+@backoff.on_exception(
+    backoff.expo,
+    (aiohttp.ClientResponseError, Exception),
+    max_tries=3,
+    giveup=lambda e: not (isinstance(e, aiohttp.ClientResponseError) and e.status == 429),
+    factor=2,
+)
+async def get_person_info_from_lusha(
+    first_name: str,
+    last_name: str,
+    company_name: str,
+    tool_config: Optional[List[Dict]] = None
+) -> dict:
+    """
+    Calls Lusha v2 GET endpoint with firstName, lastName, and companyName.
+    """
+    lusha_api_key = get_lusha_credentials_from_config(tool_config)
+    if not lusha_api_key:
+        return {"error": "No Lusha API key found."}
+
+    url = "https://api.lusha.com/v2/person"
+    headers = {"api_key": lusha_api_key}
+    params = {
+        "firstName": first_name,
+        "lastName": last_name,
+        "companyName": company_name
+    }
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, headers=headers, params=params) as response:
+            if response.status == 200:
+                return await response.json()
+            elif response.status == 429:
+                logging.warning("get_person_info_from_lusha rate limit hit")
+                raise aiohttp.ClientResponseError(
+                    request_info=response.request_info,
+                    history=response.history,
+                    status=response.status,
+                    message="Rate limit exceeded",
+                    headers=response.headers
+                )
+            else:
+                error_result = await response.json()
+                logging.warning(
+                    f"get_person_info_from_lusha failed with status {response.status}: {error_result}"
+                )
+                return {"error": error_result}