dhisana 0.0.1.dev243__py3-none-any.whl

dhisana/utils/extract_email_content_for_llm.py

@@ -0,0 +1,101 @@
+from typing import Any, Dict
+import base64
+import re
+
+def decode_base64(data: str) -> str:
+    """
+    Decode a base64- or web-safe-base64-encoded string.
+    """
+    if not data:
+        return ""
+    missing_padding = len(data) % 4
+    if missing_padding:
+        data += '=' * (4 - missing_padding)
+    return base64.urlsafe_b64decode(data).decode('utf-8', errors='ignore')
+
+def html_to_text(html: str) -> str:
+    """
+    (Optional) Convert HTML to plain text using a simple regex.
+    This is not bulletproof for all HTML, but often fine for short email bodies.
+    """
+    text = re.sub(r'<br\s*/?>', '\n', html, flags=re.IGNORECASE)     # <br> to newline
+    text = re.sub(r'<.*?>', '', text)                                # remove remaining tags
+    return text.strip()
+
+def get_text_content(payload: Dict[str, Any]) -> str:
+    """
+    Recursively extract 'text/plain' content from the Gmail message payload.
+    If no text/plain is found, fallback to 'text/html' (converted to plain text).
+    """
+    # If there's a direct 'parts' list, we may need to walk each part.
+    if 'parts' in payload:
+        extracted = []
+        for part in payload['parts']:
+            extracted.append(get_text_content(part))
+        return "\n".join(filter(None, extracted))
+    
+    # If this part has a mimeType and a body, try to decode.
+    mime_type = payload.get('mimeType', '')
+    body_data = payload.get('body', {}).get('data', '')
+    
+    # If it's text/plain, decode base64 and return
+    if mime_type == 'text/plain' and body_data:
+        return decode_base64(body_data)
+    
+    # If it's text/html (and we haven't returned text/plain yet), fallback
+    if mime_type == 'text/html' and body_data:
+        html_content = decode_base64(body_data)
+        return html_to_text(html_content)
+    
+    return ""
+
+def trim_repeated_quoted_lines(text: str) -> str:
+    """
+    (Optional) Try to remove repeated or quoted content from replies.
+    This is a naive approach—real-world heuristics can get quite complicated.
+    """
+    # Common patterns: lines starting with ">"
+    # or lines starting with "On <date>, <someone> wrote:"
+    lines = text.splitlines()
+    filtered_lines = []
+    for line in lines:
+        if line.startswith(">"):
+            continue
+        # You can add more heuristics for removing signature blocks or repeated disclaimers
+        filtered_lines.append(line)
+    return "\n".join(filtered_lines).strip()
+
+def extract_email_content_for_llm(email_details: Dict[str, Any]) -> str:
+    """
+    Cleans up, extracts, and formats the relevant text content from a single Gmail message.
+    If you want the entire thread, call the Gmail API for all messages in the thread and
+    combine them. This function handles one message in detail, recursively extracting
+    text from multiple MIME parts.
+    """
+    if not email_details or 'payload' not in email_details:
+        return "No valid email details found."
+
+    # Extract basic headers
+    headers_map = {h['name']: h['value'] for h in email_details['payload'].get('headers', [])}
+    
+    sender = headers_map.get('From', 'Unknown Sender')
+    receiver = headers_map.get('To', 'Unknown Receiver')
+    date = headers_map.get('Date', 'Unknown Date')
+    subject = headers_map.get('Subject', 'No Subject')
+    
+    # Recursively get text from payload
+    body = get_text_content(email_details['payload'])
+    
+    # Optionally remove some repeated lines
+    body = trim_repeated_quoted_lines(body)
+    
+    # Format final string
+    formatted_content = (
+        f"From: {sender}\n"
+        f"To: {receiver}\n"
+        f"Date: {date}\n"
+        f"Subject: {subject}\n\n"
+        f"{body}"
+    )
+    
+    return formatted_content

dhisana/utils/fetch_openai_config.py

@@ -0,0 +1,129 @@
+"""
+Unified OpenAI / Azure OpenAI helper (no env-fallback for secrets)
+=================================================================
+
+Resolution order
+----------------
+1. If `tool_config` has a **"openai"** block  → public OpenAI
+2. Else if it has an **"azure_openai"** block → Azure OpenAI
+3. Otherwise                            → raise ValueError
+
+`api_key` **and** `endpoint` (for Azure) must therefore be supplied in
+`tool_config`.  They will never be read from the host environment.
+
+Optional:
+  • `AZURE_OPENAI_API_VERSION` – defaults to 2025-03-01-preview
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Dict, List, Optional, Tuple, Union
+
+from openai import AsyncOpenAI, OpenAI, AzureOpenAI, AsyncAzureOpenAI
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 1. Helpers: config parsing
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _extract_config(
+    tool_config: Optional[List[Dict]], provider_name: str
+) -> Dict[str, str]:
+    """Return the config map for the requested provider name, else {}."""
+    if not tool_config:
+        return {}
+    block = next((b for b in tool_config if b.get("name") == provider_name), {})
+    return {entry["name"]: entry["value"] for entry in block.get("configuration", []) if entry}
+
+
+def _discover_credentials(
+    tool_config: Optional[List[Dict]] = None,
+) -> Tuple[str, str, Optional[str]]:
+    """
+    Return (provider, api_key, endpoint_or_None).
+
+    provider ∈ {"public", "azure"}
+    """
+    # 1️⃣ Public OpenAI
+    openai_cfg = _extract_config(tool_config, "openai")
+    if openai_cfg:
+        key = openai_cfg.get("apiKey")
+        if not key:
+            raise ValueError(
+                "OpenAI integration is not configured. Please configure the connection to OpenAI in Integrations."
+            )
+        return "public", key, None
+
+    # 2️⃣ Azure OpenAI
+    azure_cfg = _extract_config(tool_config, "azure_openai")
+    if azure_cfg:
+        key = azure_cfg.get("apiKey")
+        endpoint = azure_cfg.get("endpoint")
+        if not key or not endpoint:
+            raise ValueError(
+                "Azure OpenAI integration is not configured. Please configure the connection to Azure OpenAI in Integrations."
+            )
+        return "azure", key, endpoint
+
+    # 3️⃣ Neither block present → error
+    raise ValueError(
+        "OpenAI integration is not configured. Please configure the connection to OpenAI in Integrations."
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 2. Client factories
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _api_version() -> str:
+    """Return the Azure API version (env-controlled, no secret)."""
+    return os.getenv("AZURE_OPENAI_API_VERSION", "2025-03-01-preview")
+
+
+def create_openai_client(
+    tool_config: Optional[List[Dict]] = None,
+) -> Union[OpenAI, AzureOpenAI]:
+    """
+    Return a *synchronous* client:
+        • openai.OpenAI      – public service
+        • openai.AzureOpenAI – Azure
+    """
+    provider, key, endpoint = _discover_credentials(tool_config)
+
+    if provider == "public":
+        return OpenAI(api_key=key)
+
+    # Azure
+    return AzureOpenAI(api_key=key, azure_endpoint=endpoint, api_version=_api_version())
+
+
+def create_async_openai_client(
+    tool_config: Optional[List[Dict]] = None,
+) -> AsyncOpenAI:
+    """
+    Return an *async* client (AsyncOpenAI).
+
+    For Azure we pass both `azure_endpoint` and `api_version`.
+    """
+    provider, key, endpoint = _discover_credentials(tool_config)
+
+    if provider == "public":
+        return AsyncOpenAI(api_key=key)
+
+    return AsyncAzureOpenAI(
+        api_key=key,
+        azure_endpoint=endpoint,
+        api_version=_api_version(),
+    )
+
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# 3. Convenience helper (legacy)
+# ─────────────────────────────────────────────────────────────────────────────
+
+def get_openai_access_token(tool_config: Optional[List[Dict]] = None) -> str:
+    """Return just the API key (legacy helper)."""
+    _, key, _ = _discover_credentials(tool_config)
+    return key

dhisana/utils/field_validators.py

@@ -0,0 +1,426 @@
+import logging
+from urllib.parse import urlparse
+import urllib.parse
+import re
+
+from email_validator import validate_email, EmailNotValidError
+from fqdn import FQDN
+
+logger = logging.getLogger(__name__)
+
+# -------------------------------------------------------------------------------------
+# Utility sets and patterns
+# -------------------------------------------------------------------------------------
+PLACEHOLDER_EMAILS = {
+    "test@example.com",
+    "test@test.com",
+    "test@domain.com",
+    "user@domain.com",
+    "user@example.com",
+    "user@yourdomain.com",
+    "no-reply@example.com",
+    "no-reply@domain.com",
+    "no-reply@yourdomain.com",
+    "admin@domain.com",
+    "contact@domain.com",
+    "info@domain.com",
+    "none@none.com",
+    "none@domain.com",
+    "noemail@noemail.com",
+    "test@fake.com",
+    "test@demo.com",
+    "test@testing.com",
+    "test@local.com",
+    "fake@fake.com",
+    "email@email.com",
+    "asdf@asdf.com",
+    "qwerty@qwerty.com",
+    "xxx@xxx.com",
+    "aaaa@aaaa.com",
+    "nomail@nomail.com",
+    "dontreply@dontreply.com",
+    "asdasd@asdasd.com",
+    "abcdefg@abcdefg.com",
+    "123@123.com",
+    "test123@test.com",
+}
+
+DISPOSABLE_DOMAINS = {
+    "mailinator.com",
+    "10minutemail.com",
+    "yopmail.com",
+    "guerrillamail.com",
+    "tempmail.com",
+    "fakemailgenerator.com",
+    "mytrashmail.com",
+    "getnada.com",
+    "throwawaymail.com",
+    "sharklasers.com",
+    "maildrop.cc",
+    "discard.email",
+    "temporaryemail.com",
+    "trashmail.com",
+    "mohmal.com",
+    "mail.tm",
+    "mailsac.com",
+    "mailcatch.com",
+    "temp-mail.org",
+    "emailondeck.com",
+    "mailinabox.email",
+    "spambog.com",
+    "mintemail.com",
+    "spam4.me",
+    "spambox.us",
+    "edumail.rocks",
+    "getairmail.com",
+    "mailnesia.com",
+    "spoofmail.de",
+    "dropmail.me",
+    "tempmailaddress.com",
+    "33mail.com",
+    "incognitomail.com",
+    "tempemail.co",
+    "trbvm.com",
+    "online.ms",
+    "20mail.in",
+    "wavee.net",
+    "ephemeral.email",
+    "bccto.me",
+    "cuvox.de",
+    "dispostable.com",
+    "easytrashmail.com",
+    "email-fake.org",
+    "emailtemporario.com.br",
+    "fleckens.hu",
+    "lroid.com",
+    "mail-temporaire.fr",
+    "mailate.com",
+    "mailfever.com",
+    "mailforspam.com",
+    "mailfreeonline.com",
+    "mailhazard.com",
+    "mailimate.com",
+    "mailin8r.com",
+    "mailincubator.com",
+    "mailmoat.com",
+    "mailzilla.org",
+    "notsharingmy.info",
+    "objectmail.com",
+    "proxymail.eu",
+    "spamdecoy.net",
+    "spamfree24.org",
+    "spamgourmet.com",
+    "spamify.com",
+    "spamomatic.com",
+    "spamspot.com",
+    "superrito.com",
+    "teleworm.us",
+    "trash-amil.com",
+    "trashmail.me",
+    "trashmail.net",
+    "wegwerfemail.de",
+    "wh4f.org",
+    "zmail.ru",
+    # Add more as needed
+}
+
+SPAMMY_PATTERN = re.compile(r"(.)\1{3,}")  # e.g., 4+ repeated characters
+
+BLOCKED_DOMAINS = {
+    # URL shorteners and link forwarders
+    "bit.ly",
+    "tinyurl.com",
+    "t.co",
+    "ow.ly",
+    "is.gd",
+    "cutt.ly",
+    "bit.do",
+    "buff.ly",
+    "rebrand.ly",
+    "rebrandly.com",
+    "snip.ly",
+    "shorte.st",
+    "soo.gd",
+    "shorturl.at",
+    "adf.ly",
+    # Bio / profile link aggregators
+    "linktr.ee",
+    "linktree.com",
+    "linkin.bio",
+    "campsite.bio",
+    "bio.link",
+    "bio.site",
+    "bio.fm",
+    "milkshake.app",
+    "lnk.bio",
+    "withkoji.com",
+    "about.me",
+    "carrd.co",
+    # Large social platforms (block if used as “organization domain”)
+    "facebook.com",
+    "instagram.com",
+    "linkedin.com",
+    "youtube.com",
+    "yelp.com",
+    "twitter.com",
+    "tiktok.com",
+    "pinterest.com",
+    "reddit.com",
+    "snapchat.com",
+    "tumblr.com",
+    "vimeo.com",
+    "flickr.com",
+    "wechat.com",
+    "qq.com",
+}
+
+
+# -------------------------------------------------------------------------------------
+# Helper: Check FQDN validity
+# -------------------------------------------------------------------------------------
+def is_valid_fqdn(domain: str) -> bool:
+    """
+    Returns True if `domain` is a syntactically valid Fully Qualified Domain Name.
+    """
+    try:
+        if not domain or not isinstance(domain, str):
+            return False
+        fqdn_obj = FQDN(domain)
+        return fqdn_obj.is_valid
+    except Exception:
+        return False
+
+
+# -------------------------------------------------------------------------------------
+# Domain validation for organizations
+# -------------------------------------------------------------------------------------
+def validation_organization_domain(domain: str) -> str:
+    """
+    1. Lowercases/strips the input domain string.
+    2. Checks if the domain is in (or a subdomain of) a blocked set.
+    3. Checks if the domain is a valid FQDN.
+    4. Returns '' if blocked or invalid, otherwise returns the normalized domain.
+    """
+    if not domain or not isinstance(domain, str):
+        return ""
+
+    domain = domain.strip().lower()
+    # If domain exactly matches OR is a subdomain of any blocked domain
+    if any(domain == blocked or domain.endswith(f".{blocked}") for blocked in BLOCKED_DOMAINS):
+        return ""
+
+    # Otherwise, confirm valid FQDN
+    return domain if is_valid_fqdn(domain) else ""
+
+
+# -------------------------------------------------------------------------------------
+# Email validation & cleaning
+# -------------------------------------------------------------------------------------
+def validate_and_clean_email(email: str) -> str:
+    """
+    Return a validated, normalized email string or '' if invalid/unwanted.
+    Checks:
+      1. Syntax / deliverability via email_validator
+      2. Against placeholder/fake emails
+      3. Against disposable email domains
+      4. Spammy repeated character patterns
+    """
+    if not email or not isinstance(email, str):
+        return ""
+
+    try:
+        v = validate_email(email, check_deliverability=True)
+        normalized_email = v["email"]  # canonical form
+        local_part, domain_part = normalized_email.rsplit("@", 1)
+
+        # 1. Check entire address in placeholder set
+        if normalized_email.lower() in PLACEHOLDER_EMAILS:
+            return ""
+
+        # 2. Check domain in disposable set
+        if domain_part.lower() in DISPOSABLE_DOMAINS:
+            return ""
+
+        # 3. Check repeated/spammy pattern
+        if SPAMMY_PATTERN.search(normalized_email):
+            return ""
+
+        return normalized_email
+    except EmailNotValidError:
+        return ""
+
+
+# -------------------------------------------------------------------------------------
+# Website URL validation
+# -------------------------------------------------------------------------------------
+def validate_website_url(raw_url: str) -> str:
+    """
+    Validate a website URL (must be http/https or a raw domain).
+    If no scheme is provided but the input is a valid FQDN-like string
+    (e.g. www.google.com), automatically prefix https://.
+    Return the normalized URL without query/fragment, or '' if invalid.
+    """
+    if not raw_url or not isinstance(raw_url, str):
+        return ""
+
+    # Clean input
+    raw_url = raw_url.strip().lower()
+    
+    try:
+        parsed = urllib.parse.urlparse(raw_url)
+        
+        # If there's no scheme, try prefixing https://
+        if not parsed.scheme:
+            # Example: "www.google.com" => "https://www.google.com"
+            potential_url = f"https://{raw_url}"
+            test_parsed = urllib.parse.urlparse(potential_url)
+            
+            # If that yields a valid scheme and netloc, use it
+            if test_parsed.scheme in ["http", "https"] and test_parsed.netloc:
+                parsed = test_parsed
+            else:
+                return ""
+        
+        # Check we now have a valid scheme and netloc
+        if parsed.scheme not in ["http", "https"] or not parsed.netloc:
+            return ""
+        
+        # Normalize by removing query and fragment parts
+        normalized = urllib.parse.urlunparse(
+            (parsed.scheme, parsed.netloc, parsed.path, "", "", "")
+        )
+        return normalized
+    
+    except Exception:
+        return ""
+
+
+# -------------------------------------------------------------------------------------
+# LinkedIn URL Normalizers
+# -------------------------------------------------------------------------------------
+def normalize_linkedin_url(raw_url: str) -> str:
+    """
+    Normalize a personal LinkedIn URL to the form: https://www.linkedin.com/in/<something>
+    Must contain '/in/'. Otherwise, return ''.
+    """
+    if not raw_url or not isinstance(raw_url, str):
+        return ""
+
+    try:
+        raw_url = raw_url.strip()
+        parsed = urlparse(raw_url)
+
+        if not parsed.scheme or not parsed.netloc:
+            return ""
+
+        if "linkedin.com" not in parsed.netloc.lower():
+            return ""
+
+        url = raw_url.rstrip("/")
+        parsed = urlparse(url)
+
+        if "/in/" not in parsed.path.lower():
+            return ""
+
+        path = parsed.path.lstrip("/")
+        return f"https://www.linkedin.com/{path}".rstrip("/")
+    except Exception:
+        return ""
+
+
+def normalize_linkedin_company_url(raw_url: str) -> str:
+    """
+    Normalize a company LinkedIn URL to the form: https://www.linkedin.com/company/<something>
+    Must contain '/company/'. Otherwise, return ''.
+    """
+    if not raw_url or not isinstance(raw_url, str):
+        return ""
+
+    try:
+        raw_url = raw_url.strip()
+        parsed = urlparse(raw_url)
+
+        if not parsed.scheme or not parsed.netloc:
+            return ""
+
+        if "linkedin.com" not in parsed.netloc.lower():
+            return ""
+
+        url = raw_url.rstrip("/")
+        parsed = urlparse(url)
+
+        if "/company/" not in parsed.path.lower():
+            return ""
+
+        path = parsed.path.lstrip("/")
+        return f"https://www.linkedin.com/{path}".rstrip("/")
+    except Exception:
+        return ""
+
+
+def normalize_linkedin_company_salesnav_url(raw_url: str) -> str:
+    """
+    Normalize a company Sales Navigator URL to: https://www.linkedin.com/sales/company/<something>
+    Must contain '/sales/company/'. Otherwise, return ''.
+    """
+    if not raw_url or not isinstance(raw_url, str):
+        return ""
+
+    try:
+        raw_url = raw_url.strip()
+        parsed = urlparse(raw_url)
+
+        if not parsed.scheme or not parsed.netloc:
+            return ""
+
+        if "linkedin.com" not in parsed.netloc.lower():
+            return ""
+
+        url = raw_url.rstrip("/")
+        parsed = urlparse(url)
+
+        if "/sales/company/" not in parsed.path.lower():
+            return ""
+
+        path = parsed.path.lstrip("/")
+        return f"https://www.linkedin.com/{path}".rstrip("/")
+    except Exception:
+        return ""
+
+
+def normalize_salesnav_url(raw_url: str) -> str:
+    """
+    Normalize a Sales Navigator URL to: https://www.linkedin.com/sales/lead/<something>
+    Must contain '/sales/lead/'. Otherwise, return ''.
+    Strips anything after a comma in the URL.
+    """
+    if not raw_url or not isinstance(raw_url, str):
+        return ""
+
+    try:
+        raw_url = raw_url.strip()
+        parsed_initial = urlparse(raw_url)
+
+        if not parsed_initial.scheme or not parsed_initial.netloc:
+            return ""
+
+        if "linkedin.com" not in parsed_initial.netloc.lower():
+            return ""
+
+        # Remove trailing slash
+        url = raw_url.rstrip("/")
+
+        # Strip anything after the first comma
+        comma_idx = url.find(",")
+        if comma_idx != -1:
+            url = url[:comma_idx]
+
+        parsed = urlparse(url)
+
+        if "/sales/lead/" not in parsed.path.lower():
+            return ""
+
+        path = parsed.path.lstrip("/")
+        return f"https://www.linkedin.com/{path}".rstrip("/")
+    except Exception:
+        return ""