dhisana 0.0.1.dev243__py3-none-any.whl

dhisana/utils/trasform_json.py

@@ -0,0 +1,173 @@
+import json
+from typing import Any, Dict, List, Optional, Type
+
+from pydantic import BaseModel
+
+from dhisana.utils.assistant_tool_tag import assistant_tool
+from dhisana.utils.generate_structured_output_internal import get_structured_output_internal
+
+GLOBAL_GENERATED_PYTHON_CODE = {}
+import logging
+logger = logging.getLogger(__name__)
+
+class GeneratedPythonCode(BaseModel):
+    python_code: str
+
+
+@assistant_tool
+async def transform_json_code(
+    input_json: str, 
+    output_json: str, 
+    function_name: str, 
+    tool_config: Optional[List[Dict]] = None
+) -> str:
+    """
+    Use LLM to generate python code to transform JSON from format X to format Y.
+    Save that to a GLOBAL variable.
+
+    Args:
+        input_json (str): Example input JSON.
+        output_json (str): Example output JSON.
+        function_name (str): The generated python function should be saved as.
+        tool_config (Optional[List[Dict]]): Optional tool configuration.
+
+    Returns:
+        str: Function name that was saved to the global scope.
+    """
+    max_retries = 3
+    error_message = ""
+
+    for attempt in range(max_retries):
+        # Prepare the message
+        message = f"""
+        Given the following input and output JSON schemas, generate a Python function that transforms the input JSON to the output JSON.
+        Example Input JSON:
+        {input_json}
+        Example Output JSON:
+        {output_json}
+        Name the function as:
+        {function_name}
+        Check for NoneType in code before any concatenation and make sure errors do not happen like 
+        "unsupported operand type(s) for +: 'NoneType' and 'str'".
+        Preserve the output type to be of the type in output JSON. Convert input field to string and 
+        assign to output field if types don't match.
+        Return the function code in 'python_code'. Do not include any imports or explanations; only 
+        provide the '{function_name}' code that takes 'input_json' as input and returns the transformed 
+        'output_json' as output.
+        """
+        if error_message:
+            message += f"\nThe previous attempt returned the following error:\n{error_message}\nPlease fix the function."
+
+        # Get structured output
+        generated_python_code, status = await get_structured_output_internal(message, GeneratedPythonCode, tool_config=tool_config)
+        if status == 'SUCCESS' and generated_python_code and generated_python_code.python_code:
+            function_string = generated_python_code.python_code
+            # Execute the generated function
+            try:
+                exec(function_string, globals())
+                # Test the function
+                input_data = json.loads(input_json)
+                output_data = globals()[function_name](input_data)
+                if output_data:
+                    # Store the function code
+                    GLOBAL_GENERATED_PYTHON_CODE[function_name] = globals()[function_name]
+                    return function_name
+                else:
+                    error_message = "The function did not produce the expected output."
+            except Exception as e:
+                error_message = str(e)
+        else:
+            error_message = "Failed to generate valid Python code."
+
+        if attempt == max_retries - 1:
+            raise RuntimeError(f"Error executing generated function after {max_retries} attempts: {error_message}")
+
+
+@assistant_tool
+async def transform_json_with_type(
+    input_json_str: str, 
+    response_type: Type[BaseModel], 
+    function_name: str, 
+    tool_config: Optional[List[Dict]] = None
+):
+    """
+    Transforms the input JSON into the format specified by the given Pydantic response type.
+
+    Args:
+        input_json_str (str): The input JSON string to be transformed.
+        response_type (Type[BaseModel]): The Pydantic model defining the desired output format.
+        function_name (str): The name of the function to generate and execute.
+        tool_config (Optional[List[Dict]]): Optional tool configuration.
+
+    Returns:
+        The transformed JSON string matching the response_type format.
+    """
+    # Create a sample instance of the Pydantic model
+    sample_instance = response_type.construct()
+    # Convert the instance to JSON
+    response_type_json_str = sample_instance.json()
+    return await transform_json_code(input_json_str, response_type_json_str, function_name, tool_config=tool_config)
+
+
+# ----------------------------------------------------
+# Property Mapping (LLM-based)
+# ----------------------------------------------------
+class PropertyMapping(BaseModel):
+    input_property_name: str
+    mapped_property_name: str
+
+
+class PropertyMappingList(BaseModel):
+    properties: List[PropertyMapping]
+
+
+async def create_property_mapping(
+    sample_input: Dict[str, Any],
+    required_fields: List[str],
+    entity_type: str,
+    tool_config: Optional[List[Dict[str, Any]]] = None
+) -> Dict[str, str]:
+    """
+    Generate a property mapping from the input fields to the required fields for either a
+    Lead or an Account (Company). Calls an LLM to produce a JSON dictionary of field mappings.
+
+    :param sample_input: A sample dictionary from the input data.
+    :param required_fields: A list of fields we want to map to (e.g. ["organization_name", "first_name"]).
+    :param entity_type: "lead" or "account", used in the prompt to clarify context for the LLM.
+    :param tool_config: Optional LLM config.
+
+    :return: Dict of {"existingField": "requiredFieldName", ...}
+    """
+    # We'll only show the top-level of sample_input in the prompt for brevity
+    truncated_sample = {k: str(sample_input[k])[:128] for k in list(sample_input.keys())}
+
+    # Prepare a textual prompt for the LLM
+    user_prompt = f"""
+    The user has data representing a {entity_type} but the fields may not match the required format.
+    Required fields are: {required_fields}.
+    A sample of the input is: {json.dumps(truncated_sample, indent=2)}
+
+    Please output a JSON output mapping input_property_name to mapped_property_name.
+    You MUST map only one input property to one output property.
+    If a input property does not match any required field, you can skip mapping it. Map the best match.
+    DO NOT map the same input property to multiple output properties.
+    """
+
+    logger.info(f"Asking LLM to create property mapping for entity_type='{entity_type}'...")
+
+    response, status = await get_structured_output_internal(
+        prompt=user_prompt,
+        response_format=PropertyMappingList,
+        effort="high",
+        model="gpt-5.1-chat",
+        tool_config=tool_config
+    )
+    if status == "SUCCESS" and response and response.properties:
+        mapping = {}
+        for prop in response.properties:
+            mapping[prop.input_property_name] = prop.mapped_property_name
+        return mapping
+    else:
+        logger.warning("Could not generate property mapping from LLM. Returning empty mapping.")
+        return {}
+

dhisana/utils/web_download_parse_tools.py

@@ -0,0 +1,189 @@
+# Tools to download and parse web content.
+# Uses plyaWright to fetch HTML content from a URL, parse HTML content as text, and extract structured data from HTML content.
+
+import csv
+import logging
+import os
+from bs4 import BeautifulSoup
+import html2text
+from playwright.async_api import async_playwright
+from dhisana.utils.assistant_tool_tag import assistant_tool
+from urllib.parse import urlparse
+import re
+from datetime import datetime
+from dhisana.utils.dataframe_tools import get_structured_output
+
+
+@assistant_tool
+def parse_html_content_as_text(html_content):
+    h = html2text.HTML2Text()
+    h.ignore_links = False  # Keeps links in the markdown
+    h.ignore_images = True  # Removes images
+    return h.handle(html_content)
+
+@assistant_tool
+async def standardize_url(url):
+    parsed_url = urlparse(url)
+    if not parsed_url.scheme:
+        url = "https://" + url
+        parsed_url = urlparse(url)
+    if parsed_url.hostname and parsed_url.hostname.count('.') == 1:
+        url = url.replace(parsed_url.hostname, "www." + parsed_url.hostname)
+    return url
+
+@assistant_tool
+async def fetch_html_content(url):
+    url = await standardize_url(url)
+    async with async_playwright() as playwright:
+        browser = await playwright.chromium.launch(headless=True)
+        context = await browser.new_context()
+        page = await context.new_page()
+        logging.info(f"Requesting {url}")
+        try:
+            await page.goto(url, timeout=10000)
+            return await page.content()
+        except Exception as e:
+            logging.info(f"Failed to fetch {url}: {e}")
+            return ""
+        finally:
+            await browser.close()
+
+@assistant_tool
+async def get_html_content_from_url(url):
+    html_content = await fetch_html_content(url)
+    return await clean_html_content(html_content)
+
+@assistant_tool
+async def get_text_content_from_url(url):
+    html_content = await fetch_html_content(url)
+    return await parse_text_content(html_content)
+
+@assistant_tool
+async def parse_text_content(html_content):
+    if not html_content:
+        return ""
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for element in soup(['script', 'style', 'meta', 'code', 'svg']):
+        element.decompose()
+    return soup.get_text(separator=' ', strip=True)
+
+@assistant_tool
+async def clean_html_content(html_content):
+    if not html_content:
+        return ""
+    soup = BeautifulSoup(html_content, 'html.parser')
+    for element in soup(['script', 'style', 'meta', 'code', 'svg']):
+        element.decompose()
+    return str(soup)
+
+@assistant_tool
+async def process_files_in_folder_for_leads(folder_path: str, file_extension: str, response_list_type, response_item_type, output_file: str):
+    """
+    Process files in a folder, extract structured data, and write to a CSV file using properties from response_item_type.
+
+    Parameters:
+    - folder_path (str): The path to the folder containing files.
+    - file_extension (str): The file extension to filter files (e.g., '.html').
+    - response_list_type: The type of response expected from get_structured_output.
+    - response_item_type: The Pydantic model for each item in the response.
+    - output_file (str): The path where the output CSV file will be saved.
+
+    Returns:
+    - str: The file path of the generated CSV file.
+    """
+
+    # Ensure the parent directory of output_file exists
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+    # Use the properties from response_item_type for headers
+    keys = list(response_item_type.__fields__.keys())
+
+    # Open the CSV file in write mode initially to write the header
+    with open(output_file, 'w', newline='') as csv_file:
+        dict_writer = csv.DictWriter(csv_file, fieldnames=keys)
+        dict_writer.writeheader()
+
+    # Process each file and append to the CSV file
+    for file_name in os.listdir(folder_path):
+        if file_name.endswith(file_extension):
+            file_path = os.path.join(folder_path, file_name)
+            with open(file_path, 'r') as file:
+                html_content = file.read()
+                parsed_content = parse_html_content_as_text(html_content)
+                prompt = "Extract structured content from input. Output is in JSON Format. DO NOT make up values. Use what is provided in input. \n\n Input: " + parsed_content
+                prompt = prompt[:1040000]  # Limit prompt length to 1040000 characters
+                structured_data, result = await get_structured_output(parsed_content, response_list_type)
+                if result != 'SUCCESS':
+                    logging.warning(f"Failed to extract structured data from {file_name}: {structured_data}")
+                    continue
+                for item in structured_data.data:
+                    # Append each item to the CSV file immediately
+                    with open(output_file, 'a', newline='') as csv_file:
+                        dict_writer = csv.DictWriter(csv_file, fieldnames=keys)
+                        dict_writer.writerow(item.dict())
+
+    return output_file
+
+@assistant_tool
+async def process_files_in_folder_for_linkedin_urls(folder_path: str, file_extension: str):
+    """
+    Process files in a folder, extract LinkedIn URLs, and write to a CSV file.
+
+    Parameters:
+    - folder_path (str): The path to the folder containing files.
+    - file_extension (str): The file extension to filter files (e.g., '*.html').
+
+    Returns:
+    - str: The file path of the generated CSV file.
+    """
+    linkedin_urls = set()
+
+    for file_name in os.listdir(folder_path):
+        if file_name.endswith(file_extension):
+            file_path = os.path.join(folder_path, file_name)
+            with open(file_path, 'r') as file:
+                html_content = file.read()
+                soup = BeautifulSoup(html_content, 'html.parser')
+                for link in soup.find_all('a', href=True):
+                    url = link['href']
+                    if re.match(r'^https://www\.linkedin\.com/in/[^?]+', url):
+                        linkedin_urls.add(url.split('?')[0])  # Remove query parameters
+
+    # Write the LinkedIn URLs to a CSV file
+    csv_file_path = os.path.join(folder_path, 'linkedin_urls.csv')
+    with open(csv_file_path, 'w', newline='') as csv_file:
+        dict_writer = csv.DictWriter(csv_file, fieldnames=['id', 'linkedin_url'])
+        dict_writer.writeheader()
+        for url in linkedin_urls:
+            unique_id = datetime.now().strftime('%Y%m%d%H%M%S%f')
+            dict_writer.writerow({'id': unique_id, 'linkedin_url': url})
+
+    return csv_file_path
+
+@assistant_tool
+async def get_lead_urls_from_sales_nav_search_results(folder_path: str, file_extension: str, output_file_path: str):
+    linkedin_urls = set()
+
+    for file_name in os.listdir(folder_path):
+        if file_name.endswith(file_extension):
+            file_path = os.path.join(folder_path, file_name)
+            with open(file_path, 'r') as file:
+                html_content = file.read()
+                soup = BeautifulSoup(html_content, 'html.parser')
+                for link in soup.find_all('a', href=True):
+                    url = link['href']
+                    match = re.match(r'^/sales/lead/([^,]+),', url)
+                    if match:
+                        lead_id = match.group(1)
+                        linkedin_urls.add(lead_id)
+
+    # Write the LinkedIn URLs to a CSV file at the output_file_path
+    with open(output_file_path, 'w', newline='') as csv_file:
+        dict_writer = csv.DictWriter(csv_file, fieldnames=['id', 'linkedin_url'])
+        dict_writer.writeheader()
+        for lead_id in sorted(linkedin_urls):
+            linkedin_url = f'https://www.linkedin.com/in/{lead_id}'
+            dict_writer.writerow({'id': lead_id, 'linkedin_url': linkedin_url})
+
+    return output_file_path
+

dhisana/utils/workflow_code_model.py

@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+
+class WorkflowPythonCode(BaseModel):
+    workflow_python_code: str