dhisana 0.0.1.dev243__py3-none-any.whl

dhisana/utils/openapi_tool/convert_openai_spec_to_tool.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+import re
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+
+from fastapi import logger
+from openapi_pydantic import Parameter
+from pydantic import BaseModel, Field
+import aiohttp
+from typing import Any, Optional
+
+from .api_models import INVALID_LOCATION_TEMPL, APIProperty, APIRequestBody
+from .openapi_tool import HTTPVerb, OpenAPISpec
+
+def _format_url(url: str, path_params: dict) -> str:
+    expected_path_param = re.findall(r"{(.*?)}", url)
+    new_params = {}
+    for param in expected_path_param:
+        clean_param = param.lstrip(".;").rstrip("*")
+        val = path_params[clean_param]
+        if isinstance(val, list):
+            if param[0] == ".":
+                sep = "." if param[-1] == "*" else ","
+                new_val = "." + sep.join(val)
+            elif param[0] == ";":
+                sep = f"{clean_param}=" if param[-1] == "*" else ","
+                new_val = f"{clean_param}=" + sep.join(val)
+            else:
+                new_val = ",".join(val)
+        elif isinstance(val, dict):
+            kv_sep = "=" if param[-1] == "*" else ","
+            kv_strs = [kv_sep.join((k, v)) for k, v in val.items()]
+            if param[0] == ".":
+                sep = "."
+                new_val = "."
+            elif param[0] == ";":
+                sep = ";"
+                new_val = ";"
+            else:
+                sep = ","
+                new_val = ""
+            new_val += sep.join(kv_strs)
+        else:
+            if param[0] == ".":
+                new_val = f".{val}"
+            elif param[0] == ";":
+                new_val = f";{clean_param}={val}"
+            else:
+                new_val = val
+        new_params[param] = new_val
+    return url.format(**new_params)
+
+class APIOperation(BaseModel):
+    """A model for a single API operation."""
+
+    operation_id: str = Field(alias="operation_id")
+    """The unique identifier of the operation."""
+
+    description: Optional[str] = Field(alias="description")
+    """The description of the operation."""
+
+    base_url: str = Field(alias="base_url")
+    """The base URL of the operation."""
+
+    path: str = Field(alias="path")
+    """The path of the operation."""
+
+    method: HTTPVerb = Field(alias="method")
+    """The HTTP method of the operation."""
+
+    properties: Sequence[APIProperty] = Field(alias="properties")
+
+    # TODO: Add parse in used components to be able to specify what type of
+    # referenced object it is.
+    # """The properties of the operation."""
+    # components: Dict[str, BaseModel] = Field(alias="components")
+
+    request_body: Optional[APIRequestBody] = Field(alias="request_body")
+    """The request body of the operation."""
+
+    @staticmethod
+    def _get_properties_from_parameters(
+        parameters: List[Parameter], spec: OpenAPISpec
+    ) -> List[APIProperty]:
+        """Get the properties of the operation."""
+        properties = []
+        for param in parameters:
+            if APIProperty.is_supported_location(param.param_in):
+                properties.append(APIProperty.from_parameter(param, spec))
+            elif param.required:
+                raise ValueError(
+                    INVALID_LOCATION_TEMPL.format(
+                        location=param.param_in, name=param.name
+                    )
+                )
+            else:
+                logger.warning(
+                    INVALID_LOCATION_TEMPL.format(
+                        location=param.param_in, name=param.name
+                    )
+                    + " Ignoring optional parameter"
+                )
+                pass
+        return properties
+
+    @classmethod
+    def from_openapi_url(
+        cls,
+        spec_url: str,
+        path: str,
+        method: str,
+    ) -> "APIOperation":
+        """Create an APIOperation from an OpenAPI URL."""
+        spec = OpenAPISpec.from_url(spec_url)
+        return cls.from_openapi_spec(spec, path, method)
+
+
+    @classmethod
+    def from_openapi_spec(
+        cls,
+        spec: OpenAPISpec,
+        path: str,
+        method: str,
+    ) -> "APIOperation":
+        """Create an APIOperation from an OpenAPI spec."""
+        operation = spec.get_operation(path, method)
+        parameters = spec.get_parameters_for_operation(operation)
+        properties = cls._get_properties_from_parameters(parameters, spec)
+        operation_id = OpenAPISpec.get_cleaned_operation_id(spec, operation, path, method)
+        request_body = spec.get_request_body_for_operation(operation)
+        api_request_body = (
+            APIRequestBody.from_request_body(request_body, spec)
+            if request_body is not None
+            else None
+        )
+        description = operation.description or operation.summary
+        if not description and spec.paths is not None:
+            description = spec.paths[path].description or spec.paths[path].summary
+        return cls(
+            operation_id=operation_id,
+            description=description or "",
+            base_url=spec.base_url,
+            path=path,
+            method=method,  # type: ignore[arg-type]
+            properties=properties,
+            request_body=api_request_body,
+        )
+def _openapi_params_to_json_schema(params: List[Parameter], spec: OpenAPISpec) -> dict:
+    properties = {}
+    required = []
+    for p in params:
+        if p.param_schema:
+            schema = spec.get_schema(p.param_schema)
+        else:
+            media_type_schema = list(p.content.values())[0].media_type_schema  # type: ignore  # noqa: E501
+            schema = spec.get_schema(media_type_schema)
+        if p.description and not schema.description:
+            schema.description = p.description
+        properties[p.name] = json.loads(schema.json(exclude_none=True))
+        if p.required:
+            required.append(p.name)
+    return {"type": "object", "properties": properties, "required": required}
+
+def openapi_spec_to_openai_fn(
+    spec: OpenAPISpec,
+) -> Tuple[List[Dict[str, Any]], Callable]:
+    """Convert a valid OpenAPI spec to the JSON Schema format expected for OpenAI
+        functions.
+
+    Args:
+        spec: OpenAPI spec to convert.
+
+    Returns:
+        Tuple of the OpenAI functions JSON schema and a default function for executing
+            a request based on the OpenAI function schema.
+    """
+    if not spec.paths:
+        return [], lambda: None
+    functions = []
+    _name_to_call_map = {}
+    for path in spec.paths:
+        path_params = {
+            (p.name, p.param_in): p for p in spec.get_parameters_for_path(path)
+        }
+        for method in spec.get_methods_for_path(path):
+            request_args = {}
+            op = spec.get_operation(path, method)
+            op_params = path_params.copy()
+            for param in spec.get_parameters_for_operation(op):
+                op_params[(param.name, param.param_in)] = param
+            params_by_type = defaultdict(list)
+            for name_loc, p in op_params.items():
+                params_by_type[name_loc[1]].append(p)
+            param_loc_to_arg_name = {
+                "query": "params",
+                "header": "headers",
+                "cookie": "cookies",
+                "path": "path_params",
+            }
+            for param_loc, arg_name in param_loc_to_arg_name.items():
+                if params_by_type[param_loc]:
+                    request_args[arg_name] = _openapi_params_to_json_schema(
+                        params_by_type[param_loc], spec
+                    )
+            request_body = spec.get_request_body_for_operation(op)
+            # TODO: Support more MIME types.
+            if request_body and request_body.content:
+                media_types = {}
+                for media_type, media_type_object in request_body.content.items():
+                    if media_type_object.media_type_schema:
+                        schema = spec.get_schema(media_type_object.media_type_schema)
+                        media_types[media_type] = json.loads(
+                            schema.json(exclude_none=True)
+                        )
+                if len(media_types) == 1:
+                    media_type, schema_dict = list(media_types.items())[0]
+                    key = "json" if media_type == "application/json" else "data"
+                    request_args[key] = schema_dict
+                elif len(media_types) > 1:
+                    request_args["data"] = {"anyOf": list(media_types.values())}
+
+            api_op = APIOperation.from_openapi_spec(spec, path, method)
+            fn = {
+                    "type": "function", 
+                        "function":{
+                        "name": api_op.operation_id,
+                        "description": api_op.description,
+                        "parameters": {
+                            "type": "object",
+                            "properties": request_args,
+                        },
+                    }
+                }
+            functions.append(fn)
+            _name_to_call_map[fn["function"]["name"]] = {
+                "method": method,
+                "url": api_op.base_url + api_op.path,
+            }
+    
+    async def default_call_api(
+            name: str,
+            fn_args: dict,
+            headers: Optional[dict] = None,
+            params: Optional[dict] = None,
+            **kwargs: Any,
+        ) -> Any:
+        method = _name_to_call_map[name]["method"]
+        url = _name_to_call_map[name]["url"]
+        path_params = fn_args.pop("path_params", {})
+        url = _format_url(url, path_params)
+        if "data" in fn_args and isinstance(fn_args["data"], dict):
+            fn_args["data"] = json.dumps(fn_args["data"])
+        _kwargs = {**fn_args, **kwargs}
+        if headers is not None:
+            if "headers" in _kwargs:
+                _kwargs["headers"].update(headers)
+            else:
+                _kwargs["headers"] = headers
+        if params is not None:
+            if "params" in _kwargs:
+                _kwargs["params"].update(params)
+            else:
+                _kwargs["params"] = params
+    
+        async with aiohttp.ClientSession() as session:
+            async with session.request(method, url, **_kwargs) as response:
+                return  response.status, response.reason, await response.text()
+
+    return functions, default_call_api

dhisana/utils/openapi_tool/openapi_tool.py

@@ -0,0 +1,319 @@
+"""Utility functions for parsing an OpenAPI spec."""
+from __future__ import annotations
+
+import copy
+import json
+import logging
+import re
+from enum import Enum
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
+
+from pydantic import ValidationError
+import requests
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+class HTTPVerb(str, Enum):
+    """Enumerator of the HTTP verbs."""
+
+    GET = "get"
+    PUT = "put"
+    POST = "post"
+    DELETE = "delete"
+    OPTIONS = "options"
+    HEAD = "head"
+    PATCH = "patch"
+    TRACE = "trace"
+
+    @classmethod
+    def from_str(cls, verb: str) -> HTTPVerb:
+        """Parse an HTTP verb."""
+        try:
+            return cls(verb)
+        except ValueError:
+            raise ValueError(f"Invalid HTTP verb. Valid values are {cls.__members__}")
+
+
+if TYPE_CHECKING:
+    from openapi_pydantic import (
+        Components,
+        Operation,
+        Parameter,
+        PathItem,
+        Paths,
+        Reference,
+        RequestBody,
+        Schema,
+    )
+
+try:
+    from openapi_pydantic import OpenAPI
+except ImportError:
+    OpenAPI = object  # type: ignore
+
+
+class OpenAPISpec(OpenAPI):
+    """OpenAPI Model that removes mis-formatted parts of the spec."""
+
+    openapi: str = "3.1.0"  # overriding overly restrictive type from parent class
+
+    @property
+    def _paths_strict(self) -> Paths:
+        if not self.paths:
+            raise ValueError("No paths found in spec")
+        return self.paths
+
+    def _get_path_strict(self, path: str) -> PathItem:
+        path_item = self._paths_strict.get(path)
+        if not path_item:
+            raise ValueError(f"No path found for {path}")
+        return path_item
+
+    @property
+    def _components_strict(self) -> Components:
+        """Get components or err."""
+        if self.components is None:
+            raise ValueError("No components found in spec. ")
+        return self.components
+
+    @property
+    def _parameters_strict(self) -> Dict[str, Union[Parameter, Reference]]:
+        """Get parameters or err."""
+        parameters = self._components_strict.parameters
+        if parameters is None:
+            raise ValueError("No parameters found in spec. ")
+        return parameters
+
+    @property
+    def _schemas_strict(self) -> Dict[str, Schema]:
+        """Get the dictionary of schemas or err."""
+        schemas = self._components_strict.schemas
+        if schemas is None:
+            raise ValueError("No schemas found in spec. ")
+        return schemas
+
+    @property
+    def _request_bodies_strict(self) -> Dict[str, Union[RequestBody, Reference]]:
+        """Get the request body or err."""
+        request_bodies = self._components_strict.requestBodies
+        if request_bodies is None:
+            raise ValueError("No request body found in spec. ")
+        return request_bodies
+
+    def _get_referenced_parameter(self, ref: Reference) -> Union[Parameter, Reference]:
+        """Get a parameter (or nested reference) or err."""
+        ref_name = ref.ref.split("/")[-1]
+        parameters = self._parameters_strict
+        if ref_name not in parameters:
+            raise ValueError(f"No parameter found for {ref_name}")
+        return parameters[ref_name]
+
+    def _get_root_referenced_parameter(self, ref: Reference) -> Parameter:
+        """Get the root reference or err."""
+        from openapi_pydantic import Reference
+
+        parameter = self._get_referenced_parameter(ref)
+        while isinstance(parameter, Reference):
+            parameter = self._get_referenced_parameter(parameter)
+        return parameter
+
+    def get_referenced_schema(self, ref: Reference) -> Schema:
+        """Get a schema (or nested reference) or err."""
+        ref_name = ref.ref.split("/")[-1]
+        schemas = self._schemas_strict
+        if ref_name not in schemas:
+            raise ValueError(f"No schema found for {ref_name}")
+        return schemas[ref_name]
+
+    def get_schema(self, schema: Union[Reference, Schema]) -> Schema:
+        from openapi_pydantic import Reference
+
+        if isinstance(schema, Reference):
+            return self.get_referenced_schema(schema)
+        return schema
+
+    def _get_root_referenced_schema(self, ref: Reference) -> Schema:
+        """Get the root reference or err."""
+        from openapi_pydantic import Reference
+
+        schema = self.get_referenced_schema(ref)
+        while isinstance(schema, Reference):
+            schema = self.get_referenced_schema(schema)
+        return schema
+
+    def _get_referenced_request_body(
+        self, ref: Reference
+    ) -> Optional[Union[Reference, RequestBody]]:
+        """Get a request body (or nested reference) or err."""
+        ref_name = ref.ref.split("/")[-1]
+        request_bodies = self._request_bodies_strict
+        if ref_name not in request_bodies:
+            raise ValueError(f"No request body found for {ref_name}")
+        return request_bodies[ref_name]
+
+    def _get_root_referenced_request_body(
+        self, ref: Reference
+    ) -> Optional[RequestBody]:
+        """Get the root request Body or err."""
+        from openapi_pydantic import Reference
+
+        request_body = self._get_referenced_request_body(ref)
+        while isinstance(request_body, Reference):
+            request_body = self._get_referenced_request_body(request_body)
+        return request_body
+
+    @staticmethod
+    def _alert_unsupported_spec(obj: dict) -> None:
+        """Alert if the spec is not supported."""
+        warning_message = (
+            " This may result in degraded performance."
+            + " Convert your OpenAPI spec to 3.1.* spec"
+            + " for better support."
+        )
+        swagger_version = obj.get("swagger")
+        openapi_version = obj.get("openapi")
+        if isinstance(openapi_version, str):
+            if openapi_version != "3.1.0":
+                logger.warning(
+                    f"Attempting to load an OpenAPI {openapi_version}"
+                    f" spec. {warning_message}"
+                )
+            else:
+                pass
+        elif isinstance(swagger_version, str):
+            logger.warning(
+                f"Attempting to load a Swagger {swagger_version}"
+                f" spec. {warning_message}"
+            )
+        else:
+            raise ValueError(
+                "Attempting to load an unsupported spec:"
+                f"\n\n{obj}\n{warning_message}"
+            )
+
+    @classmethod
+    def parse_obj(cls, obj: dict) -> OpenAPISpec:
+        try:
+            cls._alert_unsupported_spec(obj)
+            return super().parse_obj(obj)
+        except ValidationError as e:
+            # We are handling possibly misconfigured specs and
+            # want to do a best-effort job to get a reasonable interface out of it.
+            new_obj = copy.deepcopy(obj)
+            for error in e.errors():
+                keys = error["loc"]
+                item = new_obj
+                for key in keys[:-1]:
+                    item = item[key]
+                item.pop(keys[-1], None)
+            return cls.parse_obj(new_obj)
+
+    @classmethod
+    def from_spec_dict(cls, spec_dict: dict) -> OpenAPISpec:
+        """Get an OpenAPI spec from a dict."""
+        return cls.parse_obj(spec_dict)
+
+    @classmethod
+    def from_text(cls, text: str) -> OpenAPISpec:
+        """Get an OpenAPI spec from a text."""
+        try:
+            spec_dict = json.loads(text)
+        except json.JSONDecodeError:
+            spec_dict = yaml.safe_load(text)
+        return cls.from_spec_dict(spec_dict)
+
+    @classmethod
+    def from_file(cls, path: Union[str, Path]) -> OpenAPISpec:
+        """Get an OpenAPI spec from a file path."""
+        path_ = path if isinstance(path, Path) else Path(path)
+        if not path_.exists():
+            raise FileNotFoundError(f"{path} does not exist")
+        with path_.open("r") as f:
+            return cls.from_text(f.read())
+
+    @classmethod
+    def from_url(cls, url: str) -> OpenAPISpec:
+        """Get an OpenAPI spec from a URL."""
+        response = requests.get(url)
+        return cls.from_text(response.text)
+
+    @property
+    def base_url(self) -> str:
+        """Get the base url."""
+        return self.servers[0].url
+
+    def get_methods_for_path(self, path: str) -> List[str]:
+        """Return a list of valid methods for the specified path."""
+        from openapi_pydantic import Operation
+
+        path_item = self._get_path_strict(path)
+        results = []
+        for method in HTTPVerb:
+            operation = getattr(path_item, method.value, None)
+            if isinstance(operation, Operation):
+                results.append(method.value)
+        return results
+
+    def get_parameters_for_path(self, path: str) -> List[Parameter]:
+        from openapi_pydantic import Reference
+
+        path_item = self._get_path_strict(path)
+        parameters = []
+        if not path_item.parameters:
+            return []
+        for parameter in path_item.parameters:
+            if isinstance(parameter, Reference):
+                parameter = self._get_root_referenced_parameter(parameter)
+            parameters.append(parameter)
+        return parameters
+
+    def get_operation(self, path: str, method: str) -> Operation:
+        """Get the operation object for a given path and HTTP method."""
+        from openapi_pydantic import Operation
+
+        path_item = self._get_path_strict(path)
+        operation_obj = getattr(path_item, method, None)
+        if not isinstance(operation_obj, Operation):
+            raise ValueError(f"No {method} method found for {path}")
+        return operation_obj
+
+    def get_parameters_for_operation(self, operation: Operation) -> List[Parameter]:
+        """Get the components for a given operation."""
+        from openapi_pydantic import Reference
+
+        parameters = []
+        if operation.parameters:
+            for parameter in operation.parameters:
+                if isinstance(parameter, Reference):
+                    parameter = self._get_root_referenced_parameter(parameter)
+                parameters.append(parameter)
+        return parameters
+
+    def get_request_body_for_operation(
+        self, operation: Operation
+    ) -> Optional[RequestBody]:
+        """Get the request body for a given operation."""
+        from openapi_pydantic import Reference
+
+        request_body = operation.requestBody
+        if isinstance(request_body, Reference):
+            request_body = self._get_root_referenced_request_body(request_body)
+        return request_body
+
+    @staticmethod
+    def get_cleaned_operation_id(spec: OpenAPISpec, operation: Operation, path: str, method: str) -> str:
+        """Get a cleaned operation id from an operation id."""
+        # Sanitize the base URL
+        base_url = spec.base_url
+        sanitized_base_url = re.sub(r"https?://(www\.)?", "", base_url)
+        sanitized_base_url = re.sub(r"[^a-zA-Z0-9]", "_", sanitized_base_url).strip("_").lower()
+    
+        # Replace all punctuation of any kind with underscore
+        path = re.sub(r"[^a-zA-Z0-9]", "_", path.lstrip("/"))
+        operation_id = f"{sanitized_base_url}_{path}_{method}"
+        operation_id = operation_id.replace("-", "_").replace(".", "_").replace("/", "_").lower()
+        return operation_id
+

dhisana/utils/parse_linkedin_messages_txt.py

@@ -0,0 +1,100 @@
+import uuid
+from typing import List
+from datetime import datetime, date
+from dhisana.schemas.sales import MessageItem
+
+DAY_NAMES = {
+    "monday", "tuesday", "wednesday", "thursday",
+    "friday", "saturday", "sunday"
+}
+
+def is_day_line(line: str) -> bool:
+    """Check if line is a simple day name (case-insensitive)."""
+    return line.strip().lower() in DAY_NAMES
+
+def parse_time_line(time_str: str) -> str:
+    """
+    Parse a time string like "6:38 PM" or "14:10" using today's date,
+    returning an ISO8601 string. Returns an empty string if it fails.
+    """
+    today_str = date.today().strftime("%Y-%m-%d")
+    for fmt in ["%I:%M %p", "%H:%M"]:
+        try:
+            dt = datetime.strptime(f"{today_str} {time_str}", f"%Y-%m-%d {fmt}")
+            return dt.isoformat()
+        except ValueError:
+            pass
+    return ""  # If we can’t parse it, return empty
+
+
+def parse_conversation(conversation_text: str) -> List[MessageItem]:
+    """
+    Given raw text containing lines like:
+      'Load more'
+      'Thursday'
+      'You'
+      '6:38 PM'
+      'Hello, ...'
+    Parse them into MessageItems with empty subject/email,
+    and return a list sorted from latest (top) to oldest (bottom).
+    """
+    # Split lines, remove empties and extra spaces
+    lines = [line.strip() for line in conversation_text.split('\n') if line.strip()]
+
+    messages: List[MessageItem] = []
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+
+        # Skip lines that say 'Load more' or day lines (Thursday, Monday, etc.)
+        if line.lower().startswith("load more") or is_day_line(line):
+            i += 1
+            continue
+
+        # This line should be the sender (e.g. "You" or "Madhukar Devaraju")
+        sender = line
+        i += 1
+        if i >= len(lines):
+            break
+
+        # Next line should be the time
+        time_line = lines[i]
+        time_iso = parse_time_line(time_line)
+        if not time_iso:
+            # If we cannot parse the time here, skip it and move on
+            i += 1
+            continue
+        i += 1
+
+        # Collect body until the next recognized "sender" or "day" or "Load more" or valid time
+        body_lines = []
+        while i < len(lines):
+            nxt = lines[i]
+            if nxt.lower().startswith("load more") or is_day_line(nxt):
+                # Reached a new block
+                break
+            if parse_time_line(nxt):
+                # If nxt is a time line, it means a new message is coming
+                break
+            # Otherwise, treat it as part of the message body
+            body_lines.append(nxt)
+            i += 1
+
+        # We have enough info to form one message
+        body_text = "\n".join(body_lines).strip()
+        message_item = MessageItem(
+            message_id=str(uuid.uuid4()),
+            thread_id=str(uuid.uuid4()),
+            sender_name=sender,
+            sender_email="",      # LinkedIn message => keep empty
+            receiver_name="",     # keep empty by default
+            receiver_email="",    # keep empty by default
+            iso_datetime=time_iso,
+            subject="",           # LinkedIn => keep empty
+            body=body_text
+        )
+        messages.append(message_item)
+
+    # Reverse the list so the latest is on top
+    messages.reverse()
+    return messages