unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/zendesk/zendesk.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from pathlib import Path
+from time import time
+from typing import Any, AsyncGenerator, Literal, Union
+
+from pydantic import BaseModel, Field, Secret
+
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.error import SourceConnectionError, ValueError
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    Indexer,
+    IndexerConfig,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.html import HtmlMixin
+
+from .client import ZendeskArticle, ZendeskClient, ZendeskTicket
+
+CONNECTOR_TYPE = "zendesk"
+
+
+class ZendeskAdditionalMetadata(BaseModel):
+    item_type: Literal["ticket", "article"]
+    content: Union[ZendeskTicket, ZendeskArticle]
+
+
+class ZendeskFileData(FileData):
+    additional_metadata: ZendeskAdditionalMetadata
+
+
+class ZendeskAccessConfig(AccessConfig):
+    api_token: str = Field(
+        description="API token for zendesk generated under Apps and Integrations"
+    )
+
+
+class ZendeskConnectionConfig(ConnectionConfig):
+    subdomain: str = Field(description="Subdomain for zendesk site, <sub-domain>.company.com")
+    email: str = Field(description="Email for zendesk site registered at the subdomain")
+    access_config: Secret[ZendeskAccessConfig]
+
+    def get_client(self) -> ZendeskClient:
+        access_config = self.access_config.get_secret_value()
+
+        return ZendeskClient(
+            email=self.email, subdomain=self.subdomain, token=access_config.api_token
+        )
+
+
+class ZendeskIndexerConfig(IndexerConfig):
+    item_type: Literal["tickets", "articles", "all"] = Field(
+        default="tickets",
+        description="Type of item from zendesk to parse, can only be `tickets` or `articles`.",
+    )
+
+
+@dataclass
+class ZendeskIndexer(Indexer):
+    connection_config: ZendeskConnectionConfig
+    index_config: ZendeskIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        """Validates connection to Zendesk API."""
+        self.connection_config.get_client()
+
+    def is_async(self) -> bool:
+        return True
+
+    def _generate_fullpath(self, identifier: str) -> Path:
+        return Path(hashlib.sha256(identifier.encode("utf-8")).hexdigest()[:16] + ".txt")
+
+    async def get_tickets(self) -> AsyncGenerator[ZendeskFileData, None]:
+        async with self.connection_config.get_client() as client:
+            async for ticket in client.get_tickets():
+                source_identifiers = SourceIdentifiers(
+                    filename=f"{ticket.id}.txt", fullpath=f"tickets/{ticket.id}.txt"
+                )
+                yield ZendeskFileData(
+                    identifier=str(ticket.id),
+                    connector_type=self.connector_type,
+                    source_identifiers=source_identifiers,
+                    additional_metadata=ZendeskAdditionalMetadata(
+                        item_type="ticket", content=ticket
+                    ),
+                    metadata=FileDataSourceMetadata(
+                        url=str(ticket.url) if ticket.url else None,
+                        date_created=ticket.created_at.isoformat() if ticket.created_at else None,
+                        date_modified=ticket.updated_at.isoformat() if ticket.updated_at else None,
+                        date_processed=str(time()),
+                    ),
+                    display_name=source_identifiers.fullpath,
+                )
+
+    async def get_articles(self) -> AsyncGenerator[ZendeskFileData, None]:
+        async with self.connection_config.get_client() as client:
+            async for article in client.get_articles():
+                source_identifiers = SourceIdentifiers(
+                    filename=f"{article.id}.html", fullpath=f"articles/{article.id}.html"
+                )
+                yield ZendeskFileData(
+                    identifier=str(article.id),
+                    connector_type=self.connector_type,
+                    source_identifiers=source_identifiers,
+                    additional_metadata=ZendeskAdditionalMetadata(
+                        item_type="article", content=article
+                    ),
+                    metadata=FileDataSourceMetadata(
+                        url=str(article.url) if article.url else None,
+                        date_created=article.created_at.isoformat() if article.created_at else None,
+                        date_modified=(
+                            article.updated_at.isoformat() if article.updated_at else None
+                        ),
+                        date_processed=str(time()),
+                    ),
+                    display_name=source_identifiers.fullpath,
+                )
+
+    async def run_async(self, **kwargs: Any) -> AsyncGenerator[ZendeskFileData, None]:
+        """Determines item type and processes accordingly asynchronously."""
+        item_type = self.index_config.item_type
+
+        if item_type == "articles":
+            async for article_file_data in self.get_articles():
+                yield article_file_data
+
+        elif item_type == "tickets":
+            async for ticket_file_data in self.get_tickets():
+                yield ticket_file_data
+
+        elif item_type == "all":
+            async for article_file_data in self.get_articles():
+                yield article_file_data
+            async for ticket_file_data in self.get_tickets():
+                yield ticket_file_data
+
+        else:
+            raise ValueError(f"Item type {item_type} is not supported by the indexer")
+
+
+class ZendeskDownloaderConfig(DownloaderConfig, HtmlMixin):
+    pass
+
+
+@dataclass
+class ZendeskDownloader(Downloader):
+    download_config: ZendeskDownloaderConfig
+    connection_config: ZendeskConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def is_async(self) -> bool:
+        return True
+
+    def download_embedded_files(
+        self, session, html: str, current_file_data: FileData
+    ) -> list[DownloadResponse]:
+        if not self.download_config.extract_files:
+            return []
+        url = current_file_data.metadata.url
+        if url is None:
+            logger.warning(
+                f"""Missing URL for file: {current_file_data.source_identifiers.filename}.
+                Skipping file extraction."""
+            )
+            return []
+        filepath = current_file_data.source_identifiers.relative_path
+        download_path = Path(self.download_dir) / filepath
+        download_dir = download_path.with_suffix("")
+        return self.download_config.extract_embedded_files(
+            url=url,
+            download_dir=download_dir,
+            original_filedata=current_file_data,
+            html=html,
+            session=session,
+        )
+
+    @requires_dependencies(["aiofiles", "bs4"], extras="zendesk")
+    async def download_article(self, article: ZendeskArticle, download_path: Path) -> None:
+        import aiofiles
+        import bs4
+
+        article_html = article.as_html()
+        soup = bs4.BeautifulSoup(article_html, "html.parser")
+        async with aiofiles.open(download_path, "w", encoding="utf8") as f:
+            await f.write(soup.prettify())
+
+    @requires_dependencies(["aiofiles"], extras="zendesk")
+    async def download_ticket(self, ticket: ZendeskTicket, download_path: Path) -> None:
+        import aiofiles
+
+        async with aiofiles.open(download_path, "w", encoding="utf8") as f:
+            await f.write(ticket.as_text())
+            async with self.connection_config.get_client() as client:
+                comments = [comment async for comment in client.get_comments(ticket_id=ticket.id)]
+                for comment in comments:
+                    await f.write(comment.as_text())
+
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        zendesk_filedata = ZendeskFileData.cast(file_data=file_data)
+
+        item_type = zendesk_filedata.additional_metadata.item_type
+        download_path = self.get_download_path(file_data=zendesk_filedata)
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+
+        if item_type == "article":
+            article = ZendeskArticle.model_validate(zendesk_filedata.additional_metadata.content)
+            await self.download_article(article=article, download_path=download_path)
+        elif item_type == "ticket":
+            ticket = ZendeskTicket.model_validate(zendesk_filedata.additional_metadata.content)
+            await self.download_ticket(ticket=ticket, download_path=download_path)
+        else:
+            raise SourceConnectionError(
+                f"Item type {item_type} cannot be handled by the downloader"
+            )
+        return super().generate_download_response(
+            file_data=zendesk_filedata, download_path=download_path
+        )
+
+
+# create entry
+zendesk_source_entry = SourceRegistryEntry(
+    connection_config=ZendeskConnectionConfig,
+    indexer_config=ZendeskIndexerConfig,
+    indexer=ZendeskIndexer,
+    downloader=ZendeskDownloader,
+    downloader_config=ZendeskDownloaderConfig,
+)

unstructured_ingest/processes/embedder.py

@@ -0,0 +1,203 @@
+from abc import ABC
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal, Optional
+
+from pydantic import BaseModel, Field, SecretStr
+
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.utils.data_prep import get_json_data
+
+if TYPE_CHECKING:
+    from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder
+
+
+class EmbedderConfig(BaseModel):
+    embedding_provider: Optional[
+        Literal[
+            "openai",
+            "azure-openai",
+            "huggingface",
+            "bedrock",
+            "vertexai",
+            "voyageai",
+            "octoai",
+            "mixedbread-ai",
+            "togetherai",
+        ]
+    ] = Field(default=None, description="Type of the embedding class to be used.")
+    embedding_api_key: Optional[SecretStr] = Field(
+        default=None,
+        description="API key for the embedding model, for the case an API key is needed.",
+    )
+    embedding_model_name: Optional[str] = Field(
+        default=None,
+        description="Embedding model name, if needed. "
+        "Chooses a particular LLM between different options, to embed with it.",
+    )
+    embedding_aws_access_key_id: Optional[str] = Field(
+        default=None, description="AWS access key used for AWS-based embedders, such as bedrock"
+    )
+    embedding_aws_secret_access_key: Optional[SecretStr] = Field(
+        default=None, description="AWS secret key used for AWS-based embedders, such as bedrock"
+    )
+    embedding_aws_region: Optional[str] = Field(
+        default="us-west-2", description="AWS region used for AWS-based embedders, such as bedrock"
+    )
+    embedding_azure_endpoint: Optional[str] = Field(
+        default=None,
+        description="Your Azure endpoint, including the resource, "
+        "e.g. `https://example-resource.azure.openai.com/`",
+    )
+    embedding_azure_api_version: Optional[str] = Field(
+        description="Azure API version", default=None
+    )
+
+    def get_huggingface_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.huggingface import (
+            HuggingFaceEmbeddingConfig,
+            HuggingFaceEmbeddingEncoder,
+        )
+
+        return HuggingFaceEmbeddingEncoder(
+            config=HuggingFaceEmbeddingConfig.model_validate(embedding_kwargs)
+        )
+
+    def get_openai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+
+        return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig.model_validate(embedding_kwargs))
+
+    def get_azure_openai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.azure_openai import (
+            AzureOpenAIEmbeddingConfig,
+            AzureOpenAIEmbeddingEncoder,
+        )
+
+        config_kwargs = {
+            "api_key": self.embedding_api_key,
+            "azure_endpoint": self.embedding_azure_endpoint,
+        }
+        if api_version := self.embedding_azure_api_version:
+            config_kwargs["api_version"] = api_version
+        if model_name := self.embedding_model_name:
+            config_kwargs["model_name"] = model_name
+
+        return AzureOpenAIEmbeddingEncoder(
+            config=AzureOpenAIEmbeddingConfig.model_validate(config_kwargs)
+        )
+
+    def get_octoai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+
+        return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig.model_validate(embedding_kwargs))
+
+    def get_bedrock_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.bedrock import (
+            BedrockEmbeddingConfig,
+            BedrockEmbeddingEncoder,
+        )
+
+        embedding_kwargs = embedding_kwargs | {
+            "aws_access_key_id": self.embedding_aws_access_key_id,
+            "aws_secret_access_key": self.embedding_aws_secret_access_key.get_secret_value(),
+            "region_name": self.embedding_aws_region,
+        }
+
+        return BedrockEmbeddingEncoder(
+            config=BedrockEmbeddingConfig.model_validate(embedding_kwargs)
+        )
+
+    def get_vertexai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.vertexai import (
+            VertexAIEmbeddingConfig,
+            VertexAIEmbeddingEncoder,
+        )
+
+        return VertexAIEmbeddingEncoder(
+            config=VertexAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
+
+    def get_voyageai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.voyageai import (
+            VoyageAIEmbeddingConfig,
+            VoyageAIEmbeddingEncoder,
+        )
+
+        return VoyageAIEmbeddingEncoder(
+            config=VoyageAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
+
+    def get_mixedbread_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.mixedbreadai import (
+            MixedbreadAIEmbeddingConfig,
+            MixedbreadAIEmbeddingEncoder,
+        )
+
+        return MixedbreadAIEmbeddingEncoder(
+            config=MixedbreadAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
+
+    def get_togetherai_embedder(self, embedding_kwargs: dict) -> "BaseEmbeddingEncoder":
+        from unstructured_ingest.embed.togetherai import (
+            TogetherAIEmbeddingConfig,
+            TogetherAIEmbeddingEncoder,
+        )
+
+        return TogetherAIEmbeddingEncoder(
+            config=TogetherAIEmbeddingConfig.model_validate(embedding_kwargs)
+        )
+
+    def get_embedder(self) -> "BaseEmbeddingEncoder":
+        kwargs: dict[str, Any] = {}
+        if self.embedding_api_key:
+            kwargs["api_key"] = self.embedding_api_key.get_secret_value()
+        if self.embedding_model_name:
+            kwargs["model_name"] = self.embedding_model_name
+        # TODO make this more dynamic to map to encoder configs
+        if self.embedding_provider == "openai":
+            return self.get_openai_embedder(embedding_kwargs=kwargs)
+
+        if self.embedding_provider == "huggingface":
+            return self.get_huggingface_embedder(embedding_kwargs=kwargs)
+
+        if self.embedding_provider == "octoai":
+            return self.get_octoai_embedder(embedding_kwargs=kwargs)
+
+        if self.embedding_provider == "bedrock":
+            return self.get_bedrock_embedder(embedding_kwargs=kwargs)
+
+        if self.embedding_provider == "vertexai":
+            return self.get_vertexai_embedder(embedding_kwargs=kwargs)
+
+        if self.embedding_provider == "voyageai":
+            return self.get_voyageai_embedder(embedding_kwargs=kwargs)
+        if self.embedding_provider == "mixedbread-ai":
+            return self.get_mixedbread_embedder(embedding_kwargs=kwargs)
+        if self.embedding_provider == "togetherai":
+            return self.get_togetherai_embedder(embedding_kwargs=kwargs)
+        if self.embedding_provider == "azure-openai":
+            return self.get_azure_openai_embedder(embedding_kwargs=kwargs)
+
+        raise ValueError(f"{self.embedding_provider} not a recognized encoder")
+
+
+@dataclass
+class Embedder(BaseProcess, ABC):
+    config: EmbedderConfig
+
+    def init(self, **kwargs: Any) -> None:
+        self.config.get_embedder().initialize()
+
+    def precheck(self) -> None:
+        embedder = self.config.get_embedder()
+        embedder.precheck()
+
+    def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
+        # TODO update base embedder classes to support async
+        embedder = self.config.get_embedder()
+        elements = get_json_data(path=elements_filepath)
+        if not elements:
+            return []
+        embedded_elements = embedder.embed_documents(elements=elements)
+        return embedded_elements

unstructured_ingest/processes/filter.py

@@ -0,0 +1,60 @@
+import fnmatch
+from abc import ABC
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+from pydantic import BaseModel, Field
+
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces.process import BaseProcess
+from unstructured_ingest.logger import logger
+
+
+class FiltererConfig(BaseModel):
+    file_glob: Optional[list[str]] = Field(
+        default=None,
+        description="file globs to limit which data_types of files are accepted",
+        examples=["*.pdf", "*.html"],
+    )
+    max_file_size: Optional[int] = Field(
+        default=None, description="Max file size to process in bytes"
+    )
+
+
+@dataclass
+class Filterer(BaseProcess, ABC):
+    config: FiltererConfig = field(default_factory=lambda: FiltererConfig())
+    filters: list[Callable[[FileData], bool]] = field(init=False, default_factory=list)
+
+    def __post_init__(self):
+        # Populate the filters based on values in config
+        if self.config.file_glob is not None:
+            self.filters.append(self.glob_filter)
+        if self.config.max_file_size:
+            self.filters.append(self.file_size_filter)
+
+    def is_async(self) -> bool:
+        return False
+
+    def file_size_filter(self, file_data: FileData) -> bool:
+        if filesize_bytes := file_data.metadata.filesize_bytes:
+            return filesize_bytes <= self.config.max_file_size
+        return True
+
+    def glob_filter(self, file_data: FileData) -> bool:
+        patterns = self.config.file_glob
+        path = file_data.source_identifiers.fullpath
+        for pattern in patterns:
+            if fnmatch.filter([path], pattern):
+                return True
+        logger.debug(f"the file {path!r} is discarded as it does not match any given glob.")
+        return False
+
+    def run(self, file_data: FileData, **kwargs: Any) -> Optional[FileData]:
+        for filter in self.filters:
+            if not filter(file_data):
+                logger.debug(
+                    f"filtered out file data due to {filter.__name__}: {file_data.identifier}"
+                )
+                return None
+        return file_data