unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/__init__.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import unstructured_ingest.processes.connectors.databricks  # noqa: F401
+import unstructured_ingest.processes.connectors.duckdb  # noqa: F401
+import unstructured_ingest.processes.connectors.elasticsearch  # noqa: F401
+import unstructured_ingest.processes.connectors.fsspec  # noqa: F401
+import unstructured_ingest.processes.connectors.ibm_watsonx  # noqa: F401
+import unstructured_ingest.processes.connectors.kafka  # noqa: F401
+import unstructured_ingest.processes.connectors.lancedb  # noqa: F401
+import unstructured_ingest.processes.connectors.qdrant  # noqa: F401
+import unstructured_ingest.processes.connectors.sql  # noqa: F401
+import unstructured_ingest.processes.connectors.weaviate  # noqa: F401
+from unstructured_ingest.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .airtable import CONNECTOR_TYPE as AIRTABLE_CONNECTOR_TYPE
+from .airtable import airtable_source_entry
+from .astradb import CONNECTOR_TYPE as ASTRA_DB_CONNECTOR_TYPE
+from .astradb import astra_db_destination_entry, astra_db_source_entry
+from .azure_ai_search import CONNECTOR_TYPE as AZURE_AI_SEARCH_CONNECTOR_TYPE
+from .azure_ai_search import azure_ai_search_destination_entry
+from .chroma import CONNECTOR_TYPE as CHROMA_CONNECTOR_TYPE
+from .chroma import chroma_destination_entry
+from .confluence import CONNECTOR_TYPE as CONFLUENCE_CONNECTOR_TYPE
+from .confluence import confluence_source_entry
+from .couchbase import CONNECTOR_TYPE as COUCHBASE_CONNECTOR_TYPE
+from .couchbase import couchbase_destination_entry, couchbase_source_entry
+from .delta_table import CONNECTOR_TYPE as DELTA_TABLE_CONNECTOR_TYPE
+from .delta_table import delta_table_destination_entry
+from .discord import CONNECTOR_TYPE as DISCORD_CONNECTOR_TYPE
+from .discord import discord_source_entry
+from .github import CONNECTOR_TYPE as GITHUB_CONNECTOR_TYPE
+from .github import github_source_entry
+from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
+from .gitlab import gitlab_source_entry
+from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
+from .google_drive import google_drive_source_entry
+from .jira import CONNECTOR_TYPE as JIRA_CONNECTOR_TYPE
+from .jira import jira_source_entry
+from .kdbai import CONNECTOR_TYPE as KDBAI_CONNECTOR_TYPE
+from .kdbai import kdbai_destination_entry
+from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE
+from .local import local_destination_entry, local_source_entry
+from .milvus import CONNECTOR_TYPE as MILVUS_CONNECTOR_TYPE
+from .milvus import milvus_destination_entry
+from .mongodb import CONNECTOR_TYPE as MONGODB_CONNECTOR_TYPE
+from .mongodb import mongodb_destination_entry, mongodb_source_entry
+from .neo4j import CONNECTOR_TYPE as NEO4J_CONNECTOR_TYPE
+from .neo4j import neo4j_destination_entry
+from .notion.connector import CONNECTOR_TYPE as NOTION_CONNECTOR_TYPE
+from .notion.connector import notion_source_entry
+from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE
+from .onedrive import onedrive_destination_entry, onedrive_source_entry
+from .outlook import CONNECTOR_TYPE as OUTLOOK_CONNECTOR_TYPE
+from .outlook import outlook_source_entry
+from .pinecone import CONNECTOR_TYPE as PINECONE_CONNECTOR_TYPE
+from .pinecone import pinecone_destination_entry
+from .redisdb import CONNECTOR_TYPE as REDIS_CONNECTOR_TYPE
+from .redisdb import redis_destination_entry
+from .salesforce import CONNECTOR_TYPE as SALESFORCE_CONNECTOR_TYPE
+from .salesforce import salesforce_source_entry
+from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE
+from .sharepoint import sharepoint_source_entry
+from .slack import CONNECTOR_TYPE as SLACK_CONNECTOR_TYPE
+from .slack import slack_source_entry
+from .vectara import CONNECTOR_TYPE as VECTARA_CONNECTOR_TYPE
+from .vectara import vectara_destination_entry
+from .zendesk.zendesk import CONNECTOR_TYPE as ZENDESK_CONNECTOR_TYPE
+from .zendesk.zendesk import zendesk_source_entry
+
+add_source_entry(source_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_source_entry)
+add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
+
+add_destination_entry(destination_type=CHROMA_CONNECTOR_TYPE, entry=chroma_destination_entry)
+
+add_source_entry(source_type=COUCHBASE_CONNECTOR_TYPE, entry=couchbase_source_entry)
+add_destination_entry(destination_type=COUCHBASE_CONNECTOR_TYPE, entry=couchbase_destination_entry)
+
+add_destination_entry(
+    destination_type=DELTA_TABLE_CONNECTOR_TYPE, entry=delta_table_destination_entry
+)
+
+
+add_source_entry(source_type=GOOGLE_DRIVE_CONNECTOR_TYPE, entry=google_drive_source_entry)
+
+add_source_entry(source_type=LOCAL_CONNECTOR_TYPE, entry=local_source_entry)
+add_destination_entry(destination_type=LOCAL_CONNECTOR_TYPE, entry=local_destination_entry)
+
+add_source_entry(source_type=ONEDRIVE_CONNECTOR_TYPE, entry=onedrive_source_entry)
+add_destination_entry(destination_type=ONEDRIVE_CONNECTOR_TYPE, entry=onedrive_destination_entry)
+
+add_destination_entry(destination_type=NEO4J_CONNECTOR_TYPE, entry=neo4j_destination_entry)
+
+add_source_entry(source_type=SALESFORCE_CONNECTOR_TYPE, entry=salesforce_source_entry)
+
+add_destination_entry(destination_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_destination_entry)
+add_source_entry(source_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_source_entry)
+
+add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry)
+add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry)
+
+add_destination_entry(destination_type=MILVUS_CONNECTOR_TYPE, entry=milvus_destination_entry)
+add_destination_entry(
+    destination_type=AZURE_AI_SEARCH_CONNECTOR_TYPE,
+    entry=azure_ai_search_destination_entry,
+)
+
+add_destination_entry(destination_type=KDBAI_CONNECTOR_TYPE, entry=kdbai_destination_entry)
+add_source_entry(source_type=AIRTABLE_CONNECTOR_TYPE, entry=airtable_source_entry)
+add_source_entry(source_type=NOTION_CONNECTOR_TYPE, entry=notion_source_entry)
+
+add_source_entry(source_type=OUTLOOK_CONNECTOR_TYPE, entry=outlook_source_entry)
+
+add_source_entry(source_type=GITLAB_CONNECTOR_TYPE, entry=gitlab_source_entry)
+
+add_source_entry(source_type=SLACK_CONNECTOR_TYPE, entry=slack_source_entry)
+
+add_destination_entry(destination_type=VECTARA_CONNECTOR_TYPE, entry=vectara_destination_entry)
+add_source_entry(source_type=CONFLUENCE_CONNECTOR_TYPE, entry=confluence_source_entry)
+
+add_source_entry(source_type=DISCORD_CONNECTOR_TYPE, entry=discord_source_entry)
+add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destination_entry)
+
+add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
+
+add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)
+add_source_entry(source_type=GITHUB_CONNECTOR_TYPE, entry=github_source_entry)

unstructured_ingest/processes/connectors/airtable.py

@@ -0,0 +1,238 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import BaseModel, Field, Secret, field_validator
+
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.error import ValueError
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    Indexer,
+    IndexerConfig,
+)
+from unstructured_ingest.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from pyairtable import Api
+    from pyairtable.api.types import RecordDict
+
+CONNECTOR_TYPE = "airtable"
+
+
+class AirtableTableMeta(BaseModel):
+    """Metadata specifying a table id, a base id which the table is stored in,
+    and an t.Optional view id in case particular rows and fields are to be ingested"""
+
+    base_id: str
+    table_id: str
+    view_id: Optional[str] = None
+
+    def get_id(self) -> str:
+        id_s = f"{self.base_id}{self.table_id}"
+        id_s = f"{id_s}{self.view_id}" if self.view_id else id_s
+        return str(uuid5(NAMESPACE_DNS, id_s))
+
+
+class AirtableAccessConfig(AccessConfig):
+    personal_access_token: str = Field(
+        description="Personal access token to authenticate into Airtable. Check: "
+        "https://support.airtable.com/docs/creating-and-using-api-keys-and-access-tokens "
+        "for more info"
+    )
+
+
+class AirtableConnectionConfig(ConnectionConfig):
+    access_config: Secret[AirtableAccessConfig]
+
+    @requires_dependencies(["pyairtable"], extras="airtable")
+    def get_client(self) -> "Api":
+        from pyairtable import Api
+
+        access_config = self.access_config.get_secret_value()
+        return Api(api_key=access_config.personal_access_token)
+
+
+class AirtableIndexerConfig(IndexerConfig):
+    list_of_paths: Optional[list[str]] = Field(
+        default=None,
+        description="""
+        A list of paths that specify the locations to ingest data from within Airtable.
+
+        If this argument is not set, the connector ingests all tables within each and every base.
+        --list-of-paths: path1 path2 path3 ….
+        path: base_id/table_id(optional)/view_id(optional)/
+
+        To obtain (base, table, view) ids in bulk, check:
+        https://airtable.com/developers/web/api/list-bases (base ids)
+        https://airtable.com/developers/web/api/get-base-schema (table and view ids)
+        https://pyairtable.readthedocs.io/en/latest/metadata.html (base, table and view ids)
+
+        To obtain specific ids from Airtable UI, go to your workspace, and copy any
+        relevant id from the URL structure:
+        https://airtable.com/appAbcDeF1ghijKlm/tblABcdEfG1HIJkLm/viwABCDEfg6hijKLM
+        appAbcDeF1ghijKlm -> base_id
+        tblABcdEfG1HIJkLm -> table_id
+        viwABCDEfg6hijKLM -> view_id
+
+        You can also check: https://support.airtable.com/docs/finding-airtable-ids
+
+        Here is an example for one --list-of-paths:
+            base1/		→ gets the entirety of all tables inside base1
+            base1/table1		→ gets all rows and columns within table1 in base1
+            base1/table1/view1	→ gets the rows and columns that are
+                                  visible in view1 for the table1 in base1
+
+        Examples to invalid airtable_paths:
+            table1          → has to mention base to be valid
+            base1/view1     → has to mention table to be valid
+                """,
+    )
+
+    @classmethod
+    def validate_path(cls, path: str):
+        components = path.split("/")
+        if len(components) > 3:
+            raise ValueError(
+                f"Path must be of the format: base_id/table_id/view_id, "
+                f"where table id and view id are optional. Got: {path}"
+            )
+
+    @field_validator("list_of_paths")
+    @classmethod
+    def validate_format(cls, v: list[str]) -> list[str]:
+        for path in v:
+            cls.validate_path(path=path)
+        return v
+
+
+@dataclass
+class AirtableIndexer(Indexer):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: AirtableConnectionConfig
+    index_config: AirtableIndexerConfig
+
+    def get_all_table_meta(self) -> list[AirtableTableMeta]:
+        client = self.connection_config.get_client()
+        bases = client.bases()
+        airtable_meta = []
+        for base in bases:
+            for table in base.schema().tables:
+                airtable_meta.append(AirtableTableMeta(base_id=base.id, table_id=table.id))
+        return airtable_meta
+
+    def get_base_tables_meta(self, base_id: str) -> list[AirtableTableMeta]:
+        client = self.connection_config.get_client()
+        base = client.base(base_id=base_id)
+        airtable_meta = []
+        for table in base.tables():
+            airtable_meta.append(AirtableTableMeta(base_id=base.id, table_id=table.id))
+        return airtable_meta
+
+    def get_meta_from_list(self) -> list[AirtableTableMeta]:
+        airtable_meta = []
+        for path in self.index_config.list_of_paths:
+            components = path.split("/")
+            if len(components) == 1:
+                airtable_meta.extend(self.get_base_tables_meta(base_id=components[0]))
+            elif len(components) == 2:
+                airtable_meta.append(
+                    AirtableTableMeta(base_id=components[0], table_id=components[1])
+                )
+            elif len(components) == 3:
+                airtable_meta.append(
+                    AirtableTableMeta(
+                        base_id=components[0], table_id=components[1], view_id=components[2]
+                    )
+                )
+            else:
+                raise ValueError(
+                    f"Path must be of the format: base_id/table_id/view_id, "
+                    f"where table id and view id are optional. Got: {path}"
+                )
+        return airtable_meta
+
+    def get_table_metas(self) -> list[AirtableTableMeta]:
+        if not self.index_config.list_of_paths:
+            return self.get_all_table_meta()
+        return self.get_meta_from_list()
+
+    def precheck(self) -> None:
+        client = self.connection_config.get_client()
+        client.request(method="HEAD", url=client.build_url("meta", "bases"))
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        table_metas = self.get_table_metas()
+        for table_meta in table_metas:
+            fullpath = (
+                f"{table_meta.base_id}/{table_meta.table_id}/{table_meta.view_id}.csv"
+                if table_meta.view_id
+                else f"{table_meta.base_id}/{table_meta.table_id}.csv"
+            )
+            yield FileData(
+                identifier=table_meta.get_id(),
+                connector_type=CONNECTOR_TYPE,
+                additional_metadata=table_meta.model_dump(),
+                source_identifiers=SourceIdentifiers(
+                    filename=str(Path(fullpath).name),
+                    fullpath=fullpath,
+                ),
+                display_name=fullpath,
+            )
+
+
+class AirtableDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class AirtableDownloader(Downloader):
+    connection_config: AirtableConnectionConfig
+    download_config: AirtableDownloaderConfig = field(default_factory=AirtableDownloaderConfig)
+    connector_type: str = CONNECTOR_TYPE
+
+    def get_table_contents(self, table_meta: AirtableTableMeta) -> list["RecordDict"]:
+        client = self.connection_config.get_client()
+        table = client.table(base_id=table_meta.base_id, table_name=table_meta.table_id)
+        table_fetch_kwargs = {"view": table_meta.view_id} if table_meta.view_id else {}
+        rows = table.all(**table_fetch_kwargs)
+        return rows
+
+    def _table_row_to_dict(self, table_row: "RecordDict") -> dict:
+        row_dict = {
+            "id": table_row["id"],
+            "created_time": table_row["createdTime"],
+        }
+        row_dict.update(table_row["fields"])
+        return row_dict
+
+    @requires_dependencies(["pandas"], extras="airtable")
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        import pandas as pd
+
+        table_meta = AirtableTableMeta.model_validate(file_data.additional_metadata)
+        table_contents = self.get_table_contents(table_meta=table_meta)
+        df = pd.DataFrame.from_dict(
+            data=[self._table_row_to_dict(table_row=row) for row in table_contents]
+        ).sort_index(axis=1)
+        download_path = self.get_download_path(file_data=file_data)
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+        df.to_csv(path_or_buf=download_path)
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+
+airtable_source_entry = SourceRegistryEntry(
+    indexer=AirtableIndexer,
+    indexer_config=AirtableIndexerConfig,
+    downloader=AirtableDownloader,
+    downloader_config=AirtableDownloaderConfig,
+    connection_config=AirtableConnectionConfig,
+)

unstructured_ingest/processes/connectors/assets/databricks_delta_table_schema.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS `elements` (
+    id STRING NOT NULL PRIMARY KEY,
+    record_id STRING NOT NULL,
+    element_id STRING NOT NULL,
+    text STRING,
+    embeddings ARRAY<FLOAT>,
+    type STRING,
+    metadata VARIANT
+);

unstructured_ingest/processes/connectors/assets/weaviate_collection_config.json

@@ -0,0 +1,23 @@
+{
+    "properties": [
+        {
+            "dataType": [
+                "text"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": true,
+            "name": "record_id",
+            "tokenization": "word"
+        },
+        {
+            "dataType": [
+                "text"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": true,
+            "name": "text",
+            "tokenization": "word"
+        }
+    ],
+    "vectorizer": "none"
+}