unstructured-ingest 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

test/integration/connectors/test_github.py

@@ -0,0 +1,49 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.github import (
+    CONNECTOR_TYPE,
+    GithubAccessConfig,
+    GithubConnectionConfig,
+    GithubDownloader,
+    GithubDownloaderConfig,
+    GithubIndexer,
+    GithubIndexerConfig,
+)
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
+@pytest.mark.asyncio
+@requires_env("GH_READ_ONLY_ACCESS_TOKEN")
+async def test_github_source(temp_dir):
+    access_token = os.environ["GH_READ_ONLY_ACCESS_TOKEN"]
+    connection_config = GithubConnectionConfig(
+        access_config=GithubAccessConfig(access_token=access_token),
+        url="dcneiner/Downloadify",
+    )
+
+    indexer = GithubIndexer(
+        connection_config=connection_config,
+        index_config=GithubIndexerConfig(file_glob=["*.txt", "*.html"]),
+    )
+
+    downloader = GithubDownloader(
+        connection_config=connection_config,
+        download_config=GithubDownloaderConfig(download_dir=temp_dir),
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="github", expected_num_files=2, validate_downloaded_files=True
+        ),
+    )

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.6.2"  # pragma: no cover
+__version__ = "0.6.4"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -31,6 +31,8 @@ from .delta_table import CONNECTOR_TYPE as DELTA_TABLE_CONNECTOR_TYPE
 from .delta_table import delta_table_destination_entry
 from .discord import CONNECTOR_TYPE as DISCORD_CONNECTOR_TYPE
 from .discord import discord_source_entry
+from .github import CONNECTOR_TYPE as GITHUB_CONNECTOR_TYPE
+from .github import github_source_entry
 from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -124,3 +126,4 @@ add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destina
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
 
 add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)
+add_source_entry(source_type=GITHUB_CONNECTOR_TYPE, entry=github_source_entry)

unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -68,7 +68,6 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
         "Databricks workspace endpoint or the "
         "Databricks accounts endpoint.",
     )
-    user_agent: str = "unstructuredio_oss"
 
     def wrap_error(self, e: Exception) -> Exception:
         from databricks.sdk.errors.base import DatabricksError
@@ -101,7 +100,9 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
         config = Config(
             host=self.host,
             **self.access_config.get_secret_value().model_dump(),
-        ).with_user_agent_extra("PyDatabricksSdk", self.user_agent)
+        ).with_user_agent_extra(
+            "PyDatabricksSdk", os.getenv("UNSTRUCTURED_USER_AGENT", "unstructuredio_oss")
+        )
 
         return WorkspaceClient(config=config)
 

unstructured_ingest/v2/processes/connectors/github.py

@@ -0,0 +1,221 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import Field, Secret, field_validator
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    Indexer,
+    IndexerConfig,
+    download_responses,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+
+if TYPE_CHECKING:
+    from github import ContentFile, GitTreeElement, Repository
+    from github import Github as GithubClient
+    from github.GithubException import GithubException
+    from requests import HTTPError
+
+CONNECTOR_TYPE = "github"
+
+
+class GithubAccessConfig(AccessConfig):
+    access_token: str = Field(description="Github acess token")
+
+
+class GithubConnectionConfig(ConnectionConfig):
+    access_config: Secret[GithubAccessConfig]
+    url: str = Field(description="Github url or repository owner/name pair")
+
+    @field_validator("url", mode="after")
+    def conform_url(cls, value: str):
+        parsed_url = urlparse(value)
+        return parsed_url.path
+
+    def get_full_url(self):
+        return f"https://github.com/{self.url}"
+
+    @requires_dependencies(["github"], extras="github")
+    def get_client(self) -> "GithubClient":
+        from github import Github as GithubClient
+
+        return GithubClient(login_or_token=self.access_config.get_secret_value().access_token)
+
+    def get_repo(self) -> "Repository":
+        client = self.get_client()
+        return client.get_repo(self.url)
+
+    def wrap_github_exception(self, e: "GithubException") -> Exception:
+        data = e.data
+        status_code = e.status
+        message = data.get("message")
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {message}")
+        if 400 <= status_code < 500:
+            return UserError(message)
+        if status_code > 500:
+            return ProviderError(message)
+        logger.debug(f"unhandled github error: {e}")
+        return e
+
+    def wrap_http_error(self, e: "HTTPError") -> Exception:
+        status_code = e.response.status_code
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {e.response.text}")
+        if 400 <= status_code < 500:
+            return UserError(e.response.text)
+        if status_code > 500:
+            return ProviderError(e.response.text)
+        logger.debug(f"unhandled http error: {e}")
+        return e
+
+    @requires_dependencies(["requests"], extras="github")
+    def wrap_error(self, e: Exception) -> Exception:
+        from github.GithubException import GithubException
+        from requests import HTTPError
+
+        if isinstance(e, GithubException):
+            return self.wrap_github_exception(e=e)
+        if isinstance(e, HTTPError):
+            return self.wrap_http_error(e=e)
+        logger.debug(f"unhandled error: {e}")
+        return e
+
+
+class GithubIndexerConfig(IndexerConfig):
+    branch: Optional[str] = Field(
+        description="Branch to index, use the default if one isn't provided", default=None
+    )
+    recursive: bool = Field(
+        description="Recursively index all files in the repository", default=True
+    )
+
+
+@dataclass
+class GithubIndexer(Indexer):
+    connection_config: GithubConnectionConfig
+    index_config: GithubIndexerConfig = field(default_factory=GithubIndexerConfig)
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        try:
+            self.connection_config.get_repo()
+        except Exception as e:
+            raise self.connection_config.wrap_error(e=e)
+
+    def get_branch(self) -> str:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        return sha
+
+    def list_files(self) -> list["GitTreeElement"]:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        git_tree = repo.get_git_tree(sha, recursive=self.index_config.recursive)
+        file_elements = [
+            element for element in git_tree.tree if element.size is not None and element.size > 0
+        ]
+        return file_elements
+
+    def convert_element(self, element: "GitTreeElement") -> FileData:
+        full_path = (
+            f"{self.connection_config.get_full_url()}/blob/{self.get_branch()}/{element.path}"
+        )
+
+        return FileData(
+            identifier=str(uuid5(NAMESPACE_DNS, full_path)),
+            connector_type=self.connector_type,
+            display_name=full_path,
+            source_identifiers=SourceIdentifiers(
+                filename=Path(element.path).name,
+                fullpath=(Path(self.get_branch()) / element.path).as_posix(),
+                rel_path=element.path,
+            ),
+            metadata=FileDataSourceMetadata(
+                url=element.url,
+                version=element.etag,
+                record_locator={},
+                date_modified=str(element.last_modified_datetime.timestamp()),
+                date_processed=str(time()),
+                filesize_bytes=element.size,
+                permissions_data=[{"mode": element.mode}],
+            ),
+        )
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for element in self.list_files():
+            yield self.convert_element(element=element)
+
+
+class GithubDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class GithubDownloader(Downloader):
+    download_config: GithubDownloaderConfig
+    connection_config: GithubConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["github"], extras="github")
+    def get_file(self, file_data: FileData) -> "ContentFile":
+        from github.GithubException import UnknownObjectException
+
+        path = file_data.source_identifiers.relative_path
+        repo = self.connection_config.get_repo()
+
+        try:
+            content_file = repo.get_contents(path)
+        except UnknownObjectException as e:
+            logger.error(f"File doesn't exists {self.connection_config.url}/{path}: {e}")
+            raise UserError(f"File not found: {path}")
+        return content_file
+
+    @requires_dependencies(["requests"], extras="github")
+    def get_contents(self, content_file: "ContentFile") -> bytes:
+        import requests
+
+        if content_file.decoded_content:
+            return content_file.decoded_content
+        download_url = content_file.download_url
+        resp = requests.get(download_url)
+        try:
+            resp.raise_for_status()
+        except requests.HTTPError as e:
+            raise self.connection_config.wrap_error(e=e)
+        return resp.content
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        content_file = self.get_file(file_data)
+        contents = self.get_contents(content_file)
+        download_path = self.get_download_path(file_data)
+        with download_path.open("wb") as f:
+            f.write(contents)
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+
+github_source_entry = SourceRegistryEntry(
+    indexer=GithubIndexer,
+    indexer_config=GithubIndexerConfig,
+    downloader=GithubDownloader,
+    downloader_config=GithubDownloaderConfig,
+    connection_config=GithubConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -1,4 +1,5 @@
 import json
+import os
 from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Generator, Optional
@@ -42,7 +43,6 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
     access_config: Secret[DatabricksDeltaTablesAccessConfig]
     server_hostname: str = Field(description="server hostname connection config value")
     http_path: str = Field(description="http path connection config value")
-    user_agent: str = "unstructuredio_oss"
 
     @requires_dependencies(["databricks"], extras="databricks-delta-tables")
     def get_credentials_provider(self) -> "oauth_service_principal":
@@ -86,7 +86,9 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
         from databricks.sql import connect
 
         connect_kwargs = connect_kwargs or {}
-        connect_kwargs["_user_agent_entry"] = self.user_agent
+        connect_kwargs["_user_agent_entry"] = os.getenv(
+            "UNSTRUCTURED_USER_AGENT", "unstructuredio_oss"
+        )
         connect_kwargs["server_hostname"] = connect_kwargs.get(
             "server_hostname", self.server_hostname
         )

{unstructured_ingest-0.6.2.dist-info → unstructured_ingest-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.6.2
+Version: 0.6.4
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,348 +22,348 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: click
-Requires-Dist: python-dateutil
-Requires-Dist: dataclasses_json
 Requires-Dist: pydantic>=2.7
-Requires-Dist: tqdm
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: pandas
+Requires-Dist: tqdm
+Requires-Dist: click
+Requires-Dist: dataclasses_json
+Requires-Dist: python-dateutil
 Requires-Dist: numpy
+Requires-Dist: pandas
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
-Requires-Dist: pandas; extra == "remote"
 Requires-Dist: numpy; extra == "remote"
+Requires-Dist: pandas; extra == "remote"
 Provides-Extra: csv
 Requires-Dist: unstructured[tsv]; extra == "csv"
-Requires-Dist: pandas; extra == "csv"
 Requires-Dist: numpy; extra == "csv"
+Requires-Dist: pandas; extra == "csv"
 Provides-Extra: doc
 Requires-Dist: unstructured[docx]; extra == "doc"
-Requires-Dist: pandas; extra == "doc"
 Requires-Dist: numpy; extra == "doc"
+Requires-Dist: pandas; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
-Requires-Dist: pandas; extra == "docx"
 Requires-Dist: numpy; extra == "docx"
+Requires-Dist: pandas; extra == "docx"
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
-Requires-Dist: pandas; extra == "epub"
 Requires-Dist: numpy; extra == "epub"
+Requires-Dist: pandas; extra == "epub"
 Provides-Extra: md
 Requires-Dist: unstructured[md]; extra == "md"
-Requires-Dist: pandas; extra == "md"
 Requires-Dist: numpy; extra == "md"
+Requires-Dist: pandas; extra == "md"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
-Requires-Dist: pandas; extra == "msg"
 Requires-Dist: numpy; extra == "msg"
+Requires-Dist: pandas; extra == "msg"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
-Requires-Dist: pandas; extra == "odt"
 Requires-Dist: numpy; extra == "odt"
+Requires-Dist: pandas; extra == "odt"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
-Requires-Dist: pandas; extra == "org"
 Requires-Dist: numpy; extra == "org"
+Requires-Dist: pandas; extra == "org"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
-Requires-Dist: pandas; extra == "pdf"
 Requires-Dist: numpy; extra == "pdf"
+Requires-Dist: pandas; extra == "pdf"
 Provides-Extra: ppt
 Requires-Dist: unstructured[pptx]; extra == "ppt"
-Requires-Dist: pandas; extra == "ppt"
 Requires-Dist: numpy; extra == "ppt"
+Requires-Dist: pandas; extra == "ppt"
 Provides-Extra: pptx
 Requires-Dist: unstructured[pptx]; extra == "pptx"
-Requires-Dist: pandas; extra == "pptx"
 Requires-Dist: numpy; extra == "pptx"
+Requires-Dist: pandas; extra == "pptx"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
-Requires-Dist: pandas; extra == "rtf"
 Requires-Dist: numpy; extra == "rtf"
+Requires-Dist: pandas; extra == "rtf"
 Provides-Extra: rst
 Requires-Dist: unstructured[rst]; extra == "rst"
-Requires-Dist: pandas; extra == "rst"
 Requires-Dist: numpy; extra == "rst"
+Requires-Dist: pandas; extra == "rst"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
-Requires-Dist: pandas; extra == "tsv"
 Requires-Dist: numpy; extra == "tsv"
+Requires-Dist: pandas; extra == "tsv"
 Provides-Extra: xlsx
 Requires-Dist: unstructured[xlsx]; extra == "xlsx"
-Requires-Dist: pandas; extra == "xlsx"
 Requires-Dist: numpy; extra == "xlsx"
+Requires-Dist: pandas; extra == "xlsx"
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
-Requires-Dist: pandas; extra == "airtable"
 Requires-Dist: numpy; extra == "airtable"
+Requires-Dist: pandas; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
-Requires-Dist: pandas; extra == "astradb"
 Requires-Dist: numpy; extra == "astradb"
+Requires-Dist: pandas; extra == "astradb"
 Provides-Extra: azure
 Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
-Requires-Dist: pandas; extra == "azure"
 Requires-Dist: numpy; extra == "azure"
+Requires-Dist: pandas; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
-Requires-Dist: pandas; extra == "azure-ai-search"
 Requires-Dist: numpy; extra == "azure-ai-search"
+Requires-Dist: pandas; extra == "azure-ai-search"
 Provides-Extra: biomed
 Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
-Requires-Dist: pandas; extra == "biomed"
 Requires-Dist: numpy; extra == "biomed"
+Requires-Dist: pandas; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
-Requires-Dist: pandas; extra == "box"
 Requires-Dist: numpy; extra == "box"
+Requires-Dist: pandas; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
-Requires-Dist: pandas; extra == "chroma"
 Requires-Dist: numpy; extra == "chroma"
+Requires-Dist: pandas; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
-Requires-Dist: pandas; extra == "clarifai"
 Requires-Dist: numpy; extra == "clarifai"
+Requires-Dist: pandas; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
-Requires-Dist: pandas; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: numpy; extra == "confluence"
+Requires-Dist: pandas; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
-Requires-Dist: pandas; extra == "couchbase"
 Requires-Dist: numpy; extra == "couchbase"
+Requires-Dist: pandas; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
-Requires-Dist: pandas; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: numpy; extra == "delta-table"
+Requires-Dist: pandas; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
-Requires-Dist: pandas; extra == "discord"
 Requires-Dist: numpy; extra == "discord"
+Requires-Dist: pandas; extra == "discord"
 Provides-Extra: dropbox
 Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
-Requires-Dist: pandas; extra == "dropbox"
 Requires-Dist: numpy; extra == "dropbox"
+Requires-Dist: pandas; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
-Requires-Dist: pandas; extra == "duckdb"
 Requires-Dist: numpy; extra == "duckdb"
+Requires-Dist: pandas; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
-Requires-Dist: pandas; extra == "elasticsearch"
 Requires-Dist: numpy; extra == "elasticsearch"
+Requires-Dist: pandas; extra == "elasticsearch"
 Provides-Extra: gcs
 Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: pandas; extra == "gcs"
 Requires-Dist: numpy; extra == "gcs"
+Requires-Dist: pandas; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
-Requires-Dist: pandas; extra == "github"
 Requires-Dist: numpy; extra == "github"
+Requires-Dist: pandas; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
-Requires-Dist: pandas; extra == "gitlab"
 Requires-Dist: numpy; extra == "gitlab"
+Requires-Dist: pandas; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
-Requires-Dist: pandas; extra == "google-drive"
 Requires-Dist: numpy; extra == "google-drive"
+Requires-Dist: pandas; extra == "google-drive"
 Provides-Extra: hubspot
 Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
-Requires-Dist: pandas; extra == "hubspot"
 Requires-Dist: numpy; extra == "hubspot"
+Requires-Dist: pandas; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
 Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
-Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
 Requires-Dist: httpx; extra == "ibm-watsonx-s3"
-Requires-Dist: pandas; extra == "ibm-watsonx-s3"
+Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: numpy; extra == "ibm-watsonx-s3"
+Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
-Requires-Dist: pandas; extra == "jira"
 Requires-Dist: numpy; extra == "jira"
+Requires-Dist: pandas; extra == "jira"
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka; extra == "kafka"
-Requires-Dist: pandas; extra == "kafka"
 Requires-Dist: numpy; extra == "kafka"
+Requires-Dist: pandas; extra == "kafka"
 Provides-Extra: kdbai
 Requires-Dist: kdbai-client>=1.4.0; extra == "kdbai"
-Requires-Dist: pandas; extra == "kdbai"
 Requires-Dist: numpy; extra == "kdbai"
+Requires-Dist: pandas; extra == "kdbai"
 Provides-Extra: lancedb
 Requires-Dist: lancedb; extra == "lancedb"
-Requires-Dist: pandas; extra == "lancedb"
 Requires-Dist: numpy; extra == "lancedb"
+Requires-Dist: pandas; extra == "lancedb"
 Provides-Extra: milvus
 Requires-Dist: pymilvus; extra == "milvus"
-Requires-Dist: pandas; extra == "milvus"
 Requires-Dist: numpy; extra == "milvus"
+Requires-Dist: pandas; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
-Requires-Dist: pandas; extra == "mongodb"
 Requires-Dist: numpy; extra == "mongodb"
+Requires-Dist: pandas; extra == "mongodb"
 Provides-Extra: neo4j
 Requires-Dist: neo4j-rust-ext; extra == "neo4j"
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
-Requires-Dist: pandas; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: numpy; extra == "neo4j"
+Requires-Dist: pandas; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: backoff; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: pandas; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: numpy; extra == "notion"
+Requires-Dist: pandas; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: pandas; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: numpy; extra == "onedrive"
+Requires-Dist: pandas; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
-Requires-Dist: pandas; extra == "opensearch"
 Requires-Dist: numpy; extra == "opensearch"
+Requires-Dist: pandas; extra == "opensearch"
 Provides-Extra: outlook
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
-Requires-Dist: pandas; extra == "outlook"
 Requires-Dist: numpy; extra == "outlook"
+Requires-Dist: pandas; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
-Requires-Dist: pandas; extra == "pinecone"
 Requires-Dist: numpy; extra == "pinecone"
+Requires-Dist: pandas; extra == "pinecone"
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary; extra == "postgres"
-Requires-Dist: pandas; extra == "postgres"
 Requires-Dist: numpy; extra == "postgres"
+Requires-Dist: pandas; extra == "postgres"
 Provides-Extra: qdrant
 Requires-Dist: qdrant-client; extra == "qdrant"
-Requires-Dist: pandas; extra == "qdrant"
 Requires-Dist: numpy; extra == "qdrant"
+Requires-Dist: pandas; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
-Requires-Dist: pandas; extra == "reddit"
 Requires-Dist: numpy; extra == "reddit"
+Requires-Dist: pandas; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
-Requires-Dist: pandas; extra == "redis"
 Requires-Dist: numpy; extra == "redis"
+Requires-Dist: pandas; extra == "redis"
 Provides-Extra: s3
 Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
-Requires-Dist: pandas; extra == "s3"
 Requires-Dist: numpy; extra == "s3"
+Requires-Dist: pandas; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
-Requires-Dist: pandas; extra == "sharepoint"
 Requires-Dist: numpy; extra == "sharepoint"
+Requires-Dist: pandas; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
-Requires-Dist: pandas; extra == "salesforce"
 Requires-Dist: numpy; extra == "salesforce"
+Requires-Dist: pandas; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
-Requires-Dist: pandas; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: numpy; extra == "sftp"
+Requires-Dist: pandas; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
-Requires-Dist: pandas; extra == "slack"
 Requires-Dist: numpy; extra == "slack"
+Requires-Dist: pandas; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
-Requires-Dist: pandas; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: numpy; extra == "snowflake"
+Requires-Dist: pandas; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
-Requires-Dist: pandas; extra == "wikipedia"
 Requires-Dist: numpy; extra == "wikipedia"
+Requires-Dist: pandas; extra == "wikipedia"
 Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"
-Requires-Dist: pandas; extra == "weaviate"
 Requires-Dist: numpy; extra == "weaviate"
+Requires-Dist: pandas; extra == "weaviate"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
-Requires-Dist: pandas; extra == "databricks-volumes"
 Requires-Dist: numpy; extra == "databricks-volumes"
+Requires-Dist: pandas; extra == "databricks-volumes"
 Provides-Extra: databricks-delta-tables
 Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
-Requires-Dist: pandas; extra == "databricks-delta-tables"
 Requires-Dist: numpy; extra == "databricks-delta-tables"
+Requires-Dist: pandas; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
-Requires-Dist: pandas; extra == "singlestore"
 Requires-Dist: numpy; extra == "singlestore"
+Requires-Dist: pandas; extra == "singlestore"
 Provides-Extra: vectara
 Requires-Dist: requests; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
-Requires-Dist: pandas; extra == "vectara"
 Requires-Dist: numpy; extra == "vectara"
+Requires-Dist: pandas; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: pyarrow; extra == "vastdb"
-Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
-Requires-Dist: pandas; extra == "vastdb"
+Requires-Dist: ibis; extra == "vastdb"
+Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: numpy; extra == "vastdb"
+Requires-Dist: pandas; extra == "vastdb"
 Provides-Extra: zendesk
-Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: bs4; extra == "zendesk"
+Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: aiofiles; extra == "zendesk"
-Requires-Dist: pandas; extra == "zendesk"
 Requires-Dist: numpy; extra == "zendesk"
+Requires-Dist: pandas; extra == "zendesk"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
-Requires-Dist: pandas; extra == "embed-huggingface"
 Requires-Dist: numpy; extra == "embed-huggingface"
+Requires-Dist: pandas; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
 Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
-Requires-Dist: pandas; extra == "embed-octoai"
 Requires-Dist: numpy; extra == "embed-octoai"
+Requires-Dist: pandas; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
-Requires-Dist: pandas; extra == "embed-vertexai"
 Requires-Dist: numpy; extra == "embed-vertexai"
+Requires-Dist: pandas; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
 Requires-Dist: voyageai; extra == "embed-voyageai"
-Requires-Dist: pandas; extra == "embed-voyageai"
 Requires-Dist: numpy; extra == "embed-voyageai"
+Requires-Dist: pandas; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
-Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Requires-Dist: numpy; extra == "embed-mixedbreadai"
+Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Provides-Extra: openai
 Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
-Requires-Dist: pandas; extra == "openai"
 Requires-Dist: numpy; extra == "openai"
+Requires-Dist: pandas; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
-Requires-Dist: pandas; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: numpy; extra == "bedrock"
+Requires-Dist: pandas; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
-Requires-Dist: pandas; extra == "togetherai"
 Requires-Dist: numpy; extra == "togetherai"
+Requires-Dist: pandas; extra == "togetherai"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{unstructured_ingest-0.6.2.dist-info → unstructured_ingest-0.6.4.dist-info}/RECORD

@@ -11,6 +11,7 @@ test/integration/connectors/test_chroma.py,sha256=yn2p8U8yE9LaF-IEKiLp2XB4T4Vqo-
 test/integration/connectors/test_confluence.py,sha256=W93znOusdvFXta8q0dqQ1rKhLafRVIqrfaFqk2FY-fo,3590
 test/integration/connectors/test_delta_table.py,sha256=r2OvLMRfJFfsyIHd1H44Kx6AgNnDjPHKofN7a01oqrY,6916
 test/integration/connectors/test_dropbox.py,sha256=jzpZ6wawLa4sC1BVoHWZJ3cHjL4DWWUEX5ee7bXUOOM,4945
+test/integration/connectors/test_github.py,sha256=cRArMUMtAujsVtR2yp2GHZktX-cSPRSsYhlnnsRhhH8,1503
 test/integration/connectors/test_google_drive.py,sha256=ubjn3wvMhgpGHQs-wT_5icGgTIx2coS6hwNkAHOCEI8,10306
 test/integration/connectors/test_jira.py,sha256=IgF_cdknJ97W3OrNfHKp3YLmRyVwY6BI8jhZwVbrebc,2076
 test/integration/connectors/test_lancedb.py,sha256=AuO2qHDay5NHxQ2b1LzbsNNp6LnUDeB2_0dWlkgAuMo,9223
@@ -111,7 +112,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=UDy7drjkPUljex5sEiDR3ZALQNnlcrCXwJShdKZ37Ek,42
+unstructured_ingest/__version__.py,sha256=0sOJ1f0sRyjdtSL0LUqCE6m6T039ttMVXeIDsmdxcPw,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=7DOnDpGvUNlCoFR7UPRGmOarqH5sFtuUOO5vf8X3oTM,31489
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -425,7 +426,7 @@ unstructured_ingest/v2/processes/embedder.py,sha256=gvlCQDsbQVgcp-2f0Qq4RiFbcr8g
 unstructured_ingest/v2/processes/filter.py,sha256=E1MLxk-XeCm3mZIuM49lJToVcSgOivmTFIZApqOEFs8,2150
 unstructured_ingest/v2/processes/partitioner.py,sha256=HxopDSbovLh_1epeGeVtuWEX7v5KG35BowwKIJ_y4e8,9910
 unstructured_ingest/v2/processes/uncompress.py,sha256=O7q0pOiL6cDBvAa9NAy_vkc873tisjK2Hoq6Z-grRRo,2430
-unstructured_ingest/v2/processes/connectors/__init__.py,sha256=l4Xq4AuzRMTqUv5TU7cE1NbhGCka4SFJFZwG1FoVotE,6666
+unstructured_ingest/v2/processes/connectors/__init__.py,sha256=iyoZrTaEoBPBn9-tczFhLcwKQQoeFbESF3QkJkVimv4,6845
 unstructured_ingest/v2/processes/connectors/airtable.py,sha256=JesWeUv_tIA7b65sE2Z-ixMKuGLlgugZRMKE38ID3zg,8959
 unstructured_ingest/v2/processes/connectors/astradb.py,sha256=SwfUcdrCbMK_LDcHG5auCCuga_luPmZVvhjuAsRtKU0,18304
 unstructured_ingest/v2/processes/connectors/azure_ai_search.py,sha256=K4g-Dh7u7Z13rheNKhnnBXcO6TUfz8RPrrusDAsgkBk,11575
@@ -434,6 +435,7 @@ unstructured_ingest/v2/processes/connectors/confluence.py,sha256=ERHYYutQsnS8eZN
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=LARX4F_1Zd1LnUMSNdvIsbqLoZXZ9kl_vMZh-dRr4XA,12305
 unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=5T4hkXHGitGprpUb20206ODcBh5wchgHkUocji5l2rk,7286
 unstructured_ingest/v2/processes/connectors/discord.py,sha256=yvLxTx0ZRIATqSYLm5d09u9a0ktZGJnGcnzapwePHK8,5301
+unstructured_ingest/v2/processes/connectors/github.py,sha256=d-sh28MVe40vyaTf8b8NkspSrV9zlUHjdSSfSZjOcOA,7772
 unstructured_ingest/v2/processes/connectors/gitlab.py,sha256=hsB5g-3tZqe7bVSqwA0nY9GY4J0cg4PBawf77GkpjZY,10039
 unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=mn1BoUXYw5j-q7jO-rzPPEv_rt1UV9LDN1M_PyREEps,19678
 unstructured_ingest/v2/processes/connectors/jira.py,sha256=_ivv0TqeVPlHG9YMihljzrpYP7OLQAa0D7nqLonaeM8,17149
@@ -455,7 +457,7 @@ unstructured_ingest/v2/processes/connectors/assets/__init__.py,sha256=47DEQpj8HB
 unstructured_ingest/v2/processes/connectors/assets/databricks_delta_table_schema.sql,sha256=dUZZDNkyvQXKqoAThRz3ek7zaUE2l_LAQimlG5WZhH4,211
 unstructured_ingest/v2/processes/connectors/assets/weaviate_collection_config.json,sha256=SJlIO0kXxy866tWQ8bEzvwLwflsoUMIS-OKlxMvHIuE,504
 unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=Oh8SwTWi66gO8BsNF6vRMoQVuegyBPPCpVozkOHEf3A,2136
-unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=EghKdkt4nGacGxulSpjhToHOl5BRLbb3xNZpJzpWNX8,8002
+unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=JktJXC9SYnKLetjsyGJWKXqg5Kml8WY9dcKyr5o_Yxs,8024
 unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=h6qDxQhWlT7H4K1CEfKag1stTiD1o97VckJZERsofqU,2970
 unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=gjICJJwhDHBLt_L-LrMlvJ3DL1DYtwFpyMLb_zYvOIg,3755
 unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=Uss3XPPaq1AsqJOEy4RJgBJw2-bTjrXH2PgtVNYd2w0,3006
@@ -562,7 +564,7 @@ unstructured_ingest/v2/processes/connectors/qdrant/local.py,sha256=cGEyv3Oy6y4BQ
 unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=hsOd2Gliyjzkb21Vv6RAiFf8NAysxd29K0AxBkkm844,5483
 unstructured_ingest/v2/processes/connectors/qdrant/server.py,sha256=odvCZWZp8DmRxLXMR7tHhW-c7UQbix1_zpFdfXfCvKI,1613
 unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=NSEZwJDHh_9kFc31LnG14iRtYF3meK2UfUlQfYnwYEQ,2059
-unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=JrQMlSavPgSD1ruy4xRHaQV5iGiG7oavtm7az0tVVZc,9054
+unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=hJ0yacutrgiCer9cJSfxcNgLlOgsozJ2yGhgy8vZAkk,9086
 unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=BATfX1PQGT2kl8jAbdNKXTojYKJxh3pJV9-h3OBnHGo,5124
 unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=am2d87kDkpTTB0VbPSX3ce9o6oM9KUQu5y9T_p1kgJw,5711
 unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=FOb08WCr0SdzylN88xDP51NdVD4ggDbjanJurwJUrgM,9374
@@ -581,9 +583,9 @@ unstructured_ingest/v2/processes/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 unstructured_ingest/v2/processes/utils/blob_storage.py,sha256=_I3OMdpUElQdIwVs7W9ORU1kncNaZ_nr6lbxeKE8uaU,1014
 unstructured_ingest/v2/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/types/file_data.py,sha256=kowOhvYy0q_-khX3IuR111AfjkdQezEfxjzK6QDH7oA,3836
-unstructured_ingest-0.6.2.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.6.2.dist-info/METADATA,sha256=yUMpJD0UXDhUG1cIIpHkjn-VU2AScEaA12wLmISmG-A,14998
-unstructured_ingest-0.6.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-unstructured_ingest-0.6.2.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.6.2.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.6.2.dist-info/RECORD,,
+unstructured_ingest-0.6.4.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.6.4.dist-info/METADATA,sha256=vsGBmdOat7mYz2HZ-RuKLYaSzpjgOaGwwn_b1qxNC7g,14998
+unstructured_ingest-0.6.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+unstructured_ingest-0.6.4.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.6.4.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.6.4.dist-info/RECORD,,