unstructured-ingest 0.6.1__py3-none-any.whl → 0.6.4__py3-none-any.whl

test/integration/connectors/test_github.py

@@ -0,0 +1,49 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.github import (
+    CONNECTOR_TYPE,
+    GithubAccessConfig,
+    GithubConnectionConfig,
+    GithubDownloader,
+    GithubDownloaderConfig,
+    GithubIndexer,
+    GithubIndexerConfig,
+)
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
+@pytest.mark.asyncio
+@requires_env("GH_READ_ONLY_ACCESS_TOKEN")
+async def test_github_source(temp_dir):
+    access_token = os.environ["GH_READ_ONLY_ACCESS_TOKEN"]
+    connection_config = GithubConnectionConfig(
+        access_config=GithubAccessConfig(access_token=access_token),
+        url="dcneiner/Downloadify",
+    )
+
+    indexer = GithubIndexer(
+        connection_config=connection_config,
+        index_config=GithubIndexerConfig(file_glob=["*.txt", "*.html"]),
+    )
+
+    downloader = GithubDownloader(
+        connection_config=connection_config,
+        download_config=GithubDownloaderConfig(download_dir=temp_dir),
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="github", expected_num_files=2, validate_downloaded_files=True
+        ),
+    )

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.6.1"  # pragma: no cover
+__version__ = "0.6.4"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -31,6 +31,8 @@ from .delta_table import CONNECTOR_TYPE as DELTA_TABLE_CONNECTOR_TYPE
 from .delta_table import delta_table_destination_entry
 from .discord import CONNECTOR_TYPE as DISCORD_CONNECTOR_TYPE
 from .discord import discord_source_entry
+from .github import CONNECTOR_TYPE as GITHUB_CONNECTOR_TYPE
+from .github import github_source_entry
 from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -124,3 +126,4 @@ add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destina
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
 
 add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)
+add_source_entry(source_type=GITHUB_CONNECTOR_TYPE, entry=github_source_entry)

unstructured_ingest/v2/processes/connectors/assets/databricks_delta_table_schema.sql

@@ -0,0 +1,10 @@
+CREATE TABLE elements (
+    id STRING NOT NULL PRIMARY KEY,
+    record_id STRING NOT NULL,
+    element_id STRING NOT NULL,
+    text STRING,
+    embeddings ARRAY<FLOAT>,
+    type STRING,
+    metadata VARIANT
+);
+

unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 from uuid import NAMESPACE_DNS, uuid5
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.errors import (
@@ -61,6 +61,7 @@ class DatabricksVolumesAccessConfig(AccessConfig):
 
 
 class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
+    access_config: Secret[DatabricksVolumesAccessConfig]
     host: Optional[str] = Field(
         default=None,
         description="The Databricks host URL for either the "
@@ -94,12 +95,17 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
     @requires_dependencies(dependencies=["databricks.sdk"], extras="databricks-volumes")
     def get_client(self) -> "WorkspaceClient":
         from databricks.sdk import WorkspaceClient
+        from databricks.sdk.core import Config
 
-        return WorkspaceClient(
+        config = Config(
             host=self.host,
             **self.access_config.get_secret_value().model_dump(),
+        ).with_user_agent_extra(
+            "PyDatabricksSdk", os.getenv("UNSTRUCTURED_USER_AGENT", "unstructuredio_oss")
         )
 
+        return WorkspaceClient(config=config)
+
 
 class DatabricksVolumesIndexerConfig(IndexerConfig, DatabricksPathMixin):
     recursive: bool = False

unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py

@@ -1,14 +1,20 @@
+import json
 import os
-import tempfile
 from contextlib import contextmanager
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field
 
-from unstructured_ingest.utils.data_prep import get_data_df, write_data
-from unstructured_ingest.v2.interfaces import Uploader, UploaderConfig
+from unstructured_ingest.utils.data_prep import get_json_data, write_data
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
+from unstructured_ingest.v2.interfaces import (
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -16,28 +22,50 @@ from unstructured_ingest.v2.processes.connector_registry import (
 from unstructured_ingest.v2.processes.connectors.databricks.volumes import DatabricksPathMixin
 from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
     DatabricksDeltaTablesConnectionConfig,
-    DatabricksDeltaTablesUploadStager,
     DatabricksDeltaTablesUploadStagerConfig,
 )
 from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 CONNECTOR_TYPE = "databricks_volume_delta_tables"
 
 if TYPE_CHECKING:
-    from pandas import DataFrame
+    pass
 
 
 class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMixin):
     database: str = Field(description="Database name", default="default")
-    table_name: str = Field(description="Table name")
+    table_name: Optional[str] = Field(description="Table name", default=None)
+
+
+class DatabricksVolumeDeltaTableStagerConfig(UploadStagerConfig):
+    pass
 
 
 @dataclass
-class DatabricksVolumeDeltaTableStager(DatabricksDeltaTablesUploadStager):
-    def write_output(self, output_path: Path, data: list[dict]) -> Path:
+class DatabricksVolumeDeltaTableStager(UploadStager):
+    upload_stager_config: DatabricksVolumeDeltaTableStagerConfig = field(
+        default_factory=DatabricksVolumeDeltaTableStagerConfig
+    )
+
+    def run(
+        self,
+        elements_filepath: Path,
+        output_dir: Path,
+        output_filename: str,
+        file_data: FileData,
+        **kwargs: Any,
+    ) -> Path:
         # To avoid new line issues when migrating from volumes into delta tables, omit indenting
         # and always write it as a json file
+        output_dir.mkdir(exist_ok=True, parents=True)
+        output_path = output_dir / output_filename
         final_output_path = output_path.with_suffix(".json")
+        data = get_json_data(path=elements_filepath)
+        for element in data:
+            element["id"] = get_enhanced_element_id(element_dict=element, file_data=file_data)
+            element[RECORD_ID_LABEL] = file_data.identifier
+            element["metadata"] = json.dumps(element.get("metadata", {}))
         write_data(path=final_output_path, data=data, indent=None)
         return final_output_path
 
@@ -49,6 +77,29 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
     connector_type: str = CONNECTOR_TYPE
     _columns: Optional[dict[str, str]] = None
 
+    def init(self, **kwargs: Any) -> None:
+        self.create_destination(**kwargs)
+
+    def create_destination(
+        self, destination_name: str = "unstructuredautocreated", **kwargs: Any
+    ) -> bool:
+        table_name = self.upload_config.table_name or destination_name
+        self.upload_config.table_name = table_name
+        connectors_dir = Path(__file__).parents[1]
+        collection_config_file = connectors_dir / "assets" / "databricks_delta_table_schema.sql"
+        with self.get_cursor() as cursor:
+            cursor.execute("SHOW TABLES")
+            table_names = [r[1] for r in cursor.fetchall()]
+            if table_name in table_names:
+                return False
+            with collection_config_file.open() as schema_file:
+                data_lines = schema_file.readlines()
+            data_lines[0] = data_lines[0].replace("elements", table_name)
+            destination_schema = "".join([line.strip() for line in data_lines])
+            logger.info(f"creating table {table_name} for user")
+            cursor.execute(destination_schema)
+            return True
+
     def precheck(self) -> None:
         with self.connection_config.get_cursor() as cursor:
             cursor.execute("SHOW CATALOGS")
@@ -68,14 +119,6 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
                         self.upload_config.database, ", ".join(databases)
                     )
                 )
-            cursor.execute(f"SHOW TABLES IN {self.upload_config.database}")
-            table_names = [r[1] for r in cursor.fetchall()]
-            if self.upload_config.table_name not in table_names:
-                raise ValueError(
-                    "Table {} not found in {}".format(
-                        self.upload_config.table_name, ", ".join(table_names)
-                    )
-                )
 
     def get_output_path(self, file_data: FileData, suffix: str = ".json") -> str:
         filename = Path(file_data.source_identifiers.filename)
@@ -98,51 +141,42 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
                 self._columns = {desc[0]: desc[1] for desc in cursor.description}
         return self._columns
 
-    def _fit_to_schema(self, df: "DataFrame", add_missing_columns: bool = True) -> "DataFrame":
-        import pandas as pd
-
-        table_columns = self.get_table_columns()
-        columns = set(df.columns)
-        schema_fields = set(table_columns.keys())
-        columns_to_drop = columns - schema_fields
-        missing_columns = schema_fields - columns
-
-        if columns_to_drop:
-            logger.info(
-                "Following columns will be dropped to match the table's schema: "
-                f"{', '.join(columns_to_drop)}"
-            )
-        if missing_columns and add_missing_columns:
-            logger.info(
-                "Following null filled columns will be added to match the table's schema:"
-                f" {', '.join(missing_columns)} "
+    def can_delete(self) -> bool:
+        existing_columns = self.get_table_columns()
+        return RECORD_ID_LABEL in existing_columns
+
+    def delete_previous_content(self, file_data: FileData) -> None:
+        logger.debug(
+            f"deleting any content with metadata "
+            f"{RECORD_ID_LABEL}={file_data.identifier} "
+            f"from delta table: {self.upload_config.table_name}"
+        )
+        with self.get_cursor() as cursor:
+            cursor.execute(
+                f"DELETE FROM {self.upload_config.table_name} WHERE {RECORD_ID_LABEL} = '{file_data.identifier}'"  # noqa: E501
             )
-
-        df = df.drop(columns=columns_to_drop)
-
-        if add_missing_columns:
-            for column in missing_columns:
-                df[column] = pd.Series()
-        return df
+            results = cursor.fetchall()
+            deleted_rows = results[0][0]
+            logger.debug(f"deleted {deleted_rows} rows from table {self.upload_config.table_name}")
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        with tempfile.TemporaryDirectory() as temp_dir:
-            df = get_data_df()
-            df = self._fit_to_schema(df=df)
-            temp_path = Path(temp_dir) / path.name
-            df.to_json(temp_path, orient="records", lines=False)
-            with self.get_cursor(staging_allowed_local_path=temp_dir) as cursor:
-                catalog_path = self.get_output_path(file_data=file_data)
-                logger.debug(f"uploading {path.as_posix()} to {catalog_path}")
-                cursor.execute(f"PUT '{temp_path.as_posix()}' INTO '{catalog_path}' OVERWRITE")
-                logger.debug(
-                    f"migrating content from {catalog_path} to "
-                    f"table {self.upload_config.table_name}"
-                )
-                columns = list(df.columns)
-                column_str = ", ".join(columns)
-                sql_statment = f"INSERT INTO `{self.upload_config.table_name}` ({column_str}) SELECT {column_str} FROM json.`{catalog_path}`"  # noqa: E501
-                cursor.execute(sql_statment)
+        if self.can_delete():
+            self.delete_previous_content(file_data=file_data)
+        with self.get_cursor(staging_allowed_local_path=path.parent.as_posix()) as cursor:
+            catalog_path = self.get_output_path(file_data=file_data)
+            logger.debug(f"uploading {path.as_posix()} to {catalog_path}")
+            cursor.execute(f"PUT '{path.as_posix()}' INTO '{catalog_path}' OVERWRITE")
+            logger.debug(
+                f"migrating content from {catalog_path} to "
+                f"table {self.upload_config.table_name}"
+            )
+            data = get_json_data(path=path)
+            columns = data[0].keys()
+            select_columns = ["PARSE_JSON(metadata)" if c == "metadata" else c for c in columns]
+            column_str = ", ".join(columns)
+            select_column_str = ", ".join(select_columns)
+            sql_statment = f"INSERT INTO `{self.upload_config.table_name}` ({column_str}) SELECT {select_column_str} FROM json.`{catalog_path}`"  # noqa: E501
+            cursor.execute(sql_statment)
 
 
 databricks_volumes_delta_tables_destination_entry = DestinationRegistryEntry(

unstructured_ingest/v2/processes/connectors/github.py

@@ -0,0 +1,221 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import Field, Secret, field_validator
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    Indexer,
+    IndexerConfig,
+    download_responses,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+
+if TYPE_CHECKING:
+    from github import ContentFile, GitTreeElement, Repository
+    from github import Github as GithubClient
+    from github.GithubException import GithubException
+    from requests import HTTPError
+
+CONNECTOR_TYPE = "github"
+
+
+class GithubAccessConfig(AccessConfig):
+    access_token: str = Field(description="Github acess token")
+
+
+class GithubConnectionConfig(ConnectionConfig):
+    access_config: Secret[GithubAccessConfig]
+    url: str = Field(description="Github url or repository owner/name pair")
+
+    @field_validator("url", mode="after")
+    def conform_url(cls, value: str):
+        parsed_url = urlparse(value)
+        return parsed_url.path
+
+    def get_full_url(self):
+        return f"https://github.com/{self.url}"
+
+    @requires_dependencies(["github"], extras="github")
+    def get_client(self) -> "GithubClient":
+        from github import Github as GithubClient
+
+        return GithubClient(login_or_token=self.access_config.get_secret_value().access_token)
+
+    def get_repo(self) -> "Repository":
+        client = self.get_client()
+        return client.get_repo(self.url)
+
+    def wrap_github_exception(self, e: "GithubException") -> Exception:
+        data = e.data
+        status_code = e.status
+        message = data.get("message")
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {message}")
+        if 400 <= status_code < 500:
+            return UserError(message)
+        if status_code > 500:
+            return ProviderError(message)
+        logger.debug(f"unhandled github error: {e}")
+        return e
+
+    def wrap_http_error(self, e: "HTTPError") -> Exception:
+        status_code = e.response.status_code
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {e.response.text}")
+        if 400 <= status_code < 500:
+            return UserError(e.response.text)
+        if status_code > 500:
+            return ProviderError(e.response.text)
+        logger.debug(f"unhandled http error: {e}")
+        return e
+
+    @requires_dependencies(["requests"], extras="github")
+    def wrap_error(self, e: Exception) -> Exception:
+        from github.GithubException import GithubException
+        from requests import HTTPError
+
+        if isinstance(e, GithubException):
+            return self.wrap_github_exception(e=e)
+        if isinstance(e, HTTPError):
+            return self.wrap_http_error(e=e)
+        logger.debug(f"unhandled error: {e}")
+        return e
+
+
+class GithubIndexerConfig(IndexerConfig):
+    branch: Optional[str] = Field(
+        description="Branch to index, use the default if one isn't provided", default=None
+    )
+    recursive: bool = Field(
+        description="Recursively index all files in the repository", default=True
+    )
+
+
+@dataclass
+class GithubIndexer(Indexer):
+    connection_config: GithubConnectionConfig
+    index_config: GithubIndexerConfig = field(default_factory=GithubIndexerConfig)
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        try:
+            self.connection_config.get_repo()
+        except Exception as e:
+            raise self.connection_config.wrap_error(e=e)
+
+    def get_branch(self) -> str:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        return sha
+
+    def list_files(self) -> list["GitTreeElement"]:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        git_tree = repo.get_git_tree(sha, recursive=self.index_config.recursive)
+        file_elements = [
+            element for element in git_tree.tree if element.size is not None and element.size > 0
+        ]
+        return file_elements
+
+    def convert_element(self, element: "GitTreeElement") -> FileData:
+        full_path = (
+            f"{self.connection_config.get_full_url()}/blob/{self.get_branch()}/{element.path}"
+        )
+
+        return FileData(
+            identifier=str(uuid5(NAMESPACE_DNS, full_path)),
+            connector_type=self.connector_type,
+            display_name=full_path,
+            source_identifiers=SourceIdentifiers(
+                filename=Path(element.path).name,
+                fullpath=(Path(self.get_branch()) / element.path).as_posix(),
+                rel_path=element.path,
+            ),
+            metadata=FileDataSourceMetadata(
+                url=element.url,
+                version=element.etag,
+                record_locator={},
+                date_modified=str(element.last_modified_datetime.timestamp()),
+                date_processed=str(time()),
+                filesize_bytes=element.size,
+                permissions_data=[{"mode": element.mode}],
+            ),
+        )
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for element in self.list_files():
+            yield self.convert_element(element=element)
+
+
+class GithubDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class GithubDownloader(Downloader):
+    download_config: GithubDownloaderConfig
+    connection_config: GithubConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["github"], extras="github")
+    def get_file(self, file_data: FileData) -> "ContentFile":
+        from github.GithubException import UnknownObjectException
+
+        path = file_data.source_identifiers.relative_path
+        repo = self.connection_config.get_repo()
+
+        try:
+            content_file = repo.get_contents(path)
+        except UnknownObjectException as e:
+            logger.error(f"File doesn't exists {self.connection_config.url}/{path}: {e}")
+            raise UserError(f"File not found: {path}")
+        return content_file
+
+    @requires_dependencies(["requests"], extras="github")
+    def get_contents(self, content_file: "ContentFile") -> bytes:
+        import requests
+
+        if content_file.decoded_content:
+            return content_file.decoded_content
+        download_url = content_file.download_url
+        resp = requests.get(download_url)
+        try:
+            resp.raise_for_status()
+        except requests.HTTPError as e:
+            raise self.connection_config.wrap_error(e=e)
+        return resp.content
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        content_file = self.get_file(file_data)
+        contents = self.get_contents(content_file)
+        download_path = self.get_download_path(file_data)
+        with download_path.open("wb") as f:
+            f.write(contents)
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+
+github_source_entry = SourceRegistryEntry(
+    indexer=GithubIndexer,
+    indexer_config=GithubIndexerConfig,
+    downloader=GithubDownloader,
+    downloader_config=GithubDownloaderConfig,
+    connection_config=GithubConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -1,4 +1,5 @@
 import json
+import os
 from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Generator, Optional
@@ -42,7 +43,6 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
     access_config: Secret[DatabricksDeltaTablesAccessConfig]
     server_hostname: str = Field(description="server hostname connection config value")
     http_path: str = Field(description="http path connection config value")
-    user_agent: str = "unstructuredio_oss"
 
     @requires_dependencies(["databricks"], extras="databricks-delta-tables")
     def get_credentials_provider(self) -> "oauth_service_principal":
@@ -86,7 +86,9 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
         from databricks.sql import connect
 
         connect_kwargs = connect_kwargs or {}
-        connect_kwargs["_user_agent_entry"] = self.user_agent
+        connect_kwargs["_user_agent_entry"] = os.getenv(
+            "UNSTRUCTURED_USER_AGENT", "unstructuredio_oss"
+        )
         connect_kwargs["server_hostname"] = connect_kwargs.get(
             "server_hostname", self.server_hostname
         )

{unstructured_ingest-0.6.1.dist-info → unstructured_ingest-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.6.1
+Version: 0.6.4
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,12 +22,12 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: click
-Requires-Dist: tqdm
+Requires-Dist: pydantic>=2.7
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: python-dateutil
+Requires-Dist: tqdm
+Requires-Dist: click
 Requires-Dist: dataclasses_json
-Requires-Dist: pydantic>=2.7
+Requires-Dist: python-dateutil
 Requires-Dist: numpy
 Requires-Dist: pandas
 Provides-Extra: remote
@@ -103,8 +103,8 @@ Requires-Dist: astrapy; extra == "astradb"
 Requires-Dist: numpy; extra == "astradb"
 Requires-Dist: pandas; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: numpy; extra == "azure"
 Requires-Dist: pandas; extra == "azure"
 Provides-Extra: azure-ai-search
@@ -139,8 +139,8 @@ Requires-Dist: couchbase; extra == "couchbase"
 Requires-Dist: numpy; extra == "couchbase"
 Requires-Dist: pandas; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: numpy; extra == "delta-table"
 Requires-Dist: pandas; extra == "delta-table"
 Provides-Extra: discord
@@ -148,8 +148,8 @@ Requires-Dist: discord.py; extra == "discord"
 Requires-Dist: numpy; extra == "discord"
 Requires-Dist: pandas; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: numpy; extra == "dropbox"
 Requires-Dist: pandas; extra == "dropbox"
 Provides-Extra: duckdb
@@ -161,9 +161,9 @@ Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Requires-Dist: numpy; extra == "elasticsearch"
 Requires-Dist: pandas; extra == "elasticsearch"
 Provides-Extra: gcs
+Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: numpy; extra == "gcs"
 Requires-Dist: pandas; extra == "gcs"
 Provides-Extra: github
@@ -185,9 +185,9 @@ Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: numpy; extra == "hubspot"
 Requires-Dist: pandas; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
-Requires-Dist: httpx; extra == "ibm-watsonx-s3"
-Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
 Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
+Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
+Requires-Dist: httpx; extra == "ibm-watsonx-s3"
 Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: numpy; extra == "ibm-watsonx-s3"
 Requires-Dist: pandas; extra == "ibm-watsonx-s3"
@@ -216,15 +216,15 @@ Requires-Dist: pymongo; extra == "mongodb"
 Requires-Dist: numpy; extra == "mongodb"
 Requires-Dist: pandas; extra == "mongodb"
 Provides-Extra: neo4j
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: numpy; extra == "neo4j"
 Requires-Dist: pandas; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: httpx; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: numpy; extra == "notion"
 Requires-Dist: pandas; extra == "notion"
@@ -264,8 +264,8 @@ Requires-Dist: redis; extra == "redis"
 Requires-Dist: numpy; extra == "redis"
 Requires-Dist: pandas; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: numpy; extra == "s3"
 Requires-Dist: pandas; extra == "s3"
 Provides-Extra: sharepoint
@@ -278,8 +278,8 @@ Requires-Dist: simple-salesforce; extra == "salesforce"
 Requires-Dist: numpy; extra == "salesforce"
 Requires-Dist: pandas; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: numpy; extra == "sftp"
 Requires-Dist: pandas; extra == "sftp"
 Provides-Extra: slack
@@ -287,8 +287,8 @@ Requires-Dist: slack_sdk[optional]; extra == "slack"
 Requires-Dist: numpy; extra == "slack"
 Requires-Dist: pandas; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: numpy; extra == "snowflake"
 Requires-Dist: pandas; extra == "snowflake"
 Provides-Extra: wikipedia
@@ -313,8 +313,8 @@ Requires-Dist: numpy; extra == "singlestore"
 Requires-Dist: pandas; extra == "singlestore"
 Provides-Extra: vectara
 Requires-Dist: requests; extra == "vectara"
-Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: numpy; extra == "vectara"
 Requires-Dist: pandas; extra == "vectara"
 Provides-Extra: vastdb
@@ -324,9 +324,9 @@ Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: numpy; extra == "vastdb"
 Requires-Dist: pandas; extra == "vastdb"
 Provides-Extra: zendesk
+Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: aiofiles; extra == "zendesk"
-Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: numpy; extra == "zendesk"
 Requires-Dist: pandas; extra == "zendesk"
 Provides-Extra: embed-huggingface
@@ -356,8 +356,8 @@ Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: numpy; extra == "openai"
 Requires-Dist: pandas; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: numpy; extra == "bedrock"
 Requires-Dist: pandas; extra == "bedrock"
 Provides-Extra: togetherai

{unstructured_ingest-0.6.1.dist-info → unstructured_ingest-0.6.4.dist-info}/RECORD

@@ -11,6 +11,7 @@ test/integration/connectors/test_chroma.py,sha256=yn2p8U8yE9LaF-IEKiLp2XB4T4Vqo-
 test/integration/connectors/test_confluence.py,sha256=W93znOusdvFXta8q0dqQ1rKhLafRVIqrfaFqk2FY-fo,3590
 test/integration/connectors/test_delta_table.py,sha256=r2OvLMRfJFfsyIHd1H44Kx6AgNnDjPHKofN7a01oqrY,6916
 test/integration/connectors/test_dropbox.py,sha256=jzpZ6wawLa4sC1BVoHWZJ3cHjL4DWWUEX5ee7bXUOOM,4945
+test/integration/connectors/test_github.py,sha256=cRArMUMtAujsVtR2yp2GHZktX-cSPRSsYhlnnsRhhH8,1503
 test/integration/connectors/test_google_drive.py,sha256=ubjn3wvMhgpGHQs-wT_5icGgTIx2coS6hwNkAHOCEI8,10306
 test/integration/connectors/test_jira.py,sha256=IgF_cdknJ97W3OrNfHKp3YLmRyVwY6BI8jhZwVbrebc,2076
 test/integration/connectors/test_lancedb.py,sha256=AuO2qHDay5NHxQ2b1LzbsNNp6LnUDeB2_0dWlkgAuMo,9223
@@ -91,8 +92,6 @@ test/unit/v2/chunkers/test_chunkers.py,sha256=HSr3_lsoMw1nkDhkjO0-NOTEomRdR9oxCr
 test/unit/v2/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/connectors/test_confluence.py,sha256=lN6nnU5qOtmsjIGcz65roepm76w4vPF7AmSzi9vqV78,1919
 test/unit/v2/connectors/test_jira.py,sha256=XEBBDSdNZWUVO5JbpiSsjazJYmbLsgXUOW-APqPRKLg,12113
-test/unit/v2/connectors/databricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/unit/v2/connectors/databricks/test_volumes_table.py,sha256=-R_EJHqv1BseGRK9VRAZhF-2EXA64LAlhycoyIu556U,1078
 test/unit/v2/connectors/ibm_watsonx/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/connectors/ibm_watsonx/test_ibm_watsonx_s3.py,sha256=WKpDKvEGalh8LYRqN9xA7CfMPOPHo_VcZbnCXdkVjho,14513
 test/unit/v2/connectors/motherduck/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -113,7 +112,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=vYkj5wI9darc7y1Fll8uAtxzlI0lqsa5gGerwBBkeIQ,42
+unstructured_ingest/__version__.py,sha256=0sOJ1f0sRyjdtSL0LUqCE6m6T039ttMVXeIDsmdxcPw,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=7DOnDpGvUNlCoFR7UPRGmOarqH5sFtuUOO5vf8X3oTM,31489
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -427,7 +426,7 @@ unstructured_ingest/v2/processes/embedder.py,sha256=gvlCQDsbQVgcp-2f0Qq4RiFbcr8g
 unstructured_ingest/v2/processes/filter.py,sha256=E1MLxk-XeCm3mZIuM49lJToVcSgOivmTFIZApqOEFs8,2150
 unstructured_ingest/v2/processes/partitioner.py,sha256=HxopDSbovLh_1epeGeVtuWEX7v5KG35BowwKIJ_y4e8,9910
 unstructured_ingest/v2/processes/uncompress.py,sha256=O7q0pOiL6cDBvAa9NAy_vkc873tisjK2Hoq6Z-grRRo,2430
-unstructured_ingest/v2/processes/connectors/__init__.py,sha256=l4Xq4AuzRMTqUv5TU7cE1NbhGCka4SFJFZwG1FoVotE,6666
+unstructured_ingest/v2/processes/connectors/__init__.py,sha256=iyoZrTaEoBPBn9-tczFhLcwKQQoeFbESF3QkJkVimv4,6845
 unstructured_ingest/v2/processes/connectors/airtable.py,sha256=JesWeUv_tIA7b65sE2Z-ixMKuGLlgugZRMKE38ID3zg,8959
 unstructured_ingest/v2/processes/connectors/astradb.py,sha256=SwfUcdrCbMK_LDcHG5auCCuga_luPmZVvhjuAsRtKU0,18304
 unstructured_ingest/v2/processes/connectors/azure_ai_search.py,sha256=K4g-Dh7u7Z13rheNKhnnBXcO6TUfz8RPrrusDAsgkBk,11575
@@ -436,6 +435,7 @@ unstructured_ingest/v2/processes/connectors/confluence.py,sha256=ERHYYutQsnS8eZN
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=LARX4F_1Zd1LnUMSNdvIsbqLoZXZ9kl_vMZh-dRr4XA,12305
 unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=5T4hkXHGitGprpUb20206ODcBh5wchgHkUocji5l2rk,7286
 unstructured_ingest/v2/processes/connectors/discord.py,sha256=yvLxTx0ZRIATqSYLm5d09u9a0ktZGJnGcnzapwePHK8,5301
+unstructured_ingest/v2/processes/connectors/github.py,sha256=d-sh28MVe40vyaTf8b8NkspSrV9zlUHjdSSfSZjOcOA,7772
 unstructured_ingest/v2/processes/connectors/gitlab.py,sha256=hsB5g-3tZqe7bVSqwA0nY9GY4J0cg4PBawf77GkpjZY,10039
 unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=mn1BoUXYw5j-q7jO-rzPPEv_rt1UV9LDN1M_PyREEps,19678
 unstructured_ingest/v2/processes/connectors/jira.py,sha256=_ivv0TqeVPlHG9YMihljzrpYP7OLQAa0D7nqLonaeM8,17149
@@ -454,14 +454,15 @@ unstructured_ingest/v2/processes/connectors/slack.py,sha256=vbBVCYEd741-n2v6eAXL
 unstructured_ingest/v2/processes/connectors/utils.py,sha256=TAd0hb1f291N-q7-TUe6JKSCGkhqDyo7Ij8zmliBZUc,2071
 unstructured_ingest/v2/processes/connectors/vectara.py,sha256=KUqgZ6D2KUOrW596ms-EekvQYDh-fXqBTa7KG-leXoo,12301
 unstructured_ingest/v2/processes/connectors/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unstructured_ingest/v2/processes/connectors/assets/databricks_delta_table_schema.sql,sha256=dUZZDNkyvQXKqoAThRz3ek7zaUE2l_LAQimlG5WZhH4,211
 unstructured_ingest/v2/processes/connectors/assets/weaviate_collection_config.json,sha256=SJlIO0kXxy866tWQ8bEzvwLwflsoUMIS-OKlxMvHIuE,504
 unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=Oh8SwTWi66gO8BsNF6vRMoQVuegyBPPCpVozkOHEf3A,2136
-unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=8fg11-32If4iQGZTT9MEl1DOWZ5s3Qgj1OOzMVaHldU,7749
+unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=JktJXC9SYnKLetjsyGJWKXqg5Kml8WY9dcKyr5o_Yxs,8024
 unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=h6qDxQhWlT7H4K1CEfKag1stTiD1o97VckJZERsofqU,2970
 unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=gjICJJwhDHBLt_L-LrMlvJ3DL1DYtwFpyMLb_zYvOIg,3755
 unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=Uss3XPPaq1AsqJOEy4RJgBJw2-bTjrXH2PgtVNYd2w0,3006
 unstructured_ingest/v2/processes/connectors/databricks/volumes_native.py,sha256=g1qYnIrML4TjN7rmC0MGrD5JzAprb6SymBHlEdOumz0,3113
-unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py,sha256=5BArD1FkLC6wRJC0LxjXxQmYfmtF7r9Zrd8CtaGgWls,6855
+unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py,sha256=LiSb66039idaRtMnTuHjR5ZqvdmmIu3ByUgFQ1a3iZQ,8264
 unstructured_ingest/v2/processes/connectors/duckdb/__init__.py,sha256=5sVvJCWhU-YkjHIwk4W6BZCanFYK5W4xTpWtQ8xzeB4,561
 unstructured_ingest/v2/processes/connectors/duckdb/base.py,sha256=VCoQ3h289BO4A2kJKZXUVB0QOcaQif-HeRgg-xXzn10,2976
 unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py,sha256=DM4pygQAnP-dtuFEFAVeBfGt0pzrfkltteCai0GKnG0,4439
@@ -563,7 +564,7 @@ unstructured_ingest/v2/processes/connectors/qdrant/local.py,sha256=cGEyv3Oy6y4BQ
 unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=hsOd2Gliyjzkb21Vv6RAiFf8NAysxd29K0AxBkkm844,5483
 unstructured_ingest/v2/processes/connectors/qdrant/server.py,sha256=odvCZWZp8DmRxLXMR7tHhW-c7UQbix1_zpFdfXfCvKI,1613
 unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=NSEZwJDHh_9kFc31LnG14iRtYF3meK2UfUlQfYnwYEQ,2059
-unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=JrQMlSavPgSD1ruy4xRHaQV5iGiG7oavtm7az0tVVZc,9054
+unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=hJ0yacutrgiCer9cJSfxcNgLlOgsozJ2yGhgy8vZAkk,9086
 unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=BATfX1PQGT2kl8jAbdNKXTojYKJxh3pJV9-h3OBnHGo,5124
 unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=am2d87kDkpTTB0VbPSX3ce9o6oM9KUQu5y9T_p1kgJw,5711
 unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=FOb08WCr0SdzylN88xDP51NdVD4ggDbjanJurwJUrgM,9374
@@ -582,9 +583,9 @@ unstructured_ingest/v2/processes/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JC
 unstructured_ingest/v2/processes/utils/blob_storage.py,sha256=_I3OMdpUElQdIwVs7W9ORU1kncNaZ_nr6lbxeKE8uaU,1014
 unstructured_ingest/v2/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/types/file_data.py,sha256=kowOhvYy0q_-khX3IuR111AfjkdQezEfxjzK6QDH7oA,3836
-unstructured_ingest-0.6.1.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.6.1.dist-info/METADATA,sha256=Babhsu1h1L0nvRFeImk9Jn-jPjnaW-jdz6mhB3jkmbI,14998
-unstructured_ingest-0.6.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-unstructured_ingest-0.6.1.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.6.1.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.6.1.dist-info/RECORD,,
+unstructured_ingest-0.6.4.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.6.4.dist-info/METADATA,sha256=vsGBmdOat7mYz2HZ-RuKLYaSzpjgOaGwwn_b1qxNC7g,14998
+unstructured_ingest-0.6.4.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+unstructured_ingest-0.6.4.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.6.4.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.6.4.dist-info/RECORD,,

test/unit/v2/connectors/databricks/test_volumes_table.py

@@ -1,44 +0,0 @@
-from pathlib import Path
-
-import pytest
-from pytest_mock import MockerFixture
-
-from unstructured_ingest.v2.processes.connectors.databricks.volumes_table import (
-    DatabricksVolumeDeltaTableStager,
-)
-
-
-@pytest.fixture
-def stager():
-    return DatabricksVolumeDeltaTableStager()
-
-
-@pytest.mark.parametrize(
-    ("output_path", "called_output_path"),
-    [
-        (
-            Path("/fake/path/output"),
-            Path("/fake/path/output.json"),
-        ),
-        (
-            Path("/fake/path/output.ndjson"),
-            Path("/fake/path/output.json"),
-        ),
-    ],
-)
-def test_write_output(
-    mocker: MockerFixture,
-    stager: DatabricksVolumeDeltaTableStager,
-    output_path: Path,
-    called_output_path: Path,
-):
-    data = [{"key1": "value1", "key2": "value2"}]
-
-    mock_get_data = mocker.patch(
-        "unstructured_ingest.v2.processes.connectors.databricks.volumes_table.write_data",
-        return_value=None,
-    )
-
-    stager.write_output(output_path, data)
-
-    mock_get_data.assert_called_once_with(path=called_output_path, data=data, indent=None)