unstructured-ingest 0.6.1__tar.gz → 0.6.4__tar.gz

{unstructured_ingest-0.6.1/unstructured_ingest.egg-info → unstructured_ingest-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.6.1
+Version: 0.6.4
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,12 +22,12 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: tqdm
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: pydantic>=2.7
+Requires-Dist: dataclasses_json
 Requires-Dist: python-dateutil
+Requires-Dist: tqdm
 Requires-Dist: click
-Requires-Dist: dataclasses_json
-Requires-Dist: pydantic>=2.7
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: remote
@@ -103,8 +103,8 @@ Requires-Dist: astrapy; extra == "astradb"
 Requires-Dist: pandas; extra == "astradb"
 Requires-Dist: numpy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: pandas; extra == "azure"
 Requires-Dist: numpy; extra == "azure"
 Provides-Extra: azure-ai-search
@@ -112,8 +112,8 @@ Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Requires-Dist: pandas; extra == "azure-ai-search"
 Requires-Dist: numpy; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Requires-Dist: pandas; extra == "biomed"
 Requires-Dist: numpy; extra == "biomed"
 Provides-Extra: box
@@ -180,15 +180,15 @@ Requires-Dist: google-api-python-client; extra == "google-drive"
 Requires-Dist: pandas; extra == "google-drive"
 Requires-Dist: numpy; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: pandas; extra == "hubspot"
 Requires-Dist: numpy; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
-Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
 Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
-Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: httpx; extra == "ibm-watsonx-s3"
+Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
+Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
 Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Requires-Dist: numpy; extra == "ibm-watsonx-s3"
 Provides-Extra: jira
@@ -216,22 +216,22 @@ Requires-Dist: pymongo; extra == "mongodb"
 Requires-Dist: pandas; extra == "mongodb"
 Requires-Dist: numpy; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: networkx; extra == "neo4j"
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
+Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: pandas; extra == "neo4j"
 Requires-Dist: numpy; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: httpx; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Requires-Dist: pandas; extra == "notion"
 Requires-Dist: numpy; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: pandas; extra == "onedrive"
 Requires-Dist: numpy; extra == "onedrive"
 Provides-Extra: opensearch
@@ -239,8 +239,8 @@ Requires-Dist: opensearch-py; extra == "opensearch"
 Requires-Dist: pandas; extra == "opensearch"
 Requires-Dist: numpy; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Requires-Dist: pandas; extra == "outlook"
 Requires-Dist: numpy; extra == "outlook"
 Provides-Extra: pinecone
@@ -264,13 +264,13 @@ Requires-Dist: redis; extra == "redis"
 Requires-Dist: pandas; extra == "redis"
 Requires-Dist: numpy; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: pandas; extra == "s3"
 Requires-Dist: numpy; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: pandas; extra == "sharepoint"
 Requires-Dist: numpy; extra == "sharepoint"
 Provides-Extra: salesforce
@@ -287,8 +287,8 @@ Requires-Dist: slack_sdk[optional]; extra == "slack"
 Requires-Dist: pandas; extra == "slack"
 Requires-Dist: numpy; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: pandas; extra == "snowflake"
 Requires-Dist: numpy; extra == "snowflake"
 Provides-Extra: wikipedia
@@ -312,21 +312,21 @@ Requires-Dist: singlestoredb; extra == "singlestore"
 Requires-Dist: pandas; extra == "singlestore"
 Requires-Dist: numpy; extra == "singlestore"
 Provides-Extra: vectara
+Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
-Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: pandas; extra == "vectara"
 Requires-Dist: numpy; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
+Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: pandas; extra == "vastdb"
 Requires-Dist: numpy; extra == "vastdb"
 Provides-Extra: zendesk
-Requires-Dist: aiofiles; extra == "zendesk"
 Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: httpx; extra == "zendesk"
+Requires-Dist: aiofiles; extra == "zendesk"
 Requires-Dist: pandas; extra == "zendesk"
 Requires-Dist: numpy; extra == "zendesk"
 Provides-Extra: embed-huggingface
@@ -334,8 +334,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Requires-Dist: pandas; extra == "embed-huggingface"
 Requires-Dist: numpy; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: pandas; extra == "embed-octoai"
 Requires-Dist: numpy; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
@@ -351,13 +351,13 @@ Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Requires-Dist: numpy; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Requires-Dist: pandas; extra == "openai"
 Requires-Dist: numpy; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: pandas; extra == "bedrock"
 Requires-Dist: numpy; extra == "bedrock"
 Provides-Extra: togetherai

unstructured_ingest-0.6.4/test/integration/connectors/test_github.py

@@ -0,0 +1,49 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.github import (
+    CONNECTOR_TYPE,
+    GithubAccessConfig,
+    GithubConnectionConfig,
+    GithubDownloader,
+    GithubDownloaderConfig,
+    GithubIndexer,
+    GithubIndexerConfig,
+)
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
+@pytest.mark.asyncio
+@requires_env("GH_READ_ONLY_ACCESS_TOKEN")
+async def test_github_source(temp_dir):
+    access_token = os.environ["GH_READ_ONLY_ACCESS_TOKEN"]
+    connection_config = GithubConnectionConfig(
+        access_config=GithubAccessConfig(access_token=access_token),
+        url="dcneiner/Downloadify",
+    )
+
+    indexer = GithubIndexer(
+        connection_config=connection_config,
+        index_config=GithubIndexerConfig(file_glob=["*.txt", "*.html"]),
+    )
+
+    downloader = GithubDownloader(
+        connection_config=connection_config,
+        download_config=GithubDownloaderConfig(download_dir=temp_dir),
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="github", expected_num_files=2, validate_downloaded_files=True
+        ),
+    )

unstructured_ingest-0.6.4/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.6.4"  # pragma: no cover

{unstructured_ingest-0.6.1 → unstructured_ingest-0.6.4}/unstructured_ingest/v2/processes/connectors/__init__.py

@@ -31,6 +31,8 @@ from .delta_table import CONNECTOR_TYPE as DELTA_TABLE_CONNECTOR_TYPE
 from .delta_table import delta_table_destination_entry
 from .discord import CONNECTOR_TYPE as DISCORD_CONNECTOR_TYPE
 from .discord import discord_source_entry
+from .github import CONNECTOR_TYPE as GITHUB_CONNECTOR_TYPE
+from .github import github_source_entry
 from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -124,3 +126,4 @@ add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destina
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
 
 add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)
+add_source_entry(source_type=GITHUB_CONNECTOR_TYPE, entry=github_source_entry)

unstructured_ingest-0.6.4/unstructured_ingest/v2/processes/connectors/assets/databricks_delta_table_schema.sql

@@ -0,0 +1,10 @@
+CREATE TABLE elements (
+    id STRING NOT NULL PRIMARY KEY,
+    record_id STRING NOT NULL,
+    element_id STRING NOT NULL,
+    text STRING,
+    embeddings ARRAY<FLOAT>,
+    type STRING,
+    metadata VARIANT
+);
+

{unstructured_ingest-0.6.1 → unstructured_ingest-0.6.4}/unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 from uuid import NAMESPACE_DNS, uuid5
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.errors import (
@@ -61,6 +61,7 @@ class DatabricksVolumesAccessConfig(AccessConfig):
 
 
 class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
+    access_config: Secret[DatabricksVolumesAccessConfig]
     host: Optional[str] = Field(
         default=None,
         description="The Databricks host URL for either the "
@@ -94,12 +95,17 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
     @requires_dependencies(dependencies=["databricks.sdk"], extras="databricks-volumes")
     def get_client(self) -> "WorkspaceClient":
         from databricks.sdk import WorkspaceClient
+        from databricks.sdk.core import Config
 
-        return WorkspaceClient(
+        config = Config(
             host=self.host,
             **self.access_config.get_secret_value().model_dump(),
+        ).with_user_agent_extra(
+            "PyDatabricksSdk", os.getenv("UNSTRUCTURED_USER_AGENT", "unstructuredio_oss")
         )
 
+        return WorkspaceClient(config=config)
+
 
 class DatabricksVolumesIndexerConfig(IndexerConfig, DatabricksPathMixin):
     recursive: bool = False

unstructured_ingest-0.6.4/unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py

@@ -0,0 +1,188 @@
+import json
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field
+
+from unstructured_ingest.utils.data_prep import get_json_data, write_data
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
+from unstructured_ingest.v2.interfaces import (
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.databricks.volumes import DatabricksPathMixin
+from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
+    DatabricksDeltaTablesConnectionConfig,
+    DatabricksDeltaTablesUploadStagerConfig,
+)
+from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.v2.utils import get_enhanced_element_id
+
+CONNECTOR_TYPE = "databricks_volume_delta_tables"
+
+if TYPE_CHECKING:
+    pass
+
+
+class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMixin):
+    database: str = Field(description="Database name", default="default")
+    table_name: Optional[str] = Field(description="Table name", default=None)
+
+
+class DatabricksVolumeDeltaTableStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class DatabricksVolumeDeltaTableStager(UploadStager):
+    upload_stager_config: DatabricksVolumeDeltaTableStagerConfig = field(
+        default_factory=DatabricksVolumeDeltaTableStagerConfig
+    )
+
+    def run(
+        self,
+        elements_filepath: Path,
+        output_dir: Path,
+        output_filename: str,
+        file_data: FileData,
+        **kwargs: Any,
+    ) -> Path:
+        # To avoid new line issues when migrating from volumes into delta tables, omit indenting
+        # and always write it as a json file
+        output_dir.mkdir(exist_ok=True, parents=True)
+        output_path = output_dir / output_filename
+        final_output_path = output_path.with_suffix(".json")
+        data = get_json_data(path=elements_filepath)
+        for element in data:
+            element["id"] = get_enhanced_element_id(element_dict=element, file_data=file_data)
+            element[RECORD_ID_LABEL] = file_data.identifier
+            element["metadata"] = json.dumps(element.get("metadata", {}))
+        write_data(path=final_output_path, data=data, indent=None)
+        return final_output_path
+
+
+@dataclass
+class DatabricksVolumeDeltaTableUploader(Uploader):
+    connection_config: DatabricksDeltaTablesConnectionConfig
+    upload_config: DatabricksVolumeDeltaTableUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+    _columns: Optional[dict[str, str]] = None
+
+    def init(self, **kwargs: Any) -> None:
+        self.create_destination(**kwargs)
+
+    def create_destination(
+        self, destination_name: str = "unstructuredautocreated", **kwargs: Any
+    ) -> bool:
+        table_name = self.upload_config.table_name or destination_name
+        self.upload_config.table_name = table_name
+        connectors_dir = Path(__file__).parents[1]
+        collection_config_file = connectors_dir / "assets" / "databricks_delta_table_schema.sql"
+        with self.get_cursor() as cursor:
+            cursor.execute("SHOW TABLES")
+            table_names = [r[1] for r in cursor.fetchall()]
+            if table_name in table_names:
+                return False
+            with collection_config_file.open() as schema_file:
+                data_lines = schema_file.readlines()
+            data_lines[0] = data_lines[0].replace("elements", table_name)
+            destination_schema = "".join([line.strip() for line in data_lines])
+            logger.info(f"creating table {table_name} for user")
+            cursor.execute(destination_schema)
+            return True
+
+    def precheck(self) -> None:
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute("SHOW CATALOGS")
+            catalogs = [r[0] for r in cursor.fetchall()]
+            if self.upload_config.catalog not in catalogs:
+                raise ValueError(
+                    "Catalog {} not found in {}".format(
+                        self.upload_config.catalog, ", ".join(catalogs)
+                    )
+                )
+            cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            cursor.execute("SHOW DATABASES")
+            databases = [r[0] for r in cursor.fetchall()]
+            if self.upload_config.database not in databases:
+                raise ValueError(
+                    "Database {} not found in {}".format(
+                        self.upload_config.database, ", ".join(databases)
+                    )
+                )
+
+    def get_output_path(self, file_data: FileData, suffix: str = ".json") -> str:
+        filename = Path(file_data.source_identifiers.filename)
+        adjusted_filename = filename if filename.suffix == suffix else f"{filename}{suffix}"
+        return os.path.join(self.upload_config.path, f"{adjusted_filename}")
+
+    @contextmanager
+    def get_cursor(self, **connect_kwargs) -> Generator[Any, None, None]:
+        with self.connection_config.get_cursor(**connect_kwargs) as cursor:
+            logger.debug(f"executing: USE CATALOG: '{self.upload_config.catalog}'")
+            cursor.execute(f"USE CATALOG '{self.upload_config.catalog}'")
+            logger.debug(f"executing: USE DATABASE: {self.upload_config.database}")
+            cursor.execute(f"USE DATABASE {self.upload_config.database}")
+            yield cursor
+
+    def get_table_columns(self) -> dict[str, str]:
+        if self._columns is None:
+            with self.get_cursor() as cursor:
+                cursor.execute(f"SELECT * from {self.upload_config.table_name} LIMIT 1")
+                self._columns = {desc[0]: desc[1] for desc in cursor.description}
+        return self._columns
+
+    def can_delete(self) -> bool:
+        existing_columns = self.get_table_columns()
+        return RECORD_ID_LABEL in existing_columns
+
+    def delete_previous_content(self, file_data: FileData) -> None:
+        logger.debug(
+            f"deleting any content with metadata "
+            f"{RECORD_ID_LABEL}={file_data.identifier} "
+            f"from delta table: {self.upload_config.table_name}"
+        )
+        with self.get_cursor() as cursor:
+            cursor.execute(
+                f"DELETE FROM {self.upload_config.table_name} WHERE {RECORD_ID_LABEL} = '{file_data.identifier}'"  # noqa: E501
+            )
+            results = cursor.fetchall()
+            deleted_rows = results[0][0]
+            logger.debug(f"deleted {deleted_rows} rows from table {self.upload_config.table_name}")
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        if self.can_delete():
+            self.delete_previous_content(file_data=file_data)
+        with self.get_cursor(staging_allowed_local_path=path.parent.as_posix()) as cursor:
+            catalog_path = self.get_output_path(file_data=file_data)
+            logger.debug(f"uploading {path.as_posix()} to {catalog_path}")
+            cursor.execute(f"PUT '{path.as_posix()}' INTO '{catalog_path}' OVERWRITE")
+            logger.debug(
+                f"migrating content from {catalog_path} to "
+                f"table {self.upload_config.table_name}"
+            )
+            data = get_json_data(path=path)
+            columns = data[0].keys()
+            select_columns = ["PARSE_JSON(metadata)" if c == "metadata" else c for c in columns]
+            column_str = ", ".join(columns)
+            select_column_str = ", ".join(select_columns)
+            sql_statment = f"INSERT INTO `{self.upload_config.table_name}` ({column_str}) SELECT {select_column_str} FROM json.`{catalog_path}`"  # noqa: E501
+            cursor.execute(sql_statment)
+
+
+databricks_volumes_delta_tables_destination_entry = DestinationRegistryEntry(
+    connection_config=DatabricksDeltaTablesConnectionConfig,
+    uploader=DatabricksVolumeDeltaTableUploader,
+    uploader_config=DatabricksVolumeDeltaTableUploaderConfig,
+    upload_stager=DatabricksVolumeDeltaTableStager,
+    upload_stager_config=DatabricksDeltaTablesUploadStagerConfig,
+)