unstructured-ingest 0.0.2.dev0__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/sharepoint.py

@@ -6,10 +6,9 @@ from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional
 from urllib.parse import quote
 
-from unstructured.documents.elements import DataSourceMetadata
+from pydantic import BaseModel, Field, Secret, SecretStr
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
-from unstructured_ingest.error import SourceConnectionNetworkError
+from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -18,6 +17,7 @@ from unstructured_ingest.v2.interfaces import (
     DownloaderConfig,
     DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
@@ -55,24 +55,38 @@ class SharepointContentType(Enum):
     LIST = "list"
 
 
-@dataclass
 class SharepointAccessConfig(AccessConfig):
-    client_cred: str
-
-
-@dataclass
-class SharepointPermissionsConfig(EnhancedDataClassJsonMixin):
-    permissions_application_id: str
-    permissions_tenant: str
-    permissions_client_cred: str = enhanced_field(sensitive=True)
-    authority_url: Optional[str] = field(repr=False, default="https://login.microsoftonline.com")
+    client_cred: str = Field(description="Sharepoint app secret")
+
+
+class SharepointPermissionsConfig(BaseModel):
+    permissions_application_id: str = Field(description="Microsoft Graph API application id")
+    permissions_tenant: str = Field(
+        description="url to get permissions data within tenant.",
+        examples=["https://contoso.onmicrosoft.com"],
+    )
+    permissions_client_cred: SecretStr = Field(
+        description="Microsoft Graph API application credentials"
+    )
+    authority_url: Optional[SecretStr] = Field(
+        repr=False,
+        default_factory=lambda: SecretStr(secret_value="https://login.microsoftonline.com"),
+        description="Permissions authority url",
+        examples=["https://login.microsoftonline.com"],
+    )
 
 
-@dataclass
 class SharepointConnectionConfig(ConnectionConfig):
-    client_id: str
-    site: str
-    access_config: SharepointAccessConfig = enhanced_field(sensitive=True)
+    client_id: str = Field(description="Sharepoint app client ID")
+    site: str = Field(
+        description="Sharepoint site url. Process either base url e.g \
+                    https://[tenant].sharepoint.com  or relative sites \
+                    https://[tenant].sharepoint.com/sites/<site_name>. \
+                    To process all sites within the tenant pass a site url as \
+                    https://[tenant]-admin.sharepoint.com.\
+                    This requires the app to be registered at a tenant level"
+    )
+    access_config: Secret[SharepointAccessConfig]
     permissions_config: Optional[SharepointPermissionsConfig] = None
 
     @requires_dependencies(["office365"], extras="sharepoint")
@@ -81,7 +95,9 @@ class SharepointConnectionConfig(ConnectionConfig):
         from office365.sharepoint.client_context import ClientContext
 
         try:
-            credentials = ClientCredential(self.client_id, self.access_config.client_cred)
+            credentials = ClientCredential(
+                self.client_id, self.access_config.get_secret_value().client_cred
+            )
             site_client = ClientContext(self.site).with_credentials(credentials)
         except Exception as e:
             logger.error(f"Couldn't set Sharepoint client: {e}")
@@ -93,11 +109,12 @@ class SharepointConnectionConfig(ConnectionConfig):
         from msal import ConfidentialClientApplication
 
         try:
+            client_credential = self.permissions_config.permissions_client_cred.get_secret_value()
             app = ConfidentialClientApplication(
-                authority=f"{self.permissions_config.authority_url}/"
+                authority=f"{self.permissions_config.authority_url.get_secret_value()}/"
                 f"{self.permissions_config.permissions_tenant}",
                 client_id=self.permissions_config.permissions_application_id,
-                client_credential=self.permissions_config.permissions_client_cred,
+                client_credential=client_credential,
             )
             token = app.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"])
         except ValueError as exc:
@@ -120,13 +137,21 @@ class SharepointConnectionConfig(ConnectionConfig):
         return client
 
 
-@dataclass
 class SharepointIndexerConfig(IndexerConfig):
-    path: Optional[str] = None
-    recursive: bool = False
-    omit_files: bool = False
-    omit_pages: bool = False
-    omit_lists: bool = False
+    path: Optional[str] = Field(
+        defaul=None,
+        description="Path from which to start parsing files. If the connector is to \
+                process all sites within the tenant this filter will be applied to \
+                all sites document libraries.",
+    )
+    recursive: bool = Field(
+        default=False,
+        description="Recursively download files in their respective folders "
+        "otherwise stop at the files in provided folder level.",
+    )
+    omit_files: bool = Field(default=False, description="Don't process files.")
+    omit_pages: bool = Field(default=False, description="Don't process site pages.")
+    omit_lists: bool = Field(default=False, description="Don't process lists.")
 
 
 @dataclass
@@ -134,6 +159,14 @@ class SharepointIndexer(Indexer):
     connection_config: SharepointConnectionConfig
     index_config: SharepointIndexerConfig = field(default_factory=lambda: SharepointIndexerConfig())
 
+    def precheck(self) -> None:
+        try:
+            site_client = self.connection_config.get_client()
+            site_client.site_pages.pages.get().execute_query()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
     def list_files(self, folder: "Folder", recursive: bool = False) -> list["File"]:
         if not recursive:
             folder.expand(["Files"]).get().execute_query()
@@ -187,7 +220,7 @@ class SharepointIndexer(Indexer):
                 fullpath=file_path,
                 rel_path=file_path.replace(self.index_config.path, ""),
             ),
-            metadata=DataSourceMetadata(
+            metadata=FileDataSourceMetadata(
                 url=url,
                 version=version,
                 date_modified=str(date_modified_dt.timestamp()) if date_modified_dt else None,
@@ -222,7 +255,7 @@ class SharepointIndexer(Indexer):
                 fullpath=fullpath,
                 rel_path=rel_path,
             ),
-            metadata=DataSourceMetadata(
+            metadata=FileDataSourceMetadata(
                 url=absolute_url,
                 version=f"{file.major_version}.{file.minor_version}",
                 date_modified=str(date_modified_dt.timestamp()) if date_modified_dt else None,
@@ -303,7 +336,7 @@ class SharepointIndexer(Indexer):
     def process_permissions(self) -> bool:
         return (
             self.connection_config.permissions_config.permissions_tenant
-            and self.connection_config.permissions_config.permissions_client_cred
+            and self.connection_config.permissions_config.permissions_client_cred.get_secret_value()
             and self.connection_config.permissions_config.permissions_application_id
         )
 
@@ -328,7 +361,6 @@ class SharepointIndexer(Indexer):
                 yield file_data
 
 
-@dataclass
 class SharepointDownloaderConfig(DownloaderConfig):
     pass
 
@@ -340,10 +372,9 @@ class SharepointDownloader(Downloader):
     connector_type: str = CONNECTOR_TYPE
 
     def get_download_path(self, file_data: FileData) -> Path:
+        download_path = super().get_download_path(file_data=file_data)
+
         content_type = file_data.additional_metadata.get("sharepoint_content_type")
-        rel_path = file_data.source_identifiers.fullpath
-        rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
-        download_path = self.download_dir / Path(rel_path)
         if content_type == SharepointContentType.SITEPAGE.value:
             # Update output extension to html if site page
             download_path = download_path.with_suffix(".html")

unstructured_ingest/v2/processes/connectors/singlestore.py

@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Any, Optional
 import numpy as np
 import pandas as pd
 from dateutil import parser
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
 from unstructured_ingest.utils.data_prep import batch_generator
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.table import convert_to_pandas_dataframe
@@ -33,18 +33,16 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "singlestore"
 
 
-@dataclass
 class SingleStoreAccessConfig(AccessConfig):
-    password: Optional[str] = None
+    password: Optional[str] = Field(default=None, description="SingleStore password")
 
 
-@dataclass
 class SingleStoreConnectionConfig(ConnectionConfig):
-    host: Optional[str] = None
-    port: Optional[int] = None
-    user: Optional[str] = None
-    database: Optional[str] = None
-    access_config: SingleStoreAccessConfig = enhanced_field(sensitive=True)
+    host: Optional[str] = Field(default=None, description="SingleStore host")
+    port: Optional[int] = Field(default=None, description="SingleStore port")
+    user: Optional[str] = Field(default=None, description="SingleStore user")
+    database: Optional[str] = Field(default=None, description="SingleStore database")
+    access_config: Secret[SingleStoreAccessConfig]
 
     @requires_dependencies(["singlestoredb"], extras="singlestore")
     def get_connection(self) -> "Connection":
@@ -55,14 +53,13 @@ class SingleStoreConnectionConfig(ConnectionConfig):
             port=self.port,
             database=self.database,
             user=self.user,
-            password=self.access_config.password,
+            password=self.access_config.get_secret_value().password,
         )
         return conn
 
 
-@dataclass
 class SingleStoreUploadStagerConfig(UploadStagerConfig):
-    drop_empty_cols: bool = False
+    drop_empty_cols: bool = Field(default=False, description="Drop any columns that have no data")
 
 
 @dataclass
@@ -112,10 +109,9 @@ class SingleStoreUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
 class SingleStoreUploaderConfig(UploaderConfig):
-    table_name: str
-    batch_size: int = 100
+    table_name: str = Field(description="SingleStore table to write contents to")
+    batch_size: int = Field(default=100, description="Batch size when writing to SingleStore")
 
 
 @dataclass

unstructured_ingest/v2/processes/connectors/sql.py

@@ -1,16 +1,16 @@
-import enum
 import json
 import uuid
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Union
 
 import numpy as np
 import pandas as pd
 from dateutil import parser
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -25,42 +25,48 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 
+if TYPE_CHECKING:
+    from sqlite3 import Connection as SqliteConnection
+
+    from psycopg2.extensions import connection as PostgresConnection
+
 CONNECTOR_TYPE = "sql"
 ELEMENTS_TABLE_NAME = "elements"
+SQLITE_DB = "sqlite"
+POSTGRESQL_DB = "postgresql"
 
 
-@dataclass
 class SQLAccessConfig(AccessConfig):
-    username: Optional[str] = None
-    password: Optional[str] = None
+    username: Optional[str] = Field(default=None, description="DB username")
+    password: Optional[str] = Field(default=None, description="DB password")
 
 
-class DatabaseType(str, enum.Enum):
-    SQLITE = "sqlite"
-    POSTGRESQL = "postgresql"
+SecreteSQLAccessConfig = Secret[SQLAccessConfig]
 
 
-@dataclass
-class SimpleSqlConfig(ConnectionConfig):
-    db_type: DatabaseType = (
-        # required default value here because of parent class
-        DatabaseType.SQLITE
+class SQLConnectionConfig(ConnectionConfig):
+    db_type: Literal["sqlite", "postgresql"] = Field(
+        default=SQLITE_DB, description="Type of the database backend"
     )
-    database: Optional[str] = None
-    host: Optional[str] = None
-    port: Optional[int] = 5432
-    access_config: Optional[SQLAccessConfig] = enhanced_field(default=None, sensitive=True)
-    connector_type: str = CONNECTOR_TYPE
+    database: Optional[str] = Field(
+        default=None,
+        description="Database name. For sqlite databases, this is the path to the .db file.",
+    )
+    host: Optional[str] = Field(default=None, description="DB host")
+    port: Optional[int] = Field(default=5432, description="DB host connection port")
+    access_config: SecreteSQLAccessConfig = Field(
+        default_factory=lambda: SecreteSQLAccessConfig(secret_value=SQLAccessConfig())
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def __post_init__(self):
-        if (self.db_type == DatabaseType.SQLITE) and (self.database is None):
+        if (self.db_type == SQLITE_DB) and (self.database is None):
             raise ValueError(
                 "A sqlite connection requires a path to a *.db file "
                 "through the `database` argument"
             )
 
 
-@dataclass
 class SQLUploadStagerConfig(UploadStagerConfig):
     pass
 
@@ -134,7 +140,7 @@ class SQLUploadStager(UploadStager):
         **kwargs: Any,
     ) -> Path:
         with open(elements_filepath) as elements_file:
-            elements_contents = json.load(elements_file)
+            elements_contents: list[dict] = json.load(elements_file)
         output_path = Path(output_dir) / Path(f"{output_filename}.json")
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -151,7 +157,7 @@ class SQLUploadStager(UploadStager):
             data["id"] = str(uuid.uuid4())
 
             # remove extraneous, not supported columns
-            [data.pop(column) for column in data if column not in _COLUMNS]
+            data = {k: v for k, v in data.items() if k in _COLUMNS}
 
             output.append(data)
 
@@ -176,37 +182,46 @@ class SQLUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
 class SQLUploaderConfig(UploaderConfig):
-    batch_size: int = 50
+    batch_size: int = Field(default=50, description="Number of records per batch")
 
 
 @dataclass
 class SQLUploader(Uploader):
     connector_type: str = CONNECTOR_TYPE
     upload_config: SQLUploaderConfig
-    connection_config: SimpleSqlConfig
+    connection_config: SQLConnectionConfig
+
+    def precheck(self) -> None:
+        try:
+            cursor = self.connection().cursor()
+            cursor.execute("SELECT 1;")
+            cursor.close()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @property
-    def connection(self):
-        if self.connection_config.db_type == DatabaseType.POSTGRESQL:
+    def connection(self) -> Callable[[], Union["SqliteConnection", "PostgresConnection"]]:
+        if self.connection_config.db_type == POSTGRESQL_DB:
             return self._make_psycopg_connection
-        elif self.connection_config.db_type == DatabaseType.SQLITE:
+        elif self.connection_config.db_type == SQLITE_DB:
             return self._make_sqlite_connection
         raise ValueError(f"Unsupported database {self.connection_config.db_type} connection.")
 
-    def _make_sqlite_connection(self):
+    def _make_sqlite_connection(self) -> "SqliteConnection":
         from sqlite3 import connect
 
         return connect(database=self.connection_config.database)
 
     @requires_dependencies(["psycopg2"], extras="postgres")
-    def _make_psycopg_connection(self):
+    def _make_psycopg_connection(self) -> "PostgresConnection":
         from psycopg2 import connect
 
+        access_config = self.connection_config.access_config.get_secret_value()
         return connect(
-            user=self.connection_config.access_config.username,
-            password=self.connection_config.access_config.password,
+            user=access_config.username,
+            password=access_config.password,
             dbname=self.connection_config.database,
             host=self.connection_config.host,
             port=self.connection_config.port,
@@ -219,9 +234,7 @@ class SQLUploader(Uploader):
         for row in data:
             parsed = []
             for column_name, value in zip(columns, row):
-                if self.connection_config.db_type == DatabaseType.SQLITE and isinstance(
-                    value, (list, dict)
-                ):
+                if self.connection_config.db_type == SQLITE_DB and isinstance(value, (list, dict)):
                     value = json.dumps(value)
                 if column_name in _DATE_COLUMNS:
                     if value is None:
@@ -240,14 +253,14 @@ class SQLUploader(Uploader):
 
         columns = tuple(df.columns)
         stmt = f"INSERT INTO {ELEMENTS_TABLE_NAME} ({','.join(columns)}) \
-                VALUES({','.join(['?' if self.connection_config.db_type==DatabaseType.SQLITE else '%s' for x in columns])})"  # noqa E501
+                VALUES({','.join(['?' if self.connection_config.db_type==SQLITE_DB else '%s' for x in columns])})"  # noqa E501
 
         for rows in pd.read_json(
             content.path, orient="records", lines=True, chunksize=self.upload_config.batch_size
         ):
             with self.connection() as conn:
                 values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
-                if self.connection_config.db_type == DatabaseType.SQLITE:
+                if self.connection_config.db_type == SQLITE_DB:
                     conn.executemany(stmt, values)
                 else:
                     with conn.cursor() as cur:
@@ -261,7 +274,7 @@ class SQLUploader(Uploader):
 
 
 sql_destination_entry = DestinationRegistryEntry(
-    connection_config=SimpleSqlConfig,
+    connection_config=SQLConnectionConfig,
     uploader=SQLUploader,
     uploader_config=SQLUploaderConfig,
     upload_stager=SQLUploadStager,

unstructured_ingest/v2/processes/connectors/weaviate.py

@@ -5,8 +5,9 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
 from dateutil import parser
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -29,27 +30,37 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "weaviate"
 
 
-@dataclass
 class WeaviateAccessConfig(AccessConfig):
-    access_token: Optional[str] = None
+    access_token: Optional[str] = Field(
+        default=None, description="Used to create the bearer token."
+    )
     api_key: Optional[str] = None
     client_secret: Optional[str] = None
     password: Optional[str] = None
 
 
-@dataclass
+SecretWeaviateAccessConfig = Secret[WeaviateAccessConfig]
+
+
 class WeaviateConnectionConfig(ConnectionConfig):
-    host_url: str
-    class_name: str
-    access_config: WeaviateAccessConfig = enhanced_field(sensitive=True)
+    host_url: str = Field(description="Weaviate instance url")
+    class_name: str = Field(
+        description="Name of the class to push the records into, e.g: Pdf-elements"
+    )
+    access_config: SecretWeaviateAccessConfig = Field(
+        default_factory=lambda: SecretWeaviateAccessConfig(secret_value=WeaviateAccessConfig())
+    )
     username: Optional[str] = None
-    anonymous: bool = False
+    anonymous: bool = Field(default=False, description="if set, all auth values will be ignored")
     scope: Optional[list[str]] = None
-    refresh_token: Optional[str] = None
-    connector_type: str = CONNECTOR_TYPE
+    refresh_token: Optional[str] = Field(
+        default=None,
+        description="Will tie this value to the bearer token. If not provided, "
+        "the authentication will expire once the lifetime of the access token is up.",
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
-@dataclass
 class WeaviateUploadStagerConfig(UploadStagerConfig):
     pass
 
@@ -147,24 +158,29 @@ class WeaviateUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
 class WeaviateUploaderConfig(UploaderConfig):
-    batch_size: int = 100
+    batch_size: int = Field(default=100, description="Number of records per batch")
 
 
 @dataclass
 class WeaviateUploader(Uploader):
     upload_config: WeaviateUploaderConfig
     connection_config: WeaviateConnectionConfig
-    client: Optional["Client"] = field(init=False)
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["weaviate"], extras="weaviate")
-    def __post_init__(self):
+    def get_client(self) -> "Client":
         from weaviate import Client
 
         auth = self._resolve_auth_method()
-        self.client = Client(url=self.connection_config.host_url, auth_client_secret=auth)
+        return Client(url=self.connection_config.host_url, auth_client_secret=auth)
+
+    def precheck(self) -> None:
+        try:
+            self.get_client()
+        except Exception as e:
+            logger.error(f"Failed to validate connection {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["weaviate"], extras="weaviate")
     def _resolve_auth_method(self):
@@ -215,8 +231,9 @@ class WeaviateUploader(Uploader):
             f"at {self.connection_config.host_url}",
         )
 
-        self.client.batch.configure(batch_size=self.upload_config.batch_size)
-        with self.client.batch as b:
+        client = self.get_client()
+        client.batch.configure(batch_size=self.upload_config.batch_size)
+        with client.batch as b:
             for e in elements_dict:
                 vector = e.pop("embeddings", None)
                 b.add_data_object(