unstructured-ingest 0.0.17__py3-none-any.whl → 0.0.19__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.0.17"  # pragma: no cover
+__version__ = "0.0.19"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/chroma.py

@@ -41,13 +41,10 @@ class ChromaAccessConfig(AccessConfig):
     )
 
 
-SecretChromaAccessConfig = Secret[ChromaAccessConfig]
-
-
 class ChromaConnectionConfig(ConnectionConfig):
     collection_name: str = Field(description="The name of the Chroma collection to write into.")
-    access_config: SecretChromaAccessConfig = Field(
-        default=SecretChromaAccessConfig(secret_value=ChromaAccessConfig())
+    access_config: Secret[ChromaAccessConfig] = Field(
+        default=ChromaAccessConfig(), validate_default=True
     )
     path: Optional[str] = Field(
         default=None, description="Location where Chroma is persisted, if not connecting via http."

unstructured_ingest/v2/processes/connectors/databricks_volumes.py

@@ -81,14 +81,9 @@ class DatabricksVolumesAccessConfig(AccessConfig):
     google_service_account: Optional[str] = None
 
 
-SecretDatabricksVolumesAccessConfig = Secret[DatabricksVolumesAccessConfig]
-
-
 class DatabricksVolumesConnectionConfig(ConnectionConfig):
-    access_config: SecretDatabricksVolumesAccessConfig = Field(
-        default_factory=lambda: SecretDatabricksVolumesAccessConfig(
-            secret_value=DatabricksVolumesAccessConfig()
-        )
+    access_config: Secret[DatabricksVolumesAccessConfig] = Field(
+        default=DatabricksVolumesAccessConfig(), validate_default=True
     )
     host: Optional[str] = Field(
         default=None,

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -76,14 +76,9 @@ class AzureAccessConfig(FsspecAccessConfig):
             raise ValueError("either connection_string or account_name must be set")
 
 
-SecretAzureAccessConfig = Secret[AzureAccessConfig]
-
-
 class AzureConnectionConfig(FsspecConnectionConfig):
     supported_protocols: list[str] = field(default_factory=lambda: ["az"], init=False)
-    access_config: SecretAzureAccessConfig = Field(
-        default_factory=lambda: SecretAzureAccessConfig(secret_value=AzureAccessConfig())
-    )
+    access_config: Secret[AzureAccessConfig]
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -36,14 +36,9 @@ class BoxAccessConfig(FsspecAccessConfig):
     )
 
 
-SecretBoxAccessConfig = Secret[BoxAccessConfig]
-
-
 class BoxConnectionConfig(FsspecConnectionConfig):
     supported_protocols: list[str] = field(default_factory=lambda: ["box"], init=False)
-    access_config: SecretBoxAccessConfig = Field(
-        default_factory=lambda: SecretBoxAccessConfig(secret_value=BoxAccessConfig())
-    )
+    access_config: Secret[BoxAccessConfig] = Field(default=BoxAccessConfig(), validate_default=True)
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -35,13 +35,10 @@ class DropboxAccessConfig(FsspecAccessConfig):
     token: Optional[str] = Field(default=None, description="Dropbox access token.")
 
 
-SecretDropboxAccessConfig = Secret[DropboxAccessConfig]
-
-
 class DropboxConnectionConfig(FsspecConnectionConfig):
     supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"], init=False)
-    access_config: SecretDropboxAccessConfig = Field(
-        default_factory=lambda: SecretDropboxAccessConfig(secret_value=DropboxAccessConfig())
+    access_config: Secret[DropboxAccessConfig] = Field(
+        default=DropboxAccessConfig(), validate_default=True
     )
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -102,7 +102,13 @@ class FsspecIndexer(Indexer):
             fs = get_filesystem_class(self.index_config.protocol)(
                 **self.connection_config.get_access_config(),
             )
-            fs.ls(path=self.index_config.path_without_protocol, detail=False)
+            files = fs.ls(path=self.index_config.path_without_protocol, detail=True)
+            valid_files = [x.get("name") for x in files if x.get("type") == "file"]
+            if not valid_files:
+                return
+            file_to_sample = valid_files[0]
+            logger.debug(f"attempting to make HEAD request for file: {file_to_sample}")
+            self.fs.head(path=file_to_sample)
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
@@ -299,8 +305,8 @@ class FsspecUploader(Uploader):
             fs = get_filesystem_class(self.upload_config.protocol)(
                 **self.connection_config.get_access_config(),
             )
-            root_dir = self.upload_config.path_without_protocol.split("/")[0]
-            fs.ls(path=root_dir, detail=False)
+            upload_path = Path(self.upload_config.path_without_protocol) / "_empty"
+            fs.write_bytes(path=str(upload_path), value=b"")
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -86,14 +86,9 @@ class GcsAccessConfig(FsspecAccessConfig):
         raise ValueError("Invalid auth token value")
 
 
-SecretGcsAccessConfig = Secret[GcsAccessConfig]
-
-
 class GcsConnectionConfig(FsspecConnectionConfig):
     supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"], init=False)
-    access_config: SecretGcsAccessConfig = Field(
-        default_factory=lambda: SecretGcsAccessConfig(secret_value=GcsAccessConfig())
-    )
+    access_config: Secret[GcsAccessConfig] = Field(default=GcsAccessConfig(), validate_default=True)
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -49,14 +49,9 @@ class S3AccessConfig(FsspecAccessConfig):
     )
 
 
-SecretS3AccessConfig = Secret[S3AccessConfig]
-
-
 class S3ConnectionConfig(FsspecConnectionConfig):
     supported_protocols: list[str] = field(default_factory=lambda: ["s3", "s3a"], init=False)
-    access_config: SecretS3AccessConfig = Field(
-        default_factory=lambda: SecretS3AccessConfig(secret_value=S3AccessConfig())
-    )
+    access_config: Secret[S3AccessConfig] = Field(default=S3AccessConfig(), validate_default=True)
     endpoint_url: Optional[str] = Field(
         default=None,
         description="Use this endpoint_url, if specified. Needed for "

unstructured_ingest/v2/processes/connectors/kdbai.py

@@ -39,12 +39,9 @@ class KdbaiAccessConfig(AccessConfig):
     )
 
 
-SecretKdbaiAccessConfig = Secret[KdbaiAccessConfig]
-
-
 class KdbaiConnectionConfig(ConnectionConfig):
-    access_config: SecretKdbaiAccessConfig = Field(
-        default=SecretKdbaiAccessConfig(secret_value=KdbaiAccessConfig())
+    access_config: Secret[KdbaiAccessConfig] = Field(
+        default=KdbaiAccessConfig(), validate_default=True
     )
     endpoint: str = Field(
         default="http://localhost:8082", description="Endpoint url where KDBAI is hosted."

unstructured_ingest/v2/processes/connectors/local.py

@@ -34,12 +34,9 @@ class LocalAccessConfig(AccessConfig):
     pass
 
 
-SecretLocalAccessConfig = Secret[LocalAccessConfig]
-
-
 class LocalConnectionConfig(ConnectionConfig):
-    access_config: SecretLocalAccessConfig = Field(
-        default_factory=lambda: SecretLocalAccessConfig(secret_value=LocalAccessConfig())
+    access_config: Secret[LocalAccessConfig] = Field(
+        default=LocalAccessConfig(), validate_default=True
     )
 
 

unstructured_ingest/v2/processes/connectors/milvus.py

@@ -36,12 +36,9 @@ class MilvusAccessConfig(AccessConfig):
     token: Optional[str] = Field(default=None, description="Milvus access token")
 
 
-SecretMilvusAccessConfig = Secret[MilvusAccessConfig]
-
-
 class MilvusConnectionConfig(ConnectionConfig):
-    access_config: SecretMilvusAccessConfig = Field(
-        default_factory=lambda: SecretMilvusAccessConfig(secret_value=MilvusAccessConfig())
+    access_config: Secret[MilvusAccessConfig] = Field(
+        default=MilvusAccessConfig(), validate_default=True
     )
     uri: Optional[str] = Field(
         default=None, description="Milvus uri", examples=["http://localhost:19530"]

unstructured_ingest/v2/processes/connectors/mongodb.py

@@ -34,12 +34,9 @@ class MongoDBAccessConfig(AccessConfig):
     uri: Optional[str] = Field(default=None, description="URI to user when connecting")
 
 
-SecretMongoDBAccessConfig = Secret[MongoDBAccessConfig]
-
-
 class MongoDBConnectionConfig(ConnectionConfig):
-    access_config: SecretMongoDBAccessConfig = Field(
-        default_factory=lambda: SecretMongoDBAccessConfig(secret_value=MongoDBAccessConfig())
+    access_config: Secret[MongoDBAccessConfig] = Field(
+        default=MongoDBAccessConfig(), validate_default=True
     )
     host: Optional[str] = Field(
         default=None,

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -36,13 +36,10 @@ class PineconeAccessConfig(AccessConfig):
     )
 
 
-SecretPineconeAccessConfig = Secret[PineconeAccessConfig]
-
-
 class PineconeConnectionConfig(ConnectionConfig):
     index_name: str = Field(description="Name of the index to connect to.")
-    access_config: SecretPineconeAccessConfig = Field(
-        default_factory=lambda: SecretPineconeAccessConfig(secret_value=PineconeAccessConfig())
+    access_config: Secret[PineconeAccessConfig] = Field(
+        default=PineconeAccessConfig(), validate_default=True
     )
 
     @requires_dependencies(["pinecone"], extras="pinecone")

unstructured_ingest/v2/processes/connectors/sql.py

@@ -40,9 +40,6 @@ class SQLAccessConfig(AccessConfig):
     password: Optional[str] = Field(default=None, description="DB password")
 
 
-SecreteSQLAccessConfig = Secret[SQLAccessConfig]
-
-
 class SQLConnectionConfig(ConnectionConfig):
     db_type: Literal["sqlite", "postgresql"] = Field(
         default=SQLITE_DB, description="Type of the database backend"
@@ -53,9 +50,7 @@ class SQLConnectionConfig(ConnectionConfig):
     )
     host: Optional[str] = Field(default=None, description="DB host")
     port: Optional[int] = Field(default=5432, description="DB host connection port")
-    access_config: SecreteSQLAccessConfig = Field(
-        default_factory=lambda: SecreteSQLAccessConfig(secret_value=SQLAccessConfig())
-    )
+    access_config: Secret[SQLAccessConfig] = Field(default=SQLAccessConfig(), validate_default=True)
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def __post_init__(self):

unstructured_ingest/v2/processes/connectors/weaviate.py

@@ -38,16 +38,13 @@ class WeaviateAccessConfig(AccessConfig):
     password: Optional[str] = None
 
 
-SecretWeaviateAccessConfig = Secret[WeaviateAccessConfig]
-
-
 class WeaviateConnectionConfig(ConnectionConfig):
     host_url: str = Field(description="Weaviate instance url")
     class_name: str = Field(
         description="Name of the class to push the records into, e.g: Pdf-elements"
     )
-    access_config: SecretWeaviateAccessConfig = Field(
-        default_factory=lambda: SecretWeaviateAccessConfig(secret_value=WeaviateAccessConfig())
+    access_config: Secret[WeaviateAccessConfig] = Field(
+        default=WeaviateAccessConfig(), validate_default=True
     )
     username: Optional[str] = None
     anonymous: bool = Field(default=False, description="if set, all auth values will be ignored")