unstructured-ingest 0.5.20__py3-none-any.whl → 0.5.23__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/sql/snowflake.py

@@ -3,8 +3,6 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import numpy as np
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.data_prep import split_dataframe
@@ -32,6 +30,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
 )
 
 if TYPE_CHECKING:
+    from pandas import DataFrame
     from snowflake.connector import SnowflakeConnection
     from snowflake.connector.cursor import SnowflakeCursor
 
@@ -174,9 +173,12 @@ class SnowflakeUploader(SQLUploader):
     connector_type: str = CONNECTOR_TYPE
     values_delimiter: str = "?"
 
+    @requires_dependencies(["pandas"], extras="snowflake")
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
     ) -> list[tuple[Any, ...]]:
+        import pandas as pd
+
         output = []
         for row in data:
             parsed = []
@@ -210,7 +212,9 @@ class SnowflakeUploader(SQLUploader):
             ]
         )
 
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
+        import numpy as np
+
         if self.can_delete():
             self.delete_by_record_id(file_data=file_data)
         else:

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -6,10 +6,8 @@ from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
 from time import time
-from typing import Any, Generator, Union
+from typing import TYPE_CHECKING, Any, Generator, Union
 
-import numpy as np
-import pandas as pd
 from dateutil import parser
 from pydantic import BaseModel, Field, Secret
 
@@ -38,6 +36,9 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 _DATE_COLUMNS = ("date_created", "date_modified", "date_processed", "last_modified")
 
 
@@ -154,13 +155,15 @@ class SQLDownloader(Downloader, ABC):
     def query_db(self, file_data: SqlBatchFileData) -> tuple[list[tuple], list[str]]:
         pass
 
-    def sql_to_df(self, rows: list[tuple], columns: list[str]) -> list[pd.DataFrame]:
+    def sql_to_df(self, rows: list[tuple], columns: list[str]) -> list["DataFrame"]:
+        import pandas as pd
+
         data = [dict(zip(columns, row)) for row in rows]
         df = pd.DataFrame(data)
         dfs = [pd.DataFrame([row.values], columns=df.columns) for index, row in df.iterrows()]
         return dfs
 
-    def get_data(self, file_data: SqlBatchFileData) -> list[pd.DataFrame]:
+    def get_data(self, file_data: SqlBatchFileData) -> list["DataFrame"]:
         rows, columns = self.query_db(file_data=file_data)
         return self.sql_to_df(rows=rows, columns=columns)
 
@@ -174,7 +177,7 @@ class SQLDownloader(Downloader, ABC):
         return f
 
     def generate_download_response(
-        self, result: pd.DataFrame, file_data: SqlBatchFileData
+        self, result: "DataFrame", file_data: SqlBatchFileData
     ) -> DownloadResponse:
         id_column = file_data.additional_metadata.id_column
         table_name = file_data.additional_metadata.table_name
@@ -231,7 +234,7 @@ class SQLUploadStager(UploadStager):
         data[RECORD_ID_LABEL] = file_data.identifier
         return data
 
-    def conform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
+    def conform_dataframe(self, df: "DataFrame") -> "DataFrame":
         for column in filter(lambda x: x in df.columns, _DATE_COLUMNS):
             df[column] = df[column].apply(parse_date_string).apply(lambda date: date.timestamp())
         for column in filter(
@@ -259,6 +262,8 @@ class SQLUploadStager(UploadStager):
         output_filename: str,
         **kwargs: Any,
     ) -> Path:
+        import pandas as pd
+
         elements_contents = get_data(path=elements_filepath)
 
         df = pd.DataFrame(
@@ -309,6 +314,8 @@ class SQLUploader(Uploader):
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
     ) -> list[tuple[Any, ...]]:
+        import pandas as pd
+
         output = []
         for row in data:
             parsed = []
@@ -323,7 +330,9 @@ class SQLUploader(Uploader):
             output.append(tuple(parsed))
         return output
 
-    def _fit_to_schema(self, df: pd.DataFrame, add_missing_columns: bool = True) -> pd.DataFrame:
+    def _fit_to_schema(self, df: "DataFrame", add_missing_columns: bool = True) -> "DataFrame":
+        import pandas as pd
+
         table_columns = self.get_table_columns()
         columns = set(df.columns)
         schema_fields = set(table_columns)
@@ -348,7 +357,9 @@ class SQLUploader(Uploader):
                 df[column] = pd.Series()
         return df
 
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
+        import numpy as np
+
         if self.can_delete():
             self.delete_by_record_id(file_data=file_data)
         else:
@@ -409,6 +420,8 @@ class SQLUploader(Uploader):
                 logger.info(f"deleted {rowcount} rows from table {self.upload_config.table_name}")
 
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data)
         self.upload_dataframe(df=df, file_data=file_data)
 

unstructured_ingest/v2/processes/connectors/sql/sqlite.py

@@ -4,9 +4,9 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator
 
-import pandas as pd
 from pydantic import Field, Secret, model_validator
 
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
     from sqlite3 import Connection as SqliteConnection
     from sqlite3 import Cursor as SqliteCursor
 
+
 CONNECTOR_TYPE = "sqlite"
 
 
@@ -132,9 +133,12 @@ class SQLiteUploader(SQLUploader):
     connection_config: SQLiteConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @requires_dependencies(["pandas"])
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
     ) -> list[tuple[Any, ...]]:
+        import pandas as pd
+
         output = []
         for row in data:
             parsed = []

unstructured_ingest/v2/processes/connectors/sql/vastdb.py

@@ -2,8 +2,6 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Optional
 
-import numpy as np
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -34,6 +32,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:
+    from pandas import DataFrame
     from vastdb import connect as VastdbConnect
     from vastdb import transaction as VastdbTransaction
     from vastdb.table import Table as VastdbTable
@@ -128,7 +127,6 @@ class VastdbDownloader(SQLDownloader):
         ids = tuple([item.identifier for item in file_data.batch_items])
 
         with self.connection_config.get_table(table_name) as table:
-
             predicate = _[id_column].isin(ids)
 
             if self.download_config.fields:
@@ -168,7 +166,7 @@ class VastdbUploadStager(SQLUploadStager):
         data[RECORD_ID_LABEL] = file_data.identifier
         return data
 
-    def conform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
+    def conform_dataframe(self, df: "DataFrame") -> "DataFrame":
         df = super().conform_dataframe(df=df)
         if self.upload_stager_config.rename_columns_map:
             df.rename(columns=self.upload_stager_config.rename_columns_map, inplace=True)
@@ -193,8 +191,9 @@ class VastdbUploader(SQLUploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    @requires_dependencies(["pyarrow"], extras="vastdb")
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    @requires_dependencies(["pyarrow", "pandas"], extras="vastdb")
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
+        import numpy as np
         import pyarrow as pa
 
         if self.can_delete():
@@ -216,7 +215,6 @@ class VastdbUploader(SQLUploader):
         )
 
         for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
-
             with self.connection_config.get_table(self.upload_config.table_name) as table:
                 pa_table = pa.Table.from_pandas(rows)
                 table.insert(pa_table)

unstructured_ingest/v2/types/file_data.py

@@ -0,0 +1,116 @@
+import json
+from pathlib import Path
+from typing import Any, Optional
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
+
+from unstructured_ingest.v2.logger import logger
+
+
+class SourceIdentifiers(BaseModel):
+    filename: str
+    fullpath: str
+    rel_path: Optional[str] = None
+
+    @property
+    def filename_stem(self) -> str:
+        return Path(self.filename).stem
+
+    @property
+    def relative_path(self) -> str:
+        return self.rel_path or self.fullpath
+
+
+class FileDataSourceMetadata(BaseModel):
+    url: Optional[str] = None
+    version: Optional[str] = None
+    record_locator: Optional[dict[str, Any]] = None
+    date_created: Optional[str] = None
+    date_modified: Optional[str] = None
+    date_processed: Optional[str] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
+    filesize_bytes: Optional[int] = None
+
+
+class FileData(BaseModel):
+    identifier: str
+    connector_type: str
+    source_identifiers: SourceIdentifiers
+    metadata: FileDataSourceMetadata = Field(default_factory=lambda: FileDataSourceMetadata())
+    additional_metadata: dict[str, Any] = Field(default_factory=dict)
+    reprocess: bool = False
+    local_download_path: Optional[str] = None
+    display_name: Optional[str] = None
+
+    @classmethod
+    def from_file(cls, path: str) -> "FileData":
+        path = Path(path).resolve()
+        if not path.exists() or not path.is_file():
+            raise ValueError(f"file path not valid: {path}")
+        with open(str(path.resolve()), "rb") as f:
+            file_data_dict = json.load(f)
+        file_data = cls.model_validate(file_data_dict)
+        return file_data
+
+    @classmethod
+    def cast(cls, file_data: "FileData", **kwargs) -> "FileData":
+        file_data_dict = file_data.model_dump()
+        return cls.model_validate(file_data_dict, **kwargs)
+
+    def to_file(self, path: str) -> None:
+        path = Path(path).resolve()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(str(path.resolve()), "w") as f:
+            json.dump(self.model_dump(), f, indent=2)
+
+
+class BatchItem(BaseModel):
+    identifier: str
+    version: Optional[str] = None
+
+
+class BatchFileData(FileData):
+    identifier: str = Field(init=False)
+    batch_items: list[BatchItem]
+    source_identifiers: Optional[SourceIdentifiers] = None
+
+    @field_validator("batch_items")
+    @classmethod
+    def check_batch_items(cls, v: list[BatchItem]) -> list[BatchItem]:
+        if not v:
+            raise ValueError("batch items cannot be empty")
+        all_identifiers = [item.identifier for item in v]
+        if len(all_identifiers) != len(set(all_identifiers)):
+            raise ValueError(f"duplicate identifiers: {all_identifiers}")
+        sorted_batch_items = sorted(v, key=lambda item: item.identifier)
+        return sorted_batch_items
+
+    @model_validator(mode="before")
+    @classmethod
+    def populate_identifier(cls, data: Any) -> Any:
+        if isinstance(data, dict) and "identifier" not in data:
+            batch_items = data["batch_items"]
+            identifier_data = json.dumps(
+                {item.identifier: item.version for item in batch_items}, sort_keys=True
+            )
+            data["identifier"] = str(uuid5(NAMESPACE_DNS, str(identifier_data)))
+        return data
+
+
+def file_data_from_file(path: str) -> FileData:
+    try:
+        return BatchFileData.from_file(path=path)
+    except ValidationError:
+        logger.debug(f"{path} not detected as batch file data")
+
+    return FileData.from_file(path=path)
+
+
+def file_data_from_dict(data: dict) -> FileData:
+    try:
+        return BatchFileData.model_validate(data)
+    except ValidationError:
+        logger.debug(f"{data} not valid for batch file data")
+
+    return FileData.model_validate(data)