unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/v2/interfaces/downloader.py

@@ -0,0 +1,89 @@
+import os
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Optional, TypedDict, TypeVar, Union
+
+from pydantic import BaseModel, Field
+
+from unstructured_ingest.v2.interfaces.connector import BaseConnector
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+class DownloaderConfig(BaseModel):
+    download_dir: Optional[Path] = Field(
+        default=None,
+        description="Where files are downloaded to, defaults to a location at"
+        "`$HOME/.cache/unstructured/ingest/<connector name>/<SHA256>`.",
+    )
+
+
+DownloaderConfigT = TypeVar("DownloaderConfigT", bound=DownloaderConfig)
+
+
+class DownloadResponse(TypedDict):
+    file_data: FileData
+    path: Path
+
+
+download_responses = Union[list[DownloadResponse], DownloadResponse]
+
+
+class Downloader(BaseProcess, BaseConnector, ABC):
+    connector_type: str
+    download_config: DownloaderConfigT
+
+    def get_download_path(self, file_data: FileData) -> Optional[Path]:
+        if not file_data.source_identifiers:
+            return None
+        rel_path = file_data.source_identifiers.relative_path
+        if not rel_path:
+            return None
+        rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
+        return self.download_dir / Path(rel_path)
+
+    @staticmethod
+    def is_float(value: str):
+        try:
+            float(value)
+            return True
+        except ValueError:
+            return False
+
+    def generate_download_response(
+        self, file_data: FileData, download_path: Path
+    ) -> DownloadResponse:
+        if (
+            file_data.metadata.date_modified
+            and self.is_float(file_data.metadata.date_modified)
+            and file_data.metadata.date_created
+            and self.is_float(file_data.metadata.date_created)
+        ):
+            date_modified = float(file_data.metadata.date_modified)
+            date_created = float(file_data.metadata.date_created)
+            os.utime(download_path, times=(date_created, date_modified))
+        file_data.local_download_path = str(download_path.resolve())
+        return DownloadResponse(file_data=file_data, path=download_path)
+
+    @property
+    def download_dir(self) -> Path:
+        if self.download_config.download_dir is None:
+            self.download_config.download_dir = (
+                Path.home()
+                / ".cache"
+                / "unstructured"
+                / "ingest"
+                / "download"
+                / self.connector_type
+            ).resolve()
+        return self.download_config.download_dir
+
+    def is_async(self) -> bool:
+        return True
+
+    @abstractmethod
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        pass
+
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        return self.run(file_data=file_data, **kwargs)

unstructured_ingest/v2/interfaces/file_data.py

@@ -0,0 +1,116 @@
+import json
+from pathlib import Path
+from typing import Any, Optional
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
+
+from unstructured_ingest.v2.logger import logger
+
+
+class SourceIdentifiers(BaseModel):
+    filename: str
+    fullpath: str
+    rel_path: Optional[str] = None
+
+    @property
+    def filename_stem(self) -> str:
+        return Path(self.filename).stem
+
+    @property
+    def relative_path(self) -> str:
+        return self.rel_path or self.fullpath
+
+
+class FileDataSourceMetadata(BaseModel):
+    url: Optional[str] = None
+    version: Optional[str] = None
+    record_locator: Optional[dict[str, Any]] = None
+    date_created: Optional[str] = None
+    date_modified: Optional[str] = None
+    date_processed: Optional[str] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
+    filesize_bytes: Optional[int] = None
+
+
+class FileData(BaseModel):
+    identifier: str
+    connector_type: str
+    source_identifiers: SourceIdentifiers
+    metadata: FileDataSourceMetadata = Field(default_factory=lambda: FileDataSourceMetadata())
+    additional_metadata: dict[str, Any] = Field(default_factory=dict)
+    reprocess: bool = False
+    local_download_path: Optional[str] = None
+    display_name: Optional[str] = None
+
+    @classmethod
+    def from_file(cls, path: str) -> "FileData":
+        path = Path(path).resolve()
+        if not path.exists() or not path.is_file():
+            raise ValueError(f"file path not valid: {path}")
+        with open(str(path.resolve()), "rb") as f:
+            file_data_dict = json.load(f)
+        file_data = cls.model_validate(file_data_dict)
+        return file_data
+
+    @classmethod
+    def cast(cls, file_data: "FileData", **kwargs) -> "FileData":
+        file_data_dict = file_data.model_dump()
+        return cls.model_validate(file_data_dict, **kwargs)
+
+    def to_file(self, path: str) -> None:
+        path = Path(path).resolve()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(str(path.resolve()), "w") as f:
+            json.dump(self.model_dump(), f, indent=2)
+
+
+class BatchItem(BaseModel):
+    identifier: str
+    version: Optional[str] = None
+
+
+class BatchFileData(FileData):
+    identifier: str = Field(init=False)
+    batch_items: list[BatchItem]
+    source_identifiers: Optional[SourceIdentifiers] = None
+
+    @field_validator("batch_items")
+    @classmethod
+    def check_batch_items(cls, v: list[BatchItem]) -> list[BatchItem]:
+        if not v:
+            raise ValueError("batch items cannot be empty")
+        all_identifiers = [item.identifier for item in v]
+        if len(all_identifiers) != len(set(all_identifiers)):
+            raise ValueError(f"duplicate identifiers: {all_identifiers}")
+        sorted_batch_items = sorted(v, key=lambda item: item.identifier)
+        return sorted_batch_items
+
+    @model_validator(mode="before")
+    @classmethod
+    def populate_identifier(cls, data: Any) -> Any:
+        if isinstance(data, dict) and "identifier" not in data:
+            batch_items = data["batch_items"]
+            identifier_data = json.dumps(
+                {item.identifier: item.version for item in batch_items}, sort_keys=True
+            )
+            data["identifier"] = str(uuid5(NAMESPACE_DNS, str(identifier_data)))
+        return data
+
+
+def file_data_from_file(path: str) -> FileData:
+    try:
+        return BatchFileData.from_file(path=path)
+    except ValidationError:
+        logger.debug(f"{path} not valid for batch file data")
+
+    return FileData.from_file(path=path)
+
+
+def file_data_from_dict(data: dict) -> FileData:
+    try:
+        return BatchFileData.model_validate(data)
+    except ValidationError:
+        logger.debug(f"{data} not valid for batch file data")
+
+    return FileData.model_validate(data)

unstructured_ingest/v2/interfaces/indexer.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from typing import Any, AsyncGenerator, Generator, Optional, TypeVar
+
+from pydantic import BaseModel
+
+from unstructured_ingest.v2.interfaces.connector import BaseConnector
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+class IndexerConfig(BaseModel):
+    pass
+
+
+IndexerConfigT = TypeVar("IndexerConfigT", bound=IndexerConfig)
+
+
+class Indexer(BaseProcess, BaseConnector, ABC):
+    connector_type: str
+    index_config: Optional[IndexerConfigT] = None
+
+    def is_async(self) -> bool:
+        return False
+
+    @abstractmethod
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        pass
+
+    async def run_async(self, **kwargs: Any) -> AsyncGenerator[FileData, None]:
+        raise NotImplementedError()

unstructured_ingest/v2/interfaces/process.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class BaseProcess(ABC):
+    def is_async(self) -> bool:
+        return False
+
+    def precheck(self) -> None:
+        pass
+
+    @abstractmethod
+    def run(self, **kwargs: Any) -> Any:
+        pass
+
+    async def run_async(self, **kwargs: Any) -> Any:
+        return self.run(**kwargs)

unstructured_ingest/v2/interfaces/processor.py

@@ -0,0 +1,88 @@
+import os
+from asyncio import Semaphore
+from pathlib import Path
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+DEFAULT_WORK_DIR = str((Path.home() / ".cache" / "unstructured" / "ingest" / "pipeline").resolve())
+
+
+class ProcessorConfig(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    reprocess: bool = Field(
+        default=False,
+        description="Reprocess a downloaded file even if the relevant structured "
+        "output .json file in output directory already exists.",
+    )
+    verbose: bool = Field(default=False)
+    tqdm: bool = Field(default=False, description="Display tqdm progress bar")
+    work_dir: str = Field(
+        default_factory=lambda: DEFAULT_WORK_DIR,
+        description="Where to place working files when processing each step",
+    )
+    num_processes: int = Field(
+        default=2, description="Number of parallel processes with which to process docs"
+    )
+    max_connections: Optional[int] = Field(
+        default=None, description="Limit of concurrent connectionts"
+    )
+    raise_on_error: bool = Field(
+        default=False,
+        description="Is set, will raise error if any doc in the pipeline fail. "
+        "Otherwise will log error and continue with other docs",
+    )
+    disable_parallelism: bool = Field(
+        default_factory=lambda: os.getenv("INGEST_DISABLE_PARALLELISM", "false").lower() == "true",
+    )
+    preserve_downloads: bool = Field(
+        default=False, description="Don't delete downloaded files after process completes"
+    )
+    download_only: bool = Field(
+        default=False, description="skip the rest of the process after files are downloaded"
+    )
+    re_download: bool = Field(
+        default=False,
+        description="If set, will re-download downloaded files "
+        "regardless of if they already exist locally",
+    )
+    uncompress: bool = Field(
+        default=False,
+        description="Uncompress any archived files. Currently supporting "
+        "zip and tar files based on file extension.",
+    )
+    iter_delete: bool = Field(
+        default=False,
+        description="If limited on memory, this can be enabled to delete "
+        "cached content as it's used and no longer needed in the pipeline.",
+    )
+    delete_cache: bool = Field(
+        default=False,
+        description="If set, will delete the cache work directory when process finishes",
+    )
+
+    # OTEL support
+    otel_endpoint: Optional[str] = Field(
+        default=None, description="OTEL endpoint to publish trace data to"
+    )
+
+    # Used to keep track of state in pipeline
+    status: dict = Field(default_factory=dict)
+    semaphore: Optional[Semaphore] = Field(init=False, default=None, exclude=True)
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.max_connections is not None:
+            self.semaphore = Semaphore(self.max_connections)
+
+    @property
+    def mp_supported(self) -> bool:
+        return not self.disable_parallelism and self.num_processes > 1
+
+    @property
+    def async_supported(self) -> bool:
+        if self.disable_parallelism:
+            return False
+        if self.max_connections is not None and isinstance(self.max_connections, int):
+            return self.max_connections > 1
+        return True

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -0,0 +1,102 @@
+import json
+from abc import ABC
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, TypeVar
+
+import ndjson
+from pydantic import BaseModel
+
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+class UploadStagerConfig(BaseModel):
+    pass
+
+
+UploadStagerConfigT = TypeVar("UploadStagerConfigT", bound=UploadStagerConfig)
+
+
+@dataclass
+class UploadStager(BaseProcess, ABC):
+    upload_stager_config: UploadStagerConfigT
+
+    def write_output(self, output_path: Path, data: list[dict]) -> None:
+        if output_path.suffix == ".json":
+            with output_path.open("w") as f:
+                json.dump(data, f, indent=2)
+        elif output_path.suffix == ".ndjson":
+            with output_path.open("w") as f:
+                ndjson.dump(data, f)
+        else:
+            raise ValueError(f"Unsupported output format: {output_path}")
+
+    def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
+        return element_dict
+
+    def get_output_path(self, output_filename: str, output_dir: Path) -> Path:
+        output_path = Path(output_filename)
+        output_filename = f"{Path(output_filename).stem}{output_path.suffix}"
+        output_path = Path(output_dir) / Path(f"{output_filename}")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        return output_path
+
+    def stream_update(self, input_file: Path, output_file: Path, file_data: FileData) -> None:
+        with input_file.open() as in_f:
+            reader = ndjson.reader(in_f)
+            with output_file.open("w") as out_f:
+                writer = ndjson.writer(out_f)
+                for element in reader:
+                    conformed_element = self.conform_dict(element_dict=element, file_data=file_data)
+                    writer.writerow(row=conformed_element)
+                    writer.f.flush()
+
+    def process_whole(self, input_file: Path, output_file: Path, file_data: FileData) -> None:
+        with input_file.open() as in_f:
+            elements_contents = json.load(in_f)
+
+        conformed_elements = [
+            self.conform_dict(element_dict=element, file_data=file_data)
+            for element in elements_contents
+        ]
+
+        with open(output_file, "w") as out_f:
+            json.dump(conformed_elements, out_f, indent=2)
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        output_file = self.get_output_path(output_filename=output_filename, output_dir=output_dir)
+        if elements_filepath.suffix == ".ndjson":
+            self.stream_update(
+                input_file=elements_filepath, output_file=output_file, file_data=file_data
+            )
+        elif elements_filepath.suffix == ".json":
+            self.process_whole(
+                input_file=elements_filepath, output_file=output_file, file_data=file_data
+            )
+        else:
+            raise ValueError(f"Unsupported file extension: {elements_filepath}")
+        return output_file
+
+    async def run_async(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        return self.run(
+            elements_filepath=elements_filepath,
+            output_dir=output_dir,
+            output_filename=output_filename,
+            file_data=file_data,
+            **kwargs,
+        )

unstructured_ingest/v2/interfaces/uploader.py

@@ -0,0 +1,53 @@
+from abc import ABC
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, TypeVar
+
+from pydantic import BaseModel
+
+from unstructured_ingest.utils.data_prep import get_data
+from unstructured_ingest.v2.interfaces.connector import BaseConnector
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+class UploaderConfig(BaseModel):
+    pass
+
+
+UploaderConfigT = TypeVar("UploaderConfigT", bound=UploaderConfig)
+
+
+@dataclass
+class UploadContent:
+    path: Path
+    file_data: FileData
+
+
+@dataclass
+class Uploader(BaseProcess, BaseConnector, ABC):
+    upload_config: UploaderConfigT
+    connector_type: str
+
+    def is_async(self) -> bool:
+        return False
+
+    def is_batch(self) -> bool:
+        return False
+
+    def run_batch(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        raise NotImplementedError()
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        data = get_data(path=path)
+        self.run_data(data=data, file_data=file_data, **kwargs)
+
+    async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        data = get_data(path=path)
+        await self.run_data_async(data=data, file_data=file_data, **kwargs)
+
+    def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        raise NotImplementedError()
+
+    async def run_data_async(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        return self.run_data(data=data, file_data=file_data, **kwargs)

unstructured_ingest/v2/logger.py

@@ -0,0 +1,126 @@
+import ast
+import json
+import os
+from logging import Formatter, Logger, StreamHandler, getLevelName, getLogger
+from typing import Any, Callable
+
+log_level = os.getenv("INGEST_LOG_LEVEL", "INFO")
+LOGGER_NAME = "unstructured_ingest.v2"
+
+
+def default_is_data_sensitive(k: str, v: Any) -> bool:
+    sensitive_fields = [
+        "account_name",
+        "client_id",
+    ]
+    sensitive_triggers = ["key", "cred", "token", "password", "oauth", "secret"]
+    return (
+        v
+        and any([s in k.lower() for s in sensitive_triggers])  # noqa: C419
+        or k.lower() in sensitive_fields
+    )
+
+
+def hide_sensitive_fields(
+    data: dict, is_sensitive_fn: Callable[[str, Any], bool] = default_is_data_sensitive
+) -> dict:
+    """
+    Will recursively look through every k, v pair in this dict and any nested ones and run
+    is_sensitive_fn to dynamically redact the value of the k, v pair. Will also check if
+    any string value can be parsed as valid json and process that dict as well and replace
+    the original string with the json.dumps() version of the redacted dict.
+    """
+    new_data = data.copy()
+    for k, v in new_data.items():
+        if is_sensitive_fn(k, v):
+            new_data[k] = "*******"
+        if isinstance(v, dict):
+            new_data[k] = hide_sensitive_fields(v)
+        if isinstance(v, str):
+            # Need to take into account strings generated via json.dumps() or simply printing a dict
+            try:
+                json_data = json.loads(v)
+                if isinstance(json_data, dict):
+                    updated_data = hide_sensitive_fields(json_data)
+                    new_data[k] = json.dumps(updated_data)
+            except json.JSONDecodeError:
+                pass
+
+    return new_data
+
+
+def redact_jsons(s: str) -> str:
+    """
+    Takes in a generic string and pulls out all valid json content. Leverages
+    hide_sensitive_fields() to redact any sensitive information and replaces the
+    original json with the new redacted format. There can be any number of valid
+    jsons in a generic string and this will work. Having extra '{' without a
+    closing '}' will cause this to break though. i.e '{ text, {"a": 3}'.
+
+    """
+    chars = list(s)
+    if "{" not in chars:
+        return s
+    i = 0
+    jsons = []
+    i = 0
+    while i < len(chars):
+        char = chars[i]
+        if char == "{":
+            stack = [char]
+            current = [char]
+            while len(stack) != 0 and i < len(chars):
+                i += 1
+                char = chars[i]
+                current.append(char)
+                if char == "{":
+                    stack.append(char)
+                if char == "}":
+                    stack.pop(-1)
+            jsons.append("".join(current))
+            continue
+        i += 1
+    for j in jsons:
+        try:
+            formatted_j = json.dumps(json.loads(j))
+        except json.JSONDecodeError:
+            lit = ast.literal_eval(j)
+            formatted_j = json.dumps(lit)
+        hidden_j = json.dumps(hide_sensitive_fields(json.loads(formatted_j)))
+        s = s.replace(j, hidden_j)
+    return s
+
+
+class SensitiveFormatter(Formatter):
+    def format(self, record):
+        s = super().format(record=record)
+        try:
+            return redact_jsons(s)
+        except Exception:
+            return f"Failed to redact: {s}"
+
+
+def remove_root_handlers(logger: Logger) -> None:
+    # NOTE(robinson): in some environments such as Google Colab, there is a root handler
+    # that doesn't not mask secrets, meaning sensitive info such as api keys appear in logs.
+    # Removing these when they exist prevents this behavior
+    if logger.root.hasHandlers():
+        for handler in logger.root.handlers:
+            logger.root.removeHandler(handler)
+
+
+def make_default_logger(level: int) -> Logger:
+    """Return a custom logger."""
+    logger = getLogger(LOGGER_NAME)
+    handler = StreamHandler()
+    handler.name = "ingest_log_handler"
+    formatter = SensitiveFormatter("%(asctime)s %(processName)-10s %(levelname)-8s %(message)s")
+    handler.setFormatter(formatter)
+    if handler.name not in [h.name for h in logger.handlers]:
+        logger.addHandler(handler)
+    logger.setLevel(level)
+    remove_root_handlers(logger)
+    return logger
+
+
+logger = make_default_logger(level=getLevelName(log_level.upper()))

unstructured_ingest/v2/main.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+from unstructured_ingest.v2.cli.cli import get_cmd
+
+
+def main():
+    ingest_cmd = get_cmd()
+    ingest_cmd()
+
+
+if __name__ == "__main__":
+    main()