unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/v2/interfaces/connector.py

@@ -0,0 +1,32 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import Any, TypeVar
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+
+
+@dataclass
+class AccessConfig(EnhancedDataClassJsonMixin):
+    """Meant to designate holding any sensitive information associated with other configs
+    and also for access specific configs."""
+
+
+AccessConfigT = TypeVar("AccessConfigT", bound=AccessConfig)
+
+
+@dataclass
+class ConnectionConfig(EnhancedDataClassJsonMixin):
+    access_config: AccessConfigT
+
+    def get_access_config(self) -> dict[str, Any]:
+        if not self.access_config:
+            return {}
+        return self.access_config.to_dict(apply_name_overload=False)
+
+
+ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)
+
+
+@dataclass
+class BaseConnector(ABC):
+    connection_config: ConnectionConfigT

unstructured_ingest/v2/interfaces/downloader.py

@@ -0,0 +1,79 @@
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional, TypedDict, TypeVar, Union
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.v2.interfaces.connector import BaseConnector
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+@dataclass
+class DownloaderConfig(EnhancedDataClassJsonMixin):
+    download_dir: Optional[Path] = None
+
+
+DownloaderConfigT = TypeVar("DownloaderConfigT", bound=DownloaderConfig)
+
+
+class DownloadResponse(TypedDict):
+    file_data: FileData
+    path: Path
+
+
+download_responses = Union[list[DownloadResponse], DownloadResponse]
+
+
+class Downloader(BaseProcess, BaseConnector, ABC):
+    connector_type: str
+    download_config: DownloaderConfigT
+
+    @staticmethod
+    def is_float(value: str):
+        try:
+            float(value)
+            return True
+        except ValueError:
+            return False
+
+    def generate_download_response(
+        self, file_data: FileData, download_path: Path
+    ) -> DownloadResponse:
+        if (
+            file_data.metadata.date_modified
+            and self.is_float(file_data.metadata.date_modified)
+            and file_data.metadata.date_created
+            and self.is_float(file_data.metadata.date_created)
+        ):
+            date_modified = float(file_data.metadata.date_modified)
+            date_created = float(file_data.metadata.date_created)
+            os.utime(download_path, times=(date_created, date_modified))
+        return DownloadResponse(file_data=file_data, path=download_path)
+
+    @property
+    def download_dir(self) -> Path:
+        if self.download_config.download_dir is None:
+            self.download_config.download_dir = (
+                Path.home()
+                / ".cache"
+                / "unstructured"
+                / "ingest"
+                / "download"
+                / self.connector_type
+            ).resolve()
+        return self.download_config.download_dir
+
+    def is_async(self) -> bool:
+        return True
+
+    def get_download_path(self, file_data: FileData) -> Optional[Path]:
+        return None
+
+    @abstractmethod
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        pass
+
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        return self.run(file_data=file_data, **kwargs)

unstructured_ingest/v2/interfaces/file_data.py

@@ -0,0 +1,49 @@
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, Optional
+
+from dataclasses_json import DataClassJsonMixin
+from unstructured.documents.elements import DataSourceMetadata
+
+
+@dataclass
+class SourceIdentifiers:
+    filename: str
+    fullpath: str
+    rel_path: Optional[str] = None
+
+    @property
+    def filename_stem(self) -> str:
+        return Path(self.filename).stem
+
+    @property
+    def relative_path(self) -> str:
+        return self.rel_path or self.fullpath
+
+
+@dataclass
+class FileData(DataClassJsonMixin):
+    identifier: str
+    connector_type: str
+    source_identifiers: Optional[SourceIdentifiers] = None
+    doc_type: Literal["file", "batch"] = field(default="file")
+    metadata: DataSourceMetadata = field(default_factory=DataSourceMetadata)
+    additional_metadata: dict[str, Any] = field(default_factory=dict)
+    reprocess: bool = False
+
+    @classmethod
+    def from_file(cls, path: str) -> "FileData":
+        path = Path(path).resolve()
+        if not path.exists() or not path.is_file():
+            raise ValueError(f"file path not valid: {path}")
+        with open(str(path.resolve()), "rb") as f:
+            file_data_dict = json.load(f)
+        file_data = FileData.from_dict(file_data_dict)
+        return file_data
+
+    def to_file(self, path: str) -> None:
+        path = Path(path).resolve()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(str(path.resolve()), "w") as f:
+            json.dump(self.to_dict(), f, indent=2)

unstructured_ingest/v2/interfaces/indexer.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Generator, Optional, TypeVar
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.v2.interfaces.connector import BaseConnector
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+@dataclass
+class IndexerConfig(EnhancedDataClassJsonMixin):
+    pass
+
+
+IndexerConfigT = TypeVar("IndexerConfigT", bound=IndexerConfig)
+
+
+class Indexer(BaseProcess, BaseConnector, ABC):
+    connector_type: str
+    index_config: Optional[IndexerConfigT] = None
+
+    def is_async(self) -> bool:
+        return False
+
+    @abstractmethod
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        pass

unstructured_ingest/v2/interfaces/process.py

@@ -0,0 +1,20 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class BaseProcess(ABC):
+    def is_async(self) -> bool:
+        return False
+
+    @abstractmethod
+    def run(self, **kwargs: Any) -> Any:
+        pass
+
+    async def run_async(self, **kwargs: Any) -> Any:
+        return self.run(**kwargs)
+
+    def check_connection(self):
+        # If the process requires external connections, run a quick check
+        pass

unstructured_ingest/v2/interfaces/processor.py

@@ -0,0 +1,48 @@
+import os
+from asyncio import Semaphore
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+
+DEFAULT_WORK_DIR = str((Path.home() / ".cache" / "unstructured" / "ingest" / "pipeline").resolve())
+
+
+@dataclass
+class ProcessorConfig(EnhancedDataClassJsonMixin):
+    reprocess: bool = False
+    verbose: bool = False
+    tqdm: bool = False
+    work_dir: str = field(default_factory=lambda: DEFAULT_WORK_DIR)
+    num_processes: int = 2
+    max_connections: Optional[int] = None
+    raise_on_error: bool = False
+    disable_parallelism: bool = field(
+        default_factory=lambda: os.getenv("INGEST_DISABLE_PARALLELISM", "false").lower() == "true"
+    )
+    preserve_downloads: bool = False
+    download_only: bool = False
+    max_docs: Optional[int] = None
+    re_download: bool = False
+    uncompress: bool = False
+
+    # Used to keep track of state in pipeline
+    status: dict = field(default_factory=dict)
+    semaphore: Optional[Semaphore] = field(init=False, default=None)
+
+    def __post_init__(self):
+        if self.max_connections is not None:
+            self.semaphore = Semaphore(self.max_connections)
+
+    @property
+    def mp_supported(self) -> bool:
+        return not self.disable_parallelism and self.num_processes > 1
+
+    @property
+    def async_supported(self) -> bool:
+        if self.disable_parallelism:
+            return False
+        if self.max_connections is not None and isinstance(self.max_connections, int):
+            return self.max_connections > 1
+        return True

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, TypeVar
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+@dataclass
+class UploadStagerConfig(EnhancedDataClassJsonMixin):
+    pass
+
+
+UploadStagerConfigT = TypeVar("UploadStagerConfigT", bound=UploadStagerConfig)
+
+
+@dataclass
+class UploadStager(BaseProcess, ABC):
+    upload_stager_config: UploadStagerConfigT
+
+    @abstractmethod
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any
+    ) -> Path:
+        pass
+
+    async def run_async(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any
+    ) -> Path:
+        return self.run(
+            elements_filepath=elements_filepath,
+            output_dir=output_dir,
+            output_filename=output_filename,
+            file_data=file_data,
+            **kwargs
+        )

unstructured_ingest/v2/interfaces/uploader.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, TypeVar
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.v2.interfaces.connector import BaseConnector
+from unstructured_ingest.v2.interfaces.file_data import FileData
+from unstructured_ingest.v2.interfaces.process import BaseProcess
+
+
+@dataclass
+class UploaderConfig(EnhancedDataClassJsonMixin):
+    pass
+
+
+UploaderConfigT = TypeVar("UploaderConfigT", bound=UploaderConfig)
+
+
+@dataclass
+class UploadContent:
+    path: Path
+    file_data: FileData
+
+
+@dataclass
+class Uploader(BaseProcess, BaseConnector, ABC):
+    upload_config: UploaderConfigT
+    connector_type: str
+
+    def is_async(self) -> bool:
+        return False
+
+    @abstractmethod
+    def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        pass
+
+    async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        return self.run(contents=[UploadContent(path=path, file_data=file_data)], **kwargs)

unstructured_ingest/v2/logger.py

@@ -0,0 +1,126 @@
+import ast
+import json
+import os
+from logging import Formatter, Logger, StreamHandler, getLevelName, getLogger
+from typing import Any, Callable
+
+log_level = os.getenv("INGEST_LOG_LEVEL", "INFO")
+LOGGER_NAME = "unstructured.ingest.v2"
+
+
+def default_is_data_sensitive(k: str, v: Any) -> bool:
+    sensitive_fields = [
+        "account_name",
+        "client_id",
+    ]
+    sensitive_triggers = ["key", "cred", "token", "password", "oauth", "secret"]
+    return (
+        v
+        and any([s in k.lower() for s in sensitive_triggers])  # noqa: C419
+        or k.lower() in sensitive_fields
+    )
+
+
+def hide_sensitive_fields(
+    data: dict, is_sensitive_fn: Callable[[str, Any], bool] = default_is_data_sensitive
+) -> dict:
+    """
+    Will recursively look through every k, v pair in this dict and any nested ones and run
+    is_sensitive_fn to dynamically redact the value of the k, v pair. Will also check if
+    any string value can be parsed as valid json and process that dict as well and replace
+    the original string with the json.dumps() version of the redacted dict.
+    """
+    new_data = data.copy()
+    for k, v in new_data.items():
+        if is_sensitive_fn(k, v):
+            new_data[k] = "*******"
+        if isinstance(v, dict):
+            new_data[k] = hide_sensitive_fields(v)
+        if isinstance(v, str):
+            # Need to take into account strings generated via json.dumps() or simply printing a dict
+            try:
+                json_data = json.loads(v)
+                if isinstance(json_data, dict):
+                    updated_data = hide_sensitive_fields(json_data)
+                    new_data[k] = json.dumps(updated_data)
+            except json.JSONDecodeError:
+                pass
+
+    return new_data
+
+
+def redact_jsons(s: str) -> str:
+    """
+    Takes in a generic string and pulls out all valid json content. Leverages
+    hide_sensitive_fields() to redact any sensitive information and replaces the
+    original json with the new redacted format. There can be any number of valid
+    jsons in a generic string and this will work. Having extra '{' without a
+    closing '}' will cause this to break though. i.e '{ text, {"a": 3}'.
+
+    """
+    chars = list(s)
+    if "{" not in chars:
+        return s
+    i = 0
+    jsons = []
+    i = 0
+    while i < len(chars):
+        char = chars[i]
+        if char == "{":
+            stack = [char]
+            current = [char]
+            while len(stack) != 0 and i < len(chars):
+                i += 1
+                char = chars[i]
+                current.append(char)
+                if char == "{":
+                    stack.append(char)
+                if char == "}":
+                    stack.pop(-1)
+            jsons.append("".join(current))
+            continue
+        i += 1
+    for j in jsons:
+        try:
+            formatted_j = json.dumps(json.loads(j))
+        except json.JSONDecodeError:
+            lit = ast.literal_eval(j)
+            formatted_j = json.dumps(lit)
+        hidden_j = json.dumps(hide_sensitive_fields(json.loads(formatted_j)))
+        s = s.replace(j, hidden_j)
+    return s
+
+
+class SensitiveFormatter(Formatter):
+    def format(self, record):
+        s = super().format(record=record)
+        try:
+            return redact_jsons(s)
+        except Exception:
+            return f"Failed to redact: {s}"
+
+
+def remove_root_handlers(logger: Logger) -> None:
+    # NOTE(robinson) - in some environments such as Google Colab, there is a root handler
+    # that doesn't not mask secrets, meaning sensitive info such as api keys appear in logs.
+    # Removing these when they exist prevents this behavior
+    if logger.root.hasHandlers():
+        for handler in logger.root.handlers:
+            logger.root.removeHandler(handler)
+
+
+def make_default_logger(level: int) -> Logger:
+    """Return a custom logger."""
+    logger = getLogger(LOGGER_NAME)
+    handler = StreamHandler()
+    handler.name = "ingest_log_handler"
+    formatter = SensitiveFormatter("%(asctime)s %(processName)-10s %(levelname)-8s %(message)s")
+    handler.setFormatter(formatter)
+    if handler.name not in [h.name for h in logger.handlers]:
+        logger.addHandler(handler)
+    logger.setLevel(level)
+    remove_root_handlers(logger)
+    return logger
+
+
+logger = make_default_logger(level=getLevelName(log_level.upper()))

unstructured_ingest/v2/main.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+from unstructured_ingest.v2.cli.cli import get_cmd
+
+
+def main():
+    ingest_cmd = get_cmd()
+    ingest_cmd()
+
+
+if __name__ == "__main__":
+    main()

unstructured_ingest/v2/pipeline/interfaces.py

@@ -0,0 +1,167 @@
+import asyncio
+import logging
+import multiprocessing as mp
+from abc import ABC
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from functools import wraps
+from pathlib import Path
+from time import time
+from typing import Any, Awaitable, Callable, Optional, TypeVar
+
+from tqdm import tqdm
+from tqdm.asyncio import tqdm as tqdm_asyncio
+
+from unstructured_ingest.v2.interfaces import BaseProcess, ProcessorConfig
+from unstructured_ingest.v2.logger import logger, make_default_logger
+
+BaseProcessT = TypeVar("BaseProcessT", bound=BaseProcess)
+iterable_input = list[dict[str, Any]]
+
+
+def timed(func):
+    @wraps(func)
+    def time_it(self, *args, **kwargs):
+        start = time()
+        try:
+            return func(self, *args, **kwargs)
+        finally:
+            if func.__name__ == "__call__":
+                reported_name = f"{self.__class__.__name__} [cls]"
+            else:
+                reported_name = func.__name__
+            logger.info(f"{reported_name} took {time() - start} seconds")
+
+    return time_it
+
+
+@dataclass
+class PipelineStep(ABC):
+    process: BaseProcessT
+    context: ProcessorConfig
+    identifier: str
+
+    def __str__(self):
+        return self.identifier
+
+    def process_serially(self, iterable: iterable_input) -> Any:
+        logger.info("processing content serially")
+        if iterable:
+            if len(iterable) == 1:
+                return [self.run(**iterable[0])]
+            if self.context.tqdm:
+                return [self.run(**it) for it in tqdm(iterable, desc=self.identifier)]
+            return [self.run(**it) for it in iterable]
+        return [self.run()]
+
+    async def _process_async(self, iterable: iterable_input) -> Any:
+        if iterable:
+            if len(iterable) == 1:
+                return [await self.run_async(**iterable[0])]
+            if self.context.tqdm:
+                return await tqdm_asyncio.gather(
+                    *[self.run_async(**i) for i in iterable], desc=self.identifier
+                )
+            return await asyncio.gather(*[self.run_async(**i) for i in iterable])
+        return [await self.run_async()]
+
+    def process_async(self, iterable: iterable_input) -> Any:
+        logger.info("processing content async")
+        return self.asyncio_run(fn=self._process_async, iterable=iterable)
+
+    def asyncio_run(
+        self, fn: Callable[[Any, Any], Awaitable[Any]], *args: Any, **kwargs: Any
+    ) -> Any:
+        current_loop = asyncio._get_running_loop()
+        if current_loop is None:
+            return asyncio.run(fn(*args, **kwargs))
+        with ThreadPoolExecutor(thread_name_prefix="asyncio") as thread_pool:
+            logger.warning(
+                f"async code being run in dedicated thread pool "
+                f"to not conflict with existing event loop: {current_loop}"
+            )
+
+            def wrapped():
+                return asyncio.run(fn(*args, **kwargs))
+
+            future = thread_pool.submit(wrapped)
+            return future.result()
+
+    def process_multiprocess(self, iterable: iterable_input) -> Any:
+        logger.info("processing content across processes")
+
+        if iterable:
+            if len(iterable) == 1:
+                return [self.process_serially(iterable)]
+            if self.context.num_processes == 1:
+                return self.process_serially(iterable)
+            with mp.Pool(
+                processes=self.context.num_processes,
+                initializer=self._init_logger,
+                initargs=(logging.DEBUG if self.context.verbose else logging.INFO,),
+            ) as pool:
+                if self.context.tqdm:
+                    return list(
+                        tqdm(
+                            pool.imap_unordered(func=self._wrap_mp, iterable=iterable),
+                            total=len(iterable),
+                            desc=self.identifier,
+                        )
+                    )
+                return pool.map(self._wrap_mp, iterable)
+        return [self.run()]
+
+    def _wrap_mp(self, input_kwargs: dict) -> Any:
+        # Allow mapping of kwargs via multiprocessing map()
+        return self.run(**input_kwargs)
+
+    def _init_logger(self, log_level: int):
+        # Init logger for each spawned process when using multiprocessing pool
+        make_default_logger(level=log_level)
+
+    @timed
+    def __call__(self, iterable: Optional[iterable_input] = None) -> Any:
+        iterable = iterable or []
+        if iterable:
+            logger.info(
+                f"Calling {self.__class__.__name__} " f"with {len(iterable)} docs",  # type: ignore
+            )
+        if self.context.async_supported and self.process.is_async():
+            return self.process_async(iterable=iterable)
+        if self.context.mp_supported:
+            return self.process_multiprocess(iterable=iterable)
+        return self.process_serially(iterable=iterable)
+
+    def _run(self, fn: Callable, **kwargs: Any) -> Optional[Any]:
+        return self.asyncio_run(fn=self.run_async, _fn=fn, **kwargs)
+
+    async def _run_async(self, fn: Callable, **kwargs: Any) -> Optional[Any]:
+        raise NotImplementedError
+
+    def run(self, _fn: Optional[Callable] = None, **kwargs: Any) -> Optional[Any]:
+        try:
+            fn = _fn or self.process.run
+            return self._run(fn=fn, **kwargs)
+        except Exception as e:
+            logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
+            if "file_data_path" in kwargs:
+                self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
+            if self.context.raise_on_error:
+                raise e
+            return None
+
+    async def run_async(self, _fn: Optional[Callable] = None, **kwargs: Any) -> Optional[Any]:
+        try:
+            fn = _fn or self.process.run_async
+            return await self._run_async(fn=fn, **kwargs)
+        except Exception as e:
+            logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
+            if "file_data_path" in kwargs:
+                self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
+            if self.context.raise_on_error:
+                raise e
+            return None
+
+    @property
+    def cache_dir(self) -> Path:
+        return Path(self.context.work_dir) / self.identifier