unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/embed/octoai.py

@@ -8,9 +8,7 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import (
+from unstructured_ingest.errors_v2 import (
     ProviderError,
     QuotaError,
     RateLimitError,
@@ -18,6 +16,8 @@ from unstructured_ingest.v2.errors import (
     UserError,
     is_internal_error,
 )
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from openai import AsyncOpenAI, OpenAI

unstructured_ingest/embed/openai.py

@@ -8,9 +8,7 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import (
+from unstructured_ingest.errors_v2 import (
     ProviderError,
     QuotaError,
     RateLimitError,
@@ -18,6 +16,8 @@ from unstructured_ingest.v2.errors import (
     UserError,
     is_internal_error,
 )
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from openai import AsyncOpenAI, OpenAI

unstructured_ingest/embed/togetherai.py

@@ -8,12 +8,12 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import (
+from unstructured_ingest.errors_v2 import (
     RateLimitError as CustomRateLimitError,
 )
-from unstructured_ingest.v2.errors import UserAuthError, UserError, is_internal_error
+from unstructured_ingest.errors_v2 import UserAuthError, UserError, is_internal_error
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from together import AsyncTogether, Together

unstructured_ingest/embed/vertexai.py

@@ -13,8 +13,8 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
+from unstructured_ingest.errors_v2 import UserAuthError, is_internal_error
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import UserAuthError, is_internal_error
 
 if TYPE_CHECKING:
     from vertexai.language_models import TextEmbeddingModel

unstructured_ingest/embed/voyageai.py

@@ -8,12 +8,12 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError, is_internal_error
-from unstructured_ingest.v2.errors import (
+from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError, is_internal_error
+from unstructured_ingest.errors_v2 import (
     RateLimitError as CustomRateLimitError,
 )
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from voyageai import AsyncClient as AsyncVoyageAIClient

unstructured_ingest/{v2/interfaces → interfaces}/downloader.py

@@ -5,9 +5,9 @@ from typing import Any, Optional, TypedDict, TypeVar, Union
 
 from pydantic import BaseModel, Field
 
-from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces.connector import BaseConnector
+from unstructured_ingest.interfaces.process import BaseProcess
 
 
 class DownloaderConfig(BaseModel):

unstructured_ingest/{v2/interfaces → interfaces}/indexer.py

@@ -3,9 +3,9 @@ from typing import Any, AsyncGenerator, Generator, Optional, TypeVar
 
 from pydantic import BaseModel
 
-from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.process import BaseProcess
-from unstructured_ingest.v2.types.file_data import FileData
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces.connector import BaseConnector
+from unstructured_ingest.interfaces.process import BaseProcess
 
 
 class IndexerConfig(BaseModel):

unstructured_ingest/{v2/interfaces → interfaces}/upload_stager.py

@@ -5,10 +5,10 @@ from typing import Any, TypeVar
 
 from pydantic import BaseModel
 
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces import BaseProcess
 from unstructured_ingest.utils import ndjson
 from unstructured_ingest.utils.data_prep import get_data, write_data
-from unstructured_ingest.v2.interfaces import BaseProcess
-from unstructured_ingest.v2.types.file_data import FileData
 
 
 class UploadStagerConfig(BaseModel):

unstructured_ingest/{v2/interfaces → interfaces}/uploader.py

@@ -5,9 +5,9 @@ from typing import Any, TypeVar
 
 from pydantic import BaseModel
 
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces import BaseConnector, BaseProcess
 from unstructured_ingest.utils.data_prep import get_data
-from unstructured_ingest.v2.interfaces import BaseConnector, BaseProcess
-from unstructured_ingest.v2.types.file_data import FileData
 
 
 class UploaderConfig(BaseModel):

unstructured_ingest/{v2/otel.py → otel.py}

@@ -13,7 +13,7 @@ from opentelemetry.sdk.trace.export import (
     SpanExportResult,
 )
 
-from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.logger import logger
 
 
 class AddTraceCallable(Protocol):

unstructured_ingest/pipeline/__init__.py

@@ -1,22 +0,0 @@
-from .doc_factory import DocFactory
-from .interfaces import PipelineContext, ReformatNode
-from .partition import Partitioner
-from .permissions import PermissionsDataCleaner
-from .pipeline import Pipeline
-from .reformat.chunking import Chunker
-from .reformat.embedding import Embedder
-from .source import Reader
-from .write import Writer
-
-__all__ = [
-    "DocFactory",
-    "Partitioner",
-    "Reader",
-    "Embedder",
-    "PipelineContext",
-    "Pipeline",
-    "Writer",
-    "Chunker",
-    "ReformatNode",
-    "PermissionsDataCleaner",
-]

unstructured_ingest/pipeline/interfaces.py

@@ -1,270 +1,211 @@
-import hashlib
-import json
+from __future__ import annotations
+
+import asyncio
 import logging
 import multiprocessing as mp
-import typing as t
+import shutil
 from abc import ABC, abstractmethod
-from dataclasses import dataclass, field
-from multiprocessing.managers import DictProxy
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Any, Awaitable, Callable, Optional, TypeVar
 
-from dataclasses_json import DataClassJsonMixin
+from tqdm import tqdm
+from tqdm.asyncio import tqdm as tqdm_asyncio
 
-from unstructured_ingest.error import SourceConnectionNetworkError
-from unstructured_ingest.interfaces import (
-    BaseDestinationConnector,
-    BaseSourceConnector,
-    PartitionConfig,
-    ProcessorConfig,
-    ReadConfig,
-    RetryStrategyConfig,
-)
-from unstructured_ingest.logger import ingest_log_streaming_init, logger
+from unstructured_ingest.interfaces import BaseProcess, ProcessorConfig, Uploader
+from unstructured_ingest.logger import logger, make_default_logger
+from unstructured_ingest.otel import OtelHandler
+from unstructured_ingest.pipeline.otel import instrument
 
-if t.TYPE_CHECKING:
-    from unstructured_ingest.ingest_backoff import RetryHandler
+BaseProcessT = TypeVar("BaseProcessT", bound=BaseProcess)
+iterable_input = list[dict[str, Any]]
 
 
 @dataclass
-class PipelineContext(ProcessorConfig):
-    """
-    Data that gets shared across each pipeline node
-    """
-
-    def __post_init__(self):
-        self._ingest_docs_map: t.Optional[DictProxy] = None
+class PipelineStep(ABC):
+    process: BaseProcessT
+    context: ProcessorConfig
+    identifier: str
 
-    @property
-    def ingest_docs_map(self) -> DictProxy:
-        if self._ingest_docs_map is None:
-            raise ValueError("ingest_docs_map never initialized")
-        return self._ingest_docs_map
+    def __str__(self):
+        return self.identifier
 
-    @ingest_docs_map.setter
-    def ingest_docs_map(self, value: DictProxy):
-        self._ingest_docs_map = value
+    def process_serially(self, iterable: iterable_input) -> Any:
+        logger.info("processing content serially")
+        if iterable:
+            if len(iterable) == 1:
+                return [self.run(**iterable[0])]
+            if self.context.tqdm:
+                return [self.run(**it) for it in tqdm(iterable, desc=self.identifier)]
+            return [self.run(**it) for it in iterable]
+        return [self.run()]
+
+    async def _process_async(self, iterable: iterable_input) -> Any:
+        if iterable:
+            if len(iterable) == 1:
+                return [await self.run_async(**iterable[0])]
+            if self.context.tqdm:
+                return await tqdm_asyncio.gather(
+                    *[self.run_async(**i) for i in iterable], desc=self.identifier
+                )
+            return await asyncio.gather(*[self.run_async(**i) for i in iterable])
+        return [await self.run_async()]
+
+    def process_async(self, iterable: iterable_input) -> Any:
+        logger.info("processing content async")
+        return self.asyncio_run(fn=self._process_async, iterable=iterable)
+
+    def asyncio_run(
+        self, fn: Callable[[Any, Any], Awaitable[Any]], *args: Any, **kwargs: Any
+    ) -> Any:
+        current_loop = asyncio._get_running_loop()
+        if current_loop is None:
+            return asyncio.run(fn(*args, **kwargs))
+        with ThreadPoolExecutor(thread_name_prefix="asyncio") as thread_pool:
+            logger.warning(
+                f"async code being run in dedicated thread pool "
+                f"to not conflict with existing event loop: {current_loop}"
+            )
 
+            def wrapped():
+                return asyncio.run(fn(*args, **kwargs))
 
-@dataclass
-class PipelineNode(DataClassJsonMixin, ABC):
-    """
-    Class that encapsulates logic to run during a single pipeline step
-    """
+            future = thread_pool.submit(wrapped)
+            return future.result()
 
-    pipeline_context: PipelineContext
+    def process_multiprocess(self, iterable: iterable_input) -> Any:
+        logger.info("processing content across processes")
 
-    def __call__(self, iterable: t.Optional[t.Iterable[t.Any]] = None) -> t.Any:
-        iterable = iterable if iterable else []
+        if iterable:
+            if len(iterable) == 1:
+                return self.process_serially(iterable)
+            if self.context.num_processes == 1:
+                return self.process_serially(iterable)
+            with mp.Pool(
+                processes=self.context.num_processes,
+                initializer=self._init_mp,
+                initargs=(
+                    logging.DEBUG if self.context.verbose else logging.INFO,
+                    self.context.otel_endpoint,
+                ),
+            ) as pool:
+                otel_context = OtelHandler.inject_context()
+                for iter in iterable:
+                    iter[OtelHandler.trace_context_key] = otel_context
+                if self.context.tqdm:
+                    return list(
+                        tqdm(
+                            pool.imap_unordered(func=self._wrap_mp, iterable=iterable),
+                            total=len(iterable),
+                            desc=self.identifier,
+                        )
+                    )
+                return pool.map(self._wrap_mp, iterable)
+        return [self.run()]
+
+    def _wrap_mp(self, input_kwargs: dict) -> Any:
+        # Allow mapping of kwargs via multiprocessing map()
+        return self.run(**input_kwargs)
+
+    def _init_mp(self, log_level: int, endpoint: Optional[str] = None) -> None:
+        # Init logger for each spawned process when using multiprocessing pool
+        make_default_logger(level=log_level)
+        otel_handler = OtelHandler(otel_endpoint=endpoint, log_out=logger.debug)
+        otel_handler.init_trace()
+
+    @instrument()
+    def __call__(self, iterable: Optional[iterable_input] = None) -> Any:
+        iterable = iterable or []
         if iterable:
             logger.info(
                 f"calling {self.__class__.__name__} " f"with {len(iterable)} docs",  # type: ignore
             )
-
-        self.initialize()
-        if not self.supported_multiprocessing():
-            if iterable:
-                self.result = self.run(iterable)
-            else:
-                self.result = self.run()
-        elif self.pipeline_context.num_processes == 1:
-            if iterable:
-                self.result = [self.run(it) for it in iterable]
-            else:
-                self.result = self.run()
         else:
-            with mp.Pool(
-                processes=self.pipeline_context.num_processes,
-                initializer=ingest_log_streaming_init,
-                initargs=(logging.DEBUG if self.pipeline_context.verbose else logging.INFO,),
-            ) as pool:
-                self.result = pool.map(self.run, iterable)
-        # Remove None which may be caused by failed docs that didn't raise an error
-        if isinstance(self.result, t.Iterable):
-            self.result = [r for r in self.result if r is not None]
-        return self.result
-
-    def supported_multiprocessing(self) -> bool:
-        return True
-
-    @abstractmethod
-    def run(self, *args, **kwargs) -> t.Optional[t.Any]:
-        pass
-
-    def initialize(self):
-        if path := self.get_path():
-            logger.info(f"creating {path}")
-            path.mkdir(parents=True, exist_ok=True)
-        ingest_log_streaming_init(logging.DEBUG if self.pipeline_context.verbose else logging.INFO)
-
-    def get_path(self) -> t.Optional[Path]:
-        return None
-
-
-@dataclass
-class DocFactoryNode(PipelineNode):
-    """
-    Encapsulated logic to generate a list of ingest docs
-    """
-
-    source_doc_connector: BaseSourceConnector
-
-    def initialize(self):
-        logger.info(
-            f"Running doc factory to generate ingest docs. "
-            f"Source connector: {self.source_doc_connector.to_json()}",
-        )
-        super().initialize()
-        self.source_doc_connector.initialize()
-
-    @abstractmethod
-    def run(self, *args, **kwargs) -> t.Iterable[dict]:
-        pass
-
-    def supported_multiprocessing(self) -> bool:
-        return False
-
-
-@dataclass
-class SourceNode(PipelineNode):
-    """A pipeline node representing logic to pull data from a source using base ingest documents.
-
-    This class encapsulates the logic for pulling data from a specified source using base ingest
-    documents. The output of this logic is expected to be in JSON format representing the data
-    itself.
-
-    Attributes:
-        read_config: A configuration object specifying how to read data from the source.
-        retry_strategy_config: Optional configuration specifying the strategy for network errors.
-
-    Properties:
-        retry_strategy: A retry handler configured based on the retry strategy configuration.
-
-    Methods:
-        initialize: Initializes the source node and logs the process.
-        run: Abstract method for downloading data associated with ingest documents.
-    """
-
-    read_config: ReadConfig
-    retry_strategy_config: t.Optional[RetryStrategyConfig] = None
+            logger.info(f"calling {self.__class__.__name__} with no inputs")
+        if self.context.async_supported and self.process.is_async():
+            return self.process_async(iterable=iterable)
+        if self.context.mp_supported:
+            return self.process_multiprocess(iterable=iterable)
+        return self.process_serially(iterable=iterable)
+
+    def _run(self, fn: Callable, **kwargs: Any) -> Optional[Any]:
+        return self.asyncio_run(fn=self.run_async, _fn=fn, **kwargs)
+
+    async def _run_async(self, fn: Callable, **kwargs: Any) -> Optional[Any]:
+        raise NotImplementedError
+
+    def run(self, _fn: Callable[..., Any] | None = None, **kwargs: Any) -> Optional[Any]:
+        kwargs = kwargs.copy()
+        otel_handler = OtelHandler(otel_endpoint=self.context.otel_endpoint, log_out=logger.debug)
+        tracer = otel_handler.get_tracer()
+        if trace_context := kwargs.pop(otel_handler.trace_context_key, {}):
+            otel_handler.attach_context(trace_context=trace_context)
+        attributes = {}
+        if file_data_path := kwargs.get("file_data_path"):
+            attributes["file_id"] = Path(file_data_path).stem
+        try:
+            with tracer.start_as_current_span(self.identifier, record_exception=True) as span:
+                otel_handler.set_attributes(span, attributes)
+                fn = _fn or self.process.run
+                return self._run(fn=fn, **kwargs)
+        except Exception as e:
+            logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
+            if "file_data_path" in kwargs:
+                self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
+            if self.context.raise_on_error:
+                raise e
+            return None
+
+    async def run_async(self, _fn: Optional[Callable] = None, **kwargs: Any) -> Optional[Any]:
+        otel_handler = OtelHandler(otel_endpoint=self.context.otel_endpoint, log_out=logger.debug)
+        try:
+            attributes = {}
+            if file_data_path := kwargs.get("file_data_path"):
+                attributes["file_id"] = Path(file_data_path).stem
+            with otel_handler.get_tracer().start_as_current_span(
+                self.identifier, record_exception=True
+            ) as span:
+                otel_handler.set_attributes(span, attributes)
+                fn = _fn or self.process.run_async
+                return await self._run_async(fn=fn, **kwargs)
+        except Exception as e:
+            logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
+            if "file_data_path" in kwargs:
+                self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
+            if self.context.raise_on_error:
+                raise e
+            return None
 
     @property
-    def retry_strategy(self) -> t.Optional["RetryHandler"]:
-        if retry_strategy_config := self.retry_strategy_config:
-            import backoff
+    def cache_dir(self) -> Path:
+        return Path(self.context.work_dir) / self.identifier
 
-            from unstructured_ingest.ingest_backoff import RetryHandler
-
-            return RetryHandler(
-                backoff.expo,
-                SourceConnectionNetworkError,
-                max_time=retry_strategy_config.max_retry_time,
-                max_tries=retry_strategy_config.max_retries,
-                logger=logger,
-                start_log_level=logger.level,
-                backoff_log_level=logger.level,
-            )
-        return None
-
-    def initialize(self):
-        logger.info("Running source node to download data associated with ingest docs")
-        super().initialize()
-
-    @abstractmethod
-    def run(self, ingest_doc_json: str) -> t.Optional[str]:
-        pass
-
-
-@dataclass
-class PartitionNode(PipelineNode):
-    """
-    Encapsulates logic to run partition on the json files as the output of the source node
-    """
-
-    partition_config: PartitionConfig
-    partition_kwargs: dict = field(default_factory=dict)
-
-    def initialize(self):
-        logger.info(
-            f"Running partition node to extract content from json files. "
-            f"Config: {self.partition_config.to_json()}, "
-            f"partition kwargs: {json.dumps(self.partition_kwargs)}]",
-        )
-        super().initialize()
-
-    def create_hash(self) -> str:
-        hash_dict = self.partition_config.to_dict()
-        hash_dict["partition_kwargs"] = self.partition_kwargs
-        return hashlib.sha256(json.dumps(hash_dict, sort_keys=True).encode()).hexdigest()[:32]
-
-    @abstractmethod
-    def run(self, json_path: str) -> t.Optional[str]:
-        pass
-
-    def get_path(self) -> Path:
-        return (Path(self.pipeline_context.work_dir) / "partitioned").resolve()
-
-
-@dataclass
-class ReformatNode(PipelineNode, ABC):
-    """
-    Encapsulated any logic to reformat the output List[Element]
-    content from partition before writing it
-    """
-
-    @abstractmethod
-    def run(self, elements_json: str) -> t.Optional[str]:
-        pass
-
-
-@dataclass
-class WriteNode(PipelineNode):
-    """
-    Encapsulated logic to write the final result to a downstream data connection
-    """
-
-    dest_doc_connector: BaseDestinationConnector
-
-    @abstractmethod
-    def run(self, json_paths: t.List[str]):
-        pass
-
-    def initialize(self):
-        logger.info(
-            f"Running write node to upload content. "
-            f"Destination connector: {self.dest_doc_connector.to_json(redact_sensitive=True)}]",
-        )
-        super().initialize()
-        self.dest_doc_connector.initialize()
-
-    def supported_multiprocessing(self) -> bool:
-        return False
+    def delete_cache(self):
+        if self.context.iter_delete and self.cache_dir.exists():
+            cache_dir = self.cache_dir
+            logger.info(f"deleting {self.identifier} cache dir {cache_dir}")
+            shutil.rmtree(cache_dir)
 
 
 @dataclass
-class CopyNode(PipelineNode):
-    """
-    Encapsulated logic to copy the final result of the pipeline to the designated output location.
-    """
+class BatchPipelineStep(PipelineStep, ABC):
+    process: Uploader
 
-    def initialize(self):
-        logger.info("Running copy node to move content to desired output location")
-        super().initialize()
+    def __call__(self, iterable: Optional[iterable_input] = None) -> Any:
+        if self.context.mp_supported and self.process.is_batch():
+            return self.run_batch(contents=iterable)
+        super().__call__(iterable=iterable)
 
     @abstractmethod
-    def run(self, json_path: str):
+    def _run_batch(self, contents: iterable_input, **kwargs) -> Any:
         pass
 
-
-@dataclass
-class PermissionsNode(PipelineNode):
-    """
-    Encapsulated logic to do operations on permissions related data.
-    """
-
-    def initialize(self):
-        logger.info("Running permissions node to cleanup the permissions folder")
-        super().initialize()
-
-    @abstractmethod
-    def run(self):
-        pass
+    def run_batch(self, contents: iterable_input, **kwargs) -> Any:
+        try:
+            return self._run_batch(contents=contents, **kwargs)
+        except Exception as e:
+            self.context.status[self.identifier] = {"step_error": str(e)}
+            if self.context.raise_on_error:
+                raise e
+            return None

unstructured_ingest/{v2/pipeline → pipeline}/otel.py

@@ -1,8 +1,8 @@
 from functools import wraps
 from typing import Callable, Optional
 
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.otel import OtelHandler
+from unstructured_ingest.logger import logger
+from unstructured_ingest.otel import OtelHandler
 
 
 def instrument(