unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/v2/cli/base/dest.py

@@ -1,85 +0,0 @@
-import logging
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base.cmd import BaseCmd
-from unstructured_ingest.v2.cli.utils.click import Dict, conform_click_options
-from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
-
-
-@dataclass
-class DestCmd(BaseCmd):
-    registry_entry: DestinationRegistryEntry
-
-    def get_registry_options(self):
-        options = []
-        configs = [
-            config
-            for config in [
-                self.registry_entry.uploader_config,
-                self.registry_entry.upload_stager_config,
-                self.registry_entry.connection_config,
-            ]
-            if config
-        ]
-        for config in configs:
-            options.extend(options_from_base_model(model=config))
-        options = self.consolidate_options(options=options)
-        return options
-
-    def cmd(self, ctx: click.Context, **options) -> None:
-        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
-        if not ctx.parent:
-            raise click.ClickException("destination command called without a parent")
-        if not ctx.parent.info_name:
-            raise click.ClickException("parent command missing info name")
-        source_cmd = ctx.parent.info_name.replace("-", "_")
-        source_options: dict = ctx.parent.params if ctx.parent else {}
-        conform_click_options(options)
-        try:
-            pipeline = self.get_pipeline(
-                src=source_cmd,
-                source_options=source_options,
-                dest=self.cmd_name,
-                destination_options=options,
-            )
-            pipeline.run()
-        except Exception as e:
-            logger.error(f"failed to run destination command {self.cmd_name}: {e}", exc_info=True)
-            raise click.ClickException(str(e)) from e
-
-    def get_cmd(self) -> click.Command:
-        # Dynamically create the command without the use of click decorators
-        fn = self.cmd
-        fn = click.pass_context(fn)
-        cmd = click.command(fn)
-        if not isinstance(cmd, click.core.Command):
-            raise ValueError(f"generated command was not of expected type Command: {type(cmd)}")
-        cmd.name = self.cli_cmd_name
-        cmd.short_help = "v2"
-        cmd.invoke_without_command = True
-        self.add_options(cmd)
-        cmd.params.append(
-            click.Option(
-                ["--custom-stager"],
-                required=False,
-                type=str,
-                default=None,
-                help="Pass a pointer to a custom upload stager to use, "
-                "must be in format '<module>:<attribute>'",
-            )
-        )
-        cmd.params.append(
-            click.Option(
-                ["--custom-stager-config-kwargs"],
-                required=False,
-                type=Dict(),
-                default=None,
-                help="Any kwargs to instantiate the configuration "
-                "associated with the customer stager",
-            )
-        )
-        return cmd

unstructured_ingest/v2/cli/base/src.py

@@ -1,85 +0,0 @@
-import logging
-from dataclasses import dataclass, field
-from typing import Any
-
-import click
-from pydantic import BaseModel
-
-from unstructured_ingest.v2.cli.base.cmd import BaseCmd
-from unstructured_ingest.v2.cli.utils.click import Group, conform_click_options
-from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model
-from unstructured_ingest.v2.interfaces import ProcessorConfig
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes import (
-    ChunkerConfig,
-    EmbedderConfig,
-    FiltererConfig,
-    PartitionerConfig,
-)
-from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
-
-
-@dataclass
-class SrcCmd(BaseCmd):
-    registry_entry: SourceRegistryEntry
-    default_configs: list[BaseModel] = field(
-        default_factory=lambda: [
-            ProcessorConfig,
-            PartitionerConfig,
-            EmbedderConfig,
-            FiltererConfig,
-            ChunkerConfig,
-        ]
-    )
-
-    def get_registry_options(self):
-        options = []
-        configs = [
-            config
-            for config in [
-                self.registry_entry.connection_config,
-                self.registry_entry.indexer_config,
-                self.registry_entry.downloader_config,
-            ]
-            if config
-        ]
-        for config in configs:
-            options.extend(options_from_base_model(model=config))
-        options = self.consolidate_options(options=options)
-        return options
-
-    def cmd(self, ctx: click.Context, **options: dict[str, Any]) -> None:
-        if ctx.invoked_subcommand:
-            return
-
-        conform_click_options(options)
-        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
-        try:
-            pipeline = self.get_pipeline(src=self.cmd_name, source_options=options)
-            pipeline.run()
-        except Exception as e:
-            logger.error(f"failed to run source command {self.cmd_name}: {e}", exc_info=True)
-            raise click.ClickException(str(e)) from e
-
-    def get_cmd(self) -> click.Group:
-        # Dynamically create the command without the use of click decorators
-        fn = self.cmd
-        fn = click.pass_context(fn)
-        cmd = click.group(fn, cls=Group)
-        if not isinstance(cmd, click.core.Group):
-            raise ValueError(f"generated src command was not of expected type Group: {type(cmd)}")
-        cmd.name = self.cli_cmd_name
-        cmd.short_help = "v2"
-        cmd.invoke_without_command = True
-        self.add_options(cmd)
-
-        # TODO remove after v1 no longer supported
-        cmd.params.append(
-            click.Option(
-                ["--output-dir"],
-                required=False,
-                type=str,
-                help="Local path to write partitioned output to",
-            )
-        )
-        return cmd

unstructured_ingest/v2/cli/cli.py

@@ -1,24 +0,0 @@
-import click
-
-from unstructured_ingest.v2.cli.cmds import dest, src
-
-
-@click.group()
-def ingest():
-    pass
-
-
-def get_cmd() -> click.Command:
-    """Construct and return a Click command object representing the main command for the CLI.
-
-    This function adds all dest_subcommand(s) to each src_subcommand, and adds all of those
-    to the main command as nested subcommands.
-    """
-    cmd = ingest
-    # Add all subcommands
-    for src_subcommand in src:
-        # Add all destination subcommands
-        for dest_subcommand in dest:
-            src_subcommand.add_command(dest_subcommand)
-        cmd.add_command(src_subcommand)
-    return cmd

unstructured_ingest/v2/logger.py

@@ -1,126 +0,0 @@
-import ast
-import json
-import os
-from logging import Formatter, Logger, StreamHandler, getLevelName, getLogger
-from typing import Any, Callable
-
-log_level = os.getenv("INGEST_LOG_LEVEL", "INFO")
-LOGGER_NAME = "unstructured_ingest.v2"
-
-
-def default_is_data_sensitive(k: str, v: Any) -> bool:
-    sensitive_fields = [
-        "account_name",
-        "client_id",
-    ]
-    sensitive_triggers = ["key", "cred", "token", "password", "oauth", "secret"]
-    return (
-        v
-        and any([s in k.lower() for s in sensitive_triggers])  # noqa: C419
-        or k.lower() in sensitive_fields
-    )
-
-
-def hide_sensitive_fields(
-    data: dict, is_sensitive_fn: Callable[[str, Any], bool] = default_is_data_sensitive
-) -> dict:
-    """
-    Will recursively look through every k, v pair in this dict and any nested ones and run
-    is_sensitive_fn to dynamically redact the value of the k, v pair. Will also check if
-    any string value can be parsed as valid json and process that dict as well and replace
-    the original string with the json.dumps() version of the redacted dict.
-    """
-    new_data = data.copy()
-    for k, v in new_data.items():
-        if is_sensitive_fn(k, v):
-            new_data[k] = "*******"
-        if isinstance(v, dict):
-            new_data[k] = hide_sensitive_fields(v)
-        if isinstance(v, str):
-            # Need to take into account strings generated via json.dumps() or simply printing a dict
-            try:
-                json_data = json.loads(v)
-                if isinstance(json_data, dict):
-                    updated_data = hide_sensitive_fields(json_data)
-                    new_data[k] = json.dumps(updated_data)
-            except json.JSONDecodeError:
-                pass
-
-    return new_data
-
-
-def redact_jsons(s: str) -> str:
-    """
-    Takes in a generic string and pulls out all valid json content. Leverages
-    hide_sensitive_fields() to redact any sensitive information and replaces the
-    original json with the new redacted format. There can be any number of valid
-    jsons in a generic string and this will work. Having extra '{' without a
-    closing '}' will cause this to break though. i.e '{ text, {"a": 3}'.
-
-    """
-    chars = list(s)
-    if "{" not in chars:
-        return s
-    i = 0
-    jsons = []
-    i = 0
-    while i < len(chars):
-        char = chars[i]
-        if char == "{":
-            stack = [char]
-            current = [char]
-            while len(stack) != 0 and i < len(chars):
-                i += 1
-                char = chars[i]
-                current.append(char)
-                if char == "{":
-                    stack.append(char)
-                if char == "}":
-                    stack.pop(-1)
-            jsons.append("".join(current))
-            continue
-        i += 1
-    for j in jsons:
-        try:
-            formatted_j = json.dumps(json.loads(j))
-        except json.JSONDecodeError:
-            lit = ast.literal_eval(j)
-            formatted_j = json.dumps(lit)
-        hidden_j = json.dumps(hide_sensitive_fields(json.loads(formatted_j)))
-        s = s.replace(j, hidden_j)
-    return s
-
-
-class SensitiveFormatter(Formatter):
-    def format(self, record):
-        s = super().format(record=record)
-        try:
-            return redact_jsons(s)
-        except Exception:
-            return f"Failed to redact: {s}"
-
-
-def remove_root_handlers(logger: Logger) -> None:
-    # NOTE(robinson): in some environments such as Google Colab, there is a root handler
-    # that doesn't not mask secrets, meaning sensitive info such as api keys appear in logs.
-    # Removing these when they exist prevents this behavior
-    if logger.root.hasHandlers():
-        for handler in logger.root.handlers:
-            logger.root.removeHandler(handler)
-
-
-def make_default_logger(level: int) -> Logger:
-    """Return a custom logger."""
-    logger = getLogger(LOGGER_NAME)
-    handler = StreamHandler()
-    handler.name = "ingest_log_handler"
-    formatter = SensitiveFormatter("%(asctime)s %(processName)-10s %(levelname)-8s %(message)s")
-    handler.setFormatter(formatter)
-    if handler.name not in [h.name for h in logger.handlers]:
-        logger.addHandler(handler)
-    logger.setLevel(level)
-    remove_root_handlers(logger)
-    return logger
-
-
-logger = make_default_logger(level=getLevelName(log_level.upper()))

unstructured_ingest/v2/main.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python3
-from unstructured_ingest.v2.cli.cli import get_cmd
-
-
-def main():
-    ingest_cmd = get_cmd()
-    ingest_cmd()
-
-
-if __name__ == "__main__":
-    main()

unstructured_ingest/v2/pipeline/interfaces.py

@@ -1,211 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import logging
-import multiprocessing as mp
-import shutil
-from abc import ABC, abstractmethod
-from concurrent.futures import ThreadPoolExecutor
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Awaitable, Callable, Optional, TypeVar
-
-from tqdm import tqdm
-from tqdm.asyncio import tqdm as tqdm_asyncio
-
-from unstructured_ingest.v2.interfaces import BaseProcess, ProcessorConfig, Uploader
-from unstructured_ingest.v2.logger import logger, make_default_logger
-from unstructured_ingest.v2.otel import OtelHandler
-from unstructured_ingest.v2.pipeline.otel import instrument
-
-BaseProcessT = TypeVar("BaseProcessT", bound=BaseProcess)
-iterable_input = list[dict[str, Any]]
-
-
-@dataclass
-class PipelineStep(ABC):
-    process: BaseProcessT
-    context: ProcessorConfig
-    identifier: str
-
-    def __str__(self):
-        return self.identifier
-
-    def process_serially(self, iterable: iterable_input) -> Any:
-        logger.info("processing content serially")
-        if iterable:
-            if len(iterable) == 1:
-                return [self.run(**iterable[0])]
-            if self.context.tqdm:
-                return [self.run(**it) for it in tqdm(iterable, desc=self.identifier)]
-            return [self.run(**it) for it in iterable]
-        return [self.run()]
-
-    async def _process_async(self, iterable: iterable_input) -> Any:
-        if iterable:
-            if len(iterable) == 1:
-                return [await self.run_async(**iterable[0])]
-            if self.context.tqdm:
-                return await tqdm_asyncio.gather(
-                    *[self.run_async(**i) for i in iterable], desc=self.identifier
-                )
-            return await asyncio.gather(*[self.run_async(**i) for i in iterable])
-        return [await self.run_async()]
-
-    def process_async(self, iterable: iterable_input) -> Any:
-        logger.info("processing content async")
-        return self.asyncio_run(fn=self._process_async, iterable=iterable)
-
-    def asyncio_run(
-        self, fn: Callable[[Any, Any], Awaitable[Any]], *args: Any, **kwargs: Any
-    ) -> Any:
-        current_loop = asyncio._get_running_loop()
-        if current_loop is None:
-            return asyncio.run(fn(*args, **kwargs))
-        with ThreadPoolExecutor(thread_name_prefix="asyncio") as thread_pool:
-            logger.warning(
-                f"async code being run in dedicated thread pool "
-                f"to not conflict with existing event loop: {current_loop}"
-            )
-
-            def wrapped():
-                return asyncio.run(fn(*args, **kwargs))
-
-            future = thread_pool.submit(wrapped)
-            return future.result()
-
-    def process_multiprocess(self, iterable: iterable_input) -> Any:
-        logger.info("processing content across processes")
-
-        if iterable:
-            if len(iterable) == 1:
-                return self.process_serially(iterable)
-            if self.context.num_processes == 1:
-                return self.process_serially(iterable)
-            with mp.Pool(
-                processes=self.context.num_processes,
-                initializer=self._init_mp,
-                initargs=(
-                    logging.DEBUG if self.context.verbose else logging.INFO,
-                    self.context.otel_endpoint,
-                ),
-            ) as pool:
-                otel_context = OtelHandler.inject_context()
-                for iter in iterable:
-                    iter[OtelHandler.trace_context_key] = otel_context
-                if self.context.tqdm:
-                    return list(
-                        tqdm(
-                            pool.imap_unordered(func=self._wrap_mp, iterable=iterable),
-                            total=len(iterable),
-                            desc=self.identifier,
-                        )
-                    )
-                return pool.map(self._wrap_mp, iterable)
-        return [self.run()]
-
-    def _wrap_mp(self, input_kwargs: dict) -> Any:
-        # Allow mapping of kwargs via multiprocessing map()
-        return self.run(**input_kwargs)
-
-    def _init_mp(self, log_level: int, endpoint: Optional[str] = None) -> None:
-        # Init logger for each spawned process when using multiprocessing pool
-        make_default_logger(level=log_level)
-        otel_handler = OtelHandler(otel_endpoint=endpoint, log_out=logger.debug)
-        otel_handler.init_trace()
-
-    @instrument()
-    def __call__(self, iterable: Optional[iterable_input] = None) -> Any:
-        iterable = iterable or []
-        if iterable:
-            logger.info(
-                f"calling {self.__class__.__name__} " f"with {len(iterable)} docs",  # type: ignore
-            )
-        else:
-            logger.info(f"calling {self.__class__.__name__} with no inputs")
-        if self.context.async_supported and self.process.is_async():
-            return self.process_async(iterable=iterable)
-        if self.context.mp_supported:
-            return self.process_multiprocess(iterable=iterable)
-        return self.process_serially(iterable=iterable)
-
-    def _run(self, fn: Callable, **kwargs: Any) -> Optional[Any]:
-        return self.asyncio_run(fn=self.run_async, _fn=fn, **kwargs)
-
-    async def _run_async(self, fn: Callable, **kwargs: Any) -> Optional[Any]:
-        raise NotImplementedError
-
-    def run(self, _fn: Callable[..., Any] | None = None, **kwargs: Any) -> Optional[Any]:
-        kwargs = kwargs.copy()
-        otel_handler = OtelHandler(otel_endpoint=self.context.otel_endpoint, log_out=logger.debug)
-        tracer = otel_handler.get_tracer()
-        if trace_context := kwargs.pop(otel_handler.trace_context_key, {}):
-            otel_handler.attach_context(trace_context=trace_context)
-        attributes = {}
-        if file_data_path := kwargs.get("file_data_path"):
-            attributes["file_id"] = Path(file_data_path).stem
-        try:
-            with tracer.start_as_current_span(self.identifier, record_exception=True) as span:
-                otel_handler.set_attributes(span, attributes)
-                fn = _fn or self.process.run
-                return self._run(fn=fn, **kwargs)
-        except Exception as e:
-            logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
-            if "file_data_path" in kwargs:
-                self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
-            if self.context.raise_on_error:
-                raise e
-            return None
-
-    async def run_async(self, _fn: Optional[Callable] = None, **kwargs: Any) -> Optional[Any]:
-        otel_handler = OtelHandler(otel_endpoint=self.context.otel_endpoint, log_out=logger.debug)
-        try:
-            attributes = {}
-            if file_data_path := kwargs.get("file_data_path"):
-                attributes["file_id"] = Path(file_data_path).stem
-            with otel_handler.get_tracer().start_as_current_span(
-                self.identifier, record_exception=True
-            ) as span:
-                otel_handler.set_attributes(span, attributes)
-                fn = _fn or self.process.run_async
-                return await self._run_async(fn=fn, **kwargs)
-        except Exception as e:
-            logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
-            if "file_data_path" in kwargs:
-                self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
-            if self.context.raise_on_error:
-                raise e
-            return None
-
-    @property
-    def cache_dir(self) -> Path:
-        return Path(self.context.work_dir) / self.identifier
-
-    def delete_cache(self):
-        if self.context.iter_delete and self.cache_dir.exists():
-            cache_dir = self.cache_dir
-            logger.info(f"deleting {self.identifier} cache dir {cache_dir}")
-            shutil.rmtree(cache_dir)
-
-
-@dataclass
-class BatchPipelineStep(PipelineStep, ABC):
-    process: Uploader
-
-    def __call__(self, iterable: Optional[iterable_input] = None) -> Any:
-        if self.context.mp_supported and self.process.is_batch():
-            return self.run_batch(contents=iterable)
-        super().__call__(iterable=iterable)
-
-    @abstractmethod
-    def _run_batch(self, contents: iterable_input, **kwargs) -> Any:
-        pass
-
-    def run_batch(self, contents: iterable_input, **kwargs) -> Any:
-        try:
-            return self._run_batch(contents=contents, **kwargs)
-        except Exception as e:
-            self.context.status[self.identifier] = {"step_error": str(e)}
-            if self.context.raise_on_error:
-                raise e
-            return None