unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/error.py

@@ -0,0 +1,49 @@
+from abc import ABC
+from functools import wraps
+
+
+class CustomError(Exception, ABC):
+    error_string: str
+
+    @classmethod
+    def wrap(cls, f):
+        """
+        Provides a wrapper for a function that catches any exception and
+        re-raises it as the customer error. If the exception itself is already an instance
+        of the custom error, re-raises original error.
+        """
+
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except BaseException as error:
+                if not isinstance(error, cls) and not issubclass(type(error), cls):
+                    raise cls(cls.error_string.format(str(error))) from error
+                raise
+
+        return wrapper
+
+
+class SourceConnectionError(CustomError):
+    error_string = "Error in getting data from upstream data source: {}"
+
+
+class SourceConnectionNetworkError(SourceConnectionError):
+    error_string = "Error in connecting to upstream data source: {}"
+
+
+class DestinationConnectionError(CustomError):
+    error_string = "Error in connecting to downstream data source: {}"
+
+
+class EmbeddingEncoderConnectionError(CustomError):
+    error_string = "Error in connecting to the embedding model provider: {}"
+
+
+class WriteError(CustomError):
+    error_string = "Error in writing to downstream data source: {}"
+
+
+class PartitionError(CustomError):
+    error_string = "Error in partitioning content: {}"

unstructured_ingest/evaluate.py

@@ -0,0 +1,338 @@
+#! /usr/bin/env python3
+
+from typing import List, Optional, Tuple, Union
+
+import click
+from unstructured.metrics.evaluate import (
+    ElementTypeMetricsCalculator,
+    ObjectDetectionMetricsCalculator,
+    TableStructureMetricsCalculator,
+    TextExtractionMetricsCalculator,
+    filter_metrics,
+    get_mean_grouping,
+)
+
+
+@click.group()
+def main():
+    pass
+
+
+@main.command()
+@click.option("--output_dir", type=str, help="Directory to structured output.")
+@click.option("--source_dir", type=str, help="Directory to source.")
+@click.option(
+    "--output_list",
+    type=str,
+    multiple=True,
+    help="Optional: list of selected structured output file names under the \
+        directory to be evaluate. If none, all files under directory will be use.",
+)
+@click.option(
+    "--source_list",
+    type=str,
+    multiple=True,
+    help="Optional: list of selected source file names under the directory \
+        to be evaluate. If none, all files under directory will be use.",
+)
+@click.option(
+    "--export_dir",
+    type=str,
+    default="metrics",
+    help="Directory to save the output evaluation metrics to. Default to \
+        your/working/dir/metrics/",
+)
+@click.option("--group_by", type=str, help="Input field for aggregration, or leave blank if none.")
+@click.option(
+    "--weights",
+    type=(int, int, int),
+    default=(2, 1, 1),
+    show_default=True,
+    help="A list of weights to the Levenshtein distance calculation. Takes input as --weights 2 2 2\
+        See text_extraction.py/calculate_edit_distance for more details.",
+)
+@click.option(
+    "--visualize",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Add the flag to show progress bar.",
+)
+@click.option(
+    "--output_type",
+    type=str,
+    default="json",
+    show_default=True,
+    help="Takes in either `txt` or `json` as output_type.",
+)
+def measure_text_extraction_accuracy_command(
+    output_dir: str,
+    source_dir: str,
+    export_dir: str,
+    weights: Tuple[int, int, int],
+    visualize: bool,
+    output_type: str,
+    output_list: Optional[List[str]] = None,
+    source_list: Optional[List[str]] = None,
+    group_by: Optional[str] = None,
+):
+    return (
+        TextExtractionMetricsCalculator(
+            documents_dir=output_dir,
+            ground_truths_dir=source_dir,
+            group_by=group_by,
+            weights=weights,
+            document_type=output_type,
+        )
+        .on_files(document_paths=output_list, ground_truth_paths=source_list)
+        .calculate(export_dir=export_dir, visualize_progress=visualize, display_agg_df=True)
+    )
+
+
+@main.command()
+@click.option("--output_dir", type=str, help="Directory to structured output.")
+@click.option("--source_dir", type=str, help="Directory to structured source.")
+@click.option(
+    "--output_list",
+    type=str,
+    multiple=True,
+    help="Optional: list of selected structured output file names under the \
+        directory to be evaluate. If none, all files under directory will be used.",
+)
+@click.option(
+    "--source_list",
+    type=str,
+    multiple=True,
+    help="Optional: list of selected source file names under the directory \
+        to be evaluate. If none, all files under directory will be used.",
+)
+@click.option(
+    "--export_dir",
+    type=str,
+    default="metrics",
+    help="Directory to save the output evaluation metrics to. Default to \
+        your/working/dir/metrics/",
+)
+@click.option(
+    "--visualize",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Add the flag to show progress bar.",
+)
+def measure_element_type_accuracy_command(
+    output_dir: str,
+    source_dir: str,
+    export_dir: str,
+    visualize: bool,
+    output_list: Optional[List[str]] = None,
+    source_list: Optional[List[str]] = None,
+):
+    return (
+        ElementTypeMetricsCalculator(
+            documents_dir=output_dir,
+            ground_truths_dir=source_dir,
+        )
+        .on_files(document_paths=output_list, ground_truth_paths=source_list)
+        .calculate(export_dir=export_dir, visualize_progress=visualize, display_agg_df=True)
+    )
+
+
+@main.command()
+@click.option(
+    "--group_by",
+    type=str,
+    required=True,
+    help="The category to group by; valid values are 'doctype' and 'connector'.",
+)
+@click.option(
+    "--data_input",
+    type=str,
+    required=True,
+    help="A datafram or path to the CSV/TSV file containing the data",
+)
+@click.option(
+    "--export_dir",
+    type=str,
+    default="metrics",
+    help="Directory to save the output evaluation metrics to. Default to \
+        your/working/dir/metrics/",
+)
+@click.option(
+    "--eval_name",
+    type=str,
+    help="Evaluated metric. Expecting one of 'text_extraction' or 'element_type'",
+)
+@click.option(
+    "--agg_name",
+    type=str,
+    help="String to use with export filename. Default is `cct` for `text_extraction` \
+        and `element-type` for `element_type`",
+)
+@click.option(
+    "--export_filename", type=str, help="Optional. Define your file name for the output here."
+)
+def get_mean_grouping_command(
+    group_by: str,
+    data_input: str,
+    export_dir: str,
+    eval_name: str,
+    agg_name: Optional[str] = None,
+    export_filename: Optional[str] = None,
+):
+    return get_mean_grouping(
+        group_by=group_by,
+        data_input=data_input,
+        export_dir=export_dir,
+        eval_name=eval_name,
+        agg_name=agg_name,
+        export_filename=export_filename,
+    )
+
+
+@main.command()
+@click.option("--output_dir", type=str, help="Directory to structured output.")
+@click.option("--source_dir", type=str, help="Directory to structured source.")
+@click.option(
+    "--output_list",
+    type=str,
+    multiple=True,
+    help="Optional: list of selected structured output file names under the \
+        directory to be evaluate. If none, all files under directory will be used.",
+)
+@click.option(
+    "--source_list",
+    type=str,
+    multiple=True,
+    help="Optional: list of selected source file names under the directory \
+        to be evaluate. If none, all files under directory will be used.",
+)
+@click.option(
+    "--export_dir",
+    type=str,
+    default="metrics",
+    help="Directory to save the output evaluation metrics to. Default to \
+        your/working/dir/metrics/",
+)
+@click.option(
+    "--visualize",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Add the flag to show progress bar.",
+)
+@click.option(
+    "--cutoff",
+    type=float,
+    show_default=True,
+    default=0.8,
+    help="The cutoff value for the element level alignment. \
+        If not set, a default value is used",
+)
+def measure_table_structure_accuracy_command(
+    output_dir: str,
+    source_dir: str,
+    export_dir: str,
+    visualize: bool,
+    output_list: Optional[List[str]] = None,
+    source_list: Optional[List[str]] = None,
+    cutoff: Optional[float] = None,
+):
+    return (
+        TableStructureMetricsCalculator(
+            documents_dir=output_dir,
+            ground_truths_dir=source_dir,
+            cutoff=cutoff,
+        )
+        .on_files(document_paths=output_list, ground_truth_paths=source_list)
+        .calculate(export_dir=export_dir, visualize_progress=visualize, display_agg_df=True)
+    )
+
+
+@main.command()
+@click.option("--output_dir", type=str, help="Directory to structured output.")
+@click.option("--source_dir", type=str, help="Directory to structured source.")
+@click.option(
+    "--output_list",
+    type=str,
+    multiple=True,
+    help=(
+        "Optional: list of selected structured output file names under the "
+        "directory to be evaluated. If none, all files under directory will be used."
+    ),
+)
+@click.option(
+    "--source_list",
+    type=str,
+    multiple=True,
+    help="Optional: list of selected source file names under the directory \
+        to be evaluate. If none, all files under directory will be used.",
+)
+@click.option(
+    "--export_dir",
+    type=str,
+    default="metrics",
+    help="Directory to save the output evaluation metrics to. Default to \
+        your/working/dir/metrics/",
+)
+@click.option(
+    "--visualize",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help="Add the flag to show progress bar.",
+)
+def measure_object_detection_metrics_command(
+    output_dir: str,
+    source_dir: str,
+    export_dir: str,
+    visualize: bool,
+    output_list: Optional[List[str]] = None,
+    source_list: Optional[List[str]] = None,
+):
+    return (
+        ObjectDetectionMetricsCalculator(
+            documents_dir=output_dir,
+            ground_truths_dir=source_dir,
+        )
+        .on_files(document_paths=output_list, ground_truth_paths=source_list)
+        .calculate(export_dir=export_dir, visualize_progress=visualize, display_agg_df=True)
+    )
+
+
+@main.command()
+@click.option(
+    "--data_input", type=str, required=True, help="Takes in path to data file as .tsv .csv .txt"
+)
+@click.option(
+    "--filter_list",
+    type=str,
+    required=True,
+    help="Takes in list of string to filter the data_input.",
+)
+@click.option(
+    "--filter_by",
+    type=str,
+    required=True,
+    help="Field from data_input to match with filter_list. Default is `filename`.",
+)
+@click.option(
+    "--export_filename", type=str, help="Export filename. Required when return_type is `file`"
+)
+@click.option("--export_dir", type=str, help="Export directory.")
+@click.option("--return_type", type=str, help="`dataframe` or `file`. Default is `file`.")
+def filter_metrics_command(
+    data_input: str,
+    filter_list: Union[str, List[str]],
+    filter_by: str = "filename",
+    export_filename: Optional[str] = None,
+    export_dir: str = "metrics",
+    return_type: str = "file",
+):
+    return filter_metrics(
+        data_input, filter_list, filter_by, export_filename, export_dir, return_type
+    )
+
+
+if __name__ == "__main__":
+    main()

unstructured_ingest/ingest_backoff/__init__.py

@@ -0,0 +1,3 @@
+from ._wrapper import RetryHandler
+
+__all__ = ["RetryHandler"]

unstructured_ingest/ingest_backoff/_common.py

@@ -0,0 +1,102 @@
+import logging
+import sys
+import traceback
+
+
+# Default startup handler
+def _log_start(details, logger, log_level):
+    max_tried = details.get("max_tries")
+    max_time = details.get("max_time")
+    if max_tried is not None and max_time is not None:
+        s = "%.1fs or %d tries"
+        s_args = [max_time, max_tried]
+    elif max_tried is not None:
+        s = "%d tries"
+        s_args = [max_tried]
+    else:
+        s = "%.1fs"
+        s_args = [max_time]
+    exception = details.get("exception")
+    if isinstance(exception, tuple):
+        exception = list(exception)
+    elif not isinstance(exception, list):
+        exception = [exception]
+    exception_s = ", ".join([e.__name__ for e in exception])
+    if log_level >= logging.INFO:
+        msg = f"Attempting %s(...), will retry for {s} given these issues: %s"
+        log_args = [details["target"].__name__] + s_args + [exception_s]
+    else:
+        msg = f"Attempting %s(%s), will retry for {s} given these issues: %s"
+        target_input_list = []
+        if args := details.get("args"):
+            target_input_list.extend([str(d) for d in args])
+        if kwargs := details.get("kwargs"):
+            target_input_list.extend([f"{k}={str(v)}" for k, v in kwargs.items()])
+        target_input = ", ".join(target_input_list) if target_input_list else ""
+        log_args = (
+            [
+                details["target"].__name__,
+                target_input,
+            ]
+            + s_args
+            + [exception_s]
+        )
+    logger.log(log_level, msg, *log_args)
+
+
+# Default backoff handler
+def _log_backoff(details, logger, log_level):
+    if log_level >= logging.INFO:
+        msg = "Backing off %s(...) for %.1fs (%s)"
+        log_args = [details["target"].__name__, details["tries"]]
+    else:
+        msg = "Backing off %.1fs seconds after %d tries calling function %s(%s) -> %s"
+        target_input_list = []
+        if args := details.get("args"):
+            target_input_list.extend([str(d) for d in args])
+        if kwargs := details.get("kwargs"):
+            target_input_list.extend([f"{k}={str(v)}" for k, v in kwargs.items()])
+        target_input = ", ".join(target_input_list) if target_input_list else ""
+        log_args = [
+            details["wait"],
+            details["tries"],
+            details["target"].__name__,
+            target_input,
+        ]
+    exc_typ, exc, _ = sys.exc_info()
+    if exc is not None:
+        exc_fmt = traceback.format_exception_only(exc_typ, exc)[-1]
+        log_args.append(exc_fmt.rstrip("\n"))
+    else:
+        log_args.append(str(details["value"]))
+    logger.log(log_level, msg, *log_args)
+
+
+# Default giveup handler
+def _log_giveup(details, logger, log_level):
+    if log_level >= logging.INFO:
+        msg = "Giving up %s(...) after %.1fs (%s)"
+        log_args = [details["target"].__name__, details["tries"]]
+    else:
+        msg = "Giving up after %d tries (%.1fs) calling function %s(%s) -> %s"
+        target_input_list = []
+        if args := details.get("args"):
+            target_input_list.extend([str(d) for d in args])
+        if kwargs := details.get("kwargs"):
+            target_input_list.extend([f"{k}={str(v)}" for k, v in kwargs.items()])
+        target_input = ", ".join(target_input_list) if target_input_list else "..."
+        log_args = [
+            details["tries"],
+            details["wait"],
+            details["target"].__name__,
+            target_input,
+        ]
+
+    exc_typ, exc, _ = sys.exc_info()
+    if exc is not None:
+        exc_fmt = traceback.format_exception_only(exc_typ, exc)[-1]
+        log_args.append(exc_fmt.rstrip("\n"))
+    else:
+        log_args.append(details["value"])
+
+    logger.log(log_level, msg, *log_args)

unstructured_ingest/ingest_backoff/_wrapper.py

@@ -0,0 +1,122 @@
+# coding:utf-8
+import logging
+from collections.abc import Iterable as IterableType
+from typing import Any, Iterable, Optional, Type, Union
+
+from backoff import _sync
+from backoff._common import _config_handlers, _prepare_logger
+from backoff._jitter import full_jitter
+from backoff._typing import (
+    _Handler,
+    _Jitterer,
+    _MaybeCallable,
+    _MaybeLogger,
+    _MaybeSequence,
+    _Predicate,
+    _WaitGenerator,
+)
+
+from unstructured_ingest.ingest_backoff._common import _log_backoff, _log_giveup, _log_start
+
+
+class RetryHandler:
+    def __init__(
+        self,
+        wait_gen: _WaitGenerator,
+        exception: _MaybeSequence[Type[Exception]],
+        *,
+        max_tries: Optional[_MaybeCallable[int]] = None,
+        max_time: Optional[_MaybeCallable[float]] = None,
+        jitter: Union[_Jitterer, None] = full_jitter,
+        giveup: _Predicate[Exception] = lambda e: False,
+        on_start: Union[_Handler, Iterable[_Handler], None] = None,
+        on_success: Union[_Handler, Iterable[_Handler], None] = None,
+        on_backoff: Union[_Handler, Iterable[_Handler], None] = None,
+        on_giveup: Union[_Handler, Iterable[_Handler], None] = None,
+        raise_on_giveup: bool = True,
+        logger: _MaybeLogger = "backoff",
+        start_log_level: int = logging.INFO,
+        backoff_log_level: int = logging.INFO,
+        giveup_log_level: int = logging.ERROR,
+        **wait_gen_kwargs: Any,
+    ):
+        prepared_logger = _prepare_logger(logger)
+        on_success = _config_handlers(on_success)
+        on_start = _config_handlers(
+            on_start,
+            default_handler=_log_start,
+            logger=prepared_logger,
+            log_level=start_log_level,
+        )
+        on_backoff = _config_handlers(
+            on_backoff,
+            default_handler=_log_backoff,
+            logger=prepared_logger,
+            log_level=backoff_log_level,
+        )
+        on_giveup = _config_handlers(
+            on_giveup,
+            default_handler=_log_giveup,
+            logger=prepared_logger,
+            log_level=giveup_log_level,
+        )
+        prepared_logger.debug(
+            "Initiating retry handler with "
+            "max_tries={}, "
+            "max_time={}, "
+            "exception={}, "
+            "start_log_level={}, "
+            "backoff_log_level={}, "
+            "giveup_log_level={}".format(
+                max_tries,
+                max_time,
+                (
+                    ", ".join([e.__name__ for e in exception])
+                    if isinstance(exception, IterableType)
+                    else exception.__name__
+                ),
+                logging.getLevelName(start_log_level),
+                logging.getLevelName(backoff_log_level),
+                logging.getLevelName(giveup_log_level),
+            ),
+        )
+        self.on_start = on_start
+        self.on_success = on_success
+        self.on_backoff = on_backoff
+        self.on_giveup = on_giveup
+        self.jitter = jitter
+        self.giveup = giveup
+        self.raise_on_giveup = raise_on_giveup
+        self.wait_gen_kwargs = wait_gen_kwargs
+        self.wait_gen = wait_gen
+        self.exception = exception
+        self.max_tries = max_tries
+        self.max_time = max_time
+
+    def __call__(self, target, *args, **kwargs):
+        _sync._call_handlers(
+            self.on_start,
+            target=target,
+            args=args,
+            kwargs=kwargs,
+            tries=None,
+            elapsed=None,
+            max_tries=self.max_tries,
+            max_time=self.max_time,
+            exception=self.exception,
+        )
+        wrapped_func = _sync.retry_exception(
+            target,
+            self.wait_gen,
+            self.exception,
+            max_tries=self.max_tries,
+            max_time=self.max_time,
+            jitter=self.jitter,
+            giveup=self.giveup,
+            on_success=self.on_success,
+            on_backoff=self.on_backoff,
+            on_giveup=self.on_giveup,
+            raise_on_giveup=self.raise_on_giveup,
+            wait_gen_kwargs=self.wait_gen_kwargs,
+        )
+        return wrapped_func(*args, **kwargs)