oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/text_dataset/dataset.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import itertools
+from typing import Any, Callable, Dict, Generator, List, Union
+
+import ads
+import ads.text_dataset.extractor as te
+import fsspec
+import pandas as pd
+from ads.text_dataset import backends
+from ads.text_dataset.options import OptionFactory, Options
+from ads.text_dataset.udfs import UDF
+from ads.text_dataset.utils import NotSupportedError
+
+
+class DataLoader:
+    """
+    DataLoader binds engine, FileProcessor and File handler(in this case it is fsspec)
+    together to produce a dataframe of parsed text from files.
+
+    This class is expected to be used mainly from TextDatasetFactory class.
+
+    Attributes
+    ----------
+    processor: `ads.text_dataset.extractor.FileProcessor`
+        processor that is used for loading data.
+
+    Examples
+    --------
+    >>> import oci
+    >>> from ads.text_dataset.dataset import TextDatasetFactory as textfactory
+    >>> from ads.text_dataset.options import Options
+    >>> df = textfactory.format('pdf').engine('pandas').read_line(
+    ...     'oci://<bucket-name>@<namespace>/<path>/*.pdf',
+    ...     storage_options={"config": oci.config.from_file(os.path.join("~/.oci", "config"))},
+    ... )
+    >>> data_gen = textfactory.format('pdf').option(Options.FILE_NAME).backend('pdfplumber').read_text(
+    ...     'oci://<bucket-name>@<namespace>/<path>/*.pdf',
+    ...     storage_options={"config": oci.config.from_file(os.path.join("~/.oci", "config"))},
+    ... )
+    >>> textfactory.format('docx').convert_to_text(
+    ...     'oci://<bucket-name>@<namespace>/<path>/*.docx',
+    ...     './extracted',
+    ...     storage_options={"config": oci.config.from_file(os.path.join("~/.oci", "config"))},
+    ... )
+    >>> textfactory.format('docx').convert_to_text(
+    ...     'oci://<bucket-name>@<namespace>/<path>/*.docx',
+    ...     'oci://<bucket-name>@<namespace>/<out_path>',
+    ...     storage_options={"config": oci.config.from_file(os.path.join("~/.oci", "config"))},
+    ... )
+    >>> meta_gen = textfactory.format('docx').metadata_schema(
+    ...     'oci://<bucket-name>@<namespace>/papers/*.pdf',
+    ...     storage_options={"config": oci.config.from_file(os.path.join("~/.oci", "config"))},
+    ... )
+    >>> df = textfactory.format('pdf').engine('pandas').option(Options.FILE_METADATA, {'extract': ['Author']}).read_text(
+    ...     'oci://<bucket-name>@<namespace>/<path>/*.pdf',
+    ...     storage_options={"config": oci.config.from_file(os.path.join("~/.oci", "config"))},
+    ...     total_files=10,
+    ... )
+    >>> df = textfactory.format('txt').engine('cudf').read_line(
+    ...     'oci://<bucket-name>@<namespace>/<path>/*.log',
+    ...      udf=r'^\[(\S+)\s(\S+)\s(\d+)\s(\d+\:\d+\:\d+)\s(\d+)]\s(\S+)\s(\S+)\s(\S+)\s(\S+)',
+    ...      df_args={"columns":["day", "month", "date", "time", "year", "type", "method", "status", "file"]},
+    ...      n_lines_per_file=10,
+    ... )
+    """
+
+    def __init__(self, engine: str = None) -> None:
+        """Initialize a DataLoader object.
+
+        Parameters
+        ----------
+        engine : str, optional
+            dataframe engine, by default None.
+
+        Returns
+        -------
+        None
+        """
+        self.engine(engine)
+        self.filemanager = fsspec
+        self.processor = te.FileProcessorFactory.get_processor("txt")
+        self.options = []
+        self._data = None
+
+    def with_processor(self, processor_type: str) -> None:
+        """Set file processor.
+
+        Parameters
+        ----------
+        processor_type : str
+            type of processor, which corresponds to format of the file.
+
+        Returns
+        -------
+        None
+        """
+        self.processor = te.FileProcessorFactory.get_processor(processor_type)()
+        return self
+
+    def engine(self, eng: str) -> None:
+        """Set engine for dataloader. Can be pandas or cudf.
+
+        Parameters
+        ----------
+        eng : str
+            name of engine
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        NotSupportedError
+            raises error if engine passed in is not supported.
+        """
+        if eng is None:
+            self._engine = None
+            self._format_output = lambda *args, **kwargs: args[0]
+            return self
+        if eng not in ["pandas", "cudf"]:
+            raise NotSupportedError("Only pandas and cudf currently.")
+        else:
+            if eng == "pandas":
+                import pandas
+
+                self._engine = pandas
+                self._format_output = pandas.DataFrame
+            else:
+                import cudf
+
+                self._engine = cudf
+                self._format_output = lambda data, **kwargs: cudf.DataFrame(
+                    [row for row in data], **kwargs
+                )  # cuDF cannot be initialized with a generator
+        return self
+
+    def backend(self, backend: Union[str, backends.Base]) -> None:
+        """Set backend used for extracting text from files.
+
+        Parameters
+        ----------
+        backend : (str | `ads.text_dataset.backends.Base`)
+            backend for extracting text from raw files.
+
+        Returns
+        -------
+        None
+        """
+        self.processor.backend(backend)
+        return self
+
+    def option(self, opt: Options, spec: Any = None) -> None:
+        """Set extraction options.
+
+        Parameters
+        ----------
+        opt : `ads.text_dataset.options.Options`
+            an option defined in `ads.text_dataset.options.Options`
+        spec : Any, optional
+            specifications that will be passed to option handler, by default None
+
+        Returns
+        -------
+        None
+        """
+        self.options.append((OptionFactory.option_handler(opt), spec))
+        return self
+
+    def __load_data__(
+        self,
+        reader: Callable,
+        path: str,
+        udf: Union[str, Callable] = None,
+        storage_options: Dict = None,
+        encoding: str = "utf-8",
+        n_rows_per_file: int = None,
+        total_rows: int = None,
+    ) -> Generator[Union[str, List[str]], None, None]:
+        storage_options = storage_options if storage_options is not None else {}
+        fhs = self.filemanager.open_files(
+            path, mode="rb", encoding=encoding, **storage_options
+        )
+        if udf is not None:
+            if isinstance(udf, str):
+                fn = UDF.from_regex(udf)
+            else:
+                fn = udf
+        else:
+            fn = lambda x: x
+
+        total_line_count = [0]
+
+        # function to apply to each element
+        def func(fh, reader):
+            out = [option(self).handle(fh, spec) for option, spec in self.options]
+            line_count = 0
+            for text in reader(fh):
+                if total_rows is None or total_line_count[0] < total_rows:
+                    if n_rows_per_file is None or line_count < n_rows_per_file:
+                        content = fn(text)
+                        if content is not None:
+                            yield out + list(content) if (
+                                isinstance(content, list) or isinstance(content, tuple)
+                            ) else out + [content]
+                            line_count += 1
+                            total_line_count[0] += 1
+
+        return itertools.chain.from_iterable((func(fh, reader) for fh in fhs))
+
+    def read_line(
+        self,
+        path: str,
+        udf: Union[str, Callable] = None,
+        n_lines_per_file: int = None,
+        total_lines: int = None,
+        df_args: Dict = None,
+        storage_options: Dict = None,
+        encoding: str = "utf-8",
+    ) -> Union[Generator[Union[str, List[str]], None, None], "DataFrame"]:
+        """Read each file into lines. If path matches multiple files, will combine lines from all files.
+
+        Parameters
+        ----------
+        path : str
+            path to data files. can have glob pattern.
+        udf : (callable | str), optional
+            user defined function for processing each line, can be a callable or regex, by default None
+        n_lines_per_file : int, optional
+            max number of lines read from each file, by default None
+        total_lines : int, optional
+            max number of lines read from all files, by default None
+        df_args : dict, optional
+            arguments passed to dataframe engine (e.g. pandas), by default None
+        storage_options : dict, optional
+            storage options for cloud storage, by default None
+        encoding : str, optional
+            encoding of files, by default 'utf-8'
+
+        Returns
+        -------
+        (Generator | DataFrame)
+            returns either a data generator or a dataframe.
+        """
+        df_args = df_args if df_args is not None else {}
+        self._data = self.__load_data__(
+            self.processor.read_line,
+            path,
+            udf,
+            storage_options,
+            encoding,
+            n_lines_per_file,
+            total_lines,
+        )
+        return self._format_output(self._data, **df_args)
+
+    def read_text(
+        self,
+        path: str,
+        udf: Union[str, Callable] = None,
+        total_files: int = None,
+        storage_options: Dict = None,
+        df_args: Dict = None,
+        encoding: str = "utf-8",
+    ) -> Union[Generator[Union[str, List[str]], None, None], "DataFrame"]:
+        """Read each file into a text string. If path matches multiple files, each file corresponds to one record.
+
+        Parameters
+        ----------
+        path : str
+            path to data files. can have glob pattern.
+        udf : (callable | str), optional
+            user defined function for processing each line, can be a callable or regex, by default None
+        total_files : int, optional
+            max number of files to read, by default None
+        df_args : dict, optional
+            arguments passed to dataframe engine (e.g. pandas), by default None
+        storage_options : dict, optional
+            storage options for cloud storage, by default None
+        encoding : str, optional
+            encoding of files, by default 'utf-8'
+
+        Returns
+        -------
+        (Generator | DataFrame)
+            returns either a data generator or a dataframe.
+        """
+        df_args = df_args if df_args is not None else {}
+        self._data = self.__load_data__(
+            self.processor.read_text,
+            path,
+            udf,
+            storage_options,
+            encoding,
+            1,
+            total_files,
+        )
+        return self._format_output(self._data, **df_args)
+
+    def convert_to_text(
+        self,
+        src_path: str,
+        dst_path: str,
+        encoding: str = "utf-8",
+        storage_options: Dict = None,
+    ) -> None:
+        """Convert files to plain text files.
+
+        Parameters
+        ----------
+        src_path : str
+            path to source data file(s). can use glob pattern
+        dst_path: str
+            local folder or cloud storage (e.g., OCI object storage) prefix to save converted text files
+        encoding: str, optional
+            encoding for files, by default utf-8
+        storage_options : Dict, optional
+            storage options for cloud storage, by default None
+
+        Returns
+        -------
+        None
+        """
+        storage_options = storage_options if storage_options is not None else {}
+        fhs = self.filemanager.open_files(
+            src_path, mode="rb", encoding=encoding, **storage_options
+        )
+        for fh in fhs:
+            self.processor.convert_to_text(
+                fh,
+                dst_path,
+                storage_options=storage_options,
+            )
+
+    def metadata_all(
+        self, path: str, storage_options: Dict = None, encoding: str = "utf-8"
+    ) -> Generator[Dict[str, Any], None, None]:
+        """Get metadata of all files that matches the given path. Return a generator.
+
+        Parameters
+        ----------
+        path : str
+            path to data files. can use glob pattern.
+        storage_options : Dict, optional
+            storage options for cloud storage, by default None
+        encoding : str, optional
+            encoding of files, by default 'utf-8'
+
+        Returns
+        -------
+        Generator
+            generator of extracted metedata from files.
+        """
+        storage_options = storage_options if storage_options is not None else {}
+        fhs = self.filemanager.open_files(
+            path, mode="rb", encoding=encoding, **storage_options
+        )
+        return (self.processor.get_metadata(fh) for fh in fhs)
+
+    def metadata_schema(
+        self,
+        path: str,
+        n_files: int = 1,
+        storage_options: Dict = None,
+        encoding: str = "utf-8",
+    ) -> List[str]:
+        """
+        Get available fields in metadata by looking at the first `n_files` that
+        matches the given path.
+
+        Parameters
+        ----------
+        path: str
+            path to data files. can have glob pattern
+        n_files: int, optional
+            number of files to look up, default to be 1
+        storage_options: dict, optional
+            storage options for cloud storage, by default None
+        encoding: str, optional
+            encoding of files, by default utf-8
+
+        Returns
+        -------
+        List[str]
+            list of available fields in metadata
+        """
+
+        metadata = self.metadata_all(
+            path, storage_options=storage_options, encoding=encoding
+        )
+        fields = set()
+        for _ in range(n_files):
+            try:
+                fields.update(list(next(metadata).keys()))
+            except StopIteration:
+                break
+        return list(fields)
+
+    # ----- not currently used, but in case we want to consider chaining in the future -----
+    def _transform(self, udf, udf_type="fn"):  # pragma: no cover
+        if udf_type == "fn":
+            func = UDF.from_lambda(udf)
+        elif udf_type == "regex":
+            func = UDF.from_regex(udf)
+        else:
+            raise NotImplementedError("Other types of UDF not yet supported.")
+
+        # convert df into iterator
+        if isinstance(self._data, pd.DataFrame) or isinstance(self._data, pd.Series):
+            self._data = (
+                row.values if len(row.values) > 1 else row.values[0]
+                for i, row in self._data.iterrows()
+            )
+
+        self._data = (func(row) for row in self._data)
+        self._data = (row for row in self._data if row is not None)
+        return self
+
+
+class TextDatasetFactory:
+    """A class that generates a dataloader given a file format."""
+
+    @staticmethod
+    def format(format_name: str) -> DataLoader:
+        """
+        Instantiates DataLoader class and seeds it with the right kind of FileProcessor.
+        Eg. PDFProcessor for pdf. The FileProcessorFactory returns the processor based
+        on the format Type.
+
+        Parameters
+        ----------
+        format_name : str
+            name of format
+
+        Returns
+        -------
+        `ads.text_dataset.dataset.DataLoader`
+            a `DataLoader` object.
+        """
+        return DataLoader().with_processor(format_name)

ads/text_dataset/extractor.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+import os
+from typing import Dict, Generator, List, Union
+
+from ads.text_dataset import backends
+from ads.text_dataset.backends import OITCC, Base, PDFPlumber, Tika
+from ads.text_dataset.utils import NotSupportedError
+from fsspec.core import OpenFile
+
+logger = logging.getLogger("ads.text_dataset")
+
+
+class FileProcessor:
+    """
+    Base class for all the file processor. Files are opened using fsspec library.
+    The default implementation in the base class assumes text files.
+
+    This class is expected to be used inside `ads.text_dataset.dataset.DataLoader`.
+    """
+
+    backend_map = {"default": Base, "tika": Tika}
+
+    def __init__(self, backend: Union[str, backends.Base] = "default") -> None:
+        self.backend(backend)
+
+    def backend(self, backend: Union[str, backends.Base]) -> None:
+        """Set backend for file processor.
+
+        Parameters
+        ----------
+        backend : `ads.text_dataset.backends.Base`
+            a backend for file processor
+
+        Returns
+        -------
+        None
+
+        Raises
+        ------
+        NotSupportedError
+            when specified backend is not supported.
+        """
+        if isinstance(backend, str) and backend in self.backend_map:
+            self._backend = self.backend_map[backend]()
+        elif isinstance(backend, Base):
+            self._backend = backend
+        else:
+            raise NotSupportedError(
+                f"backend {backend} is not recognized or not a subclass of ads.text_dataset.backends.Base."
+            )
+        return self
+
+    def read_line(
+        self, fhandler: OpenFile, **format_reader_kwargs: Dict
+    ) -> Generator[Union[str, List[str]], None, None]:
+        """Yields lines from a file.
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            file handler returned by `fsspec`
+
+        Returns
+        -------
+        Generator
+            a generator that yields lines from a file
+        """
+        return self._backend.read_line(fhandler, **format_reader_kwargs)
+
+    def read_text(
+        self, fhandler: OpenFile, **format_reader_kwargs: Dict
+    ) -> Generator[Union[str, List[str]], None, None]:
+        """Yield contents from the entire file.
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            a file handler returned by fsspec
+
+        Returns
+        -------
+        Generator
+            a generator that yield text from a file
+        """
+        return self._backend.read_text(fhandler, **format_reader_kwargs)
+
+    def convert_to_text(
+        self,
+        fhandler: OpenFile,
+        dst_path: str,
+        fname: str = None,
+        storage_options: Dict = None,
+    ) -> str:
+        """Convert input file to a text file.
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            a file handler returned by `fsspec`
+        dst_path: str
+            local folder or cloud storage (e.g. OCI object storage) prefix to save converted text files
+        fname: str, optional
+            filename for converted output, relative to dirname or prefix, by default None
+        storage_options: dict, optional
+            storage options for cloud storage, by default None
+
+        Returns
+        -------
+        str
+            path to saved output
+        """
+        return self._backend.convert_to_text(fhandler, dst_path, fname, storage_options)
+
+    def get_metadata(self, fhandler: OpenFile) -> Dict:
+        """Get metadata of a file.
+
+        Parameters
+        ----------
+        fhandler : `fsspec.core.OpenFile`
+            a file handler returned by fsspec
+
+        Returns
+        -------
+        dict
+            dictionary of metadata
+        """
+        return self._backend.get_metadata(fhandler)
+
+
+class PDFProcessor(FileProcessor):
+    """
+    Extracts text content from PDF
+    """
+
+    backend_map = {"tika": Tika, "pdfplumber": PDFPlumber, "default": Tika}
+
+
+class WordProcessor(FileProcessor):
+    """
+    Extracts text content from doc or docx format.
+    """
+
+    backend_map = {"default": Tika, "tika": Tika}
+
+
+class FileProcessorFactory:
+    """Factory that manages all file processors.
+    Provides functionality to get a processor corresponding to a given file type,
+    or register custom processor for a specific file format.
+
+    Examples
+    --------
+    >>> from ads.text_dataset.extractor import FileProcessor, FileProcessorFactory
+    >>> FileProcessorFactory.get_processor('pdf')
+    >>> class CustomProcessor(FileProcessor):
+    ... # custom logic here
+    ... pass
+    >>> FileProcessorFactory.register('new_format', CustomProcessor)
+    """
+
+    processor_map = {
+        "pdf": PDFProcessor,
+        "docx": WordProcessor,
+        "doc": WordProcessor,
+        "txt": FileProcessor,
+    }
+
+    @classmethod
+    def register(cls, fmt: str, processor: FileProcessor) -> None:
+        """Register custom file processor for a file format.
+
+        Parameters
+        ----------
+        fmt : str
+            file format
+        processor : `FileProcessor`
+            custom processor
+
+        Raises
+        ------
+        TypeError
+            raised when processor is not a subclass of `FileProcessor`.
+        """
+        if issubclass(processor, FileProcessor):
+            cls.processor_map[fmt] = processor
+        else:
+            raise TypeError(f"Processor must inherit from FileProcessor class.")
+
+    @staticmethod
+    def get_processor(format):
+        if format in FileProcessorFactory.processor_map:
+            return FileProcessorFactory.processor_map[format]
+        else:
+            logger.warning(
+                f"""
+                Format {format} is not supported natively.
+                A generic FileProcessor is returned.
+                You can define and register a custom processor.
+                """
+            )
+            return FileProcessor

ads/text_dataset/options.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from enum import Enum, auto
+from typing import Any, List, Dict
+
+from fsspec.core import OpenFile
+
+
+class Options(Enum):
+    FILE_NAME = auto()
+    FILE_METADATA = auto()
+
+
+class OptionHandler:
+    def __init__(self, dataloader: "ads.text_dataset.dataset.DataLoader") -> None:
+        self.dataloader = dataloader
+
+    def handle(self, fhandler: OpenFile, spec: Any) -> Any:
+        raise NotImplementedError()
+
+
+class FileOption(OptionHandler):
+    def handle(self, fhandler: OpenFile, spec: Any) -> Any:
+        return fhandler.path
+
+
+class MetadataOption(OptionHandler):
+    def handle(self, fhandler: OpenFile, spec: Dict) -> List:
+        metadata = self.dataloader.processor.get_metadata(fhandler)
+        return [metadata.get(k, None) for k in spec["extract"]]
+
+
+class OptionFactory:
+
+    option_handlers = {
+        Options.FILE_NAME: FileOption,
+        Options.FILE_METADATA: MetadataOption,
+    }
+
+    @staticmethod
+    def option_handler(option: Options) -> OptionHandler:
+        handler = OptionFactory.option_handlers.get(option, None)
+        if handler is None:
+            raise RuntimeError(f"Option {option} Not Recognized.")
+        return handler
+
+    @classmethod
+    def register_option(cls, option: Options, handler) -> None:
+        cls.option_handlers[option] = handler