mseep-txtai 9.1.1__py3-none-any.whl

txtai/pipeline/data/tabular.py

@@ -0,0 +1,155 @@
+"""
+Tabular module
+"""
+
+import os
+
+# Conditional import
+try:
+    import pandas as pd
+
+    PANDAS = True
+except ImportError:
+    PANDAS = False
+
+from ..base import Pipeline
+
+
+class Tabular(Pipeline):
+    """
+    Splits tabular data into rows and columns.
+    """
+
+    def __init__(self, idcolumn=None, textcolumns=None, content=False):
+        """
+        Creates a new Tabular pipeline.
+
+        Args:
+            idcolumn: column name to use for row id
+            textcolumns: list of columns to combine as a text field
+            content: if True, a dict per row is generated with all fields. If content is a list, a subset of fields
+                     is included in the generated rows.
+        """
+
+        if not PANDAS:
+            raise ImportError('Tabular pipeline is not available - install "pipeline" extra to enable')
+
+        self.idcolumn = idcolumn
+        self.textcolumns = textcolumns
+        self.content = content
+
+    def __call__(self, data):
+        """
+        Splits data into rows and columns.
+
+        Args:
+            data: input data
+
+        Returns:
+            list of (id, text, tag)
+        """
+
+        items = [data] if not isinstance(data, list) else data
+
+        # Combine all rows into single return element
+        results = []
+        dicts = []
+
+        for item in items:
+            # File path
+            if isinstance(item, str):
+                _, extension = os.path.splitext(item)
+                extension = extension.replace(".", "").lower()
+
+                if extension == "csv":
+                    df = pd.read_csv(item)
+
+                results.append(self.process(df))
+
+            # Dict
+            if isinstance(item, dict):
+                dicts.append(item)
+
+            # List of dicts
+            elif isinstance(item, list):
+                df = pd.DataFrame(item)
+                results.append(self.process(df))
+
+        if dicts:
+            df = pd.DataFrame(dicts)
+            results.extend(self.process(df))
+
+        return results[0] if not isinstance(data, list) else results
+
+    def process(self, df):
+        """
+        Extracts a list of (id, text, tag) tuples from a dataframe.
+
+        Args:
+            df: DataFrame to extract content from
+
+        Returns:
+            list of (id, text, tag)
+        """
+
+        rows = []
+
+        # Columns to use for text
+        columns = self.textcolumns
+        if not columns:
+            columns = list(df.columns)
+            if self.idcolumn:
+                columns.remove(self.idcolumn)
+
+        # Transform into (id, text, tag) tuples
+        for index, row in df.iterrows():
+            uid = row[self.idcolumn] if self.idcolumn else index
+            uid = uid if uid is not None else index
+            text = self.concat(row, columns)
+
+            rows.append((uid, text, None))
+
+            # Also add row for content
+            if isinstance(self.content, list):
+                row = {column: self.column(value) for column, value in row.to_dict().items() if column in self.content}
+                rows.append((uid, row, None))
+            elif self.content:
+                row = {column: self.column(value) for column, value in row.to_dict().items()}
+                rows.append((uid, row, None))
+
+        return rows
+
+    def concat(self, row, columns):
+        """
+        Builds a text field from row using columns.
+
+        Args:
+            row: input row
+            columns: list of columns to join together
+
+        Returns:
+            text
+        """
+
+        parts = []
+        for column in columns:
+            column = self.column(row[column])
+            if column:
+                parts.append(str(column))
+
+        return ". ".join(parts) if parts else None
+
+    def column(self, value):
+        """
+        Applies column standardization logic:
+            - Replace NaN values with None
+
+        Args:
+            value: input value
+
+        Returns:
+            formatted value
+        """
+
+        # Check for null - treat lists as not null
+        return None if not isinstance(value, list) and pd.isnull(value) else value

txtai/pipeline/data/textractor.py

@@ -0,0 +1,139 @@
+"""
+Textractor module
+"""
+
+import contextlib
+import os
+import tempfile
+
+from urllib.parse import urlparse
+from urllib.request import urlopen, Request
+
+from .filetohtml import FileToHTML
+from .htmltomd import HTMLToMarkdown
+from .segmentation import Segmentation
+
+
+class Textractor(Segmentation):
+    """
+    Extracts text from files.
+    """
+
+    # pylint: disable=R0913
+    def __init__(
+        self,
+        sentences=False,
+        lines=False,
+        paragraphs=False,
+        minlength=None,
+        join=False,
+        sections=False,
+        cleantext=True,
+        chunker=None,
+        headers=None,
+        backend="available",
+        **kwargs
+    ):
+        super().__init__(sentences, lines, paragraphs, minlength, join, sections, cleantext, chunker, **kwargs)
+
+        # Get backend parameter - handle legacy tika flag
+        backend = "tika" if "tika" in kwargs and kwargs["tika"] else None if "tika" in kwargs else backend
+
+        # File to HTML pipeline
+        self.html = FileToHTML(backend) if backend else None
+
+        # HTML to Markdown pipeline
+        self.markdown = HTMLToMarkdown(self.paragraphs, self.sections)
+
+        # HTTP headers
+        self.headers = headers if headers else {}
+
+    def text(self, text):
+        # Check if text is a valid file path or url
+        path, exists = self.valid(text)
+
+        if not path:
+            # Not a valid file path, treat input as data
+            html = text
+
+        elif self.html:
+            # Use FileToHTML pipeline, if available
+            # Retrieve remote file, if necessary
+            path = path if exists else self.download(path)
+
+            # Parse content to HTML
+            html = self.html(path)
+
+            # FiletoHTML pipeline returns None when input is already HTML
+            html = html if html else self.retrieve(path)
+
+            # Delete temporary file
+            if not exists:
+                os.remove(path)
+
+        else:
+            # Read data from url/path
+            html = self.retrieve(path)
+
+        # HTML to Markdown
+        return self.markdown(html)
+
+    def valid(self, path):
+        """
+        Checks if path is a valid local file or web url. Returns path if valid along with a flag
+        denoting if the path exists locally.
+
+        Args:
+            path: path to check
+
+        Returns:
+            (path, exists)
+        """
+
+        # Convert file urls to local paths
+        path = path.replace("file://", "")
+
+        # Check if this is a local file path or local file url
+        exists = os.path.exists(path)
+
+        # Consider local files and HTTP urls valid
+        return (path if exists or urlparse(path).scheme in ("http", "https") else None, exists)
+
+    def download(self, url):
+        """
+        Downloads content of url to a temporary file.
+
+        Args:
+            url: input url
+
+        Returns:
+            temporary file path
+        """
+
+        with tempfile.NamedTemporaryFile(mode="wb", delete=False) as output:
+            path = output.name
+
+            # Retrieve and write data to temporary file
+            output.write(self.retrieve(url))
+
+        return path
+
+    def retrieve(self, url):
+        """
+        Retrieves content from url.
+
+        Args:
+            url: input url
+
+        Returns:
+            data
+        """
+
+        # Local file
+        if os.path.exists(url):
+            with open(url, "rb") as f:
+                return f.read()
+
+        # Remote file
+        with contextlib.closing(urlopen(Request(url, headers=self.headers))) as connection:
+            return connection.read()

txtai/pipeline/data/tokenizer.py

@@ -0,0 +1,112 @@
+"""
+Tokenizer module
+"""
+
+import re
+import string
+
+import regex
+
+from ..base import Pipeline
+
+
+class Tokenizer(Pipeline):
+    """
+    Tokenizes text into tokens using one of the following methods.
+
+      1. Backwards compatible tokenization that only accepts alphanumeric tokens from the Latin alphabet.
+
+      2. Split using word boundary rules from the Unicode Text Segmentation algorithm (see Unicode Standard Annex #29).
+         This is similar to the standard tokenizer in Apache Lucene and works well for most languages.
+    """
+
+    # fmt: off
+    # English Stop Word List (Standard stop words used by Apache Lucene)
+    STOP_WORDS = {"a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it",
+                  "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these",
+                  "they", "this", "to", "was", "will", "with"}
+    # fmt: on
+
+    @staticmethod
+    def tokenize(text, lowercase=True, emoji=True, alphanum=True, stopwords=True):
+        """
+        Tokenizes text into a list of tokens. The default backwards compatible parameters filter out English stop words and only
+        accept alphanumeric tokens.
+
+        Args:
+            text: input text
+            lowercase: lower cases all tokens if True, defaults to True
+            emoji: tokenize emoji in text if True, defaults to True
+            alphanum: requires 2+ character alphanumeric tokens if True, defaults to True
+            stopwords: removes provided stop words if a list, removes default English stop words if True, defaults to True
+
+        Returns:
+            list of tokens
+        """
+
+        # Create a tokenizer with backwards compatible settings
+        return Tokenizer(lowercase, emoji, alphanum, stopwords)(text)
+
+    def __init__(self, lowercase=True, emoji=True, alphanum=False, stopwords=False):
+        """
+        Creates a new tokenizer. The default parameters segment text per Unicode Standard Annex #29.
+
+        Args:
+            lowercase: lower cases all tokens if True, defaults to True
+            emoji: tokenize emoji in text if True, defaults to True
+            alphanum: requires 2+ character alphanumeric tokens if True, defaults to False
+            stopwords: removes provided stop words if a list, removes default English stop words if True, defaults to False
+        """
+
+        # Lowercase
+        self.lowercase = lowercase
+
+        # Text segmentation
+        self.alphanum, self.segment = None, None
+        if alphanum:
+            # Alphanumeric regex that accepts tokens that meet following rules:
+            #  - Strings to be at least 2 characters long AND
+            #  - At least 1 non-trailing alpha character in string
+            # Note: The standard Python re module is much faster than regex for this expression
+            self.alphanum = re.compile(r"^\d*[a-z][\-.0-9:_a-z]{1,}$")
+        else:
+            # Text segmentation per Unicode Standard Annex #29
+            pattern = r"\w\p{Extended_Pictographic}\p{WB:RegionalIndicator}" if emoji else r"\w"
+            self.segment = regex.compile(rf"[{pattern}](?:\B\S)*", flags=regex.WORD)
+
+        # Stop words
+        self.stopwords = stopwords if isinstance(stopwords, list) else Tokenizer.STOP_WORDS if stopwords else False
+
+    def __call__(self, text):
+        """
+        Tokenizes text into a list of tokens.
+
+        Args:
+            text: input text
+
+        Returns:
+            list of tokens
+        """
+
+        # Check for None and skip processing
+        if text is None:
+            return None
+
+        # Lowercase
+        text = text.lower() if self.lowercase else text
+
+        if self.alphanum:
+            # Text segmentation using standard split
+            tokens = [token.strip(string.punctuation) for token in text.split()]
+
+            # Filter on alphanumeric strings.
+            tokens = [token for token in tokens if re.match(self.alphanum, token)]
+        else:
+            # Text segmentation per Unicode Standard Annex #29
+            tokens = regex.findall(self.segment, text)
+
+        # Stop words
+        if self.stopwords:
+            tokens = [token for token in tokens if token not in self.stopwords]
+
+        return tokens

txtai/pipeline/factory.py

@@ -0,0 +1,77 @@
+"""
+Pipeline factory module
+"""
+
+import inspect
+import sys
+import types
+
+from ..util import Resolver
+
+from .base import Pipeline
+
+
+class PipelineFactory:
+    """
+    Pipeline factory. Creates new Pipeline instances.
+    """
+
+    @staticmethod
+    def get(pipeline):
+        """
+        Gets a new instance of pipeline class.
+
+        Args:
+            pclass: Pipeline instance class
+
+        Returns:
+            Pipeline class
+        """
+
+        # Local pipeline if no package
+        if "." not in pipeline:
+            return PipelineFactory.list()[pipeline]
+
+        # Attempt to load custom pipeline
+        return Resolver()(pipeline)
+
+    @staticmethod
+    def create(config, pipeline):
+        """
+        Creates a new Pipeline instance.
+
+        Args:
+            config: Pipeline configuration
+            pipeline: Pipeline instance class
+
+        Returns:
+            Pipeline
+        """
+
+        # Resolve pipeline
+        pipeline = PipelineFactory.get(pipeline)
+
+        # Return functions directly, otherwise create pipeline instance
+        return pipeline if isinstance(pipeline, types.FunctionType) else pipeline(**config)
+
+    @staticmethod
+    def list():
+        """
+        Lists callable pipelines.
+
+        Returns:
+            {short name: pipeline class}
+        """
+
+        pipelines = {}
+
+        # Get handle to pipeline module
+        pipeline = sys.modules[".".join(__name__.split(".")[:-1])]
+
+        # Get list of callable pipelines
+        for x in inspect.getmembers(pipeline, inspect.isclass):
+            if issubclass(x[1], Pipeline) and [y for y, _ in inspect.getmembers(x[1], inspect.isfunction) if y == "__call__"]:
+                # short name: pipeline class
+                pipelines[x[0].lower()] = x[1]
+
+        return pipelines

txtai/pipeline/hfmodel.py

@@ -0,0 +1,111 @@
+"""
+Hugging Face Transformers model wrapper module
+"""
+
+from ..models import Models
+from .tensors import Tensors
+
+
+class HFModel(Tensors):
+    """
+    Pipeline backed by a Hugging Face Transformers model.
+    """
+
+    def __init__(self, path=None, quantize=False, gpu=False, batch=64):
+        """
+        Creates a new HFModel.
+
+        Args:
+            path: optional path to model, accepts Hugging Face model hub id or local path,
+                  uses default model for task if not provided
+            quantize: if model should be quantized, defaults to False
+            gpu: True/False if GPU should be enabled, also supports a GPU device id
+            batch: batch size used to incrementally process content
+        """
+
+        # Default model path
+        self.path = path
+
+        # Quantization flag
+        self.quantization = quantize
+
+        # Get tensor device reference
+        self.deviceid = Models.deviceid(gpu)
+        self.device = Models.device(self.deviceid)
+
+        # Process batch size
+        self.batchsize = batch
+
+    def prepare(self, model):
+        """
+        Prepares a model for processing. Applies dynamic quantization if necessary.
+
+        Args:
+            model: input model
+
+        Returns:
+            model
+        """
+
+        if self.deviceid == -1 and self.quantization:
+            model = self.quantize(model)
+
+        return model
+
+    def tokenize(self, tokenizer, texts):
+        """
+        Tokenizes text using tokenizer. This method handles overflowing tokens and automatically splits
+        them into separate elements. Indices of each element is returned to allow reconstructing the
+        transformed elements after running through the model.
+
+        Args:
+            tokenizer: Tokenizer
+            texts: list of text
+
+        Returns:
+            (tokenization result, indices)
+        """
+
+        # Pre-process and split on newlines
+        batch, positions = [], []
+        for x, text in enumerate(texts):
+            elements = [t + " " for t in text.split("\n") if t]
+            batch.extend(elements)
+            positions.extend([x] * len(elements))
+
+        # Run tokenizer
+        tokens = tokenizer(batch, padding=True)
+
+        inputids, attention, indices = [], [], []
+        for x, ids in enumerate(tokens["input_ids"]):
+            if len(ids) > tokenizer.model_max_length:
+                # Remove padding characters, if any
+                ids = [i for i in ids if i != tokenizer.pad_token_id]
+
+                # Split into model_max_length chunks
+                for chunk in self.batch(ids, tokenizer.model_max_length - 1):
+                    # Append EOS token if necessary
+                    if chunk[-1] != tokenizer.eos_token_id:
+                        chunk.append(tokenizer.eos_token_id)
+
+                    # Set attention mask
+                    mask = [1] * len(chunk)
+
+                    # Append padding if necessary
+                    if len(chunk) < tokenizer.model_max_length:
+                        pad = tokenizer.model_max_length - len(chunk)
+                        chunk.extend([tokenizer.pad_token_id] * pad)
+                        mask.extend([0] * pad)
+
+                    inputids.append(chunk)
+                    attention.append(mask)
+                    indices.append(positions[x])
+            else:
+                inputids.append(ids)
+                attention.append(tokens["attention_mask"][x])
+                indices.append(positions[x])
+
+        tokens = {"input_ids": inputids, "attention_mask": attention}
+
+        # pylint: disable=E1102
+        return ({name: self.tensor(tensor).to(self.device) for name, tensor in tokens.items()}, indices)

txtai/pipeline/hfpipeline.py

@@ -0,0 +1,96 @@
+"""
+Hugging Face Transformers pipeline wrapper module
+"""
+
+import inspect
+
+from transformers import pipeline
+
+from ..models import Models
+from ..util import Resolver
+
+from .tensors import Tensors
+
+
+class HFPipeline(Tensors):
+    """
+    Light wrapper around Hugging Face Transformers pipeline component for selected tasks. Adds support for model
+    quantization and minor interface changes.
+    """
+
+    def __init__(self, task, path=None, quantize=False, gpu=False, model=None, **kwargs):
+        """
+        Loads a new pipeline model.
+
+        Args:
+            task: pipeline task or category
+            path: optional path to model, accepts Hugging Face model hub id, local path or (model, tokenizer) tuple.
+                  uses default model for task if not provided
+            quantize: if model should be quantized, defaults to False
+            gpu: True/False if GPU should be enabled, also supports a GPU device id
+            model: optional existing pipeline model to wrap
+            kwargs: additional keyword arguments to pass to pipeline model
+        """
+
+        if model:
+            # Check if input model is a Pipeline or a HF pipeline
+            self.pipeline = model.pipeline if isinstance(model, HFPipeline) else model
+        else:
+            # Get device
+            deviceid = Models.deviceid(gpu) if "device_map" not in kwargs else None
+            device = Models.device(deviceid) if deviceid is not None else None
+
+            # Split into model args, pipeline args
+            modelargs, kwargs = self.parseargs(**kwargs)
+
+            # Transformer pipeline task
+            if isinstance(path, (list, tuple)):
+                # Derive configuration, if possible
+                config = path[1] if path[1] and isinstance(path[1], str) else None
+
+                # Load model
+                model = Models.load(path[0], config, task)
+
+                self.pipeline = pipeline(task, model=model, tokenizer=path[1], device=device, model_kwargs=modelargs, **kwargs)
+            else:
+                self.pipeline = pipeline(task, model=path, device=device, model_kwargs=modelargs, **kwargs)
+
+            # Model quantization. Compresses model to int8 precision, improves runtime performance. Only supported on CPU.
+            if deviceid == -1 and quantize:
+                # pylint: disable=E1101
+                self.pipeline.model = self.quantize(self.pipeline.model)
+
+        # Detect unbounded tokenizer typically found in older models
+        Models.checklength(self.pipeline.model, self.pipeline.tokenizer)
+
+    def parseargs(self, **kwargs):
+        """
+        Inspects the pipeline method and splits kwargs into model args and pipeline args.
+
+        Args:
+            kwargs: all keyword arguments
+
+        Returns:
+            (model args, pipeline args)
+        """
+
+        # Get pipeline method arguments
+        args = inspect.getfullargspec(pipeline).args
+
+        # Resolve torch dtype, if necessary
+        dtype = kwargs.get("torch_dtype")
+        if dtype and isinstance(dtype, str) and dtype != "auto":
+            kwargs["torch_dtype"] = Resolver()(dtype)
+
+        # Split into modelargs and kwargs
+        return ({arg: value for arg, value in kwargs.items() if arg not in args}, {arg: value for arg, value in kwargs.items() if arg in args})
+
+    def maxlength(self):
+        """
+        Gets the max length to use for generate calls.
+
+        Returns:
+            max length
+        """
+
+        return Models.maxlength(self.pipeline.model, self.pipeline.tokenizer)

txtai/pipeline/image/__init__.py

@@ -0,0 +1,7 @@
+"""
+Image imports
+"""
+
+from .caption import Caption
+from .imagehash import ImageHash
+from .objects import Objects