indexify 0.0.43__py3-none-any.whl → 0.2__py3-none-any.whl

indexify/graph.py

@@ -1,133 +0,0 @@
-import itertools
-import json
-from collections import defaultdict
-from typing import Any, Dict, List, Optional, Type, Union
-
-import cloudpickle
-from pydantic import BaseModel
-
-from .extractor_sdk import Content, Extractor, extractor
-from .runner import Runner
-
-
-@extractor(description="id function")
-def _id(content: Content) -> List[Content]:
-    return [content]
-
-
-def load_graph(graph: bytes) -> "Graph":
-    return cloudpickle.loads(graph)
-
-
-class Graph:
-    def __init__(
-        self, name: str, input: Type[BaseModel], start_node: extractor, runner: Runner
-    ):
-        # TODO check for cycles
-        self.name = name
-
-        self.nodes: Dict[str, Union[extractor, Extractor]] = {}
-        self.params: Dict[str, Any] = {}
-
-        self.edges: Dict[str, List[(str, str)]] = defaultdict(list)
-
-        self.nodes["start"] = _id
-        self.nodes["end"] = _id
-
-        self._topo_counter = defaultdict(int)
-
-        self._start_node = None
-        self._input = input
-
-        self.runner = runner
-
-    def get_extractor(self, name: str) -> Extractor:
-        return self.nodes[name]
-
-    def _node(self, extractor: Extractor, params: Any = None) -> "Graph":
-        name = extractor.name
-
-        # if you've already inserted a node just ignore the new insertion.
-        if name in self.nodes:
-            return
-
-        self.nodes[name] = extractor
-        self.params[name] = extractor.__dict__.get("params", None)
-
-        # assign each node a rank of 1 to init the graph
-        self._topo_counter[name] = 1
-
-        return self
-
-    def serialize(self):
-        return cloudpickle.dumps(self)
-
-    def add_edge(
-        self,
-        from_node: Type[Extractor],
-        to_node: Type[Extractor],
-        prefilter_predicates: Optional[str] = None,
-    ) -> "Graph":
-
-        self._node(from_node)
-        self._node(to_node)
-
-        from_node_name = from_node.name
-        to_node_name = to_node.name
-
-        self.edges[from_node_name].append((to_node_name, prefilter_predicates))
-
-        self._topo_counter[to_node_name] += 1
-
-        return self
-
-    """
-    Connect nodes as a fan out from one `from_node` to multiple `to_nodes` and respective `prefilter_predicates`.
-    Note: The user has to match the sizes of the lists to make sure they line up otherwise a None is used as a default.
-    """
-
-    def steps(
-        self,
-        from_node: extractor,
-        to_nodes: List[extractor],
-        prefilter_predicates: List[str] = [],
-    ) -> "Graph":
-        print(f"{to_nodes}, {prefilter_predicates}, {prefilter_predicates}")
-        for t_n, p in itertools.zip_longest(
-            to_nodes, prefilter_predicates, fillvalue=None
-        ):
-            self.step(from_node=from_node, to_node=t_n, prefilter_predicates=p)
-
-        return self
-
-    def add_param(self, node: extractor, params: Dict[str, Any]):
-        try:
-            # check if the params can be serialized since the server needs this
-            json.dumps(params)
-        except Exception:
-            raise Exception(f"For node {node.name}, cannot serialize params as json.")
-
-        self.params[node.name] = params
-
-    def run(self, wf_input, local):
-        self._assign_start_node()
-        self.runner.run(self, wf_input=wf_input)
-        pass
-
-    def clear_cache_for_node(self, node: Union[extractor, Extractor]):
-        if node.name not in self.nodes.keys():
-            raise Exception(f"Node with name {node.name} not found in graph")
-
-        self.runner.deleted_from_memo(node.name)
-
-    def clear_cache_for_all_nodes(self):
-        for node_name in self.nodes:
-            self.runner.deleted_from_memo(node_name=node_name)
-
-    def get_result(self, node: Union[extractor, Extractor]) -> Any:
-        return self.runner.results[node.name]
-
-    def _assign_start_node(self):
-        # this method should be called before a graph can be run
-        nodes = sorted(self._topo_counter.items(), key=lambda x: x[1])
-        self._start_node = nodes[0][0]

indexify/local_runner.py

@@ -1,128 +0,0 @@
-import hashlib
-import os
-import pickle
-import shutil
-from collections import defaultdict
-from pathlib import Path
-from typing import Any, Callable, Dict, Optional, Union
-
-from indexify.extractor_sdk.data import BaseData, Feature
-from indexify.extractor_sdk.extractor import Extractor, extractor
-from indexify.graph import Graph
-from indexify.runner import Runner
-
-
-class LocalRunner(Runner):
-    def __init__(self):
-        self.results: Dict[str, Any] = defaultdict(
-            list
-        )  # TODO should the Any be Content?
-
-    def run(self, g, wf_input: BaseData):
-        return self._run(g, _input=wf_input, node_name=g._start_node)
-
-    # graph is getting some files which are files, some lables and the MIME type of the bytes
-    # those bytes have to be a python type
-
-    # _input needs to be serializable into python object (ie json for ex) and Feature
-    def _run(self, g: Graph, _input: BaseData, node_name: str):
-        print(f"---- Starting node {node_name}")
-        print(f"node_name {node_name}")
-
-        extractor_construct: Callable = g.nodes[node_name]
-        params = g.params.get(node_name, None)
-
-        # NOTE: User should clear cache for nodes they would like to re-rerun
-        input_hash = hashlib.sha256(str(_input).encode()).hexdigest()
-        memo_output = self.get_from_memo(node_name, input_hash)
-        if memo_output is None:
-            print("=== FYI Writing output to cache")
-            res = extractor_construct().extract(input=_input, params=params)
-            self.put_into_memo(node_name, input_hash, pickle.dumps(res))
-        else:
-            print("=== Reading output from cache")
-            res = pickle.loads(memo_output)
-
-        if not isinstance(res, list):
-            res = [res]
-
-        res_data = [i for i in res if not isinstance(i, Feature)]
-        res_features = [i for i in res if isinstance(i, Feature)]
-
-        self.results[node_name].extend(res_data)
-
-        for f in res_features:
-            _input.meta[f.name] = f.value
-
-        # this assume that if an extractor emits features then the next edge will always process
-        # the edges
-        data_to_process = res_data
-        if len(res_features) > 0:
-            data_to_process.append(_input)
-
-        for out_edge, pre_filter_predicate in g.edges[node_name]:
-            # TODO there are no reductions yet, each recursion finishes it's path and returns
-            for r in data_to_process:
-                if self._prefilter_content(
-                    content=r, prefilter_predicate=pre_filter_predicate
-                ):
-                    continue
-
-                self._run(g, _input=r, node_name=out_edge)
-
-    """
-    Returns True if content should be filtered
-    """
-
-    def _prefilter_content(
-        self, content: BaseData, prefilter_predicate: Optional[str]
-    ) -> bool:
-        if prefilter_predicate is None:
-            return False
-
-        atoms = prefilter_predicate.split("and")
-        if len(atoms) == 0:
-            return False
-
-        # TODO For now only support `and` and `=` and `string values`
-        bools = []
-        metadata = content.get_features()["metadata"]
-        for atom in atoms:
-            l, r = atom.split("=")
-            if l in metadata:
-                bools.append(metadata[l] != r)
-
-        return all(bools)
-
-    def get_result(self, node: Union[extractor, Extractor]) -> Any:
-        node_name = node.name
-        return self.results[node_name]
-
-    def deleted_from_memo(self, node_name):
-        path_prefix = f"./indexify_local_runner_cache/{node_name}"
-
-        if os.path.exists(path_prefix) and os.path.isdir(path_prefix):
-            shutil.rmtree(path_prefix)
-
-    def get_from_memo(self, node_name, input_hash):
-        path_prefix = f"./indexify_local_runner_cache/{node_name}"
-        file_name = f"{input_hash}"
-        file_path = f"{path_prefix}/{file_name}"
-
-        if not os.path.exists(file_path):
-            return None
-
-        with open(file_path, "rb") as f:
-            return f.read()
-
-    def put_into_memo(self, node_name, input_hash, output):
-        path_prefix = f"./indexify_local_runner_cache/{node_name}"
-        file_name = f"{input_hash}"
-        file_path = f"{path_prefix}/{file_name}"
-
-        os.makedirs(path_prefix, exist_ok=True)
-
-        Path(file_path).touch()
-
-        with open(file_path, "wb") as f:
-            return f.write(output)

indexify/runner.py

@@ -1,22 +0,0 @@
-from abc import ABC
-from typing import Any, Union
-
-from indexify.extractor_sdk.data import BaseData
-from indexify.extractor_sdk.extractor import Extractor, extractor
-
-
-class Runner(ABC):
-    def run(self, g, wf_input: BaseData):
-        raise NotImplementedError()
-
-    def get_result(self, node: Union[extractor, Extractor]) -> Any:
-        raise NotImplementedError()
-
-    def deleted_from_memo(self, node_name):
-        raise NotImplementedError()
-
-    def get_from_memo(self, node_name, input_hash):
-        raise NotImplementedError()
-
-    def put_into_memo(self, node_name, input_hash, output):
-        raise NotImplementedError()

indexify/utils.py

@@ -1,7 +0,0 @@
-from enum import Enum
-
-
-def json_set_default(obj):
-    if isinstance(obj, set):
-        return list(obj)
-    raise TypeError

indexify-0.0.43.dist-info/METADATA

@@ -1,66 +0,0 @@
-Metadata-Version: 2.1
-Name: indexify
-Version: 0.0.43
-Summary: Python Client for Indexify
-Home-page: https://github.com/tensorlakeai/indexify
-License: Apache 2.0
-Author: Diptanu Gon Choudhury
-Author-email: diptanuc@gmail.com
-Requires-Python: >=3.9,<4.0
-Classifier: License :: Other/Proprietary License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: cloudpickle (>=3,<4)
-Requires-Dist: httpx[http2] (>=0,<1)
-Requires-Dist: pydantic (>=2.8,<3.0)
-Requires-Dist: pyyaml (>=6,<7)
-Project-URL: Repository, https://github.com/tensorlakeai/indexify
-Description-Content-Type: text/markdown
-
-# Indexify Python Client
-
-
-[![PyPI version](https://badge.fury.io/py/indexify.svg)](https://badge.fury.io/py/indexify)
-
-## Installation
-
-This is the Python client for interacting with the Indexify service.
-
-To install it, simply run:
-
-```shell
-pip install indexify
-```
-
-## Usage
-
-See the [getting started](https://docs.getindexify.com/getting_started/) guide for examples of how to use the client.
-Look at the [examples](https://github.com/tensorlakeai/indexify/tree/main/examples) directory for more examples.
-
-## Development
-
-To install the client from this repository for development:
-
-```shell
-cd "path to this repository"
-pip install -e .
-```
-
-Install and run the `poetry` package manager:
-
-```shell
-pip install poetry
-poetry install
-```
-
-More information at [https://python-poetry.org/docs/](https://python-poetry.org/docs/).
-
-
-
-### Environment Variables
-
-IndexifyClient uses httpx under the hood, so there are many environment variables that can be used to configure the client. More information on supported environment variables can be found [here](https://www.python-httpx.org/environment_variables/).
-

indexify-0.0.43.dist-info/RECORD

@@ -1,25 +0,0 @@
-indexify/__init__.py,sha256=e4s2395B3gEGrZk2u5OZO2RtrXYFYUTItaM3mtlusBE,493
-indexify/base_client.py,sha256=HwT2KJNq8j-KiPVA9RJm-yearSjxifRjXTcP1zUVeo8,2784
-indexify/client.py,sha256=p4WDmYR94DjU0EqosuCKNGjbfh11qUID6TxDhTK6Uk4,26001
-indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
-indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
-indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
-indexify/error.py,sha256=3umTeYb0ugtUyehV1ibfvaeACxAONPyWPc-1HRN4d1M,856
-indexify/exceptions.py,sha256=vjd5SPPNFIEW35GorSIodsqvm9RKHQm9kdp8t9gv-WM,111
-indexify/extraction_policy.py,sha256=927BBtZBDPsLMm01uQDPCZnj3Pwmjh6L6QLHb4ShQKk,2076
-indexify/extractor_sdk/__init__.py,sha256=DOL-wJvIspWPqjFRBpmhMbnsMZC2JY-NtNwQGiE6IqU,348
-indexify/extractor_sdk/data.py,sha256=JpX9WdTpiuK72wn6QYhtqj5p5JiJu4waBrK-Hi7lNsA,2742
-indexify/extractor_sdk/extractor.py,sha256=1SFYXW_vCZt7WdsBfHlfxpuZv6inrsyqRpLCiKvdmX0,9896
-indexify/extractor_sdk/utils.py,sha256=bW_D2eMWTzcAYZ8Lv7LUKGgOD0cyW77E6gNO3y7iNNA,6234
-indexify/extractors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-indexify/extractors/embedding.py,sha256=Be6X4odSHbkAEm2myxB04RN-Mvb2bFk8uWXxUpY-Z6E,1859
-indexify/extractors/pdf_parser.py,sha256=DwHXVbdy-3SDPOo1U_tdp6g5NWs0Tde5h0GoMGTi1WA,2787
-indexify/graph.py,sha256=UdvrpNc-SdD3U27Ee9aTMMYcSOUz__WQWc31oFHV4yQ,3963
-indexify/local_runner.py,sha256=uuMJbnT4qYMSySxsB3lEC7FSjYnJFh5eNZ00zu5gLNw,4387
-indexify/runner.py,sha256=VVmLGF1kAmEuE461Hs0QJFnSvVWtUzYhhQfB1KptYPU,637
-indexify/settings.py,sha256=LSaWZ0ADIVmUv6o6dHWRC3-Ry5uLbCw2sBSg1e_U7UM,99
-indexify/utils.py,sha256=rDN2lrsAs9noJEIjfx6ukmC2SAIyrlUt7QU-kaBjujM,125
-indexify-0.0.43.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.0.43.dist-info/METADATA,sha256=FHsxTsqQwFL7_ezp2EUjjMTj7fP1Oyma4G_AlTQDq7w,1913
-indexify-0.0.43.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-indexify-0.0.43.dist-info/RECORD,,