bulk-chain 1.0.0__tar.gz → 1.1.0__tar.gz

{bulk_chain-1.0.0 → bulk_chain-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bulk_chain
-Version: 1.0.0
+Version: 1.1.0
 Summary: A lightweight, no-strings-attached Chain-of-Thought framework for your LLM, ensuring reliable results for bulk input requests.
 Home-page: https://github.com/nicolay-r/bulk-chain
 Author: Nicolay Rusnachenko
@@ -15,9 +15,8 @@ Classifier: Topic :: Text Processing :: Linguistic
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: tqdm
 
-# bulk-chain 1.0.0
+# bulk-chain 1.1.0
 ![](https://img.shields.io/badge/Python-3.9-brightgreen.svg)
 [![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/nicolay-r/bulk-chain/blob/master/bulk_chain_tutorial.ipynb)
 [![twitter](https://img.shields.io/twitter/url/https/shields.io.svg?style=social)](https://x.com/nicolayr_/status/1847969224636961033)
@@ -78,16 +77,36 @@ Below, is an example on how to declare your own schema:
 
 # Usage
 
-Preliminary steps:
+## 🤖 Prepare 
 
-1. Define your [schema](#chain-of-thought-schema) ([Example for Sentiment Analysis](/ext/schema/thor_cot_schema.json)))
-2. Wrap or pick **LLM model** from the [<b>Third-party providers hosting</b>↗️](https://github.com/nicolay-r/nlp-thirdgate?tab=readme-ov-file#llm).
+1. [schema](#chain-of-thought-schema)
+    * [Example for Sentiment Analysis](test/schema/thor_cot_schema.json)
+2. **LLM model** from the [<b>Third-party providers hosting</b>↗️](https://github.com/nicolay-r/nlp-thirdgate?tab=readme-ov-file#llm).
+3. Data (iter of dictionaries)
 
+## 🚀 Launch
 
+> **API**: For more details see the [**related Wiki page**](https://github.com/nicolay-r/bulk-chain/wiki)
 
-## API
-
-Please take a look at the [**related Wiki page**](https://github.com/nicolay-r/bulk-chain/wiki)
+```python
+from bulk_chain.core.utils import dynamic_init
+from bulk_chain.api import iter_content
+
+content_it = iter_content(
+    # 1. Your schema.              
+    schema="YOUR_SCHEMA.json",
+    # 2. Your third-party model implementation.
+    llm=dynamic_init(class_filepath="replicate_104.py", class_name="Replicate")(api_token="<API-KEY>"),
+    # 3. Customize your inference and result providing modes: 
+    infer_mode="batch_async", 
+    return_mode="batch",
+    # 4. Your iterator of dictionaries
+    input_dicts_it=YOUR_DATA_IT,
+)
+    
+for content in content_it:
+    # Handle your LLM responses here ...
+```
 
 
 # Embed your LLM

{bulk_chain-1.0.0 → bulk_chain-1.1.0}/README.md

@@ -1,4 +1,4 @@
-# bulk-chain 1.0.0
+# bulk-chain 1.1.0
 ![](https://img.shields.io/badge/Python-3.9-brightgreen.svg)
 [![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/nicolay-r/bulk-chain/blob/master/bulk_chain_tutorial.ipynb)
 [![twitter](https://img.shields.io/twitter/url/https/shields.io.svg?style=social)](https://x.com/nicolayr_/status/1847969224636961033)
@@ -59,16 +59,36 @@ Below, is an example on how to declare your own schema:
 
 # Usage
 
-Preliminary steps:
+## 🤖 Prepare 
 
-1. Define your [schema](#chain-of-thought-schema) ([Example for Sentiment Analysis](/ext/schema/thor_cot_schema.json)))
-2. Wrap or pick **LLM model** from the [<b>Third-party providers hosting</b>↗️](https://github.com/nicolay-r/nlp-thirdgate?tab=readme-ov-file#llm).
+1. [schema](#chain-of-thought-schema)
+    * [Example for Sentiment Analysis](test/schema/thor_cot_schema.json)
+2. **LLM model** from the [<b>Third-party providers hosting</b>↗️](https://github.com/nicolay-r/nlp-thirdgate?tab=readme-ov-file#llm).
+3. Data (iter of dictionaries)
 
+## 🚀 Launch
 
+> **API**: For more details see the [**related Wiki page**](https://github.com/nicolay-r/bulk-chain/wiki)
 
-## API
-
-Please take a look at the [**related Wiki page**](https://github.com/nicolay-r/bulk-chain/wiki)
+```python
+from bulk_chain.core.utils import dynamic_init
+from bulk_chain.api import iter_content
+
+content_it = iter_content(
+    # 1. Your schema.              
+    schema="YOUR_SCHEMA.json",
+    # 2. Your third-party model implementation.
+    llm=dynamic_init(class_filepath="replicate_104.py", class_name="Replicate")(api_token="<API-KEY>"),
+    # 3. Customize your inference and result providing modes: 
+    infer_mode="batch_async", 
+    return_mode="batch",
+    # 4. Your iterator of dictionaries
+    input_dicts_it=YOUR_DATA_IT,
+)
+    
+for content in content_it:
+    # Handle your LLM responses here ...
+```
 
 
 # Embed your LLM

bulk_chain-1.1.0/bulk_chain/api.py

@@ -0,0 +1,186 @@
+import asyncio
+import collections
+import logging
+import os
+from itertools import chain
+
+from bulk_chain.core.llm_base import BaseLM
+from bulk_chain.core.service_asyncio import AsyncioService
+from bulk_chain.core.service_batch import BatchIterator
+from bulk_chain.core.service_data import DataService
+from bulk_chain.core.service_dict import DictionaryService
+from bulk_chain.core.service_json import JsonService
+from bulk_chain.core.service_schema import SchemaService
+from bulk_chain.core.utils import attempt_wrapper
+
+
+INFER_MODES = {
+    "single": lambda llm, batch, **kwargs: [llm.ask(prompt) for prompt in batch],
+    "single_stream": lambda llm, batch, **kwargs: [llm.ask_stream(prompt) for prompt in batch],
+    "batch": lambda llm, batch, **kwargs: llm.ask(batch),
+    "batch_async": lambda llm, batch, **kwargs: AsyncioService.run_tasks(
+        batch=batch, async_handler=llm.ask_async, event_loop=kwargs.get("event_loop")
+    ),
+    "batch_stream_async": lambda llm, batch, **kwargs: AsyncioService.run_tasks(
+        batch=batch, async_handler=llm.ask_stream_async, event_loop=kwargs.get("event_loop")
+    ),
+}
+
+
+CWD = os.getcwd()
+
+
+def _iter_batch_prompts(c, batch_content_it, **kwargs):
+    for ind_in_batch, entry in enumerate(batch_content_it):
+        content = DataService.get_prompt_text(
+            prompt=entry[c]["prompt"],
+            data_dict=entry,
+            handle_missed_func=kwargs["handle_missed_value_func"])
+        yield ind_in_batch, content
+
+
+def __handle_agen_to_gen(handle, batch, event_loop):
+    """ This handler provides conversion of the async generator to generator (sync).
+    """
+
+    def __wrap_with_index(async_gens):
+        async def wrapper(index, agen):
+            async for item in agen:
+                yield index, item
+        return [wrapper(i, agen) for i, agen in enumerate(async_gens)]
+
+    agen_list = handle(batch, event_loop=event_loop)
+
+    it = AsyncioService.async_gen_to_iter(
+        gen=AsyncioService.merge_generators(*__wrap_with_index(agen_list)),
+        loop=event_loop)
+
+    for ind_in_batch, chunk in it:
+        yield ind_in_batch, str(chunk)
+
+
+def __handle_gen(handle, batch, event_loop):
+    """ This handler deals with the iteration of each individual element of the batch.
+    """
+
+    def _iter_entry_content(entry):
+        if isinstance(entry, str):
+            yield entry
+        elif isinstance(entry, collections.abc.Iterable):
+            for chunk in map(lambda item: str(item), entry):
+                yield chunk
+        else:
+            raise Exception(f"Non supported type `{type(entry)}` for handling output from batch")
+
+    for ind_in_batch, entry in enumerate(handle(batch, event_loop=event_loop)):
+        for chunk in _iter_entry_content(entry=entry):
+            yield ind_in_batch, chunk
+
+
+def _iter_chunks(p_column, batch_content_it, **kwargs):
+    handler = __handle_agen_to_gen if kwargs["infer_mode"] == "batch_stream_async" else __handle_gen
+    p_batch = [item[p_column] for item in batch_content_it]
+    it = handler(handle=kwargs["handle_batch_func"], batch=p_batch, event_loop=kwargs["event_loop"])
+    for ind_in_batch, chunk in it:
+        yield ind_in_batch, chunk
+
+
+def _infer_batch(batch, batch_ind, schema, return_mode, cols=None, **kwargs):
+    assert (isinstance(batch, list))
+
+    if len(batch) == 0:
+        return batch
+
+    if cols is None:
+        first_item = batch[0]
+        cols = list(first_item.keys()) if cols is None else cols
+
+    for c in cols:
+
+        # Handling prompt column.
+        if c in schema.p2r:
+            content_it = _iter_batch_prompts(c=c, batch_content_it=iter(batch), **kwargs)
+            for ind_in_batch, prompt in content_it:
+                batch[ind_in_batch][c] = prompt
+
+        # Handling column for inference.
+        if c in schema.r2p:
+            content_it = _iter_chunks(p_column=schema.r2p[c], batch_content_it=iter(batch), **kwargs)
+            # Register values.
+            for item in batch:
+                item[c] = []
+            for ind_in_batch, chunk in content_it:
+                # Append batch.
+                batch[ind_in_batch][c].append(chunk)
+                # Returning (optional).
+                if return_mode == "chunk":
+                    global_ind = batch_ind * len(batch) + ind_in_batch
+                    yield [global_ind, c, chunk]
+
+            # Convert content to string.
+            for item in batch:
+                item[c] = "".join(item[c])
+
+    if return_mode == "record":
+        for record in batch:
+            yield record
+
+    if return_mode == "batch":
+        yield batch
+
+
+def iter_content(input_dicts_it, llm, schema, batch_size=1, limit_prompt=None,
+                 infer_mode="batch", return_mode="batch", attempts=1, event_loop=None,
+                 **kwargs):
+    """ This method represent Python API aimed at application of `llm` towards
+        iterator of input_dicts via cache_target that refers to the SQLite using
+        the given `schema`
+    """
+    assert (infer_mode in INFER_MODES.keys())
+    assert (return_mode in ["batch", "chunk", "record"])
+    assert (isinstance(llm, BaseLM))
+
+    # Setup event loop.
+    event_loop = asyncio.get_event_loop_policy().get_event_loop() \
+        if event_loop is None else event_loop
+
+    # Quick initialization of the schema.
+    if isinstance(schema, str):
+        schema = JsonService.read(schema)
+    if isinstance(schema, dict):
+        schema = SchemaService(json_data=schema)
+
+    prompts_it = map(
+        lambda data: DictionaryService.custom_update(src_dict=dict(data), other_dict=schema.cot_args),
+        input_dicts_it
+    )
+
+    handle_batch_func = lambda batch, **handle_kwargs: INFER_MODES[infer_mode](
+        llm,
+        DataService.limit_prompts(batch, limit=limit_prompt),
+        **handle_kwargs
+    )
+
+    # Optional wrapping into attempts.
+    if attempts > 1:
+        # Optional setup of the logger.
+        logger = logging.getLogger(__name__)
+        logging.basicConfig(level=logging.INFO)
+
+        attempt_dec = attempt_wrapper(attempts=attempts,
+                                      delay_sec=kwargs.get("attempt_delay_sec", 1),
+                                      logger=logger)
+        handle_batch_func = attempt_dec(handle_batch_func)
+
+    content_it = (_infer_batch(batch=batch,
+                               batch_ind=batch_ind,
+                               infer_mode=infer_mode,
+                               handle_batch_func=handle_batch_func,
+                               handle_missed_value_func=lambda *_: None,
+                               return_mode=return_mode,
+                               schema=schema,
+                               event_loop=event_loop,
+                               **kwargs)
+                  for batch_ind, batch in enumerate(BatchIterator(prompts_it, batch_size=batch_size)))
+
+    yield from chain.from_iterable(content_it)

bulk_chain-1.1.0/bulk_chain/core/llm_base.py

@@ -0,0 +1,24 @@
+class BaseLM(object):
+
+    def __init__(self, **kwargs):
+        pass
+
+    def ask(self, content):
+        """ Assumes to return str.
+        """
+        raise NotImplemented()
+
+    def ask_stream(self, content):
+        """ Assumes to return generator.
+        """
+        raise NotImplemented()
+
+    async def ask_async(self, prompt):
+        """ Assumes to return co-routine.
+        """
+        raise NotImplemented()
+
+    async def ask_stream_async(self, batch):
+        """ Assumes to return AsyncGenerator.
+        """
+        raise NotImplemented()

bulk_chain-1.1.0/bulk_chain/core/service_asyncio.py

@@ -0,0 +1,65 @@
+import asyncio
+from typing import AsyncGenerator, Any
+
+
+class AsyncioService:
+
+    @staticmethod
+    async def _run_tasks_async(batch, async_handler):
+        tasks = [async_handler(prompt) for prompt in batch]
+        return await asyncio.gather(*tasks)
+
+    @staticmethod
+    async def _run_generator(gen, output_queue, idx):
+        try:
+            async for item in gen:
+                await output_queue.put((idx, item))
+        finally:
+            await output_queue.put((idx, StopAsyncIteration))
+
+
+    @staticmethod
+    def run_tasks(event_loop, **tasks_kwargs):
+        return event_loop.run_until_complete(AsyncioService._run_tasks_async(**tasks_kwargs))
+
+    @staticmethod
+    async def merge_generators(*gens: AsyncGenerator[Any, None]) -> AsyncGenerator[Any, None]:
+
+        output_queue = asyncio.Queue()
+        tasks = [
+            asyncio.create_task(AsyncioService._run_generator(gen, output_queue, idx))
+            for idx, gen in enumerate(gens)
+        ]
+
+        finished = set()
+        while len(finished) < len(tasks):
+            idx, item = await output_queue.get()
+            if item is StopAsyncIteration:
+                finished.add(idx)
+            else:
+                yield item
+
+        for task in tasks:
+            task.cancel()
+
+    @staticmethod
+    def async_gen_to_iter(gen, loop=None):
+        """ This approach is limited. Could be considered as legacy.
+            https://stackoverflow.com/questions/71580727/translating-async-generator-into-sync-one/78573267#78573267
+        """
+
+        loop_created = False
+        if loop is None:
+            loop_created = True
+            loop = asyncio.new_event_loop()
+
+        asyncio.set_event_loop(loop)
+        try:
+            while True:
+                try:
+                    yield loop.run_until_complete(gen.__anext__())
+                except StopAsyncIteration:
+                    break
+        finally:
+            if loop_created:
+                loop.close()

{bulk_chain-1.0.0 → bulk_chain-1.1.0}/bulk_chain/core/service_batch.py

@@ -1,8 +1,8 @@
 class BatchIterator:
 
     def __init__(self, data_iter, batch_size, end_value=None, filter_func=None):
-        assert(isinstance(batch_size, int) and batch_size > 0)
-        assert(callable(end_value) or end_value is None)
+        assert (isinstance(batch_size, int) and batch_size > 0)
+        assert (callable(end_value) or end_value is None)
         self.__data_iter = data_iter
         self.__index = 0
         self.__batch_size = batch_size

{bulk_chain-1.0.0 → bulk_chain-1.1.0}/bulk_chain/core/utils.py

@@ -1,6 +1,7 @@
 import importlib
 import logging
 import sys
+import time
 from collections import Counter
 from os.path import dirname, join, basename
 
@@ -60,10 +61,10 @@ def auto_import(name, is_class=False):
     return m() if is_class else m
 
 
-def dynamic_init(class_dir, class_filepath, class_name=None):
+def dynamic_init(class_filepath, class_name=None):
 
     # Registering path.
-    target = join(class_dir, dirname(class_filepath))
+    target = join(dirname(class_filepath))
     logger.info(f"Adding sys path for `{target}`")
     sys.path.insert(1, target)
     class_path_list = class_filepath.split('/')
@@ -89,3 +90,21 @@ def optional_limit_iter(it_data, limit=None):
         if limit is not None and counter["returned"] > limit:
             break
         yield data
+
+
+def attempt_wrapper(attempts, delay_sec=1, logger=None):
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            for i in range(attempts):
+                try:
+                    # Do action.
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    if logger is not None:
+                        logger.info(f"Unable to infer the result. Try {i} out of {attempts}.")
+                        logger.info(e)
+                    if delay_sec is not None:
+                        time.sleep(delay_sec)
+            raise Exception(f"Failed after {attempts} attempts")
+        return wrapper
+    return decorator

{bulk_chain-1.0.0 → bulk_chain-1.1.0}/bulk_chain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bulk_chain
-Version: 1.0.0
+Version: 1.1.0
 Summary: A lightweight, no-strings-attached Chain-of-Thought framework for your LLM, ensuring reliable results for bulk input requests.
 Home-page: https://github.com/nicolay-r/bulk-chain
 Author: Nicolay Rusnachenko
@@ -15,9 +15,8 @@ Classifier: Topic :: Text Processing :: Linguistic
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: tqdm
 
-# bulk-chain 1.0.0
+# bulk-chain 1.1.0
 ![](https://img.shields.io/badge/Python-3.9-brightgreen.svg)
 [![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/nicolay-r/bulk-chain/blob/master/bulk_chain_tutorial.ipynb)
 [![twitter](https://img.shields.io/twitter/url/https/shields.io.svg?style=social)](https://x.com/nicolayr_/status/1847969224636961033)
@@ -78,16 +77,36 @@ Below, is an example on how to declare your own schema:
 
 # Usage
 
-Preliminary steps:
+## 🤖 Prepare 
 
-1. Define your [schema](#chain-of-thought-schema) ([Example for Sentiment Analysis](/ext/schema/thor_cot_schema.json)))
-2. Wrap or pick **LLM model** from the [<b>Third-party providers hosting</b>↗️](https://github.com/nicolay-r/nlp-thirdgate?tab=readme-ov-file#llm).
+1. [schema](#chain-of-thought-schema)
+    * [Example for Sentiment Analysis](test/schema/thor_cot_schema.json)
+2. **LLM model** from the [<b>Third-party providers hosting</b>↗️](https://github.com/nicolay-r/nlp-thirdgate?tab=readme-ov-file#llm).
+3. Data (iter of dictionaries)
 
+## 🚀 Launch
 
+> **API**: For more details see the [**related Wiki page**](https://github.com/nicolay-r/bulk-chain/wiki)
 
-## API
-
-Please take a look at the [**related Wiki page**](https://github.com/nicolay-r/bulk-chain/wiki)
+```python
+from bulk_chain.core.utils import dynamic_init
+from bulk_chain.api import iter_content
+
+content_it = iter_content(
+    # 1. Your schema.              
+    schema="YOUR_SCHEMA.json",
+    # 2. Your third-party model implementation.
+    llm=dynamic_init(class_filepath="replicate_104.py", class_name="Replicate")(api_token="<API-KEY>"),
+    # 3. Customize your inference and result providing modes: 
+    infer_mode="batch_async", 
+    return_mode="batch",
+    # 4. Your iterator of dictionaries
+    input_dicts_it=YOUR_DATA_IT,
+)
+    
+for content in content_it:
+    # Handle your LLM responses here ...
+```
 
 
 # Embed your LLM

{bulk_chain-1.0.0 → bulk_chain-1.1.0}/bulk_chain.egg-info/SOURCES.txt

@@ -6,18 +6,18 @@ bulk_chain/api.py
 bulk_chain.egg-info/PKG-INFO
 bulk_chain.egg-info/SOURCES.txt
 bulk_chain.egg-info/dependency_links.txt
-bulk_chain.egg-info/requires.txt
 bulk_chain.egg-info/top_level.txt
 bulk_chain/core/__init__.py
 bulk_chain/core/llm_base.py
+bulk_chain/core/service_asyncio.py
 bulk_chain/core/service_batch.py
 bulk_chain/core/service_data.py
 bulk_chain/core/service_dict.py
 bulk_chain/core/service_json.py
 bulk_chain/core/service_schema.py
 bulk_chain/core/utils.py
-test/test.py
 test/test_api.py
+test/test_api_batching.py
 test/test_api_streaming.py
-test/test_args_seeking.py
-test/test_provider_batching.py
+test/test_replicate_async_baseline.py
+test/test_replicate_async_batch_async.py

{bulk_chain-1.0.0 → bulk_chain-1.1.0}/setup.py

@@ -15,7 +15,7 @@ def get_requirements(filenames):
 
 setup(
     name='bulk_chain',
-    version='1.0.0',
+    version='1.1.0',
     python_requires=">=3.6",
     description='A lightweight, no-strings-attached Chain-of-Thought framework for your LLM, '
                 'ensuring reliable results for bulk input requests.',
@@ -36,5 +36,4 @@ setup(
              'chain-of-thought, '
              'reasoning',
     packages=find_packages(),
-    install_requires=get_requirements(['dependencies.txt'])
 )

bulk_chain-1.1.0/test/test_api.py

@@ -0,0 +1,67 @@
+import unittest
+from os.path import join
+
+from bulk_chain.api import iter_content
+from utils import current_dir, DEFAULT_REMOTE_LLM
+
+
+class TestAPI(unittest.TestCase):
+
+
+    @staticmethod
+    def it_data(n):
+        for i in range(n):
+            yield {"ind": i, "text": "X invent sanctions against Y", "entity": "X"}
+
+    def test_single(self):
+        data_it = iter_content(input_dicts_it=self.it_data(20),
+                               llm=DEFAULT_REMOTE_LLM,
+                               batch_size=1,
+                               infer_mode="single",
+                               return_mode="batch",
+                               schema=join(current_dir, "schema/default.json"))
+
+        for data in data_it:
+            print(data)
+
+    def test_single_stream(self):
+        """ Returns individual chunks.
+        """
+        data_it = iter_content(input_dicts_it=self.it_data(20),
+                               llm=DEFAULT_REMOTE_LLM,
+                               batch_size=1,
+                               infer_mode="single_stream",
+                               return_mode="chunk",
+                               schema=join(current_dir, "schema/default.json"))
+
+        for data in data_it:
+            print(data)
+
+    def test_batch_async(self):
+        """ Return batches that passed async at the Replicate.
+        """
+        data_it = iter_content(input_dicts_it=self.it_data(20),
+                               llm=DEFAULT_REMOTE_LLM,
+                               batch_size=5,
+                               infer_mode="batch_async",
+                               return_mode="batch",
+                               schema=join(current_dir, "schema/default.json"))
+
+        for batch in data_it:
+            for item in batch:
+                print(item)
+
+    def test_batch_stream_async(self):
+        data_it = iter_content(input_dicts_it=self.it_data(20),
+                               llm=DEFAULT_REMOTE_LLM,
+                               batch_size=5,
+                               infer_mode="batch_stream_async",
+                               return_mode="chunk",
+                               schema=join(current_dir, "schema/default.json"))
+
+        for chunk_info in data_it:
+            print(chunk_info)
+
+
+if __name__ == '__main__':
+    unittest.main()

bulk_chain-1.0.0/test/test_provider_batching.py → bulk_chain-1.1.0/test/test_api_batching.py

@@ -1,17 +1,15 @@
 import unittest
-from os.path import join
 
 from tqdm import tqdm
 
-from bulk_chain.api import CWD, iter_content
+from bulk_chain.api import iter_content
 from bulk_chain.core.utils import dynamic_init
 from utils import iter_test_jsonl_samples
 
 
 class TestProviderBatching(unittest.TestCase):
 
-    llm = dynamic_init(class_dir=join(CWD),
-                       class_filepath="providers/transformers_flan_t5.py",
+    llm = dynamic_init(class_filepath="providers/transformers_flan_t5.py",
                        class_name="FlanT5")(model_name="nicolay-r/flan-t5-tsa-thor-base",
                                             max_new_tokens=128)
 
@@ -19,9 +17,9 @@ class TestProviderBatching(unittest.TestCase):
         input_dicts_it = iter_test_jsonl_samples()
         data_it = iter_content(input_dicts_it=input_dicts_it,
                                llm=self.llm,
+                               infer_mode="batch",
                                batch_size=10,
                                return_batch=False,
-                               handle_missed_value_func=lambda *_: None,
                                schema="schema/thor_cot_schema.json")
 
         for item in tqdm(data_it):

bulk_chain-1.1.0/test/test_api_streaming.py

@@ -0,0 +1,21 @@
+import unittest
+
+from bulk_chain.api import iter_content
+from utils import iter_test_jsonl_samples, DEFAULT_REMOTE_LLM
+
+
+class TestAPI_Streaming(unittest.TestCase):
+
+    def test_content_iter_mode(self):
+
+        input_dicts_it = iter_test_jsonl_samples()
+        data_it = iter_content(input_dicts_it=input_dicts_it,
+                               llm=DEFAULT_REMOTE_LLM,
+                               batch_size=1,
+                               infer_mode="single_stream",
+                               return_mode="chunk",
+                               attempts=2,
+                               schema="schema/thor_cot_schema.json")
+
+        for ind_in_batch, col, item in data_it:
+            print("\t".join([str(ind_in_batch), str(col), item]))

bulk_chain-1.1.0/test/test_replicate_async_baseline.py

@@ -0,0 +1,11 @@
+from timeit import default_timer as timer
+from utils import DEFAULT_REMOTE_LLM
+
+start = timer()
+r = ["".join([str(s) for s in DEFAULT_REMOTE_LLM.ask(f"what's the color of the {p}")])
+     for p in ["sky", "ground", "water"]]
+end = timer()
+
+total = sum(len(i) for i in r)
+print(f"Completed [time: {end - start}]: {len(r[0])}, {len(r[1])}, {len(r[2])}")
+print(f"TPS: {total / (end-start)}")

bulk_chain-1.1.0/test/test_replicate_async_batch_async.py

@@ -0,0 +1,37 @@
+from timeit import default_timer as timer
+import asyncio
+
+from utils import DEFAULT_REMOTE_LLM
+
+
+async def infer_item(prompt):
+    content = []
+    for chunk in DEFAULT_REMOTE_LLM.ask(prompt):
+        content.append(str(chunk))
+    return content
+
+
+async def coro_infer_llm(prompt):
+    print(f"launch: {prompt}")
+    r = None
+    for response in asyncio.as_completed([infer_item(prompt)]):
+        r = await response
+    return "".join(r)
+
+
+async def main():
+    batch = [f"what's the color of the {p}" for p in ["sky", "ground", "water"]]
+    routines = [coro_infer_llm(p) for p in batch]
+    return await asyncio.gather(*routines)
+
+
+start = timer()
+r = asyncio.run(main())
+end = timer()
+
+total = sum(len(i) for i in r)
+print(r[0])
+print(r[1])
+print(r[2])
+print(f"Completed [time: {end - start}]: {len(r[0])}, {len(r[1])}, {len(r[2])}")
+print(f"TPS: {total / (end-start)}")

bulk_chain-1.0.0/bulk_chain/api.py

@@ -1,143 +0,0 @@
-import collections
-import os
-from itertools import chain
-
-from bulk_chain.core.llm_base import BaseLM
-from bulk_chain.core.service_batch import BatchIterator
-from bulk_chain.core.service_data import DataService
-from bulk_chain.core.service_dict import DictionaryService
-from bulk_chain.core.service_json import JsonService
-from bulk_chain.core.service_schema import SchemaService
-from bulk_chain.core.utils import dynamic_init, find_by_prefix
-
-
-INFER_MODES = {
-    "batch": lambda llm, batch, limit_prompt=None: llm.ask_core(
-        DataService.limit_prompts(batch, limit=limit_prompt))
-}
-
-
-CWD = os.getcwd()
-
-
-def _iter_entry_content(entry, entry_info=None, **kwargs):
-
-    if isinstance(entry, str):
-        kwargs.get("callback_str_func", lambda *_: None)(entry, entry_info)
-        yield entry
-    elif isinstance(entry, collections.abc.Iterable):
-        h = kwargs.get("callback_stream_func", lambda *_: None)
-        h(None, entry_info | {"action": "start"})
-        for chunk in map(lambda item: str(item), entry):
-            yield chunk
-            h(chunk, entry_info)
-        h(None, entry_info | {"action": "end"})
-    else:
-        raise Exception(f"Non supported type `{type(entry)}` for handling output from batch")
-
-
-def _iter_batch_prompts(c, batch_content_it, **kwargs):
-    for ind_in_batch, entry in enumerate(batch_content_it):
-        content = DataService.get_prompt_text(
-            prompt=entry[c]["prompt"],
-            data_dict=entry,
-            handle_missed_func=kwargs["handle_missed_value_func"])
-        yield ind_in_batch, content
-
-
-def _iter_batch_responses(p_column, c, batch_content_it, **kwargs):
-    p_batch = [item[p_column] for item in batch_content_it]
-    # TODO. This part could be async.
-    # TODO. ind_in_batch might be a part of the async return.
-    for ind_in_batch, entry in enumerate(kwargs["handle_batch_func"](p_batch)):
-        yield ind_in_batch, _iter_entry_content(entry=entry, entry_info={"ind": ind_in_batch, "param": c}, **kwargs)
-
-
-def _infer_batch(batch, schema, return_mode, cols=None, **kwargs):
-    assert (isinstance(batch, list))
-
-    if len(batch) == 0:
-        return batch
-
-    if cols is None:
-        first_item = batch[0]
-        cols = list(first_item.keys()) if cols is None else cols
-
-    for c in cols:
-
-        # Handling prompt column.
-        if c in schema.p2r:
-            content_it = _iter_batch_prompts(c=c, batch_content_it=iter(batch), **kwargs)
-            for ind_in_batch, prompt in content_it:
-                batch[ind_in_batch][c] = prompt
-
-        # Handling column for inference.
-        if c in schema.r2p:
-            content_it = _iter_batch_responses(c=c, p_column=schema.r2p[c], batch_content_it=iter(batch), **kwargs)
-            for ind_in_batch, chunk_it in content_it:
-
-                chunks = []
-                for chunk in chunk_it:
-                    chunks.append(chunk)
-
-                    if return_mode == "chunk":
-                        yield [ind_in_batch, c, chunk]
-
-                batch[ind_in_batch][c] = "".join(chunks)
-
-    if return_mode == "record":
-        for record in batch:
-            yield record
-
-    if return_mode == "batch":
-        yield batch
-
-
-def iter_content(input_dicts_it, llm, schema, batch_size=1, limit_prompt=None, return_mode="batch", **kwargs):
-    """ This method represent Python API aimed at application of `llm` towards
-        iterator of input_dicts via cache_target that refers to the SQLite using
-        the given `schema`
-    """
-    assert (return_mode in ["batch", "chunk"])
-    assert (isinstance(llm, BaseLM))
-
-    # Quick initialization of the schema.
-    if isinstance(schema, str):
-        schema = JsonService.read(schema)
-    if isinstance(schema, dict):
-        schema = SchemaService(json_data=schema)
-
-    prompts_it = map(
-        lambda data: DictionaryService.custom_update(src_dict=dict(data), other_dict=schema.cot_args),
-        input_dicts_it
-    )
-
-    content_it = (_infer_batch(batch=batch,
-                               handle_batch_func=lambda batch: INFER_MODES["batch"](llm, batch, limit_prompt),
-                               return_mode=return_mode,
-                               schema=schema,
-                               **kwargs)
-                  for batch in BatchIterator(prompts_it, batch_size=batch_size))
-
-    yield from chain.from_iterable(content_it)
-
-
-def init_llm(adapter, **model_kwargs):
-    """ This method perform dynamic initialization of LLM from third-party resource.
-    """
-    assert (isinstance(adapter, str))
-
-    # List of the Supported models and their API wrappers.
-    models_preset = {
-        "dynamic": lambda: dynamic_init(class_dir=CWD, class_filepath=llm_model_name,
-                                        class_name=llm_model_params)(**model_kwargs)
-    }
-
-    # Initialize LLM model.
-    params = adapter.split(':')
-    llm_model_type = params[0]
-    llm_model_name = params[1] if len(params) > 1 else params[-1]
-    llm_model_params = ':'.join(params[2:]) if len(params) > 2 else None
-    llm = find_by_prefix(d=models_preset, key=llm_model_type)()
-
-    return llm, llm_model_name

bulk_chain-1.0.0/bulk_chain/core/llm_base.py

@@ -1,50 +0,0 @@
-import logging
-import time
-
-
-class BaseLM(object):
-
-    def __init__(self, name=None, attempts=None, delay_sec=1, enable_log=True,
-                 support_batching=False, **kwargs):
-
-        self.__name = name
-        self.__attempts = 1 if attempts is None else attempts
-        self.__delay_sec = delay_sec
-        self.__support_batching = support_batching
-
-        if enable_log:
-            self.__logger = logging.getLogger(__name__)
-            logging.basicConfig(level=logging.INFO)
-
-    def ask_core(self, batch):
-
-        for i in range(self.__attempts):
-            try:
-                if self.__support_batching:
-                    # Launch in batch mode.
-                    content = batch
-                else:
-                    # Launch in non-batch mode.
-                    assert len(batch) == 1, "The LM does not support batching," \
-                                            f" while size of the content is {len(batch)} which is not equal 1. " \
-                                            f"Please enable batch-supporting or set required inference settings."
-                    content = batch[0]
-
-                response = self.ask(content)
-
-                # Wrapping into batch the response in the case of non-batching mode.
-                return response if self.__support_batching else [response]
-
-            except Exception as e:
-                if self.__logger is not None:
-                    self.__logger.info("Unable to infer the result. Try {} out of {}.".format(i, self.__attempts))
-                    self.__logger.info(e)
-                time.sleep(self.__delay_sec)
-
-        raise Exception("Can't infer")
-
-    def ask(self, content):
-        raise NotImplemented()
-
-    def name(self):
-        return self.__name.replace("/", "_")

bulk_chain-1.0.0/bulk_chain.egg-info/requires.txt

@@ -1 +0,0 @@
-tqdm

bulk_chain-1.0.0/test/test.py

@@ -1,62 +0,0 @@
-import random
-import time
-import logging
-
-
-def setup_logger_behaviour(name: str) -> logging.Logger:
-    root_handlers = logging.getLogger().handlers # gets root logger
-    current_logger = logging.getLogger(name)   # gets current logger
-    if not root_handlers:  # if root logger has no handlers then create streaming handeler only
-        new_handler = logging.StreamHandler()
-        new_handler.terminator = ""
-        new_handler.setFormatter(logging.Formatter("%(message)s"))
-        current_logger.addHandler(new_handler)
-        current_logger.propagate = False
-        current_logger.setLevel(logging.INFO)
-        return current_logger
-
-    # Remove  exixting Handlers from the current logger
-    for handler in current_logger.handlers[:]:
-        current_logger.removeHandler(handler)
-
-    for handler_r in root_handlers:  # if root logger has handlers
-        if type(handler_r) is logging.StreamHandler:  # if root logger has streaming handler
-            new_handler = logging.StreamHandler() 
-            new_handler.terminator = ""   # This will stop the printing in new line
-            new_handler.setFormatter(logging.Formatter("%(message)s")) # This will set the format
-            current_logger.addHandler(new_handler)
-        elif type(handler_r) is logging.FileHandler:  # if root logger has file handler
-            new_handler = logging.FileHandler( # create new file handler
-                handler_r.baseFilename,        # with same filename and other properties
-                handler_r.mode,
-                handler_r.encoding,
-                handler_r.delay,
-                handler_r.errors,
-            )
-            new_handler.terminator = ""  # This will stop the printing in new line
-            new_handler.setFormatter(logging.Formatter("%(message)s")) # This will set the format
-            current_logger.addHandler(new_handler)
-        else:
-            continue
-    current_logger.propagate = False  # Don't propagate to root logger
-    return current_logger
-
-# Configure the logger
-logger =logging.getLogger(__name__)
-class FakeStreamingDataGenerator:
-
-    def stream_data(self):
-        while True:
-            data = random.randint(0, 100)
-            yield data
-            time.sleep(0.5)
-
-# Example usage:
-generator = FakeStreamingDataGenerator()
-stream = generator.stream_data()
-
-logger = setup_logger_behaviour(__name__) # call you set up function here
-while True:
-    chunk = next(stream)
-    # Replacing print with logger
-    logger.info(chunk)  # Best practice now

bulk_chain-1.0.0/test/test_api.py

@@ -1,34 +0,0 @@
-import unittest
-from os.path import join
-
-from bulk_chain.api import iter_content, CWD
-from bulk_chain.core.utils import dynamic_init
-from utils import current_dir, API_TOKEN
-
-
-class TestAPI(unittest.TestCase):
-
-    llm = dynamic_init(class_dir=join(CWD, ".."),
-                       class_filepath="providers/replicate_104.py",
-                       class_name="Replicate")(api_token=API_TOKEN,
-                                               model_name="deepseek-ai/deepseek-r1")
-
-    @staticmethod
-    def it_data(n):
-        for i in range(n):
-            yield {"ind": i, "text": "X invent sanctions against Y", "entity": "X"}
-
-    def test_iter(self):
-        data_it = iter_content(input_dicts_it=self.it_data(20),
-                               llm=self.llm,
-                               batch_size=1,
-                               handle_missed_value_func=lambda *_: None,
-                               return_mode="batch",
-                               schema=join(current_dir, "schema/default.json"))
-
-        for data in data_it:
-            print(data)
-
-
-if __name__ == '__main__':
-    unittest.main()

bulk_chain-1.0.0/test/test_api_streaming.py

@@ -1,52 +0,0 @@
-import unittest
-from os.path import join
-
-from tqdm import tqdm
-
-from bulk_chain.api import CWD, iter_content
-from bulk_chain.core.utils import dynamic_init
-from utils import API_TOKEN, iter_test_jsonl_samples
-
-
-class TestAPI_Streaming(unittest.TestCase):
-
-    llm = dynamic_init(class_dir=join(CWD, ".."),
-                       class_filepath="providers/replicate_104.py",
-                       class_name="Replicate")(api_token=API_TOKEN,
-                                               model_name="meta/meta-llama-3-70b-instruct",
-                                               stream=True)
-
-    def test_callback_mode(self):
-
-        def callback(chunk, info):
-            if chunk is None and info["action"] == "start":
-                print(f"\n{info['param']} (batch_ind={info['ind']}):\n")
-                return
-            if chunk is None and info["action"] == "end":
-                print("\n\n")
-                return
-            print(chunk, end="")
-
-        input_dicts_it = iter_test_jsonl_samples()
-        data_it = iter_content(input_dicts_it=input_dicts_it,
-                               llm=self.llm,
-                               return_batch=False,
-                               callback_stream_func=callback,
-                               handle_missed_value_func=lambda *_: None,
-                               schema="schema/thor_cot_schema.json")
-
-        for _ in tqdm(data_it):
-            print("\n|NEXT ENTRY|\n")
-
-    def test_content_iter_mode(self):
-
-        input_dicts_it = iter_test_jsonl_samples()
-        data_it = iter_content(input_dicts_it=input_dicts_it,
-                               llm=self.llm,
-                               batch_size=1,
-                               return_mode="chunk",
-                               handle_missed_value_func=lambda *_: None,
-                               schema="schema/thor_cot_schema.json")
-
-        for ind_in_batch, col, item in data_it:
-            print("\t".join([str(ind_in_batch), str(col), item]))

bulk_chain-1.0.0/test/test_args_seeking.py

@@ -1,26 +0,0 @@
-import unittest
-
-from bulk_chain.core.utils import iter_params
-
-
-class TestArgumentsSeeking(unittest.TestCase):
-
-    def test(self):
-        params = list(iter_params("X is a {x} and p is {text} and for {k}"))
-
-        line = ",".join(["{{{x}}}".format(x=x) for x in params])
-        print(line)
-        d_params = {}
-        for param in params:
-            d_params[param] = 2
-        print(d_params)
-
-        z = line.format(**d_params)
-        print(z)
-
-        b = list(iter_params("X"))
-        print(b)
-
-
-if __name__ == '__main__':
-    unittest.main()