praisonaiagents 0.0.73__py3-none-any.whl → 0.0.74__py3-none-any.whl

praisonaiagents/knowledge/chunking.py

@@ -7,21 +7,19 @@ class Chunking:
     
     CHUNKER_PARAMS = {
         'token': ['chunk_size', 'chunk_overlap', 'tokenizer'],
-        'word': ['chunk_size', 'chunk_overlap', 'tokenizer'],
-        'sentence': ['chunk_size', 'chunk_overlap', 'tokenizer'],
-        'semantic': ['chunk_size', 'embedding_model', 'tokenizer'],
-        'sdpm': ['chunk_size', 'embedding_model', 'tokenizer'],
-        'late': ['chunk_size', 'embedding_model', 'tokenizer'],
-        'recursive': ['chunk_size', 'tokenizer']
+        'sentence': ['chunk_size', 'chunk_overlap', 'tokenizer_or_token_counter'],
+        'recursive': ['chunk_size', 'tokenizer_or_token_counter'],
+        'semantic': ['chunk_size', 'embedding_model'],
+        'sdpm': ['chunk_size', 'embedding_model'],
+        'late': ['chunk_size', 'embedding_model'],
     }
     
     @cached_property
     def SUPPORTED_CHUNKERS(self) -> Dict[str, Any]:
         """Lazy load chunker classes."""
         try:
-            from chonkie.chunker import (
+            from chonkie import (
                 TokenChunker,
-                WordChunker,
                 SentenceChunker,
                 SemanticChunker,
                 SDPMChunker,
@@ -35,7 +33,6 @@ class Chunking:
             
         return {
             'token': TokenChunker,
-            'word': WordChunker,
             'sentence': SentenceChunker,
             'semantic': SemanticChunker,
             'sdpm': SDPMChunker,
@@ -48,7 +45,7 @@ class Chunking:
         chunker_type: str = 'token',
         chunk_size: int = 512,
         chunk_overlap: int = 128,
-        tokenizer: str = "gpt2",
+        tokenizer_or_token_counter: str = "gpt2",
         embedding_model: Optional[Union[str, Any]] = None,
         **kwargs
     ):
@@ -62,7 +59,7 @@ class Chunking:
         self.chunker_type = chunker_type
         self.chunk_size = chunk_size
         self.chunk_overlap = chunk_overlap
-        self.tokenizer = tokenizer
+        self.tokenizer_or_token_counter = tokenizer_or_token_counter
         self._embedding_model = embedding_model
         self.kwargs = kwargs
         
@@ -89,11 +86,10 @@ class Chunking:
         if 'chunk_overlap' in allowed_params:
             params['chunk_overlap'] = self.chunk_overlap
             
-        if 'tokenizer' in allowed_params:
-            if self.chunker_type in ['semantic', 'sdpm', 'late']:
-                params['tokenizer'] = self.embedding_model.get_tokenizer_or_token_counter()
-            else:
-                params['tokenizer'] = self.tokenizer
+        if 'tokenizer_or_token_counter' in allowed_params:
+            params['tokenizer_or_token_counter'] = self.tokenizer_or_token_counter
+        elif 'tokenizer' in allowed_params:
+            params['tokenizer'] = self.tokenizer_or_token_counter
                 
         if 'embedding_model' in allowed_params:
             params['embedding_model'] = self.embedding_model
@@ -115,63 +111,82 @@ class Chunking:
         
         return self._chunker
     
-    def _get_overlap_refinery(self, context_size: Optional[int] = None, **kwargs):
-        """Lazy load the overlap refinery."""
-        try:
-            from chonkie.refinery import OverlapRefinery
-        except ImportError:
-            raise ImportError("Failed to import OverlapRefinery from chonkie.refinery")
+    # NOTE: OverlapRefinery is not supported, disabled for now
+    # As soon as Chonkie is updated to support it, we can re-enable it! 
+    # Track in https://github.com/chonkie-inc/chonkie/issues/21
+
+    # def _get_overlap_refinery(self, context_size: Optional[int] = None, **kwargs):
+    #     """Lazy load the overlap refinery."""
+    #     try:
+    #         from chonkie.refinery import OverlapRefinery
+    #     except ImportError:
+    #         raise ImportError("Failed to import OverlapRefinery from chonkie.refinery")
             
-        if context_size is None:
-            context_size = self.chunk_overlap
+    #     if context_size is None:
+    #         context_size = self.chunk_overlap
             
-        return OverlapRefinery(
-            context_size=context_size,
-            tokenizer=self.chunker.tokenizer,
-            **kwargs
-        )
+    #     return OverlapRefinery(
+    #         context_size=context_size,
+    #         tokenizer=self.chunker.tokenizer,
+    #         **kwargs
+    #     )
     
-    def add_overlap_context(
-        self,
-        chunks: List[Any],
-        context_size: int = None,
-        mode: str = "suffix",
-        merge_context: bool = True
-    ) -> List[Any]:
-        """Add overlap context to chunks using OverlapRefinery."""
-        refinery = self._get_overlap_refinery(
-            context_size=context_size,
-            mode=mode,
-            merge_context=merge_context
-        )
-        return refinery.refine(chunks)
+    # def add_overlap_context(
+    #     self,
+    #     chunks: List[Any],
+    #     context_size: int = None,
+    #     mode: str = "suffix",
+    #     merge_context: bool = True
+    # ) -> List[Any]:
+    #     """Add overlap context to chunks using OverlapRefinery."""
+    #     refinery = self._get_overlap_refinery(
+    #         context_size=context_size,
+    #         mode=mode,
+    #         merge_context=merge_context
+    #     )
+    #     return refinery.refine(chunks)
     
     def chunk(
         self,
         text: Union[str, List[str]],
-        add_context: bool = False,
-        context_params: Optional[Dict[str, Any]] = None
+        # Disable context for now, as it's not supported
+        # add_context: bool = False,
+        # context_params: Optional[Dict[str, Any]] = None
+        **kwargs # Added to maintain compatibility with the original `chunk` method signature
     ) -> Union[List[Any], List[List[Any]]]:
         """Chunk text using the configured chunking strategy."""
         chunks = self.chunker(text)
         
-        if add_context:
-            context_params = context_params or {}
-            if isinstance(text, str):
-                chunks = self.add_overlap_context(chunks, **context_params)
-            else:
-                chunks = [self.add_overlap_context(c, **context_params) for c in chunks]
+        # NOTE: OverlapRefinery is not supported, disabled for now
+        # As soon as Chonkie is updated to support it, we can re-enable it! 
+        # Track in https://github.com/chonkie-inc/chonkie/issues/21
+
+        # if add_context:
+        #     context_params = context_params or {}
+        #     if isinstance(text, str):
+        #         chunks = self.add_overlap_context(chunks, **context_params)
+        #     else:
+        #         chunks = [self.add_overlap_context(c, **context_params) for c in chunks]
+
+        if 'add_context' in kwargs or 'context_params' in kwargs:
+            import warnings
+            warnings.warn(
+                "The `add_context` and `context_params` parameters are currently not supported for Chonkie as of version 1.0.2. They would be added in the future. Track in https://github.com/chonkie-inc/chonkie/issues/21",
+                UserWarning
+            )
                 
         return chunks
     
     def __call__(
         self,
         text: Union[str, List[str]],
-        add_context: bool = False,
-        context_params: Optional[Dict[str, Any]] = None
+        # Disable context for now, as it's not supported
+        # add_context: bool = False,
+        # context_params: Optional[Dict[str, Any]] = None
+        **kwargs # Added to maintain compatibility with the original `chunk` method signature
     ) -> Union[List[Any], List[List[Any]]]:
         """Make the Chunking instance callable."""
-        return self.chunk(text, add_context, context_params)
+        return self.chunk(text, **kwargs)
     
     def __repr__(self) -> str:
         """String representation of the Chunking instance."""

{praisonaiagents-0.0.73.dist-info → praisonaiagents-0.0.74.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: praisonaiagents
-Version: 0.0.73
+Version: 0.0.74
 Summary: Praison AI agents for completing complex tasks with Self Reflection Agents
 Author: Mervin Praison
 Requires-Dist: pydantic
@@ -15,7 +15,7 @@ Provides-Extra: knowledge
 Requires-Dist: mem0ai>=0.1.0; extra == "knowledge"
 Requires-Dist: chromadb==0.5.23; extra == "knowledge"
 Requires-Dist: markitdown; extra == "knowledge"
-Requires-Dist: chonkie; extra == "knowledge"
+Requires-Dist: chonkie>=1.0.2; extra == "knowledge"
 Provides-Extra: llm
 Requires-Dist: litellm>=1.50.0; extra == "llm"
 Requires-Dist: pydantic>=2.4.2; extra == "llm"

{praisonaiagents-0.0.73.dist-info → praisonaiagents-0.0.74.dist-info}/RECORD

@@ -7,7 +7,7 @@ praisonaiagents/agents/__init__.py,sha256=_1d6Pqyk9EoBSo7E68sKyd1jDRlN1vxvVIRpoM
 praisonaiagents/agents/agents.py,sha256=uAOHyn77noFvg3sYVFRhQUuc1LDpCMpfLND8CKOXAd4,37971
 praisonaiagents/agents/autoagents.py,sha256=olYDn--rlJp-SckxILqmREkkgNlzCgEEcAUzfMj-54E,13518
 praisonaiagents/knowledge/__init__.py,sha256=xL1Eh-a3xsHyIcU4foOWF-JdWYIYBALJH9bge0Ujuto,246
-praisonaiagents/knowledge/chunking.py,sha256=FzoNY0q8MkvG4gADqk4JcRhmH3lcEHbRdonDgitQa30,6624
+praisonaiagents/knowledge/chunking.py,sha256=G6wyHa7_8V0_7VpnrrUXbEmUmptlT16ISJYaxmkSgmU,7678
 praisonaiagents/knowledge/knowledge.py,sha256=fQNREDiwdoisfIxJBLVkteXgq_8Gbypfc3UaZbxf5QY,13210
 praisonaiagents/llm/__init__.py,sha256=ttPQQJQq6Tah-0updoEXDZFKWtJAM93rBWRoIgxRWO8,689
 praisonaiagents/llm/llm.py,sha256=1WjHumxzuc8sj81NQ4uVEIetUOrb-i58HYLQW7vjV3M,87921
@@ -39,7 +39,7 @@ praisonaiagents/tools/xml_tools.py,sha256=iYTMBEk5l3L3ryQ1fkUnNVYK-Nnua2Kx2S0dxN
 praisonaiagents/tools/yaml_tools.py,sha256=uogAZrhXV9O7xvspAtcTfpKSQYL2nlOTvCQXN94-G9A,14215
 praisonaiagents/tools/yfinance_tools.py,sha256=s2PBj_1v7oQnOobo2fDbQBACEHl61ftG4beG6Z979ZE,8529
 praisonaiagents/tools/train/data/generatecot.py,sha256=H6bNh-E2hqL5MW6kX3hqZ05g9ETKN2-kudSjiuU_SD8,19403
-praisonaiagents-0.0.73.dist-info/METADATA,sha256=6Cwjw1SNW5j4ribad7S2mOlXZKW9o_6HRJhFSBVKmPY,970
-praisonaiagents-0.0.73.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-praisonaiagents-0.0.73.dist-info/top_level.txt,sha256=_HsRddrJ23iDx5TTqVUVvXG2HeHBL5voshncAMDGjtA,16
-praisonaiagents-0.0.73.dist-info/RECORD,,
+praisonaiagents-0.0.74.dist-info/METADATA,sha256=322am_MaL8PT1WHZn-uOxB7or2WJpHgnwLOdFVMxi9o,977
+praisonaiagents-0.0.74.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+praisonaiagents-0.0.74.dist-info/top_level.txt,sha256=_HsRddrJ23iDx5TTqVUVvXG2HeHBL5voshncAMDGjtA,16
+praisonaiagents-0.0.74.dist-info/RECORD,,