sunholo 0.61.7__tar.gz → 0.61.9__tar.gz

{sunholo-0.61.7 → sunholo-0.61.9}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.61.7
+Version: 0.61.9
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.7.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.9.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.61.7 → sunholo-0.61.9}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.61.7'
+version = '0.61.9'
 
 setup(
     name='sunholo',

sunholo-0.61.9/sunholo/chunker/__init__.py

@@ -0,0 +1 @@
+from .data_to_embed_pubsub import data_to_embed_pubsub, direct_file_to_embed

{sunholo-0.61.7 → sunholo-0.61.9}/sunholo/chunker/data_to_embed_pubsub.py

@@ -11,11 +11,36 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
+import pathlib
+
 from ..logging import log
 from ..pubsub import process_pubsub_message
 from .message_data import handle_gcs_message, handle_google_drive_message, handle_github_message, handle_http_message, handle_json_content_message
 from .publish import process_docs_chunks_vector_name
+from .splitter import chunk_doc_to_docs
+
 from ..llamaindex.import_files import llamaindex_chunker_check
+from . import loaders
+
+def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
+    """
+    Send direct files to chunking embed pipeline
+
+    
+    
+    """
+    log.info(f"Sending direct file upload {file_name} to loaders.read_file_to_documents {metadata}")
+    docs = loaders.read_file_to_documents(file_name, metadata=metadata)
+    if docs is None:
+        log.warning(f"loaders.read_file_to_documents docs2 failed to load file {metadata}")
+
+        return None
+
+    chunks = chunk_doc_to_docs(docs, file_name.suffix, vector_name=vector_name)
+    
+    return format_chunk_return(chunks, metadata, vector_name)
+
+
 
 def data_to_embed_pubsub(data: dict):
     """Triggered from a message on a Cloud Pub/Sub topic.
@@ -62,7 +87,11 @@ def process_chunker_data(message_data, metadata, vector_name):
 
     else: 
         chunks, metadata = handle_json_content_message(message_data, metadata, vector_name) 
+    
+    return format_chunk_return(chunks, metadata, vector_name)
+
 
+def format_chunk_return(chunks, metadata, vector_name):
     # to be really sure
     if metadata:
         metadata["vector_name"] = vector_name

{sunholo-0.61.7 → sunholo-0.61.9}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.61.7
+Version: 0.61.9
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.7.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.9.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

sunholo-0.61.7/sunholo/chunker/__init__.py

@@ -1 +0,0 @@
-from .data_to_embed_pubsub import data_to_embed_pubsub