sunholo 0.61.7__py3-none-any.whl → 0.61.9__py3-none-any.whl

sunholo/chunker/__init__.py

@@ -1 +1 @@
-from .data_to_embed_pubsub import data_to_embed_pubsub
+from .data_to_embed_pubsub import data_to_embed_pubsub, direct_file_to_embed

sunholo/chunker/data_to_embed_pubsub.py

@@ -11,11 +11,36 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
+import pathlib
+
 from ..logging import log
 from ..pubsub import process_pubsub_message
 from .message_data import handle_gcs_message, handle_google_drive_message, handle_github_message, handle_http_message, handle_json_content_message
 from .publish import process_docs_chunks_vector_name
+from .splitter import chunk_doc_to_docs
+
 from ..llamaindex.import_files import llamaindex_chunker_check
+from . import loaders
+
+def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
+    """
+    Send direct files to chunking embed pipeline
+
+    
+    
+    """
+    log.info(f"Sending direct file upload {file_name} to loaders.read_file_to_documents {metadata}")
+    docs = loaders.read_file_to_documents(file_name, metadata=metadata)
+    if docs is None:
+        log.warning(f"loaders.read_file_to_documents docs2 failed to load file {metadata}")
+
+        return None
+
+    chunks = chunk_doc_to_docs(docs, file_name.suffix, vector_name=vector_name)
+    
+    return format_chunk_return(chunks, metadata, vector_name)
+
+
 
 def data_to_embed_pubsub(data: dict):
     """Triggered from a message on a Cloud Pub/Sub topic.
@@ -62,7 +87,11 @@ def process_chunker_data(message_data, metadata, vector_name):
 
     else: 
         chunks, metadata = handle_json_content_message(message_data, metadata, vector_name) 
+    
+    return format_chunk_return(chunks, metadata, vector_name)
+
 
+def format_chunk_return(chunks, metadata, vector_name):
     # to be really sure
     if metadata:
         metadata["vector_name"] = vector_name

{sunholo-0.61.7.dist-info → sunholo-0.61.9.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.61.7
+Version: 0.61.9
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.7.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.9.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.61.7.dist-info → sunholo-0.61.9.dist-info}/RECORD

@@ -21,8 +21,8 @@ sunholo/bots/__init__.py,sha256=EMFd7e2z68l6pzYOnkzHbLd2xJRvxTKFRNCTuhZ8hIw,130
 sunholo/bots/discord.py,sha256=cCFae5K1BCa6JVkWGLh_iZ9qFO1JpXb6K4eJrlDfEro,2442
 sunholo/bots/github_webhook.py,sha256=5pQPRLM_wxxcILVaIzUDV8Kt7Arcm2dL1r1kMMHA524,9629
 sunholo/bots/webapp.py,sha256=EIMxdAJ_xtufwJmvnn7N_Fb_1hZ9DjhJ0Kf_hp02vEU,1926
-sunholo/chunker/__init__.py,sha256=UhQBZTKwDfBXm0TPv4LvsGc5pdUGCbYzi3qPTOkU4gw,55
-sunholo/chunker/data_to_embed_pubsub.py,sha256=IY9SBRA7IO77QJBEgQuO1FiSCd6Dfm-TMEf1Ey-pLoo,3065
+sunholo/chunker/__init__.py,sha256=yWYwpejyYxDpZv1joTrFMsh2SWAkd0z7a1VKtmOfMhA,77
+sunholo/chunker/data_to_embed_pubsub.py,sha256=h4X373NfRLaZvSMKm8RjrNKsPAmx8kK-91uyXYu3Gx8,3887
 sunholo/chunker/doc_handling.py,sha256=rIyknpzDyj5A0u_DqSQVD_CXLRNZPOU6TCL4bhCdjOI,8563
 sunholo/chunker/images.py,sha256=Xmh1vwHrVhoXm5iH2dhCc52O8YgdzE8KrDSdL-pGnp8,1861
 sunholo/chunker/loaders.py,sha256=xiToUVgPz2ZzcqpUAq7aNP3PTenb_rBUAFzu0JPycIg,10268
@@ -96,9 +96,9 @@ sunholo/utils/parsers.py,sha256=OrHmASqIbI45atVOhiGodgLvnfrzkvVzyHnSvAXD89I,3841
 sunholo/utils/user_ids.py,sha256=SQd5_H7FE7vcTZp9AQuQDWBXd4FEEd7TeVMQe1H4Ny8,292
 sunholo/vertex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/vertex/init_vertex.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
-sunholo-0.61.7.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.61.7.dist-info/METADATA,sha256=49tEfz5t9jCW7XjVFyy-4Ip7_JbDUdTxxZviBqtBVaA,8057
-sunholo-0.61.7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-sunholo-0.61.7.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.61.7.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.61.7.dist-info/RECORD,,
+sunholo-0.61.9.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.61.9.dist-info/METADATA,sha256=gjQ1Kx5ibCS76YAO3pDR7KCJd7xlxxAK46k2Ei4AQAY,8057
+sunholo-0.61.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sunholo-0.61.9.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.61.9.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.61.9.dist-info/RECORD,,