unaiverse 0.1.12__py3-none-any.whl

unaiverse/networking/p2p/lib_types.py

@@ -0,0 +1,311 @@
+"""
+       █████  █████ ██████   █████           █████ █████   █████ ██████████ ███████████    █████████  ██████████
+      ░░███  ░░███ ░░██████ ░░███           ░░███ ░░███   ░░███ ░░███░░░░░█░░███░░░░░███  ███░░░░░███░░███░░░░░█
+       ░███   ░███  ░███░███ ░███   ██████   ░███  ░███    ░███  ░███  █ ░  ░███    ░███ ░███    ░░░  ░███  █ ░ 
+       ░███   ░███  ░███░░███░███  ░░░░░███  ░███  ░███    ░███  ░██████    ░██████████  ░░█████████  ░██████   
+       ░███   ░███  ░███ ░░██████   ███████  ░███  ░░███   ███   ░███░░█    ░███░░░░░███  ░░░░░░░░███ ░███░░█   
+       ░███   ░███  ░███  ░░█████  ███░░███  ░███   ░░░█████░    ░███ ░   █ ░███    ░███  ███    ░███ ░███ ░   █
+       ░░████████   █████  ░░█████░░████████ █████    ░░███      ██████████ █████   █████░░█████████  ██████████
+        ░░░░░░░░   ░░░░░    ░░░░░  ░░░░░░░░ ░░░░░      ░░░      ░░░░░░░░░░ ░░░░░   ░░░░░  ░░░░░░░░░  ░░░░░░░░░░ 
+                 A Collectionless AI Project (https://collectionless.ai)
+                 Registration/Login: https://unaiverse.io
+                 Code Repositories:  https://github.com/collectionlessai/
+                 Main Developers:    Stefano Melacci (Project Leader), Christian Di Maio, Tommaso Guidi
+"""
+import json
+import ctypes
+import logging
+from threading import Lock
+from typing import List, Any
+
+from .golibp2p import GoLibP2P  # Assuming this class loads the library
+
+logger = logging.getLogger('LIB-TYPES')
+
+
+class TypeInterface:
+    """
+    Helper class for converting between Python types and Go types using ctypes.
+    """
+    def __init__(self, libp2p_instance: GoLibP2P):
+        self.__freed_pointers: set[int] = set()  # Track freed pointers to prevent double-free errors
+        self.__freed_pointers_lock: Any = Lock()  # A threading lock
+        self.libp2p: GoLibP2P = libp2p_instance  # Store the shared library object instance
+
+    def to_go_string(self, s: str) -> bytes:
+        """
+        Converts a Python string to a UTF-8 encoded Python 'bytes' object.
+
+        This 'bytes' object is suitable for direct use with ctypes when passing
+        to a C function expecting a 'char*' (ctypes.c_char_p), as ctypes
+        will automatically pass a pointer to the byte string's data.
+
+        Args:
+            s: The Python string.
+
+        Returns:
+            A Python 'bytes' object containing the UTF-8 encoded string.
+        """
+        if s is None:
+            s = ""
+        return s.encode("utf-8")
+
+    def from_go_string(self, cstr: bytes) -> str:
+        """
+        Converts a C char pointer (Go string) to a Python string.
+
+        Args:
+            cstr: The C char pointer.
+
+        Returns:
+            The decoded Python string.
+        """
+        if not cstr:
+            return ""
+        return cstr.decode("utf-8")
+
+    def to_go_int(self, i: int) -> ctypes.c_int:
+        """
+        Converts a Python integer to a Go-compatible ctypes integer.
+
+        Args:
+            i: The Python integer.
+
+        Returns:
+            A ctypes.c_int equivalent.
+        """
+        return ctypes.c_int(i)
+
+    def from_go_int(self, val: ctypes.c_int) -> int:
+        """
+        Converts a ctypes.c_int from Go to a Python integer.
+
+        Args:
+            val: The ctypes.c_int value.
+
+        Returns:
+            The corresponding Python integer.
+        """
+        return int(val)
+
+    def to_go_float(self, f: float) -> ctypes.c_float:
+        """
+        Converts a Python float to a Go-compatible ctypes float.
+
+        Args:
+            f: The Python float.
+
+        Returns:
+            A ctypes.c_float equivalent.
+        """
+        return ctypes.c_float(f)
+
+    def from_go_float(self, val: ctypes.c_float) -> float:
+        """
+        Converts a ctypes.c_float from Go to a Python float.
+
+        Args:
+            val: The ctypes.c_float value.
+
+        Returns:
+            The corresponding Python float.
+        """
+        return float(val)
+
+    def to_go_bool(self, b: bool) -> ctypes.c_int:
+        """
+        Converts a Python boolean to a Go-compatible integer (1 if True, 0 if False).
+
+        Args:
+            b: The Python boolean.
+
+        Returns:
+            A ctypes.c_int (1 or 0).
+        """
+        return ctypes.c_int(1 if b else 0)
+
+    def from_go_bool(self, val: ctypes.c_int) -> bool:
+        """
+        Converts a Go-compatible integer (ctypes.c_int) to a Python boolean.
+
+        Args:
+            val: The ctypes.c_int value.
+
+        Returns:
+            True if the value equals 1, False otherwise.
+        """
+        return val == 1
+
+    def to_go_bytes(self, b: bytes) -> ctypes.c_char_p:
+        """
+        Converts a Python bytes object to a Go-compatible C char pointer.
+
+        Args:
+            b: The Python bytes.
+
+        Returns:
+            A ctypes.c_char_p pointing to the byte data.
+        """
+        if b is None:
+            b = b""
+        buf = ctypes.create_string_buffer(b, len(b))
+        return ctypes.cast(buf, ctypes.c_char_p)
+
+    def from_go_bytes(self, cptr: ctypes.c_char_p, length: int) -> bytes:
+        """
+        Converts a Go pointer representing a byte array to a Python bytes object.
+
+        Args:
+            cptr: The C pointer to the byte array.
+            length: The number of bytes to read.
+
+        Returns:
+            A Python bytes object containing the read data.
+        """
+        if not cptr or length <= 0:
+            return bytes()
+        return ctypes.string_at(cptr, length)
+
+    def from_go_ptr_to_json(self, c_void_ptr_val: int) -> Any:
+        """
+        Converts a C void* pointer (returned by Go as int) pointing to a
+        null-terminated C string containing JSON into a Python object.
+
+        It reads the string, parses it as JSON, and crucially frees the C memory
+        using the provided FreeString function from the Go library.
+
+        Args:
+            c_void_ptr_val: The integer value representing the C pointer address.
+
+        Returns:
+            The parsed Python object from the JSON string.
+
+        Raises:
+            GoLibError: If the pointer is NULL, reading/decoding fails, or
+                        JSON parsing fails.
+            TypeError: When go_lib is not a valid ctypes library object.
+        """
+
+        if not c_void_ptr_val:  # Check if the address is NULL (0)
+            raise print("Received a NULL pointer from Go function")
+
+        try:
+
+            # --- Double-Free Check (Before Reading/Casting) ---
+            self.__freed_pointers_lock.acquire()  # Acquire lock if using threading
+            if c_void_ptr_val in self.__freed_pointers:
+
+                # This indicates a serious logic error elsewhere - the pointer
+                # was already freed but somehow passed here again.
+                logger.warning(f"🔥🔥🔥 ATTEMPT TO PROCESS ALREADY FREED POINTER {hex(c_void_ptr_val)}! 🔥🔥🔥")
+
+                # Raising an error is safer than trying to read potentially invalid memory.
+                logger.error(f"Attempt to process pointer {hex(c_void_ptr_val)} which was already freed")
+                raise Exception(f"Attempt to process pointer {hex(c_void_ptr_val)} which was already freed")
+            self.__freed_pointers_lock.release()  # Release lock if using threading
+
+            # --- Cast void* to c_char_p and Read String ---
+            try:
+
+                # Perform the cast only when needed for reading
+                c_char_ptr_for_read = ctypes.cast(c_void_ptr_val, ctypes.c_char_p)
+                raw_bytes = ctypes.string_at(c_char_ptr_for_read)
+                json_string = raw_bytes.decode('utf-8')
+
+                # Logger.debug(f"Read string (len={len(json_string)})
+                # from pointer {hex(c_void_ptr_val)}: %.100s...", json_string)
+            except (ctypes.ArgumentError, ValueError, UnicodeDecodeError) as read_err:
+                logger.error(f"Failed to read/decode string from pointer {hex(c_void_ptr_val)}: "
+                             f"{read_err}", exc_info=False)
+
+                # Even if reading fails, the pointer itself *might* still be valid C memory
+                # that Go expects us to free. We will proceed to free it in finally.
+                raise Exception(f"Failed to read string from pointer {hex(c_void_ptr_val)}: {read_err}") from read_err
+            except Exception as unexpected_read_err:  # Catch other potential ctypes issues
+                logger.error(f"Unexpected error reading C string from pointer {hex(c_void_ptr_val)}: "
+                             f"{unexpected_read_err}", exc_info=True)
+                raise Exception(f"Unexpected error reading C string from pointer {hex(c_void_ptr_val)}: "
+                                f"{unexpected_read_err}") from unexpected_read_err
+
+            # --- Check for Empty String ---
+
+            # --- Parse JSON ---
+            try:
+
+                # Now that we have the string, parse it
+                logger.debug(f"Parsing JSON from string: {json_string}")
+                parsed_data = json.loads(json_string)
+                logger.debug(f"Parsed JSON data: {parsed_data}")
+
+                # Logger.debug(f"Successfully parsed JSON from pointer {hex(c_void_ptr_val)}")
+                return parsed_data  # Return the parsed Python object
+
+            except json.JSONDecodeError as json_err:
+                logger.error(f"Failed to decode JSON from pointer {hex(c_void_ptr_val)}: {json_err}", exc_info=False)
+
+                # Again, the pointer is likely valid C memory, but the content is bad.
+                # Let the block handle freeing.
+                raise Exception(f"Failed to decode JSON from pointer {hex(c_void_ptr_val)}: {json_err}") from json_err
+
+        finally:
+
+            # --- CRITICAL: Free C Memory ---
+            # This block executes even if errors occurred during read/parse,
+            # ensuring we attempt to free any non-NULL pointer received from Go.
+            with self.__freed_pointers_lock:
+                if c_void_ptr_val:
+                    logger.info(f"🐍 FINALLY: Freeing pointer {hex(c_void_ptr_val)}...")
+                    if c_void_ptr_val in self.__freed_pointers:
+
+                        # This check is technically redundant if the initial check worked,
+                        # but provides an extra safety layer in case of concurrency issues
+                        # (if freed_pointers is shared without locks - which it shouldn't be).
+                        logger.warning(f"🔥🔥🔥 DOUBLE FREE DETECTED in finally block for "
+                                       f"{hex(c_void_ptr_val)}! Skipping FreeString call again. 🔥🔥🔥")
+                    else:
+
+                        # Add before calling free
+                        try:
+                            self.libp2p.FreeString(c_void_ptr_val)  # Pass the original void* value
+                            logger.info(f"✅ FINALLY: FreeString successful for {hex(c_void_ptr_val)}.")
+                        except Exception as free_err:
+
+                            # Log if FreeString fails, but don't raise from finally
+                            # as it might hide the original error.
+                            logger.critical(f"🚨 FAILED TO FREE C MEMORY for pointer "
+                                            f"{hex(c_void_ptr_val)} via FreeString: {free_err}", exc_info=True)
+
+                            # Consider removing from freed_pointers if free failed?
+                            # freed_pointers.discard(c_void_ptr_val) # Maybe, to allow retry? Risky.
+                            # But if FreeString fails, the pointer is likely invalid anyway.
+
+    def to_go_json(self, data: Any) -> bytes:
+        """
+    Encodes a Python object to a JSON string, returning a UTF-8 encoded
+    Python 'bytes' object.
+
+    This 'bytes' object is suitable for direct use with ctypes when passing
+    to a C function expecting a 'char*' (ctypes.c_char_p).
+
+    Args:
+        data: The Python object (e.g., dict, list) to encode.
+
+    Returns:
+        A Python 'bytes' object containing the JSON string, UTF-8 encoded.
+    """
+        json_str = json.dumps(data)
+        return self.to_go_string(json_str)
+
+    def from_go_string_to_list(self, cstr: ctypes.c_char_p) -> List[Any]:
+        """
+        Decodes a JSON-encoded list from a Go C char pointer into a Python list.
+
+        Args:
+            cstr: The Go string (C char pointer) containing a JSON list.
+
+        Returns:
+            A Python list representing the JSON data.
+        """
+        s = self.from_go_string(cstr)
+
+        return json.loads(s)

unaiverse/networking/p2p/message_pb2.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: message.proto
+# Protobuf Python Version: 4.25.3
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rmessage.proto\x12\x03p2p\x1a\x1cgoogle/protobuf/struct.proto\"\xe8\x01\n\x07Message\x12\x0e\n\x06sender\x18\x01 \x01(\t\x12\x14\n\x0c\x63ontent_type\x18\x02 \x01(\t\x12\x0f\n\x07\x63hannel\x18\x03 \x01(\t\x12\x11\n\tpiggyback\x18\x04 \x01(\t\x12\x15\n\rtimestamp_net\x18\x05 \x01(\t\x12\x31\n\rstream_sample\x18\x06 \x01(\x0b\x32\x18.p2p.StreamSampleContentH\x00\x12\x16\n\x0cjson_content\x18\x07 \x01(\tH\x00\x12&\n\x0cstats_update\x18\x08 \x01(\x0b\x32\x0e.p2p.StatBatchH\x00\x42\t\n\x07\x63ontent\"\x90\x01\n\x13StreamSampleContent\x12\x36\n\x07samples\x18\x01 \x03(\x0b\x32%.p2p.StreamSampleContent.SamplesEntry\x1a\x41\n\x0cSamplesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.p2p.StreamSample:\x02\x38\x01\"e\n\x0cStreamSample\x12\x1d\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32\x0f.p2p.SampleData\x12\x10\n\x08\x64\x61ta_tag\x18\x02 \x01(\x05\x12\x16\n\tdata_uuid\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\x0c\n\n_data_uuid\"\xb2\x01\n\nSampleData\x12&\n\x0btensor_data\x18\x01 \x01(\x0b\x32\x0f.p2p.TensorDataH\x00\x12$\n\nimage_data\x18\x02 \x01(\x0b\x32\x0e.p2p.ImageDataH\x00\x12\"\n\ttext_data\x18\x03 \x01(\x0b\x32\r.p2p.TextDataH\x00\x12\"\n\tfile_data\x18\x04 \x01(\x0b\x32\r.p2p.FileDataH\x00\x42\x0e\n\x0c\x64\x61ta_payload\"8\n\nTensorData\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\r\n\x05\x64type\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x05\"\x19\n\tImageData\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\"\x18\n\x08TextData\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\t\"@\n\x08\x46ileData\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x10\n\x08\x66ilename\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\"j\n\nStatUpdate\x12\x0f\n\x07peer_id\x18\x01 \x01(\t\x12\x11\n\tstat_name\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12%\n\x05value\x18\x04 \x01(\x0b\x32\x16.google.protobuf.Value\"-\n\tStatBatch\x12 \n\x07updates\x18\x01 \x03(\x0b\x32\x0f.p2p.StatUpdateB\x0cZ\n./proto-gob\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'message_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'Z\n./proto-go'
+  _globals['_STREAMSAMPLECONTENT_SAMPLESENTRY']._options = None
+  _globals['_STREAMSAMPLECONTENT_SAMPLESENTRY']._serialized_options = b'8\001'
+  _globals['_MESSAGE']._serialized_start=53
+  _globals['_MESSAGE']._serialized_end=285
+  _globals['_STREAMSAMPLECONTENT']._serialized_start=288
+  _globals['_STREAMSAMPLECONTENT']._serialized_end=432
+  _globals['_STREAMSAMPLECONTENT_SAMPLESENTRY']._serialized_start=367
+  _globals['_STREAMSAMPLECONTENT_SAMPLESENTRY']._serialized_end=432
+  _globals['_STREAMSAMPLE']._serialized_start=434
+  _globals['_STREAMSAMPLE']._serialized_end=535
+  _globals['_SAMPLEDATA']._serialized_start=538
+  _globals['_SAMPLEDATA']._serialized_end=716
+  _globals['_TENSORDATA']._serialized_start=718
+  _globals['_TENSORDATA']._serialized_end=774
+  _globals['_IMAGEDATA']._serialized_start=776
+  _globals['_IMAGEDATA']._serialized_end=801
+  _globals['_TEXTDATA']._serialized_start=803
+  _globals['_TEXTDATA']._serialized_end=827
+  _globals['_FILEDATA']._serialized_start=829
+  _globals['_FILEDATA']._serialized_end=893
+  _globals['_STATUPDATE']._serialized_start=895
+  _globals['_STATUPDATE']._serialized_end=1001
+  _globals['_STATBATCH']._serialized_start=1003
+  _globals['_STATBATCH']._serialized_end=1048
+# @@protoc_insertion_point(module_scope)