naeural-client 2.7.31__py3-none-any.whl → 2.7.40__py3-none-any.whl

naeural_client/_ver.py

@@ -1,4 +1,4 @@
-__VER__ = "2.7.31"
+__VER__ = "2.7.40"
 
 if __name__ == "__main__":
   with open("pyproject.toml", "rt") as fd:

naeural_client/ipfs/__init__.py

@@ -0,0 +1 @@
+from .r1fs import R1FSEngine, log_info

naeural_client/ipfs/r1fs.py

@@ -0,0 +1,555 @@
+"""
+Ratio1 base IPFS utility functions.
+
+"""
+import subprocess
+import json
+from datetime import datetime
+import base64
+import time
+import os
+import tempfile
+import uuid
+
+from threading import Lock
+
+__VER__ = "0.2.1"
+
+
+class IPFSCt:
+  EE_IPFS_RELAY_ENV_KEY = "EE_IPFS_RELAY"
+  EE_SWARM_KEY_CONTENT_BASE64_ENV_KEY = "EE_SWARM_KEY_CONTENT_BASE64"
+  R1FS_DOWNLOADS = "ipfs_downloads"
+  R1FS_UPLOADS = "ipfs_uploads"
+  TEMP_DOWNLOAD = os.path.join("./_local_cache/_output", R1FS_DOWNLOADS)
+  TEMP_UPLOAD = os.path.join("./_local_cache/_output", R1FS_UPLOADS)
+
+ERROR_TAG = "Unknown"
+
+COLOR_CODES = {
+  "g": "\033[92m",
+  "r": "\033[91m",
+  "b": "\033[94m",
+  "y": "\033[93m",
+  "m": "\033[95m",
+  'd': "\033[90m", # dark gray
+  "reset": "\033[0m"
+}
+
+def log_info(msg: str, color="reset", **kwargs):
+  timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+  color_code = COLOR_CODES.get(color, COLOR_CODES["reset"])
+  reset_code = COLOR_CODES["reset"]
+  print(f"{color_code}[{timestamp}] {msg}{reset_code}", flush=True)
+  return
+
+class SimpleLogger:
+  def P(self, *args, **kwargs):
+    log_info(*args, **kwargs)
+    return
+  
+def require_ipfs_started(method):
+  """
+  decorator to ensure the IPFS is started before executing the method.
+  
+  parameters
+  ----------
+  method : callable
+      the method to be decorated.
+  
+  returns
+  -------
+  callable
+      the wrapped method that checks the 'started' attribute.
+  
+  raises
+  ------
+  RuntimeError
+      if the instance's 'started' attribute is False.
+  """
+  def wrapper(self, *args, **kwargs):
+    if not self.ipfs_started:
+      raise RuntimeError(f"{method.__name__} FAILED. R1FS.ipfs_started=={self.ipfs_started}")
+    return method(self, *args, **kwargs)
+  return wrapper  
+
+
+
+class R1FSEngine:
+  _lock: Lock = Lock()
+  __instances = {}
+
+  def __new__(
+    cls, 
+    name: str = "default", 
+    logger: any = None, 
+    downloads_dir: str = None,
+    uploads_dir: str = None,
+    base64_swarm_key: str = None, 
+    ipfs_relay: str = None,   
+    debug=False,     
+  ):
+    with cls._lock:
+      if name not in cls.__instances:
+        instance = super(R1FSEngine, cls).__new__(cls)
+        instance._build(
+          name=name, logger=logger, downloads_dir=downloads_dir, uploads_dir=uploads_dir,
+          base64_swarm_key=base64_swarm_key, ipfs_relay=ipfs_relay, debug=debug,
+        )
+        cls.__instances[name] = instance
+      else:
+        instance = cls.__instances[name]
+    return instance
+    
+  def _build(
+    self, 
+    name: str = "default",
+    logger: any = None, 
+    downloads_dir: str = None,
+    uploads_dir: str = None,
+    base64_swarm_key: str = None, 
+    ipfs_relay: str = None,   
+    debug=False,     
+  ):
+    """
+    Initialize the IPFS wrapper with a given logger function.
+    By default, it uses the built-in print function for logging.
+    """
+    self.__name = name
+    if logger is None:
+      logger = SimpleLogger()
+
+    self.logger = logger
+
+    self.__ipfs_started = False
+    self.__ipfs_address = None
+    self.__ipfs_id = None
+    self.__ipfs_agent = None
+    self.__uploaded_files = {}
+    self.__downloaded_files = {}
+    self.__base64_swarm_key = base64_swarm_key
+    self.__ipfs_relay = ipfs_relay
+    self.__downloads_dir = downloads_dir
+    self.__uploads_dir = uploads_dir
+    self.__debug = debug
+    
+    self.startup()
+    return
+  
+  def startup(self):
+    
+    if self.__downloads_dir is None:
+      if hasattr(self.logger, "get_output_folder"):
+        self.__downloads_dir = os.path.join(
+          self.logger.get_output_folder(),
+          IPFSCt.R1FS_DOWNLOADS
+        )
+      else:
+        self.__downloads_dir = IPFSCt.TEMP_DOWNLOAD
+    #end if downloads_dir    
+    os.makedirs(self.__downloads_dir, exist_ok=True)    
+    
+    if self.__uploads_dir is None:
+      if hasattr(self.logger, "get_output_folder"):
+        self.__uploads_dir = os.path.join(
+          self.logger.get_output_folder(),
+          IPFSCt.R1FS_UPLOADS
+        )
+      else:
+        self.__uploads_dir = IPFSCt.TEMP_UPLOAD
+    os.makedirs(self.__uploads_dir, exist_ok=True)    
+    
+    self.maybe_start_ipfs(
+      base64_swarm_key=self.__base64_swarm_key,
+      ipfs_relay=self.__ipfs_relay,
+    )
+    return
+    
+    
+  def P(self, s, *args, **kwargs):
+    s = "[R1FS] " + s
+    self.logger.P(s, *args, **kwargs)
+    return
+  
+  def Pd(self, s, *args, **kwargs):
+    if self.__debug:
+      s = "[R1FS][DEBUG] " + s
+      self.logger.P(s, *args, **kwargs)
+    return
+    
+  @property
+  def ipfs_id(self):
+    return self.__ipfs_id
+  
+  @property
+  def ipfs_address(self):
+    return self.__ipfs_address
+  
+  @property
+  def ipfs_agent(self):
+    return self.__ipfs_agent
+  
+  @property
+  def ipfs_started(self):
+    return self.__ipfs_started
+  
+  @property
+  def uploaded_files(self):
+    return self.__uploaded_files
+  
+  @property
+  def downloaded_files(self):
+    return self.__downloaded_files
+  
+  def get_unique_name(self, prefix="r1fs", suffix=""):
+    str_id = str(uuid.uuid4()).replace("-", "")[:8]
+    return f"{prefix}_{str_id}{suffix}"
+  
+  def get_unique_upload_name(self, prefix="r1fs", suffix=""):
+    return os.path.join(self.__uploads_dir, self.get_unique_name(prefix, suffix))
+  
+  def get_unique_or_complete_upload_name(self, fn=None, prefix="r1fs", suffix=""):
+    if fn is not None and os.path.dirname(fn) == "":
+      return os.path.join(self.__uploads_dir, fn)
+    return self.get_unique_upload_name(prefix, suffix)
+  
+
+  def __run_command(
+    self, 
+    cmd_list: list, 
+    raise_on_error=True,
+    timeout=60,
+    verbose=False,
+  ):
+    """
+    Run a shell command using subprocess.run with a timeout.
+    Logs the command and its result. If verbose is enabled,
+    prints command details. Raises an exception on error if raise_on_error is True.
+    """
+    failed = False
+    output = ""
+    cmd_str = " ".join(cmd_list)
+    self.Pd(f"Running command: {cmd_str}", color='d')
+    try:
+      result = subprocess.run(
+        cmd_list, 
+        capture_output=True, 
+        text=True, 
+        timeout=timeout,
+      )
+    except subprocess.TimeoutExpired as e:
+      failed = True
+      self.P(f"Command timed out after {timeout} seconds: {cmd_str}", color='r')
+      if raise_on_error:
+        raise Exception(f"Timeout expired for '{cmd_str}'") from e
+    
+    if result.returncode != 0:
+      failed = True
+      self.P(f"Command error: {result.stderr.strip()}", color='r')
+      if raise_on_error:
+        raise Exception(f"Error while running '{cmd_str}': {result.stderr.strip()}")
+    
+    if not failed:
+      if verbose:
+        self.Pd(f"Command output: {result.stdout.strip()}")
+      output = result.stdout.strip()
+    return output
+  
+
+  def __get_id(self) -> str:
+    """
+    Get the IPFS peer ID via 'ipfs id' (JSON output).
+    Returns the 'ID' field as a string.
+    """
+    output = self.__run_command(["ipfs", "id"])
+    try:
+      data = json.loads(output)
+      self.__ipfs_id = data.get("ID", ERROR_TAG)
+      self.__ipfs_address = data.get("Addresses", [ERROR_TAG,ERROR_TAG])[1]
+      self.__ipfs_agent = data.get("AgentVersion", ERROR_TAG)
+      return data.get("ID", ERROR_TAG)
+    except json.JSONDecodeError:
+      raise Exception("Failed to parse JSON from 'ipfs id' output.")
+
+  @require_ipfs_started
+  def __pin_add(self, cid: str) -> str:
+    """
+    Explicitly pin a CID (and fetch its data) so it appears in the local pinset.
+    """
+    res = self.__run_command(["ipfs", "pin", "add", cid])
+    self.Pd(f"{res}")
+    return res  
+
+  
+  # Public methods
+  
+  def add_json(self, data, fn=None, tempfile=False) -> bool:
+    """
+    Add a JSON object to IPFS.
+    """
+    try:
+      json_data = json.dumps(data)
+      if tempfile:
+        self.Pd("Using tempfile for JSON")
+        with tempfile.NamedTemporaryFile(
+          mode='w', suffix='.json', delete=False
+        ) as f:
+          f.write(json_data)
+        fn = f.name
+      else:
+        fn = self.get_unique_or_complete_upload_name(fn=fn, suffix=".json")
+        self.Pd(f"Using unique name for JSON: {fn}")
+        with open(fn, "w") as f:
+          f.write(json_data)
+      #end if tempfile
+      cid = self.add_file(fn)
+      return cid
+    except Exception as e:
+      self.P(f"Error adding JSON to IPFS: {e}", color='r')
+      return None
+    
+    
+  def add_yaml(self, data, fn=None, tempfile=False) -> bool:
+    """
+    Add a YAML object to IPFS.
+    """
+    try:
+      import yaml
+      yaml_data = yaml.dump(data)
+      if tempfile:
+        self.Pd("Using tempfile for YAML")
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f:
+          f.write(yaml_data)
+        fn = f.name
+      else:
+        fn = self.get_unique_or_complete_upload_name(fn=fn, suffix=".yaml")
+        self.Pd(f"Using unique name for YAML: {fn}")
+        with open(fn, "w") as f:
+          f.write(yaml_data)
+      cid = self.add_file(fn)
+      return cid
+    except Exception as e:
+      self.P(f"Error adding YAML to IPFS: {e}", color='r')
+      return None
+    
+    
+  def add_pickle(self, data, fn=None, tempfile=False) -> bool:
+    """
+    Add a Pickle object to IPFS.
+    """
+    try:
+      import pickle
+      if tempfile:
+        self.Pd("Using tempfile for Pickle")
+        with tempfile.NamedTemporaryFile(mode='wb', suffix='.pkl', delete=False) as f:
+          pickle.dump(data, f)
+        fn = f.name
+      else:
+        fn = self.get_unique_or_complete_upload_name(fn=fn, suffix=".pkl")
+        self.Pd(f"Using unique name for pkl: {fn}")
+        with open(fn, "wb") as f:
+          pickle.dump(data, f)
+      cid = self.add_file(fn)
+      return cid
+    except Exception as e:
+      self.P(f"Error adding Pickle to IPFS: {e}", color='r')
+      return None
+
+
+  @require_ipfs_started
+  def add_file(self, file_path: str) -> str:
+    """
+    This method adds a file to IPFS and returns the CID of the wrapped folder.
+    
+    Parameters
+    ----------
+    file_path : str
+        The path to the file to be added.
+    
+    Returns
+    -------
+    str
+        The CID of the wrapped folder
+      
+    """
+    assert os.path.isfile(file_path), f"File not found: {file_path}"
+    
+    output = self.__run_command(["ipfs", "add", "-q", "-w", file_path])
+    # "ipfs add -w <file>" typically prints two lines:
+    #   added <hash_of_file> <filename>
+    #   added <hash_of_wrapped_folder> <foldername?>
+    # We want the *last* line's CID (the wrapped folder).
+    lines = output.strip().split("\n")
+    if not lines:
+      raise Exception("No output from 'ipfs add -w -q'")
+    folder_cid = lines[-1].strip()
+    self.__uploaded_files[folder_cid] = file_path
+    # now we pin the folder
+    res = self.__pin_add(folder_cid)
+    self.P(f"Added file {file_path} as <{folder_cid}>")
+    return folder_cid
+
+
+  @require_ipfs_started
+  def get_file(self, cid: str, local_folder: str = None, pin=True) -> str:
+    """
+    Get a file from IPFS by CID and save it to a local folder.
+    If no local folder is provided, the default downloads directory is used.
+    Returns the full path of the downloaded file.
+    
+    Parameters
+    ----------
+    cid : str
+        The CID of the file to download.
+        
+    local_folder : str
+        The local folder to save the
+            
+    """
+    if pin:
+      pin_result = self.__pin_add(cid)
+      
+    if local_folder is None:
+      local_folder = self.__downloads_dir # default downloads directory
+      os.makedirs(local_folder, exist_ok=True)
+      local_folder = os.path.join(local_folder, cid) # add the CID as a subfolder
+      
+    self.Pd(f"Downloading file {cid} to {local_folder}")
+    self.__run_command(["ipfs", "get", cid, "-o", local_folder])
+    # now we need to get the file from the folder
+    folder_contents = os.listdir(local_folder)
+    if len(folder_contents) != 1:
+      raise Exception(f"Expected one file in {local_folder}, found {folder_contents}")
+    # get the full path of the file
+    out_local_filename = os.path.join(local_folder, folder_contents[0])
+    self.P(f"Downloaded <{cid}> to {out_local_filename}")
+    self.__downloaded_files[cid] = out_local_filename
+    return out_local_filename
+
+
+
+
+
+  @require_ipfs_started
+  def list_pins(self):
+    """
+    List pinned CIDs via 'ipfs pin ls --type=recursive'.
+    Returns a list of pinned CIDs.
+    """
+    output = self.__run_command(["ipfs", "pin", "ls", "--type=recursive"])
+    pinned_cids = []
+    for line in output.split("\n"):
+      line = line.strip()
+      if not line:
+        continue
+      parts = line.split()
+      if len(parts) > 0:
+        pinned_cids.append(parts[0])
+    return pinned_cids
+
+
+  
+
+  def maybe_start_ipfs(
+    self, 
+    base64_swarm_key: str = None, 
+    ipfs_relay: str = None
+  ) -> bool:
+    """
+    This method initializes the IPFS repository if needed, connects to a relay, and starts the daemon.
+    """
+    if self.ipfs_started:
+      return
+    
+    if base64_swarm_key is None:
+      base64_swarm_key = os.getenv(IPFSCt.EE_SWARM_KEY_CONTENT_BASE64_ENV_KEY)
+      if base64_swarm_key is not None:
+        self.P("Found IPFS swarm key in environment variable.", color='d')
+      
+    if ipfs_relay is None:
+      ipfs_relay = os.getenv(IPFSCt.EE_IPFS_RELAY_ENV_KEY)
+      if ipfs_relay is not None:
+        self.P("Found IPFS relay in environment variable.", color='d')
+      
+    
+    if not base64_swarm_key or not ipfs_relay:
+      self.P("Missing env values EE_SWARM_KEY_CONTENT_BASE64 and EE_IPFS_RELAY.", color='r')
+      return False
+    
+    self.__base64_swarm_key = base64_swarm_key
+    self.__ipfs_relay = ipfs_relay
+    hidden_base64_swarm_key = base64_swarm_key[:8] + "..." + base64_swarm_key[-8:]
+    msg = f"Starting R1FS <{self.__name}>:"
+    msg += f"\n  Relay:    {self.__ipfs_relay}"
+    msg += f"\n  Download: {self.__downloads_dir}"
+    msg += f"\n  Upload:   {self.__uploads_dir}"
+    msg += f"\n  SwarmKey: {hidden_base64_swarm_key}"
+    self.P(msg, color='m')
+    
+    ipfs_repo = os.path.expanduser("~/.ipfs")
+    os.makedirs(ipfs_repo, exist_ok=True)
+    config_path = os.path.join(ipfs_repo, "config")
+    swarm_key_path = os.path.join(ipfs_repo, "swarm.key")
+
+    if not os.path.isfile(config_path):
+      # Repository is not initialized; write the swarm key and init.
+      try:
+        decoded_key = base64.b64decode(base64_swarm_key)
+        with open(swarm_key_path, "wb") as f:
+          f.write(decoded_key)
+        os.chmod(swarm_key_path, 0o600)
+        self.P("Swarm key written successfully.", color='g')
+      except Exception as e:
+        self.P(f"Error writing swarm.key: {e}", color='r')
+        return False
+
+      try:
+        self.P("Initializing IPFS repository...")
+        self.__run_command(["ipfs", "init"])
+      except Exception as e:
+        self.P(f"Error during IPFS init: {e}", color='r')
+        return False
+    else:
+      self.P(f"IPFS repository already initialized in {config_path}.", color='g')
+
+    try:
+      self.P("Removing public IPFS bootstrap nodes...")
+      self.__run_command(["ipfs", "bootstrap", "rm", "--all"])
+    except Exception as e:
+      self.P(f"Error removing bootstrap nodes: {e}", color='r')
+
+    # Check if daemon is already running by attempting to get the node id.
+    try:
+      # explicit run no get_id
+      result = self.__run_command(["ipfs", "id"])
+      self.__ipfs_id = json.loads(result)["ID"]
+      self.__ipfs_address = json.loads(result)["Addresses"][1]
+      self.__ipfs_agent = json.loads(result)["AgentVersion"]
+      self.P("IPFS daemon running", color='g')
+            
+    except Exception:
+      try:
+        self.P("Starting IPFS daemon in background...")
+        subprocess.Popen(["ipfs", "daemon"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        time.sleep(5)
+      except Exception as e:
+        self.P(f"Error starting IPFS daemon: {e}", color='r')
+        return
+
+    try:
+      my_id = self.__get_id()
+      assert my_id != ERROR_TAG, "Failed to get IPFS ID."
+      self.P("IPFS ID: " + my_id, color='g')
+      self.P(f"Connecting to relay: {ipfs_relay}")
+      result = self.__run_command(["ipfs", "swarm", "connect", ipfs_relay])
+      relay_ip = ipfs_relay.split("/")[2]
+      if "connect" in result.lower() and "success" in result.lower():
+        self.P(f"R1FS connected to: {relay_ip}", color='g', boxed=True)
+        self.__ipfs_started = True
+      else:
+        self.P("Relay connection result did not indicate success.", color='r')
+    except Exception as e:
+      self.P(f"Error connecting to relay: {e}", color='r')
+      
+    return self.ipfs_started
+    

{naeural_client-2.7.31.dist-info → naeural_client-2.7.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: naeural_client
-Version: 2.7.31
+Version: 2.7.40
 Summary: `naeural_client` is the Python SDK required for client app development for the Naeural Edge Protocol Edge Protocol framework
 Project-URL: Homepage, https://github.com/NaeuralEdgeProtocol/naeural_client
 Project-URL: Bug Tracker, https://github.com/NaeuralEdgeProtocol/naeural_client/issues

{naeural_client-2.7.31.dist-info → naeural_client-2.7.40.dist-info}/RECORD

@@ -1,5 +1,5 @@
 naeural_client/__init__.py,sha256=YimqgDbjLuywsf8zCWE0EaUXH4MBUrqLxt0TDV558hQ,632
-naeural_client/_ver.py,sha256=txvzfbhMSo8ZJdttNBeVblWMbw5beX2aZ0Rf_eZ4JtA,331
+naeural_client/_ver.py,sha256=w-s-yz0eHjAKD9mUqSByv0yLn__Zw5dFOhJPHU7DI3w,331
 naeural_client/base_decentra_object.py,sha256=C4iwZTkhKNBS4VHlJs5DfElRYLo4Q9l1V1DNVSk1fyQ,4412
 naeural_client/plugins_manager_mixin.py,sha256=X1JdGLDz0gN1rPnTN_5mJXR8JmqoBFQISJXmPR9yvCo,11106
 naeural_client/base/__init__.py,sha256=hACh83_cIv7-PwYMM3bQm2IBmNqiHw-3PAfDfAEKz9A,259
@@ -60,6 +60,8 @@ naeural_client/io_formatter/default/__init__.py,sha256=zOm2tsOk6fXvyCXxsXDnsNs6B
 naeural_client/io_formatter/default/a_dummy.py,sha256=qr9eUizQ-NN5jdXVzkaZKMaf9KS41MpPN_iDoTN_Qd0,1148
 naeural_client/io_formatter/default/aixp1.py,sha256=MX0TeUR4APA-qN3vUC6uzcz8Pssz5lgrQWo7td5Ri1A,3052
 naeural_client/io_formatter/default/default.py,sha256=gEy78cP2D5s0y8vQh4aHuxqz7D10gGfuiKF311QhrpE,494
+naeural_client/ipfs/__init__.py,sha256=vXEDLUNUO6lOTMGa8iQ9Zf7ajIQq9GZuvYraAHt3meE,38
+naeural_client/ipfs/r1fs.py,sha256=u_Dy05wH_Xt4GTvxjLwSoz5hzRz0Tc2Tr-0c2BxwXtg,16503
 naeural_client/logging/__init__.py,sha256=b79X45VC6c37u32flKB2GAK9f-RR0ocwP0JDCy0t7QQ,33
 naeural_client/logging/base_logger.py,sha256=qqqMX30Vmh5Dz8YYaeL_ainQPTP5FsX1Y4QMbsIG5Rg,69599
 naeural_client/logging/small_logger.py,sha256=m12hCb_H4XifJYYfgCAOUDkcXm-h4pSODnFf277OFVI,2937
@@ -86,8 +88,8 @@ naeural_client/utils/comm_utils.py,sha256=4cS9llRr_pK_3rNgDcRMCQwYPO0kcNU7AdWy_L
 naeural_client/utils/config.py,sha256=lAbWe3UMi40BOdsAIZIb-fYtb4LwG3MIYg0EOA1ITr8,10340
 naeural_client/utils/dotenv.py,sha256=_AgSo35n7EnQv5yDyu7C7i0kHragLJoCGydHjvOkrYY,2008
 naeural_client/utils/oracle_sync/oracle_tester.py,sha256=X-923ccjkr6_kzbbiuAAcWSIhMtBDOH2VURjTh55apQ,27235
-naeural_client-2.7.31.dist-info/METADATA,sha256=0GqYz-TDWNENqA9JIZesUQajhYoUFL39urhid0YxPlo,12354
-naeural_client-2.7.31.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-naeural_client-2.7.31.dist-info/entry_points.txt,sha256=CTua17GUrRa4aXeafezGC9TiWKGKQzwTjQmB2jyj22g,91
-naeural_client-2.7.31.dist-info/licenses/LICENSE,sha256=cvOsJVslde4oIaTCadabXnPqZmzcBO2f2zwXZRmJEbE,11311
-naeural_client-2.7.31.dist-info/RECORD,,
+naeural_client-2.7.40.dist-info/METADATA,sha256=cRiEi7AbNAdspHTFOmzlUxQwk6MzIoKtTYwudP65_fo,12354
+naeural_client-2.7.40.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+naeural_client-2.7.40.dist-info/entry_points.txt,sha256=CTua17GUrRa4aXeafezGC9TiWKGKQzwTjQmB2jyj22g,91
+naeural_client-2.7.40.dist-info/licenses/LICENSE,sha256=cvOsJVslde4oIaTCadabXnPqZmzcBO2f2zwXZRmJEbE,11311
+naeural_client-2.7.40.dist-info/RECORD,,