cmdbox 0.6.4.1__py3-none-any.whl → 0.6.5__py3-none-any.whl

cmdbox/app/app.py

@@ -118,6 +118,13 @@ class CmdBoxApp:
         args = argparse.Namespace(**{k:common.chopdq(v) for k,v in args_dict.items()})
         eargsparsetime = time.perf_counter()
 
+        if args.debug_attach:
+            import debugpy
+            debugpy.listen(("", args.debug_attach_port))
+            self.default_logger.info(f"Waiting for debugger attach on port {args.debug_attach_port}...")
+            debugpy.wait_for_client()
+            self.default_logger.info("Debugger attached.")
+
         ret = dict(success=f"Start command. {args}")
         if args.saveopt:
             if args.useopt is None:

cmdbox/app/client.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from cmdbox.app import filer
+from cmdbox.app import common, filer
 from cmdbox.app.commons import convert, redis_client
 import base64
 import logging
@@ -196,12 +196,13 @@ class Client(object):
                 if download_file.exists():
                     self.logger.warning(f"download_file {download_file} already exists.")
                     return dict(warn=f"download_file {download_file} already exists.")
-                with open(download_file, "wb") as f:
+                def _wd(f):
                     f.write(base64.b64decode(res_json["success"]["data"]))
                     del res_json["success"]["data"]
                     res_json["success"]["download_file"] = str(download_file.absolute())
+                common.save_file(download_file, _wd, mode='wb')
         return res_json
-    
+
     def file_upload(self, svpath:str, upload_file:Path, scope:str="client", client_data:Path=None, mkdir:bool=False, orverwrite:bool=False,
                     retry_count:int=3, retry_interval:int=5, timeout:int=60):
         """

cmdbox/app/common.py

@@ -80,8 +80,9 @@ def load_yml(yml_path:Path) -> dict:
     Returns:
         dict: 読み込んだYAMLファイルの内容
     """
-    with open(yml_path, encoding="utf-8") as f:
+    def _r(f):
         return yaml.safe_load(f)
+    return load_file(yml_path, _r)
 
 def save_yml(yml_path:Path, data:dict) -> None:
     """
@@ -91,8 +92,83 @@ def save_yml(yml_path:Path, data:dict) -> None:
         yml_path (Path): YAMLファイルのパス
         data (dict): 書き込むデータ
     """
-    with open(yml_path, 'w') as f:
+    def _w(f):
         yaml.dump(data, f, default_flow_style=False, sort_keys=False)
+    save_file(yml_path, _w)
+
+def load_file(file_path:Path, func, mode='r', encoding='utf-8') -> None:
+    """
+    ファイルを読み込みます。読み込み時に排他ロックします。
+
+    Args:
+        file_path (Path): ファイルのパス
+        func: 読み込んだデータを処理する関数
+        mode (str, optional): ファイルモード. Defaults to 'r'.
+        encoding (str, optional): エンコーディング. Defaults to 'utf-8'.
+    """
+    lock_file = f'{file_path}.lock'
+    try:
+        with open(lock_file, 'w') as fd:
+            try:
+                if sys.platform == 'win32':
+                    import msvcrt
+                    msvcrt.locking(fd.fileno(), msvcrt.LK_LOCK, 0)
+                else:
+                    import fcntl
+                    fcntl.lockf(fd, fcntl.LOCK_EX)
+                if 'b' in mode:
+                    with open(file_path, mode) as f:
+                        return func(f)
+                else:
+                    with open(file_path, mode, encoding=encoding) as f:
+                        return func(f)
+            finally:
+                if sys.platform == 'win32':
+                    msvcrt.locking(fd.fileno(), msvcrt.LK_UNLCK, 0)
+                else:
+                    fcntl.lockf(fd, fcntl.LOCK_UN)
+    finally:
+        try:
+            os.remove(lock_file)
+        except:
+            pass
+
+def save_file(file_path:Path, func, mode='w', encoding='utf-8') -> None:
+    """
+    ファイルに書き込みます。書き込み時に排他ロックします。
+
+    Args:
+        file_path (Path): ファイルのパス
+        func: 書き込む関数
+        mode (str, optional): ファイルモード. Defaults to 'w'.
+        encoding (str, optional): エンコーディング. Defaults to 'utf-8'.
+    """
+    lock_file = f'{file_path}.lock'
+    try:
+        with open(lock_file, 'w') as fd:
+            try:
+                if sys.platform == 'win32':
+                    import msvcrt
+                    msvcrt.locking(fd.fileno(), msvcrt.LK_LOCK, 0)
+                else:
+                    import fcntl
+                    fcntl.lockf(fd, fcntl.LOCK_EX)
+                if 'b' in mode:
+                    with open(file_path, mode) as f:
+                        func(f)
+                else:
+                    with open(file_path, mode, encoding=encoding) as f:
+                        func(f)
+            finally:
+                if sys.platform == 'win32':
+                    msvcrt.locking(fd.fileno(), msvcrt.LK_UNLCK, 0)
+                else:
+                    fcntl.lockf(fd, fcntl.LOCK_UN)
+    finally:
+        try:
+            os.remove(lock_file)
+        except:
+            pass
 
 class CommonValue:
     _map = dict()
@@ -251,8 +327,7 @@ def load_config(mode:str, debug:bool=False, data=HOME_DIR, webcall:bool=False, v
     if not log_conf_path.exists():
         log_conf_path = Path(version.__file__).parent / f"logconf_{ver.__appid__}.yml"
         log_name = mode
-    with open(log_conf_path) as f:
-        log_config = yaml.safe_load(f)
+    log_config = load_yml(log_conf_path)
     std_key = None
     set_common_value('webcall', webcall)
     for k, h in log_config['handlers'].items():
@@ -343,8 +418,9 @@ def saveopt(opt:dict, opt_path:Path, webmode:bool=False) -> None:
                 k = r['opt']
                 d = r['default'] if 'default' in r else None
                 opt[k] = d if k not in lopts else lopts.get(k, d)
-    with open(opt_path, 'w') as f:
+    def _w(f):
         json.dump(opt, f, indent=4, default=default_json_enc)
+    save_file(opt_path, _w)
 
 def saveuser(user_data:dict, user_path:Path) -> None:
     """
@@ -552,9 +628,10 @@ def print_format(data:dict, format:bool, tm:float, output_json:str=None, output_
                 pass
     if output_json is not None:
         try:
-            with open(output_json, 'a' if output_json_append else 'w', encoding='utf-8') as f:
+            def _w(f):
                 json.dump(data, f, default=default_json_enc, ensure_ascii=False)
                 print('', file=f)
+            save_file(Path(output_json), _w, mode='a' if output_json_append else 'w')
         except Exception as e:
             pass
     return txt
@@ -584,8 +661,9 @@ def download_file(url:str, save_path:Path) -> Path:
         Path: 保存したファイルのパス
     """
     r = requests.get(url, stream=True)
-    with open(save_path, 'wb') as f:
+    def _w(f):
         f.write(r.content)
+    save_file(Path(save_path), _w)
     return save_path
 
 def cmd(cmd:str, logger:logging.Logger, slise:int=100, newenv:Dict=None) -> Tuple[int, str, str]:

cmdbox/app/commons/convert.py

@@ -99,8 +99,10 @@ def npy2imgfile(npy, output_image_file:Path=None, image_type:str='jpeg') -> byte
     image_type = 'jpeg' if image_type == 'jpg' else image_type
     img_byte = img2byte(image, format=image_type)
     if output_image_file is not None:
-        with open(output_image_file, 'wb') as f:
+        from cmdbox.app import common
+        def _w(f):
             f.write(img_byte)
+        common.save_file(output_image_file, _w, mode='wb')
     return img_byte
 
 def bytes2b64str(img:bytes) -> str:

cmdbox/app/edge.py

@@ -515,9 +515,9 @@ class Edge(object):
                     except Exception as e:
                         raise HTTPException(status_code=500, detail=f'Failed to get token. {e}')
 
-                if not hasattr(self, 'thUvicorn') or not self.thUvicorn.is_alive():
-                    self.thUvicorn = web.ThreadedUvicorn(config=Config(app=fastapi, host='localhost', port=oauth2_port))
-                    self.thUvicorn.start()
+                if not hasattr(self, 'thHttp') or not self.thHttp.is_alive():
+                    self.thHttp = web.ThreadedASGI(config=Config(app=fastapi, host='localhost', port=oauth2_port))
+                    self.thHttp.start()
                     time.sleep(1)
 
                 # OAuth2認証のリクエストを送信
@@ -581,9 +581,9 @@ class Edge(object):
                     except Exception as e:
                         raise HTTPException(status_code=500, detail=f'Failed to get token. {e}')
 
-                if not hasattr(self, 'thUvicorn') or not self.thUvicorn.is_alive():
-                    self.thUvicorn = web.ThreadedUvicorn(config=Config(app=fastapi, host='localhost', port=oauth2_port))
-                    self.thUvicorn.start()
+                if not hasattr(self, 'thHttp') or not self.thHttp.is_alive():
+                    self.thHttp = web.ThreadedASGI(config=Config(app=fastapi, host='localhost', port=oauth2_port))
+                    self.thHttp.start()
                     time.sleep(1)
 
                 # OAuth2認証のリクエストを送信
@@ -652,9 +652,9 @@ class Edge(object):
                     except Exception as e:
                         raise HTTPException(status_code=500, detail=f'Failed to get token. {e}')
 
-                if not hasattr(self, 'thUvicorn') or not self.thUvicorn.is_alive():
-                    self.thUvicorn = web.ThreadedUvicorn(self.logger, Config(app=fastapi, host='localhost', port=oauth2_port), {})
-                    self.thUvicorn.start()
+                if not hasattr(self, 'thHttp') or not self.thHttp.is_alive():
+                    self.thHttp = web.ThreadedASGI(config=Config(app=fastapi, host='localhost', port=oauth2_port))
+                    self.thHttp.start()
                     time.sleep(1)
 
                 # OAuth2認証のリクエストを送信
@@ -734,9 +734,9 @@ class Edge(object):
                     except Exception as e:
                         raise HTTPException(status_code=500, detail=f'Failed to get token. {e}')
 
-                if not hasattr(self, 'thUvicorn') or not self.thUvicorn.is_alive():
-                    self.thUvicorn = web.ThreadedUvicorn(self.logger, Config(app=fastapi, host='localhost', port=saml_port), {})
-                    self.thUvicorn.start()
+                if not hasattr(self, 'thHttp') or not self.thHttp.is_alive():
+                    self.thHttp = web.ThreadedASGI(config=Config(app=fastapi, host='localhost', port=saml_port))
+                    self.thHttp.start()
                     time.sleep(1)
 
                 # SAML認証のリクエストを送信

cmdbox/app/features/cli/_cmdbox_vision_install.py

@@ -0,0 +1,214 @@
+from cmdbox import version
+from cmdbox.app import common, client, feature
+from cmdbox.app.commons import convert, redis_client
+from cmdbox.app.options import Options
+from cmdbox.app.features.cli import cmdbox_vision_start
+from pathlib import Path
+from typing import Dict, Any, Tuple, List, Union
+import argparse
+import glob
+import logging
+import pip
+import requests
+import shutil
+import subprocess
+import sys
+
+
+class VisionInstall(cmdbox_vision_start.VisionStart):
+    def get_mode(self) -> Union[str, List[str]]:
+        """
+        この機能のモードを返します
+
+        Returns:
+            Union[str, List[str]]: モード
+        """
+        return 'vision'
+
+    def get_cmd(self):
+        """
+        この機能のコマンドを返します
+
+        Returns:
+            str: コマンド
+        """
+        return 'install'
+    
+    def get_option(self):
+        """
+        この機能のオプションを返します
+
+        Returns:
+            Dict[str, Any]: オプション
+        """
+        return dict(
+            use_redis=self.USE_REDIS_MEIGHT, nouse_webmode=True, use_agent=False,
+            description_ja="画像/動画の推論エンジンをインストールします。",
+            description_en="Installs the image/video inference engine.",
+            choice=[
+                dict(opt="host", type=Options.T_STR, default=self.default_host, required=True, multi=False, hide=True, choice=None, web="mask",
+                     description_ja="Redisサーバーのサービスホストを指定します。",
+                     description_en="Specify the service host of the Redis server."),
+                dict(opt="port", type=Options.T_INT, default=self.default_port, required=True, multi=False, hide=True, choice=None, web="mask",
+                     description_ja="Redisサーバーのサービスポートを指定します。",
+                     description_en="Specify the service port of the Redis server."),
+                dict(opt="password", type=Options.T_STR, default=self.default_pass, required=True, multi=False, hide=True, choice=None, web="mask",
+                     description_ja=f"Redisサーバーのアクセスパスワード(任意)を指定します。省略時は `{self.default_pass}` を使用します。",
+                     description_en=f"Specify the access password of the Redis server (optional). If omitted, `{self.default_pass}` is used."),
+                dict(opt="svname", type=Options.T_STR, default=self.default_svname, required=True, multi=False, hide=True, choice=None, web="readonly",
+                     description_ja="サーバーのサービス名を指定します。省略時は `server` を使用します。",
+                     description_en="Specify the service name of the inference server. If omitted, `server` is used."),
+                dict(opt="retry_count", type=Options.T_INT, default=3, required=False, multi=False, hide=True, choice=None,
+                     description_ja="Redisサーバーへの再接続回数を指定します。0以下を指定すると永遠に再接続を行います。",
+                     description_en="Specifies the number of reconnections to the Redis server.If less than 0 is specified, reconnection is forever."),
+                dict(opt="retry_interval", type=Options.T_INT, default=5, required=False, multi=False, hide=True, choice=None,
+                     description_ja="Redisサーバーに再接続までの秒数を指定します。",
+                     description_en="Specifies the number of seconds before reconnecting to the Redis server."),
+                dict(opt="timeout", type=Options.T_INT, default="300", required=False, multi=False, hide=True, choice=None,
+                     description_ja="サーバーの応答が返ってくるまでの最大待ち時間を指定。",
+                     description_en="Specify the maximum waiting time until the server responds."),
+                dict(opt="client_only", type=Options.T_BOOL, default=False, required=False, multi=False, hide=True, choice=[True, False],
+                     description_ja="サーバーへの接続を行わないようにします。",
+                     description_en="Do not make connections to the server."),
+                dict(opt="vision_engine", type=Options.T_STR, default="sam2", required=True, multi=False, hide=False,
+                     choice=["sam2"],
+                     description_ja="使用するVisionエンジンを指定します。",
+                     description_en="Specify the Vision engine to use."),
+            ]
+        )
+
+    def get_svcmd(self):
+        """
+        この機能のサーバー側のコマンドを返します
+
+        Returns:
+            str: サーバー側のコマンド
+        """
+        return 'vision_install'
+
+    def apprun(self, logger:logging.Logger, args:argparse.Namespace, tm:float, pf:List[Dict[str, float]]=[]) -> Tuple[int, Dict[str, Any], Any]:
+        """
+        この機能の実行を行います
+
+        Args:
+            logger (logging.Logger): ロガー
+            args (argparse.Namespace): 引数
+            tm (float): 実行開始時間
+            pf (List[Dict[str, float]]): 呼出元のパフォーマンス情報
+
+        Returns:
+            Tuple[int, Dict[str, Any], Any]: 終了コード, 結果, オブジェクト
+        """
+        if args.vision_engine is None:
+            msg = dict(warn=f"Please specify the --vision_engine option.")
+            common.print_format(msg, False, tm, args.output_json, args.output_json_append, pf=pf)
+            return self.RESP_WARN, msg, None
+        if args.vision_engine == 'sam2':
+            vision_engine_b64 = convert.str2b64str(args.vision_engine)
+            if args.client_only:
+                # クライアントのみの場合は、サーバーに接続せずに実行
+                ret = self.install(common.random_string(), args.vision_engine, logger)
+            else:
+                cl = client.Client(logger, redis_host=args.host, redis_port=args.port, redis_password=args.password, svname=args.svname)
+                ret = cl.redis_cli.send_cmd(self.get_svcmd(), [vision_engine_b64],
+                                            retry_count=args.retry_count, retry_interval=args.retry_interval, timeout=args.timeout, nowait=False)
+            common.print_format(ret, False, tm, None, False, pf=pf)
+            if 'success' not in ret:
+                    return self.RESP_WARN, ret, None
+            return self.RESP_SUCCESS, ret, None
+
+        msg = dict(warn=f"Unsupported vision engine: {args.vision_engine}")
+        common.print_format(msg, False, tm, args.output_json, args.output_json_append, pf=pf)
+        return self.RESP_WARN, msg, None
+
+    def is_cluster_redirect(self):
+        """
+        クラスター宛のメッセージの場合、メッセージを転送するかどうかを返します
+
+        Returns:
+            bool: メッセージを転送する場合はTrue
+        """
+        return False
+
+    def svrun(self, data_dir:Path, logger:logging.Logger, redis_cli:redis_client.RedisClient, msg:List[str],
+              sessions:Dict[str, Dict[str, Any]]) -> int:
+        """
+        この機能のサーバー側の実行を行います
+
+        Args:
+            data_dir (Path): データディレクトリ
+            logger (logging.Logger): ロガー
+            redis_cli (redis_client.RedisClient): Redisクライアント
+            msg (List[str]): 受信メッセージ
+            sessions (Dict[str, Dict[str, Any]]): セッション情報
+        
+        Returns:
+            int: 終了コード
+        """
+        if logger.level == logging.DEBUG:
+            logger.debug(f"audit write svrun msg: {msg}")
+        vision_engine = convert.b64str2str(msg[2])
+        ret = self.install(msg[1], vision_engine, logger)
+
+        if 'success' not in ret:
+            redis_cli.rpush(msg[1], ret)
+            return self.RESP_WARN
+        return self.RESP_SUCCESS
+
+    def install(self, reskey:str, vision_engine:str, logger:logging.Logger) -> Dict[str, Any]:
+        """
+        SAML2をインストールします
+
+        Args:
+            reskey (str): レスポンスキー
+            vision_engine (str): Visionエンジン
+            logger (logging.Logger): ロガー
+
+        Returns:
+            Dict[str, Any]: 結果
+        """
+        try:
+            if vision_engine == 'sam2':
+                #===============================================================
+                # SAM2のモデルファイルのダウンロード
+                sam2_dir = Path(version.__file__).parent / '.sam2' / 'model'
+                # すでにダウンローダーが存在する場合は削除
+                if sam2_dir.exists():
+                    shutil.rmtree(sam2_dir)
+                for mk, mv in self.VISION_MODEL.items():
+                    model_file = sam2_dir / mv['path']
+                    model_file.parent.mkdir(parents=True, exist_ok=True)
+                    # モデルファイルを保存
+                    logger.info(f"Downloading.. : {mv['url']}")
+                    responce = requests.get(f"{mv['url']}", allow_redirects=True)
+                    if responce.status_code != 200:
+                        _msg = f"Failed to download SAM2 model: {responce.status_code} {responce.reason}. {mv['url']}"
+                        logger.error(_msg, exc_info=True)
+                        return dict(warn=_msg)
+                    def _wm(f):
+                        f.write(responce.content)
+                    common.save_file(model_file, _wm, mode='wb')
+                #===============================================================
+                # SAM2のpythonライブラリのインストール
+                whl_url = f'git+https://github.com/facebookresearch/sam2.git'
+                # whlファイルをpipでインストール
+                if logger.level == logging.DEBUG:
+                    logger.debug(f"pip install {whl_url}")
+                rescode = pip.main(['install', str(whl_url)])  # pipのインストール
+                logger.info(f"Install wheel: {whl_url}")
+                if rescode != 0:
+                    _msg = f"Failed to install SAM2 python library: Possible whl not in the environment. {whl_url}"
+                    logger.error(_msg, exc_info=True)
+                    return dict(warn=_msg)
+                #===============================================================
+                # 成功時の処理
+                rescode, _msg = (self.RESP_SUCCESS, dict(success=f'Success to install SAM2 python library. {whl_url}'))
+                return dict(success=_msg)
+            else:
+                _msg = f"Unsupported vision engine: {vision_engine}"
+                logger.error(_msg, exc_info=True)
+                return dict(warn=_msg)
+        except Exception as e:
+            _msg = f"Failed to install vision engine: {e}"
+            logger.warning(_msg, exc_info=True)
+            return dict(warn=_msg)