kmisc 2.1.114__tar.gz → 2.1.116__tar.gz

{kmisc-2.1.114 → kmisc-2.1.116}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kmisc
-Version: 2.1.114
+Version: 2.1.116
 Summary: Enginering useful library
 Home-page: https://github.com/kagepark/kmisc
 Author: Kage Park

{kmisc-2.1.114 → kmisc-2.1.116}/kmisc/__init__.py

@@ -34,7 +34,10 @@ from http.cookies import Morsel # This module for requests when you use build by
 import xml.etree.ElementTree as ET
 from email.mime.text import MIMEText
 from email.mime.base import MIMEBase
-from multiprocessing import Process, Queue
+import multiprocessing
+Process=multiprocessing.Process
+Queue=multiprocessing.Queue
+#from multiprocessing import Process, Queue
 from email.mime.multipart import MIMEMultipart
 try:
     from kmport import *
@@ -64,6 +67,7 @@ pipe_file=None
 log_new_line='\n'
 url_group = re.compile('^(https|http|ftp)://([^/\r\n]+)(/[^\r\n]*)?')
 cdrom_ko=['sr_mod','cdrom','libata','ata_piix','ata_generic','usb-storage']
+env_kmp=Environment(name='__KmP__')
 
 def Abs(*inps,**opts):
     default=opts.get('default',None)
@@ -2734,35 +2738,119 @@ def net_start_single_server(server_port,main_func_name,server_ip='',timeout=0,ma
     ssoc.close()
     return rc
 
-def kmp(mp={},func=None,name=None,timeout=0,quit=False,log_file=None,log_screen=True,log_raw=False, argv=[],queue=None):
+def kmp(mp={},func=None,name=None,timeout=0,quit=False,log_file=None,log_screen=True,log_raw=False, argv=[],queue=False):
+    #Log
+    def LLOG(msg):
+        if 'log' in mp and 'queue' in mp['log']:
+            mp['log']['queue'].put(msg)
+            time.sleep(2)
+
+    # clean thread
+    def kill_thread(thread):
+        if not thread.is_alive():
+            return
+        tid = ctypes.c_long(thread.ident)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(SystemExit))
+        if res > 1:
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+
     # Clean
+    def terminate_process(pobj,max_retry=3):
+        def kill_process(pobj,max_retry):
+            for i in range(max_retry):
+                if isinstance(pobj,dict) and 'mp' in pobj:
+                    pobj=pobj['mp']
+                if type(pobj).__name__ == 'Process':
+                    try:
+                        if pobj.is_alive():
+                            pobj.terminate()
+                            pobj.join()
+                        else:
+                            return True
+                    except:
+                        pass
+                    time.sleep(3)
+                else:
+                    break
+            return False
+        if isinstance(pobj,dict):
+            return kill_process(pobj,max_retry)
+        elif isinstance(pobj,list):
+            rc=True
+            for pp in pobj:
+                if not kill_process(pp,max_retry):
+                    rc=False
+            return rc
+
     for n in [k for k in mp]:
-        if quit is True:
-            if n != 'log':
-                mp[n]['mp'].terminate()
-                if 'log' in mp:
-                    mp['log']['queue'].put('\nterminate function {}'.format(n))
-        else:
-            if mp[n]['timeout'] > 0 and TIME().Int() > mp[n]['timeout']:
-                mp[n]['mp'].terminate()
-                if 'log' in mp:
-                    mp['log']['queue'].put('\ntimeout function {}'.format(n))
-        if not mp[n]['mp'].is_alive():
+        if isinstance(mp[k],dict):
+            timeout=mp[n].get('timeout',0)
+            if quit is True or timeout > 0 and TIME().Int() > timeout:
+                if n != 'log':
+                    if 'mp' in mp[n]:
+                        LLOG('\nterminate function {}'.format(n))
+                        if terminate_process(mp[n]):
+                            del mp[n]
+        elif isinstance(mp[k],list):
+            for mm in mp[n]:
+                if not isinstance(mm,dict): continue
+                timeout=mm.get('timeout',0)
+                if quit is True or timeout > 0 and TIME().Int() > timeout:
+                    if n != 'log':
+                        if 'mp' in mm:
+                            LLOG('\nterminate function {}'.format(n))
+                            terminate_process(mm)
             del mp[n]
-    if quit is True and 'log' in mp:
-        mp['log']['queue'].put('\nterminate function log')
-        TIME().Sleep(2)
-        mp['log']['mp'].terminate()
+    env_mp=env_kmp.get()
+    for n in [k for k in env_mp]:
+        if isinstance(env_mp[n],dict):
+            timeout=env_mp[n].get('timeout',0)
+            if quit is True or timeout > 0 and TIME().Int() > timeout:
+                if n != 'log':
+                    if 'mp' in env_mp[n]:
+                        LLOG('\nterminate function {}'.format(n))
+                        if terminate_process(env_mp[n]):
+                            env_kmp.remove(n)
+        elif isinstance(env_mp[n],list):
+            for mm in env_mp[n]:
+                if not isinstance(mm,dict): continue
+                timeout=mm.get('timeout',0)
+                if quit is True or timeout > 0 and TIME().Int() > timeout:
+                    if n != 'log':
+                        if 'mp' in mm:
+                            LLOG('\nterminate function {}'.format(n))
+                            terminate_process(mm)
+            env_kmp.remove(n)
+    if quit is True:
+        #cleanup log
+        if 'log' in mp:
+            mp['log']['stop']=True # stop signal
+            if 'queue' in mp['log']:
+                mp['log']['queue'].put('__exit__all__') #stop message
+            if 'mp' in mp['log']:
+                if terminate_process(mp['log']['mp']):  # terminate
+                    del mp['log']
+        #cleanup all child process again
+        for c in multiprocessing.active_children():
+            c.terminate()
+            c.join()
+        #kill still remained
+        for c in multiprocessing.active_children():
+            c.kill()
+            c.join()
         return
 
     # LOG
-    def logging(ql,log_file=None,log_screen=True,raw=False):
+    def logging(ql,log_file=None,log_screen=False,raw=False,mp={}):
         while True:
-            #if not ql.empty():
+            if isinstance(mp,dict) and mp.get('log',{}).get('stop'): # if stop signal then break
+                break
             if ql.empty():
                 TIME().Sleep(0.01)
             else:
                 ll=ql.get()
+                if ll == '__exit__all__':
+                    break # if message are exit message then quit
                 if raw:
                     log_msg=ll
                 else:
@@ -2776,14 +2864,17 @@ def kmp(mp={},func=None,name=None,timeout=0,quit=False,log_file=None,log_screen=
                     sys.stdout.write(log_msg)
                     sys.stdout.flush()
 
-    if 'log' not in mp or not mp['log']['mp'].is_alive():
-        #log=multiprocessing.Queue()
-        log=Queue()
-        #lqp=multiprocessing.Process(name='log',target=logging,args=(log,log_file,log_screen,log_raw,))
-        lqp=Process(name='log',target=logging,args=(log,log_file,log_screen,log_raw,))
-        lqp.daemon = True
-        mp.update({'log':{'mp':lqp,'start':TIME().Int(),'timeout':0,'queue':log}})
-        lqp.start()
+    #Start Log Daemon
+    if log_screen or log_file:
+        if 'log' not in mp:
+            log=Queue()
+            lqp=Process(name='log',target=logging,args=(log,log_file,log_screen,log_raw,mp,))
+            lqp.daemon = True
+            mp['log']={'mp':lqp,'start':TIME().Int(),'timeout':0,'queue':log}
+            lqp.start()
+        else:
+            if not mp['log']['mp'].is_alive():
+                mp['log']['mp'].start()
 
     # Functions
     if func:
@@ -2791,22 +2882,18 @@ def kmp(mp={},func=None,name=None,timeout=0,quit=False,log_file=None,log_screen=
             name=func.__name__
         if name not in mp:
             if argv:
-                #mf=multiprocessing.Process(name=name,target=func,args=tuple(argv))
                 mf=Process(name=name,target=func,args=tuple(argv))
             else:
-                #mf=multiprocessing.Process(name=name,target=func)
                 mf=Process(name=name,target=func)
             if timeout > 0:
                 timeout=TIME().Int()+timeout
-            
-#            for aa in argv:
-#                if type(aa).__name__ == 'Queue':
-#                    mp.update({name:{'mp':mf,'timeout':timeout,'start':now(),'queue':aa}})
-            if name not in mp:
+            if name in mp:
+                printf('Already multiprocess has same name',logfile=log_file,mode='d')
+            else:
                 if queue and type(queue).__name__ == 'Queue':
-                    mp.update({name:{'mp':mf,'timeout':timeout,'start':TIME().Int(),'queue':queue}})
+                    mp[name]={'mp':mf,'timeout':timeout,'start':TIME().Int(),'queue':queue}
                 else:
-                    mp.update({name:{'mp':mf,'timeout':timeout,'start':TIME().Int()}})
+                    mp[name]={'mp':mf,'timeout':timeout,'start':TIME().Int()}
             mf.start()
     return mp
 

{kmisc-2.1.114 → kmisc-2.1.116}/kmisc.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: kmisc
-Version: 2.1.114
+Version: 2.1.116
 Summary: Enginering useful library
 Home-page: https://github.com/kagepark/kmisc
 Author: Kage Park