remote-run-everything 2.0.3__py3-none-any.whl → 2.0.5__py3-none-any.whl

remote_run_everything/__init__.py

@@ -1,5 +1,4 @@
 from remote_run_everything.deploy.by_http import ByHttp
-from remote_run_everything.deploy.by_http_server import ByHttpServer, cherrypy_in_daemon
 
 from remote_run_everything.db.crude_duck import CrudeDuck
 from remote_run_everything.db.crud_sqlalchemy import Crud

remote_run_everything/db/crud_sqlalchemy.py

@@ -1,10 +1,16 @@
 import os, pymysql
 from urllib.parse import quote_plus
-
-from sqlalchemy import create_engine, select, update, and_, insert, delete
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy import create_engine, select, update, and_, insert, delete, text
 
 
 class Crud:
+    def auto_tab(self, engine):
+        Base = automap_base()
+        Base.prepare(autoload_with=engine)
+        # FinIndi = Base.classes.fin_indi
+        return Base.classes
+
     def sqlite_engine(self, dbpath):
         dir = os.path.dirname(dbpath)
         os.makedirs(dir, exist_ok=True)
@@ -50,10 +56,11 @@ class Crud:
             cols = mod.__dict__['__annotations__'].keys()
         else:
             cols = [i for i in mod.__dict__.keys() if not i.startswith("__")]
+        cols = [i for i in cols if i != '_sa_class_manager']
         return cols
 
     def insert_many(self, engine, mod, l):
-        if len(l)==0:return
+        if len(l) == 0: return
         cols = self.table_columns(mod)
         with engine.connect() as conn:
             for dic in l:
@@ -78,20 +85,53 @@ class Crud:
             conn.execute(stmt)
             conn.commit()
 
+    def update(self, engine, mod, cond, dic):
+        cols = self.table_columns(mod)
+        dic = {k: v for k, v in dic.items() if k in cols}
+        with engine.connect() as conn:
+            stmt = update(mod).where(cond).values(dic)
+            conn.execute(stmt)
+            conn.commit()
+
+    # cond = and_(BdhPrice.date == dic['date'], BdhPrice.goodsName == dic['goodsName'])
+    # cond1 = and_(FinIndi.year == '2020')
+    # cond2 = or_(FinIndi.year == "2021")
+    # cond = or_(cond1, cond2)
     def upsert(self, engine, mod, cond, dic):
-        id = self.exist_id(engine, mod, cond)
-        if id is not None:
-            self.update_by_id(engine, mod, id, dic)
+        exists = self.query_cond(engine, mod, cond)
+        if len(exists) > 0:
+            self.update(engine, mod, cond, dic)
             return
         self.insert_one(engine, mod, dic)
 
     def delete_by_id(self, engine, mod, id):
+        cond = and_(mod.id == id)
         with engine.connect() as conn:
-            stmt = delete(mod).where(mod.id == id)
+            stmt = delete(mod).where(cond)
             conn.execute(stmt)
             conn.commit()
 
     def delete(self, engine, mod, cond):
-        id = self.exist_id(engine, mod, cond)
-        if id is not None:
-            self.delete_by_id(engine, mod, id)
+        with engine.connect() as conn:
+            stmt = delete(mod).where(cond)
+            conn.execute(stmt)
+            conn.commit()
+
+    def query_cond(self, engine, mod, cond):
+        mdic = lambda cols, row: {i[0]: i[1] for i in zip(cols, row)}
+        with engine.connect() as ses:
+            stmt = select(mod).where(cond)
+            rows = ses.execute(stmt)
+            cols = rows.keys()
+            return [mdic(cols, r) for r in rows]
+
+    def query_sql(self, engine, sql):
+        mdic = lambda cols, row: {i[0]: i[1] for i in zip(cols, row)}
+        with engine.connect() as ses:
+            rows = ses.execute(text(sql))
+            cols = rows.keys()
+            return [mdic(cols, r) for r in rows]
+
+    def execute_sql(self, engine, sql):
+        with engine.connect() as ses:
+            return ses.execute(text(sql))

remote_run_everything/db/crude_duck.py

@@ -48,6 +48,9 @@ class CrudeDuck:
         scheme = {i[0]: i[1] for i in con.sql(sql).fetchall()}
         return scheme
 
+    def quot_comma(self, l):
+        return ','.join([f"'{i}'" for i in l])
+
     def max_id(self, con, table):
         sql = f'select max(id) from {table}'
         a = con.sql(sql).fetchone()

remote_run_everything/deploy/by_http.py

@@ -37,9 +37,9 @@ class ByHttp:
         c.insert_many(eg, mod, add_l)
 
     def up(self, disallow_keys=None):
-        if os.name == "nt":
-            return self.up_win(disallow_keys)
-        assert self.dbpath.endswith(".db"), "dbpath should be xxx.db"
+        # if os.name == "nt":
+        #     return self.up_win(disallow_keys)
+        # assert self.dbpath.endswith(".db"), "dbpath should be xxx.db"
         c = Crud()
         eg = c.sqlite_engine(self.dbpath)
         c.create_table(eg, Up)
@@ -62,21 +62,4 @@ class ByHttp:
         con.commit()
         c.insert_many(eg, mod, add_l)
 
-    def up_win(self, disallow_keys=None):
-        from mongo_emb import PyMongoEmb
-        if not os.path.exists(self.dbpath):
-            os.makedirs(self.dbpath)
-        assert os.path.isdir(self.dbpath), "dbpath should be dir"
-        path = os.path.normpath(self.dbpath)
-        db = PyMongoEmb(path)
-        col = db['up']
-        loc_files = self.t.all_local_path(self.local, disallow_keys)
-        for dic in loc_files:
-            dic['host'] = self.host
-            # time path host
-            if col.find_one(dic) is not None:
-                continue
-            if os.path.normpath(os.path.dirname(dic['path'])) == path: continue
-            self.t.push(self.host, dic['path'], self.local, self.remote)
-            print("up==", dic)
-            col.update_one({"host": self.host, "path": dic['path']}, {"$set": {"time": dic['time']}}, upsert=True)
+

remote_run_everything/deploy/by_http_tool.py

@@ -5,19 +5,24 @@ import os, glob, arrow, requests
 
 class ByHttpTool:
     def push(self, host, f, local, remote):
-        b64 = Common().readb64(f)
-        push_url = f"{host}/wb64"
-        path = self.loc2remote(f, local, remote)
-        pay = {"b64": b64, "path": path}
-        res = requests.post(push_url, json=pay).json()
-        return res
+        # b64 = Common().readb64(f)
+        # push_url = f"{host}/wb64"
+        # pay = {"b64": b64, "path": path}
+        # res = requests.post(push_url, json=pay).json()
+        rpath = self.loc2remote(f, local, remote)
+        lpath = f
+        self.upload_file(host, rpath, lpath)
+        return
 
     def pull(self, host, f, local, remote):
-        res = requests.post(f"{host}/rb64", json={"path": f}).json()
-        b64 = res['data']
-        path = self.remote2loc(f, local, remote)
-        Common().writeb64(path, b64)
-        return path
+        rpath = f
+        lpath = self.remote2loc(f, local, remote)
+        self.download_file(host, rpath, lpath)
+        # res = requests.post(f"{host}/rb64", json={"path": f}).json()
+        # b64 = res['data']
+        # path = self.remote2loc(f, local, remote)
+        # Common().writeb64(path, b64)
+        return
 
     def all_remote_path(self, host, root, disallow_keys=None):
         url = f"{host}/iterdir"
@@ -62,3 +67,22 @@ class ByHttpTool:
     def remote2loc(self, f, local, remote):
         rela = os.path.relpath(f, remote)
         return f"{local}/{rela}"
+
+    def download_file(self, host, rpath, lpath):
+        ldir = os.path.dirname(lpath)
+        os.makedirs(ldir, exist_ok=True)
+        url = f"{host}/downfile"
+        pay = {"path": rpath}
+        # NOTE the stream=True parameter below
+        with requests.post(url, data=pay, stream=True) as r:
+            r.raise_for_status()
+            with open(lpath, 'wb') as f:
+                for chunk in r.iter_content(chunk_size=8192):
+                    f.write(chunk)
+
+    def upload_file(self, host, rpath, lpath):
+        url = f"{host}/upfile"
+        files = {'files': open(lpath, 'rb')}
+        r = requests.post(url, files=files, data={'path': rpath})
+        if "ok" in r.text: return "ok"
+        return "fail"

remote_run_everything/tools/common1.py

@@ -1,7 +1,7 @@
 import jinja2, requests, os
 import pandas as pd
 import base64
-import os, signal, glob, arrow
+import os, signal, glob, arrow, uuid, hashlib
 
 
 class Common1:
@@ -94,6 +94,10 @@ class Common1:
             if dif.days > n:
                 os.remove(f)
 
+    def str2uuid(self, s):
+        hex_string = hashlib.md5(s.encode("UTF-8")).hexdigest()
+        return str(uuid.UUID(hex=hex_string))
+
 
 if __name__ == '__main__':
     g = Common1()

remote_run_everything/tools/sqlacodegen_go_struct.py

@@ -1,6 +1,10 @@
 import os
 from dataclasses import dataclass
 
+'''
+sqlacodegen mssql+pyodbc://sa:a@127.0.0.1:1433/pstarback?driver=ODBC+Driver+17+for+SQL+Server > ./test.py
+'''
+
 
 @dataclass
 class Tab:
@@ -105,5 +109,5 @@ class Sql2go:
 
 
 if __name__ == '__main__':
-    s = SqlRead("./pstarback.py")
+    s = Sql2go("./pstarback.py")
     s.write_go()

{remote_run_everything-2.0.3.dist-info → remote_run_everything-2.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: remote_run_everything
-Version: 2.0.3
+Version: 2.0.5
 Summary: Deploy Tools
 Author-email: Wang Qi <wangmarkqi@gmail.com>
 License-Expression: MIT

{remote_run_everything-2.0.3.dist-info → remote_run_everything-2.0.5.dist-info}/RECORD

@@ -1,13 +1,12 @@
-remote_run_everything/__init__.py,sha256=QZgJLX-Wjt31-7gK1IREJ68ipzznnKs8Qiwd0F3jBMQ,850
+remote_run_everything/__init__.py,sha256=s0ZYAkpui5x47uOzKMkghoVW9-yW6Y4xvoEntyB0_cc,760
 remote_run_everything/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 remote_run_everything/db/backup.py,sha256=_mDUNQo1q6Gc75-diwadrooZsdV89KGG9mIcS4rmcBQ,1152
-remote_run_everything/db/crud_sqlalchemy.py,sha256=c6zxrIRbNT1Lsunpu6zisy7mrsRCs93Gm3hxDTHGCtg,3608
-remote_run_everything/db/crude_duck.py,sha256=qo3KSUTTVkSmVMfyK4UmMkJS4fQdj_xN57zwDpwzxEE,4290
+remote_run_everything/db/crud_sqlalchemy.py,sha256=cDyDGHwE-1TUBZrXMVDTjIa4Z0ZUQvM9m9sk-eIwaSM,5192
+remote_run_everything/db/crude_duck.py,sha256=OYkE26_Js_PFr0DoBOBCZ6tYxsp7BhhNvGMn7-rJgWA,4370
 remote_run_everything/db/kv_store.py,sha256=Df2Lp2l5ip0YQuTdVnlQbkZT_Sz01j9D5bJ8PYaSFRY,3357
 remote_run_everything/deploy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-remote_run_everything/deploy/by_http.py,sha256=bDbngxTTkGnMAvUw7yJ9aQAdMbvDUJlLnbBGhTAO8U8,3343
-remote_run_everything/deploy/by_http_server.py,sha256=_CVOQvUMnZKIJ8eVYXcnAPNBBR9lwSE_Kk6cu41sxuE,1726
-remote_run_everything/deploy/by_http_tool.py,sha256=x2GUi7x4MIhTcaxIsk-nXDQBgUdSO_rSN5z81bnEAoA,2178
+remote_run_everything/deploy/by_http.py,sha256=q08g3qCoDcu7y1SfEfRsWU2rlbJAYZDHYEZ7Utprf8k,2471
+remote_run_everything/deploy/by_http_tool.py,sha256=w0BAcp9-nEIyKmCSBLoZzFMsILzf91nLDiH0iDMOr0w,3110
 remote_run_everything/deploy/record_mod.py,sha256=29L-RTnvANpjbWRND7GGz5sdCzbz1Ba9QxdS858bdAY,725
 remote_run_everything/nosql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 remote_run_everything/nosql/no_sql.py,sha256=1-dT6LhrayQqLqZ4ES2tmWfXYCEXOnqwNzKgyXWYRG4,2343
@@ -16,13 +15,13 @@ remote_run_everything/nosql/no_sql_pg.py,sha256=XCrEnHKp5RVHPOyPOMICY2PnvNMkkIID
 remote_run_everything/nosql/no_sql_tool.py,sha256=xRu7vsDgr3JQNSo6CLNTtNNxlg4fl-Q7_vw4y9lklWI,2590
 remote_run_everything/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 remote_run_everything/tools/common.py,sha256=UozWsdBwf7mGnvv5hLiaIfalhDaSaMjDOBjky8mo2w8,1747
-remote_run_everything/tools/common1.py,sha256=XEpQBp0zhRs2jM9t7z7b_WFV1j00O3s9T9UlYes5r3c,3502
+remote_run_everything/tools/common1.py,sha256=ffDHzjiLCc9vc8slaHOmH8xlzXOAtLdtVjIRkBcN7y4,3659
 remote_run_everything/tools/decorators.py,sha256=SIacNAs7afgkU0D09J-s-YscCVnxSG8Qj9vSL4VzCHw,1988
-remote_run_everything/tools/sqlacodegen_go_struct.py,sha256=NFXRDPOWU7sP9V3ZAtfPQvLl-jJ7SL3qcrYrIrrLKOI,3119
+remote_run_everything/tools/sqlacodegen_go_struct.py,sha256=0xWJVCjFyEF2VjC1aGo6DmIqEQQ0Q46CfBAb9FSCUuE,3237
 remote_run_everything/vsconf/conf_txt.py,sha256=nhFuKLlts-sCIBmfr0IKv1pP-qPUvQQrsRRg21q5Gd4,2418
 remote_run_everything/vsconf/core.py,sha256=HmSEzXjGPY7R64rwfAV024YxMHwmBkLin6lGace4U0M,833
-remote_run_everything-2.0.3.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
-remote_run_everything-2.0.3.dist-info/METADATA,sha256=HT0UNlT3V-3sDl9g0mii35olbOu3Ll9kzT6IqEQGzzc,3121
-remote_run_everything-2.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-remote_run_everything-2.0.3.dist-info/top_level.txt,sha256=1TUcAqPgSwiVBqUHz-1pZFXvRpr9cudEYlmfw_mztRY,22
-remote_run_everything-2.0.3.dist-info/RECORD,,
+remote_run_everything-2.0.5.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
+remote_run_everything-2.0.5.dist-info/METADATA,sha256=Z0SSxn3qhvpwoMFV-GN4l6lEvquPzr7lfNjNAf2QVHs,3121
+remote_run_everything-2.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+remote_run_everything-2.0.5.dist-info/top_level.txt,sha256=1TUcAqPgSwiVBqUHz-1pZFXvRpr9cudEYlmfw_mztRY,22
+remote_run_everything-2.0.5.dist-info/RECORD,,

remote_run_everything/deploy/by_http_server.py

@@ -1,56 +0,0 @@
-import cherrypy
-from remote_run_everything.deploy.by_http_tool import ByHttpTool
-from remote_run_everything.tools.common import Common
-
-from cherrypy.process.plugins import Daemonizer
-
-
-def cherrypy_in_daemon(myapp, port, prefix):
-    cherrypy.config.update({
-        "server.socket_port": port,
-    })
-    Daemonizer(cherrypy.engine).subscribe()
-    cherrypy.tree.mount(myapp(), prefix)
-    cherrypy.engine.start()
-    cherrypy.engine.block()
-
-
-class ByHttpServer:
-
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @cherrypy.tools.json_in()
-    def rb64(self):
-        try:
-            args = cherrypy.request.json
-            path = args['path']
-            data = Common().readb64(path)
-            return {"status": "ok", "data": data}
-        except Exception as e:
-            return {"status": "fail", "data": str(e)}
-
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @cherrypy.tools.json_in()
-    def wb64(self):
-        try:
-            args = cherrypy.request.json
-            path = args['path']
-            b64 = args['b64']
-            Common().writeb64(path, b64)
-            return {"status": "ok", "data": path}
-        except Exception as e:
-            return {"status": "fail", "data": str(e)}
-
-    @cherrypy.expose
-    @cherrypy.tools.json_out()
-    @cherrypy.tools.json_in()
-    def iterdir(self):
-        try:
-            args = cherrypy.request.json
-            root = args['root']
-            disallow = args.get("disallow_keys", [])
-            data = ByHttpTool().all_local_path(root, disallow)
-            return {"status": "ok", "data": data}
-        except Exception as e:
-            return {"status": "fail", "data": str(e)}