remote-run-everything 2.0.2__py3-none-any.whl

remote_run_everything/__init__.py

@@ -0,0 +1,16 @@
+from remote_run_everything.deploy.by_http import ByHttp
+from remote_run_everything.deploy.by_http_server import ByHttpServer, cherrypy_in_daemon
+
+from remote_run_everything.db.crude_duck import CrudeDuck
+from remote_run_everything.db.crud_sqlalchemy import Crud
+from remote_run_everything.db.kv_store import KvStore
+from remote_run_everything.db.backup import BackUp
+
+from remote_run_everything.tools.common import Common
+from remote_run_everything.tools.sqlacodegen_go_struct import Sql2go
+from remote_run_everything.tools.decorators import cache_by_1starg, cache_by_name, cache_by_rkey,cache_by_nth_arg
+
+from remote_run_everything.nosql.no_sql import Nosql
+from remote_run_everything.nosql.no_sql_pg import NosqlPg
+from remote_run_everything.nosql.no_sql_mysql import NosqlMysql
+from remote_run_everything.vsconf.core import VsConf

remote_run_everything/db/backup.py

@@ -0,0 +1,38 @@
+from bson import BSON, decode_all
+import pymongo
+import os, sys
+
+
+class BackUp:
+    def __init__(self,  root):
+        self.root = root
+        os.makedirs(self.root, exist_ok=True)
+
+    def mongo_dump(self,cli,dbs):
+        for d in dbs:
+            db = cli[d]
+            dir = os.path.join(self.root, d)
+            os.makedirs(dir, exist_ok=True)
+            cols = db.list_collection_names()
+            for col in cols:
+                sr = db[col]
+                # Dump.
+                with open(f'{dir}/{col}.bson', 'wb+') as f:
+                    for doc in sr.find():
+                        f.write(BSON.encode(doc))
+
+    def mongo_restore(self,cli,dbs):
+        for d in dbs:
+            db = cli[d]
+            dir = os.path.join(self.root, d)
+            files = os.listdir(dir)
+            cols = [i.split('.')[0] for i in files]
+            print(cols)
+            for name in cols:
+                file = f"{dir}/{name}.bson"
+                col = db[name]
+                with open(file, 'rb') as f:
+                    data = decode_all(f.read())
+                    col.insert_many(data)
+
+

remote_run_everything/db/crud_sqlalchemy.py

@@ -0,0 +1,97 @@
+import os, pymysql
+from urllib.parse import quote_plus
+
+from sqlalchemy import create_engine, select, update, and_, insert, delete
+
+
+class Crud:
+    def sqlite_engine(self, dbpath):
+        dir = os.path.dirname(dbpath)
+        os.makedirs(dir, exist_ok=True)
+        url = f"sqlite:///{dbpath}"
+        engine = create_engine(url, future=True, connect_args={'timeout': 30, 'check_same_thread': False})
+        return engine
+
+    def pg_url(self, user, pwd, host, port, db):
+        f = lambda x: quote_plus(x)
+        return f"postgresql://{f(user)}:{f(pwd)}@{f(host)}:{port}/{db}"
+
+    def pg_engine(self, user, pwd, host, port, db):
+        return create_engine(self.pg_url(user, pwd, host, port, db), pool_recycle=3600, pool_size=80, max_overflow=-1,
+                             echo=False,
+                             future=True)
+
+    def mysql_url(self, user, pwd, host, port, db):
+        f = lambda x: quote_plus(x)
+        return f"mysql+pymysql://{f(user)}:{f(pwd)}@{f(host)}:{port}/{db}"
+
+    def mysql_engine(self, user, pwd, host, port, db):
+        # print(f"sqlacodegen {url} >> {db}.py")
+        return create_engine(self.mysql_url(user, pwd, host, port, db), pool_recycle=3600, pool_size=80,
+                             max_overflow=-1, echo=False,
+                             future=True)
+
+    def drop_table(self, engine, mod):
+        mod.__table__.drop(engine)
+
+    def create_table(self, engine, mod):
+        mod.__table__.create(engine, checkfirst=True)
+
+    def exist_id(self, engine, mod, cond):
+        with engine.connect() as conn:
+            stmt = select(mod).where(cond).limit(1)
+            id = conn.scalar(stmt)
+            if id is not None:
+                return id
+            return None
+
+    def table_columns(self, mod):
+        if "__annotations__" in mod.__dict__:
+            cols = mod.__dict__['__annotations__'].keys()
+        else:
+            cols = [i for i in mod.__dict__.keys() if not i.startswith("__")]
+        return cols
+
+    def insert_many(self, engine, mod, l):
+        if len(l)==0:return
+        cols = self.table_columns(mod)
+        with engine.connect() as conn:
+            for dic in l:
+                dic = {k: v for k, v in dic.items() if k in cols}
+                stmt = insert(mod).values(dic)
+                conn.execute(stmt)
+            conn.commit()
+
+    def insert_one(self, engine, mod, dic):
+        cols = self.table_columns(mod)
+        dic = {k: v for k, v in dic.items() if k in cols}
+        with engine.connect() as conn:
+            stmt = insert(mod).values(dic)
+            conn.execute(stmt)
+            conn.commit()
+
+    def update_by_id(self, engine, mod, id, dic):
+        cols = self.table_columns(mod)
+        dic = {k: v for k, v in dic.items() if k in cols}
+        with engine.connect() as conn:
+            stmt = update(mod).where(mod.id == id).values(dic)
+            conn.execute(stmt)
+            conn.commit()
+
+    def upsert(self, engine, mod, cond, dic):
+        id = self.exist_id(engine, mod, cond)
+        if id is not None:
+            self.update_by_id(engine, mod, id, dic)
+            return
+        self.insert_one(engine, mod, dic)
+
+    def delete_by_id(self, engine, mod, id):
+        with engine.connect() as conn:
+            stmt = delete(mod).where(mod.id == id)
+            conn.execute(stmt)
+            conn.commit()
+
+    def delete(self, engine, mod, cond):
+        id = self.exist_id(engine, mod, cond)
+        if id is not None:
+            self.delete_by_id(engine, mod, id)

remote_run_everything/db/crude_duck.py

@@ -0,0 +1,122 @@
+import duckdb, os, arrow
+
+
+class CrudeDuck:
+
+    def install_sql_ext(self, dbpath):
+        dir = os.path.dirname(dbpath)
+        os.makedirs(dir, exist_ok=True)
+        sql = f"ATTACH '{dbpath}' AS db (TYPE sqlite,journal_mode wal);use db;"
+        con = duckdb.connect()
+        con.install_extension("sqlite")
+        con.load_extension("sqlite")
+        con.sql(sql)
+        return con
+
+    def install_pg_ext(self, user, pwd, host, port, dbname):
+        sql = f'''ATTACH 'dbname={dbname} user={user} 
+            host={host} port={port} connect_timeout=10 password={pwd}'
+             AS {dbname} (TYPE postgres);use {dbname};'''
+        con = duckdb.connect()
+        con.install_extension("postgres")
+        con.load_extension("postgres")
+        con.sql(sql)
+        return con
+
+    def install_mysql_ext(self, user, pwd, host, port, dbname):
+        con = duckdb.connect()
+        con.install_extension("mysql")
+        con.load_extension("mysql")
+        sql = f"ATTACH 'host={host} user={user} password={pwd} port={port} database={dbname}' AS msqldb (TYPE MYSQL);"
+        con.sql(sql)
+        con.sql(f"USE msqldb;")
+        return con
+
+    def scheme(self, con, db, table, dbtype):
+        # for mysql db== dbname ; for pg db==public for sqlite db=main
+        if dbtype == "mysql":
+            db = db
+        elif dbtype == "sqlite3":
+            db = "main"
+        elif dbtype == "pg":
+            db = "public"
+        else:
+            db = db
+        sql = f'''  SELECT column_name, data_type FROM information_schema.columns
+    WHERE table_schema = '{db}' AND table_name   = '{table}';
+            '''
+        scheme = {i[0]: i[1] for i in con.sql(sql).fetchall()}
+        return scheme
+
+    def max_id(self, con, table):
+        sql = f'select max(id) from {table}'
+        a = con.sql(sql).fetchone()
+        if a is None or a[0] is None: return 0
+        return a[0] + 1
+
+    def sql_from_ty(self, ty, v):
+        if v is None: return None
+        ty = ty.upper()
+        if ty in ['BIGINT', "TINYINT", "INTEGER", "BOOLEAN"]:
+            return str(int(v))
+        if ty in ["VARCHAR"]:
+            return f"'{str(v)}'"
+        if "TIMESTAMP" in ty:
+            return f"'{arrow.get(v).format()[:19]}'"
+        return None
+
+    def dic2sql(self, data, scheme):
+        dic = {}
+        for k, v in data.items():
+            if k not in scheme.keys(): continue
+            tyv = self.sql_from_ty(scheme[k], v)
+            if tyv is not None:            dic[k] = tyv
+        if len(dic) == 0:
+            return "", ""
+        cols = ", ".join(dic.keys())
+        values = ", ".join(dic.values())
+        return f"({cols})", f"({values})"
+
+    def list2sql(self, l, scheme):
+        cols = ""
+        values = ""
+        for data in l:
+            dic = {}
+            for k, v in data.items():
+                if k not in scheme.keys(): continue
+                tyv = self.sql_from_ty(scheme[k], v)
+                if tyv is not None:   dic[k] = tyv
+            if len(dic) == 0:
+                continue
+            cols = ", ".join(dic.keys())
+            s = ", ".join(dic.values())
+            values = values + f"({s}),"
+        return f"({cols})", values
+
+    def insert_many(self, con, db, table, data, dbtype=""):
+        sche = self.scheme(con, db, table, dbtype)
+        cols, values = self.list2sql(data, sche)
+        if values == "": return
+        sql = f'insert into {table} {cols} values {values}'
+        con.execute(sql)
+
+    def insert_one(self, con, db, table, data, dbtype=""):
+        sche = self.scheme(con, db, table, dbtype)
+        cols, values = self.dic2sql(data, sche)
+        if values == "": return
+        sql = f'insert into {table} {cols} values {values}'
+        con.execute(sql)
+
+    def drop_table(self, con, table):
+        sql = f"drop table if exists {table}"
+        con.execute(sql)
+
+    def delete_by_ids(self, con, table, ids):
+        ids = [str(i) for i in ids]
+        ids = ','.join(ids)
+        sql = f"delete from  {table} where id in ({ids})"
+        con.execute(sql)
+
+    def delete_by_id(self, con, table, id):
+        sql = f"delete from  {table} where id = {id}"
+        con.execute(sql)

remote_run_everything/db/kv_store.py

@@ -0,0 +1,109 @@
+import sqlite3, os, json, arrow
+
+'''
+kv = KeyValueStore('test.db')  # uses SQLite
+
+print(len(kv))                 # 0 item
+kv['hello1'] = 'you1'
+
+del kv['hello1']
+print(len(kv))                 # 2 items remaining
+print('hello1' in kv)          # False, it has just been deleted!
+
+kv['hello3'] = 'newvalue'      # redefine an already present key/value
+
+print(kv.keys())               # ['hello2', 'hello3']
+print(kv.values())             # ['you2', 'newvalue']
+print(kv.items())              # [('hello2', 'you2'), ('hello3', 'newvalue')]
+
+for k in kv:
+    print(k, kv[k])
+
+kv.close()  
+'''
+
+
+class KvStore(dict):
+    def __init__(self, filename=None):
+        self.db_path = self.default_db_path(filename)
+        self.conn = sqlite3.connect(self.db_path, isolation_level=None)
+        self.conn.execute('pragma journal_mode=wal')
+        self.conn.execute("CREATE TABLE IF NOT EXISTS kv (key text unique, value text)")
+
+    def default_db_path(self, db_path):
+        if db_path is None:
+            db_path = "D://wq/temp/decor.db" if os.name == 'nt' else "/data/temp/decor.db"
+        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        return db_path
+
+    def close(self):
+        self.conn.close()
+
+    def commit(self):
+        self.conn.commit()
+
+    def __len__(self):
+        rows = self.conn.execute('SELECT COUNT(*) FROM kv').fetchone()[0]
+        return rows if rows is not None else 0
+
+    def iterkeys(self):
+        c = self.conn.cursor()
+        for row in c.execute('SELECT key FROM kv'):
+            yield row[0]
+
+    def itervalues(self):
+        c = self.conn.cursor()
+        for row in c.execute('SELECT value FROM kv'):
+            yield row[0]
+
+    def iteritems(self):
+        c = self.conn.cursor()
+        for row in c.execute('SELECT key, value FROM kv'):
+            yield row[0], row[1]
+
+    def keys(self):
+        return list(self.iterkeys())
+
+    def values(self):
+        return list(self.itervalues())
+
+    def items(self):
+        return list(self.iteritems())
+
+    def __contains__(self, key):
+        key = str(key)
+        return self.conn.execute('SELECT 1 FROM kv WHERE key = ?', (key,)).fetchone() is not None
+
+    def __getitem__(self, key):
+        key = str(key)
+        item = self.conn.execute('SELECT value FROM kv WHERE key = ?', (key,)).fetchone()
+        if item is None:
+            return None
+        v = json.loads(item[0])
+        return v
+
+    def __setitem__(self, key, value):
+        key = str(key)
+        value = json.dumps(value)
+        self.conn.execute('REPLACE INTO kv (key, value) VALUES (?,?)', (key, value))
+
+    def __delitem__(self, key):
+        key = str(key)
+        if key in self:
+            self.conn.execute('DELETE FROM kv WHERE key = ?', (key,))
+
+    def __iter__(self):
+        return self.iterkeys()
+
+    def read_with_ex(self, key, ex):
+        res = self.__getitem__(key)
+        if res is None: return None
+        if not (isinstance(res, dict) and "time" in res.keys()): return None
+        dif = arrow.now() - arrow.get(res['time'])
+        if dif.seconds >= ex:
+            self.__delitem__(key)
+            return None
+        return res['v']
+
+    def write_with_ex(self, k, v):
+        self.__setitem__(k, {"v": v, "time": arrow.now().format()})

remote_run_everything/deploy/by_http.py

@@ -0,0 +1,82 @@
+import os.path
+
+from remote_run_everything.db.crud_sqlalchemy import Crud
+from remote_run_everything.deploy.record_mod import Up, Down
+from remote_run_everything.db.crude_duck import CrudeDuck
+from remote_run_everything.deploy.by_http_tool import ByHttpTool
+
+
+class ByHttp:
+    def __init__(self, host, local, remote, dbpath):
+        self.host = host
+        self.local = local
+        self.remote = remote
+        self.dbpath = dbpath
+        self.t = ByHttpTool()
+
+    def down(self, disallow_keys=None):
+        assert self.dbpath.endswith(".db"), "dbpath should be xxx.db"
+        c = Crud()
+        eg = c.sqlite_engine(self.dbpath)
+        c.create_table(eg, Down)
+        con = CrudeDuck().install_sql_ext(self.dbpath)
+        remote_files = self.t.all_remote_path(self.host, self.remote, disallow_keys)
+        mod = Down
+        add_l = []
+        for dic in remote_files:
+            path = dic['path']
+            t = dic['time']
+            sql = f"select * from down where path='{path}' and time='{t}' "
+            res = con.sql(sql).fetchone()
+            if res is not None: continue
+            print("down", dic)
+            con.execute(f"delete from down where path='{path}' ")
+            self.t.pull(self.host, path, self.local, self.remote)
+            add_l.append(dic)
+        con.commit()
+        c.insert_many(eg, mod, add_l)
+
+    def up(self, disallow_keys=None):
+        if os.name == "nt":
+            return self.up_win(disallow_keys)
+        assert self.dbpath.endswith(".db"), "dbpath should be xxx.db"
+        c = Crud()
+        eg = c.sqlite_engine(self.dbpath)
+        c.create_table(eg, Up)
+        con = CrudeDuck().install_sql_ext(self.dbpath)
+        mod = Up
+        loc_files = self.t.all_local_path(self.local, disallow_keys)
+        add_l = []
+        for dic in loc_files:
+            path = dic['path']
+            t = dic['time']
+            sql = f"select * from up where path='{path}' and time='{t}' and host='{self.host}' "
+            res = con.sql(sql).fetchone()
+            if res is not None: continue
+            print("up==", dic)
+            sql = f"delete from  up where path='{path}' and  host='{self.host}'"
+            con.execute(sql).commit()
+            self.t.push(self.host, path, self.local, self.remote)
+            dic['host'] = self.host
+            add_l.append(dic)
+        con.commit()
+        c.insert_many(eg, mod, add_l)
+
+    def up_win(self, disallow_keys=None):
+        from mongo_emb import PyMongoEmb
+        if not os.path.exists(self.dbpath):
+            os.makedirs(self.dbpath)
+        assert os.path.isdir(self.dbpath), "dbpath should be dir"
+        path = os.path.normpath(self.dbpath)
+        db = PyMongoEmb(path)
+        col = db['up']
+        loc_files = self.t.all_local_path(self.local, disallow_keys)
+        for dic in loc_files:
+            dic['host'] = self.host
+            # time path host
+            if col.find_one(dic) is not None:
+                continue
+            if os.path.normpath(os.path.dirname(dic['path'])) == path: continue
+            self.t.push(self.host, dic['path'], self.local, self.remote)
+            print("up==", dic)
+            col.update_one({"host": self.host, "path": dic['path']}, {"$set": {"time": dic['time']}}, upsert=True)

remote_run_everything/deploy/by_http_server.py

@@ -0,0 +1,56 @@
+import cherrypy
+from remote_run_everything.deploy.by_http_tool import ByHttpTool
+from remote_run_everything.tools.common import Common
+
+from cherrypy.process.plugins import Daemonizer
+
+
+def cherrypy_in_daemon(myapp, port, prefix):
+    cherrypy.config.update({
+        "server.socket_port": port,
+    })
+    Daemonizer(cherrypy.engine).subscribe()
+    cherrypy.tree.mount(myapp(), prefix)
+    cherrypy.engine.start()
+    cherrypy.engine.block()
+
+
+class ByHttpServer:
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_in()
+    def rb64(self):
+        try:
+            args = cherrypy.request.json
+            path = args['path']
+            data = Common().readb64(path)
+            return {"status": "ok", "data": data}
+        except Exception as e:
+            return {"status": "fail", "data": str(e)}
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_in()
+    def wb64(self):
+        try:
+            args = cherrypy.request.json
+            path = args['path']
+            b64 = args['b64']
+            Common().writeb64(path, b64)
+            return {"status": "ok", "data": path}
+        except Exception as e:
+            return {"status": "fail", "data": str(e)}
+
+    @cherrypy.expose
+    @cherrypy.tools.json_out()
+    @cherrypy.tools.json_in()
+    def iterdir(self):
+        try:
+            args = cherrypy.request.json
+            root = args['root']
+            disallow = args.get("disallow_keys", [])
+            data = ByHttpTool().all_local_path(root, disallow)
+            return {"status": "ok", "data": data}
+        except Exception as e:
+            return {"status": "fail", "data": str(e)}

remote_run_everything/deploy/by_http_tool.py

@@ -0,0 +1,64 @@
+from remote_run_everything.tools.common import Common
+
+import os, glob, arrow, requests
+
+
+class ByHttpTool:
+    def push(self, host, f, local, remote):
+        b64 = Common().readb64(f)
+        push_url = f"{host}/wb64"
+        path = self.loc2remote(f, local, remote)
+        pay = {"b64": b64, "path": path}
+        res = requests.post(push_url, json=pay).json()
+        return res
+
+    def pull(self, host, f, local, remote):
+        res = requests.post(f"{host}/rb64", json={"path": f}).json()
+        b64 = res['data']
+        path = self.remote2loc(f, local, remote)
+        Common().writeb64(path, b64)
+        return path
+
+    def all_remote_path(self, host, root, disallow_keys=None):
+        url = f"{host}/iterdir"
+        res = requests.post(url, json={"root": root}).json()
+        l = []
+        for i in res['data']:
+            path = i['path']
+            if path == "": continue
+            if not self.contain_disallow(path, disallow_keys):
+                t = arrow.get(i['time']).format()
+                l.append({"path": path, "time": t})
+        return l
+
+    def contain_disallow(self, path, disallow_keys):
+        if disallow_keys is None or len(disallow_keys) == 0:
+            return False
+        for k in disallow_keys:
+            if k in path:
+                return True
+        return False
+
+    def all_local_path(self, root, disallow_keys=None):
+        '''服务端提供的接口
+        '''
+        if not os.path.exists(root):
+            return []
+        files = glob.glob(f'{root}/**/*', recursive=True)
+        res = []
+        for path in files:
+            if os.path.isdir(path):
+                continue
+            if not self.contain_disallow(path, disallow_keys):
+                t = os.path.getmtime(path)
+                t = arrow.get(t).format()
+                res.append({"path": path, "time": t})
+        return res
+
+    def loc2remote(self, f, local, remote):
+        rela = os.path.relpath(f, local)
+        return f"{remote}/{rela}"
+
+    def remote2loc(self, f, local, remote):
+        rela = os.path.relpath(f, remote)
+        return f"{local}/{rela}"

remote_run_everything/deploy/record_mod.py

@@ -0,0 +1,26 @@
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+import datetime
+
+from sqlalchemy import Integer, DECIMAL, DateTime, Index, Integer, String, Column, Text
+import json, arrow, sys, os
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class Up(Base):
+    __tablename__ = 'up'
+    id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
+    host = Column(String(50), default="")
+    path = Column(String(50), default="")
+    time = Column(String(50), default="")
+
+
+class Down(Base):
+    __tablename__ = 'down'
+    id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
+    path = Column(String(50), default="")
+    time = Column(String(50), default="")
+
+