remote-run-everything 1.9__tar.gz → 2.0.2__tar.gz

remote_run_everything-2.0.2/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

remote_run_everything-2.0.2/PKG-INFO

@@ -0,0 +1,109 @@
+Metadata-Version: 2.4
+Name: remote_run_everything
+Version: 2.0.2
+Summary: Deploy Tools
+Author-email: Wang Qi <wangmarkqi@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/wangmarkqi/remote_run_everything
+Project-URL: Issues, https://github.com/wangmarkqi/remote_run_everything/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# remote_run_everthing 各类实用代码集合封装
+
+## 安装
+```shell
+ pip install -U --index-url https://test.pypi.org/simple/ remote_run_everything
+```
+
+## 运维功能
+```python
+# 服务端代码
+from remote_run_everything import cherrypy_in_daemon,ByHttpServer
+cherrypy_in_daemon(ByHttpServer,8888,"/deploy")
+
+# 上推代码
+from remote_run_everything import ByHttp 
+def test_up():
+    host = "http://x.x.x.x:8888/deploy"
+    local = "D://project/demand/shop"
+    remote = "/data/mypy/shop"
+    db = "D://wq/temp/shop.db"
+    bh = ByHttp(host, local, remote, db)
+    bh.up(['node_modules', ".pyc", ".idea"])
+
+# 下拉代码
+def test_down():
+    host = "http://x.x.x.x:8888/deploy"
+    local = "D://project/demand/shop"
+    remote = "/data/mypy/shop"
+    db = "D://wq/temp/shop.db"
+    bh = ByHttp(host, local, remote, db)
+    bh.down(['node_modules', ".pyc", ".idea"])
+```
+
+## 缓存功能
+```python
+from remote_run_everything import cache_by_1starg,cache_by_name,cache_by_rkey 
+@cache_by_name("asdf", 1)
+def test1():
+    print("运行了函数!!!!!!!!!!!!!!!!")
+    return {"a": "adaf"}
+@cache_by_1starg("asdf", 1)
+def test2(arg1):
+    print("运行了函数!!!!!!!!!!!!!!!!")
+    return {"a": "adaf"}
+@cache_by_rkey(1)
+def test2(rkey="xx"):
+    print("运行了函数!!!!!!!!!!!!!!!!")
+    return {"a": "adaf"}
+```
+## KV数据库
+```python
+from remote_run_everything import KvStore
+kv = KvStore('test.db') 
+print(len(kv))         
+kv['hello1'] = 'you1'
+del kv['hello1']
+print(len(kv))        
+print('hello1' in kv) 
+kv['hello3'] = 'newvalue'
+print(kv.keys())        
+print(kv.values())     
+print(kv.items())     
+for k in kv:
+    print(k, kv[k])
+```
+## Mongodb like 数据库
+```python
+from remote_run_everything import Nosql,NosqlPg,NosqlMysql
+db = Nosql()
+t = "test"
+col = db['test']
+dic = {"a": 2, "b": 456, 'c': "adf", "d": "2020-02-02"}
+col.insert_one(dic)
+dic = {"a": 56, "b": 456, 'c': "adf", "d": "2020-07-02"}
+col.insert_one(dic)
+q = {"a": 2,"b":{"$gt":1}}
+print(col.find(q))
+db.drop_db()
+```
+## 进程管理
+```python
+class ProcessManage:
+    # nosql is instance of Nosql or Nosqlmysql or Nqsqlpg
+    def __init__(self, nosql):
+        self.db = nosql
+        self.col = self.db['pid']
+
+    # 只需要在程序中引入这个函数,启动后会把pid存入数据库,然后有了pid,结合psutil,什么都有了
+    def save_pid(self, name, cmd):
+        dic = {"name": name, "cmd": cmd, "pid": os.getpid()}
+        print("save pid", dic)
+        self.col.insert_one(dic)
+
+```

remote_run_everything-2.0.2/README.md

@@ -0,0 +1,94 @@
+# remote_run_everthing 各类实用代码集合封装
+
+## 安装
+```shell
+ pip install -U --index-url https://test.pypi.org/simple/ remote_run_everything
+```
+
+## 运维功能
+```python
+# 服务端代码
+from remote_run_everything import cherrypy_in_daemon,ByHttpServer
+cherrypy_in_daemon(ByHttpServer,8888,"/deploy")
+
+# 上推代码
+from remote_run_everything import ByHttp 
+def test_up():
+    host = "http://x.x.x.x:8888/deploy"
+    local = "D://project/demand/shop"
+    remote = "/data/mypy/shop"
+    db = "D://wq/temp/shop.db"
+    bh = ByHttp(host, local, remote, db)
+    bh.up(['node_modules', ".pyc", ".idea"])
+
+# 下拉代码
+def test_down():
+    host = "http://x.x.x.x:8888/deploy"
+    local = "D://project/demand/shop"
+    remote = "/data/mypy/shop"
+    db = "D://wq/temp/shop.db"
+    bh = ByHttp(host, local, remote, db)
+    bh.down(['node_modules', ".pyc", ".idea"])
+```
+
+## 缓存功能
+```python
+from remote_run_everything import cache_by_1starg,cache_by_name,cache_by_rkey 
+@cache_by_name("asdf", 1)
+def test1():
+    print("运行了函数!!!!!!!!!!!!!!!!")
+    return {"a": "adaf"}
+@cache_by_1starg("asdf", 1)
+def test2(arg1):
+    print("运行了函数!!!!!!!!!!!!!!!!")
+    return {"a": "adaf"}
+@cache_by_rkey(1)
+def test2(rkey="xx"):
+    print("运行了函数!!!!!!!!!!!!!!!!")
+    return {"a": "adaf"}
+```
+## KV数据库
+```python
+from remote_run_everything import KvStore
+kv = KvStore('test.db') 
+print(len(kv))         
+kv['hello1'] = 'you1'
+del kv['hello1']
+print(len(kv))        
+print('hello1' in kv) 
+kv['hello3'] = 'newvalue'
+print(kv.keys())        
+print(kv.values())     
+print(kv.items())     
+for k in kv:
+    print(k, kv[k])
+```
+## Mongodb like 数据库
+```python
+from remote_run_everything import Nosql,NosqlPg,NosqlMysql
+db = Nosql()
+t = "test"
+col = db['test']
+dic = {"a": 2, "b": 456, 'c': "adf", "d": "2020-02-02"}
+col.insert_one(dic)
+dic = {"a": 56, "b": 456, 'c': "adf", "d": "2020-07-02"}
+col.insert_one(dic)
+q = {"a": 2,"b":{"$gt":1}}
+print(col.find(q))
+db.drop_db()
+```
+## 进程管理
+```python
+class ProcessManage:
+    # nosql is instance of Nosql or Nosqlmysql or Nqsqlpg
+    def __init__(self, nosql):
+        self.db = nosql
+        self.col = self.db['pid']
+
+    # 只需要在程序中引入这个函数,启动后会把pid存入数据库,然后有了pid,结合psutil,什么都有了
+    def save_pid(self, name, cmd):
+        dic = {"name": name, "cmd": cmd, "pid": os.getpid()}
+        print("save pid", dic)
+        self.col.insert_one(dic)
+
+```

remote_run_everything-2.0.2/pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "remote_run_everything"
+version = "2.0.2"
+authors = [
+  { name="Wang Qi", email="wangmarkqi@gmail.com" },
+]
+description = "Deploy Tools"
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "MIT"
+license-files = ["LICEN[CS]E*"]
+
+[project.urls]
+Homepage = 'https://github.com/wangmarkqi/remote_run_everything'
+Issues = "https://github.com/wangmarkqi/remote_run_everything/issues"
+

remote_run_everything-2.0.2/remote_run_everything/__init__.py

@@ -0,0 +1,16 @@
+from remote_run_everything.deploy.by_http import ByHttp
+from remote_run_everything.deploy.by_http_server import ByHttpServer, cherrypy_in_daemon
+
+from remote_run_everything.db.crude_duck import CrudeDuck
+from remote_run_everything.db.crud_sqlalchemy import Crud
+from remote_run_everything.db.kv_store import KvStore
+from remote_run_everything.db.backup import BackUp
+
+from remote_run_everything.tools.common import Common
+from remote_run_everything.tools.sqlacodegen_go_struct import Sql2go
+from remote_run_everything.tools.decorators import cache_by_1starg, cache_by_name, cache_by_rkey,cache_by_nth_arg
+
+from remote_run_everything.nosql.no_sql import Nosql
+from remote_run_everything.nosql.no_sql_pg import NosqlPg
+from remote_run_everything.nosql.no_sql_mysql import NosqlMysql
+from remote_run_everything.vsconf.core import VsConf

remote_run_everything-2.0.2/remote_run_everything/db/backup.py

@@ -0,0 +1,38 @@
+from bson import BSON, decode_all
+import pymongo
+import os, sys
+
+
+class BackUp:
+    def __init__(self,  root):
+        self.root = root
+        os.makedirs(self.root, exist_ok=True)
+
+    def mongo_dump(self,cli,dbs):
+        for d in dbs:
+            db = cli[d]
+            dir = os.path.join(self.root, d)
+            os.makedirs(dir, exist_ok=True)
+            cols = db.list_collection_names()
+            for col in cols:
+                sr = db[col]
+                # Dump.
+                with open(f'{dir}/{col}.bson', 'wb+') as f:
+                    for doc in sr.find():
+                        f.write(BSON.encode(doc))
+
+    def mongo_restore(self,cli,dbs):
+        for d in dbs:
+            db = cli[d]
+            dir = os.path.join(self.root, d)
+            files = os.listdir(dir)
+            cols = [i.split('.')[0] for i in files]
+            print(cols)
+            for name in cols:
+                file = f"{dir}/{name}.bson"
+                col = db[name]
+                with open(file, 'rb') as f:
+                    data = decode_all(f.read())
+                    col.insert_many(data)
+
+

remote_run_everything-2.0.2/remote_run_everything/db/crud_sqlalchemy.py

@@ -0,0 +1,97 @@
+import os, pymysql
+from urllib.parse import quote_plus
+
+from sqlalchemy import create_engine, select, update, and_, insert, delete
+
+
+class Crud:
+    def sqlite_engine(self, dbpath):
+        dir = os.path.dirname(dbpath)
+        os.makedirs(dir, exist_ok=True)
+        url = f"sqlite:///{dbpath}"
+        engine = create_engine(url, future=True, connect_args={'timeout': 30, 'check_same_thread': False})
+        return engine
+
+    def pg_url(self, user, pwd, host, port, db):
+        f = lambda x: quote_plus(x)
+        return f"postgresql://{f(user)}:{f(pwd)}@{f(host)}:{port}/{db}"
+
+    def pg_engine(self, user, pwd, host, port, db):
+        return create_engine(self.pg_url(user, pwd, host, port, db), pool_recycle=3600, pool_size=80, max_overflow=-1,
+                             echo=False,
+                             future=True)
+
+    def mysql_url(self, user, pwd, host, port, db):
+        f = lambda x: quote_plus(x)
+        return f"mysql+pymysql://{f(user)}:{f(pwd)}@{f(host)}:{port}/{db}"
+
+    def mysql_engine(self, user, pwd, host, port, db):
+        # print(f"sqlacodegen {url} >> {db}.py")
+        return create_engine(self.mysql_url(user, pwd, host, port, db), pool_recycle=3600, pool_size=80,
+                             max_overflow=-1, echo=False,
+                             future=True)
+
+    def drop_table(self, engine, mod):
+        mod.__table__.drop(engine)
+
+    def create_table(self, engine, mod):
+        mod.__table__.create(engine, checkfirst=True)
+
+    def exist_id(self, engine, mod, cond):
+        with engine.connect() as conn:
+            stmt = select(mod).where(cond).limit(1)
+            id = conn.scalar(stmt)
+            if id is not None:
+                return id
+            return None
+
+    def table_columns(self, mod):
+        if "__annotations__" in mod.__dict__:
+            cols = mod.__dict__['__annotations__'].keys()
+        else:
+            cols = [i for i in mod.__dict__.keys() if not i.startswith("__")]
+        return cols
+
+    def insert_many(self, engine, mod, l):
+        if len(l)==0:return
+        cols = self.table_columns(mod)
+        with engine.connect() as conn:
+            for dic in l:
+                dic = {k: v for k, v in dic.items() if k in cols}
+                stmt = insert(mod).values(dic)
+                conn.execute(stmt)
+            conn.commit()
+
+    def insert_one(self, engine, mod, dic):
+        cols = self.table_columns(mod)
+        dic = {k: v for k, v in dic.items() if k in cols}
+        with engine.connect() as conn:
+            stmt = insert(mod).values(dic)
+            conn.execute(stmt)
+            conn.commit()
+
+    def update_by_id(self, engine, mod, id, dic):
+        cols = self.table_columns(mod)
+        dic = {k: v for k, v in dic.items() if k in cols}
+        with engine.connect() as conn:
+            stmt = update(mod).where(mod.id == id).values(dic)
+            conn.execute(stmt)
+            conn.commit()
+
+    def upsert(self, engine, mod, cond, dic):
+        id = self.exist_id(engine, mod, cond)
+        if id is not None:
+            self.update_by_id(engine, mod, id, dic)
+            return
+        self.insert_one(engine, mod, dic)
+
+    def delete_by_id(self, engine, mod, id):
+        with engine.connect() as conn:
+            stmt = delete(mod).where(mod.id == id)
+            conn.execute(stmt)
+            conn.commit()
+
+    def delete(self, engine, mod, cond):
+        id = self.exist_id(engine, mod, cond)
+        if id is not None:
+            self.delete_by_id(engine, mod, id)

remote_run_everything-2.0.2/remote_run_everything/db/crude_duck.py

@@ -0,0 +1,122 @@
+import duckdb, os, arrow
+
+
+class CrudeDuck:
+
+    def install_sql_ext(self, dbpath):
+        dir = os.path.dirname(dbpath)
+        os.makedirs(dir, exist_ok=True)
+        sql = f"ATTACH '{dbpath}' AS db (TYPE sqlite,journal_mode wal);use db;"
+        con = duckdb.connect()
+        con.install_extension("sqlite")
+        con.load_extension("sqlite")
+        con.sql(sql)
+        return con
+
+    def install_pg_ext(self, user, pwd, host, port, dbname):
+        sql = f'''ATTACH 'dbname={dbname} user={user} 
+            host={host} port={port} connect_timeout=10 password={pwd}'
+             AS {dbname} (TYPE postgres);use {dbname};'''
+        con = duckdb.connect()
+        con.install_extension("postgres")
+        con.load_extension("postgres")
+        con.sql(sql)
+        return con
+
+    def install_mysql_ext(self, user, pwd, host, port, dbname):
+        con = duckdb.connect()
+        con.install_extension("mysql")
+        con.load_extension("mysql")
+        sql = f"ATTACH 'host={host} user={user} password={pwd} port={port} database={dbname}' AS msqldb (TYPE MYSQL);"
+        con.sql(sql)
+        con.sql(f"USE msqldb;")
+        return con
+
+    def scheme(self, con, db, table, dbtype):
+        # for mysql db== dbname ; for pg db==public for sqlite db=main
+        if dbtype == "mysql":
+            db = db
+        elif dbtype == "sqlite3":
+            db = "main"
+        elif dbtype == "pg":
+            db = "public"
+        else:
+            db = db
+        sql = f'''  SELECT column_name, data_type FROM information_schema.columns
+    WHERE table_schema = '{db}' AND table_name   = '{table}';
+            '''
+        scheme = {i[0]: i[1] for i in con.sql(sql).fetchall()}
+        return scheme
+
+    def max_id(self, con, table):
+        sql = f'select max(id) from {table}'
+        a = con.sql(sql).fetchone()
+        if a is None or a[0] is None: return 0
+        return a[0] + 1
+
+    def sql_from_ty(self, ty, v):
+        if v is None: return None
+        ty = ty.upper()
+        if ty in ['BIGINT', "TINYINT", "INTEGER", "BOOLEAN"]:
+            return str(int(v))
+        if ty in ["VARCHAR"]:
+            return f"'{str(v)}'"
+        if "TIMESTAMP" in ty:
+            return f"'{arrow.get(v).format()[:19]}'"
+        return None
+
+    def dic2sql(self, data, scheme):
+        dic = {}
+        for k, v in data.items():
+            if k not in scheme.keys(): continue
+            tyv = self.sql_from_ty(scheme[k], v)
+            if tyv is not None:            dic[k] = tyv
+        if len(dic) == 0:
+            return "", ""
+        cols = ", ".join(dic.keys())
+        values = ", ".join(dic.values())
+        return f"({cols})", f"({values})"
+
+    def list2sql(self, l, scheme):
+        cols = ""
+        values = ""
+        for data in l:
+            dic = {}
+            for k, v in data.items():
+                if k not in scheme.keys(): continue
+                tyv = self.sql_from_ty(scheme[k], v)
+                if tyv is not None:   dic[k] = tyv
+            if len(dic) == 0:
+                continue
+            cols = ", ".join(dic.keys())
+            s = ", ".join(dic.values())
+            values = values + f"({s}),"
+        return f"({cols})", values
+
+    def insert_many(self, con, db, table, data, dbtype=""):
+        sche = self.scheme(con, db, table, dbtype)
+        cols, values = self.list2sql(data, sche)
+        if values == "": return
+        sql = f'insert into {table} {cols} values {values}'
+        con.execute(sql)
+
+    def insert_one(self, con, db, table, data, dbtype=""):
+        sche = self.scheme(con, db, table, dbtype)
+        cols, values = self.dic2sql(data, sche)
+        if values == "": return
+        sql = f'insert into {table} {cols} values {values}'
+        con.execute(sql)
+
+    def drop_table(self, con, table):
+        sql = f"drop table if exists {table}"
+        con.execute(sql)
+
+    def delete_by_ids(self, con, table, ids):
+        ids = [str(i) for i in ids]
+        ids = ','.join(ids)
+        sql = f"delete from  {table} where id in ({ids})"
+        con.execute(sql)
+
+    def delete_by_id(self, con, table, id):
+        sql = f"delete from  {table} where id = {id}"
+        con.execute(sql)

remote_run_everything-2.0.2/remote_run_everything/db/kv_store.py

@@ -0,0 +1,109 @@
+import sqlite3, os, json, arrow
+
+'''
+kv = KeyValueStore('test.db')  # uses SQLite
+
+print(len(kv))                 # 0 item
+kv['hello1'] = 'you1'
+
+del kv['hello1']
+print(len(kv))                 # 2 items remaining
+print('hello1' in kv)          # False, it has just been deleted!
+
+kv['hello3'] = 'newvalue'      # redefine an already present key/value
+
+print(kv.keys())               # ['hello2', 'hello3']
+print(kv.values())             # ['you2', 'newvalue']
+print(kv.items())              # [('hello2', 'you2'), ('hello3', 'newvalue')]
+
+for k in kv:
+    print(k, kv[k])
+
+kv.close()  
+'''
+
+
+class KvStore(dict):
+    def __init__(self, filename=None):
+        self.db_path = self.default_db_path(filename)
+        self.conn = sqlite3.connect(self.db_path, isolation_level=None)
+        self.conn.execute('pragma journal_mode=wal')
+        self.conn.execute("CREATE TABLE IF NOT EXISTS kv (key text unique, value text)")
+
+    def default_db_path(self, db_path):
+        if db_path is None:
+            db_path = "D://wq/temp/decor.db" if os.name == 'nt' else "/data/temp/decor.db"
+        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        return db_path
+
+    def close(self):
+        self.conn.close()
+
+    def commit(self):
+        self.conn.commit()
+
+    def __len__(self):
+        rows = self.conn.execute('SELECT COUNT(*) FROM kv').fetchone()[0]
+        return rows if rows is not None else 0
+
+    def iterkeys(self):
+        c = self.conn.cursor()
+        for row in c.execute('SELECT key FROM kv'):
+            yield row[0]
+
+    def itervalues(self):
+        c = self.conn.cursor()
+        for row in c.execute('SELECT value FROM kv'):
+            yield row[0]
+
+    def iteritems(self):
+        c = self.conn.cursor()
+        for row in c.execute('SELECT key, value FROM kv'):
+            yield row[0], row[1]
+
+    def keys(self):
+        return list(self.iterkeys())
+
+    def values(self):
+        return list(self.itervalues())
+
+    def items(self):
+        return list(self.iteritems())
+
+    def __contains__(self, key):
+        key = str(key)
+        return self.conn.execute('SELECT 1 FROM kv WHERE key = ?', (key,)).fetchone() is not None
+
+    def __getitem__(self, key):
+        key = str(key)
+        item = self.conn.execute('SELECT value FROM kv WHERE key = ?', (key,)).fetchone()
+        if item is None:
+            return None
+        v = json.loads(item[0])
+        return v
+
+    def __setitem__(self, key, value):
+        key = str(key)
+        value = json.dumps(value)
+        self.conn.execute('REPLACE INTO kv (key, value) VALUES (?,?)', (key, value))
+
+    def __delitem__(self, key):
+        key = str(key)
+        if key in self:
+            self.conn.execute('DELETE FROM kv WHERE key = ?', (key,))
+
+    def __iter__(self):
+        return self.iterkeys()
+
+    def read_with_ex(self, key, ex):
+        res = self.__getitem__(key)
+        if res is None: return None
+        if not (isinstance(res, dict) and "time" in res.keys()): return None
+        dif = arrow.now() - arrow.get(res['time'])
+        if dif.seconds >= ex:
+            self.__delitem__(key)
+            return None
+        return res['v']
+
+    def write_with_ex(self, k, v):
+        self.__setitem__(k, {"v": v, "time": arrow.now().format()})