ct 0.10.8.114__py3-none-any.whl

cantools/scripts/pubsub/bots.py

@@ -0,0 +1,143 @@
+import datetime, json, os
+from cantools import config
+from cantools.util import log, write
+from cantools.web import fetch, send_mail
+from .actor import Actor
+try:
+	import psutil
+except ImportError as e:
+	pass # google crap engine (get it if you need it!)
+from six import with_metaclass
+
+class BotMeta(type):
+	def __new__(cls, name, bases, attrs):
+		bc = type.__new__(cls, name, bases, attrs)
+		if name != "Bot":
+			name != "Monitor" and log("Initializing Bot Class: %s"%(name,), important=True)
+			config.pubsub.bots.update(name.lower(), bc)
+		return bc
+
+class Bot(with_metaclass(BotMeta, Actor)):
+	num = 0
+	def __init__(self, server, channel, name=None):
+		Bot.num += 1
+		self.name = name or (self.__class__.__name__ + str(Bot.num))
+		self.server = server
+		self.channel = channel # often we only care about one channel
+		self.channels = set()
+		self._set_defaults()
+		log("Bot Spawned: '%s'"%(self.name,), 2)
+		channel.join(self)
+		self.server.bots[self.name] = self
+
+	def pub(self, message):
+		self.server.publish({
+			"message": message,
+			"channel": self.channel.name
+		}, self)
+
+	def write(self, obj): # receive message from channel
+		getattr(self, "on_%s"%(obj["action"],))(obj["data"])
+
+	def _default_handler(self, action):
+		def _h(*args):
+			log('Bot %s handling %s: "%s"'%(self.name, action, json.dumps(args)), 3)
+		return _h
+
+	def _set_defaults(self):
+		for action in ["channel", "publish", "subscribe", "unsubscribe", "pm", "error", "meta"]:
+			hname = "on_%s"%(action,)
+			if not hasattr(self, hname):
+				setattr(self, hname, self._default_handler(action))
+
+class Monitor(Bot):
+	def __init__(self, server, channel, name="monitor"): # only one monitor
+		import event # here for google crap engine
+		self.current = {}
+		self.alert = {}
+		Bot.__init__(self, server, channel, name)
+		event.timeout(config.admin.monitor.interval, self._tick)
+
+	def _datedir(self):
+		n = datetime.datetime.now()
+		lp = os.path.join("logs", "monitor")
+		yp = os.path.join(lp, str(n.year))
+		mp = os.path.join(yp, str(n.month))
+		dp = os.path.join(mp, str(n.day))
+		if not os.path.isdir(lp):
+			os.mkdir(lp)
+		if not os.path.isdir(yp):
+			os.mkdir(yp)
+		if not os.path.isdir(mp):
+			os.mkdir(mp)
+		if not os.path.isdir(dp):
+			os.mkdir(dp)
+		return os.path.join(dp, str(n.hour))
+
+	def log(self, data):
+		self.pub(data)
+		if config.admin.monitor.log:
+			write(data, self._datedir(), True, append=True, newline=True)
+
+	def _cpu(self):
+		c = self.current["cpu"] = psutil.cpu_percent()
+		if self.alert.get("cpu"):
+			if c < config.admin.monitor.thresholds.cpu:
+				del self.alert["cpu"]
+				log("CPU calmed down")
+				send_mail(config.admin.contacts, subject="High CPU", body="just ended")
+		else:
+			if c >= config.admin.monitor.thresholds.cpu:
+				self.alert["cpu"] = True
+				log("CPU just started going crazy")
+				send_mail(config.admin.contacts, subject="High CPU", body="just started")
+
+	def _tick(self):
+		self._cpu()
+		dioc = psutil.disk_io_counters()
+		nioc = psutil.net_io_counters()
+		dmon = fetch(config.admin.host, "/_report",
+			config.admin.port, True, protocol=config.admin.protocol)
+		data = {
+			"gc": dmon["gc"],
+			"cpu": self.current["cpu"],
+			"read": dioc.read_time,
+			"write": dioc.write_time,
+			"sent": nioc.bytes_sent,
+			"recv": nioc.bytes_recv,
+			"process_memory": dmon["mem"],
+			"threads": dmon["threads"],
+			"stack_frames": dmon["stack_frames"],
+			"virtual_memory": psutil.virtual_memory().percent,
+			"swap_memory": psutil.swap_memory().percent,
+			"connections": len(psutil.net_connections()),
+			"web_connections": dmon["web"]["connections"],
+			"admin_connections": dmon["admin"]["connections"],
+			"web_requests": dmon["web"]["requests"],
+			"admin_requests": dmon["admin"]["requests"],
+			"totals": {
+				"web": {
+					"connections": dmon["web"]["total_connections"],
+					"requests": dmon["web"]["total_requests"],
+					"rolls": dmon["web"]["rolls"]
+				},
+				"admin": {
+					"connections": dmon["admin"]["total_connections"],
+					"requests": dmon["admin"]["total_requests"],
+					"rolls": dmon["admin"]["rolls"]
+				}
+			},
+			"devices": {
+				"web": dmon["web"]["devices"],
+				"admin": dmon["admin"]["devices"]
+			},
+			"ips": {
+				"web": dmon["web"]["ips"],
+				"admin": dmon["admin"]["ips"]
+			}
+		}
+		if config.admin.monitor.proxy:
+			data["ips"]["proxy"] = fetch(config.admin.host, "/_report",
+				config.admin.monitor.proxy, True)["ips"]
+		self.log(data)
+		return True

cantools/scripts/pubsub/channel.py

@@ -0,0 +1,85 @@
+from cantools import config
+
+def diffmerge(orig, diff): # move this somewhere else....
+    for k, v in diff.items():
+        if type(v) == dict:
+            if k not in orig:
+                orig[k] = {}
+            diffmerge(orig[k], v)
+        else:
+            orig[k] = v
+
+class PubSubChannel(object):
+    def __init__(self, name, server):
+        self._log = server._log
+        self.server = server
+        self.name = name
+        self.users = set()
+        self.history = []
+        self.metadata = {}
+        self._log('NEW CHANNEL: "%s"'%(name,), 1, True)
+
+    def data(self):
+        return {
+            "name": self.name,
+            "users": [u.name for u in self.users]
+        }
+
+    def _broadcast(self, obj):
+        for user in self.users:
+            if config.pubsub.echo or user.name != obj["data"]["user"]:
+                user.write(obj)
+        for user in list(self.server.admins.values()):
+            user.write(obj)
+
+    def meta(self, subobj):
+        subobj["channel"] = self.name
+        self._broadcast({
+            "action": "meta",
+            "data": subobj
+        })
+
+    def chmeta(self, subobj):
+        subobj["channel"] = self.name # _usually_ unnecessary
+        diffmerge(self.metadata, subobj["meta"])
+#        self.metadata = subobj["meta"]
+        self._broadcast({
+            "action": "chmeta",
+            "data": subobj
+        })
+
+    def write(self, subobj):
+        subobj["channel"] = self.name
+        obj = {
+            "action": "publish",
+            "data": subobj
+        }
+        self._broadcast(obj)
+        self.history.append(subobj)
+        self.history = self.history[-config.pubsub.history:]
+
+    def leave(self, user):
+        if user in self.users:
+            self.users.remove(user)
+            user.leave(self)
+            self._broadcast({
+                "action": "unsubscribe",
+                "data": {
+                    "user": user.name,
+                    "channel": self.name
+                }
+            })
+
+    def join(self, user):
+        data = {
+            "user": user.name,
+            "channel": self.name
+        }
+        if config.pubsub.meta:
+            data["meta"] = user.meta
+        self._broadcast({
+            "action": "subscribe",
+            "data": data
+        })
+        self.users.add(user)
+        user.join(self)

cantools/scripts/pubsub/ps.py

@@ -0,0 +1,145 @@
+import os, sys
+from base64 import b64encode
+from dez.network.websocket import WebSocketDaemon
+from cantools import config
+from cantools.util import log, set_log
+from .user import PubSubUser
+from .channel import PubSubChannel
+
+class PubSub(WebSocketDaemon):
+    def __init__(self, *args, **kwargs):
+        if config.pubsub.log:
+            set_log(os.path.join("logs", config.pubsub.log))
+        kwargs["b64"] = config.pubsub.b64
+        kwargs["isJSON"] = True
+        kwargs["report_cb"] = self._log
+        kwargs["cb"] = self.connect
+        kwargs["certfile"] = config.ssl.pubsubcert or config.ssl.certfile
+        kwargs["keyfile"] = config.ssl.pubsubkey or config.ssl.keyfile
+        kwargs["cacerts"] = config.ssl.pubsubcacerts or config.ssl.cacerts
+        if "silent" in kwargs:
+            self.silent = kwargs["silent"]
+            del kwargs["silent"]
+        else:
+            self.silent = False
+        WebSocketDaemon.__init__(self, *args, **kwargs)
+        self.bots = {}
+        self.users = {}
+        self.admins = {}
+        self.channels = {}
+        self.loadBots()
+        config.admin.update("pw", config.cache("admin password? "))
+        self._log("Initialized PubSub Server @ %s:%s"%(self.hostname, self.port), important=True)
+
+    def loadBots(self):
+        self._log("Loading Bots: %s"%(config.pubsub.botnames,))
+        sys.path.insert(0, "bots") # for dynamically loading bot modules
+        for bname in config.pubsub.botnames:
+            self._log("Importing Bot: %s"%(bname,), 2)
+            __import__(bname) # config modified in pubsub.bots.BotMeta.__new__()
+
+    def newUser(self, u):
+        if not u.name: # on dc?
+            self._log("user disconnected without registering")
+            u.conn.close()
+        elif u.name.startswith("__admin__") and u.name.endswith(b64encode(config.admin.pw.encode()).decode()):
+            self.admins[u.name] = u
+            self.snapshot(u)
+        else:
+            self.users[u.name] = u
+
+    def client(self, name):
+        return self.users.get(name) or self.bots.get(name) or self.admins.get(name)
+
+    def snapshot(self, admin):
+        admin.write({
+            "action": "snapshot",
+            "data": {
+                "bots": [b.data() for b in list(self.bots.values())],
+                "users": [u.data() for u in list(self.users.values())],
+                "admins": [a.data() for a in list(self.admins.values())],
+                "channels": [c.data() for c in list(self.channels.values())]
+            }
+        })
+
+    def pm(self, data, user):
+        recipient = self.client(data["user"])
+        if not recipient:
+            return user._error("no such user!")
+        recipient.write({
+            "action": "pm",
+            "data": {
+                "user": user.name,
+                "message": data["message"]
+            }
+        })
+
+    def subscribe(self, channel, user):
+        self._check_channel(channel)
+        chan = self.channels[channel]
+        chan.join(user)
+        self._log('SUBSCRIBE: "%s" -> "%s"'%(user.name, channel), 2)
+        data = {
+            "channel": channel,
+            "presence": [u.name for u in chan.users],
+            "history": chan.history
+        }
+        if config.pubsub.meta:
+            data["meta"] = {
+                "channel": chan.metadata,
+                "users": [u.meta for u in chan.users]
+            }
+        user.write({
+            "action": "channel",
+            "data": data
+        })
+
+    def unsubscribe(self, channel, user):
+        if self._check_channel(channel, True) and user in self.channels[channel].users:
+            self.channels[channel].leave(user)
+            self._log('UNSUBSCRIBE: "%s" -> "%s"'%(user.name, channel), 2)
+        else:
+            self._log('FAILED UNSUBSCRIBE: "%s" -> "%s"'%(user.name, channel), 2)
+
+    def meta(self, data, user):
+        channel = data["channel"]
+        self._check_channel(channel)
+        user.meta = data["meta"]
+        self.channels[channel].meta({
+            "meta": user.meta,
+            "user": user.name
+        })
+
+    def chmeta(self, data, user):
+        channel = data["channel"]
+        self._check_channel(channel)
+        self.channels[channel].chmeta(data)
+
+    def publish(self, data, user):
+        channel = data["channel"]
+        self._check_channel(channel)
+        self.channels[channel].write({
+            "message": data["message"],
+            "user": user.name
+        })
+
+    def _new_channel(self, channel):
+        self.channels[channel] = PubSubChannel(channel, self)
+        # check for bots...
+        botname = channel.split("_")[0]
+        if botname in config.pubsub.bots:
+            self._log("Generating Bot '%s' for channel '%s'"%(botname, channel), 2)
+            config.pubsub.bots[botname](self, self.channels[channel])
+
+    def _check_channel(self, channel, justBool=False):
+        condition = channel in self.channels
+        if not condition and not justBool:
+            self._new_channel(channel)
+        return condition
+
+    def _log(self, data, level=0, important=False):
+        if not self.silent:
+            log(data, level=level, important=important)
+
+    def connect(self, conn):
+        PubSubUser(conn, self, self._log)

cantools/scripts/pubsub/user.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+from .actor import Actor
+
+class PubSubUser(Actor):
+    id = 0
+    def __init__(self, conn, server, logger):
+        PubSubUser.id += 1
+        self.id = PubSubUser.id
+        self.conn = conn
+        self.server = server
+        self._log = logger
+        self.channels = set()
+        self.conn.set_cb(self._register)
+        self._log('NEW CONNECTION (%s)'%(self.id,), 1, True)
+
+    def write(self, data):
+        data["data"]["datetime"] = str(datetime.utcnow())
+        self.conn.write(data)
+
+    def _read(self, obj):
+        if obj["action"] == "close":
+            return self.conn.close()
+        getattr(self.server, obj["action"])(obj["data"], self)
+
+    def _close(self):
+        if self.name in self.server.users:
+            del self.server.users[self.name]
+        elif self.name in self.server.admins:
+            del self.server.admins[self.name]
+        for channel in list(self.channels):
+            channel.leave(self)
+
+    def _error(self, message):
+        self.write({
+            "action": "error",
+            "data": {
+                "message": message
+            }
+        })
+
+    def _register(self, obj):
+        name = obj.get("data")
+        if not name:
+            self._log("no name provided! assigning index (%s)."%(self.id,))
+            name = str(self.id)
+        self._log('REGISTER: "%s"'%(name,), 1, True)
+        self.name = name
+        self.meta = obj.get("meta")
+        self.server.newUser(self)
+        self.conn.set_cb(self._read)
+        self.conn.set_close_cb(self._close)

cantools/scripts/start.py

@@ -0,0 +1,53 @@
+"""
+### Usage: ctstart [--web_backend=BACKEND] [--port=PORT] [--datastore=DS_PATH]
+
+### Options:
+    -h, --help            show this help message and exit
+    -w WEB_BACKEND, --web_backend=WEB_BACKEND
+                          web backend. options: dez, gae. (default: dez)
+    -p PORT, --port=PORT  select your port (default=8080)
+    -a ADMIN_PORT, --admin_port=ADMIN_PORT
+                          select your port (default=8002)
+    -d DATASTORE, --datastore=DATASTORE
+                          select your datastore file (default=sqlite:///data.db)
+    -o, --overwrite_password
+                          overwrite admin password (default=False)
+"""
+
+from optparse import OptionParser
+from cantools import config
+from cantools.util import error
+
+def go():
+	parser = OptionParser("ctstart [--web_backend=BACKEND] [--port=PORT] [--datastore=DS_PATH]")
+	parser.add_option("-w", "--web_backend", dest="web_backend", default=config.web.server,
+		help="web backend. options: dez, gae. (default: %s)"%(config.web.server,))
+	parser.add_option("-p", "--port", dest="port", default=config.web.port,
+		help="select your port (default=%s)"%(config.web.port,))
+	parser.add_option("-a", "--admin_port", dest="admin_port", default=config.admin.port,
+		help="select your port (default=%s)"%(config.admin.port,))
+	parser.add_option("-d", "--datastore", dest="datastore", default=config.db.main,
+		help="select your datastore file (default=%s)"%(config.db.main,))
+	parser.add_option("-o", "--overwrite_password", action="store_true", dest="overwrite_password",
+		default=False, help="overwrite admin password (default=False)")
+	options, args = parser.parse_args()
+
+	config.web.update("port", int(options.port))
+	config.admin.update("port", int(options.admin_port))
+	if options.overwrite_password:
+		config.update("newpass", True)
+
+	if options.web_backend == "gae":
+		import subprocess
+		cmd = 'dev_appserver.py . --host=%s --port=%s --admin_port=%s --datastore_path=%s'%(config.web.host,
+			options.port, options.admin_port, options.datastore)
+		print(cmd)
+		subprocess.call(cmd, shell=True)
+	elif options.web_backend == "dez":
+		from cantools.web import run_dez_webserver
+		run_dez_webserver()
+	else:
+		error("invalid web_backend: %s"%(options.web_backend,))
+
+if __name__ == "__main__":
+	go()

cantools/scripts/util.py

@@ -0,0 +1,24 @@
+"""
+### Usage: ctutil [module] [function] [..parameters]
+
+### Options:
+    -h, --help            show this help message and exit
+"""
+
+from optparse import OptionParser
+from cantools import util
+log = util.log
+error = util.error
+
+def run():
+	parser = OptionParser("ctutil [module] [function] [..parameters]")
+	args = parser.parse_args()[1]
+	args or error("what module?")
+	module = args.pop(0)
+	args or error("what function?")
+	function = args.pop(0)
+	#log("calling %s.%s(%s)"%(module, function, ", ".join(args)))
+	getattr(getattr(util, module), function)(*args)
+
+if __name__ == "__main__":
+	run()

cantools/util/__init__.py

@@ -0,0 +1,78 @@
+import os, sys
+from fyg.util import read, write, writejson, confirm, batch, indir, rm # backwards compat
+from fyg.util.reporting import set_log, close_log, log, set_error, error, start_timer, end_timer
+from .system import cp, sym, mkdir, sed, py, pymod, cmd, output
+from .data import getxls, gettsv, getcsv, getcsv_from_data, flatten, arr2csv, token
+from .media import transcode, segment, hlsify, shouldMoveMoov, crop, resizep2, thumb, dlp, repitch
+from .admin import certs, screener
+from .apper import android
+from .ai import tox, vox, tellme
+
+def init_basic():
+    os.path.isdir("emails") and sys.path.append("emails")
+
+def init_rel():
+    import rel
+    from rel.util import loudListen
+    from cantools import config
+    rcfg = config.rel
+    if rcfg.sleep:
+        rel.set_sleep(rcfg.sleep)
+    if rcfg.turbo:
+        rel.set_turbo(rcfg.turbo)
+    rel.set_verbose(rcfg.verbose)
+    loudListen(rcfg.loudlisten)
+
+def init_gae():
+    try:
+        from google import appengine
+    except: # running outside of dev_appserver
+        gae_p = None
+        for p in os.environ["PATH"].split(":"):
+            if p.endswith("google_appengine"):
+                gae_p = p
+                break
+        if not gae_p:
+            from cantools import config
+            gae_p = config.cache("\n".join([
+                "can't find google - please enter the path to google_appengine",
+                " - if you DON'T have a copy, get one here:",
+                "   https://cloud.google.com/appengine/downloads#Google_App_Engine_SDK_for_Python",
+                " - if you DO have a copy, enter the path to it below",
+                "so what's the path? "]), password=False)
+            if not gae_p:
+                error("can't find google_appengine in path! please add and retry.")
+        sys.path.append(gae_p)
+        if 'google' in sys.modules:
+            del sys.modules['google']
+        import dev_appserver
+        dev_appserver.fix_sys_path()
+        sys.path.insert(0, ".")
+
+# accesses ct gae app running locally or elsewhere (other than app engine production, aka gcloud)
+def init_ndb():
+    from cantools import config
+    from google.appengine.ext.remote_api import remote_api_stub
+    remote_api_stub.ConfigureRemoteDatastore("", "/remote_api",
+        lambda : ("user@email.com", "password"),
+        servername="%s:%s"%(config.web.host, config.web.port))
+
+# these are essentially not recommended. google's
+# rules for operating outside of their native environment
+# are crappy and generally get worse with time. these
+# _sort of_ work, but easily get messed up. it's whatever.
+def init_ndb_cloud(url=None):
+    from cantools import config
+    from google.appengine.ext.remote_api import remote_api_stub
+    if not url:
+        app_id = read("app.yaml", True)[0].split(": ")[1].strip()
+        url = "%s.appspot.com"%(app_id,)
+    remote_api_stub.ConfigureRemoteApiForOAuth(url, "/_ah/remote_api")
+
+def init_ndb_depped(datastore_file="/dev/null"):
+    from google.appengine.api import apiproxy_stub_map, datastore_file_stub
+    app_id = read("app.yaml", True)[0].split(": ")[1].strip()
+    os.environ['APPLICATION_ID'] = app_id
+    apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() 
+    stub = datastore_file_stub.DatastoreFileStub(app_id, datastore_file, '/')
+    apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', stub)