ct 0.10.8.114__py3-none-any.whl

cantools/web/dez_server/controller.py

@@ -0,0 +1,129 @@
+import os, sys, platform
+from dez.network import SocketController, daemon_wrapper
+from dez.http.server.shield import Shield
+from dez.logging import get_logger_getter
+from cantools import config
+from .daemons import Web, Admin
+from .response import Response
+from .cron import Cron
+from cantools import config, __version__
+from ..util import *
+from ...util import read, write, log as syslog
+
+logger_getter = get_logger_getter("httpd", syslog, config.log.allow)
+CTR = None
+
+class DController(SocketController):
+	def __init__(self, *args, **kwargs):
+		self.logger = logger_getter("Controller")
+		SocketController.__init__(self, *args, **kwargs)
+		self.handlers = {}
+		self.modules = {}
+		self.blcount = 0
+		self.blchunk = getattr(config.web.shield, "chunk", 5)
+		self.logger.info("cantools: %s"%(__version__,))
+		self.logger.info("Python: %s"%(sys.version.split(' ')[0],))
+		self.logger.info("System: " + " > ".join([part for part in platform.uname() if part]))
+
+	def _respond(self, resp, *args, **kwargs):
+		if resp == "nothread": # "on start nothread" cron
+			kwargs["noLoad"] = True
+		elif resp: # regular request
+			kwargs["response"] = resp
+			localvars.response = resp
+		else: # regular cron
+			kwargs["noLoad"] = True
+			kwargs["threaded"] = True
+		do_respond(*args, **kwargs)
+
+	def register_handler(self, args, kwargs):
+		self.logger.info("register handler: %s"%(self.curpath,))
+		self.handlers[self.curpath] = lambda resp : self._respond(resp, *args, **kwargs)
+
+	def trigger_handler(self, rule, target, req=None, option=None):
+		self.curpath = rule
+		if rule not in self.handlers:
+			if target in self.modules:
+				self.logger.info("linking module: %s"%(target,))
+				self.handlers[rule] = self.modules[target]
+			else:
+				self.logger.info("importing module: %s"%(target,))
+				__import__(target)
+				self.modules[target] = self.handlers[rule]
+		self.handlers[rule](req and Response(req) or option)
+
+	def blup(self):
+		wcfg = config.web
+		bl = wcfg.blacklist.obj()
+		blen = len(bl.keys())
+		self.logger.warn("saving %s IPs in black.list"%(blen,))
+		write(bl, "black.list", isjson=True)
+		if wcfg.report and self.blcount != blen:
+			self.blcount = blen
+			if not blen % self.blchunk:
+				from cantools.web import email_admins
+				email_admins("sketch IPs blacklisted", "sketch count: %s"%(blen,))
+		wcfg.blacklister and wcfg.blacklister.update(bl)
+
+class PaperShield(object):
+	def __init__(self):
+		self.logger = logger_getter("PaperShield")
+		self.default = { "reason": "I'm just a paper shield!" }
+		self.ips = {}
+
+	def __call__(self, path, ip, fspath=False, count=True):
+		self.logger.access('NOOP > paperShield("%s", "%s", fspath=%s, count=%s)'%(path, ip, fspath, count))
+
+	def suss(self, ip, reason):
+		self.logger.access("suss(%s) -> %s"%(ip, reason))
+		self.ips[ip] = { "reason": reason }
+
+	def ip(self, ip):
+		return self.ips.get(ip, self.default)
+
+def setBlacklist():
+	bl = {}
+	for preban in config.web.blacklist:
+		bl[preban] = "config ban"
+	if os.path.isfile("black.list"):
+		try:
+			bsaved = read("black.list", isjson=True)
+		except: # old style
+			bsaved = {}
+			for line in read("black.list", lines=True):
+				bsaved[line.strip()] = "legacy ban"
+		bsaved and bl.update(bsaved)
+	config.web.update("blacklist", bl)
+
+def getController():
+	global CTR
+	if not CTR:
+		# controller
+		CTR = DController()
+
+		shield = None
+		shfg = config.web.shield
+		if shfg: # web/admin share shield and blacklist
+			setBlacklist()
+			shield = Shield(config.web.blacklist, logger_getter, CTR.blup,
+				getattr(shfg, "limit", 400),
+				getattr(shfg, "interval", 2))
+		config.web.update("shield", shield or PaperShield())
+		mempad = config.mempad
+
+		# web
+		CTR.web = CTR.register_address(config.web.host,
+			config.web.port, dclass=daemon_wrapper(Web, logger_getter, shield, mempad))
+		CTR.web.controller = CTR
+
+		# admin
+		config.admin.update("pw",
+			config.cache("admin password? ", overwrite=config.newpass))
+		CTR.admin = CTR.register_address(config.admin.host,
+			config.admin.port, dclass=daemon_wrapper(Admin, logger_getter, shield, mempad))
+		CTR.admin.controller = CTR
+
+		# cron
+		Cron(CTR, logger_getter)
+
+	return CTR

cantools/web/dez_server/cron.py

@@ -0,0 +1,115 @@
+import time, os
+from datetime import datetime, timedelta
+from rel import timeout
+from cantools import config
+from cantools.util import read, write
+from ..util import do_respond
+
+secsPerUnit = {
+    "days": 60 * 60 * 24,
+    "hours": 60 * 60,
+    "minutes": 60,
+    "mins": 60,
+    "seconds": 1
+}
+
+class Rule(object):
+    def __init__(self, controller, scheduler, url, rule, logger_getter):
+        self.logger = logger_getter("Rule(%s -> %s)"%(rule, url))
+        self.controller = controller
+        self.scheduler = scheduler
+        self.url = url
+        self.rule = rule
+        self.exact = len(rule) == 5 # 17:00
+        self.words = rule.split(" ")
+        self.timer = timeout(None, self.trigger)
+        self.parse()
+
+    def trigger(self):
+        self.logger.info("trigger: %s (%s)"%(self.url, getattr(self, "seconds", self.rule)))
+        self.controller.trigger_handler(self.url, self.url[1:],
+            option=self.rule.endswith("nothread") and "nothread")
+        if not self.exact and not self.rule.startswith("on start"):
+            self.scheduler.update(self)
+        return True
+
+    def start(self):
+        if self.rule.startswith("on start"):
+            return
+        self.logger.info("start (%s seconds)"%(self.seconds,))
+        self.timer.add(self.seconds)
+        if self.exact:
+            self.timer.delay = 60 * 60 * 24
+        else:
+            ff = self.scheduler.ff(self)
+            if ff:
+                self.logger.info("rescheduled for %s seconds"%(self.timer.delay - ff,))
+                self.timer.expiration -= ff
+
+    def parse(self):
+        self.logger.info("parse")
+        if self.exact:
+            hours, mins = self.rule.split(":")
+            n = datetime.now()
+            t = datetime(n.year, n.month, n.day, int(hours), int(mins))
+            self.seconds = (t - n).seconds
+        elif self.rule.startswith("on start"):
+            self.logger.info("triggering start script")
+            self.trigger()
+        elif len(self.words) == 3 and self.words[0] == "every": # every [NUMBER] [UNIT]
+            num = int(self.words[1])
+            unit = self.words[2]
+            self.seconds = num * secsPerUnit[unit]
+        else: # we can implement more later...
+            self.logger.error("can't parse: %s"%(self.rule,))
+
+class Scheduler(object):
+    def __init__(self, logger_getter):
+        self.logger_getter = logger_getter
+        self.logger = logger_getter("Scheduler")
+        self.tsfile = os.path.join(os.path.abspath("."), "cron.ts")
+        self.lasts = read(self.tsfile, isjson=True, default={})
+        self.logger.info("initialized with %s timestamps"%(len(self.lasts.keys()),))
+        self.logger.info("saving state to timestamp file at %s"%(self.tsfile,))
+
+    def update(self, rule):
+        self.lasts[rule.url] = time.time()
+        self.logger.info("updating %s timestamp to %s"%(rule.url, self.lasts[rule.url]))
+        write(self.lasts, self.tsfile, isjson=True)
+
+    def ff(self, rule):
+        if rule.url in self.lasts:
+            ff = min(rule.seconds, time.time() - self.lasts[rule.url])
+            self.logger.info("fast-forwarding %s %s seconds"%(rule.url, ff))
+            return ff
+        elif config.cron.catchup:
+            self.logger.info("catching up %s"%(rule.url,))
+            return rule.seconds
+
+class Cron(object):
+    def __init__(self, controller, logger_getter):
+        self.logger_getter = logger_getter
+        self.logger = logger_getter("Cron")
+        self.scheduler = Scheduler(logger_getter)
+        self.controller = controller
+        self.timers = {}
+        self.parse()
+        self.start()
+
+    def parse(self):
+        self.logger.info("parse")
+        url = None
+        for line in read("cron.yaml", True):
+            if line.startswith("- description: "):
+                self.logger.info("initializing %s"%(line[15:],))
+            elif line.startswith("  url: "):
+                url = line[7:].strip()
+            elif url:
+                self.timers[url] = Rule(self.controller, self.scheduler,
+                    url, line[12:].strip(), self.logger_getter)
+                url = None
+
+    def start(self):
+        self.logger.info("start")
+        for rule in list(self.timers.values()):
+            rule.start()

cantools/web/dez_server/daemons.py

@@ -0,0 +1,64 @@
+import sys, json, gc, os, inspect, threading
+try:
+    import psutil
+except ImportError as e:
+    pass # google crap engine (get it if you need it!)
+from dez.memcache import get_memcache
+from dez.http.application import HTTPApplication
+from .routes import static, cb
+from cantools import config
+sys.path.insert(0, ".") # for dynamically loading modules
+
+A_STATIC = {
+    "dynamic": { "/": "_/dynamic", "/css/": "_/css", "/js/CT/": "js/CT", "/logs/": "logs", "/logs": "logs" },
+    "static": { "/": "_/static", "/css/": "_/css", "/js/CT/": "js/CT", "/logs/": "logs", "/logs": "logs" },
+    "production": { "/": "_/production", "/css/": "_/css", "/logs/": "logs", "/logs": "logs" },
+}
+A_CB = { "/admin": "admin", "/_db": "_db" }
+
+class CTWebBase(HTTPApplication):
+    def __init__(self, bind_address, port, logger_getter, static=static, cb=cb, whitelist=[], blacklist=[], shield=False, mempad=0):
+        isprod = config.mode == "production"
+        HTTPApplication.__init__(self, bind_address, port, logger_getter, "dez/cantools",
+            config.ssl.certfile, config.ssl.keyfile, config.ssl.cacerts,
+            isprod, config.web.rollz, isprod, whitelist, blacklist, shield, mempad, config.web.xorigin)
+        self.memcache = get_memcache()
+        self.handlers = {}
+        for key, val in list(static.items()):
+            self.add_static_rule(key, val)
+        for key, val in list(cb.items()):
+            self.add_cb_rule(key, self._handler(key, val))
+
+    def _handler(self, rule, target):
+        self.logger.info("setting handler: %s %s"%(rule, target))
+        def h(req):
+            self.logger.info("triggering handler: %s %s"%(rule, target))
+            self.controller.trigger_handler(rule, target, req)
+        return h
+
+class Web(CTWebBase):
+    def __init__(self, bind_address, port, logger_getter, shield, mempad):
+        self.logger = logger_getter("Web")
+        wcfg = config.web
+        CTWebBase.__init__(self, bind_address, port, logger_getter,
+            whitelist=wcfg.whitelist, blacklist=wcfg.blacklist, shield=shield, mempad=mempad)
+
+class Admin(CTWebBase):
+    def __init__(self, bind_address, port, logger_getter, shield, mempad):
+        self.logger = logger_getter("Admin")
+        acfg = config.admin
+        CTWebBase.__init__(self, bind_address, port, logger_getter, # share shield/blacklist
+            A_STATIC[config.mode], A_CB, acfg.whitelist, config.web.blacklist, shield, mempad)
+        self.add_cb_rule("/_report", self.report)
+
+    def report(self, req):
+        report = json.dumps({
+            "threads": threading.active_count(),
+            "stack_frames": len(inspect.stack()),
+            "web": self.controller.web.daemon.counter.report(),
+            "admin": self.daemon.counter.report(),
+            "gc": len(gc.get_objects()),
+            "mem": psutil.Process(os.getpid()).memory_percent()
+        })
+        self.logger.info(report)
+        self.daemon.respond(req, report)

cantools/web/dez_server/mail.py

@@ -0,0 +1,24 @@
+from catmail import Mailer, Reader, Scanner
+from catmail import config as catfyg
+from ..util import config
+
+catfyg.set({
+	"html": config.mailhtml,
+	"gmailer": config.gmailer,
+	"scantick": config.mailscantick,
+	"verbose": config.mailoud,
+	"cache": config.cache
+})
+catfyg.admin.update("contacts", config.admin.contacts)
+catfyg.admin.update("reportees", config.admin.reportees)
+
+mailer = Mailer(config.mailer, config.mailername)
+send_mail = mailer.mail
+email_admins = mailer.admins
+email_reportees = mailer.reportees
+
+reader = Reader(config.mailer)
+check_inbox = reader.inbox
+
+scanner = Scanner(reader)
+on_mail = scanner.on

cantools/web/dez_server/response.py

@@ -0,0 +1,63 @@
+import json
+from rel import abort_branch, timeout
+from dez.http.server import HTTPResponse
+from cantools import config
+from ..util import *
+
+files = {}
+
+class Response(object):
+    def __init__(self, request):
+        self.id = request.id
+        self.ip = request.real_ip
+        self.request = request
+        self.response = HTTPResponse(request)
+
+    def _read(self):
+        b = self.request.body or json.dumps(rec_conv(self.request.qs_params))
+#        print(b)
+        ctype = self.request.headers.get("content-type")
+        if ctype and ctype.startswith("multipart/form-data"):
+            if type(b) != bytes:
+                b = b.encode()
+            obj = {}
+            splitter, body = b.rsplit(b"\r\n", 2)[0].split(b"\r\n", 1)
+            for chunk in body.split(b"\r\n%s\r\n"%(splitter,)):
+                nameline, data = chunk.split(b"\r\n\r\n", 1)
+                nameline = nameline.decode()
+                name = nameline.split("; filename=")[0][:-1].split('name="')[1]
+                if "filename=" in nameline:
+                    signature = "%s%s"%(self.id, name)
+                    files[signature] = data
+                    obj[name] = signature
+                else:
+                    obj[name] = data
+            b = json.dumps(rec_conv(obj))
+        return b
+
+    def _send(self, *args, **kwargs):
+        wcfg = config.web
+        if wcfg.xorigin:
+            self.response.headers["Access-Control-Allow-Origin"] = "*"
+        if wcfg.csp:
+            if wcfg.csp.startswith("auto"):
+                d = wcfg.domain
+                if wcfg.csp != "autosub":
+                    d = ".".join(d.split(".")[-2:])
+                wcfg.update("csp", "default-src 'self' %s *.%s"%(d, d))
+            self.response.headers["Content-Security-Policy"] = wcfg.csp
+        self.response.write(*args, **kwargs)
+
+    def _close(self):
+        timeout(.001, self.response.dispatch)
+        abort_branch()
+
+    def _header(self, *args, **kwargs):
+        self.response.__setitem__(*args, **kwargs)
+
+    def set_cbs(self):
+        set_read(self._read)
+        set_header(self._header)
+        set_send(self._send)
+        set_close(self._close)
+        set_redir(self.response.redirect)

cantools/web/dez_server/routes.py

@@ -0,0 +1,21 @@
+from ...util import read
+
+url = None
+static = {}
+cb = {}
+
+for line in read("app.yaml", True):
+	if line.startswith("- url: "):
+		url = line[7:].strip()
+	elif url:
+		if line.startswith("  static_dir: "):
+			target = line[14:].strip()
+			if "*" in url: # bad regex detection...
+				static[url] = target
+			else:
+				url = url.rstrip("/")
+				static[url] = static[url + "/"] = target
+			url = None
+		elif line.startswith("  script: "):
+			cb[url] = line[10:].split(".")[0]
+			url = None

cantools/web/dez_server/server.py

@@ -0,0 +1,229 @@
+import os, rel, ssl, sys, json
+from dez.http import fetch as dfetch, post as dpost
+from ..util import *
+from ...util import set_log, set_error, init_rel
+from .mail import send_mail, email_admins, email_reportees, mailer, reader, check_inbox, scanner, on_mail
+from .sms import send_sms
+from .controller import getController
+from cantools import config
+
+def fdup():
+	import resource
+	from cantools.util import log # gives us logger set in run_dez_webserver()
+	log("checking the number of file descriptors allowed on your system", important=True)
+	soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+	half = int(hard / 2)
+	log("soft: %s. hard: %s. half: %s"%(soft, hard, half))
+	if soft < half:
+		log("increasing soft limit to half of hard limit")
+		resource.setrlimit(resource.RLIMIT_NOFILE, (half, hard))
+		log("new limits! soft: %s. hard: %s"%resource.getrlimit(resource.RLIMIT_NOFILE))
+
+def log_kernel():
+	from cantools.web.util import log
+	log(json.dumps(rel.report()), "kernel")
+	return True
+
+TMSNAP = None
+
+def log_tracemalloc():
+	global TMSNAP
+	import tracemalloc
+	from cantools.util import log
+	snapshot = tracemalloc.take_snapshot()
+	log("[LINEMALLOC START]", important=True)
+	if TMSNAP:
+		lines = snapshot.compare_to(TMSNAP, 'lineno')
+	else:
+		lines = snapshot.statistics("lineno")
+	TMSNAP = snapshot
+	for line in lines[:10]:
+		log(line)
+	log("[LINEMALLOC END]", important=True)
+	return True
+
+PROC = None
+
+def log_openfiles():
+	global PROC
+	from cantools.util import log
+	if not PROC:
+		import psutil
+		PROC = psutil.Process(os.getpid())
+	ofz = PROC.open_files()
+	if config.log.oflist:
+		log("OPEN FILES: %s"%(ofz,), important=True)
+	else:
+		log("OPEN FILE COUNT: %s"%(len(ofz),), important=True)
+	return True
+
+def quit():
+	from cantools.util import log
+	if config.web.errlog:
+		log("closing error log", important=True)
+		sys.stderr.close()
+	log("quitting - goodbye!", important=True)
+
+def run_dez_webserver():
+	if not config.ssl.verify and hasattr(ssl, "_https_verify_certificates"):
+		ssl._https_verify_certificates(False)
+	c = getController()
+	setlog(c.web.logger.simple)
+	if config.web.log:
+		set_log(os.path.join("logs", config.web.log))
+	init_rel()
+	clog = config.log
+	if clog.openfiles:
+		rel.timeout(clog.openfiles, log_openfiles)
+	if clog.tracemalloc:
+		import tracemalloc
+		tracemalloc.start()
+		rel.timeout(clog.tracemalloc, log_tracemalloc)
+	if "kernel" in clog.allow:
+		rel.timeout(1, log_kernel)
+	set_error(fail)
+	if config.fdup:
+		fdup()
+	if config.web.errlog:
+		sys.stderr = open(os.path.join("logs", config.web.errlog), "a")
+	c.start(quit)
+
+def dweb():
+	return getController().web
+
+def respond(*args, **kwargs):
+	getController().register_handler(args, kwargs)
+
+def _ctjson(result):
+	from cantools.util import log # gives us logger set in run_dez_webserver()
+	result = result.decode()
+	code = result[0]
+	if code not in "0123":
+		log("response encoded:")
+		log(result)
+		log("attempting decode")
+		result = dec(result)
+	if code in "02":
+		from cantools.util import log
+		log("request failed!! : %s"%(result,), important=True)
+	elif code == "3":
+		return rec_conv(json.loads(result[1:]), True)
+	else:
+		return json.loads(result[1:])
+
+def parse_url_parts(host, path, port, protocol):
+	if "://" in host:
+		protocol, host = host.split("://", 1)
+		if "/" in host:
+			host, path = host.split("/", 1)
+			path = "/" + path
+		else:
+			path = "/"
+	if ":" in host:
+		host, port = host.split(":")
+		port = int(port)
+	elif not port:
+		port = protocol == "https" and 443 or 80
+	return host, path, port, protocol
+
+def fetch(host, path="/", port=None, asjson=False, cb=None, timeout=1, asyn=False, protocol="http", ctjson=False, qsp=None, fakeua=False, retries=5):
+	from cantools.util import log # gives us logger set in run_dez_webserver()
+	host, path, port, protocol = parse_url_parts(host, path, port, protocol)
+	if qsp:
+		path += "?"
+		for k, v in list(qsp.items()):
+			path += "%s=%s&"%(k, v)
+		path = path[:-1]
+	gkwargs = {}
+	headers = {}
+	if fakeua:
+		if type(fakeua) is str:
+			headers['User-Agent'] = fakeua
+		else:
+			headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36'
+		gkwargs["headers"] = headers
+	if asyn or cb: # asyn w/o cb works, will just log
+		secure = protocol == "https"
+		if ctjson:
+			orig_cb = cb or log
+			cb = lambda v : orig_cb(_ctjson(v))
+		return dfetch(host, path, port, secure, headers, cb, timeout, asjson)
+	if timeout:
+		gkwargs["timeout"] = timeout
+	furl = "%s://%s:%s%s"%(protocol, host, port, path)
+	log("fetch %s"%(furl,))
+	return syncreq(furl, "get", asjson, ctjson, retries, gkwargs)
+
+def post(host, path="/", port=80, data=None, protocol="http", asjson=False, ctjson=False, text=None, cb=None):
+	if ctjson:
+		data = rec_conv(data)
+	if cb:
+		if ctjson:
+			orig_cb = cb
+			cb = lambda v : orig_cb(_ctjson(v))
+		host, path, port, protocol = parse_url_parts(host, path, port, protocol)
+		return dpost(host, path, port, protocol == "https", data=data, text=text, cb=cb)
+	url = "://" in host and host or "%s://%s:%s%s"%(protocol, host, port, path)
+	log("post %s"%(url,))
+	kwargs = {}
+	if data:
+		kwargs["json"] = data
+	elif text:
+		kwargs["data"] = text
+	return syncreq(url, "post", asjson, ctjson, rekwargs=kwargs)
+
+def _dosyncreq(requester, url, asjson, ctjson, rekwargs):
+	result = requester(url, **rekwargs).content
+	if ctjson:
+		return _ctjson(result)
+	return asjson and json.loads(result) or result
+
+def syncreq(url, method="get", asjson=False, ctjson=False, retries=5, rekwargs={}):
+	import time, requests
+	attempt = 1
+	requester = getattr(requests, method)
+	while attempt < retries:
+		try:
+			return _dosyncreq(requester, url, asjson, ctjson, rekwargs)
+		except requests.exceptions.ConnectionError:
+			log("syncreq(%s %s) attempt #%s failed"%(method, url, attempt))
+			time.sleep(1)
+		attempt += 1
+	return _dosyncreq(requester, url, asjson, ctjson, rekwargs) # final try-less try
+
+# file uploads
+def read_file(data_field):
+	from .response import files
+	return files.pop(data_field)
+
+# memcache stuff
+def getmem(key, tojson=True):
+	return dweb().memcache.get(key, tojson)
+
+def setmem(key, val, fromjson=True):
+	dweb().memcache.set(key, val, fromjson)
+
+def delmem(key):
+	dweb().memcache.rm(key)
+
+def clearmem():
+	dweb().memcache.clear()
+
+def getcache():
+	c = {}
+	orig = dweb().memcache.cache
+	for k, v in list(orig.items()):
+		if v.__class__ == set:
+			v = list(v)
+		elif v.__class__ == dict and "ttl" in v: # countdown
+			v = str(v)
+		c[k] = v
+	return c
+
+set_getmem(getmem)
+set_setmem(setmem)
+set_delmem(delmem)
+set_clearmem(clearmem)
+
+if __name__ == "__main__":
+	run_dez_webserver()

cantools/web/dez_server/sms.py

@@ -0,0 +1,12 @@
+from .mail import send_mail
+from cantools import config
+
+carriers = {
+	'at&t': 'mms.att.net',
+	'verizon': 'vtext.com',
+	'tmobile': 'tmomail.net',
+	'sprint': 'page.nextel.com'
+}
+
+def send_sms(number, subject="hello", body="goodbye", carrier="at&t"):
+	send_mail(to="%s@%s"%(number, carriers[carrier]), subject=subject, body=body)