ct 0.10.8.114__py3-none-any.whl → 0.10.8.115__py3-none-any.whl

cantools/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.10.8.114"
+__version__ = "0.10.8.115"
 
 from . import util, hooks
 from . import config as cfgmod
@@ -10,15 +10,40 @@ if config.web.server == "gae":
 	util.init_gae()
 else:
 	util.init_basic()
-from . import geo
-from .scripts import builder, deploy, init, pubsub, start, index, migrate, doc, bench, util
-
-ctstart = start.go
-ctdeploy = deploy.run
-ctpubsub = pubsub.get_addr_and_start
-ctinit = init.parse_and_make
-ctindex = index.go
-ctmigrate = migrate.go
-ctdoc = doc.build
-ctbench = bench.run
-ctutil = util.run
+#from . import geo
+from .scripts import builder, pubsub
+
+def ctstart():
+	from .scripts import start
+	start.go()
+
+def ctdeploy():
+	from .scripts import deploy
+	deploy.run()
+
+def ctpubsub():
+	pubsub.get_addr_and_start()
+
+def ctinit():
+	from .scripts import init
+	init.parse_and_make()
+
+def ctindex():
+	from .scripts import index
+	index.go()
+
+def ctmigrate():
+	from .scripts import migrate
+	migrate.go()
+
+def ctdoc():
+	from .scripts import doc
+	doc.build()
+
+def ctbench():
+	from .scripts import bench
+	bench.run()
+
+def ctutil():
+	from .scripts import util
+	util.run()

cantools/_db.py

@@ -69,7 +69,7 @@ def response():
 			keys = cgi_get("keys", required=False)
 			exporter = cgi_get("exporter", default="export")
 			if mname:
-				order = cgi_get("order", default="index")
+				order = cgi_get("order", required=False)
 				if config.web.server == "gae":
 					order = getattr(get_model(mname), order)
 				res = get_page(mname, int(cgi_get("limit")), int(cgi_get("offset")), order,

cantools/cfg.py

@@ -69,6 +69,7 @@ cfg = {
 		"protocol": "http",
 		"xorigin": False,
 		"shield": False,
+		"debug": False,
 		"csp": None,
 		"log": None,
 		"errlog": None,
@@ -78,6 +79,9 @@ cfg = {
 		"blacklist": [],
 		"whitelist": []
 	},
+	"proxy": { # user, gateway, minport, maxport
+		"active": False
+	},
 	"ssl": {
 		"verify": True,
 		"certfile": None,
@@ -97,6 +101,15 @@ cfg = {
 		"alter": False, # add new columns to tables - sqlite only!
 		"echo": False,
 		"public": True, # read from db without credentials via _db.py web handler
+		"nopoly": False,
+		"jsontext": True,
+		"arraytext": True,
+		"stringsize": 500,
+		"flatkeysize": 80,
+		"index": {
+			"key": False,
+			"named": False
+		},
 		"pool": {
 			"null": True,
 			"size": 10,

cantools/config.py

@@ -57,9 +57,9 @@ for line in lines:
 		items.append([key, val])
 
 for key, val in items:
-	if key in ["ENCODE", "DB_ECHO", "DB_PUBLIC", "DB_REFCOUNT", "DB_CACHE", "DB_POOL_NULL", "GEO_TEST", "REL_VERBOSE", "REL_LOUDLISTEN", "MEMCACHE_REQUEST", "MEMCACHE_DB", "PUBSUB_ECHO", "PUBSUB_META", "PUBSUB_B64", "SSL_VERIFY", "ADMIN_MONITOR_LOG", "WEB_XORIGIN", "LOG_TIMESTAMP", "BUILD_PROD_CLOSURE", "BUILD_PROD_B64", "GMAILER", "MAILHTML", "MAILOUD"]:
+	if key in ["ENCODE", "DB_ECHO", "DB_PUBLIC", "DB_REFCOUNT", "DB_CACHE", "DB_JSONTEXT", "DB_ARRAYTEXT", "DB_NOPOLY", "DB_POOL_NULL", "DB_INDEX_KEYS", "DB_INDEX_NAMED", "PROXY_ACTIVE", "GEO_TEST", "REL_VERBOSE", "REL_LOUDLISTEN", "MEMCACHE_REQUEST", "MEMCACHE_DB", "PUBSUB_ECHO", "PUBSUB_META", "PUBSUB_B64", "SSL_VERIFY", "ADMIN_MONITOR_LOG", "WEB_DEBUG", "WEB_XORIGIN", "LOG_TIMESTAMP", "BUILD_PROD_CLOSURE", "BUILD_PROD_B64", "GMAILER", "MAILHTML", "MAILOUD"]:
 		val = val == "True"
-	elif key in ["PUBSUB_HISTORY", "MEMPAD", "WEB_SHIELD_CHUNK", "WEB_SHIELD_LIMIT", "WEB_SHIELD_INTERVAL", "MEMCACHE_PROX_TIMEOUT", "DB_POOL_SIZE", "DB_POOL_OVERFLOW", "DB_POOL_RECYCLE", "LOG_OPENFILES", "LOG_TRACEMALLOC", "MAILSCANTICK"]:
+	elif key in ["PUBSUB_HISTORY", "MEMPAD", "WEB_SHIELD_CHUNK", "WEB_SHIELD_LIMIT", "WEB_SHIELD_INTERVAL", "PROXY_MINPORT", "PROXY_MAXPORT", "MEMCACHE_PROX_TIMEOUT", "DB_POOL_SIZE", "DB_POOL_OVERFLOW", "DB_POOL_RECYCLE", "DB_STRINGSIZE", "DB_FLATKEYSIZE", "LOG_OPENFILES", "LOG_TRACEMALLOC", "MAILSCANTICK"]:
 		val = int(val)
 	elif key in ["REL_SLEEP", "REL_TURBO"]:
 		val = float(val)
@@ -92,11 +92,16 @@ for key, val in items:
 		c.update(target, val)
 
 config.update("cache", pc)
-config.db.update("main", config.db[config.web.server])
+if not config.db.main:
+	config.db.update("main", config.db[config.web.server])
+if "DBPW" in config.db.main:
+	config.db.update("main", config.db.main.replace("DBPW", config.cache("database password? ")))
 for prop in ["deep", "flush", "timestamp", "allow"]:
 	confyg.log.update(prop, config.log[prop])
-for prop in ["cache", "refcount", "main", "test", "blob", "alter", "echo"]:
+for prop in ["cache", "refcount", "main", "test", "blob", "alter", "echo", "jsontext", "arraytext", "stringsize", "flatkeysize"]:
 	dbcfg.update(prop, config.db[prop])
+for prop in ["key", "named"]:
+	dbcfg.index.update(prop, config.db.index[prop])
 for prop in ["null", "size", "recycle", "overflow"]:
 	dbcfg.pool.update(prop, config.db.pool[prop])
 

cantools/db/__init__.py

@@ -5,12 +5,15 @@ if config.web.server == "gae":
 	from .gae.model import *
 elif config.web.server == "dez":
 	from rel import tick
+	if not config.db.nopoly:
+		from databae.poly import ModelBase, TimeStampedBase
 	from databae import *
 	from cantools.web import set_pre_close, cgi_dump
 	def scoper(threadId):
 		if threadId == "MainThread":
 			threadId = tick()
 		return "%s%s"%(threadId, cgi_dump())
+	session = seshman.get()
 	set_scoper(scoper)
 	set_pre_close(seshman.close)
 else:

cantools/db/gae/model.py

@@ -1,6 +1,5 @@
 from datetime import datetime
 from .properties import *
-from six import with_metaclass
 
 class CTMeta(ndb.MetaModel):
     def __new__(cls, name, bases, attrs):
@@ -22,7 +21,7 @@ class CTMeta(ndb.MetaModel):
         modelsubs[lname] = super(CTMeta, cls).__new__(cls, name, bases, attrs)
         return modelsubs[lname]
 
-class ModelBase(with_metaclass(CTMeta, ndb.Model)):
+class ModelBase(ndb.Model, metaclass=CTMeta):
     index = Integer()
 
     def __eq__(self, other):

cantools/db/wp.py

@@ -45,7 +45,7 @@ def trydb(attempts=5, wait=0.4):
 		db = setdb()
 		if db:
 			return db
-		log("trydb failed retry #" + attempt, important=True)
+		log("trydb failed retry #%s"%(attempt,), important=True)
 		time.sleep(wait)
 
 def getdb(subdb=False):

cantools/scripts/index.py

@@ -44,16 +44,9 @@ Run this in 'index' mode on a database with lots of missing index values.
 from getpass import getpass
 from optparse import OptionParser
 from fyg.util import log, error, batch
-from cantools.db import get_schema, get_model, put_multi, delete_multi, unpad_key
+from cantools.db import get_schema, get_model, put_multi, delete_multi, unpad_key, session, func
 from cantools.web import fetch
 from cantools import config
-if config.web.server == "dez":
-	from cantools.db import session, func, refresh_counter
-
-try:
-    input = raw_input # py2/3 compatibility
-except NameError:
-    pass
 
 counts = { "_counters": 0 }
 RETRIES = 5
@@ -93,6 +86,7 @@ def refmap():
 	return rmap
 
 def do_batch(chunk, reference):
+	from databae.lookup import refresh_counter
 	log("refreshing %s %s keys"%(len(chunk), reference), 1)
 	i = 0
 	rc = []

cantools/scripts/init.py

@@ -248,7 +248,7 @@ class Builder(object):
 				for dname, fnames in list(mod.init.syms.items()):
 					for fname in fnames:
 						sym(os.path.join(mod.__ct_mod_path__, dname, fname),
-							os.path.join(dname, fname))
+							os.path.join(dname, fname.strip("./")))
 
 	def vcignore(self):
 		log("configuring version control path exclusion", 1)

cantools/scripts/pubsub/actor.py

@@ -1,4 +1,6 @@
-class Actor(object):
+from fyg.util import Named
+
+class Actor(Named):
     def data(self):
         return {
             "name": self.name,

cantools/scripts/pubsub/bots.py

@@ -1,13 +1,7 @@
 import datetime, json, os
 from cantools import config
 from cantools.util import log, write
-from cantools.web import fetch, send_mail
 from .actor import Actor
-try:
-	import psutil
-except ImportError as e:
-	pass # google crap engine (get it if you need it!)
-from six import with_metaclass
 
 class BotMeta(type):
 	def __new__(cls, name, bases, attrs):
@@ -17,7 +11,7 @@ class BotMeta(type):
 			config.pubsub.bots.update(name.lower(), bc)
 		return bc
 
-class Bot(with_metaclass(BotMeta, Actor)):
+class Bot(Actor, metaclass=BotMeta):
 	num = 0
 	def __init__(self, server, channel, name=None):
 		Bot.num += 1
@@ -26,7 +20,7 @@ class Bot(with_metaclass(BotMeta, Actor)):
 		self.channel = channel # often we only care about one channel
 		self.channels = set()
 		self._set_defaults()
-		log("Bot Spawned: '%s'"%(self.name,), 2)
+		self.log("Bot Spawned")
 		channel.join(self)
 		self.server.bots[self.name] = self
 
@@ -41,7 +35,7 @@ class Bot(with_metaclass(BotMeta, Actor)):
 
 	def _default_handler(self, action):
 		def _h(*args):
-			log('Bot %s handling %s: "%s"'%(self.name, action, json.dumps(args)), 3)
+			self.log("handling", action, ":", args)
 		return _h
 
 	def _set_defaults(self):
@@ -74,25 +68,29 @@ class Monitor(Bot):
 			os.mkdir(dp)
 		return os.path.join(dp, str(n.hour))
 
-	def log(self, data):
+	def report(self, data):
 		self.pub(data)
 		if config.admin.monitor.log:
 			write(data, self._datedir(), True, append=True, newline=True)
 
 	def _cpu(self):
+		import psutil
+		from cantools.web import send_mail
 		c = self.current["cpu"] = psutil.cpu_percent()
 		if self.alert.get("cpu"):
 			if c < config.admin.monitor.thresholds.cpu:
 				del self.alert["cpu"]
-				log("CPU calmed down")
+				self.log("CPU calmed down")
 				send_mail(config.admin.contacts, subject="High CPU", body="just ended")
 		else:
 			if c >= config.admin.monitor.thresholds.cpu:
 				self.alert["cpu"] = True
-				log("CPU just started going crazy")
+				self.log("CPU just started going crazy")
 				send_mail(config.admin.contacts, subject="High CPU", body="just started")
 
 	def _tick(self):
+		import psutil
+		from cantools.web import fetch
 		self._cpu()
 		dioc = psutil.disk_io_counters()
 		nioc = psutil.net_io_counters()
@@ -139,5 +137,5 @@ class Monitor(Bot):
 		if config.admin.monitor.proxy:
 			data["ips"]["proxy"] = fetch(config.admin.host, "/_report",
 				config.admin.monitor.proxy, True)["ips"]
-		self.log(data)
+		self.report(data)
 		return True

cantools/scripts/start.py

@@ -44,8 +44,8 @@ def go():
 		print(cmd)
 		subprocess.call(cmd, shell=True)
 	elif options.web_backend == "dez":
-		from cantools.web import run_dez_webserver
-		run_dez_webserver()
+		from cantools.web import run_bw
+		run_bw()
 	else:
 		error("invalid web_backend: %s"%(options.web_backend,))
 

cantools/util/__init__.py

@@ -7,9 +7,11 @@ from .media import transcode, segment, hlsify, shouldMoveMoov, crop, resizep2, t
 from .admin import certs, screener
 from .apper import android
 from .ai import tox, vox, tellme
+from .package import pipper
 
 def init_basic():
     os.path.isdir("emails") and sys.path.append("emails")
+    sys.path.insert(0, ".") # for dynamically loading modules
 
 def init_rel():
     import rel

cantools/util/admin.py

@@ -78,6 +78,20 @@ def pcheck(pname, target, starter):
 		cmd("killall screen; %s"%(starter,))
 		return True
 
+def pkill(pname, force=False):
+	pblock = output("ps -ef | grep %s | egrep -v 'screener|pkill|grep'"%(pname,))
+	if not pblock:
+		log("no '%s' processes!"%(pname,))
+	else:
+		plines = pblock.split("\n")
+		log("found %s '%s' processes"%(len(plines), pname), important=True)
+		if plines:
+			procs = [[w for w in line.split(" ") if w][1] for line in plines]
+			if force or confirm("kill %s '%s' processes"%(len(procs), pname)):
+				for proc in procs:
+					cmd("kill -9 %s"%(proc,))
+	log("goodbye")
+
 def binpath(bpath="/usr/bin/"):
 	log("checking %s core modules"%(len(coremods),), important=True)
 	missings = []
@@ -295,11 +309,12 @@ def upcheck(*procs):
 	log("goodbye")
 	close_log()
 
-def vitals(clean=False, thresh=90, dpath="."):
+def vitals(clean=False, thresh=90, dpath=".", conlim=0):
+	from cantools.web import email_admins
 	if clean == "False":
 		clean = False
 	thresh = int(thresh)
-	from cantools.web import email_admins
+	conlim = int(conlim)
 	os.chdir(dpath)
 	set_log("cron-vitals.log")
 	log("scanning vitals", important=True)
@@ -310,9 +325,12 @@ def vitals(clean=False, thresh=90, dpath="."):
 	lz.append("inode usage: %s%%"%(inodes,))
 	memuse = float(output("free | grep Mem | awk '{print $3/$2 * 100.0}'"))
 	lz.append("memory usage: %s%%"%(memuse,))
+	if conlim:
+		concount = int(output("lsof -i | wc -l"))
+		lz.append("connections: %s"%(concount,))
 	for l in lz:
 		log(l)
-	if hdrive > thresh or inodes > thresh or memuse > thresh:
+	if hdrive > thresh or inodes > thresh or memuse > thresh or (conlim and concount > conlim):
 		log("threshold exceeded - notifying admins")
 		if hdrive > thresh and clean:
 			log("cleaning up!")
@@ -333,9 +351,9 @@ def sslredirect(port=80):
 def blacklist(ip):
 	if not confirm("blacklist %s?"%(ip,)):
 		return log("okay, bye!")
-	from cantools.web import controller
+	from cantools.web import shield
 	from cantools import config
-	controller.setBlacklist()
+	shield.setBlacklist()
 	blist = config.web.blacklist.obj()
 	if ip in blist:
 		return log("%s is already blacklisted! reason: %s"%(ip, blist[ip]))

cantools/web/__init__.py

@@ -1,9 +1,13 @@
-from .util import config
+from cantools import config
 
 if config.web.server == "gae":
 	from .gae_server import *
 elif config.web.server == "dez":
-	from cantools.web.dez_server import *
+	from .bw import *
+	from babyweb import *
+	from babyweb import mail
+	from babyweb.sms import send_sms
+	from babyweb.mail import send_mail, email_admins, email_reportees, mailer, reader, check_inbox, scanner, on_mail
 else:
 	from cantools import util
 	util.error("no web server specified")

cantools/web/bw.py

@@ -0,0 +1,70 @@
+import json, gc, os, inspect, threading, psutil
+from babyweb.daemons import WebBase, initWebs, logger_getter
+from babyweb.server import run_dez_webserver
+from babyweb.config import config as babyfyg
+from babyweb.logger import log, syslog
+from ..util import init_rel
+from cantools import config
+
+A_STATIC = {
+    "dynamic": { "/": "_/dynamic", "/css/": "_/css", "/js/CT/": "js/CT", "/logs/": "logs", "/logs": "logs" },
+    "static": { "/": "_/static", "/css/": "_/css", "/js/CT/": "js/CT", "/logs/": "logs", "/logs": "logs" },
+    "production": { "/": "_/production", "/css/": "_/css", "/logs/": "logs", "/logs": "logs" },
+}
+A_CB = { "/admin": "admin", "/_db": "_db" }
+
+def setcfg():
+	syslog("passing configuration to babyweb")
+	for prop in ["cache", "encode", "mempad", "web", "cron", "scrambler"]:
+		babyfyg.update(prop, config[prop])
+	for prop in ["contacts", "reportees"]:
+		babyfyg.admin.update(prop, config.admin[prop])
+	for prop in ["verify", "certfile", "keyfile", "cacerts"]:
+		babyfyg.ssl.update(prop, config.ssl[prop])
+	for prop in ["oflist", "openfiles", "tracemalloc", "allow"]:
+		babyfyg.log.update(prop, config.log[prop])
+	for prop in ["active", "user", "gateway", "minport", "maxport"]:
+		babyfyg.proxy.update(prop, config.proxy[prop])
+	babyfyg.update("memcache", config.memcache.request)
+	bmfg = babyfyg.mail
+	bmfg.update("mailer", config.mailer)
+	bmfg.update("gmailer", config.gmailer)
+	bmfg.update("name", config.mailername)
+	bmfg.update("html", config.mailhtml)
+	bmfg.update("verbose", config.mailoud)
+	bmfg.update("scantick", config.mailscantick)
+
+setcfg()
+
+class Admin(WebBase):
+    def __init__(self, bind_address, port, logger_getter, shield, mempad):
+        self.logger = logger_getter("Admin")
+        acfg = config.admin
+        WebBase.__init__(self, bind_address, port, logger_getter, # share shield/blacklist
+            A_STATIC[config.mode], A_CB, acfg.whitelist, config.web.blacklist, shield, mempad)
+        self.add_cb_rule("/_report", self.report)
+
+    def report(self, req):
+        report = json.dumps({
+            "threads": threading.active_count(),
+            "stack_frames": len(inspect.stack()),
+            "web": self.controller.webs["web"].daemon.counter.report(),
+            "admin": self.daemon.counter.report(),
+            "gc": len(gc.get_objects()),
+            "mem": psutil.Process(os.getpid()).memory_percent()
+        })
+        self.logger.info(report)
+        self.daemon.respond(req, report)
+
+def run_bw():
+	syslog("initializing web server")
+	init_rel()
+	initWebs({
+		"admin": {
+			"daemon": Admin,
+			"config": config.admin
+		}
+	})
+	config.admin.update("pw",
+		config.cache("admin password? ", overwrite=config.newpass))
+	run_dez_webserver()

cantools/web/gae_server.py

@@ -60,9 +60,4 @@ def delmem(key):
 
 def clearmem():
     from google.appengine.api import memcache
-    memcache.flush_all()
-
-set_getmem(getmem)
-set_setmem(setmem)
-set_delmem(delmem)
-set_clearmem(clearmem)
+    memcache.flush_all()