ct 0.10.8.114__py3-none-any.whl

cantools/config.py

@@ -0,0 +1,131 @@
+import os
+from fyg import Config, PCache, config as confyg
+from databae import config as dbcfg
+from fyg.util import read
+from .cfg import cfg
+
+pc = PCache(".ctp")
+
+def _getpass(val, ptype):
+	if "{PASSWORD}" in val:
+		val = val.replace("{PASSWORD}", pc("enter password (%s): "%(ptype,)))
+	return val
+
+config = Config(cfg)
+
+def include_plugins():
+	plugs = {}
+	def loadp(plugin):
+		plugin = plugin.split("/")[-1]
+		if plugin not in plugs:
+			try:
+				mod = plugs[plugin] = __import__(plugin)
+			except:
+				print("missing plugin: %s (fine if refreshing project for 1st time)"%(plugin,))
+				return
+			if hasattr(mod.init, "requires"):
+				for p in mod.init.requires:
+					loadp(p)
+			if hasattr(mod.init, "cfg"):
+				config.update(plugin, mod.init.cfg)
+	for plugin in config.plugin.modules:
+		loadp(plugin)
+
+config.plugin.update("modules", [])
+config.plugin.update("repos", [])
+
+items = []
+lines = read("ct.cfg", True)
+gcpath = os.path.join(os.path.expanduser("~"), "ct.cfg")
+if os.path.isfile(gcpath):
+	print("loading global configuration at: %s"%(gcpath,))
+	lines = read(gcpath, True) + lines
+for line in lines:
+	if line.startswith("#"):
+		continue
+	try:
+		key, val = [term.strip() for term in line.split(" = ", 1)]
+	except Exception as e:
+		print("failed to parse config on line:", line)
+		raise e
+	if key == "PLUGIN_MODULES":
+		mods = val.split("|")
+		config.plugin.update("modules", [p.split("/")[-1] for p in mods])
+		config.plugin.update("repos", ["/" in p and p or "%s/%s"%(config.plugin.base, p) for p in mods])
+		include_plugins()
+	else:
+		items.append([key, val])
+
+for key, val in items:
+	if key in ["ENCODE", "DB_ECHO", "DB_PUBLIC", "DB_REFCOUNT", "DB_CACHE", "DB_POOL_NULL", "GEO_TEST", "REL_VERBOSE", "REL_LOUDLISTEN", "MEMCACHE_REQUEST", "MEMCACHE_DB", "PUBSUB_ECHO", "PUBSUB_META", "PUBSUB_B64", "SSL_VERIFY", "ADMIN_MONITOR_LOG", "WEB_XORIGIN", "LOG_TIMESTAMP", "BUILD_PROD_CLOSURE", "BUILD_PROD_B64", "GMAILER", "MAILHTML", "MAILOUD"]:
+		val = val == "True"
+	elif key in ["PUBSUB_HISTORY", "MEMPAD", "WEB_SHIELD_CHUNK", "WEB_SHIELD_LIMIT", "WEB_SHIELD_INTERVAL", "MEMCACHE_PROX_TIMEOUT", "DB_POOL_SIZE", "DB_POOL_OVERFLOW", "DB_POOL_RECYCLE", "LOG_OPENFILES", "LOG_TRACEMALLOC", "MAILSCANTICK"]:
+		val = int(val)
+	elif key in ["REL_SLEEP", "REL_TURBO"]:
+		val = float(val)
+	if key == "DB":
+		config.db.update(config.web.server, _getpass(val, "db"))
+	elif key == "DB_TEST":
+		config.db.update("test", _getpass(val, "test db"))
+	elif key == "ADMINPWGAE":
+		# this property only exists for local finagling of a gae database (datastore).
+		# this is _not_ for production and frankly should _not_ even be committed to
+		# a repository. use with care!
+		config.admin.update("pw", val)
+	else:
+		if key == "SCRAMBLER":
+			config.update("customscrambler", True)
+		target = key.lower()
+		c = config
+		if target in ["pubsub_botnames", "log_allow", "geo_user_geonames", "geo_user_google", "admin_contacts", "admin_reportees", "web_rollz", "admin_monitor_geo", "build_dependencies", "build_exclude", "build_include", "build_notjs", "web_whitelist", "admin_whitelist", "web_blacklist", "web_eflags"]: # TODO: use default list detection instead
+			val = val.split("|")
+			if target == "web_rollz":
+				rollz = {}
+				for v in val:
+					flag, domain = v.split(":")
+					rollz[flag] = domain
+				val = rollz
+		if "_" in target:
+			path, target = target.rsplit("_", 1)
+			for part in path.split("_"):
+				c = c.sub(part)
+		c.update(target, val)
+
+config.update("cache", pc)
+config.db.update("main", config.db[config.web.server])
+for prop in ["deep", "flush", "timestamp", "allow"]:
+	confyg.log.update(prop, config.log[prop])
+for prop in ["cache", "refcount", "main", "test", "blob", "alter", "echo"]:
+	dbcfg.update(prop, config.db[prop])
+for prop in ["null", "size", "recycle", "overflow"]:
+	dbcfg.pool.update(prop, config.db.pool[prop])
+
+# set protocol based on certs
+if config.ssl.certfile:
+	config.web.update("protocol", "https")
+	config.admin.update("protocol", "https")
+
+# extract mailer name from MAILER
+if config.mailer and " <" in config.mailer:
+	mn, m = config.mailer[:-1].split(" <")
+	config.update("mailer", m)
+	config.update("mailername", mn)
+
+def mod_and_repo(plug):
+	repo = "/" in plug and plug or "%s/%s"%(config.plugin.base, plug)
+	if repo == plug:
+		plug = plug.split("/")[1]
+	return plug, repo
+
+def mods_and_repos(plugs):
+	mz, rz = [], [] # slicker way to do this?
+	for plug in plugs:
+		m, r = mod_and_repo(plug)
+		mz.append(m)
+		rz.append(r)
+	return mz, rz
+
+def include_plugin(plug):
+	mod, repo = mod_and_repo(plug)
+	config.plugin.repos.append(repo)
+	config.plugin.modules.append(mod)

cantools/db/__init__.py

@@ -0,0 +1,18 @@
+from ..config import config
+
+# later, switch these imports on 'db' instead of 'web.server'
+if config.web.server == "gae":
+	from .gae.model import *
+elif config.web.server == "dez":
+	from rel import tick
+	from databae import *
+	from cantools.web import set_pre_close, cgi_dump
+	def scoper(threadId):
+		if threadId == "MainThread":
+			threadId = tick()
+		return "%s%s"%(threadId, cgi_dump())
+	set_scoper(scoper)
+	set_pre_close(seshman.close)
+else:
+	from fyg.util import error
+	error("no data backend specified")

cantools/db/admin.py

@@ -0,0 +1,27 @@
+from databae import get_model, get_schema, put_multi
+from fyg.util import log
+
+def index(kind, i=0): # be careful with this!
+	kinds = kind == "*" and list(get_schema().keys()) or [kind]
+	puts = []
+	for kind in kinds:
+		mod = get_model(kind)
+		schema = get_schema(kind)
+		q = mod.query()
+		for prop in ["created", "modified", "date"]:
+			if prop in schema:
+				q.order(getattr(mod, prop))
+				break
+		items = q.fetch() # gae doesn't support all()...
+		log("assigning sequential index properties to %s %s records"%(len(items), kind), important=True)
+		for n in range(len(items)):
+			item = items[n]
+			i += 1
+			item.index = i
+			if n and not n % 100:
+				log("processed %s %s entities"%(n, kind), 1)
+		log("processed %s %s entities"%(len(items), kind))
+		puts += items
+	log("saving %s records"%(len(puts),), important=True)
+	put_multi(puts)
+	return i

cantools/db/gae/model.py

@@ -0,0 +1,127 @@
+from datetime import datetime
+from .properties import *
+from six import with_metaclass
+
+class CTMeta(ndb.MetaModel):
+    def __new__(cls, name, bases, attrs):
+        lname = name.lower()
+        if lname != "modelbase":
+            if "label" not in attrs:
+                for label in ["name", "title"]:
+                    if label in attrs:
+                        attrs["label"] = label
+                        break
+                if "label" not in attrs:
+                    attrs["label"] = "key"
+            schema = attrs["_schema"] = merge_schemas(bases, attrs["label"])
+            for key, val in list(attrs.items()):
+                if getattr(val, "_ct_type", None):
+                    schema[key] = val._ct_type
+                    if val._ct_type == "key":
+                        schema["_kinds"][key] = getattr(val, "_kinds", "*") # always * for now...
+        modelsubs[lname] = super(CTMeta, cls).__new__(cls, name, bases, attrs)
+        return modelsubs[lname]
+
+class ModelBase(with_metaclass(CTMeta, ndb.Model)):
+    index = Integer()
+
+    def __eq__(self, other):
+        return self.id() == (other and hasattr(other, "id") and other.id())
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return 0 # ensures proper set-uniquification
+
+    def put(self):
+        if not self.index:
+            self.index = self.__class__.query().count() + 1
+        super(ModelBase, self).put()
+
+    def rm(self):
+        self.key.delete()
+
+    def collection(self, entity_model, property_name, fetch=True, keys_only=False, data=False):
+        q = entity_model.query(ndb.GenericProperty(property_name) == self.key)
+        if not fetch:
+            return q
+        if not data:
+            return q.fetch(1000, keys_only=keys_only)
+        return [d.data() for d in q.fetch(1000)]
+
+    def modeltype(self):
+        return self.__class__.__name__.lower()
+
+    def id(self):
+        return self.key.urlsafe()
+
+    def export(self):
+        cols = {}
+        mt = self.modeltype()
+        cols["key"] = self.id()
+        cols["ctkey"] = ct_key(mt, self.index)
+        cols["oldkey"] = str(self.key.to_old_key())
+        cols["index"] = self.index
+        cols["modelName"] = mt
+        cols["_label"] = self.label
+        for cname, prop in list(self._schema.items()):
+            if not cname.startswith("_"):
+                val = getattr(self, cname)
+                if prop.startswith("key"):
+                    if type(val) is list:
+                        val = [v.urlsafe() for v in val]
+                    elif hasattr(val, "urlsafe"):
+                        val = val.urlsafe()
+                elif prop == "blob":
+                    val = bool(val)
+                elif val and prop == "datetime":
+                    val = str(val)[:19]
+                cols[cname] = val
+        cols["label"] = cols[self.label] or "%s %s"%(mt, self.index)
+        return cols
+
+def getall(entity=None, query=None, keys_only=False):
+    cursor = None
+    ents = []
+    while True:
+        q = query or entity.query()
+        batch, cursor, more = q.fetch_page(1000, start_cursor=cursor, keys_only=keys_only)
+        ents += batch
+        if len(batch) < 1000:
+            break
+    return ents
+
+def get(key):
+    return Key(urlsafe=key).get()
+
+def get_page(modelName, limit, offset, order='index', filters={}):
+    schema = get_schema(modelName)
+    mod = get_model(modelName)
+    query = mod.query()
+    for key, obj in list(filters.items()):
+        val = obj["value"]
+        comp = obj["comparator"]
+        prop = getattr(mod, key)
+        if schema[key] == "key" and not isinstance(val, KeyWrapper):
+            val = KeyWrapper(val)
+        elif schema[key] == "datetime" and not isinstance(val, datetime):
+            val = datetime.strptime(val, "%Y-%m-%d %H:%M:%S")
+        if comp == "like":
+            query.filter(func.lower(prop).like(val.lower()))
+        else:
+            query.filter(operators[comp](prop, val))
+    return [d.export() for d in query.order(order).fetch(limit, offset=offset)]
+
+def edit(data):
+    ent = "key" in data and get(data["key"]) or get_model(data["modelName"])()
+    for propname, val in list(data.items()):
+        if propname in ent._schema:
+            if val:
+                if propname in ent._schema["_kinds"]: # foreignkey
+                    val = KeyWrapper(val)
+                elif ent._schema[propname] == "datetime" and not isinstance(val, datetime):
+                    val = datetime.strptime(val, "%Y-%m-%d %H:%M:%S")
+            setattr(ent, propname, val)
+    ent.put()
+    return ent

cantools/db/gae/properties.py

@@ -0,0 +1,35 @@
+from google.appengine.ext import ndb
+from databae.util import *
+
+# column properties
+Integer = ndb.IntegerProperty
+Float = ndb.FloatProperty
+Boolean = ndb.BooleanProperty
+String = ndb.StringProperty
+Text = ndb.TextProperty
+Binary = ndb.BlobProperty
+Date = ndb.DateProperty
+Time = ndb.TimeProperty
+DateTime = ndb.DateTimeProperty
+ForeignKey = ndb.KeyProperty
+
+# ctypes
+Integer._ct_type = "integer"
+Float._ct_type = "float"
+Boolean._ct_type = "boolean"
+String._ct_type = "string"
+Text._ct_type = "text"
+Binary._ct_type = "blob"
+Date._ct_type = "date"
+Time._ct_type = "time"
+DateTime._ct_type = "datetime"
+ForeignKey._ct_type = "key"
+
+# entity keys
+Key = ndb.Key
+KeyWrapper = ndb.Key
+
+# funcs
+get_multi = ndb.get_multi
+put_multi = ndb.put_multi
+delete_multi = ndb.delete_multi

cantools/db/wp.py

@@ -0,0 +1,99 @@
+import pymysql, time
+from fyg.util import log, read, error
+
+class DBWrapper(object):
+	def __init__(self):
+		self._h, self._u, self._p, self._d = read(".c").strip().split("|")
+
+	def db(self):
+		try:
+			self._db.ping()
+		except:
+			self._db = pymysql.connect(host=self._h,
+				user=self._u, passwd=self._p, db=self._d)
+		return self._db
+
+	def query(self, q, fetch=True, silent=False):
+		silent or log("executing query: %s"%(q,))
+		db = self.db()
+		cur = db.cursor()
+		cur.execute(q)
+		if fetch:
+			rowz = cur.fetchall()
+		cur.close()
+		if fetch:
+			return rowz
+
+	def commit(self):
+		self.db().commit()
+
+_db = None
+
+def setdb():
+	global _db
+	if not _db:
+		try:
+			_db = DBWrapper()
+		except:
+			log("NO DATABASE CONNECTION!!!!!", important=True)
+	return _db
+
+setdb()
+
+def trydb(attempts=5, wait=0.4):
+	for attempt in range(attempts):
+		db = setdb()
+		if db:
+			return db
+		log("trydb failed retry #" + attempt, important=True)
+		time.sleep(wait)
+
+def getdb(subdb=False):
+	db = trydb()
+	return subdb and db.db() or db
+
+def dbcommit():
+	getdb().commit()
+
+def query(q, fetch=True, silent=False):
+	return getdb().query(q, fetch, silent)
+
+def memprop(email, prop):
+	res = query("select %s from wp_users where user_email = '%s';"%(prop, email))
+	if not res:
+		error("no matching member: %s"%(email,))
+	return res[0][0]
+
+def memid(email):
+	return memprop(email, "ID")
+
+def getmeta(pid, key):
+	return query("select meta_value from wp_usermeta where user_id = '%s' and meta_key = '%s';"%(pid, key))
+
+def setmeta(pid, key, val):
+	mres = query("select * from wp_usermeta where user_id = '%s' and meta_key = '%s';"%(pid, key))
+	if mres:
+		query("update wp_usermeta set meta_value = '%s' where umeta_id = '%s';"%(val, mres[0][0]), False)
+	else:
+		query("insert into wp_usermeta(user_id,meta_key,meta_value) values(%s,'%s','%s');"%(pid, key, val), False)
+
+def transmeta(pid, key, transformer):
+	mres = query("select * from wp_usermeta where user_id = '%s' and meta_key = '%s';"%(pid, key))
+	mid, uid, k, v = mres[0]
+	val = transformer(v)
+	query("update wp_usermeta set meta_value = '%s' where umeta_id = '%s';"%(val, mid), False)
+
+def bymeta(key, val, props=["user_email"], like=True):
+	cond = (like and "like '%%%s%%'" or "= '%s'")%(val,)
+	q = "select %s from wp_users join wp_usermeta on wp_users.ID = wp_usermeta.user_id where wp_usermeta.meta_key = '%s' and wp_usermeta.meta_value %s;"%(", ".join(props), key, cond)
+	return query(q)
+
+def capability2emails(capability):
+	return [row[0] for row in bymeta("wp_capabilities", capability)]
+
+def id2email(memid):
+	e = query("select user_email from wp_users where ID = '%s';"%(memid,))
+	return e and e[0][0] or "[account %s not found]"%(memid,)
+
+def ip2mems(ip):
+	return [id2email(m[0]) for m in query("select user_id from wp_usermeta where meta_value = '%s';"%(ip,))]

cantools/geo.py

@@ -0,0 +1,188 @@
+import os
+try:
+    from urllib import parse # py3
+except:
+    import urllib as parse # py2.7
+from cantools.web import fetch
+from cantools.util import log, read, write, writejson
+from cantools import config
+
+zcpath = os.path.join("logs", "json", "geo")
+
+class Geo(object):
+	apis = {
+		"latlng": {
+			"user": "google",
+			"host": "maps.googleapis.com",
+			"path": "/maps/api/geocode/json?sensor=false&address={0}",
+			"sig": "key",
+			"property": "results",
+			"index": 0
+		},
+		"where": {
+			"user": "google",
+			"host": "maps.googleapis.com",
+			"path": "/maps/api/geocode/json?sensor=false&address={0}",
+			"sig": "key",
+			"property": "results",
+			"index": 0
+		},
+		"zip": {
+			"geonames": {
+				"user": "geonames",
+				"host": "ws.geonames.org",
+				"path": "/findNearbyPostalCodesJSON?radius=1&lat={0}&lng={1}",
+				"sig": "username",
+				"property": "postalCodes",
+				"index": 0
+			},
+			"google": {
+				"user": "google",
+				"host": "maps.googleapis.com",
+				"path": "/maps/api/geocode/json?sensor=false&latlng={0},{1}",
+				"sig": "key",
+				"property": "results",
+				"index": 0
+			}
+		}[config.geo.zip]
+	}
+
+	def __init__(self):
+		self.wherecache = {} # TODO : persist!
+		self.cache = read("%s.json"%(zcpath,), isjson=True, default={})
+		for addr in self.cache:
+			self.cache[addr]["count"] = 0
+
+	def _no_cache(self, addr):
+		return not (addr in self.cache and self.cache[addr]["lat"] and self.cache[addr]["lng"])
+
+	def _fetch(self, api, *args): # api is latlng/zip
+		path = self.apis[api]["path"].format(*[parse.quote(str(a).replace(" ", "+")) for a in args])
+		host = self.apis[api]["host"]
+		user = self.apis[api]["user"]
+		prop = self.apis[api]["property"]
+		keys = config.geo.user[user]
+		onum = num = self.apis[api]["index"]
+		sig = self.apis[api]["sig"]
+		kwargs = { "asjson": True }
+		result = []
+		while True:
+			fullpath = path
+			if keys[num]:
+				fullpath += "&%s=%s"%(sig, keys[num])
+				if user == "google":
+					kwargs["protocol"] = "https"
+			raw = fetch(host, fullpath, **kwargs)
+			result = raw.get(prop, [])
+			if len(result):
+				break
+			log("0-length %s result (got: '%s') - changing user"%(api, raw), important=True)
+			num = (num + 1) % len(keys)
+			if num == onum:
+				log("exhausted key list -- returning [] :'(")
+				break
+		self.apis[api]["index"] = num
+		return result
+
+	def where(self, place):
+		place = place.title()
+		if place not in self.wherecache:
+			if config.geo.where.nom:
+				res = fetch("https://nominatim.openstreetmap.org/search", asjson=True, qsp={
+					"limit": 1,
+					"format": "json",
+					"q": place.replace(" ", "%%20")
+				}, fakeua=config.geo.where.ua)
+				if res:
+					places = res[0]["display_name"].split(", ")
+					curplace = places.pop(0)
+					while places:
+						self.wherecache[curplace] = places[0]
+						curplace = places.pop(0)
+					self.wherecache[curplace] = "earth"
+				else:
+					self.wherecache[place] = "earth"
+			else:
+				results = self._fetch("where", place)
+				if not results:
+					return "space"
+				places = results[0]["address_components"]
+				self.wherecache[place] = len(places) > 1 and places[1]["long_name"] or "earth"
+		return self.wherecache[place]
+
+	def address2latlng(self, address):
+		if self._no_cache(address):
+			log("finding lat/lng for %s"%(address,), 3)
+			results = self._fetch("latlng", address)
+			if not len(results):
+				log("no results!!!", 4)
+				self.cache[address] = {
+					"lat": None,
+					"lng": None
+				}
+			else:
+				loc = results[0]['geometry']['location']
+				self.cache[address] = {
+					"lat": loc["lat"],
+					"lng": loc["lng"]
+				}
+			self.savecache()
+		return [self.cache[address]['lat'], self.cache[address]['lng']]
+
+	def latlng2zip(self, lat, lng):
+		result = self._fetch("zip", lat, lng)
+		log("finding zip for lat %s and lng %s"%(lat, lng), 3)
+		if not len(result):
+			log("can't find zipcode!!!", important=True)
+			return None
+		if config.geo.zip == "google":
+			return [c["short_name"] for c in result[0]["address_components"] if c["types"][0] == "postal_code"][0]
+		return result[0]["postalCode"]
+
+	def addr2zip(self, addr, allowNone=False):
+		log("finding zip for '%s'"%(addr,), 3)
+		if config.geo.test:
+			log("test mode! returning nonsense :)")
+			return '12345'
+		if self._no_cache(addr):
+			self.address2latlng(addr)
+		d = self.cache[addr]
+		if not d.get("zip", allowNone):
+			d["count"] = 1
+			d["zip"] = d["lat"] and self.latlng2zip(d["lat"], d["lng"])
+			self.savecache()
+		else:
+			d["count"] += 1
+			log("address referenced %s times"%(d["count"],), 4)
+		log("found zip: %s"%(d["zip"],), 4)
+		return d["zip"]
+
+	def savecache(self, pretty=False):
+		log("saving zipcode / latlng (geo) cache (pretty: %s)"%(pretty,), important=True)
+		if pretty:
+			writejson(self.cache, zcpath)
+		else:
+			write(self.cache, "%s.json"%(zcpath,), True)
+
+	def distance(self, lat1, lng1, lat2, lng2):
+		log("approximating distance between [%s %s] and [%s %s]"%(lat1, lng1, lat2, lng2))
+		latd = lat1 - lat2
+		lngd = lng1 - lng2
+		latsq = latd ** 2
+		lngsq = lngd ** 2
+		tsq = latsq + lngsq
+		hypot = tsq**(1.0/2)
+		miles = hypot * 69.11;
+		log("lat diff: %s square: %s"%(latd, latsq))
+		log("lng diff: %s square: %s"%(lngd, lngsq))
+		log("sum of squares: %s hypotenuse: %s"%(tsq, hypot))
+		log("about %s miles"%(miles,))
+		return miles
+
+geo = Geo()
+where = geo.where
+address2latlng = geo.address2latlng
+latlng2zip = geo.latlng2zip
+addr2zip = geo.addr2zip
+savecache = geo.savecache
+distance = geo.distance

cantools/hooks.py

@@ -0,0 +1,13 @@
+class HookBox(object):
+	def __init__(self):
+		self.hooks = []
+
+	def __call__(self, data):
+		for hook in self.hooks:
+			hook(data)
+
+	def register(self, cb):
+		self.hooks.append(cb)
+
+memhook = HookBox()
+dbhook = HookBox()