ct 0.10.8.114__py3-none-any.whl

cantools/util/data.py

@@ -0,0 +1,109 @@
+def getxls(data, name=None, parse=True):
+	import xlrd, datetime
+	wb = xlrd.open_workbook(file_contents=data)
+	s = wb.sheet_by_name(name or wb.sheet_names()[0])
+	if not parse:
+		return s
+	d = [[c.value for c in s.row(0)]]
+	for r in range(1, s.nrows):
+		row = s.row(r)
+		rl = []
+		for c in row:
+			if c.ctype == 3: # date
+				rl.append(datetime.datetime(*xlrd.xldate_as_tuple(c.value, wb.datemode)))
+			else:
+				rl.append(c.value)
+		d.append(rl)
+	return d
+
+def _svlines(data):
+	return data.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+
+def gettsv(data):
+	return [l.split("\t") for l in _svlines(data) if l]
+
+def getcsv_from_data(data, unquote=False): # deprecated! doesn't work _that_ well.
+	return [[unquote and i.strip('"') or i for i in l.split(",")] for l in _svlines(data) if l]
+
+# eh, module reads it better sometimes (especially if we write(_svlines(read())) first)
+def getcsv(fname):
+	import csv
+	d = []
+	f = open(fname, "r")
+	reader = csv.reader(f)
+	for row in reader:
+		d.append(row)
+	f.close()
+	return d
+
+def flatten(obj):
+	keys = []
+	vals = []
+	for key, val in list(obj.items()):
+		if isinstance(val, dict):
+			subkeys, subvals = flatten(val)
+			for subkey in subkeys:
+				fullsubkey = "%s.%s"%(key, subkey)
+				if fullsubkey not in keys:
+					keys.append(fullsubkey)
+			vals += subvals
+		else:
+			if key not in keys:
+				keys.append(key)
+			vals.append(val)
+	return keys, vals
+
+def arr2csv(arr):
+	return "\n".join([",".join(flatten(arr[0])[0])] + [",".join([str(i) for i in flatten(obj)[1]]) for obj in arr])
+
+def token(n=10):
+	import random, string
+	samp = string.ascii_letters + string.digits
+	pop = len(samp)
+	s = ""
+	while n:
+		chunk = min(n, pop)
+		s += "".join(random.sample(samp, chunk))
+		n -= chunk
+	return s
+
+# below (props, drower, tsv) for tsv dumping
+def props(mod):
+	from model import db
+	s = db.get_schema(mod)
+	return filter(lambda p : p not in mod._data_omit and p not in ["index",
+		"key"] and not p.startswith("_") and s[p] != "blob", s.keys())
+
+def dcell(e, p):
+	v = getattr(e, p)
+	t = e._schema[p]
+	if t == "key":
+		v = v and v.get().labeler() or None
+	elif t == "keylist":
+		from model import db
+		v = v and [i.labeler() for i in db.get_multi(v)] or []
+	return '"%s"'%(str(v),)
+
+def drower(e, pz):
+	return map(lambda p : dcell(e, p), pz)
+
+def rgen(ents, pz, rower=None):
+	rower = rower or drower
+	return [pz] + map(lambda e : rower(e, pz), ents)
+
+def tsv(ents, pz, rower=None):
+	return "\n".join(map(lambda r : "\t".join(r),
+		rgen(ents, pz, rower)))
+
+def ez2csv(ents, pz, rower=None):
+	return "\n".join(map(lambda r : ",".join(r),
+		rgen(ents, pz, rower)))
+
+def spreadsheet(fmat, ents, pz, rower=None):
+	return (fmat == "tsv" and tsv or ez2csv)(ents, pz, rower)
+
+def rgb2hex(rgbstr="rgb(224, 62, 45)"):
+	return '%02x%02x%02x' % tuple([int(c) for c in rgbstr[4:-1].split(", ")])
+
+def hex2rgb(h="#ffaabb"):
+	return "rgb%s"%(tuple(int(h[i:i+2], 16) for i in (1, 3, 5)),)

cantools/util/media.py

@@ -0,0 +1,303 @@
+from __future__ import absolute_import # for io.BytesIO
+from io import BytesIO
+import os, magic
+from fyg.util import log, read, write, rm
+from .system import cmd, output, cp, mkdir
+try:
+	from PIL import Image
+except:
+	pass#print("no PIL.Image")
+
+#
+# general
+#
+_ii = 0
+def get(url, ext="jpg", pref=""):
+	global _ii
+	from cantools.web import fetch
+	_ii += 1
+	f = "%s%s.%s"%(pref, _ii, ext)
+	cp(fetch(url), f)
+	return f
+
+def dl(ilist, ext="jpg", pref=""):
+	fnames = []
+	for url in read(ilist).split("\n"):
+		if " " in url:
+			fnames.append([get(u, ext, pref) for u in url.split(" ")])
+		else:
+			fnames.append(get(url, ext, pref))
+	return fnames
+
+def o2f(outname, inames, ext="jpg", pref=""):
+	if outname.endswith(".list"):
+		inames = dl(outname, ext, pref)
+		outname = outname.split(".").pop(0)
+	return outname, inames
+
+#
+# video (ffmpeg)
+#
+
+def dlp(url, opath=None, tarform="mp4", size="+size"):
+	ytc = "yt-dlp -f %s -S %s"%(tarform, size)
+	if opath:
+		ytc = "%s -o %s"%(ytc, opath)
+	cmd("%s %s"%(ytc, url))
+
+def thumb(vname, src=".", dest=".", vext="mp4", forceDest=False, overwrite=False):
+	if "." in vname:
+		vpath = vname
+		vname = vname.split(".")[0]
+	else:
+		vpath = "%s/%s.%s"%(src, vname, vext)
+	dpath = os.path.join(dest, vname.rsplit("/", 1)[0])
+	dname = "%s/%s.jpg"%(dest, vname)
+	if forceDest:
+		os.path.isdir(dpath) or mkdir(dpath, True)
+	if overwrite or not os.path.exists(dname):
+		cmd('ffmpeg -i %s -vf "thumbnail" -frames:v 1 %s'%(vpath, dname))
+
+def concat(outname, *vnames):
+	outname, vnames = o2f(outname, vnames, "mp4")
+	flist = "\n".join(map(lambda vn : "file %s"%(vn,), vnames))
+	cp(flist, "files.list")
+	cmd("ffmpeg -f concat -i files.list -c copy %s.mp4"%(outname,))
+
+TRANS = "ffmpeg -y -i %s -loglevel error -stats -c:a aac -movflags +faststart -f mp4 _tmp"
+BASELINE = "ffmpeg -y -i %s -loglevel error -stats -c:a aac -profile:v baseline -level 3.0 -movflags +faststart -f mp4 _tmp"
+SLOW = "ffmpeg -y -i %s -loglevel error -stats -c:v libx264 -preset veryslow -c:a aac -movflags +faststart -f mp4 _tmp"
+FAST = "ffmpeg -y -i %s -loglevel error -stats -c:v copy -c:a copy -movflags +faststart -f mp4 _tmp"
+#SEG = "ffmpeg -i %s -loglevel error -stats -map 0 -codec:v libx264 -codec:a aac -f ssegment -segment_list %s/list.m3u8 -segment_list_flags +live -segment_time 10 %s/%%03d.ts"
+SEG = 'ffmpeg -i %s -loglevel error -stats -c:v libx264 -c:a copy -r 30 -x264opts "keyint=60:min-keyint=60" -forced-idr 1 -f ssegment -segment_list %s/list.m3u8 -segment_time 10 %s/%%03d.ts'
+MOOV = "ffmpeg -v trace -i %s 2>&1 | grep -e \"'mdat' parent\" -e \"'moov' parent\""
+
+def shouldMoveMoov(fpath):
+	return "moov" in output(MOOV%(fpath,)).split("\n").pop()
+
+def transcode(orig, tmp=False, fast=True, slow=False, baseline=False):
+	if tmp: # orig is _data_, not _path_
+		log("media.transcode > writing to tmp file", 1)
+		data = orig
+		orig = "_tctmp"
+		write(data, orig, binary=True)
+	log("media.transcode > optimizing for mobile (%s)"%(orig,), 1)
+	cmd((baseline and BASELINE or slow and SLOW or fast and FAST or TRANS)%(orig,))
+	data = read(binary=True)
+	if not tmp:
+		log("media.transcode > overwriting original; removing tmp file", 1)
+		write(data, orig, binary=True)
+	rm("_tmp")
+	if tmp:
+		rm(orig)
+		return data
+
+def segment(orig, p):
+	log("media.segment > segmenting to %s (hls)"%(p,), 1)
+	cmd(SEG%(orig, p, p))
+
+def hlsify(blobpath, check=False):
+	p = blobpath.replace("blob/", "blob/hls/")
+	isd = os.path.isdir(p)
+	if check:
+		return isd
+	if not isd:
+		log("transcode > attempt with video: '%s'"%(blobpath,), important=True)
+		mkdir(p, True)
+		segment(blobpath, p)
+		log("transcode > done!")
+
+#
+# audio (ffmpeg)
+#
+
+def repitch(fname, pitch, ext="mp3"):
+	log("repitch(%s, %s, %s)"%(pitch, fname, ext))
+	cmd('ffmpeg -i %s.%s -af "rubberband=pitch=%s" %s-tmp.%s'%(fname,
+		ext, pitch, fname, ext))
+	cmd("mv %s-tmp.%s %s.%s"%(fname, ext, fname, ext))
+
+def mp3ize(ext=".wav", path="."):
+	fnames = os.listdir(path)
+	log("you asked for it! scanning %s files for %s"%(len(fnames), ext), important=True)
+	for fname in fnames:
+		if fname.endswith(ext):
+			cmd('ffmpeg -i "%s" "%s.mp3"'%(fname, fname[:-4]))
+
+#
+# images (identify, convert, montage, PIL)
+#
+
+class Tiler(object):
+	def __init__(self, images, outname, vertical=True):
+		self.vertical = vertical
+		self.outname = outname
+		self.images = images
+		self.final = []
+		self.geos = {}
+		self.reduce()
+		self.assess()
+		self.resize()
+
+	def dims(self, ipath):
+		xy = output("identify %s"%(ipath,)).split(" ")[2].split("x")
+		return {
+			"x": int(xy[0]),
+			"y": int(xy[1])
+		}
+
+	def reduce(self):
+		for i in range(len(self.images)):
+			iline = self.images[i]
+			if type(iline) == list:
+				newname = "".join([n.split(".").pop(0) for n in iline])
+				Tiler(iline, newname, not self.vertical).render()
+				self.images[i] = "%s.jpg"%(newname,)
+
+	def assess(self):
+		dim = self.vertical and "x" or "y"
+		self.smallest = None
+		for image in self.images:
+			self.geos[image] = self.dims(image)
+			if not self.smallest or self.geos[image][dim] < self.geos[self.smallest][dim]:
+				self.smallest = image
+
+	def _resizer(self):
+		sdim = str(self.geos[self.smallest][self.vertical and "x" or "y"])
+		resizer = ['convert "%s" -resize']
+		resizer.append(self.vertical and sdim or ("x" + sdim))
+		resizer.append('"%s"')
+		return " ".join(resizer)
+
+	def resize(self):
+		resizer = self._resizer()
+		for image in self.images:
+			if image == self.smallest:
+				self.final.append(image)
+			else:
+				sname = image.replace(".jpg", "s.jpg")
+				cmd(resizer%(image, sname))
+				self.final.append(sname)
+
+	def _renderer(self):
+		montager = ['montage "%s" -geometry +2+2']
+		self.vertical and montager.append("-tile 1x")
+		montager.append('"%s.jpg"')
+		return " ".join(montager)
+
+	def render(self):
+		cmd(self._renderer()%('" "'.join(self.final), self.outname))
+
+def autotile(outname, vertical=True, inames=[]):
+	outname, inames = o2f(outname, inames)
+	Tiler(inames or os.listdir(), outname, vertical).render()
+
+def vtile(outname, *inames):
+	autotile(outname, inames=inames)
+
+def htile(outname, *inames):
+	autotile(outname, False, inames)
+
+def crop(img, constraint):
+	w = img.size[0]
+	h = img.size[1]
+	smaller = min(w, h)
+	if smaller == w: # clean these up...
+		fromx = 0
+		tox = w
+		fromy = (h - w) / 2.0
+		toy = fromy + w
+	else:
+		fromy = 0
+		toy = h
+		fromx = (w - h) / 2.0
+		tox = fromx + h
+	return img.crop((int(fromx), int(fromy), int(tox),
+		int(toy))).resize((constraint, constraint), Image.ANTIALIAS)
+
+def jpgize(path, fromform=None, overwrite=False):
+	opath = path
+	if not overwrite and input("overwrite %s? [Y/n] "%(path,)).lower().startswith("n"):
+		opath = input("ok, what should we call the output file? ")
+	if fromform == "TIFF":
+		Image.open(BytesIO(read(path, binary=True))).save(opath, format="JPEG")
+	else:
+		cmd("convert %s tmp.jpg; mv tmp.jpg %s"%(path, opath))
+
+_p2 = []
+class ImageResizer(object):
+	def __init__(self, img):
+		self.img = img
+		self.high = max(img.size)
+		[self.width, self.height] = img.size
+		log("loaded with size: %sx%s"%img.size)
+
+	def closest(self, n):
+		for p in _p2:
+			if n >= p:
+				return p
+
+	def s2p2(self):
+		divisor = self.high / self.closest(self.high)
+		return (self.closest(self.width / divisor), self.closest(self.height / divisor))
+
+	def resize(self):
+		dims = self.s2p2()
+		log("resizing to: %sx%s"%dims)
+		rimg = self.img.resize(dims)
+		with BytesIO() as outbytes:
+			rimg.save(outbytes, format=self.img.format)
+			return outbytes.getvalue()
+
+def _initp2():
+	if not _p2:
+		n = 16
+		while n <= 1024:
+			_p2.append(n)
+			n *= 2
+		_p2.reverse()
+
+def resizep2(data):
+	_initp2()
+	return ImageResizer(Image.open(BytesIO(data))).resize()
+
+def p2(path, overwrite=False):
+	opath = path
+	resized = resizep2(read(path, binary=True))
+	if not overwrite and input("overwrite %s? [Y/n] "%(path,)).lower().startswith("n"):
+		opath = input("ok, what should we call the output file? ")
+	write(resized, opath, binary=True)
+
+def imeta(f, silent=False):
+	_initp2()
+	magstr = magic.from_file(f)
+	if "image" not in magstr:
+		return
+	fm = magstr.split(" ").pop(0)
+	d = {}
+	m = {
+		"format": fm,
+		"dims": d
+	}
+	w = None
+	h = None
+	if fm == "JPEG":
+		[w, h] = magstr.split(", ")[-2].split("x")
+	elif fm == "PNG":
+		[w, h] = magstr.split(", ")[1].split(" x ")
+	elif fm == "TIFF":
+		parts = magstr.split("=")
+		[w, h] = [parts[-1], parts[2].split(", ").pop(0)]
+	if w:
+		d["width"] = w = int(w)
+		d["height"] = h = int(h)
+	m["p2"] = w in _p2 and h in _p2
+	silent or log(m)
+	return m
+
+def scanp2(overwrite=False):
+	for f in os.listdir():
+		m = imeta(f, True)
+		if m:
+			print(f, m)
+			overwrite and not m["p2"] and p2(f)

cantools/util/package.py

@@ -0,0 +1,125 @@
+import os, sys, sysconfig, importlib
+from cantools.util import cmd, log, pymod, read, write
+
+def managedpip():
+    fname = os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")
+    print("checking for", fname)
+    isext = os.path.isfile(fname)
+    print("found:", isext)
+    return isext
+
+def virtenv():
+    invenv = sys.prefix != sys.base_prefix
+    print("virtual environment:", invenv)
+    return invenv
+
+def pipper(execute=False, force=False):
+    p = "pip install -e ."
+    valid = True
+    notvenv = not virtenv()
+    if managedpip() and notvenv:
+        log("your Python is externally managed by your OS")
+        if force or input("install anyway? [y/N] ").lower().startswith("y"):
+            p += " --break-system-packages --use-pep517"
+        else:
+            valid = False
+    def dopip():
+        if valid:
+            pymod(p, notvenv)
+        else:
+            log("pip skip!")
+    if execute:
+        dopip()
+    else:
+        log(p)
+    return dopip
+
+def pushv(version):
+    cmd("git add -u")
+    cmd('git commit -m "vpush %s"'%(version,))
+    cmd("git push")
+
+def upv(oldver, newver, pname, fnames=None):
+    fnames = fnames or [
+        "setup.py",
+        "about.txt",
+        os.path.join(pname, "__init__.py"),
+        os.path.join(pname, "version.py"),
+    ]
+    for fname in fnames:
+        if os.path.exists(fname):
+            log("updating: %s"%(fname,), 1)
+            txt = read(fname)
+            if fname == "setup.py": # extra careful
+                txt = txt.replace('version="%s"'%(oldver,), 'version="%s"'%(newver,))
+            else:
+                txt = txt.replace(oldver, newver)
+            write(txt, fname)
+
+def incv(curver):
+    log("current version: %s"%(curver,))
+    vnumz = [int(n) for n in curver.split(".")]
+    while len(vnumz) < 4:
+        vnumz.append(0)
+    vnumz[-1] += 1
+    minor = ".".join([str(n) for n in vnumz])
+    vnumz[-2] += 1
+    major = ".".join([str(n) for n in vnumz[:-1]])
+    log("How should we increment? You have three options:", important=True)
+    log("Major ('M' - %s)"%(major,), 1)
+    log("Minor ('m' - %s)"%(minor,), 1)
+    log("Custom ('c')", 1)
+    selection = input("So? (M/m/c - default: m): ")
+    if selection == "c":
+        version = input("ok, then. what's the new version #? ")
+    elif selection == "M":
+        version = major
+    else: # default (minor)
+        version = minor
+    log("going with: %s"%(version,), important=True)
+    return version
+
+def pwheel(pname, version, withCheese=False):
+    log("laying egg (universal py2/3 wheel)", important=True)
+    pymod("build -n", True)
+    if withCheese or input("push to cheese shop? [N/y] ").lower().startswith("y"):
+        log("pushing to cheese shop", important=True)
+        cmd("twine upload dist/%s-%s-py3-none-any.whl"%(pname, version))
+
+def here():
+    return os.path.abspath(".").split(os.path.sep)[-1]
+
+def getmod():
+    h = here()
+    return os.path.isdir(h) and h or input("module name? ")
+
+def getver(modname=None):
+    v = importlib.import_module(modname or getmod()).__version__
+    print(v)
+    return v
+
+def vpush(modname=None, packname=None, curver=None, doxxer=None):
+    modname = modname or getmod()
+    curver = curver or getver(modname)
+    log("vpushin %s!"%(modname,), important=True)
+    version = incv(curver)
+    upv(curver, version, modname)
+    if doxxer:
+        doxxer()
+    elif input("generate documentation? [y/N] ").lower().startswith("y"):
+        cmd("ctdoc")
+    else:
+        print("okay, i'll just update your README.md directly")
+        upv(curver, version, modname, ["README.md"])
+    pushv(version)
+    pwheel(packname or modname, version)
+    log("we did it (%s -> %s)!"%(curver, version), important=True)
+
+def refresh_plugins():
+    pcmd = pipper()
+    def refresher():
+        cmd("git pull")
+        pcmd()
+    from cantools.util.admin import plugdirs
+    plugdirs(refresher, input("search directory? [default: .ctplug] ") or ".ctplug",
+        input("filter by preface? [suggested: ct] "))

cantools/util/system.py

@@ -0,0 +1,73 @@
+import os, sys, subprocess, platform
+try:
+	from commands import getoutput   # py2
+except:
+	from subprocess import getoutput # py3
+from fyg.util import log, read, write, rm
+
+def cp(content, fname): # more write than copy, buuuut...
+	log("writing %s"%(fname,), 2)
+	write(content, fname, binary=hasattr(content, "decode"))
+
+def _init_win_sym():
+	def win_sym(src, dest):
+		mode = os.path.isdir(src) and "J" or "H"
+		cmd("mklink /%s %s %s"%(mode, dest, src))
+	os.symlink = win_sym
+
+def sym(src, dest, safe=False):
+	log("symlinking %s to %s"%(src, dest), 2)
+	if not hasattr(os, "symlink"):
+		_init_win_sym()
+	try:
+		os.symlink(src, dest)
+	except Exception as e:
+		log("symlinking failed (%s) - file exists: %s"%(e, dest), 3)
+		if not safe:
+			try:
+				rm(dest)
+				sym(src, dest, True)
+			except:
+				log("ok, really failed - skipping", 3)
+
+def mkdir(pname, recursive=False):
+	log("new directory: %s"%(pname,), 2)
+	if recursive:
+		os.makedirs(pname)
+	else:
+		os.mkdir(pname)
+
+def sed(fname, flag, replacement, target=None):
+	write(read(fname).replace(flag, replacement), target or fname)
+
+def sudoed(cline, sudo=False):
+	if sudo and platform.system() != "Windows" and os.geteuid(): # !root
+		cline = "sudo %s"%(cline,)
+	return cline
+
+def cmd(cline, sudo=False, silent=False):
+	cline = sudoed(cline, sudo)
+	silent or log('issuing command: "%s"'%(cline,), 2)
+	subprocess.call(cline, shell=True)
+
+def output(cline, sudo=False, silent=False, loud=False):
+	cline = sudoed(cline, sudo)
+	silent or log('getting output for: "%s"'%(cline,), 2)
+	output = getoutput(cline)
+	loud and log(output)
+	return output
+
+def envget(name):
+	return output("echo $%s"%(name,))
+
+def envset(name, val):
+	os.environ[name] = val
+	#cmd("export %s=%s"%(name, val))
+
+PYVER = sys.version_info[0] == 2 and "python" or "python3"
+
+def py(cline, sudo=False):
+	cmd("%s %s"%(PYVER, cline), sudo)
+
+def pymod(mod, sudo=False):
+	py("-m %s"%(mod,), sudo)

cantools/web/__init__.py

@@ -0,0 +1,9 @@
+from .util import config
+
+if config.web.server == "gae":
+	from .gae_server import *
+elif config.web.server == "dez":
+	from cantools.web.dez_server import *
+else:
+	from cantools import util
+	util.error("no web server specified")

cantools/web/dez_server/__init__.py

@@ -0,0 +1 @@
+from .server import *