pydan 1.7__tar.gz

usr/local/lib/python3.8/dist-packages/pydan/jdata.py

@@ -0,0 +1,471 @@
+#!/usr/bin/env python3
+
+# Data Helpers
+
+import re
+import collections
+import os
+import sys
+import io
+import binascii
+import random
+from datetime import datetime,timezone
+import base64
+import platform # platform.node() -> hostname
+# for yaml
+#from ruamel.yaml.comments import CommentedMap
+import ruamel.yaml
+# for json
+import json
+# for xml
+import dicttoxml
+# self tools
+from pydan import run
+
+# repair LANG=C -> utf8
+if(sys.getfilesystemencoding()=="ascii" and sys.getdefaultencoding()=="utf-8"):
+	sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
+
+c={
+	"none":"\033[0m",
+	"header":"\033[38;5;166m", # orange
+	"group":"\033[1;34m",      # light blue
+	"item":"\033[0;36m",       # cian
+	"null":"\033[1;30m",       # gray
+	"date":"\033[38;5;105m",   # xviolet
+	"str":"\033[0;32m",        # green
+	"int":"\033[1;32m",        # light green
+	"float":"\033[38;5;112m",  # xolive
+	"bool":"\033[1;36m",       # white
+	"unk":"\033[0;31m",        # red
+	"varint":"\033[1;35m",     # magentab
+	"varenv":"\033[0;35m",     # magenta
+	"varcmd":"\033[1;31m",     # red
+	"err":"\033[1;31m",        # light red
+	"important":"\033[1;33m",  # light red
+}
+
+#{{{ colprint: Imprime un dict por pantalla con colores según el tipo de datos
+def colprint(d,header=None,key=None,indent="",crop=True):
+	indentchars="    "
+	if(header): print(c["header"]+header+c["none"]);indent=indent+indentchars
+
+	# Si es diccionario iteramos
+	if(type(d)==dict or type(d)==collections.OrderedDict): # or type(d)==CommentedMap):
+	#if(type(d)==dict or type(d)==collections.OrderedDict or type(d)==CommentedMap):
+	#if hasattr(d,'__iter__') and type(d)!=str:
+		if(key!=None): print(indent+c["group"]+key+":"+c["none"]); indent+=indentchars
+		if(len(d)==0): print(indent+c["null"]+"empty"+c["none"])
+		else:
+			for k in d: colprint(d[k], key=k, indent=indent)
+		return
+
+	# Si es lista listamos con [indice]
+	#if(type(d)==list):
+	#	i=0
+	#	print(indent+c["group"]+key+":"+c["none"])
+	#	indent=indent+indentchars
+	#	# Lista de strings
+	#	#if type(key[0])==str:
+	#	#	for k in d:
+	#	#		print(indent+c["item"]+"["+str(i)+"]: "+c["none"], end='')
+
+	#	#for k in d:
+	#	#	#print(indent+c["group"]+key+"["+str(i)+"]:"+c["none"])
+	#	#	#colprint(k, indent=indent+indentchars)
+	#	#	#print(indent+c["item"]+"["+str(i)+"]: "+c["none"], end='')
+	#	#	colprint(k, indent=indent+c["item"]+"["+str(i)+"]: ")
+	#	#	i=i+1
+	#	#	#print(k);
+	#
+	#	for k in d:
+	#		#print(indent+c["group"]+key+"["+str(i)+"]:"+c["none"])
+	#		print(indent+c["group"]+"["+str(i)+"]:"+c["none"])
+	#		colprint(k, indent=indent+indentchars)
+	#	#	#print(indent+c["item"]+"["+str(i)+"]: "+c["none"], end='')
+	#		#colprint(k, indent=indent+c["item"]+"["+str(i)+"]: ")
+	#		i=i+1
+	#		#print(k);
+	#	return
+
+	if type(d)==list or type(d)==tuple:
+		# Vacia
+		if len(d)==0:
+			if(key!=None):
+				print(indent+c["group"]+key+"[]: "+c["null"]+"empty"+c["none"]);
+			else:
+				print(indent+c["null"]+"empty"+c["none"])
+			return
+
+		# Lista de strings o ints
+		if type(d[0])==str or type(d[0])==int:
+			i=0
+			if key!=None:
+				print(indent+c["group"]+key+":"+c["none"])
+				for k in d:
+					colprint(str(k), key="["+str(i)+"]", indent=indent+indentchars)
+					i+=1
+			else:
+				for k in d:
+					colprint(str(k), key="["+str(i)+"]", indent=indent)
+					i+=1
+			return
+
+		#for k in d:
+		#	#print(indent+c["group"]+key+"["+str(i)+"]:"+c["none"])
+		#	#colprint(k, indent=indent+indentchars)
+		#	#print(indent+c["item"]+"["+str(i)+"]: "+c["none"], end='')
+		#	colprint(k, indent=indent+c["item"]+"["+str(i)+"]: ")
+		#	i=i+1
+		#	#print(k);
+
+		# Lista de objetos
+		if(type(d[0])==dict or type(d[0])==collections.OrderedDict):
+			i=0
+			for k in d:
+				print(indent+c["group"]+key+"["+str(i)+"]:"+c["none"])
+				colprint(k, indent=indent+indentchars)
+				i=i+1
+			return
+
+		print(c["err"]+"¿list of "+type(d[0]).__name__+"?"+c["none"])
+		return
+		#print("unknown "+type(d).__name__)
+		#return
+
+	# Mostramos key
+	if(key!=None): print(indent+c["item"]+key+": ", end='')
+	else: print(indent, end='')
+
+	# Mostramos valor
+	t=type(d)
+	if(t==str):
+		p=re.compile("\$([a-zA-Z0-9_]*)\$")
+		vars=set(p.findall(d))
+		for k in vars: d=re.sub("\$"+k+"\$", c["varenv"]+"$"+k+"$"+c["str"], d)
+		p=re.compile("\%([a-zA-Z0-9_]*)\%")
+		vars=set(p.findall(d))
+		for k in vars: d=re.sub("\%"+k+"\%", c["varint"]+"%"+k+"%"+c["str"], d)
+		p=re.compile("\(\(([^)]*)\)\)")
+		vars=set(p.findall(d))
+		for k in vars: d=re.sub("\(\(.*\)\)", c["varcmd"]+"(("+k+"))"+c["str"], d)
+		if crop and len(d)>4000:
+			print(c["important"]+"###DATA-"+str(len(d))+"###"+c["none"])
+		else:
+			print(c["str"]+d+c["none"])
+	elif(t==datetime):
+		dtstr=re.sub("\+00:00", "Z", d.isoformat())
+		#dtstr=d.__str__()
+		print(c["date"]+dtstr+c["none"])
+	elif(t==int):
+		print(c["int"]+str(d)+c["none"])
+	elif(t==float):
+		print(c["float"]+str(d)+c["none"])
+	elif(t==bool):
+		print(c["bool"]+str(d)+c["none"])
+#	elif(t==list):
+#		print
+#		for i in d:
+#			colprint(i,header="x",indent=indent)
+#			#print(indent+indentchars+c["str"]+i)
+#			#print(d)
+	elif(t==type(None)):
+		print(c["null"]+"null"+c["none"])
+	else:
+		print(c["err"]+"¿"+str(t.__name__)+"?"+c["none"])
+#}}}
+
+# Por cada item en d reemplaza $var$ por el valor de la variable en la lista varlist
+#{{{ replacevars
+def replacevars(d, varlist, unknownempty=False, emptyremove=False, unknownnull=False, emptynull=False):
+	#o=collections.OrderedDict()
+	o={}
+	for k in d:
+		if(type(d[k])==dict or type(d[k])==collections.OrderedDict):
+			d[k]=replacevars(d[k], varlist, unknownempty=unknownempty, emptyremove=emptyremove, unknownnull=unknownnull, emptynull=emptynull)
+			o[k]=d[k]
+			continue
+		else:
+			if(type(d[k])==str):
+				#if(d[k][0]=='$'):
+				#	var=d[k][1:-1]
+				#	v=varlist.get(var)
+				#	if(v): o[k]=v
+				#else:
+				#	o[k]=d[k]
+
+				# Buscamos variables en varlist
+				p=re.compile("\$([a-zA-Z0-9_]*)\$")
+				vars=set(p.findall(d[k]))
+				# No hay variables
+				if(vars==None):
+					o[k]=d[k]
+					continue
+				# Hay variables
+				for sk in vars:
+					val=varlist.get(sk)
+					if(val==None):
+						if(unknownempty): val=""
+						else:
+							if(unknownnull):
+								if emptynull: val="$null$"
+								else: val="null"
+							else:
+								continue
+					d[k]=re.sub("\$"+sk+"\$", str(val), d[k])
+
+				if(d[k]=="$null$" and emptynull): continue
+				if(d[k]=="" and emptyremove): continue
+				o[k]=d[k]
+			else:
+				o[k]=d[k]
+	return o
+#}}}
+
+# Por cada item en d reemplaza $e$var$ por variable de entorno
+#{{{ replaceenv
+def replaceenv(d, unknownempty=False, emptyremove=None):
+	o=collections.OrderedDict()
+	for k in d:
+		if(type(d[k])==dict or type(d[k])==collections.OrderedDict):
+			d[k]=replaceenv(d[k], unknownempty=unknownempty, emptyremove=emptyremove)
+			o[k]=d[k]
+			continue
+		else:
+			if(type(d[k])==str):
+				p=re.compile("\$e$([a-zA-Z0-9_]*)\$")
+				vars=set(p.findall(d[k]))
+				# No hay variables
+				if(vars==None):
+					o[k]=d[k]
+					continue
+				# Hay variables
+				for sk in vars:
+					if (sk=="HOSTNAME"): val=platform.node()
+					else: val=os.environ.get(sk)
+					if(val==None):
+						if(unknownempty): val=""
+						else: continue
+					d[k]=re.sub("\$e\$"+sk+"\$", val, d[k])
+
+				if(d[k]=="" and emptyremove): continue
+				o[k]=d[k]
+			else:
+				o[k]=d[k]
+	return o
+#}}}
+
+# Por cada item en d reemplaza ((cmd)) por la ejecución de cmd
+# Por cada item en d reemplaza $b$file$ por el fichero en b64
+# Por cada item en d reemplaza $t$file$ por el contenido del fichero
+#{{{ runvars
+def runvars(d, unknownempty=False, emptyremove=None):
+	o=collections.OrderedDict()
+	for k in d:
+		if(type(d[k])==dict or type(d[k])==collections.OrderedDict):
+			d[k]=runvars(d[k], unknownempty=unknownempty, emptyremove=emptyremove)
+			o[k]=d[k]
+			continue
+		else:
+			if(type(d[k])==str):
+				modified=False
+
+				# Comando a ejecutar
+				r_buscacmd=re.compile("\(\((.*)\)\)")
+				vars=set(r_buscacmd.findall(d[k]))
+				# No hay variables
+				if(vars==None):
+					o[k]=d[k]
+				#	continue
+				else:
+					modified=True
+					# Hay variables
+					for sk in vars:
+						# b|fichero -> base64 fichero
+						#sk=re.sub(r"^t\|(.*)$", r"cat '\g<1>'", sk)
+						sk=re.sub(r"^b\|(.*)$", r"cat '\g<1>'|base64 -w0", sk)
+						pars=["bash","-c",sk]
+						ex=run.cmd(pars)
+						#if ex.retcode==0:
+						val=ex.out
+						if(val==None):
+							if(unknownempty): val=""
+							else: continue
+						#d[k]=re.sub("\(\("+sk+"\)\)", val, d[k])
+						d[k]=re.sub("\(\(.*\)\)", val, d[k])
+
+				# Fichero base64
+				r_buscacmd=re.compile(r"$b$(.*)$")
+				vars=set(r_buscacmd.findall(d[k]))
+				# No hay variables
+				if(vars==None):
+					o[k]=d[k]
+				else:
+					modified=True
+					# Hay variables
+					for sk in vars:
+						f=open(sk, "rb")
+						fdata=base64.b64encode(f.read()).decode()
+						d[k]=re.sub(r"$b$.*$", fdata, d[k])
+
+				# Fichero directo
+				r_buscacmd=re.compile(r"$t$(.*)$")
+				vars=set(r_buscacmd.findall(d[k]))
+				# No hay variables
+				if(vars==None):
+					o[k]=d[k]
+				else:
+					modified=True
+					# Hay variables
+					for sk in vars:
+						f=open(sk, "r")
+						fdata=f.read()
+						d[k]=re.sub(r"$t$.*$", fdata, d[k])
+
+				if modified and d[k]=="" and emptyremove: continue
+				o[k]=d[k]
+			else:
+				o[k]=d[k]
+	return o
+#}}}
+
+# Values of dict contained in another dict
+def contained(d1, d2):
+	for k in d1:
+		if d2.get(k)==None: return False
+		if(type(d1[k])==dict or type(d1[k])==collections.OrderedDict):
+			if not contained(d1[k], d2[k]): return False
+		else:
+			if d1[k]!=d2[k]: return False
+	return True
+
+
+#{{{ xml tools
+
+# XML -> dict
+def fromxml(xmldata):
+	return xmltodict.parse("<xml>"+xmldata+"</xml>", disable_entities=True)[xml]
+
+#}}}
+
+#{{{ json tools
+def fromjson(jsondata):
+	return json.loads(jsondata,object_pairs_hook=json_parser_hook)
+
+def readjson(jsonfile):
+	f=open(jsonfile,"rt")
+	json=f.read()
+	data=fromjson(json)
+	return data
+
+def tojson(data,indent=None):
+	return json.dumps(data,separators=(',',':'),default=json_serializer_hook,indent=indent)
+
+def writejson(data,filename,tabs=0,spaces=0):
+	f=open(filename, "w")
+	indent=None
+	if spaces!=0:
+		indent=spaces
+	if (tabs==1):
+		indent='\t'
+	jsondata=tojson(data,indent=indent)
+	f.write(jsondata)
+	f.write("\n")
+	f.close()
+
+# json -> dict
+def json_parser_hook(js):
+	#out=collections.OrderedDict(js)
+	out=dict(js)
+	for (key, value) in out.items():
+		# Hora con timezone sin milisegundos
+		try:
+			dt=re.sub("Z$", "UTC",value)
+			dt=datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S%Z")
+			#dt=dt.replace(tzinfo=datetime.timezone(datetime.timedelta(0)))
+			dt=dt.replace(tzinfo=timezone.utc)
+			out[key]=dt
+			continue
+		except: pass
+		# Hora con timezone con milisegundos
+		try:
+			dt=re.sub("Z$", "UTC",value)
+			dt=datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%Z")
+			dt=dt.replace(tzinfo=timezone.utc)
+			out[key]=dt
+			continue
+		except: pass
+		# Hora sin timezone sin milisegundos
+		try:
+			out[key]=datetime.strptime(value, "%Y-%m-%dT%H:%M:%S")
+			continue
+		except: pass
+		# Hora sin timezone con milisegundos
+		try:
+			out[key]=datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f")
+			continue
+		except: pass
+	return out
+
+# dict -> json
+def json_serializer_hook(o):
+	if isinstance(o, datetime):
+		return re.sub("\+00:00", "Z", o.isoformat())
+
+#}}}
+
+#{{{ yaml tools
+def fromyaml(yamldata):
+	yaml=yamldata.replace("\t", "  ")
+	data=ruamel.yaml.load(yaml, Loader=ruamel.yaml.Loader)
+	return data
+
+def readrawyaml(filename):
+	# Cargar fichero yaml en dict
+	import collections
+	ruamel.yaml.representer.RoundTripRepresenter.add_representer(
+	collections.OrderedDict, ruamel.yaml.representer.RoundTripRepresenter.represent_ordereddict)
+	f=open(filename,"rt")
+	yamldata=f.read()
+	data=fromyaml(yamldata)
+	return data
+
+def readyaml(filename):
+	data=readrawyaml(filename)
+	if data.get("yaml-include"):
+		for f in data["yaml-include"]:
+			if os.path.isfile(f):
+				subdata=readyaml(f)
+				data.update(subdata)
+			else:
+				path=os.path.dirname(filename)
+				f=path+os.path.sep+f
+				if os.path.isfile(f):
+					subdata=read(f)
+					data.update(subdata)
+		#data["include"]=None
+		data.pop("yaml-include")
+	return data
+
+def yamlupdate(filename,var,val,node=None):
+	yaml=ruamel.yaml.YAML()
+	yaml.width=4096
+	f=open(filename,"r")
+	data=yaml.load(f.read())
+	f.close()
+	datamod=data
+	if node:
+		for n in node:
+			datamod=datamod.get(n)
+	datamod[var]=val
+	f=open(filename, "w")
+	yaml.dump(data, f)
+	f.close()
+
+def writeyaml(data,filename):
+	yaml=ruamel.yaml.YAML()
+	f=open(filename, "w")
+	yaml.dump(data, f)
+	f.close()
+
+#}}}

usr/local/lib/python3.8/dist-packages/pydan/run.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import subprocess
+import types
+import sys
+import os
+import time
+
+def cmd(cmd):
+	ret=types.SimpleNamespace()
+
+	if sys.platform.startswith('win'):
+		cmdproc=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,creationflags=0x00000200)
+	else:
+		cmdproc=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,preexec_fn=os.setpgrp)
+	t1=time.time()
+	binout,binerr=cmdproc.communicate()
+	t2=time.time()
+	ret.time=t2-t1
+	ret.out=binout.decode().rstrip()+binerr.decode().rstrip()
+	ret.retcode=cmdproc.returncode
+	return ret

usr/local/lib/python3.8/dist-packages/pydan/script.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+
+# Directorio del script principal
+#scriptdir=sys.path[0]
+
+# Cambiamos al directorio del script principal
+#os.chdir(scriptdir)
+
+# Utilizamos scriptdir/lib como path para los import
+#sys.path.insert(1,scriptdir+'/lib')
+
+# Cambiamos el nombre del proceso al del script
+import setproctitle
+setproctitle.setproctitle(os.path.basename(sys.argv[0]))
+
+# Excepciones coloreadas
+import colored_traceback.always
+
+# Exportamos scriptdir al script principal
+#import __main__
+#__main__.scriptdir=scriptdir
+
+def parsecmdline():
+	cmdline={}
+	cmdline["params"]=[]
+	cmdline["options"]={}
+	i=0
+	while i < len(sys.argv)-1:
+		i+=1
+		s=sys.argv[i]
+		if '=' in s:
+			keyval=s.split("=")
+			cmdline["options"][keyval[0]]=keyval[1]
+		elif s[0]=='-':
+			i+=1
+			val=sys.argv[i]
+			key=s[1:]
+			if (key[0]=='-'):
+				key=key[1:]
+			cmdline["options"][key]=val
+		else:
+			cmdline["params"].append(s)
+	if not cmdline["params"]:
+		cmdline["cmd"]=""
+	else:
+		cmdline["cmd"]=cmdline["params"][0]
+	return cmdline

usr/local/lib/python3.8/dist-packages/pydan/utils.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import os
+import platform
+
+def creation_date(path_to_file):
+	"""
+	Try to get the date that a file was created, falling back to when it was
+	last modified if that isn't possible.
+	See http://stackoverflow.com/a/39501288/1709587 for explanation.
+	"""
+	if platform.system() == 'Windows':
+		return os.path.getctime(path_to_file)
+	else:
+		stat = os.stat(path_to_file)
+		try:
+			return stat.st_birthtime
+		except AttributeError:
+			# We're probably on Linux. No easy way to get creation dates here,
+			# so we'll settle for when its content was last modified.
+			return stat.st_mtime

usr/local/lib/python3.8/dist-packages/pydan-1.7.egg-info/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 1.2
+Name: pydan
+Version: 1.7
+Summary: Python3 Data Utils
+Home-page: UNKNOWN
+Author: Jose Biosca Martín
+Author-email: jbiosca@zone-sys.net
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Requires-Python: >=3.6

usr/local/lib/python3.8/dist-packages/pydan-1.7.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+setup.py
+pydan.egg-info/PKG-INFO
+pydan.egg-info/SOURCES.txt
+pydan.egg-info/dependency_links.txt
+pydan.egg-info/requires.txt
+pydan.egg-info/top_level.txt
+src/__init__.py
+src/jdata.py
+src/run.py
+src/script.py
+src/utils.py
+test/test-jdata.py

usr/local/lib/python3.8/dist-packages/pydan-1.7.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

usr/local/lib/python3.8/dist-packages/pydan-1.7.egg-info/requires.txt

@@ -0,0 +1,6 @@
+colored_traceback
+dicttoxml
+ruamel.yaml
+setproctitle
+setuptools
+xmltodict

usr/local/lib/python3.8/dist-packages/pydan-1.7.egg-info/top_level.txt

@@ -0,0 +1 @@
+pydan