zacahuil 0.1.0__tar.gz

zacahuil-0.1.0/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: zacahuil
+Version: 0.1.0
+Summary: Cliente del sistema de almacenamiendo en memoria ESS v1
+Author-email: Julio Salazar <juliocsp03@gmail.com>
+Requires-Python: >=3.8
+License-File: LICENSE
+Dynamic: license-file

zacahuil-0.1.0/README.md

@@ -0,0 +1,15 @@
+
+# Cliente de sistema de almacenamiento basado en memoria
+
+Es un cliente para el sistema de almacenamiento en memoria "ESS v1". El cliente es capaz de cargar en memoria datos locales, enviar datos al sistema de almacenamiento, cargar los datos en la memoria local y enviarlos al sistema de almacenamiento, y descargar datos del sistema de almacenamiento. 
+
+
+## Authors
+
+- [@juliocsp03](https://www.github.com/juliocsp03)
+
+
+## License
+
+[MIT](https://choosealicense.com/licenses/mit/)
+

zacahuil-0.1.0/pyproject.toml

@@ -0,0 +1,12 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "zacahuil"
+version = "0.1.0"
+description = "Cliente del sistema de almacenamiendo en memoria ESS v1"
+authors = [
+  { name="Julio Salazar", email="juliocsp03@gmail.com" },
+]
+requires-python = ">=3.8"

zacahuil-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

zacahuil-0.1.0/zacahuil/acts.py

@@ -0,0 +1,205 @@
+import requests
+from . import fsOperations as fso
+from multiprocessing import shared_memory, Pool
+# import mem
+# import sub
+# import pub
+from . import mem
+from . import sub
+from . import pub
+
+url = "localhost"
+urlStorage = "localhost"
+chunk_size = 1048576
+workers = 1
+
+def loadRes(path, s = False, load_to_memory = True):
+	mem.remove_shm_from_resource_tracker()
+	metadata = fso.getMetadata(path)
+	segments = []
+	for index, resource in enumerate(metadata["resources"]): # GENERA LOS ESPACIOS DE MEMEORIA
+		try: # SI YA EXISTEN SE AGREGA UN CAMPO 'local' CON VALOR 'True'
+			shm = shared_memory.SharedMemory(name = resource['hash'], create = False)
+			metadata["resources"][index]['local'] = True
+			shm.close()
+		except FileNotFoundError: # SI NO EXISTEN SE AGREGA UN CAMPO 'local' CON VALOR 'False'
+			mem.createLocalSharedMemorySpace(resource['hash'], resource['size'])
+			metadata["resources"][index]['local'] = False
+		except Exception as e:
+			return
+	for file in metadata["resources"]: # OBTENER LOS SEGMENTOS DE LOS ARCHIVOS QUE NO ESTÁN EN MEMORIA
+		if(file['local'] == False):
+			segments+=fso.array_of_segments(file['size'], file["path"], file['hash'], chunk_size)
+
+	if (load_to_memory):
+		with Pool(processes = workers) as p: # CARGARLOS EN MEMORIA A TRAVÉS DE TRABAJADORES
+			p.map(mem.loadToLocalSharedMemory, segments)
+	
+	for index, resource in enumerate(metadata["resources"]):
+		if(resource['local'] == False):
+			metadata["resources"][index]['local'] = True
+	if s:
+		return metadata, segments
+	else: 
+		return metadata
+
+def loadAndUploadToCloud(path):
+	res, segments = loadRes(path, True)
+	# print(segments)
+	for index, resource in enumerate(res["resources"]):
+		verification = verify(resource['hash'])
+		if (verification):
+			# print("Registrado")
+			# print(verification['file'])
+			if (verification['file']['state']=="reserving"):
+				print("Reservando espacio en memoria del servidor")
+				sub.subToTopic(resource['hash'])
+				verification['file']['state']="ready"
+				res['resources'][index] = verification['file']
+			elif (verification['file']['state']=="ready"):
+				res['resources'][index] = verification['file']
+				pass
+			else:
+				return False
+		else:
+			resource.pop("local")
+			resource['state'] = "reserving"
+			register = registry(resource)
+			pub.pubTopic(resource['hash'], "loading")
+			if (register["created"]):
+				response_reserve = requests.post(f"{urlStorage}/reserve", json={"name":resource['hash'], "size": resource['size']})
+				segments_res = list(filter(lambda x : x['segment'] == resource['hash'], segments))
+				for segment in segments_res:
+					print(segment)
+					data = None
+					with open(segment['origin'], "rb") as file:
+						file.seek(segment['start_position'])
+						data = file.read(segment['chunk_size'])
+					files = {"files": data}
+					response_fill = requests.post(f"{urlStorage}/fill", files = files, data = segment)
+				resUpdateState = update(resource['hash'], "ready")
+				pub.pubTopic(resource['hash'], "ready")
+	return res
+
+def uploadToCloud(path):
+	res, segments = loadRes(path, True, False)
+	# print(segments)
+	for index, resource in enumerate(res["resources"]):
+		verification = verify(resource['hash'])
+		if (verification):
+			# print("Registrado")
+			# print(verification['file'])
+			if (verification['file']['state']=="reserving"):
+				print("Reservando espacio en memoria del servidor")
+				sub.subToTopic(resource['hash'])
+				verification['file']['state']="ready"
+				res['resources'][index] = verification['file']
+			elif (verification['file']['state']=="ready"):
+				res['resources'][index] = verification['file']
+				print("Archivo listo para consumo")
+				pass
+			else:
+				return False
+		else:
+			resource.pop("local")
+			resource['state'] = "reserving"
+			register = registry(resource)
+			pub.pubTopic(resource['hash'], "loading")
+			if (register["created"]):
+				response_reserve = requests.post(f"{urlStorage}/reserve", json={"name":resource['hash'], "size": resource['size']})
+				segments_res = list(filter(lambda x : x['segment'] == resource['hash'], segments))
+				for segment in segments_res:
+					# print(segment)
+					data = None
+					with open(segment['origin'], "rb") as file:
+						file.seek(segment['start_position'])
+						data = file.read(segment['chunk_size'])
+					files = {"files": data}
+					response_fill = requests.post(f"{urlStorage}/fill", files = files, data = segment)
+				resUpdateState = update(resource['hash'], "ready")
+				pub.pubTopic(resource['hash'], "ready")
+	return res
+
+def downloadFromCloud(hash):
+	mem.remove_shm_from_resource_tracker()
+	md = verify(hash)
+	md = md['file']
+	# print(md)
+	mem.createLocalSharedMemorySpace(md['hash'], md['size'])
+	segments_to_get = fso.array_of_segments_for_server_file(md['size'], md['hash'], chunk_size)
+	print("*"*100)
+	filename = f"{md['hash']+md['extension']}"
+	with open(filename, "w") as file:
+		pass
+	for segment in segments_to_get:
+		response = requests.post(f"{urlStorage}/chunk", data = segment)
+		# print(type(response.content))
+		with open(f"{filename}", "r+b") as f:
+			f.seek(segment['start_position'])
+			f.write(response.content)
+		mem.loadDowloadedToLocalSharedMemory(md['hash'], segment['start_position'], segment['end_position'], response.content)
+	downloadedFile = getRes(md['hash'])
+	return downloadedFile
+
+def verify(hash):
+	try:
+		response = requests.post(f"{url}/hash", json = {"hash": hash})
+		if response.json()['registered']:
+			return response.json()
+		else: 
+			return False
+	except requests.exceptions.RequestException as err:
+		print(f"Error al realizar la petición POST de verificación: {err}")
+		return {"message": "Error"}
+
+def update(hash, state):
+	try:
+		response = requests.post(f"{url}/update-file-state", json = {"hash": hash, "state": state})
+		if response.json():
+			return response.json()
+		else: 
+			return False
+	except requests.exceptions.RequestException as err:
+		print(f"Error al realizar la petición POST de actualización de estado: {err}")
+		return {"message": "Error"}
+
+def registry(rmd):
+	try:
+		response = requests.post(f"{url}/register-file", json=rmd)
+		return response.json()
+	except requests.exceptions.RequestException as err:
+		print(f"Error al realizar la petición POST de registro: {err}")
+
+def getRes(file):
+	return mem.getData(file)
+
+def saveFile(path, data, name):
+	fso.saveFile(path, data, name)
+
+def setWorkers(w):
+	global workers 
+	workers = w
+
+def setChunkSize(cs):
+	global chunk_size
+	chunk_size = cs
+
+def setUrl(u):
+	global url
+	url = u
+
+def setStorageUrl(u):
+	global urlStorage
+	urlStorage = u
+
+def getWorkers():
+	return workers
+
+def getChunkSize():
+	return chunk_size
+
+def getUrl():
+	return url
+
+def getStorageUrl():
+	return urlStorage

zacahuil-0.1.0/zacahuil/fsOperations.py

@@ -0,0 +1,93 @@
+import os
+import hashlib
+
+def getMetadata(res):
+    type = None
+    data = []
+    if (os.path.isdir(res)):
+        data = dirs(res)
+        type = "directorio"
+    elif (os.path.isfile(res)):
+        data.append(files(res))
+        type = "archivo"
+    else:
+        print("Entrada inválida")
+    return {"type": type, "resources": data}
+
+def getSegmentData(segment):
+    data = None
+    with open(segment['origin'], "rb") as file:
+        file.seek(segment['start_position'])
+        data = file.read(segment['chunk_size'])
+    return data
+
+def files(res):
+    with open(res, "rb") as file:
+        digest = getFileHash(res)
+    return {"hash": digest, "path": res, "size": os.path.getsize(res), "extension": os.path.splitext(res)[1], "state": "none", "temperature": "none", "owners": "none" }
+
+def getFileHash(path):
+    hash_func = getattr(hashlib, "sha256")()
+    with open(path, "rb") as file:
+        while chunk := file.read(1048576):
+            hash_func.update(chunk)
+    return hash_func.hexdigest()
+
+def saveFile(path, data, name):
+     with open(f"{path}{name}", "wb") as file:
+          file.write(data)
+
+def dirs(res):
+    list_metadata = []
+    for element in os.listdir(res):
+        if os.path.isdir(res + '/' + element):
+            list_metadata = list_metadata + dirs(res + '/' + element)
+        elif os.path.isfile(res + '/' + element):
+            list_metadata.append(files(f"{res}/{element}"))
+    return list_metadata
+
+def array_of_segments(file_size, path, segment_name, chunk_size):
+	# INICIALIZA POSICIÓN EN 0
+	position = 0
+	# DEFINE EL TAMAÑO DEL SEGMENTO
+	chunk = chunk_size
+    # OBTIENE EL NOMBRE Y EXTENSIÓN DEL ARCHIVO
+	filename, extension = os.path.splitext(os.path.basename(path))
+	# DEFINE ARREGLO DE POSICIONES
+	positions = []
+	while position < file_size:
+		# DEFINE POSICIÓN FINAL DE UN CHUNK
+		end_position = position + chunk if ((position + chunk) < file_size) else position + (file_size - position)
+		# AGREGA ELEMENTO AL ARREGLO DE POSICIONES
+		positions.append({"origin": path, "filename": filename, "extension": extension,"start_position": position, "end_position": end_position, "file_size": file_size, "segment": segment_name, "chunk_size": chunk})
+		# ACTUALIZA LA POSICIÓN
+		position = position + chunk
+	# RETORNA LAS POSICIONES
+	return positions
+
+def array_of_segments_for_server_file(file_size, segment_name, chunk_size):
+	# INICIALIZA POSICIÓN EN 0
+	position = 0
+	# DEFINE EL TAMAÑO DEL SEGMENTO
+	chunk = chunk_size
+    # OBTIENE EL NOMBRE Y EXTENSIÓN DEL ARCHIVO
+	# filename, extension = os.path.splitext(os.path.basename(path))
+	# DEFINE ARREGLO DE POSICIONES
+	positions = []
+	while position < file_size:
+		# DEFINE POSICIÓN FINAL DE UN CHUNK
+		end_position = position + chunk if ((position + chunk) < file_size) else position + (file_size - position)
+		# AGREGA ELEMENTO AL ARREGLO DE POSICIONES
+		positions.append({"start_position": position, "end_position": end_position, "file_size": file_size, "segment": segment_name, "chunk_size": chunk})
+		# ACTUALIZA LA POSICIÓN
+		position = position + chunk
+	# RETORNA LAS POSICIONES
+	return positions
+
+
+# if __name__ == '__main__':
+#     response = get_metadata("/home/julio/objects")
+#     # response = metadata_extractor("/home/julio/objects/one/f.txt")
+#     # response = dirs("/home/julio/objects")
+#     # print()
+#     print(json.dumps(response))

zacahuil-0.1.0/zacahuil/mem.py

@@ -0,0 +1,50 @@
+from multiprocessing import shared_memory, Pool
+from multiprocessing import resource_tracker
+
+def createLocalSharedMemorySpace(name, size):
+	# self.remove_shm_from_resource_tracker()
+	shm = shared_memory.SharedMemory(create=True, size = size, name = name)
+	resource_tracker.unregister(shm._name, 'shared_memory')
+	shm.close()
+	return shm.name
+
+def loadToLocalSharedMemory(segment):
+	shm = shared_memory.SharedMemory(name = segment['segment'], create=False)
+	with open(segment['origin'], "rb") as file:
+		file.seek(segment['start_position'])
+		shm.buf[segment['start_position']:segment['end_position']] = file.read(segment['chunk_size'])
+	shm.close()
+
+def loadDowloadedToLocalSharedMemory(shm_name, start_position, end_position, data):
+	shm = shared_memory.SharedMemory(name = shm_name, create=False)
+	shm.buf[start_position:end_position] = data
+	shm.close()
+
+
+def getData(file):
+	remove_shm_from_resource_tracker()
+	shm = shared_memory.SharedMemory(name = file, create=False)
+	data = bytes(shm.buf[:])
+	shm.close()
+	return data
+
+def remove_shm_from_resource_tracker():
+	"""Monkey-patch multiprocessing.resource_tracker so SharedMemory won't be tracked
+
+	More details at: https://bugs.python.org/issue38119
+	"""
+
+	def fix_register(name, rtype):
+		if rtype == "shared_memory":
+			return
+		return resource_tracker._resource_tracker.register(self, name, rtype)
+	resource_tracker.register = fix_register
+
+	def fix_unregister(name, rtype):
+		if rtype == "shared_memory":
+			return
+		return resource_tracker._resource_tracker.unregister(self, name, rtype)
+	resource_tracker.unregister = fix_unregister
+
+	if "shared_memory" in resource_tracker._CLEANUP_FUNCS:
+		del resource_tracker._CLEANUP_FUNCS["shared_memory"]

zacahuil-0.1.0/zacahuil/pub.py

@@ -0,0 +1,68 @@
+import paho.mqtt.client as mqtt
+import time
+import random
+
+# broker = "148.247.201.226"  
+broker = "localhost"
+port = 1883
+topic = "files/"
+pub_confirmed = False
+def on_connect(client, userdata, flags, rc, properties = None):
+    if rc == 0:
+        # print("Conectado al broker MQTT!")
+        pass
+    else:
+        print(f"Fallo al conectar, código de retorno: {rc}")
+
+def on_publish(client, userdata, mid, rc, properties=None):
+    global pub_confirmed
+    if rc == 0:
+        # print(f"Mensaje con ID {mid} publicado y confirmado.")
+        pub_confirmed = True
+    else:
+        # print(f"Fallo al publicar el mensaje con ID {mid}.")
+        pub_confirmed = False
+
+client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2)
+
+def pubTopic(hash = "test-hash", value = "test value"):
+	client.on_connect = on_connect
+	client.on_publish = on_publish
+	try:
+		client.connect(broker, port, 60)
+	except Exception as e:
+		print(f"Error al conectar: {e}")
+		exit()
+
+	client.loop_start()
+
+	try:	
+		result = client.publish(f"{topic}{hash}", value, retain=True, qos = 1)
+		timeout = time.time() + 10
+		while not pub_confirmed and time.time() < timeout:
+			time.sleep(0.1)
+		if (pub_confirmed):
+			# print(f"Publicado exitosamente '{value}' en el tema '{topic}{hash}'")
+			client.loop_stop()
+			client.disconnect()
+			# print("Publicador detenido.")
+			return {"pub": True}
+		else:
+			# print(f"Fallo al publicar mensaje al tema {topic}")
+			client.loop_stop()
+			client.disconnect()
+			# print("Publicador detenido.")
+			return {"pub": False}
+		# if result[0] == 0:
+		# 	print(f"Publicado exitosamente '{value}' en el tema '{topic}{hash}'")
+		# 	return {"pub": True}
+		# else:
+		# 	# print(f"Fallo al publicar mensaje al tema {topic}")
+		# 	return {"pub": False}
+		# client.loop_stop()
+		# client.disconnect()
+		# print("Publicador detenido.")
+
+	except KeyboardInterrupt:
+		print("Fallo inesperado.")
+		

zacahuil-0.1.0/zacahuil/sub.py

@@ -0,0 +1,46 @@
+import paho.mqtt.client as mqtt
+import time
+
+# broker = "148.247.201.226"
+broker = "localhost"
+port = 1883
+topic = "files/"
+hash = "ejemplo"
+file_state = None
+waiting = True
+
+def on_connect(client, userdata, flags, rc, properties = None):
+    if rc == 0:
+        client.subscribe(f"{topic}{hash}", qos = 1)
+    else:
+        print(f"Fallo al conectar, código: {rc}")
+
+def on_message(client, userdata, msg):
+    print(f"Mensaje recibido: {msg.payload.decode()} en el tema: {msg.topic}")
+    if (msg.payload.decode() == "ready"):
+        print("ENTRANDO A CANCELAR")
+        global waiting
+        waiting = False
+        # client.loop_stop()
+
+client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2)
+
+def subToTopic(topic_hash):
+	global hash
+	hash = topic_hash
+
+	client.on_connect = on_connect
+	client.on_message = on_message
+
+	try:
+		client.connect(broker, port)
+		# client.loop_forever()
+		client.loop_start()
+		while waiting:
+			time.sleep(0.5)
+	except Exception as e:
+		print(f"Error al conectar: {e}")
+		# exit()
+	finally:
+		client.loop_stop()
+		client.disconnect()

zacahuil-0.1.0/zacahuil.egg-info/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: zacahuil
+Version: 0.1.0
+Summary: Cliente del sistema de almacenamiendo en memoria ESS v1
+Author-email: Julio Salazar <juliocsp03@gmail.com>
+Requires-Python: >=3.8
+License-File: LICENSE
+Dynamic: license-file

zacahuil-0.1.0/zacahuil.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+README.md
+pyproject.toml
+zacahuil/__init__.py
+zacahuil/acts.py
+zacahuil/fsOperations.py
+zacahuil/mem.py
+zacahuil/pub.py
+zacahuil/sub.py
+zacahuil.egg-info/PKG-INFO
+zacahuil.egg-info/SOURCES.txt
+zacahuil.egg-info/dependency_links.txt
+zacahuil.egg-info/top_level.txt

zacahuil-0.1.0/zacahuil.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

zacahuil-0.1.0/zacahuil.egg-info/top_level.txt

@@ -0,0 +1 @@
+zacahuil