kax 0.1.0__tar.gz

kax-0.1.0/PKG-INFO

@@ -0,0 +1,41 @@
+Metadata-Version: 2.3
+Name: kax
+Version: 0.1.0
+Summary: Biblioteca cliente de sistema de almacenamiento en memoria.
+Author: Julio Salazar
+Author-email: julio.salazar@cinvestav.mx
+Requires-Python: >=3.10
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: certifi (==2025.6.15)
+Requires-Dist: charset-normalizer (==3.4.2)
+Requires-Dist: idna (==3.10)
+Requires-Dist: paho-mqtt (==2.1.0)
+Requires-Dist: python-dateutil (==2.9.0.post0)
+Requires-Dist: pytz (==2025.2)
+Requires-Dist: requests (==2.32.4)
+Requires-Dist: six (==1.17.0)
+Requires-Dist: tzdata (==2025.2)
+Requires-Dist: urllib3 (==2.5.0)
+Description-Content-Type: text/markdown
+
+
+# Biblioteca cliente de sistema de almacenamiento basado en memoria
+
+Biblioteca cliente para el sistema de almacenamiento en memoria "ESS v1". El cliente es capaz de cargar en datos memoria de forma local, enviar datos al sistema de almacenamiento en memoria y descargar datos del sistema de almacenamiento. 
+
+UTILIZAR EN SISTEMA OPERATIVO LINUX.
+
+## Authors
+
+- [@juliocsp03](https://www.github.com/juliocsp03)
+
+
+## License
+
+[MIT](https://choosealicense.com/licenses/mit/)
+
+

kax-0.1.0/README.md

@@ -0,0 +1,16 @@
+
+# Biblioteca cliente de sistema de almacenamiento basado en memoria
+
+Biblioteca cliente para el sistema de almacenamiento en memoria "ESS v1". El cliente es capaz de cargar en datos memoria de forma local, enviar datos al sistema de almacenamiento en memoria y descargar datos del sistema de almacenamiento. 
+
+UTILIZAR EN SISTEMA OPERATIVO LINUX.
+
+## Authors
+
+- [@juliocsp03](https://www.github.com/juliocsp03)
+
+
+## License
+
+[MIT](https://choosealicense.com/licenses/mit/)
+

kax-0.1.0/kax/acts.py

@@ -0,0 +1,211 @@
+import requests
+from multiprocessing import shared_memory, Pool
+from . import fsOperations as fso
+from . import mem
+from . import sub
+from . import pub
+
+url = "localhost"
+urlStorage = "localhost"
+chunk_size = 1048576
+workers = 1
+originFile = None
+use_workers = True
+
+def loadRes(path, s = False, load_to_memory = True):
+	mem.remove_shm_from_resource_tracker()
+	metadata = fso.getMetadata(path)
+	# print(json.dumps(metadata))
+	segments = []
+	for index, resource in enumerate(metadata["resources"]): # GENERA LOS ESPACIOS DE MEMEORIA
+		try: # SI YA EXISTEN SE AGREGA UN CAMPO 'local' CON VALOR 'True'
+			shm = shared_memory.SharedMemory(name = resource['hash'], create = False)
+			metadata["resources"][index]['local'] = True
+			shm.close()
+		except FileNotFoundError: # SI NO EXISTEN SE AGREGA UN CAMPO 'local' CON VALOR 'False'
+			if (load_to_memory):
+				mem.createLocalSharedMemorySpace(resource['hash'], resource['size'])
+			metadata["resources"][index]['local'] = False
+		except Exception as e:
+			return
+	for file in metadata["resources"]: # OBTENER LOS SEGMENTOS DE LOS ARCHIVOS QUE NO ESTÁN EN MEMORIA
+		if(file['local'] == False or file['local'] == True):
+			segments+=fso.array_of_segments(file['size'], file["path"], file['hash'], chunk_size)
+
+	if (load_to_memory):
+		with Pool(processes = workers) as p: # CARGARLOS EN MEMORIA A TRAVÉS DE TRABAJADORES
+			p.map(mem.loadToLocalSharedMemory, segments)
+	
+	for index, resource in enumerate(metadata["resources"]):
+		if(resource['local'] == False):
+			metadata["resources"][index]['local'] = True
+	if s:
+		return metadata, segments
+	else:
+		return metadata
+
+def loadAndUploadToCloud(path):
+	res, segments = loadRes(path, True)
+	res = uploadToCloud(res, segments)
+	return res
+
+
+def uploadToCloudFromDisk(path):
+	res, segments = loadRes(path, True, False)
+	res = uploadToCloud(res, segments, "Disk")
+	return res
+
+def uploadToCloudFromMemory(res):
+	# res, segments = loadRes(path, True, False)
+	segments = []
+	segments+=fso.array_of_segments(res['resources'][0]['size'], res['resources'][0]["path"], res['resources'][0]['hash'], chunk_size)
+	res = uploadToCloud(res, segments, "Memory")
+	return res
+
+def uploadToCloud(res = None, segments = False, origin = "Memory"):
+	global originFile
+	originFile = origin
+	mem.remove_shm_from_resource_tracker()
+	for index, resource in enumerate(res["resources"]):
+		# print("Verificando en la nube el recurso: ", resource['hash'])
+		verification = verify(resource['hash'])
+		# print(verification)
+		if verification:
+			# print("Verificación: ", verification) 
+			if (verification['file']['state']=="reserving"):
+				sub.subToTopic(resource['hash'])
+				verification['file']['state']="ready"
+				res['resources'][index] = verification['file']
+			elif (verification['file']['state']=="ready"):
+				res['resources'][index] = verification['file']
+				# print("Archivo listo para consumo")
+				pass
+			else:
+				return False
+		else:
+			if 'local' in resource: resource.pop("local")
+			resource['state'] = "reserving"
+			register = registry(resource)
+			pub.pubTopic(resource['hash'], "loading")
+			if (register["created"]):
+				response_reserve = requests.post(f"{urlStorage}/reserve", json={"name":resource['hash'], "size": resource['size']})
+				segments_res = list(filter(lambda x : f"{x['segment']}" == resource['hash'], segments))
+				if use_workers:
+					with Pool(processes = workers) as p:
+						p.map(sendToCloud, segments_res)
+				else:
+					data = None
+					for segment in segments_res:
+						if (originFile == "Memory"):
+							data = mem.getChunk(segment['segment'], segment['start_position'], segment['end_position'])
+						else:
+							with open(segment['origin'], "rb") as file:
+								file.seek(segment['start_position'])
+								data = file.read(segment['chunk_size'])
+						files = {"files": data}
+						response_fill = requests.post(f"{urlStorage}/fill", files = files, data = segment)
+				resUpdateState = update(resource['hash'], "ready")
+				pub.pubTopic(resource['hash'], "ready")
+	originFile = None
+	return res
+
+def sendToCloud(segment):
+	global originFile
+	# print("worker: ", os.getpid(), "\tSegment: ", segment['segment'])
+	data = None
+	if (originFile == "Memory"):
+		data = mem.getChunk(segment['segment'], segment['start_position'], segment['end_position'])
+	else:
+		with open(segment['origin'], "rb") as file:
+			file.seek(segment['start_position'])
+			data = file.read(segment['chunk_size'])
+	files = {"files": data}
+	response_fill = requests.post(f"{urlStorage}/fill", files = files, data = segment)
+
+def downloadFromCloud(hash):
+	mem.remove_shm_from_resource_tracker()
+	md = verify(hash)
+	md = md['file']
+	original_hash = md['hash']
+	mem.createLocalSharedMemorySpace(md['hash'], md['size'])
+	segments_to_get = fso.array_of_segments_for_server_file(md['size'], original_hash, chunk_size)
+	for segment in segments_to_get:
+		response = requests.post(f"{urlStorage}/chunk", data = segment)
+		mem.loadDowloadedToLocalSharedMemory(md['hash'], segment['start_position'], segment['end_position'], response.content)
+	return {"message": True, "file": md['hash']}
+
+def verify(hash):
+	try:
+		response = requests.post(f"{url}/hash", json = {"hash": hash})
+		if response.json()['registered']:
+			return response.json()
+		else: 
+			return False
+	except requests.exceptions.RequestException as err:
+		return {"message": "Error"}
+
+def update(hash, state):
+	try:
+		response = requests.post(f"{url}/update-file-state", json = {"hash": hash, "state": state})
+		if response.json():
+			return response.json()
+		else: 
+			return False
+	except requests.exceptions.RequestException as err:
+		return {"message": "Error"}
+
+def registry(rmd):
+	try:
+		response = requests.post(f"{url}/register-file", json=rmd)
+		return response.json()
+	except requests.exceptions.RequestException as err:
+		pass
+
+def removeShm(shm_name):
+	mem.removeShm(shm_name)
+
+def getRes(file):
+	return mem.getData(file)
+
+def saveFile(path, data, name):
+	fso.saveFile(path, data, name)
+
+def setWorkers(w):
+	global workers 
+	workers = w
+
+def setChunkSize(cs):
+	global chunk_size
+	chunk_size = cs
+
+def setUrl(u):
+	global url
+	url = u
+
+def setStorageUrl(u):
+	global urlStorage
+	urlStorage = u
+
+def getWorkers():
+	return workers
+
+def getChunkSize():
+	return chunk_size
+
+def getUrl():
+	return url
+
+def getStorageUrl():
+	return urlStorage
+
+def setBrokerUrl(url):
+	pub.setBrokerUrl(url)
+	sub.setBrokerUrl(url)
+
+def setBrokerPort(p):
+	pub.setBrokerPort(p)
+	sub.setBrokerPort(p)
+
+def setUseWorkers(value):
+	global use_workers
+	use_workers = value

kax-0.1.0/kax/fsOperations.py

@@ -0,0 +1,93 @@
+import os
+import hashlib
+
+def getMetadata(res):
+    type = None
+    data = []
+    if (os.path.isdir(res)):
+        data = dirs(res)
+        type = "directorio"
+    elif (os.path.isfile(res)):
+        data.append(files(res))
+        type = "archivo"
+    else:
+        print("Entrada inválida")
+    return {"type": type, "resources": data}
+
+def getSegmentData(segment):
+    data = None
+    with open(segment['origin'], "rb") as file:
+        file.seek(segment['start_position'])
+        data = file.read(segment['chunk_size'])
+    return data
+
+def files(res):
+    with open(res, "rb") as file:
+        digest = getFileHash(res)
+    return {"hash": digest, "path": res, "size": os.path.getsize(res), "extension": os.path.splitext(res)[1], "state": "none", "temperature": "none", "owners": "none" }
+
+def getFileHash(path):
+    hash_func = getattr(hashlib, "sha256")()
+    with open(path, "rb") as file:
+        while chunk := file.read(1048576):
+            hash_func.update(chunk)
+    return hash_func.hexdigest()
+
+def saveFile(path, data, name):
+     with open(f"{path}{name}", "wb") as file:
+          file.write(data)
+
+def dirs(res):
+    list_metadata = []
+    for element in os.listdir(res):
+        if os.path.isdir(res + '/' + element):
+            list_metadata = list_metadata + dirs(res + '/' + element)
+        elif os.path.isfile(res + '/' + element):
+            list_metadata.append(files(f"{res}/{element}"))
+    return list_metadata
+
+def array_of_segments(file_size, path, segment_name, chunk_size):
+	# INICIALIZA POSICIÓN EN 0
+	position = 0
+	# DEFINE EL TAMAÑO DEL SEGMENTO
+	chunk = chunk_size
+    # OBTIENE EL NOMBRE Y EXTENSIÓN DEL ARCHIVO
+	filename, extension = os.path.splitext(os.path.basename(path))
+	# DEFINE ARREGLO DE POSICIONES
+	positions = []
+	while position < file_size:
+		# DEFINE POSICIÓN FINAL DE UN CHUNK
+		end_position = position + chunk if ((position + chunk) < file_size) else position + (file_size - position)
+		# AGREGA ELEMENTO AL ARREGLO DE POSICIONES
+		positions.append({"origin": path, "filename": filename, "extension": extension,"start_position": position, "end_position": end_position, "file_size": file_size, "segment": segment_name, "chunk_size": chunk})
+		# ACTUALIZA LA POSICIÓN
+		position = position + chunk
+	# RETORNA LAS POSICIONES
+	return positions
+
+def array_of_segments_for_server_file(file_size, segment_name, chunk_size):
+	# INICIALIZA POSICIÓN EN 0
+	position = 0
+	# DEFINE EL TAMAÑO DEL SEGMENTO
+	chunk = chunk_size
+    # OBTIENE EL NOMBRE Y EXTENSIÓN DEL ARCHIVO
+	# filename, extension = os.path.splitext(os.path.basename(path))
+	# DEFINE ARREGLO DE POSICIONES
+	positions = []
+	while position < file_size:
+		# DEFINE POSICIÓN FINAL DE UN CHUNK
+		end_position = position + chunk if ((position + chunk) < file_size) else position + (file_size - position)
+		# AGREGA ELEMENTO AL ARREGLO DE POSICIONES
+		positions.append({"start_position": position, "end_position": end_position, "file_size": file_size, "segment": segment_name, "chunk_size": chunk})
+		# ACTUALIZA LA POSICIÓN
+		position = position + chunk
+	# RETORNA LAS POSICIONES
+	return positions
+
+
+# if __name__ == '__main__':
+#     response = get_metadata("/home/julio/objects")
+#     # response = metadata_extractor("/home/julio/objects/one/f.txt")
+#     # response = dirs("/home/julio/objects")
+#     # print()
+#     print(json.dumps(response))

kax-0.1.0/kax/mem.py

@@ -0,0 +1,71 @@
+from multiprocessing import shared_memory, Pool
+from multiprocessing import resource_tracker
+
+def createLocalSharedMemorySpace(name, size):
+	# self.remove_shm_from_resource_tracker()
+	try:
+		shm = shared_memory.SharedMemory(create=True, size = size, name = name)
+		# resource_tracker.unregister(shm._name, 'shared_memory')
+		# shm.close()
+	except Exception as e:
+		removeShm(name)
+		shm = shared_memory.SharedMemory(create=True, size = size, name = name)
+	finally:
+		resource_tracker.unregister(shm._name, 'shared_memory')
+		shm.close()
+		return shm.name
+	# shm = shared_memory.SharedMemory(create=True, size = size, name = name)
+	# resource_tracker.unregister(shm._name, 'shared_memory')
+	# shm.close()
+	# return shm.name
+
+def loadToLocalSharedMemory(segment):
+	shm = shared_memory.SharedMemory(name = segment['segment'], create=False)
+	with open(segment['origin'], "rb") as file:
+		file.seek(segment['start_position'])
+		shm.buf[segment['start_position']:segment['end_position']] = file.read(segment['chunk_size'])
+	shm.close()
+
+def loadDowloadedToLocalSharedMemory(shm_name, start_position, end_position, data):
+	shm = shared_memory.SharedMemory(name = shm_name, create=False)
+	shm.buf[start_position:end_position] = data
+	shm.close()
+
+def getChunk(shm_name, start_position, end_position):
+	shm = shared_memory.SharedMemory(name = shm_name, create=False)
+	data = bytes(shm.buf[int(start_position):int(end_position)])
+	shm.close()
+	return data
+
+def getData(file):
+	shm = shared_memory.SharedMemory(name = file, create=False)
+	data = bytes(shm.buf[:])
+	resource_tracker.unregister(shm._name, 'shared_memory')
+	shm.close()
+	return data
+
+def removeShm(shm_name):
+	shm = shared_memory.SharedMemory(name = shm_name, create=False)
+	shm.close()
+	shm.unlink()
+
+def remove_shm_from_resource_tracker():
+	"""Monkey-patch multiprocessing.resource_tracker so SharedMemory won't be tracked
+
+	More details at: https://bugs.python.org/issue38119
+	"""
+
+	def fix_register(name, rtype):
+		if rtype == "shared_memory":
+			return
+		return resource_tracker._resource_tracker.register(self, name, rtype)
+	resource_tracker.register = fix_register
+
+	def fix_unregister(name, rtype):
+		if rtype == "shared_memory":
+			return
+		return resource_tracker._resource_tracker.unregister(self, name, rtype)
+	resource_tracker.unregister = fix_unregister
+
+	if "shared_memory" in resource_tracker._CLEANUP_FUNCS:
+		del resource_tracker._CLEANUP_FUNCS["shared_memory"]

kax-0.1.0/kax/pub.py

@@ -0,0 +1,76 @@
+import paho.mqtt.client as mqtt
+import time
+
+# broker = "148.247.201.226"  
+broker = "localhost"
+port = 1883
+topic = "files/"
+pub_confirmed = False
+def on_connect(client, userdata, flags, rc, properties = None):
+	if rc == 0:
+		# print("Conectado al broker MQTT!")
+		pass
+	else:
+		# print(f"Fallo al conectar, código de retorno: {rc}")
+		pass
+
+def on_publish(client, userdata, mid, rc, properties=None):
+    global pub_confirmed
+    if rc == 0:
+        # print(f"Mensaje con ID {mid} publicado y confirmado.")
+        pub_confirmed = True
+    else:
+        # print(f"Fallo al publicar el mensaje con ID {mid}.")
+        pub_confirmed = False
+
+client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2)
+
+def pubTopic(hash = "test-hash", value = "test value"):
+	client.on_connect = on_connect
+	client.on_publish = on_publish
+	try:
+		client.connect(broker, port, 60)
+	except Exception as e:
+		# print(f"Error al conectar: {e}")
+		exit()
+
+	client.loop_start()
+
+	try:	
+		result = client.publish(f"{topic}{hash}", value, retain=True, qos = 1)
+		timeout = time.time() + 10
+		while not pub_confirmed and time.time() < timeout:
+			time.sleep(0.1)
+		if (pub_confirmed):
+			# print(f"Publicado exitosamente '{value}' en el tema '{topic}{hash}'")
+			client.loop_stop()
+			client.disconnect()
+			# print("Publicador detenido.")
+			return {"pub": True}
+		else:
+			# print(f"Fallo al publicar mensaje al tema {topic}")
+			client.loop_stop()
+			client.disconnect()
+			# print("Publicador detenido.")
+			return {"pub": False}
+		# if result[0] == 0:
+		# 	print(f"Publicado exitosamente '{value}' en el tema '{topic}{hash}'")
+		# 	return {"pub": True}
+		# else:
+		# 	# print(f"Fallo al publicar mensaje al tema {topic}")
+		# 	return {"pub": False}
+		# client.loop_stop()
+		# client.disconnect()
+		# print("Publicador detenido.")
+
+	except KeyboardInterrupt:
+		# print("Fallo inesperado.")
+		pass
+		
+def setBrokerUrl(url):
+	global broker
+	broker = url
+
+def setBrokerPort(p):
+	global port
+	port = p

kax-0.1.0/kax/sub.py

@@ -0,0 +1,55 @@
+import paho.mqtt.client as mqtt
+import time
+
+# broker = "148.247.201.226"
+broker = "localhost"
+port = 1883
+topic = "files/"
+hash = "ejemplo"
+file_state = None
+waiting = True
+
+def on_connect(client, userdata, flags, rc, properties = None):
+	if rc == 0:
+		client.subscribe(f"{topic}{hash}", qos = 1)
+	else:
+		# print(f"Fallo al conectar, código: {rc}")
+		pass
+
+def on_message(client, userdata, msg):
+    # print(f"Mensaje recibido: {msg.payload.decode()} en el tema: {msg.topic}")
+    if (msg.payload.decode() == "ready"):
+        # print("ENTRANDO A CANCELAR")
+        global waiting
+        waiting = False
+        # client.loop_stop()
+
+client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2)
+
+def subToTopic(topic_hash):
+	global hash
+	hash = topic_hash
+
+	client.on_connect = on_connect
+	client.on_message = on_message
+
+	try:
+		client.connect(broker, port)
+		# client.loop_forever()
+		client.loop_start()
+		while waiting:
+			time.sleep(0.3)
+	except Exception as e:
+		# print(f"Error al conectar: {e}")
+		exit()
+	finally:
+		client.loop_stop()
+		client.disconnect()
+
+def setBrokerUrl(url):
+	global broker
+	broker = url
+
+def setBrokerPort(p):
+	global port
+	port = p

kax-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "kax"
+version = "0.1.0"
+description = "Biblioteca cliente de sistema de almacenamiento en memoria."
+authors = [
+    {name = "Julio Salazar", email = "julio.salazar@cinvestav.mx"}
+]
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "certifi==2025.6.15",
+    "charset-normalizer==3.4.2",
+    "idna==3.10",
+    "paho-mqtt==2.1.0",
+    "python-dateutil==2.9.0.post0",
+    "pytz==2025.2",
+    "requests==2.32.4",
+    "six==1.17.0",
+    "tzdata==2025.2",
+	"urllib3==2.5.0"
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"