acelerai 0.0.1__tar.gz

acelerai-0.0.1/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.2
+Name: acelerai
+Version: 0.0.1
+Summary: short package description
+Author: DaniloAraneda
+Author-email: danilo@alert2gain.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10.7
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: requires-python
+Dynamic: summary

acelerai-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

acelerai-0.0.1/setup.py

@@ -0,0 +1,20 @@
+import setuptools
+
+
+setuptools.setup(
+    name = "acelerai",
+    version = "0.0.1",
+    author = "DaniloAraneda",
+    author_email = "danilo@alert2gain.com",
+    description = "short package description",
+    classifiers = [
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    package_dir = {"": "src"},
+    packages = setuptools.find_packages(where="src"),
+    include_dirs=[],
+    python_requires = ">=3.10.7",
+    requires=[]
+)

acelerai-0.0.1/src/acelerai.egg-info/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.2
+Name: acelerai
+Version: 0.0.1
+Summary: short package description
+Author: DaniloAraneda
+Author-email: danilo@alert2gain.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10.7
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: requires-python
+Dynamic: summary

acelerai-0.0.1/src/acelerai.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+setup.py
+src/acelerai.egg-info/PKG-INFO
+src/acelerai.egg-info/SOURCES.txt
+src/acelerai.egg-info/dependency_links.txt
+src/acelerai.egg-info/top_level.txt
+src/acelerai_inputstream/__init__.py
+src/acelerai_inputstream/inputstream.py
+src/acelerai_inputstream/inputstream_client.py
+test/test.py
+test/test_app.py

acelerai-0.0.1/src/acelerai.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

acelerai-0.0.1/src/acelerai.egg-info/top_level.txt

@@ -0,0 +1 @@
+acelerai_inputstream

acelerai-0.0.1/src/acelerai_inputstream/__init__.py

@@ -0,0 +1,39 @@
+import json
+import os
+from acelerai_inputstream.inputstream import INSERTION_MODE, Inputstream
+from acelerai_inputstream.inputstream_client import InputstreamClient
+
+__results_path = os.environ.get("A2G_RESULT_PATH","a2g_results")
+__payload_path = os.environ.get("A2G_PAYLOAD_PATH", "payload.json")
+
+__mode = os.environ.get("EXEC_LOCATION", "LOCAL")
+
+def save_result(key:str, value, path = None):
+    """
+    Save the result in the file
+    :param key: The key to be used to save the result
+    :param value: The value to be saved
+    :param path: The path to save the result, if None, the default path is used
+    """
+    result_path = __results_path
+    if path is not None and __mode == "LOCAL":
+        result_path = path
+
+    if __mode == "LOCAL":
+        if not os.path.exists(result_path): os.makedirs(result_path)
+
+    open(f"{result_path}/{key}", 'w+').write(json.dumps(value))
+
+def get_payload(path = None) -> dict | None:
+    """
+    Get the payload from the file, if the file does not exist, return None
+    :param path: The path to the payload file, if None, the default path is used
+    """
+    payload_path = __payload_path
+    if path is not None and __mode == "LOCAL":
+        payload_path = path
+
+    if not os.path.exists(payload_path): return None
+    return json.loads(open(payload_path).read())
+
+

acelerai-0.0.1/src/acelerai_inputstream/inputstream.py

@@ -0,0 +1,263 @@
+import copy
+from datetime import datetime
+from enum import Enum
+from uuid import UUID
+from dateutil import parser
+
+
+# Enums
+class FileIndexFieldType(Enum):
+    Datetime = 0
+    String = 1
+    Number = 2
+    Integer = 3
+
+class DateBucketSize(Enum):
+    Minute = 0
+    Hour = 1
+    Day = 2
+    Week = 3
+    Month = 4
+    Year = 5
+
+class InputstreamStatus(Enum):
+    ToDiscover = 0
+    Undiscovered = 1
+    Exposed = 2
+    ToDiscoverAgain = 3
+
+class InputstreamStorage(Enum):
+    Collection = 0
+    TimeSeriesCollection = 1
+    File = 2
+
+class InputstreamProtocol(Enum):
+    MQTT = 0
+    HTTP = 1
+    BOTH = 2
+
+class RealTimeMode(Enum):
+    OFF = 0
+    ON = 1
+
+class IndexType(Enum):
+    Unique = 0
+    Search = 1
+
+class SortType(Enum):
+    Ascending = 0
+    Descending = 1
+
+class SourceType(Enum):
+    MySQL=0
+    MongoDB=1
+    SQLServer=2
+    Snowflake=3
+    Oracle=4
+    PostgresSQL=5
+    Firebase=6
+    BigQuery=7
+
+class DataType(Enum):
+    TypeNumber = 0
+    TypeBoolean = 1
+    TypeString = 2
+    TypeDateType = 3
+    TypeDate = 4
+    TypeList = 5
+
+class InputstreamType(Enum):
+    InSystem = 0
+    Native = 1
+
+# Models
+class DynamicField:
+    DataType: DataType
+    Field: str
+    ValueDefault: str
+
+    def __init__(self, **kwargs) -> None:
+        kwargs["DataType"] = DataType(kwargs.pop('DataType'))
+        self.__dict__ = kwargs
+
+    def get_dict(self):
+        data = copy.deepcopy(self.__dict__)
+        return data 
+
+class DataConnection:
+    ConnectionString: str
+    DataSourceName: str
+    Query : str
+    SourceType: SourceType
+    DynamicFields: list[DynamicField]
+    DatabaseName: str
+
+    def __init__(self,**kwargs)-> None:
+        kwargs["SourceType"] = SourceType(kwargs.pop('SourceType'))
+        self.__dict__ = kwargs
+    
+    def get_dict(self):
+        data = copy.deepcopy(self.__dict__)
+        return data
+
+
+class IndexField:
+    Name: str
+    FieldType: FileIndexFieldType
+    DoubleBucketSize: float
+    DateBucketSize: DateBucketSize
+
+    def __init__(self, from_response=False, **kwargs):
+        if from_response:
+            kwargs["Name"] = kwargs.pop('name')
+            kwargs["DoubleBucketSize"] = kwargs.pop('doubleBucketSize')
+            kwargs["FieldType"] = FileIndexFieldType(kwargs.pop('fieldType'))
+            kwargs["DateBucketSize"] = DateBucketSize(kwargs.pop('dateBucketSize'))
+        else:
+            kwargs["FieldType"] = FileIndexFieldType(kwargs.pop('FieldType'))
+            kwargs["DateBucketSize"] = DateBucketSize(kwargs.pop('DateBucketSize'))
+        self.__dict__ = kwargs
+
+    def get_dict(self):
+        data = copy.deepcopy(self.__dict__)
+        return data
+
+
+class CollectionIndexField:
+    Name: str
+    SortType: SortType
+
+    def __init__(self, from_response=False, **kwargs):
+        if from_response:
+            kwargs["Name"] = kwargs.pop('name')
+            kwargs["SortType"] = SortType(kwargs.pop('sortType'))
+        else:
+            kwargs["SortType"] = SortType(kwargs["SortType"])
+        self.__dict__ = kwargs
+
+    def get_dict(self):
+        data = copy.deepcopy(self.__dict__)
+        return data
+
+
+class CollectionIndex:
+    Name: str
+    Fields: list[CollectionIndexField]
+    Size: int
+    IndexUse: int
+    SinceUse: datetime
+    IndexType: IndexType
+    DateCreated: datetime
+    IsCompound: bool
+
+    def __init__(self, from_response=False, **kwargs):
+        if from_response:
+            kwargs["Name"] = kwargs.pop('name')
+            kwargs["DateCreated"] = kwargs.pop('dateCreated')
+            kwargs["Fields"] = [CollectionIndexField(from_response=True, **x) for x in kwargs.pop('fields')]
+            kwargs["IndexType"] = IndexType(kwargs.pop('indexType'))
+        else:
+            kwargs["DateCreated"] = kwargs.pop('DateCreated')
+            kwargs["Fields"] = [CollectionIndexField(**x) for x in kwargs["Fields"]]
+            kwargs["IndexType"] = IndexType(kwargs.pop('IndexType'))
+        self.__dict__ = kwargs
+
+    def get_dict(self):
+        data = copy.deepcopy(self.__dict__)
+        data["Fields"] = [x.get_dict() for x in self.Fields]
+        return data
+
+class Inputstream:
+    Id: UUID
+    SubscriptionId: UUID
+    Name: str
+    CollectionName: str
+    Schema: str
+    SchemaSample: str
+    SampleDate: datetime
+    Status: InputstreamStatus
+    InputstreamType: InputstreamType
+    DataConnection: DataConnection
+    Tags: list[str]
+    Ikey: str
+    CollectionIndexes: list[CollectionIndex]
+    FilesIndex: list[IndexField]
+    Storage: InputstreamStorage
+    Protocol: InputstreamProtocol
+    RealTimeMode: RealTimeMode
+    Size: int
+    MaxNDocsByFile: int
+    AllowAnyOrigin: bool
+    FileConsolidatorCron: str
+    Removed: bool
+    CreatedOn: datetime
+    RemovedOn: datetime | None
+
+
+    def __init__(self, from_response = False, **kwargs):
+        if from_response: self.from_response(**kwargs)
+        else:
+            kwargs["Id"] = UUID(kwargs.pop('Id'))
+
+            kwargs["Status"]       = InputstreamStatus(kwargs.pop('Status'))
+            kwargs["DataConnection"] = DataConnection(**kwargs["DataConnection"])
+            kwargs["InputstreamType"] = InputstreamType(kwargs.pop('InputstreamType'))
+            kwargs["Storage"]      = InputstreamStorage(kwargs.pop('Storage'))
+            kwargs["Protocol"]     = InputstreamProtocol(kwargs.pop('Protocol'))
+            kwargs["RealTimeMode"] = RealTimeMode(kwargs.pop('RealTimeMode'))
+
+            kwargs["FilesIndex"]        = [IndexField(**x) for x in kwargs["FilesIndex"]]
+            kwargs["CollectionIndexes"] = [CollectionIndex(**x) for x in kwargs["CollectionIndexes"]]
+
+            self.__dict__ = kwargs
+
+    
+    def from_response(self, **kwargs) -> None:
+        kwargs["Id"]           = UUID(kwargs.pop('id'))
+        kwargs["SubscriptionId"]            = UUID(kwargs.pop('subscriptionId'))
+
+        kwargs["Name"]                      = kwargs.pop('name')
+        kwargs["CollectionName"]            = kwargs.pop('collectionName')
+        kwargs["Schema"]                    = kwargs.pop('schema')
+        kwargs["SchemaSample"]              = kwargs.pop('schemaSample')
+        kwargs["Tags"]                      = kwargs.pop('tags')
+        kwargs["Ikey"]                      = kwargs.pop('ikey')
+        kwargs["Size"]                      = kwargs.pop('size')
+        kwargs["MaxNDocsByFile"]            = kwargs.pop('maxNDocsByFile')
+        kwargs["AllowAnyOrigin"]            = kwargs.pop('allowAnyOrigin')
+        kwargs["FileConsolidatorCron"]      = kwargs.pop('fileConsolidatorCron')
+        kwargs["Removed"]                   = kwargs.pop('removed')
+
+        kwargs["Status"]       = InputstreamStatus(kwargs.pop('status'))
+        kwargs["Storage"]      = InputstreamStorage(kwargs.pop('storage'))
+        kwargs["Protocol"]     = InputstreamProtocol(kwargs.pop('protocol'))
+        kwargs["RealTimeMode"] = RealTimeMode(kwargs.pop('realTimeMode'))
+        kwargs["InputstreamType"] = InputstreamType(kwargs.pop('inputstreamType'))
+
+        kwargs["FilesIndex"]        = [IndexField(from_response=True,**x) for x in kwargs.pop("filesIndex")]
+        kwargs["CollectionIndexes"] = [CollectionIndex(from_response=True, **x) for x in kwargs.pop("collectionIndexes")]
+
+        kwargs["SampleDate"]  = kwargs.pop('sampleDate')
+        kwargs["CreatedOn"]   = kwargs.pop('createdOn') 
+        kwargs["RemovedOn"]   = kwargs.pop('removedOn') 
+
+        self.__dict__ = kwargs
+
+    def get_dict(self):
+        data = copy.deepcopy(self.__dict__)
+        data["Id"] = str(data["Id"])
+        data["FilesIndex"] = [x.get_dict() for x in self.FilesIndex]
+        data["CollectionIndexes"] = [x.get_dict() for x in self.CollectionIndexes]
+        return data
+
+
+class INSERTION_MODE(Enum):
+    """
+    Enum for insertion modes, available modes: 
+    REPLACE: if a document collides with an existing document by a unique index, the existing document is replaced with the new document. Otherwise, the new document is inserted.
+    INSERT_UNORDERED: insert all documents that did not have a collision with an existing document by a unique index.
+    TRANSACTION: insert all documents in a transaction, if a document collides with an existing document by a unique index, the transaction is aborted and no document is inserted.
+    """
+    REPLACE = 0
+    INSERT_UNORDERED = 1
+    TRANSACTION = 2

acelerai-0.0.1/src/acelerai_inputstream/inputstream_client.py

@@ -0,0 +1,706 @@
+import logging
+logger = logging.getLogger("Inputstream")
+
+import asyncio
+from datetime import datetime, timedelta, date
+from enum import Enum
+import gzip
+import hashlib
+import os
+import json
+import struct
+from uuid import UUID
+import requests
+from acelerai_inputstream.inputstream import INSERTION_MODE, Inputstream, InputstreamStatus, InputstreamType
+#from jsonschema import Draft4Validator
+import fastjsonschema
+from dateutil import parser
+import httpx
+import gzip
+from decimal import Decimal
+import msgpack
+
+global __mode
+__mode = os.environ.get("EXEC_LOCATION", "LOCAL")
+SEM = asyncio.Semaphore(20)  # Limitar concurrencia a 20 conexiones
+
+def custom_encoder(obj):
+    """Convierte tipos no serializables como datetime y Decimal."""
+    if isinstance(obj, datetime):
+        return obj.isoformat()  # Serializar datetime como cadena ISO 8601
+    if isinstance(obj, Decimal):
+        return float(obj)  # Serializar Decimal como flotante
+    raise TypeError(f"Object of type {type(obj).__name__} is not serializable")
+
+def decode_datetime(obj):
+    """Deserializa cadenas ISO 8601 a objetos datetime."""
+    for key, value in obj.items():
+        if isinstance(value, str):
+            try:
+                obj[key] = datetime.fromisoformat(value)  # Deserializar datetime
+            except ValueError:
+                pass
+        elif isinstance(value, float):
+            obj[key] = Decimal(value)  # Convertir flotantes de regreso a Decimal
+    return obj
+
+def load_full_object(file_path):
+    """Carga completamente el objeto desde un archivo MessagePack en memoria."""
+    try:
+        with open(file_path, "rb") as file:
+            # Cargar todos los registros en memoria como una lista
+            unpacker = msgpack.Unpacker(file, raw=False)
+            data = [record for record in unpacker]  # Deserializar todos los registros
+        return data
+    except Exception as e:
+        logger.error(f"Error al cargar el archivo: {e}", exc_info=True)
+        return None
+
+if __mode != "LOCAL":
+    # Environment production
+    DATA_URL        = os.environ.get("DATA_URL"         , "https://stream.aceler.ai")
+    QUERY_MANAGER   = os.environ.get("QUERY_MANAGER"    , "https://stream.aceler.ai")  
+    INPUTSTREAM_URL = os.environ.get("INPUTSTREAM_URL"  , "https://apigw.aceler.ai")
+    verify_https = True
+else:
+    # Environment development
+    DATA_URL        = os.environ.get("DATA_URL", "https://localhost:1008")
+    QUERY_MANAGER   = os.environ.get("QUERY_MANAGER", "https://localhost:8000")
+    INPUTSTREAM_URL = os.environ.get("INPUTSTREAM_URL", "https://localhost:1006")
+    verify_https = False
+
+packer = msgpack.Packer(default=custom_encoder)  # Configurar el hook de serialización
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Enum):
+            return obj.value
+        
+        elif isinstance(obj, datetime):
+            return obj.isoformat()
+        
+        elif isinstance(obj,date):
+            return obj.isoformat()
+        
+        elif isinstance(obj, UUID):
+            return str(obj)
+        else:
+            return super().default(obj)
+
+
+class CustomJsonDecoder(json.JSONDecoder):
+    def __init__(self, *args ,**kargs):
+        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kargs)
+
+    def object_hook(self, obj:dict):
+        for k, v in obj.items():
+            if isinstance(v, str) and 'T' in v and '-' in v and ':' in v and len(v) < 40:
+                try:
+                    dv = parser.parse(v)
+                    dt = dv.replace(tzinfo=None)
+                    obj[k] = dt
+                except:
+                    pass
+            elif isinstance(v, str) and '-' in v and len(v) < 11:
+                try:
+                    obj[k] = parser.parse(v).date()
+                except:
+                    pass
+        return obj
+
+
+class CacheManager:
+    duration_inputstream:int
+    duration_data:int
+
+    def __init__(self, cache_options: dict | None = None):
+        if cache_options is None:
+            self.duration_data = 60 * 24
+            self.duration_inputstream = 60 * 24
+        else:
+            self.duration_data = cache_options.get("duration_data", 60 * 24)
+            self.duration_inputstream = cache_options.get("duration_inputstream", 60 * 24)
+
+        # create cache directories
+        if not os.path.exists(".acelerai_cache"):
+            os.mkdir(".acelerai_cache")
+            os.mkdir(".acelerai_cache/data")
+        
+    def get_inputstream(self, ikey:str) -> Inputstream | None:
+        """
+        return Inputstream if exists in cache and is not expired
+        otherwise return None
+        params:
+            ikey: str
+        """
+        file_name = f".acelerai_cache/inputstreams/{ikey}.json"
+        if os.path.exists(file_name):
+            logger.info(f"Inputstream - Ikey: {ikey}, Checking cache expiration...")
+            data = json.loads(open(file_name, "r").read(), cls=CustomJsonDecoder)
+            if datetime.utcnow() < data["duration"]:
+                logger.info(f"Inputstream - Ikey: {ikey}, from cache")
+                return Inputstream(**data["inputstream"])
+            else:
+                logger.info(f"Inputstream - Ikey: {ikey}, Cache expired, removing file...")
+                os.remove(file_name)
+                return None
+        return None
+
+    def set_inputstream(self, inputstream:Inputstream):
+        cache_register = {
+            "inputstream": inputstream.get_dict(),
+            "duration": datetime.utcnow() + timedelta(minutes=self.duration_inputstream)
+        }
+
+        file_name = inputstream.Ikey
+        if not os.path.exists(f".acelerai_cache/inputstreams/"): os.mkdir(f".acelerai_cache/inputstreams/")
+        open(f".acelerai_cache/inputstreams/{file_name}.msgpack", "w+").write(json.dumps(cache_register, cls=CustomJSONEncoder))
+
+    def get_data(self, ikey:str, hash_query:str) -> list[dict] | None:
+        """
+        return data if exists in cache and is not expired
+        otherwise return None
+        params:
+            ikey: str
+            query: dict
+        """
+        file_name = f".acelerai_cache/data/{ikey}/{hash_query}.msgpack"
+        index_ttl_file = f".acelerai_cache/data/ttl_index.json"
+        if os.path.exists(file_name) and os.path.exists(index_ttl_file):
+
+            # check if cache is expired
+            logger.info(f"Data - Ikey: {ikey}, Checking cache expiration...")
+            index = json.loads(open(index_ttl_file, "r").read(), cls=CustomJsonDecoder)
+            ttl_key = f"{ikey}_{hash_query}"
+            if ttl_key in index:
+                ttl = index[ttl_key]
+                if datetime.utcnow() > ttl:
+                    logger.info(f"Data - Ikey: {ikey}, Cache expired, removing file...")
+                    os.remove(file_name)
+                    return None
+
+            # recover data from cache
+            try:
+                logger.info(f"Data - Ikey: {ikey}, Recovering data from cache...")
+                data = load_full_object(file_name)
+                logger.info(f"Data - Ikey: {ikey}, from cache")
+                return data
+            except Exception as e:
+                if os.path.exists(file_name): os.remove(file_name)
+                raise Exception(f"Error reading cache file: {file_name} - {e}", stack_info=True)
+        else:
+            if os.path.exists(file_name): os.remove(file_name)
+            return None
+    
+    def set_data(self, ikey:str, hash_query:str):
+        # update ttl index
+        ttl_key = f"{ikey}_{hash_query}"
+        ttl = datetime.utcnow() + timedelta(minutes=self.duration_data)
+        index_file = f".acelerai_cache/data/ttl_index.json"
+        if os.path.exists(index_file):
+            index = json.loads(open(index_file, "r").read(), cls=CustomJsonDecoder)
+            index[ttl_key] = ttl
+            open(index_file, "w+").write(json.dumps(index, cls=CustomJSONEncoder))
+        else:
+            open(index_file, "w+").write(json.dumps({ttl_key: ttl}, cls=CustomJSONEncoder))
+
+
+class AcelerAIHttpClient():
+
+    def __init__(self, token:str):
+        self.token = token
+        self.lock = asyncio.Lock()
+
+    def get_inputstream_by_ikey(self, ikey:str) -> Inputstream:
+        try:
+            headers = { "Authorization": f"A2G {self.token}"}
+            # proxies = {'https': 'http://127.0.0.1:1000'}
+
+            res = requests.get(INPUTSTREAM_URL + f"/Inputstream/Ikey/{ikey}", headers=headers, verify=verify_https)
+            logger.info(f"Getting inputstream with ikey: {ikey} from ACELER.AI...")
+            if res.status_code != 200:
+                if res.status_code == 404: raise Exception("Inputstream not found, please check your ikey")
+                if res.status_code == 401: raise Exception("Unauthorized: please check your token or access permissions")
+                if res.status_code == 403: raise Exception("Forbidden: please check your access permissions")
+                raise Exception(f"Error getting inputstream, {res.status_code} {res.text}")
+            content = res.json(cls=CustomJsonDecoder)
+            if not content["success"]: raise Exception(content["errorMessage"])
+            return Inputstream(from_response=True, **content["data"])
+        except Exception as e:
+            raise e
+        
+    async def _write_to_file(self, ikey, query, data):
+        query_str = json.dumps(query, cls=CustomJSONEncoder)
+        query_hash = hashlib.sha256(query_str.encode()).hexdigest()
+
+        # save data
+        file_name = f".acelerai_cache/data/{ikey}/{query_hash}.msgpack"
+
+        async with self.lock:  # Garantiza que solo una tarea escriba a la vez
+            with open(file_name, "ab") as file:
+                for record in data:
+                    file.write(packer.pack(record))
+    
+    async def fetch_page(self, ikey:str, query:dict,delete_id:bool=True, page:int=1, page_size:int=1000):
+        async with SEM:
+            headers = {
+                "Authorization": f"A2G {self.token}",
+                "ikey": ikey,
+                'Content-Type': 'application/json'
+            }
+       
+            my_body = {
+                "delete_id": delete_id,
+                "query": json.dumps(query, cls=CustomJSONEncoder) 
+            }
+
+            buffer = b""  # Buffer para ensamblar datos incompletos
+            s = datetime.utcnow()
+            timeout = httpx.Timeout(60.0, connect=10.0, read=600.0)
+
+            async with httpx.AsyncClient(http2=True, verify=False, timeout=timeout) as client:
+                async with client.stream("POST", f"{QUERY_MANAGER}/QueryData/Find", 
+                    json=my_body, 
+                    headers=headers,
+                    params={"page": page, "page_size": page_size }) as response:
+
+                    if response.status_code != 200:
+                        msg = ''
+                        async for chunk in response.aiter_bytes():
+                            if chunk:
+                                msg += chunk.decode("utf-8")
+
+                        raise Exception(f"{response.status_code} {msg}")
+                    
+                    async for chunk in response.aiter_bytes():
+                        buffer += chunk  # Agregar los datos al buffer
+                        while len(buffer) >= 4:  # Asegurarse de que al menos 4 bytes están disponibles
+                            obj_length = struct.unpack(">I", buffer[:4])[0]
+
+                            if len(buffer) < 4 + obj_length:
+                                break  # Esperar más datos si el objeto no está completo
+
+                            obj_data = buffer[4: 4 + obj_length]
+                            buffer = buffer[4 + obj_length:]  # Actualizar el buffer
+                            decompressed_data = gzip.decompress(obj_data)  # Descomprimir los datos
+                            data = msgpack.unpackb(decompressed_data, object_hook=decode_datetime)  # Deserializar el objeto
+                            
+                            await self._write_to_file(ikey, query, data)
+            
+                    logger.info(f"Page {page} downloaded")
+            
+    def find_one(self, ikey:str, query:dict) -> list[dict]:
+        try:
+            headers = {
+                "Authorization": f"A2G {self.token}",
+                "ikey": ikey,
+                'Content-Type': 'application/json'
+            }
+            res = requests.post(QUERY_MANAGER + "/QueryData/FindOne", 
+                data=json.dumps(query, cls=CustomJSONEncoder),
+                headers=headers, 
+                verify=verify_https
+            )
+            if res.status_code != 200: raise Exception(f"Error getting inputstream {res.status_code} {res.content}")
+            content = res.json(cls=CustomJsonDecoder)
+            if not content["success"]: raise Exception(content["errorMessage"])
+            return content["data"]
+        except Exception as e:
+            raise e
+        
+    def aggregate(self, ikey:str, pipeline: list[dict]) -> list[dict]:
+        try:
+            headers = {
+                "Authorization": f"A2G {self.token}",
+                "ikey": ikey,
+                'Content-Type': 'application/json'
+            }
+
+            if not all(isinstance(x, dict) for x in pipeline):      raise Exception("Invalid pipeline, the steps must be dictionaries")
+            if len(pipeline) == 0:                                  raise Exception("Invalid pipeline, length must be greater than 0" )
+            if any("$out" in x or "$merge" in x for x in pipeline): raise Exception("Invalid pipeline, write operations not allowed"  )
+
+            res = requests.post(f"{QUERY_MANAGER}/QueryData/ExecutionPlanningAggregate", 
+                data = json.dumps(pipeline, cls=CustomJSONEncoder), 
+                headers=headers, 
+                verify=verify_https
+            )
+            if res.status_code != 200: 
+                raise Exception(f"Error getting execution planning {res.status_code} {res.content}")
+            
+            content = res.json(cls=CustomJsonDecoder)
+            if not content["success"]: raise Exception(content["errorMessage"])
+            
+            total_query     = content["data"]["total"]
+            page_size       = content["data"]["size"]
+
+            total_batchs    = (total_query // page_size) + 1
+            logger.info(f"Total documents to download {total_query}.")
+            logger.info(f"Batch 1/{total_batchs}")
+
+            downloaded_docs = 0
+            page            = 1
+            total_batchs    = (total_query // page_size) + 1
+            docs            = []
+            while downloaded_docs < total_query:
+                res = requests.post(f"{QUERY_MANAGER}/QueryData/Aggregate", 
+                    data=json.dumps(pipeline, cls=CustomJSONEncoder),
+                    headers=headers, 
+                    verify=verify_https
+                )
+
+                if res.status_code != 200: raise Exception(f"Error getting inputstream data {res.status_code} {res.content}")
+                content = res.json(cls=CustomJsonDecoder)
+                if not content["success"]: raise Exception(content["errorMessage"])
+                logger.info(f"Batch {page}/{total_batchs}")
+                downloaded_docs += content["data"]["size"]
+                docs += content["data"]["data"]
+                page += 1
+                
+            logger.info(f"Data downloaded, total docs: {total_query}")    
+            return docs#content["data"]
+        except Exception as e:
+            raise e
+
+    def insert(self, ikey:str, data:list[dict], mode:INSERTION_MODE, wait_response:bool) -> tuple[int, str]:
+        try:
+            headers = {
+                "Authorization": f"A2G {self.token}",
+                "ikey": ikey
+            }
+
+            if mode == INSERTION_MODE.REPLACE: 
+                headers["Replace"] = "true"
+                headers["Transaction"] = "false"
+
+            elif mode == INSERTION_MODE.INSERT_UNORDERED: 
+                headers["Replace"] = "false"
+                headers["Transaction"] = "false"
+
+            elif mode == INSERTION_MODE.TRANSACTION:
+                headers["Replace"] = "false"
+                headers["Transaction"] = "true"
+
+            if wait_response: headers["WaitResponse"] = "true"
+
+            res = requests.post(DATA_URL + "/Data/Insert", headers=headers, json=data, verify=verify_https)
+            if res.status_code != 200: raise Exception(f"Error to insert data in inputstream {res.status_code} {res.text}")
+            return res.status_code, res.text
+        except Exception as e:
+            raise e
+
+    async def insert_data_native(self, ikey:str,table: str, data:list[dict], start: int, end:int, wait_response = False, cache:bool=True):
+        async with SEM:
+            """
+            validate data against inputstream JsonSchema and insert into inputstream collection
+            params:
+                ikey: str
+                table_name: str
+                data: list[dict]
+            """
+            timeout = httpx.Timeout(60.0, connect=10.0, read=600.0)
+            async with httpx.AsyncClient(http2=True, verify=False, timeout=timeout) as client:
+                headers = {
+                    "Authorization": f"A2G {self.token}",
+                    "ikey": ikey,
+                    'Content-Type': 'text/plain',
+                }
+
+                if type(data) is not list: raise Exception("Data must be a list of dictionaries")
+
+                """Envia un lote de datos al servidor"""
+                my_body = {
+                    "list_data": data[start:end],
+                    "table_name": table,
+                }
+                async with client.stream("POST",f"{QUERY_MANAGER}/QueryData/InsertAll",
+                    content=json.dumps(my_body, default=str),
+                    headers=headers,
+                ) as response:
+                    if response.status_code != 200:
+                        msg = ""
+                        async for chunk in response.aiter_bytes():
+                            if chunk:
+                                msg += chunk.decode("utf-8")
+                        raise Exception(f"{response.status_code} {msg}")
+
+    def remove_documents(self, ikey:str, query:dict) -> int:
+        try:
+            logger.info("Removing data...")
+            headers = {
+                "Authorization": f"A2G {self.token}",
+                "ikey": ikey,
+                'Content-Type': 'application/json'
+            }
+
+            if len(query) == 0: 
+                raise Exception("Query is empty, please provide a valid query, if you desire to delete all documents, use the delete_all method.")
+
+            response = requests.post(f"{QUERY_MANAGER}/QueryData/RemoveDocuments", 
+                data=json.dumps(query, cls=CustomJSONEncoder), 
+                headers=headers, 
+                verify=verify_https
+            )
+            if response.status_code != 200: 
+                raise Exception(f"Error to remove data in inputstream {response.status_code} {response.content}")
+            res_object = response.json(cls=CustomJsonDecoder)
+            if not res_object["success"]: raise Exception(res_object["errorMessage"])
+
+            content = res_object["data"]
+            deleted_docs = content["docs_affected"]
+            logger.info(f"Operation complete, total docs deleted: {deleted_docs}")
+
+            return deleted_docs
+        except Exception as e:
+            raise e
+
+    def clear_inputstream(self, ikey:str) -> int:
+        try:
+            logger.info("Removing all data...")
+            headers = {
+                "Authorization": f"A2G {self.token}",
+                "ikey": ikey
+            }
+
+            response = requests.post(f"{QUERY_MANAGER}/QueryData/Clear", headers=headers, verify=verify_https)
+            if response.status_code != 200:
+                raise Exception(f"Error to remove all data in inputstream {response.status_code} {response.content}")
+            res_object = response.json(cls=CustomJsonDecoder)
+            if not res_object["success"]: raise Exception(res_object["errorMessage"])
+            
+            content = res_object["data"]
+            deleted_docs = content["docs_affected"]
+            logger.info(f"Operation complete, total docs deleted: {deleted_docs}")
+            return deleted_docs
+        except Exception as e:
+            raise e
+
+
+class InputstreamClient:
+
+    def __init__(self, token:str, cache_options:dict = None):
+        """
+        Constructor for LocalInputstream
+        :param token: Token to authenticate with ACELER.AI
+        :param cache_options: { duration_data: int, duration_inputstream: int } | None
+        """        
+        self.acelerai_client = AcelerAIHttpClient(token) 
+        self.cache_manager = CacheManager(cache_options)
+        self.__mode = os.environ.get("EXEC_LOCATION", "LOCAL")
+
+    def __get_inputstream(self, ikey:str) -> Inputstream:
+        inputstream = self.acelerai_client.get_inputstream_by_ikey(ikey)
+        return inputstream
+    
+    async def __allPages(self, ikey, query,delete_id):
+        headers = {
+            "Authorization": f"A2G {self.acelerai_client.token}",
+            "ikey": ikey,
+            'Content-Type': 'application/json'
+        }
+
+        logger.info("Getting Execution Planning...")
+
+        res = requests.post(f"{QUERY_MANAGER}/QueryData/ExecutionPlanningFind", 
+            data = json.dumps(query, cls=CustomJSONEncoder), 
+            headers=headers, 
+            verify=verify_https
+        )
+
+        logger.info(f"Getting Status {res.status_code}")
+
+        if res.status_code != 200:
+            msg = ''
+            async for chunk in res.aiter_bytes():
+                if chunk:
+                    msg += chunk.decode("utf-8")
+            raise Exception(f"{res.status_code} {msg}")
+
+        inputstream = self.acelerai_client.get_inputstream_by_ikey(ikey)
+        logger.info(f"Getting inputstream with ikey: {ikey}, Name {inputstream.Name}  from ACELER.AI...")
+        
+        content = res.json(cls=CustomJsonDecoder)
+
+        total_query     = content["total"]
+        page_size:int   = content["size"]
+
+        if total_query == 0:
+            logger.info("No data found with the query provided.")
+            return []
+        
+        page: int = 1
+
+        if content['stage'] != None:
+            stage = content["stage"].replace('_',' -> ')
+            logger.info(f"The query stages are {stage}")
+            logger.info(F"The index used in query is {content['indexName']}")
+        
+        elif inputstream.InputstreamType !=InputstreamType.Native:
+            logger.info(f"Complex query the explain was not saved")
+        
+        tasks = []
+        total_pages = (total_query + page_size - 1) // page_size
+
+        if not os.path.exists(f".acelerai_cache/")               : os.mkdir(f".acelerai_cache/")
+        if not os.path.exists(f".acelerai_cache/data/")          : os.mkdir(f".acelerai_cache/data/")
+        if not os.path.exists(f".acelerai_cache/data/{ikey}/")   : os.mkdir(f".acelerai_cache/data/{ikey}/")
+
+        logger.info('Downloading data, please wait...')
+        start_time = datetime.utcnow()
+        for page in range(total_pages):
+            tasks.append(self.acelerai_client.fetch_page(ikey, query, delete_id, page, page_size))
+        await asyncio.gather(*tasks)
+
+        end_time = datetime.utcnow()
+        logger.info(f"Total time for downloading {total_pages} pages: {round((end_time - start_time).total_seconds()/60, 2)} minutes")
+    
+    async def __get_data(self, ikey:str, query, mode:str, cache:bool, delete_id:bool=True) -> list[dict]:
+        try:
+            logger.info(f"Cache: {cache}")
+            logger.info(f"Mode: {self.__mode}")
+            query_str = json.dumps(query, cls=CustomJSONEncoder)
+            query_hash = hashlib.sha256(query_str.encode()).hexdigest()
+
+            logger.info(f"Getting data sdadsadsadasdsad...")
+
+            data = self.cache_manager.get_data(ikey, query_hash) if cache and self.__mode=='LOCAL' else None
+            if data is None:
+                if   mode == "find"     :   await self.__allPages(ikey, query,delete_id)
+                elif mode == "find_one" :   data = self.acelerai_client.find_one(ikey, query)
+                elif mode == "aggregate":   data = self.acelerai_client.aggregate(ikey, query)
+                
+                self.cache_manager.set_data(ikey, query_hash)
+
+                output_file = f".acelerai_cache/data/{ikey}/{query_hash}.msgpack"
+                data = load_full_object(output_file)
+
+            return data
+        except Exception as e:
+            raise e
+
+    def get_inputstream_schema(self, ikey:str) -> dict:
+        """
+        return Inputstream schema
+        params:
+            ikey: str
+            cache: bool = True -> if True, use cache if exists and is not expired
+        """
+        inputstream = self.__get_inputstream(ikey)
+        return json.loads(inputstream.Schema)
+
+    async def find(self, ikey:str, query:dict,cache:bool=True,delete_id:bool=True):
+        """
+        return data from inputstream
+        params:
+            ikey: str
+            query: dict
+            cache: bool = True -> if True, use cache if exists and is not expired
+        """
+        mode = "find"
+        return await self.__get_data(ikey, query, mode, cache,delete_id)
+
+    def find_one(self, ikey:str, query:dict, cache:bool=True):
+        """
+        return one data from inputstream
+        params:
+            collection: str
+            query: dict
+        """
+        mode = "find_one"
+        return self.__get_data(ikey, query, mode, cache)   
+
+    def get_data_aggregate(self, ikey:str, query: list[dict], cache:bool=True):
+        """
+        return data from inputstream
+        params:
+            ikey: str
+            query: list[dict]
+        """
+        mode = "aggregate"
+        return self.__get_data(ikey, query, mode, cache)  
+
+    async def __validate_data_async(self, d, schema, schema_validator):
+        try:
+            schema_validator(d)
+            return None  # Si no hay errores, retornamos None
+        except Exception as e:
+            return f"Error validating data: {e}"  # Devolvemos el error
+
+    async def insert_data(self, ikey:str, data:list[dict], table: str=None, mode:INSERTION_MODE = INSERTION_MODE.REPLACE, wait_response = True, batch_size:int=1000, cache:bool=True):
+        """
+        validate data against inputstream JsonSchema and insert into inputstream collection
+        params:
+            ikey: str
+            data: list[dict]
+            table: str -> table is required only when inserting to a local inputstream
+            mode: INSERTION_MODE = INSERTION_MODE.REPLACE -> insertion mode
+            wait_response: bool = True -> if True, wait for response from server
+            batch_size: int = 1000 -> batch size for insert data
+            cache: bool = True -> if True, use cache if exists and is not expired
+        """
+        start = datetime.utcnow()
+        inputstream:Inputstream = self.__get_inputstream(ikey)
+        end = datetime.utcnow()
+
+        logger.info(f'Demoró {(end - start).total_seconds()} segs en obtener el inputstream')
+
+        if type(data) is not list: raise Exception("Data must be a list of dictionaries")
+        data_parsed = json.loads(json.dumps(data, cls=CustomJSONEncoder)) 
+
+        if inputstream.InputstreamType != InputstreamType.Native:
+            if inputstream.Status == InputstreamStatus.Undiscovered: #raise Exception("Inputstream undiscovered")
+                logger.info("Inputstream undiscovered, you must discovered the schema first")
+                return False
+            elif inputstream.Status == InputstreamStatus.Exposed: #raise Exception("Inputstream undiscovered")
+                start = datetime.utcnow()
+
+                schema = json.loads(inputstream.Schema)
+                schema_validator = fastjsonschema.compile(schema)
+                #schema_validator = Draft4Validator(schema=schema)
+                
+                resultados = await asyncio.gather(*(self.__validate_data_async(d, schema, schema_validator) for d in data_parsed))
+
+                # Manejo de errores
+                errores = [error for error in resultados if error is not None]
+                if errores:
+                    for error in errores:
+                        logger.error(error)
+                    raise Exception("Hubo errores durante la validación de datos.")
+                
+                end = datetime.utcnow()
+
+                logger.info(f'Demoró {(end - start).total_seconds()} segs en validar los datos')
+
+
+            for i in range(0, len(data_parsed), batch_size):
+                code, message = self.acelerai_client.insert(ikey, data_parsed[i:i+batch_size], mode, wait_response)
+                batch_size_aux = batch_size if i+batch_size < len(data_parsed) else len(data_parsed) - i
+                logger.info(f"batch {(i//batch_size) + 1}, docs: [{i} - {i+batch_size_aux}] - {code} - {message}")
+        else:
+            logger.info('Inserting data, please wait...')
+            if not table or table=='': raise Exception("Table name is required when inserting to a native inputstream")
+            start_time = datetime.utcnow()
+            await asyncio.gather(*(self.acelerai_client.insert_data_native(ikey, table, data_parsed, i, min(i + batch_size, len(data_parsed)), wait_response, cache) for i in range(0, len(data_parsed), batch_size)))
+            endt = datetime.utcnow()
+            logger.info(f'{len(data)} registries inserted successfully in {(endt - start_time).total_seconds() / 60} minutes')
+
+    def remove_documents(self, ikey:str, query:dict) -> int:
+        """
+        delete data from inputstream
+        params:
+            ikey: str
+            query: dict
+        """
+        docs = self.acelerai_client.remove_documents(ikey, query)
+        return docs
+ 
+    def clear_inputstream(self, ikey:str) -> int:
+        """
+        delete all data from inputstream
+        params:
+            ikey: str
+        """
+        docs = self.acelerai_client.clear_inputstream(ikey)
+        return docs
+    

acelerai-0.0.1/test/test.py

@@ -0,0 +1,20 @@
+import os
+# os.environ["DATA_URL"]          =  "https://localhost:1008"
+# os.environ["QUERY_MANAGER"]     =  "http://localhost:1012"
+# os.environ["INPUTSTREAM_URL"]   =  "https://localhost:1000"
+
+from  acelerai_inputstream import InputstreamClient
+import logging
+import pandas as pd
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
+
+#Para desactivar los logs de la librería
+logging.getLogger('InputstreamClient').disabled = False
+
+#pat_danilo='324738f335354e9d80394c31ce1d644c'
+client = InputstreamClient(token='084a45f10e3b4952a4f3a7df04f546e5') 
+
+input_cycle=client.find(ikey='74c337b0e99a45ac8a9c', query= {}, cache=False)
+df=pd.DataFrame(input_cycle)
+#print(len(df))

acelerai-0.0.1/test/test_app.py

@@ -0,0 +1,12 @@
+from acelerai_inputstream import InputstreamClient
+import logging
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
+
+#Para desactivar los logs de la librería
+#logging.getLogger('InputstreamClient').disabled = True
+client = InputstreamClient(token='7ffba0a8bccb4498be8a811b43bd9400') 
+input_cycle=client.find(ikey='31feaa0361b44ebb8b70', query= {}, cache=False)
+
+
+