erioon 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

erioon/functions.py

@@ -0,0 +1,350 @@
+import msgpack
+from azure.storage.blob import ContainerClient
+import json
+import uuid
+from io import BytesIO
+import datetime
+from threading import Thread
+
+def create_container_if_not_exists(container_name, container_url):
+    """
+    Checks if the Blob Storage container exists; if not, creates it.
+    
+    Args:
+        container_name: Name of the container to check/create.
+        container_url: SAS URL to the blob container.
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    if not container_client.exists():
+        container_client.create_container()
+
+def get_shard_file_name(user_id_cont, database, collection, container_url, next_shard_number=False):
+    """
+    Determines the filename of the current (or next) shard MessagePack file for writing data.
+    
+    The filename format is: {database}/{collection}/{collection}_{shard_number}.msgpack
+
+    Args:
+        user_id_cont: User identifier/context.
+        database: Database name.
+        collection: Collection name.
+        container_url: Blob Storage container SAS URL.
+        next_shard_number: If True, returns filename for the next shard (increment shard number).
+
+    Returns:
+        Filename string of the shard to be used.
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    
+    base_shard_name = f"{database}/{collection}/{collection}"
+    
+    files = container_client.list_blobs(name_starts_with=base_shard_name)
+    existing_shards = [int(blob.name.split('_')[-1].split('.')[0]) for blob in files if blob.name.endswith('.msgpack')]
+    
+    if existing_shards:
+        next_shard = max(existing_shards) + 1 if next_shard_number else max(existing_shards)
+    else:
+        next_shard = 1
+
+    return f"{base_shard_name}_{next_shard}.msgpack"
+
+def get_shard_limit(user_id_cont, database, collection, container_url):
+    """
+    Retrieves the maximum number of records allowed in a single shard from the
+    collection_settings.json file, or returns a default limit if file doesn't exist.
+
+    Args:
+        user_id_cont: User identifier/context.
+        database: Database name.
+        collection: Collection name.
+        container_url: Blob Storage container SAS URL.
+
+    Returns:
+        Integer shard limit (default 100000).
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    config_blob_client = container_client.get_blob_client(blob=f"{database}/{collection}/collection_settings.json")
+    
+    if not config_blob_client.exists():
+        return 100000
+    
+    config_data = json.loads(config_blob_client.download_blob().readall())
+    return config_data.get("shard_limit", 100000)
+
+def create_msgpack_file(user_id_cont, database, collection, data, container_url):
+    """
+    Writes the given record data into the appropriate MessagePack shard file.
+    Automatically manages shard rollover based on shard size limit.
+
+    Args:
+        user_id_cont: User identifier/context.
+        database: Database name.
+        collection: Collection name.
+        data: The record data dict to store.
+        container_url: Blob Storage container SAS URL.
+
+    Returns:
+        The filename of the shard where the record was stored.
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    
+    msgpack_filename = get_shard_file_name(user_id_cont, database, collection, container_url)
+
+    msgpack_blob_client = container_client.get_blob_client(blob=msgpack_filename)
+
+    existing_records = []
+    max_records_per_shard = get_shard_limit(user_id_cont, database, collection, container_url)
+
+    if msgpack_blob_client.exists():
+        with BytesIO(msgpack_blob_client.download_blob().readall()) as existing_file:
+            existing_records = msgpack.unpackb(existing_file.read(), raw=False)
+
+    if len(existing_records) >= max_records_per_shard:
+        msgpack_filename = get_shard_file_name(user_id_cont, database, collection, container_url, next_shard_number=True)
+        msgpack_blob_client = container_client.get_blob_client(blob=msgpack_filename)
+        existing_records = []  
+
+    existing_records.append(data)
+
+    with BytesIO() as out_file:
+        out_file.write(msgpack.packb(existing_records, use_bin_type=True))
+        out_file.seek(0)
+        msgpack_blob_client.upload_blob(out_file, overwrite=True)
+
+    return msgpack_filename
+
+def save_logs(user_id_cont, database, collection, method, log_type, log_message, count, container_url):
+    """
+    Saves an individual log entry into logs.json inside the container.
+    Each log entry is keyed by a UUID and includes metadata and timestamp.
+
+    Args:
+        user_id_cont: User identifier/context.
+        database: Database name.
+        collection: Collection name.
+        method: HTTP method or operation type (e.g. POST, GET).
+        log_type: Log type e.g. SUCCESS, ERROR.
+        log_message: Detailed message or data for the log.
+        count: Number of records affected or relevant.
+        container_url: Blob Storage container SAS URL.
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    blob_path = f"{database}/{collection}/logs.json"
+    index_blob_client = container_client.get_blob_client(blob=blob_path)
+
+    try:
+        existing_blob = index_blob_client.download_blob().readall()
+        logs_data = json.loads(existing_blob)
+    except Exception:
+        logs_data = {}
+
+    log_id = str(uuid.uuid4())
+
+    logs_data[log_id] = {
+        "timestamp": datetime.datetime.now().isoformat(),
+        "method": method.upper(),
+        "type": log_type.upper(),
+        "log": log_message,
+        "count": count
+    }
+
+    # Upload updated logs
+    index_blob_client.upload_blob(
+        data=json.dumps(logs_data, indent=2),
+        overwrite=True
+    )
+    
+def async_log(user_id, db, collection, method, status, message, count, container_url):
+    """
+    Executes the save_logs function asynchronously in a separate thread,
+    allowing non-blocking log operations.
+
+    Args:
+        user_id: User identifier/context.
+        db: Database name.
+        collection: Collection name.
+        method: Operation method.
+        status: Log status (SUCCESS, ERROR, etc.).
+        message: Log message or data.
+        count: Number of affected records.
+        container_url: Blob Storage container SAS URL.
+    """
+    Thread(target=save_logs, args=(user_id, db, collection, method, status, message, count, container_url)).start()
+
+def generate_unique_id(existing_ids):
+    """
+    Generates a new UUID string that does not collide with any IDs in existing_ids.
+
+    Args:
+        existing_ids: Iterable of already existing _id strings.
+
+    Returns:
+        Unique UUID string not in existing_ids.
+    """
+    while True:
+        new_id = str(uuid.uuid4()) 
+        if new_id not in existing_ids:
+            return new_id
+
+def update_index_file_insert(user_id_cont, database, collection, record_id, shard_number, container_url):
+    """
+    Updates index.json to register a newly inserted record_id under the appropriate shard.
+
+    The index.json structure is a list of dicts mapping shard names to list of record IDs:
+    [
+        { "collection_1": ["id1", "id2", ...] },
+        { "collection_2": ["id3", "id4", ...] }
+    ]
+
+    Args:
+        user_id_cont: User identifier/context.
+        database: Database name.
+        collection: Collection name.
+        record_id: The _id of the inserted record.
+        shard_number: The shard number where the record was stored.
+        container_url: Blob Storage container SAS URL.
+
+    Returns:
+        The record_id inserted.
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    index_blob_client = container_client.get_blob_client(blob=f"{database}/{collection}/index.json")
+
+    index_data = []
+
+    if index_blob_client.exists():
+        try:
+            index_data = json.loads(index_blob_client.download_blob().readall())
+        except Exception:
+            index_data = []
+
+    shard_key = f"{collection}_{shard_number}"
+    shard_found = False
+
+    for shard in index_data:
+        if shard_key in shard:
+            shard[shard_key].append(record_id)
+            shard_found = True
+            break
+
+    if not shard_found:
+        index_data.append({shard_key: [record_id]})
+
+    index_blob_client.upload_blob(json.dumps(index_data), overwrite=True)
+
+    return record_id
+
+def update_index_file_delete(user_id_cont, database, collection, record_id, shard_number, container_url):
+    """
+    Removes a record_id from the index.json under the correct shard upon deletion.
+
+    Cleans up empty shard entries after removal.
+
+    Args:
+        user_id_cont: User identifier/context.
+        database: Database name.
+        collection: Collection name.
+        record_id: The _id of the deleted record.
+        shard_number: The shard number from which to remove the record.
+        container_url: Blob Storage container SAS URL.
+
+    Returns:
+        The record_id deleted.
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    index_blob_client = container_client.get_blob_client(blob=f"{database}/{collection}/index.json")
+
+    index_data = []
+
+    if index_blob_client.exists():
+        try:
+            index_data = json.loads(index_blob_client.download_blob().readall())
+        except Exception:
+            index_data = []
+
+    shard_key = f"{collection}_{shard_number}"
+
+    for shard in index_data:
+        if shard_key in shard:
+            if record_id in shard[shard_key]:
+                shard[shard_key].remove(record_id)
+                if not shard[shard_key]:
+                    index_data.remove(shard)
+            break
+
+    index_blob_client.upload_blob(json.dumps(index_data), overwrite=True)
+
+    return record_id
+
+def calculate_shard_number(user_id_cont, database, collection, container_url):
+    """
+    Determines the shard number for storing a new record.
+
+    Logic:
+    - Lists existing shard files in the collection directory.
+    - Extracts shard numbers from filenames.
+    - Returns the highest shard number found, or 1 if none found.
+
+    Args:
+        user_id_cont: User identifier/context.
+        database: Database name.
+        collection: Collection name.
+        container_url: Blob Storage container SAS URL.
+
+    Returns:
+        Integer shard number to use.
+    """
+    container_client = ContainerClient.from_container_url(container_url)
+    
+    directory_path = f"{database}/{collection}/"
+    blob_list = container_client.list_blobs(name_starts_with=directory_path)
+    
+    shard_numbers = []
+    for blob in blob_list:
+        try:
+            parts = blob.name.split("_")
+            if blob.name.endswith(".msgpack"):
+                num = int(parts[1].split(".")[0])
+                shard_numbers.append(num)
+        except Exception:
+            continue 
+    if shard_numbers:
+        next_shard = max(shard_numbers)
+    else:
+        next_shard = 1
+    return next_shard
+
+def check_nested_key(data, key_path, value):
+    """
+    Recursively checks whether a nested key in a dictionary or list of dictionaries
+    matches the specified value.
+
+    Args:
+        data (dict or list): The data structure (dict or list of dicts) to search.
+        key_path (str): Dot-separated path to the nested key (e.g. "a.b.c").
+        value: The value to compare against.
+
+    Returns:
+        bool: True if the key exists at the nested path and equals the value, else False.
+    """
+    keys = key_path.split('.') 
+    
+    if not keys:
+        return False
+    
+    current_key = keys[0]
+    remaining_keys = keys[1:]
+
+    if isinstance(data, dict):
+        if current_key in data:
+            if not remaining_keys:
+                if data[current_key] == value:
+                    return True
+            else:
+                return check_nested_key(data[current_key], '.'.join(remaining_keys), value)
+    elif isinstance(data, list):
+        for item in data:
+            if isinstance(item, dict):
+                if check_nested_key(item, '.'.join(remaining_keys), value):
+                    return True
+    return False

erioon/ping.py

@@ -0,0 +1,37 @@
+from erioon.functions import async_log
+from azure.storage.blob import ContainerClient
+
+def handle_connection_ping(user_id, db_id, coll_id, container_url):
+    """
+    Checks if a specific collection exists within an Blob Storage container 
+    and logs the status of the connection attempt asynchronously.
+
+    Parameters:
+    - user_id (str): Identifier of the user making the request.
+    - db_id (str): Database identifier (used as a folder prefix).
+    - coll_id (str): Collection identifier (used as a folder prefix).
+    - container_url (str): URL of the Blob Storage container.
+
+    Returns:
+    - tuple(dict, int): A tuple containing a status dictionary and an HTTP status code.
+        - If collection is found, returns status "OK" and HTTP 200.
+        - If collection is missing, returns status "KO" with HTTP 404.
+        - On any exception, returns status "KO" with HTTP 500.
+    """
+    try:
+        container_client = ContainerClient.from_container_url(container_url)
+        directory_path = f"{db_id}/{coll_id}/"
+
+        blobs = container_client.list_blobs(name_starts_with=directory_path)
+        blob_names = [blob.name for blob in blobs]
+
+        if not blob_names:
+            async_log(user_id, db_id, coll_id, "PING", "ERROR", f"No collection {coll_id} found.", 1, container_url)
+            return {"status": "KO", "error": f"No collection {coll_id} found."}, 404
+
+        async_log(user_id, db_id, coll_id, "PING", "SUCCESS", "Connection successful", 1, container_url)
+        return {"status": "OK", "message": "Connection successful"}, 200
+
+    except Exception as e:
+        async_log(user_id, db_id, coll_id, "PING", "ERROR", f"Connection failed: {str(e)}", 1, container_url)
+        return {"status": "KO", "error": "Connection failed", "message": str(e)}, 500

erioon/read.py

@@ -0,0 +1,241 @@
+import io
+import msgpack
+from azure.storage.blob import ContainerClient
+from erioon.functions import async_log
+from sklearn.neighbors import KNeighborsClassifier
+
+
+def handle_get_all(user_id, db_id, coll_id, limit, container_url):
+    """
+    Retrieves up to a specified number of records from a collection stored in Blob Storage 
+    and logs the operation status asynchronously.
+
+    Parameters:
+    - user_id (str): Identifier of the user making the request.
+    - db_id (str): Database identifier (used as the directory prefix).
+    - coll_id (str): Collection identifier (subdirectory under the database).
+    - limit (int): Maximum number of records to retrieve (must not exceed 1,000,000).
+    - container_url (str): URL to the Blob Storage container.
+
+    Behavior:
+    - Scans all blobs in the specified collection path (`db_id/coll_id/`).
+    - Reads shard files, each containing a list of records.
+    - Skips duplicate records by checking their `_id`.
+    - Stops reading once the record limit is reached.
+    - Skips empty or non-conforming blobs gracefully.
+
+    Returns:
+    - tuple(dict, int): A tuple containing:
+        - A status dictionary with:
+            - "status": "OK" or "KO"
+            - "count": number of records returned (0 if none)
+            - "results": list of records (only for successful responses)
+            - "error": error message (on failure)
+        - HTTP status code:
+            - 200 if data is successfully returned.
+            - 404 if collection is missing or no data found.
+            - 500 on unexpected errors.
+    """
+    if limit > 1_000_000:
+        async_log(user_id, db_id, coll_id, "GET", "ERROR", "Limit of 1,000,000 exceeded", 1, container_url)
+        return {"status": "KO", "count": 0, "error": "Limit of 1,000,000 exceeded"}, 404
+
+    directory_path = f"{db_id}/{coll_id}/"
+    container_client = ContainerClient.from_container_url(container_url)
+
+    blob_list = container_client.list_blobs(name_starts_with=directory_path)
+    blob_names = [blob.name for blob in blob_list]
+
+    if not blob_names:
+        async_log(user_id, db_id, coll_id, "GET", "ERROR", f"No collection {coll_id} found.", 1, container_url)
+        return {"status": "KO", "count": 0, "error": f"No collection {coll_id} found."}, 404
+
+    results = []
+    seen_ids = set()
+
+    for blob in blob_names:
+        try:
+            if blob.endswith(".msgpack"):
+                blob_client = container_client.get_blob_client(blob)
+                msgpack_data = blob_client.download_blob().readall()
+
+                if not msgpack_data:
+                    continue
+
+                with io.BytesIO(msgpack_data) as buffer:
+                    unpacked_data = msgpack.unpackb(buffer.read(), raw=False)
+                    if isinstance(unpacked_data, list):
+                        for record in unpacked_data:
+                            if record["_id"] in seen_ids:
+                                continue
+
+                            results.append(record)
+                            seen_ids.add(record["_id"])
+
+                            if len(results) >= limit:
+                                async_log(user_id, db_id, coll_id, "GET", "SUCCESS", f"OK", len(results), container_url)
+                                return {"status": "OK", "count": len(results), "results": results}, 200
+
+        except Exception:
+            continue
+
+    if results:
+        async_log(user_id, db_id, coll_id, "GET", "SUCCESS", f"OK", len(results), container_url)
+        return {"status": "OK", "count": len(results), "results": results}, 200
+
+    async_log(user_id, db_id, coll_id, "GET", "ERROR", "No data found", 1, container_url)
+    return {"status": "KO", "count": 0, "error": "No data found"}, 404
+
+
+def handle_get_data(user_id, db_id, coll_id, search_criteria, limit, container_url):
+    """
+    Searches for records within a collection in Blob Storage that match specified search criteria,
+    and logs the query attempt asynchronously.
+
+    Parameters:
+    - user_id (str): Identifier of the user making the request.
+    - db_id (str): Database identifier (used as the directory prefix).
+    - coll_id (str): Collection identifier (subdirectory under the database).
+    - search_criteria (list[dict]): A list of key-value conditions to match (supports dot notation for nested keys).
+    - limit (int): Maximum number of matching records to return.
+    - container_url (str): URL to the Blob Storage container.
+
+    Behavior:
+    - Iterates over blobs in the collection path (`db_id/coll_id/`).
+    - Filters shard blobs containing lists of records.
+    - Each record is checked against all `search_criteria`.
+        - Supports nested key matching using dot notation (e.g., "user.name").
+    - Skips duplicates based on `_id`.
+    - Stops when enough matching records are found or blobs are exhausted.
+    - Handles and skips corrupted or unreadable blobs gracefully.
+
+    Returns:
+    - tuple(dict, int): A tuple containing:
+        - A status dictionary with:
+            - "status": "OK" or "KO"
+            - "count": number of records returned (0 if none)
+            - "results": list of records (only for successful responses)
+            - "error": error message (on failure)
+        - HTTP status code:
+            - 200 if matching records are found.
+            - 404 if the collection or matching data is not found.
+            - 500 on unexpected errors.
+    """
+    directory_path = f"{db_id}/{coll_id}/"
+    container_client = ContainerClient.from_container_url(container_url)
+
+    blob_list = container_client.list_blobs(name_starts_with=directory_path)
+    blob_names = [blob.name for blob in blob_list]
+
+    if not blob_names:
+        async_log(user_id, db_id, coll_id, "GET", "ERROR", f"No collection {coll_id} found.", 1, container_url)
+        return {"status": "KO", "count": 0, "error": f"No collection {coll_id} found."}, 404
+
+    results = []
+    seen_ids = set()
+
+    for blob in blob_names:
+        try:
+            if blob.endswith(".msgpack"):
+                blob_client = container_client.get_blob_client(blob)
+                msgpack_data = blob_client.download_blob().readall()
+
+                if not msgpack_data:
+                    continue
+
+                with io.BytesIO(msgpack_data) as buffer:
+                    unpacked_data = msgpack.unpackb(buffer.read(), raw=False)
+                    if isinstance(unpacked_data, list):
+                        for record in unpacked_data:
+                            if record["_id"] in seen_ids:
+                                continue
+
+                            match_found = False
+                            for criteria in search_criteria:
+                                print(criteria)
+                                key, value = list(criteria.items())[0]
+
+                                if key == "_id" and record.get("_id") == value:
+                                    match_found = True
+                                else:
+                                    keys = key.split(".")
+                                    nested_value = record
+                                    for k in keys:
+                                        if isinstance(nested_value, dict) and k in nested_value:
+                                            nested_value = nested_value[k]
+                                        else:
+                                            nested_value = None
+                                            break
+                                    if nested_value == value:
+                                        match_found = True
+
+                                if match_found:
+                                    results.append(record)
+                                    seen_ids.add(record["_id"])
+                                    if len(results) >= limit:
+                                        async_log(user_id, db_id, coll_id, "GET", "SUCCESS", "OK", len(results), container_url)
+                                        return {"status": "OK", "count": len(results), "results": results}, 200
+                                    break
+
+        except Exception:
+            continue
+
+    async_log(user_id, db_id, coll_id, "GET", "ERROR", "No matching record found", 1, container_url)
+    return {"status": "KO", "count": 0, "error": "No matching record found"}, 404
+
+
+def handle_classify_vector(user_id, db_id, coll_id, container_url, k=3):
+    # 1. Retrieve all data from collection
+    response, status = handle_get_all(user_id, db_id, coll_id, limit=10000, container_url=container_url)
+    if status != 200:
+        return {"status": "KO", "message": "Failed to fetch data for classification", "error": response.get("error", "")}, status
+
+    records = response.get("results", [])
+    if not records:
+        return {"status": "KO", "message": "No data found for classification"}, 404
+
+    # 2. Prepare dataset for classification
+    vectors = []
+    labels = []
+    unknown_vectors = []
+    unknown_ids = []
+
+    for rec in records:
+        vec = rec.get("vector")
+        meta = rec.get("metadata", {})
+        if not vec:
+            continue
+        vectors.append(vec)
+        labels.append(meta.get("class", "unknown"))
+    
+    # If some records don’t have classes, you might want to separate them or remove them
+    known_vectors = []
+    known_labels = []
+    for v, l in zip(vectors, labels):
+        if l != "unknown" and l is not None:
+            known_vectors.append(v)
+            known_labels.append(l)
+        else:
+            unknown_vectors.append(v)
+            unknown_ids.append(rec.get("_id"))
+
+    if not known_vectors:
+        return {"status": "KO", "message": "No labeled data for training classification"}, 404
+
+    # 3. Train k-NN classifier on known labeled vectors
+    knn = KNeighborsClassifier(n_neighbors=k)
+    knn.fit(known_vectors, known_labels)
+
+    # 4. Predict class for unknown vectors (if any)
+    predictions = {}
+    if unknown_vectors:
+        predicted_labels = knn.predict(unknown_vectors)
+        for _id, pred in zip(unknown_ids, predicted_labels):
+            predictions[_id] = pred
+
+    # 5. Return predictions (or full classification result)
+    return {
+        "status": "OK",
+        "message": f"Classification done on {len(unknown_vectors)} unknown vectors",
+        "predictions": predictions,
+    }, 200