erioon 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

erioon/functions.py

@@ -1,350 +0,0 @@
-import msgpack
-from azure.storage.blob import ContainerClient
-import json
-import uuid
-from io import BytesIO
-import datetime
-from threading import Thread
-
-def create_container_if_not_exists(container_name, container_url):
-    """
-    Checks if the Blob Storage container exists; if not, creates it.
-    
-    Args:
-        container_name: Name of the container to check/create.
-        container_url: SAS URL to the blob container.
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    if not container_client.exists():
-        container_client.create_container()
-
-def get_shard_file_name(user_id_cont, database, collection, container_url, next_shard_number=False):
-    """
-    Determines the filename of the current (or next) shard MessagePack file for writing data.
-    
-    The filename format is: {database}/{collection}/{collection}_{shard_number}.msgpack
-
-    Args:
-        user_id_cont: User identifier/context.
-        database: Database name.
-        collection: Collection name.
-        container_url: Blob Storage container SAS URL.
-        next_shard_number: If True, returns filename for the next shard (increment shard number).
-
-    Returns:
-        Filename string of the shard to be used.
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    
-    base_shard_name = f"{database}/{collection}/{collection}"
-    
-    files = container_client.list_blobs(name_starts_with=base_shard_name)
-    existing_shards = [int(blob.name.split('_')[-1].split('.')[0]) for blob in files if blob.name.endswith('.msgpack')]
-    
-    if existing_shards:
-        next_shard = max(existing_shards) + 1 if next_shard_number else max(existing_shards)
-    else:
-        next_shard = 1
-
-    return f"{base_shard_name}_{next_shard}.msgpack"
-
-def get_shard_limit(user_id_cont, database, collection, container_url):
-    """
-    Retrieves the maximum number of records allowed in a single shard from the
-    collection_settings.json file, or returns a default limit if file doesn't exist.
-
-    Args:
-        user_id_cont: User identifier/context.
-        database: Database name.
-        collection: Collection name.
-        container_url: Blob Storage container SAS URL.
-
-    Returns:
-        Integer shard limit (default 100000).
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    config_blob_client = container_client.get_blob_client(blob=f"{database}/{collection}/collection_settings.json")
-    
-    if not config_blob_client.exists():
-        return 100000
-    
-    config_data = json.loads(config_blob_client.download_blob().readall())
-    return config_data.get("shard_limit", 100000)
-
-def create_msgpack_file(user_id_cont, database, collection, data, container_url):
-    """
-    Writes the given record data into the appropriate MessagePack shard file.
-    Automatically manages shard rollover based on shard size limit.
-
-    Args:
-        user_id_cont: User identifier/context.
-        database: Database name.
-        collection: Collection name.
-        data: The record data dict to store.
-        container_url: Blob Storage container SAS URL.
-
-    Returns:
-        The filename of the shard where the record was stored.
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    
-    msgpack_filename = get_shard_file_name(user_id_cont, database, collection, container_url)
-
-    msgpack_blob_client = container_client.get_blob_client(blob=msgpack_filename)
-
-    existing_records = []
-    max_records_per_shard = get_shard_limit(user_id_cont, database, collection, container_url)
-
-    if msgpack_blob_client.exists():
-        with BytesIO(msgpack_blob_client.download_blob().readall()) as existing_file:
-            existing_records = msgpack.unpackb(existing_file.read(), raw=False)
-
-    if len(existing_records) >= max_records_per_shard:
-        msgpack_filename = get_shard_file_name(user_id_cont, database, collection, container_url, next_shard_number=True)
-        msgpack_blob_client = container_client.get_blob_client(blob=msgpack_filename)
-        existing_records = []  
-
-    existing_records.append(data)
-
-    with BytesIO() as out_file:
-        out_file.write(msgpack.packb(existing_records, use_bin_type=True))
-        out_file.seek(0)
-        msgpack_blob_client.upload_blob(out_file, overwrite=True)
-
-    return msgpack_filename
-
-def save_logs(user_id_cont, database, collection, method, log_type, log_message, count, container_url):
-    """
-    Saves an individual log entry into logs.json inside the container.
-    Each log entry is keyed by a UUID and includes metadata and timestamp.
-
-    Args:
-        user_id_cont: User identifier/context.
-        database: Database name.
-        collection: Collection name.
-        method: HTTP method or operation type (e.g. POST, GET).
-        log_type: Log type e.g. SUCCESS, ERROR.
-        log_message: Detailed message or data for the log.
-        count: Number of records affected or relevant.
-        container_url: Blob Storage container SAS URL.
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    blob_path = f"{database}/{collection}/logs.json"
-    index_blob_client = container_client.get_blob_client(blob=blob_path)
-
-    try:
-        existing_blob = index_blob_client.download_blob().readall()
-        logs_data = json.loads(existing_blob)
-    except Exception:
-        logs_data = {}
-
-    log_id = str(uuid.uuid4())
-
-    logs_data[log_id] = {
-        "timestamp": datetime.datetime.now().isoformat(),
-        "method": method.upper(),
-        "type": log_type.upper(),
-        "log": log_message,
-        "count": count
-    }
-
-    # Upload updated logs
-    index_blob_client.upload_blob(
-        data=json.dumps(logs_data, indent=2),
-        overwrite=True
-    )
-    
-def async_log(user_id, db, collection, method, status, message, count, container_url):
-    """
-    Executes the save_logs function asynchronously in a separate thread,
-    allowing non-blocking log operations.
-
-    Args:
-        user_id: User identifier/context.
-        db: Database name.
-        collection: Collection name.
-        method: Operation method.
-        status: Log status (SUCCESS, ERROR, etc.).
-        message: Log message or data.
-        count: Number of affected records.
-        container_url: Blob Storage container SAS URL.
-    """
-    Thread(target=save_logs, args=(user_id, db, collection, method, status, message, count, container_url)).start()
-
-def generate_unique_id(existing_ids):
-    """
-    Generates a new UUID string that does not collide with any IDs in existing_ids.
-
-    Args:
-        existing_ids: Iterable of already existing _id strings.
-
-    Returns:
-        Unique UUID string not in existing_ids.
-    """
-    while True:
-        new_id = str(uuid.uuid4()) 
-        if new_id not in existing_ids:
-            return new_id
-
-def update_index_file_insert(user_id_cont, database, collection, record_id, shard_number, container_url):
-    """
-    Updates index.json to register a newly inserted record_id under the appropriate shard.
-
-    The index.json structure is a list of dicts mapping shard names to list of record IDs:
-    [
-        { "collection_1": ["id1", "id2", ...] },
-        { "collection_2": ["id3", "id4", ...] }
-    ]
-
-    Args:
-        user_id_cont: User identifier/context.
-        database: Database name.
-        collection: Collection name.
-        record_id: The _id of the inserted record.
-        shard_number: The shard number where the record was stored.
-        container_url: Blob Storage container SAS URL.
-
-    Returns:
-        The record_id inserted.
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    index_blob_client = container_client.get_blob_client(blob=f"{database}/{collection}/index.json")
-
-    index_data = []
-
-    if index_blob_client.exists():
-        try:
-            index_data = json.loads(index_blob_client.download_blob().readall())
-        except Exception:
-            index_data = []
-
-    shard_key = f"{collection}_{shard_number}"
-    shard_found = False
-
-    for shard in index_data:
-        if shard_key in shard:
-            shard[shard_key].append(record_id)
-            shard_found = True
-            break
-
-    if not shard_found:
-        index_data.append({shard_key: [record_id]})
-
-    index_blob_client.upload_blob(json.dumps(index_data), overwrite=True)
-
-    return record_id
-
-def update_index_file_delete(user_id_cont, database, collection, record_id, shard_number, container_url):
-    """
-    Removes a record_id from the index.json under the correct shard upon deletion.
-
-    Cleans up empty shard entries after removal.
-
-    Args:
-        user_id_cont: User identifier/context.
-        database: Database name.
-        collection: Collection name.
-        record_id: The _id of the deleted record.
-        shard_number: The shard number from which to remove the record.
-        container_url: Blob Storage container SAS URL.
-
-    Returns:
-        The record_id deleted.
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    index_blob_client = container_client.get_blob_client(blob=f"{database}/{collection}/index.json")
-
-    index_data = []
-
-    if index_blob_client.exists():
-        try:
-            index_data = json.loads(index_blob_client.download_blob().readall())
-        except Exception:
-            index_data = []
-
-    shard_key = f"{collection}_{shard_number}"
-
-    for shard in index_data:
-        if shard_key in shard:
-            if record_id in shard[shard_key]:
-                shard[shard_key].remove(record_id)
-                if not shard[shard_key]:
-                    index_data.remove(shard)
-            break
-
-    index_blob_client.upload_blob(json.dumps(index_data), overwrite=True)
-
-    return record_id
-
-def calculate_shard_number(user_id_cont, database, collection, container_url):
-    """
-    Determines the shard number for storing a new record.
-
-    Logic:
-    - Lists existing shard files in the collection directory.
-    - Extracts shard numbers from filenames.
-    - Returns the highest shard number found, or 1 if none found.
-
-    Args:
-        user_id_cont: User identifier/context.
-        database: Database name.
-        collection: Collection name.
-        container_url: Blob Storage container SAS URL.
-
-    Returns:
-        Integer shard number to use.
-    """
-    container_client = ContainerClient.from_container_url(container_url)
-    
-    directory_path = f"{database}/{collection}/"
-    blob_list = container_client.list_blobs(name_starts_with=directory_path)
-    
-    shard_numbers = []
-    for blob in blob_list:
-        try:
-            parts = blob.name.split("_")
-            if blob.name.endswith(".msgpack"):
-                num = int(parts[1].split(".")[0])
-                shard_numbers.append(num)
-        except Exception:
-            continue 
-    if shard_numbers:
-        next_shard = max(shard_numbers)
-    else:
-        next_shard = 1
-    return next_shard
-
-def check_nested_key(data, key_path, value):
-    """
-    Recursively checks whether a nested key in a dictionary or list of dictionaries
-    matches the specified value.
-
-    Args:
-        data (dict or list): The data structure (dict or list of dicts) to search.
-        key_path (str): Dot-separated path to the nested key (e.g. "a.b.c").
-        value: The value to compare against.
-
-    Returns:
-        bool: True if the key exists at the nested path and equals the value, else False.
-    """
-    keys = key_path.split('.') 
-    
-    if not keys:
-        return False
-    
-    current_key = keys[0]
-    remaining_keys = keys[1:]
-
-    if isinstance(data, dict):
-        if current_key in data:
-            if not remaining_keys:
-                if data[current_key] == value:
-                    return True
-            else:
-                return check_nested_key(data[current_key], '.'.join(remaining_keys), value)
-    elif isinstance(data, list):
-        for item in data:
-            if isinstance(item, dict):
-                if check_nested_key(item, '.'.join(remaining_keys), value):
-                    return True
-    return False

erioon/ping.py

@@ -1,37 +0,0 @@
-from erioon.functions import async_log
-from azure.storage.blob import ContainerClient
-
-def handle_connection_ping(user_id, db_id, coll_id, container_url):
-    """
-    Checks if a specific collection exists within an Blob Storage container 
-    and logs the status of the connection attempt asynchronously.
-
-    Parameters:
-    - user_id (str): Identifier of the user making the request.
-    - db_id (str): Database identifier (used as a folder prefix).
-    - coll_id (str): Collection identifier (used as a folder prefix).
-    - container_url (str): URL of the Blob Storage container.
-
-    Returns:
-    - tuple(dict, int): A tuple containing a status dictionary and an HTTP status code.
-        - If collection is found, returns status "OK" and HTTP 200.
-        - If collection is missing, returns status "KO" with HTTP 404.
-        - On any exception, returns status "KO" with HTTP 500.
-    """
-    try:
-        container_client = ContainerClient.from_container_url(container_url)
-        directory_path = f"{db_id}/{coll_id}/"
-
-        blobs = container_client.list_blobs(name_starts_with=directory_path)
-        blob_names = [blob.name for blob in blobs]
-
-        if not blob_names:
-            async_log(user_id, db_id, coll_id, "PING", "ERROR", f"No collection {coll_id} found.", 1, container_url)
-            return {"status": "KO", "error": f"No collection {coll_id} found."}, 404
-
-        async_log(user_id, db_id, coll_id, "PING", "SUCCESS", "Connection successful", 1, container_url)
-        return {"status": "OK", "message": "Connection successful"}, 200
-
-    except Exception as e:
-        async_log(user_id, db_id, coll_id, "PING", "ERROR", f"Connection failed: {str(e)}", 1, container_url)
-        return {"status": "KO", "error": "Connection failed", "message": str(e)}, 500

erioon/read.py

@@ -1,241 +0,0 @@
-import io
-import msgpack
-from azure.storage.blob import ContainerClient
-from erioon.functions import async_log
-from sklearn.neighbors import KNeighborsClassifier
-
-
-def handle_get_all(user_id, db_id, coll_id, limit, container_url):
-    """
-    Retrieves up to a specified number of records from a collection stored in Blob Storage 
-    and logs the operation status asynchronously.
-
-    Parameters:
-    - user_id (str): Identifier of the user making the request.
-    - db_id (str): Database identifier (used as the directory prefix).
-    - coll_id (str): Collection identifier (subdirectory under the database).
-    - limit (int): Maximum number of records to retrieve (must not exceed 1,000,000).
-    - container_url (str): URL to the Blob Storage container.
-
-    Behavior:
-    - Scans all blobs in the specified collection path (`db_id/coll_id/`).
-    - Reads shard files, each containing a list of records.
-    - Skips duplicate records by checking their `_id`.
-    - Stops reading once the record limit is reached.
-    - Skips empty or non-conforming blobs gracefully.
-
-    Returns:
-    - tuple(dict, int): A tuple containing:
-        - A status dictionary with:
-            - "status": "OK" or "KO"
-            - "count": number of records returned (0 if none)
-            - "results": list of records (only for successful responses)
-            - "error": error message (on failure)
-        - HTTP status code:
-            - 200 if data is successfully returned.
-            - 404 if collection is missing or no data found.
-            - 500 on unexpected errors.
-    """
-    if limit > 1_000_000:
-        async_log(user_id, db_id, coll_id, "GET", "ERROR", "Limit of 1,000,000 exceeded", 1, container_url)
-        return {"status": "KO", "count": 0, "error": "Limit of 1,000,000 exceeded"}, 404
-
-    directory_path = f"{db_id}/{coll_id}/"
-    container_client = ContainerClient.from_container_url(container_url)
-
-    blob_list = container_client.list_blobs(name_starts_with=directory_path)
-    blob_names = [blob.name for blob in blob_list]
-
-    if not blob_names:
-        async_log(user_id, db_id, coll_id, "GET", "ERROR", f"No collection {coll_id} found.", 1, container_url)
-        return {"status": "KO", "count": 0, "error": f"No collection {coll_id} found."}, 404
-
-    results = []
-    seen_ids = set()
-
-    for blob in blob_names:
-        try:
-            if blob.endswith(".msgpack"):
-                blob_client = container_client.get_blob_client(blob)
-                msgpack_data = blob_client.download_blob().readall()
-
-                if not msgpack_data:
-                    continue
-
-                with io.BytesIO(msgpack_data) as buffer:
-                    unpacked_data = msgpack.unpackb(buffer.read(), raw=False)
-                    if isinstance(unpacked_data, list):
-                        for record in unpacked_data:
-                            if record["_id"] in seen_ids:
-                                continue
-
-                            results.append(record)
-                            seen_ids.add(record["_id"])
-
-                            if len(results) >= limit:
-                                async_log(user_id, db_id, coll_id, "GET", "SUCCESS", f"OK", len(results), container_url)
-                                return {"status": "OK", "count": len(results), "results": results}, 200
-
-        except Exception:
-            continue
-
-    if results:
-        async_log(user_id, db_id, coll_id, "GET", "SUCCESS", f"OK", len(results), container_url)
-        return {"status": "OK", "count": len(results), "results": results}, 200
-
-    async_log(user_id, db_id, coll_id, "GET", "ERROR", "No data found", 1, container_url)
-    return {"status": "KO", "count": 0, "error": "No data found"}, 404
-
-
-def handle_get_data(user_id, db_id, coll_id, search_criteria, limit, container_url):
-    """
-    Searches for records within a collection in Blob Storage that match specified search criteria,
-    and logs the query attempt asynchronously.
-
-    Parameters:
-    - user_id (str): Identifier of the user making the request.
-    - db_id (str): Database identifier (used as the directory prefix).
-    - coll_id (str): Collection identifier (subdirectory under the database).
-    - search_criteria (list[dict]): A list of key-value conditions to match (supports dot notation for nested keys).
-    - limit (int): Maximum number of matching records to return.
-    - container_url (str): URL to the Blob Storage container.
-
-    Behavior:
-    - Iterates over blobs in the collection path (`db_id/coll_id/`).
-    - Filters shard blobs containing lists of records.
-    - Each record is checked against all `search_criteria`.
-        - Supports nested key matching using dot notation (e.g., "user.name").
-    - Skips duplicates based on `_id`.
-    - Stops when enough matching records are found or blobs are exhausted.
-    - Handles and skips corrupted or unreadable blobs gracefully.
-
-    Returns:
-    - tuple(dict, int): A tuple containing:
-        - A status dictionary with:
-            - "status": "OK" or "KO"
-            - "count": number of records returned (0 if none)
-            - "results": list of records (only for successful responses)
-            - "error": error message (on failure)
-        - HTTP status code:
-            - 200 if matching records are found.
-            - 404 if the collection or matching data is not found.
-            - 500 on unexpected errors.
-    """
-    directory_path = f"{db_id}/{coll_id}/"
-    container_client = ContainerClient.from_container_url(container_url)
-
-    blob_list = container_client.list_blobs(name_starts_with=directory_path)
-    blob_names = [blob.name for blob in blob_list]
-
-    if not blob_names:
-        async_log(user_id, db_id, coll_id, "GET", "ERROR", f"No collection {coll_id} found.", 1, container_url)
-        return {"status": "KO", "count": 0, "error": f"No collection {coll_id} found."}, 404
-
-    results = []
-    seen_ids = set()
-
-    for blob in blob_names:
-        try:
-            if blob.endswith(".msgpack"):
-                blob_client = container_client.get_blob_client(blob)
-                msgpack_data = blob_client.download_blob().readall()
-
-                if not msgpack_data:
-                    continue
-
-                with io.BytesIO(msgpack_data) as buffer:
-                    unpacked_data = msgpack.unpackb(buffer.read(), raw=False)
-                    if isinstance(unpacked_data, list):
-                        for record in unpacked_data:
-                            if record["_id"] in seen_ids:
-                                continue
-
-                            match_found = False
-                            for criteria in search_criteria:
-                                print(criteria)
-                                key, value = list(criteria.items())[0]
-
-                                if key == "_id" and record.get("_id") == value:
-                                    match_found = True
-                                else:
-                                    keys = key.split(".")
-                                    nested_value = record
-                                    for k in keys:
-                                        if isinstance(nested_value, dict) and k in nested_value:
-                                            nested_value = nested_value[k]
-                                        else:
-                                            nested_value = None
-                                            break
-                                    if nested_value == value:
-                                        match_found = True
-
-                                if match_found:
-                                    results.append(record)
-                                    seen_ids.add(record["_id"])
-                                    if len(results) >= limit:
-                                        async_log(user_id, db_id, coll_id, "GET", "SUCCESS", "OK", len(results), container_url)
-                                        return {"status": "OK", "count": len(results), "results": results}, 200
-                                    break
-
-        except Exception:
-            continue
-
-    async_log(user_id, db_id, coll_id, "GET", "ERROR", "No matching record found", 1, container_url)
-    return {"status": "KO", "count": 0, "error": "No matching record found"}, 404
-
-
-def handle_classify_vector(user_id, db_id, coll_id, container_url, k=3):
-    # 1. Retrieve all data from collection
-    response, status = handle_get_all(user_id, db_id, coll_id, limit=10000, container_url=container_url)
-    if status != 200:
-        return {"status": "KO", "message": "Failed to fetch data for classification", "error": response.get("error", "")}, status
-
-    records = response.get("results", [])
-    if not records:
-        return {"status": "KO", "message": "No data found for classification"}, 404
-
-    # 2. Prepare dataset for classification
-    vectors = []
-    labels = []
-    unknown_vectors = []
-    unknown_ids = []
-
-    for rec in records:
-        vec = rec.get("vector")
-        meta = rec.get("metadata", {})
-        if not vec:
-            continue
-        vectors.append(vec)
-        labels.append(meta.get("class", "unknown"))
-    
-    # If some records don’t have classes, you might want to separate them or remove them
-    known_vectors = []
-    known_labels = []
-    for v, l in zip(vectors, labels):
-        if l != "unknown" and l is not None:
-            known_vectors.append(v)
-            known_labels.append(l)
-        else:
-            unknown_vectors.append(v)
-            unknown_ids.append(rec.get("_id"))
-
-    if not known_vectors:
-        return {"status": "KO", "message": "No labeled data for training classification"}, 404
-
-    # 3. Train k-NN classifier on known labeled vectors
-    knn = KNeighborsClassifier(n_neighbors=k)
-    knn.fit(known_vectors, known_labels)
-
-    # 4. Predict class for unknown vectors (if any)
-    predictions = {}
-    if unknown_vectors:
-        predicted_labels = knn.predict(unknown_vectors)
-        for _id, pred in zip(unknown_ids, predicted_labels):
-            predictions[_id] = pred
-
-    # 5. Return predictions (or full classification result)
-    return {
-        "status": "OK",
-        "message": f"Classification done on {len(unknown_vectors)} unknown vectors",
-        "predictions": predictions,
-    }, 200