OpenGeodeWeb-Back 6.12.5rc2__tar.gz → 6.13.0__tar.gz

{opengeodeweb_back-6.12.5rc2 → opengeodeweb_back-6.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenGeodeWeb-Back
-Version: 6.12.5rc2
+Version: 6.13.0
 Summary: OpenGeodeWeb-Back is an open source framework that proposes handy python functions and wrappers for the OpenGeode ecosystem
 Author-email: Geode-solutions <team-web@geode-solutions.com>
 Project-URL: Homepage, https://github.com/Geode-solutions/OpenGeodeWeb-Back

{opengeodeweb_back-6.12.5rc2 → opengeodeweb_back-6.13.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "OpenGeodeWeb-Back"
-version = "6.12.5-rc.2"
+version = "6.13.0"
 dynamic = ["dependencies"]
 authors = [{ name = "Geode-solutions", email = "team-web@geode-solutions.com" }]
 description = "OpenGeodeWeb-Back is an open source framework that proposes handy python functions and wrappers for the OpenGeode ecosystem"

{opengeodeweb_back-6.12.5rc2 → opengeodeweb_back-6.13.0}/src/OpenGeodeWeb_Back.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenGeodeWeb-Back
-Version: 6.12.5rc2
+Version: 6.13.0
 Summary: OpenGeodeWeb-Back is an open source framework that proposes handy python functions and wrappers for the OpenGeode ecosystem
 Author-email: Geode-solutions <team-web@geode-solutions.com>
 Project-URL: Homepage, https://github.com/Geode-solutions/OpenGeodeWeb-Back

{opengeodeweb_back-6.12.5rc2 → opengeodeweb_back-6.13.0}/src/opengeodeweb_back/routes/blueprint_routes.py

@@ -4,6 +4,7 @@ import os
 import time
 import shutil
 import math
+import typing
 from threading import Timer
 
 # Third party imports
@@ -63,6 +64,18 @@ def allowed_files() -> flask.Response:
     return flask.make_response({"extensions": list(extensions)}, 200)
 
 
+def _write_stream(path: str, stream: typing.IO[bytes], mode: str = "wb") -> None:
+    read_buffer_size = 1024 * 1024
+    with open(path, mode) as destination:
+        while chunk := stream.read(read_buffer_size):
+            destination.write(chunk)
+
+
+def _finalize_upload(filename: str) -> flask.Response:
+    print(f"{filename=}", flush=True)
+    return flask.make_response({"message": "File uploaded"}, 201)
+
+
 @routes.route(
     schemas_dict["upload_file"]["route"],
     methods=schemas_dict["upload_file"]["methods"],
@@ -73,18 +86,43 @@ def upload_file() -> flask.Response:
     if not os.path.exists(UPLOAD_FOLDER_PATH):
         os.makedirs(UPLOAD_FOLDER_PATH, exist_ok=True)
 
-    file = flask.request.files["file"]
-    if file.filename is None:
+    # Multipart callers (e.g. Vease) still send the whole file as a "file" form
+    # part. Everything else PUTs raw bytes with ?filename= as a query param:
+    # either the whole file in one request, or one of several chunks (when
+    # ?chunk_index=/?total_chunks= are also present) that get appended in
+    # order and assembled into the final file once the last one arrives. This
+    # keeps every request under cloud hosting's hard request-size limit.
+    if flask.request.mimetype == "multipart/form-data":
+        file = flask.request.files["file"]
+        if file.filename is None:
+            flask.abort(400, "Filename is required")
+        filename = werkzeug.utils.secure_filename(os.path.basename(file.filename))
+        file_path = os.path.join(UPLOAD_FOLDER_PATH, filename)
+        file.save(file_path)
+        return _finalize_upload(filename)
+
+    raw_filename = flask.request.args.get("filename")
+    if not raw_filename:
         flask.abort(400, "Filename is required")
-    filename = werkzeug.utils.secure_filename(os.path.basename(file.filename))
-    print(f"{filename=}", flush=True)
+    filename = werkzeug.utils.secure_filename(os.path.basename(raw_filename))
     file_path = os.path.join(UPLOAD_FOLDER_PATH, filename)
-    file.save(file_path)
-    if filename.lower().endswith(".csv.json"):
-        shutil.copyfile(
-            file_path, os.path.join(UPLOAD_FOLDER_PATH, filename[:-9] + ".json")
-        )
-    return flask.make_response({"message": "File uploaded"}, 201)
+
+    total_chunks = flask.request.args.get("total_chunks", type=int)
+    if total_chunks is None:
+        _write_stream(file_path, flask.request.stream)
+        return _finalize_upload(filename)
+
+    chunk_index = flask.request.args.get("chunk_index", type=int)
+    if chunk_index is None or not 0 <= chunk_index < total_chunks:
+        flask.abort(400, "Invalid chunk_index")
+
+    part_path = f"{file_path}.part"
+    _write_stream(part_path, flask.request.stream, "wb" if chunk_index == 0 else "ab")
+    if chunk_index < total_chunks - 1:
+        return flask.make_response({"message": "Chunk received"}, 200)
+
+    os.replace(part_path, file_path)
+    return _finalize_upload(filename)
 
 
 @routes.route(
@@ -256,32 +294,54 @@ def extract_valid_attribute_values(
     attribute_manager: og.AttributeManager,
     attribute_name: str,
     item_index: int,
-) -> list[float]:
-    attribute_ids_matching_name = attribute_manager.attribute_ids_matching_name(
-        attribute_name
-    )
-    if not isinstance(attribute_ids_matching_name, list):
-        return []
-    component_attribute = attribute_manager.find_generic_attribute(
-        attribute_ids_matching_name[0]
-    )
-    if component_attribute is None:
-        return []
-    if not component_attribute.is_genericable():
-        return []
+) -> tuple[list[float], bool]:
+    attribute_ids = attribute_manager.attribute_ids_matching_name(attribute_name)
+    if not isinstance(attribute_ids, list):
+        return [], False
+    attribute = attribute_manager.find_generic_attribute(attribute_ids[0])
+    if (
+        attribute is None
+        or not attribute.is_genericable()
+        or not attribute.properties().transferable
+    ):
+        return [], False
+
+    nb_items = attribute.nb_items()
+    default_values = getattr(attribute, "default_values", None)
+    no_value = default_values().no_value if default_values else None
+    if no_value is None and (
+        "Point" in attribute.type() or "Vector" in attribute.type()
+    ):
+        no_value = [0.0] * nb_items
+
+    value_getter = getattr(
+        attribute,
+        "value",
+        lambda element_index: [
+            attribute.generic_item_value(element_index, i) for i in range(nb_items)
+        ],
+    )
+
     valid_values: list[float] = []
+    has_nan = False
     for element_index in range(attribute_manager.nb_elements()):
-        value = component_attribute.generic_item_value(element_index, item_index)
-        if value is not None and not math.isnan(value):
+        value = attribute.generic_item_value(element_index, item_index)
+        if (
+            value is None
+            or math.isnan(value)
+            or (no_value is not None and value_getter(element_index) == no_value)
+        ):
+            has_nan = True
+        else:
             valid_values.append(value)
-    return valid_values
+    return valid_values, has_nan
 
 
 def attributes_metadata(
     manager: og.AttributeManager | list[og.AttributeManager],
-) -> list[dict[str, str | int | float | list[float]]]:
+) -> list[dict[str, str | int | float | bool | list[float]]]:
     attribute_managers = manager if isinstance(manager, list) else [manager]
-    attributes: list[dict[str, str | int | float | list[float]]] = []
+    attributes: list[dict[str, str | int | float | bool | list[float]]] = []
     first_manager = attribute_managers[0]
     for id in first_manager.attribute_ids():
         attribute = first_manager.find_generic_attribute(id)
@@ -290,18 +350,20 @@ def attributes_metadata(
         attribute_name = attribute.name()
         if attribute_name is None:
             continue
-        if not attribute.is_genericable():
+        if not attribute.is_genericable() or not attribute.properties().transferable:
             continue
         nb_items = attribute.nb_items()
         min_values, max_values = [], []
+        attribute_has_nan = False
         for item_index in range(nb_items):
             valid_values: list[float] = []
             for attribute_manager in attribute_managers:
-                valid_values.extend(
-                    extract_valid_attribute_values(
-                        attribute_manager, attribute_name, item_index
-                    )
+                extracted_values, has_nan = extract_valid_attribute_values(
+                    attribute_manager, attribute_name, item_index
                 )
+                valid_values.extend(extracted_values)
+                if has_nan:
+                    attribute_has_nan = True
             if valid_values:
                 min_values.append(min(valid_values))
                 max_values.append(max(valid_values))
@@ -316,6 +378,7 @@ def attributes_metadata(
                 "max_value": max(max_values),
                 "min_values": min_values,
                 "max_values": max_values,
+                "no_data": attribute_has_nan,
             }
         )
     return attributes

{opengeodeweb_back-6.12.5rc2 → opengeodeweb_back-6.13.0}/tests/test_routes.py

@@ -2,6 +2,7 @@
 import os
 
 # Third party imports
+import opengeode as og
 from werkzeug.datastructures import FileStorage
 from flask.testing import FlaskClient
 from werkzeug.test import TestResponse
@@ -13,6 +14,7 @@ import zipfile
 from opengeodeweb_microservice.database.data import Data
 from opengeodeweb_microservice.database.connection import get_session
 from opengeodeweb_back import geode_functions, test_utils
+from opengeodeweb_back.routes.blueprint_routes import extract_valid_attribute_values
 from opengeodeweb_back.geode_objects.geode_polygonal_surface3d import (
     GeodePolygonalSurface3D,
 )
@@ -79,6 +81,90 @@ def test_upload_file(client: FlaskClient, filename: str = "test.og_brep") -> Non
     assert response.status_code == 201
 
 
+def test_upload_file_raw(client: FlaskClient, filename: str = "test.og_brep") -> None:
+    file = os.path.join(data_dir, filename)
+    with open(file, "rb") as opened_file:
+        file_bytes = opened_file.read()
+
+    raw_filename = "raw_upload_test.og_brep"
+    response = client.put(
+        f"/opengeodeweb_back/upload_file?filename={raw_filename}",
+        data=file_bytes,
+        content_type="application/octet-stream",
+    )
+    assert response.status_code == 201
+
+    uploaded_path = os.path.join(data_dir, raw_filename)
+    try:
+        with open(uploaded_path, "rb") as uploaded_file:
+            assert uploaded_file.read() == file_bytes
+    finally:
+        os.remove(uploaded_path)
+
+
+def test_upload_file_raw_missing_filename(client: FlaskClient) -> None:
+    response = client.put(
+        f"/opengeodeweb_back/upload_file",
+        data=b"some raw bytes",
+        content_type="application/octet-stream",
+    )
+    assert response.status_code == 400
+
+
+def test_upload_file_chunked(
+    client: FlaskClient, filename: str = "test.og_brep"
+) -> None:
+    file = os.path.join(data_dir, filename)
+    with open(file, "rb") as opened_file:
+        file_bytes = opened_file.read()
+
+    chunk_filename = "chunked_upload_test.og_brep"
+    chunk_size = max(1, len(file_bytes) // 3)
+    chunks = [
+        file_bytes[index : index + chunk_size]
+        for index in range(0, len(file_bytes), chunk_size)
+    ]
+    total_chunks = len(chunks)
+
+    uploaded_path = os.path.join(data_dir, chunk_filename)
+    try:
+        for chunk_index, chunk in enumerate(chunks):
+            response = client.put(
+                f"/opengeodeweb_back/upload_file"
+                f"?filename={chunk_filename}"
+                f"&chunk_index={chunk_index}"
+                f"&total_chunks={total_chunks}",
+                data=chunk,
+                content_type="application/octet-stream",
+            )
+            if chunk_index < total_chunks - 1:
+                assert response.status_code == 200
+                assert not os.path.exists(uploaded_path)
+            else:
+                assert response.status_code == 201
+
+        with open(uploaded_path, "rb") as uploaded_file:
+            assert uploaded_file.read() == file_bytes
+    finally:
+        if os.path.exists(uploaded_path):
+            os.remove(uploaded_path)
+        part_path = f"{uploaded_path}.part"
+        if os.path.exists(part_path):
+            os.remove(part_path)
+
+
+def test_upload_file_chunked_invalid_chunk_index(client: FlaskClient) -> None:
+    response = client.put(
+        "/opengeodeweb_back/upload_file"
+        "?filename=invalid_chunk_index.og_brep"
+        "&chunk_index=2"
+        "&total_chunks=2",
+        data=b"some raw bytes",
+        content_type="application/octet-stream",
+    )
+    assert response.status_code == 400
+
+
 def test_missing_files(client: FlaskClient) -> None:
     route = f"/opengeodeweb_back/missing_files"
 
@@ -261,6 +347,8 @@ def test_vertex_attribute_names(client: FlaskClient, test_id: str) -> None:
         assert "nb_items" in attribute
         assert "min_values" in attribute
         assert "max_values" in attribute
+        assert "no_data" in attribute
+        assert isinstance(attribute["no_data"], bool)
     print(
         f"[ATTRIBUTES]: ",
         [attribute["nb_items"] for attribute in attributes],
@@ -297,6 +385,8 @@ def test_cell_attribute_names(client: FlaskClient, test_id: str) -> None:
         assert "nb_items" in attribute
         assert "min_values" in attribute
         assert "max_values" in attribute
+        assert "no_data" in attribute
+        assert isinstance(attribute["no_data"], bool)
     print(
         f"[ATTRIBUTES]: ",
         [attribute["nb_items"] for attribute in attributes],
@@ -333,6 +423,8 @@ def test_polygon_attribute_names(client: FlaskClient, test_id: str) -> None:
         assert "nb_items" in attribute
         assert "min_values" in attribute
         assert "max_values" in attribute
+        assert "no_data" in attribute
+        assert isinstance(attribute["no_data"], bool)
     print(
         f"[ATTRIBUTES]: ",
         [attribute["nb_items"] for attribute in attributes],
@@ -370,6 +462,8 @@ def test_polyhedron_attribute_names(client: FlaskClient, test_id: str) -> None:
         assert "nb_items" in attribute
         assert "min_values" in attribute
         assert "max_values" in attribute
+        assert "no_data" in attribute
+        assert isinstance(attribute["no_data"], bool)
         if attribute["attribute_name"] == "Range":
             assert attribute["min_value"] == 0.0
             assert attribute["max_value"] == 579.0
@@ -411,6 +505,8 @@ def test_edge_attribute_names(client: FlaskClient, test_id: str) -> None:
         assert "nb_items" in attribute
         assert "min_values" in attribute
         assert "max_values" in attribute
+        assert "no_data" in attribute
+        assert isinstance(attribute["no_data"], bool)
     print(
         f"[ATTRIBUTES]: ",
         [attribute["nb_items"] for attribute in attributes],
@@ -485,7 +581,7 @@ def test_model_components(client: FlaskClient) -> None:
     assert isinstance(mesh_components, list)
     assert len(mesh_components) > 0
     for mesh_component in mesh_components:
-        assert isinstance(mesh_component, object)
+        assert isinstance(mesh_component, dict)
         assert isinstance(mesh_component["geode_id"], str)
         assert isinstance(mesh_component["viewer_id"], int)
         assert isinstance(mesh_component["name"], str)
@@ -501,7 +597,7 @@ def test_model_components(client: FlaskClient) -> None:
     collection_components = response.get_json()["collection_components"]
     assert isinstance(collection_components, list)
     for collection_component in collection_components:
-        assert isinstance(collection_component, object)
+        assert isinstance(collection_component, dict)
         assert isinstance(collection_component["geode_id"], str)
         assert isinstance(collection_component["name"], str)
         assert isinstance(collection_component["items"], list)
@@ -649,15 +745,18 @@ def _load_brep_components(client: FlaskClient) -> tuple[str, dict[str, list[str]
     response = test_save_viewable_file(client, "BRep", "cube.og_brep")
     assert response.status_code == 200
     model_id: str = response.get_json()["id"]
-    mesh_components: list[dict] = response.get_json()["mesh_components"]
+    mesh_components: list[dict[str, object]] = response.get_json()["mesh_components"]
     by_type: dict[str, list[str]] = {}
-    for mc in mesh_components:
-        component_type = mc["type"]
-        by_type.setdefault(component_type, []).append(mc["geode_id"])
+    for mesh_component in mesh_components:
+        component_type = mesh_component["type"]
+        geode_id = mesh_component["geode_id"]
+        assert isinstance(component_type, str)
+        assert isinstance(geode_id, str)
+        by_type.setdefault(component_type, []).append(geode_id)
     return model_id, by_type
 
 
-def _assert_attributes_response(response) -> None:
+def _assert_attributes_response(response: TestResponse) -> None:
     assert response.status_code == 200
     attributes = response.get_json()["attributes"]
     assert isinstance(attributes, list)
@@ -668,6 +767,8 @@ def _assert_attributes_response(response) -> None:
         assert "nb_items" in attribute
         assert "min_values" in attribute
         assert "max_values" in attribute
+        assert "no_data" in attribute
+        assert isinstance(attribute["no_data"], bool)
 
 
 def test_model_component_vertex_attribute_names(client: FlaskClient) -> None:
@@ -732,3 +833,47 @@ def test_model_component_polyhedron_attribute_names(client: FlaskClient) -> None
         return {"id": model_id, "component_ids": block_ids}
 
     test_utils.test_route_wrong_params(client, route, get_full_data)
+
+
+def test_extract_valid_attribute_values_with_sentinel_no_value() -> None:
+    mesh = og.TriangulatedSurface3D.create()
+    builder = og.TriangulatedSurfaceBuilder3D.create(mesh)
+    vertex_0 = builder.create_point(og.Point3D([0, 0, 0]))
+    vertex_1 = builder.create_point(og.Point3D([1, 0, 0]))
+    builder.create_triangle([vertex_0, vertex_1, vertex_0])
+
+    attribute_manager = mesh.vertex_attribute_manager()
+    values_config = og.AttributeValuesDouble()
+    values_config.no_value = -999.0
+    attribute_id = attribute_manager.create_attribute_variable_double(
+        "variable_double", values_config, og.AttributeProperties()
+    )
+    attribute = attribute_manager.find_attribute_variable_double(attribute_id)
+    attribute.set_value(vertex_0, -999.0)
+    attribute.set_value(vertex_1, 42.0)
+
+    valid_values, has_nan = extract_valid_attribute_values(
+        attribute_manager, "variable_double", 0
+    )
+    assert has_nan is True
+    assert valid_values == [42.0]
+
+
+def test_extract_valid_attribute_values_with_non_transferable_attribute() -> None:
+    mesh = og.TriangulatedSurface3D.create()
+    builder = og.TriangulatedSurfaceBuilder3D.create(mesh)
+    vertex_0 = builder.create_point(og.Point3D([0, 0, 0]))
+    builder.create_triangle([vertex_0, vertex_0, vertex_0])
+
+    attribute_manager = mesh.vertex_attribute_manager()
+    properties = og.AttributeProperties()
+    properties.transferable = False
+    attribute_manager.create_attribute_variable_double(
+        "internal_attribute", og.AttributeValuesDouble(), properties
+    )
+
+    valid_values, has_nan = extract_valid_attribute_values(
+        attribute_manager, "internal_attribute", 0
+    )
+    assert has_nan is False
+    assert valid_values == []