OpenGeodeWeb-Back 6.13.0rc2__tar.gz → 6.13.0rc4__tar.gz

{opengeodeweb_back-6.13.0rc2 → opengeodeweb_back-6.13.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenGeodeWeb-Back
-Version: 6.13.0rc2
+Version: 6.13.0rc4
 Summary: OpenGeodeWeb-Back is an open source framework that proposes handy python functions and wrappers for the OpenGeode ecosystem
 Author-email: Geode-solutions <team-web@geode-solutions.com>
 Project-URL: Homepage, https://github.com/Geode-solutions/OpenGeodeWeb-Back

{opengeodeweb_back-6.13.0rc2 → opengeodeweb_back-6.13.0rc4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "OpenGeodeWeb-Back"
-version = "6.13.0-rc.2"
+version = "6.13.0-rc.4"
 dynamic = ["dependencies"]
 authors = [{ name = "Geode-solutions", email = "team-web@geode-solutions.com" }]
 description = "OpenGeodeWeb-Back is an open source framework that proposes handy python functions and wrappers for the OpenGeode ecosystem"

{opengeodeweb_back-6.13.0rc2 → opengeodeweb_back-6.13.0rc4}/src/OpenGeodeWeb_Back.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenGeodeWeb-Back
-Version: 6.13.0rc2
+Version: 6.13.0rc4
 Summary: OpenGeodeWeb-Back is an open source framework that proposes handy python functions and wrappers for the OpenGeode ecosystem
 Author-email: Geode-solutions <team-web@geode-solutions.com>
 Project-URL: Homepage, https://github.com/Geode-solutions/OpenGeodeWeb-Back

{opengeodeweb_back-6.13.0rc2 → opengeodeweb_back-6.13.0rc4}/src/opengeodeweb_back/routes/blueprint_routes.py

@@ -4,6 +4,7 @@ import os
 import time
 import shutil
 import math
+import typing
 from threading import Timer
 
 # Third party imports
@@ -63,6 +64,18 @@ def allowed_files() -> flask.Response:
     return flask.make_response({"extensions": list(extensions)}, 200)
 
 
+def _write_stream(path: str, stream: typing.IO[bytes], mode: str = "wb") -> None:
+    read_buffer_size = 1024 * 1024
+    with open(path, mode) as destination:
+        while chunk := stream.read(read_buffer_size):
+            destination.write(chunk)
+
+
+def _finalize_upload(filename: str) -> flask.Response:
+    print(f"{filename=}", flush=True)
+    return flask.make_response({"message": "File uploaded"}, 201)
+
+
 @routes.route(
     schemas_dict["upload_file"]["route"],
     methods=schemas_dict["upload_file"]["methods"],
@@ -73,8 +86,12 @@ def upload_file() -> flask.Response:
     if not os.path.exists(UPLOAD_FOLDER_PATH):
         os.makedirs(UPLOAD_FOLDER_PATH, exist_ok=True)
 
-    # Multipart callers (e.g. Vease) still send the file as a "file" form part;
-    # streaming callers PUT the raw bytes as the body with ?filename= as a query param.
+    # Multipart callers (e.g. Vease) still send the whole file as a "file" form
+    # part. Everything else PUTs raw bytes with ?filename= as a query param:
+    # either the whole file in one request, or one of several chunks (when
+    # ?chunk_index=/?total_chunks= are also present) that get appended in
+    # order and assembled into the final file once the last one arrives. This
+    # keeps every request under cloud hosting's hard request-size limit.
     if flask.request.mimetype == "multipart/form-data":
         file = flask.request.files["file"]
         if file.filename is None:
@@ -82,22 +99,30 @@ def upload_file() -> flask.Response:
         filename = werkzeug.utils.secure_filename(os.path.basename(file.filename))
         file_path = os.path.join(UPLOAD_FOLDER_PATH, filename)
         file.save(file_path)
-    else:
-        raw_filename = flask.request.args.get("filename")
-        if not raw_filename:
-            flask.abort(400, "Filename is required")
-        filename = werkzeug.utils.secure_filename(os.path.basename(raw_filename))
-        file_path = os.path.join(UPLOAD_FOLDER_PATH, filename)
-        chunk_size = 1024 * 1024
-        with open(file_path, "wb") as destination:
-            while chunk := flask.request.stream.read(chunk_size):
-                destination.write(chunk)
-    print(f"{filename=}", flush=True)
-    if filename.lower().endswith(".csv.json"):
-        shutil.copyfile(
-            file_path, os.path.join(UPLOAD_FOLDER_PATH, filename[:-9] + ".json")
-        )
-    return flask.make_response({"message": "File uploaded"}, 201)
+        return _finalize_upload(filename)
+
+    raw_filename = flask.request.args.get("filename")
+    if not raw_filename:
+        flask.abort(400, "Filename is required")
+    filename = werkzeug.utils.secure_filename(os.path.basename(raw_filename))
+    file_path = os.path.join(UPLOAD_FOLDER_PATH, filename)
+
+    total_chunks = flask.request.args.get("total_chunks", type=int)
+    if total_chunks is None:
+        _write_stream(file_path, flask.request.stream)
+        return _finalize_upload(filename)
+
+    chunk_index = flask.request.args.get("chunk_index", type=int)
+    if chunk_index is None or not 0 <= chunk_index < total_chunks:
+        flask.abort(400, "Invalid chunk_index")
+
+    part_path = f"{file_path}.part"
+    _write_stream(part_path, flask.request.stream, "wb" if chunk_index == 0 else "ab")
+    if chunk_index < total_chunks - 1:
+        return flask.make_response({"message": "Chunk received"}, 200)
+
+    os.replace(part_path, file_path)
+    return _finalize_upload(filename)
 
 
 @routes.route(
@@ -270,23 +295,42 @@ def extract_valid_attribute_values(
     attribute_name: str,
     item_index: int,
 ) -> tuple[list[float], bool]:
-    attribute_ids_matching_name = attribute_manager.attribute_ids_matching_name(
-        attribute_name
-    )
-    if not isinstance(attribute_ids_matching_name, list):
+    attribute_ids = attribute_manager.attribute_ids_matching_name(attribute_name)
+    if not isinstance(attribute_ids, list):
         return [], False
-    component_attribute = attribute_manager.find_generic_attribute(
-        attribute_ids_matching_name[0]
-    )
-    if component_attribute is None:
-        return [], False
-    if not component_attribute.is_genericable():
+    attribute = attribute_manager.find_generic_attribute(attribute_ids[0])
+    if (
+        attribute is None
+        or not attribute.is_genericable()
+        or not attribute.properties().transferable
+    ):
         return [], False
+
+    nb_items = attribute.nb_items()
+    default_values = getattr(attribute, "default_values", None)
+    no_value = default_values().no_value if default_values else None
+    if no_value is None and (
+        "Point" in attribute.type() or "Vector" in attribute.type()
+    ):
+        no_value = [0.0] * nb_items
+
+    value_getter = getattr(
+        attribute,
+        "value",
+        lambda element_index: [
+            attribute.generic_item_value(element_index, i) for i in range(nb_items)
+        ],
+    )
+
     valid_values: list[float] = []
     has_nan = False
     for element_index in range(attribute_manager.nb_elements()):
-        value = component_attribute.generic_item_value(element_index, item_index)
-        if value is None or math.isnan(value):
+        value = attribute.generic_item_value(element_index, item_index)
+        if (
+            value is None
+            or math.isnan(value)
+            or (no_value is not None and value_getter(element_index) == no_value)
+        ):
             has_nan = True
         else:
             valid_values.append(value)
@@ -306,7 +350,7 @@ def attributes_metadata(
         attribute_name = attribute.name()
         if attribute_name is None:
             continue
-        if not attribute.is_genericable():
+        if not attribute.is_genericable() or not attribute.properties().transferable:
             continue
         nb_items = attribute.nb_items()
         min_values, max_values = [], []

{opengeodeweb_back-6.13.0rc2 → opengeodeweb_back-6.13.0rc4}/tests/test_routes.py

@@ -2,6 +2,7 @@
 import os
 
 # Third party imports
+import opengeode as og
 from werkzeug.datastructures import FileStorage
 from flask.testing import FlaskClient
 from werkzeug.test import TestResponse
@@ -13,6 +14,7 @@ import zipfile
 from opengeodeweb_microservice.database.data import Data
 from opengeodeweb_microservice.database.connection import get_session
 from opengeodeweb_back import geode_functions, test_utils
+from opengeodeweb_back.routes.blueprint_routes import extract_valid_attribute_values
 from opengeodeweb_back.geode_objects.geode_polygonal_surface3d import (
     GeodePolygonalSurface3D,
 )
@@ -109,6 +111,60 @@ def test_upload_file_raw_missing_filename(client: FlaskClient) -> None:
     assert response.status_code == 400
 
 
+def test_upload_file_chunked(
+    client: FlaskClient, filename: str = "test.og_brep"
+) -> None:
+    file = os.path.join(data_dir, filename)
+    with open(file, "rb") as opened_file:
+        file_bytes = opened_file.read()
+
+    chunk_filename = "chunked_upload_test.og_brep"
+    chunk_size = max(1, len(file_bytes) // 3)
+    chunks = [
+        file_bytes[index : index + chunk_size]
+        for index in range(0, len(file_bytes), chunk_size)
+    ]
+    total_chunks = len(chunks)
+
+    uploaded_path = os.path.join(data_dir, chunk_filename)
+    try:
+        for chunk_index, chunk in enumerate(chunks):
+            response = client.put(
+                f"/opengeodeweb_back/upload_file"
+                f"?filename={chunk_filename}"
+                f"&chunk_index={chunk_index}"
+                f"&total_chunks={total_chunks}",
+                data=chunk,
+                content_type="application/octet-stream",
+            )
+            if chunk_index < total_chunks - 1:
+                assert response.status_code == 200
+                assert not os.path.exists(uploaded_path)
+            else:
+                assert response.status_code == 201
+
+        with open(uploaded_path, "rb") as uploaded_file:
+            assert uploaded_file.read() == file_bytes
+    finally:
+        if os.path.exists(uploaded_path):
+            os.remove(uploaded_path)
+        part_path = f"{uploaded_path}.part"
+        if os.path.exists(part_path):
+            os.remove(part_path)
+
+
+def test_upload_file_chunked_invalid_chunk_index(client: FlaskClient) -> None:
+    response = client.put(
+        "/opengeodeweb_back/upload_file"
+        "?filename=invalid_chunk_index.og_brep"
+        "&chunk_index=2"
+        "&total_chunks=2",
+        data=b"some raw bytes",
+        content_type="application/octet-stream",
+    )
+    assert response.status_code == 400
+
+
 def test_missing_files(client: FlaskClient) -> None:
     route = f"/opengeodeweb_back/missing_files"
 
@@ -777,3 +833,47 @@ def test_model_component_polyhedron_attribute_names(client: FlaskClient) -> None
         return {"id": model_id, "component_ids": block_ids}
 
     test_utils.test_route_wrong_params(client, route, get_full_data)
+
+
+def test_extract_valid_attribute_values_with_sentinel_no_value() -> None:
+    mesh = og.TriangulatedSurface3D.create()
+    builder = og.TriangulatedSurfaceBuilder3D.create(mesh)
+    vertex_0 = builder.create_point(og.Point3D([0, 0, 0]))
+    vertex_1 = builder.create_point(og.Point3D([1, 0, 0]))
+    builder.create_triangle([vertex_0, vertex_1, vertex_0])
+
+    attribute_manager = mesh.vertex_attribute_manager()
+    values_config = og.AttributeValuesDouble()
+    values_config.no_value = -999.0
+    attribute_id = attribute_manager.create_attribute_variable_double(
+        "variable_double", values_config, og.AttributeProperties()
+    )
+    attribute = attribute_manager.find_attribute_variable_double(attribute_id)
+    attribute.set_value(vertex_0, -999.0)
+    attribute.set_value(vertex_1, 42.0)
+
+    valid_values, has_nan = extract_valid_attribute_values(
+        attribute_manager, "variable_double", 0
+    )
+    assert has_nan is True
+    assert valid_values == [42.0]
+
+
+def test_extract_valid_attribute_values_with_non_transferable_attribute() -> None:
+    mesh = og.TriangulatedSurface3D.create()
+    builder = og.TriangulatedSurfaceBuilder3D.create(mesh)
+    vertex_0 = builder.create_point(og.Point3D([0, 0, 0]))
+    builder.create_triangle([vertex_0, vertex_0, vertex_0])
+
+    attribute_manager = mesh.vertex_attribute_manager()
+    properties = og.AttributeProperties()
+    properties.transferable = False
+    attribute_manager.create_attribute_variable_double(
+        "internal_attribute", og.AttributeValuesDouble(), properties
+    )
+
+    valid_values, has_nan = extract_valid_attribute_values(
+        attribute_manager, "internal_attribute", 0
+    )
+    assert has_nan is False
+    assert valid_values == []