sera-2 1.21.2__py3-none-any.whl → 1.24.1__py3-none-any.whl

sera/libs/search_helper.py

@@ -0,0 +1,359 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Sequence
+
+import msgspec
+from litestar import status_codes
+from litestar.exceptions import HTTPException
+
+from sera.misc import to_snake_case
+from sera.models import Cardinality, Class, DataProperty, ObjectProperty
+from sera.typing import FieldName, doc
+
+if TYPE_CHECKING:
+    from sera.libs.base_service import QueryResult
+
+"""Providing helpers to implement search functionality in HTTP POST request."""
+
+
+class QueryOp(str, Enum):
+    lt = "lt"
+    lte = "lte"
+    gt = "gt"
+    gte = "gte"
+    eq = "eq"
+    ne = "ne"
+    # select records where values are in the given list
+    in_ = "in"
+    not_in = "not_in"
+    # for full text search
+    fuzzy = "fuzzy"
+
+
+class QueryClause(msgspec.Struct):
+    field: FieldName
+    op: QueryOp
+    value: Annotated[Any, doc("query value")]
+
+
+class FieldOrderClause(msgspec.Struct):
+    field: FieldName
+    order: Annotated[Literal["asc", "desc"], doc("order direction, 'asc' or 'desc'")]
+    prop: Optional[str] = None
+
+
+class FieldGroupClause(msgspec.Struct):
+    field: FieldName
+    prop: Optional[str] = None
+
+
+# queries conditions written in CNF form.
+QueryCondition = Annotated[
+    Sequence[QueryClause],
+    doc("query operations written in CNF form"),
+]
+
+
+class JoinCondition(msgspec.Struct):
+    # name of the property in the primary class
+    prop: str
+    join_type: Annotated[
+        Literal["inner", "left", "full"],
+        doc("join type, 'inner', 'left', or 'full'"),
+    ] = msgspec.field(default="inner")
+    fields: Sequence[FieldName] = msgspec.field(default_factory=list)
+    conditions: QueryCondition = msgspec.field(default_factory=list)
+
+
+class Query(msgspec.Struct):
+    # list of fields to return in the results
+    fields: Sequence[FieldName] = msgspec.field(default_factory=list)
+    # conditions to filter the records
+    conditions: QueryCondition = msgspec.field(default_factory=list)
+    # sort the records by a field or multiple fields
+    sorted_by: Sequence[FieldOrderClause] = msgspec.field(default_factory=list)
+    # group the records by a field or multiple fields
+    group_by: Sequence[FieldGroupClause] = msgspec.field(default_factory=list)
+    # join with another classes
+    join_conditions: Sequence[JoinCondition] = msgspec.field(default_factory=list)
+    # whether to return unique records
+    unique: bool = False
+    # maximum number of records to return
+    limit: Annotated[int, msgspec.Meta(le=1000, ge=1)] = 10
+    # number of records to skip before returning results
+    offset: Annotated[int, msgspec.Meta(ge=0)] = 0
+    # whether to return the total number of records that match the query
+    return_total: bool = False
+
+    def validate_and_normalize(
+        self,
+        cls: Class,
+        allowed_fields: set[str],
+        allowed_join_fields: Optional[dict[str, set[str]]] = None,
+        debug: bool = False,
+    ):
+        """Validate query against the schema and normalize the field values.
+
+        Args:
+            cls: The class schema
+            allowed_fields: The set of allowed search field names
+            allowed_join_fields: The dict of allowed search fields in joined tables
+            debug: Whether to enable debug mode
+        """
+        if allowed_join_fields is None:
+            allowed_join_fields = {}
+
+        for field_name in self.fields:
+            if field_name not in cls.properties:
+                if debug:
+                    raise HTTPException(
+                        status_code=status_codes.HTTP_400_BAD_REQUEST,
+                        detail=f"Invalid field name: {field_name}",
+                    )
+                continue
+        for groupfield in self.group_by:
+            if groupfield.field not in allowed_fields:
+                if debug:
+                    raise HTTPException(
+                        status_code=status_codes.HTTP_400_BAD_REQUEST,
+                        detail=f"Invalid group by field name: {groupfield.field}",
+                    )
+                continue
+        for sortfield in self.sorted_by:
+            if sortfield.field not in allowed_fields:
+                if debug:
+                    raise HTTPException(
+                        status_code=status_codes.HTTP_400_BAD_REQUEST,
+                        detail=f"Invalid sort by field name: {sortfield.field}",
+                    )
+                continue
+        for clause in self.conditions:
+            # TODO: check if the operation is allowed for the field
+            if clause.field not in allowed_fields:
+                if debug:
+                    raise HTTPException(
+                        status_code=status_codes.HTTP_400_BAD_REQUEST,
+                        detail=f"Invalid condition field name: {clause.field}",
+                    )
+                continue
+
+            # normalize the value based on the field type.
+            prop = cls.properties[clause.field]
+            assert isinstance(prop, DataProperty)
+            if prop.datatype.pytype.is_enum_type():
+                # skip enum types -- we directly use it as string.
+                continue
+            clause.value = FieldTypeValidator.typemap[prop.datatype.pytype.type](
+                clause.field, clause.value
+            )
+
+        for join_clause in self.join_conditions:
+            if join_clause.prop not in allowed_join_fields:
+                if debug:
+                    raise HTTPException(
+                        status_code=status_codes.HTTP_400_BAD_REQUEST,
+                        detail=f"Invalid join property: {join_clause.prop}",
+                    )
+                continue
+
+            target_prop = cls.properties[join_clause.prop]
+            if (
+                isinstance(target_prop, DataProperty)
+                and target_prop.db is not None
+                and target_prop.db.foreign_key is not None
+            ):
+                # we have this case where ID is also a foreign key
+                target_class = target_prop.db.foreign_key
+            elif (
+                isinstance(target_prop, ObjectProperty)
+                and target_prop.target.db is not None
+            ):
+                target_class = target_prop.target
+            else:
+                if debug:
+                    raise HTTPException(
+                        status_code=status_codes.HTTP_400_BAD_REQUEST,
+                        detail=f"Invalid join property: {join_clause.prop}",
+                    )
+                continue
+
+            for field in join_clause.fields:
+                if field not in target_class.properties:
+                    if debug:
+                        raise HTTPException(
+                            status_code=status_codes.HTTP_400_BAD_REQUEST,
+                            detail=f"Invalid join field: {join_clause.prop}.{field}",
+                        )
+                    continue
+                for condition in join_clause.conditions:
+                    if condition.field not in allowed_join_fields[join_clause.prop]:
+                        if debug:
+                            raise HTTPException(
+                                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                                detail=f"Invalid join condition field: {join_clause.prop}.{condition.field}",
+                            )
+                        continue
+
+                    # normalize the value based on the field type.
+                    target_prop = target_class.properties[condition.field]
+                    assert isinstance(target_prop, DataProperty)
+                    if target_prop.datatype.pytype.is_enum_type():
+                        # skip enum types -- we directly use it as string.
+                        continue
+                    condition.value = FieldTypeValidator.typemap[
+                        target_prop.datatype.pytype.type
+                    ](condition.field, condition.value)
+
+    def prepare_results(
+        self, cls: Class, dataschema: dict[str, type], result: QueryResult
+    ) -> dict:
+        output = {}
+        if result.total is not None:
+            output["total"] = result.total
+
+        for join_clause in self.join_conditions:
+            prop = cls.properties[join_clause.prop]
+            if (
+                isinstance(prop, DataProperty)
+                and prop.db is not None
+                and prop.db.foreign_key is not None
+            ):
+                target_cls = prop.db.foreign_key
+                cardinality = Cardinality.ONE_TO_ONE
+                # the property storing the SQLAlchemy relationship of the foreign key
+                source_relprop = prop.name + "_relobj"
+            else:
+                assert isinstance(prop, ObjectProperty)
+                target_cls = prop.target
+                cardinality = prop.cardinality
+                source_relprop = prop.name
+
+            target_name = target_cls.name
+            assoc_targetrel_name = to_snake_case(target_name)
+
+            deser_func = dataschema[target_name].from_db
+
+            if target_name not in output:
+                output[target_name] = []
+
+            if cardinality == Cardinality.MANY_TO_MANY:
+                # for many-to-many, we have a middle object (association tables)
+                # because it's a list, we don't need to handle outer join because we don't have null values in the list
+                output[target_name].extend(
+                    deser_func(getattr(item, assoc_targetrel_name))
+                    for record in result.records
+                    for item in getattr(record, source_relprop)
+                )
+            elif cardinality == Cardinality.ONE_TO_MANY:
+                # A -> B is 1:N, A.id is stored in B, this does not supported in SERA yet so we do not need
+                # to implement it
+                raise NotImplementedError()
+            else:
+                if join_clause.join_type != "inner":
+                    output[target_name].extend(
+                        deser_func((val := getattr(record, source_relprop)))
+                        for record in result.records
+                        if val is not None
+                    )
+                else:
+                    output[target_name].extend(
+                        deser_func(getattr(record, source_relprop))
+                        for record in result.records
+                    )
+
+        deser_func = dataschema[cls.name].from_db
+        output[cls.name] = [deser_func(record) for record in result.records]
+
+        return output
+
+
+class FieldTypeValidator:
+
+    @staticmethod
+    def normalize_str(field: str, val: Any):
+        if not isinstance(val, str):
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid value for field '{field}': Expected string",
+            )
+        return val
+
+    @staticmethod
+    def normalize_int(field: str, val: Any):
+        if not isinstance(val, int):
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid value for field '{field}': Expected int",
+            )
+        return val
+
+    @staticmethod
+    def normalize_float(field: str, val: Any):
+        if not isinstance(val, float):
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid value for field '{field}': Expected float",
+            )
+        return val
+
+    @staticmethod
+    def normalize_bool(field: str, val: Any):
+        if not isinstance(val, bool):
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid value for field '{field}': Expected bool",
+            )
+        return val
+
+    @staticmethod
+    def normalize_date(field: str, val: Any):
+        if not isinstance(val, str):
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid value for field '{field}': Expected date",
+            )
+
+        try:
+            # Parse ISO format date string to date object
+            parsed_date = (
+                datetime.fromisoformat(val.replace("Z", "+00:00"))
+                .astimezone(timezone.utc)
+                .date()
+            )
+            return parsed_date
+        except ValueError:
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid date format for field '{field}': Expected ISO format (YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS)",
+            )
+
+    @staticmethod
+    def normalize_datetime(field: str, val: Any):
+        if not isinstance(val, str):
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid value for field '{field}': Expected date",
+            )
+
+        try:
+            # Parse ISO format date string to date object
+            parsed_dt = datetime.fromisoformat(val.replace("Z", "+00:00")).astimezone(
+                timezone.utc
+            )
+            return parsed_dt
+        except ValueError:
+            raise HTTPException(
+                status_code=status_codes.HTTP_400_BAD_REQUEST,
+                detail=f"Invalid date format for field '{field}': Expected ISO format (YYYY-MM-DDTHH:MM:SS)",
+            )
+
+    typemap = {
+        "str": normalize_str,
+        "int": normalize_int,
+        "float": normalize_float,
+        "bool": normalize_bool,
+        "date": normalize_date,
+        "datetime": normalize_datetime,
+    }

sera/make/make_app.py

@@ -7,6 +7,7 @@ from typing import Annotated
 
 from codegen.models import DeferredVar, PredefinedFn, Program, expr, stmt
 from loguru import logger
+
 from sera.make.make_python_api import make_python_api
 from sera.make.make_python_model import (
     make_python_data_model,
@@ -14,10 +15,8 @@ from sera.make.make_python_model import (
     make_python_relational_model,
 )
 from sera.make.make_python_services import make_python_service_structure
-from sera.make.make_typescript_model import (
-    make_typescript_data_model,
-    make_typescript_enum,
-)
+from sera.make.make_typescript_model import make_typescript_data_model
+from sera.make.ts_frontend.make_enums import make_typescript_enums
 from sera.misc import Formatter
 from sera.models import App, DataCollection, parse_schema
 from sera.typing import Language
@@ -170,7 +169,7 @@ def make_app(
         # generate services
         make_python_service_structure(app, collections)
     elif language == Language.Typescript:
-        make_typescript_enum(schema, app.models)
+        make_typescript_enums(schema, app.models)
         make_typescript_data_model(schema, app.models)
 
     Formatter.get_instance().process()

sera/make/make_python_api.py

@@ -22,7 +22,7 @@ def make_python_api(app: App, collections: Sequence[DataCollection]):
         route = app.api.pkg("routes").pkg(collection.get_pymodule_name())
 
         controllers = []
-        controllers.append(make_python_get_api(collection, route))
+        controllers.append(make_python_search_api(collection, route))
         controllers.append(make_python_get_by_id_api(collection, route))
         controllers.append(make_python_has_api(collection, route))
         controllers.append(make_python_create_api(collection, route))
@@ -58,6 +58,12 @@ def make_python_api(app: App, collections: Sequence[DataCollection]):
                                     ]
                                 ),
                             ),
+                            PredefinedFn.keyword_assignment(
+                                "tags",
+                                PredefinedFn.list(
+                                    [expr.ExprConstant(collection.get_pymodule_name())]
+                                ),
+                            ),
                         ],
                     ),
                 ),
@@ -107,7 +113,7 @@ def make_main(target_pkg: Package, routes: Sequence[Module]):
     outmod.write(program)
 
 
-def make_python_get_api(
+def make_python_search_api(
     collection: DataCollection, target_pkg: Package
 ) -> tuple[Module, str]:
     """Make an endpoint for querying resources"""
@@ -117,10 +123,8 @@ def make_python_get_api(
     import_helper = ImportHelper(program, GLOBAL_IDENTS)
 
     program.import_("__future__.annotations", True)
-    program.import_("typing.Annotated", True)
-    program.import_("litestar.get", True)
-    program.import_("litestar.Request", True)
-    program.import_("litestar.params.Parameter", True)
+    program.import_("litestar.post", True)
+    program.import_(app.config.path + ".schema", True)
     program.import_(app.config.path + ".API_DEBUG", True)
     program.import_(
         app.services.path
@@ -128,34 +132,43 @@ def make_python_get_api(
         True,
     )
     program.import_(
-        app.models.data.path + f".{collection.get_pymodule_name()}.{collection.name}",
+        app.models.path + ".data_schema.dataschema",
         True,
     )
-    program.import_("sera.libs.api_helper.parse_query", True)
-
-    func_name = "get_"
+    program.import_("sera.libs.search_helper.Query", True)
 
-    queryable_fields = []
-    for propname, (
-        convert_func,
-        convert_func_import,
-    ) in collection.get_queryable_fields():
-        program.import_(convert_func_import, True)
-        queryable_fields.append(
-            (expr.ExprConstant(propname), expr.ExprIdent(convert_func))
-        )
+    func_name = "search"
 
     program.root(
         stmt.LineBreak(),
-        lambda ast00: ast00.assign(
+        lambda ast: ast.assign(
             DeferredVar.simple("QUERYABLE_FIELDS"),
-            PredefinedFn.dict(queryable_fields),
+            PredefinedFn.set(
+                [
+                    expr.ExprConstant(propname)
+                    for propname in collection.get_queryable_fields()
+                ]
+            ),
+        ),
+        stmt.LineBreak(),
+        lambda ast: ast.assign(
+            DeferredVar.simple("JOIN_QUERYABLE_FIELDS"),
+            PredefinedFn.dict(
+                [
+                    (
+                        expr.ExprConstant(propname),
+                        PredefinedFn.set([expr.ExprConstant(f) for f in fields]),
+                    )
+                    for propname, fields in collection.get_join_queryable_fields().items()
+                ]
+            ),
         ),
+        stmt.LineBreak(),
         stmt.PythonDecoratorStatement(
             expr.ExprFuncCall(
-                expr.ExprIdent("get"),
+                expr.ExprIdent("post"),
                 [
-                    expr.ExprConstant("/"),
+                    expr.ExprConstant("/q"),
                 ],
             )
         ),
@@ -163,44 +176,8 @@ def make_python_get_api(
             func_name,
             [
                 DeferredVar.simple(
-                    "limit",
-                    expr.ExprIdent(
-                        'Annotated[int, Parameter(default=10, description="The maximum number of records to return")]'
-                    ),
-                ),
-                DeferredVar.simple(
-                    "offset",
-                    type=expr.ExprIdent(
-                        'Annotated[int, Parameter(default=0, description="The number of records to skip before returning results")]'
-                    ),
-                ),
-                DeferredVar.simple(
-                    "unique",
-                    expr.ExprIdent(
-                        'Annotated[bool, Parameter(default=False, description="Whether to return unique results only")]'
-                    ),
-                ),
-                DeferredVar.simple(
-                    "sorted_by",
-                    expr.ExprIdent(
-                        "Annotated[list[str], Parameter(default=tuple(), description=\"list of field names to sort by, prefix a field with '-' to sort that field in descending order\")]"
-                    ),
-                ),
-                DeferredVar.simple(
-                    "group_by",
-                    expr.ExprIdent(
-                        'Annotated[list[str], Parameter(default=tuple(), description="list of field names to group by")]'
-                    ),
-                ),
-                DeferredVar.simple(
-                    "fields",
-                    expr.ExprIdent(
-                        'Annotated[list[str], Parameter(default=tuple(), description="list of field names to include in the results")]'
-                    ),
-                ),
-                DeferredVar.simple(
-                    "request",
-                    expr.ExprIdent("Request"),
+                    "data",
+                    expr.ExprIdent("Query"),
                 ),
                 DeferredVar.simple(
                     "session",
@@ -223,19 +200,27 @@ def make_python_get_api(
                     [],
                 ),
             ),
-            lambda ast101: ast101.assign(
-                DeferredVar.simple("query", expr.ExprIdent("ServiceQuery")),
+            stmt.SingleExprStatement(
                 expr.ExprFuncCall(
-                    expr.ExprIdent("parse_query"),
+                    PredefinedFn.attr_getter(
+                        expr.ExprIdent("data"),
+                        expr.ExprIdent("validate_and_normalize"),
+                    ),
                     [
-                        expr.ExprIdent("request"),
+                        PredefinedFn.item_getter(
+                            PredefinedFn.attr_getter(
+                                expr.ExprIdent("schema"), expr.ExprIdent("classes")
+                            ),
+                            expr.ExprConstant(collection.cls.name),
+                        ),
                         expr.ExprIdent("QUERYABLE_FIELDS"),
+                        expr.ExprIdent("JOIN_QUERYABLE_FIELDS"),
                         PredefinedFn.keyword_assignment(
                             "debug",
                             expr.ExprIdent("API_DEBUG"),
                         ),
                     ],
-                ),
+                )
             ),
             lambda ast102: ast102.assign(
                 DeferredVar.simple("result"),
@@ -243,67 +228,34 @@ def make_python_get_api(
                     expr.ExprFuncCall(
                         PredefinedFn.attr_getter(
                             expr.ExprIdent("service"),
-                            expr.ExprIdent("get"),
+                            expr.ExprIdent("search"),
                         ),
-                        [
-                            expr.ExprIdent("query"),
-                            PredefinedFn.keyword_assignment(
-                                "limit", expr.ExprIdent("limit")
-                            ),
-                            PredefinedFn.keyword_assignment(
-                                "offset", expr.ExprIdent("offset")
-                            ),
-                            PredefinedFn.keyword_assignment(
-                                "unique", expr.ExprIdent("unique")
-                            ),
-                            PredefinedFn.keyword_assignment(
-                                "sorted_by", expr.ExprIdent("sorted_by")
-                            ),
-                            PredefinedFn.keyword_assignment(
-                                "group_by", expr.ExprIdent("group_by")
-                            ),
-                            PredefinedFn.keyword_assignment(
-                                "fields", expr.ExprIdent("fields")
-                            ),
-                            PredefinedFn.keyword_assignment(
-                                "session", expr.ExprIdent("session")
-                            ),
-                        ],
+                        [expr.ExprIdent("data"), expr.ExprIdent("session")],
                     )
                 ),
             ),
             lambda ast103: ast103.return_(
-                PredefinedFn.dict(
+                expr.ExprFuncCall(
+                    PredefinedFn.attr_getter(
+                        expr.ExprIdent("data"),
+                        expr.ExprIdent("prepare_results"),
+                    ),
                     [
-                        (
-                            PredefinedFn.attr_getter(
-                                expr.ExprIdent(collection.name),
-                                expr.ExprIdent("__name__"),
-                            ),
-                            PredefinedFn.map_list(
-                                PredefinedFn.attr_getter(
-                                    expr.ExprIdent("result"), expr.ExprIdent("records")
-                                ),
-                                lambda item: expr.ExprMethodCall(
-                                    expr.ExprIdent(collection.name),
-                                    "from_db",
-                                    [item],
-                                ),
-                            ),
-                        ),
-                        (
-                            expr.ExprConstant("total"),
+                        PredefinedFn.item_getter(
                             PredefinedFn.attr_getter(
-                                expr.ExprIdent("result"), expr.ExprIdent("total")
+                                expr.ExprIdent("schema"), expr.ExprIdent("classes")
                             ),
+                            expr.ExprConstant(collection.cls.name),
                         ),
-                    ]
+                        expr.ExprIdent("dataschema"),
+                        expr.ExprIdent("result"),
+                    ],
                 )
             ),
         ),
     )
 
-    outmod = target_pkg.module("get")
+    outmod = target_pkg.module("search")
     outmod.write(program)
 
     return outmod, func_name