sera-2 1.19.3__tar.gz → 1.20.12__tar.gz

{sera_2-1.19.3 → sera_2-1.20.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sera-2
-Version: 1.19.3
+Version: 1.20.12
 Summary: 
 Author: Binh Vu
 Author-email: bvu687@gmail.com
@@ -9,8 +9,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: black (==25.1.0)
-Requires-Dist: codegen-2 (>=2.12.0,<3.0.0)
-Requires-Dist: graph-wrapper (>=1.7.2,<2.0.0)
+Requires-Dist: codegen-2 (>=2.12.2,<3.0.0)
+Requires-Dist: graph-wrapper (>=1.7.3,<2.0.0)
 Requires-Dist: isort (==6.0.1)
 Requires-Dist: litestar (>=2.15.1,<3.0.0)
 Requires-Dist: loguru (>=0.7.0,<0.8.0)
@@ -18,7 +18,7 @@ Requires-Dist: msgspec (>=0.19.0,<0.20.0)
 Requires-Dist: serde2 (>=1.9.2,<2.0.0)
 Requires-Dist: sqlalchemy[asyncio] (>=2.0.41,<3.0.0)
 Requires-Dist: tqdm (>=4.67.1,<5.0.0)
-Requires-Dist: typer (>=0.12.3,<0.13.0)
+Requires-Dist: typer (>=0.16.0,<0.17.0)
 Project-URL: Repository, https://github.com/binh-vu/sera
 Description-Content-Type: text/markdown
 

{sera_2-1.19.3 → sera_2-1.20.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sera-2"
-version = "1.19.3"
+version = "1.20.12"
 description = ""
 authors = ["Binh Vu <bvu687@gmail.com>"]
 readme = "README.md"
@@ -9,17 +9,17 @@ repository = "https://github.com/binh-vu/sera"
 
 [tool.poetry.dependencies]
 python = "^3.12"
-codegen-2 = "^2.12.0"
+codegen-2 = "^2.12.2"
 msgspec = "^0.19.0"
 litestar = "^2.15.1"
 loguru = "^0.7.0"
-typer = "^0.12.3"
+typer = "^0.16.0"
 black = "==25.1.0"
 sqlalchemy = { extras = ["asyncio"], version = "^2.0.41" }
 isort = "==6.0.1"
 serde2 = "^1.9.2"
 tqdm = "^4.67.1"
-graph-wrapper = "^1.7.2"
+graph-wrapper = "^1.7.3"
 
 [build-system]
 requires = ["poetry-core"]

{sera_2-1.19.3 → sera_2-1.20.12}/sera/libs/api_helper.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import re
-from typing import Collection, Generic, cast
+from typing import Callable, Collection, Generic, TypeVar, cast
 
 from litestar import Request, status_codes
 from litestar.connection import ASGIConnection
@@ -15,16 +15,34 @@ from litestar.typing import FieldDefinition
 from msgspec import Struct
 
 from sera.libs.base_service import Query, QueryOp
-from sera.libs.middlewares.uscp import STATE_SYSTEM_CONTROLLED_PROP_KEY
+from sera.libs.middlewares.uscp import SKIP_UPDATE_SYSTEM_CONTROLLED_PROPS_KEY
 from sera.typing import T
 
 # for parsing field names and operations from query string
-FIELD_REG = re.compile(r"(?P<name>[a-zA-Z_0-9]+)(?:\[(?P<op>[a-zA-Z0-9]+)\])?")
+FIELD_REG = re.compile(r"(?P<name>[a-zA-Z_0-9]+)(?:\[(?P<op>[a-zA-Z_0-9]+)\])?")
 QUERY_OPS = {op.value for op in QueryOp}
 KEYWORDS = {"field", "limit", "offset", "unique", "sorted_by", "group_by"}
 
 
-def parse_query(request: Request, fields: set[str], debug: bool) -> Query:
+class TypeConversion:
+
+    to_int = int
+    to_float = float
+
+    @staticmethod
+    def to_bool(value: str) -> bool:
+        if value == "1":
+            return True
+        elif value == "0":
+            return False
+        raise ValueError(f"Invalid boolean value: {value}")
+
+
+def parse_query(
+    request: Request,
+    fields: dict[str, Callable[[str], str | int | bool | float]],
+    debug: bool,
+) -> Query:
     """Parse query for retrieving records that match a query.
 
     If a field name collides with a keyword, you can add `_` to the field name.
@@ -56,6 +74,7 @@ def parse_query(request: Request, fields: set[str], debug: bool) -> Query:
                 continue
 
             # Process based on operation or default to equality check
+            # TODO: validate if the operation is allowed for the field
             if not operation:
                 operation = QueryOp.eq
             else:
@@ -65,6 +84,21 @@ def parse_query(request: Request, fields: set[str], debug: bool) -> Query:
                         detail=f"Invalid operation: {operation}",
                     )
                 operation = QueryOp(operation)
+
+            try:
+                norm_func = fields[field_name]
+                if isinstance(v, list):
+                    v = [norm_func(x) for x in v]
+                else:
+                    v = norm_func(v)
+            except ValueError:
+                if debug:
+                    raise HTTPException(
+                        status_code=status_codes.HTTP_400_BAD_REQUEST,
+                        detail=f"Invalid value for field {field_name}: {v}",
+                    )
+                continue
+
             query[field_name] = {operation: v}
         else:
             # Invalid field name format
@@ -78,7 +112,10 @@ def parse_query(request: Request, fields: set[str], debug: bool) -> Query:
     return query
 
 
-class SingleAutoUSCP(MsgspecDTO[T], Generic[T]):
+S = TypeVar("S", bound=Struct)
+
+
+class SingleAutoUSCP(MsgspecDTO[S], Generic[S]):
     """Auto Update System Controlled Property DTO"""
 
     @classmethod
@@ -99,9 +136,14 @@ class SingleAutoUSCP(MsgspecDTO[T], Generic[T]):
             "data_backend"
         ]  # pyright: ignore
         obj = backend.populate_data_from_raw(value, self.asgi_connection)
-        obj.update_system_controlled_props(
-            self.asgi_connection.scope["state"][STATE_SYSTEM_CONTROLLED_PROP_KEY]
-        )
+        if self.asgi_connection.scope["state"][SKIP_UPDATE_SYSTEM_CONTROLLED_PROPS_KEY]:
+            # Skip updating system-controlled properties
+            # TODO: dirty fix as this assumes every struct has _is_scp_updated property. find a
+            # better solution and fix me!
+            obj._is_scp_updated = True
+            return obj
+
+        obj.update_system_controlled_props(self.asgi_connection)
         return obj
 
 

{sera_2-1.19.3 → sera_2-1.20.12}/sera/libs/base_service.py

@@ -17,15 +17,15 @@ from sera.typing import FieldName, T, doc
 
 
 class QueryOp(str, Enum):
-    lt = "<"
-    lte = "<="
-    gt = ">"
-    gte = ">="
-    eq = "="
-    ne = "!="
+    lt = "lt"
+    lte = "lte"
+    gt = "gt"
+    gte = "gte"
+    eq = "eq"
+    ne = "ne"
     # select records where values are in the given list
     in_ = "in"
-    not_in = "not in"
+    not_in = "not_in"
     # for full text search
     fuzzy = "fuzzy"
 
@@ -94,15 +94,41 @@ class BaseAsyncService(Generic[ID, R]):
             )
         if unique:
             q = q.distinct()
-        if sorted_by:
-            for field in sorted_by:
-                if field.startswith("-"):
-                    q = q.order_by(getattr(self.orm_cls, field[1:]).desc())
+        for field in sorted_by:
+            if field.startswith("-"):
+                q = q.order_by(getattr(self.orm_cls, field[1:]).desc())
+            else:
+                q = q.order_by(getattr(self.orm_cls, field))
+        for field in group_by:
+            q = q.group_by(getattr(self.orm_cls, field))
+
+        for field, conditions in query.items():
+            for op, value in conditions.items():
+                # TODO: check if the operation is valid for the field.
+                if op == QueryOp.eq:
+                    q = q.where(getattr(self.orm_cls, field) == value)
+                elif op == QueryOp.ne:
+                    q = q.where(getattr(self.orm_cls, field) != value)
+                elif op == QueryOp.lt:
+                    q = q.where(getattr(self.orm_cls, field) < value)
+                elif op == QueryOp.lte:
+                    q = q.where(getattr(self.orm_cls, field) <= value)
+                elif op == QueryOp.gt:
+                    q = q.where(getattr(self.orm_cls, field) > value)
+                elif op == QueryOp.gte:
+                    q = q.where(getattr(self.orm_cls, field) >= value)
+                elif op == QueryOp.in_:
+                    q = q.where(getattr(self.orm_cls, field).in_(value))
+                elif op == QueryOp.not_in:
+                    q = q.where(~getattr(self.orm_cls, field).in_(value))
                 else:
-                    q = q.order_by(getattr(self.orm_cls, field))
-        if group_by:
-            for field in group_by:
-                q = q.group_by(getattr(self.orm_cls, field))
+                    assert op == QueryOp.fuzzy
+                    # Assuming fuzzy search is implemented as a full-text search
+                    q = q.where(
+                        func.to_tsvector(getattr(self.orm_cls, field)).match(value)
+                    )
+
+        print(">>>", q)
 
         cq = select(func.count()).select_from(q.subquery())
         rq = q.limit(limit).offset(offset)

{sera_2-1.19.3 → sera_2-1.20.12}/sera/libs/middlewares/uscp.py

@@ -6,7 +6,7 @@ from litestar.connection.base import UserT
 from litestar.middleware import AbstractMiddleware
 from litestar.types import ASGIApp, Message, Receive, Scope, Scopes, Send
 
-STATE_SYSTEM_CONTROLLED_PROP_KEY = "system_controlled_properties"
+SKIP_UPDATE_SYSTEM_CONTROLLED_PROPS_KEY = "skip_uscp_1157"
 
 
 class USCPMiddleware(AbstractMiddleware):
@@ -21,7 +21,6 @@ class USCPMiddleware(AbstractMiddleware):
     def __init__(
         self,
         app: ASGIApp,
-        get_system_controlled_props: Callable[[UserT], dict],
         skip_update_system_controlled_props: Callable[[UserT], bool],
         exclude: str | list[str] | None = None,
         exclude_opt_key: str | None = None,
@@ -39,17 +38,13 @@ class USCPMiddleware(AbstractMiddleware):
                 either or both 'ScopeType.HTTP' and 'ScopeType.WEBSOCKET'.
         """
         super().__init__(app, exclude, exclude_opt_key, scopes)
-        self.get_system_controlled_props = get_system_controlled_props
         self.skip_update_system_controlled_props = skip_update_system_controlled_props
 
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
         user = scope["user"]
-        if self.skip_update_system_controlled_props(user):
-            scope["state"][STATE_SYSTEM_CONTROLLED_PROP_KEY] = None
-        else:
-            scope["state"][STATE_SYSTEM_CONTROLLED_PROP_KEY] = (
-                self.get_system_controlled_props(user)
-            )
+        scope["state"][SKIP_UPDATE_SYSTEM_CONTROLLED_PROPS_KEY] = (
+            self.skip_update_system_controlled_props(user)
+        )
         await self.app(scope, receive, send)
 
 

{sera_2-1.19.3 → sera_2-1.20.12}/sera/make/make_python_api.py

@@ -4,6 +4,7 @@ from typing import Sequence
 
 from codegen.models import DeferredVar, ImportHelper, PredefinedFn, Program, expr, stmt
 from loguru import logger
+from msgspec import convert
 
 from sera.misc import assert_not_null, to_snake_case
 from sera.models import App, DataCollection, Module, Package
@@ -134,11 +135,21 @@ def make_python_get_api(
 
     func_name = "get_"
 
+    queryable_fields = []
+    for propname, (
+        convert_func,
+        convert_func_import,
+    ) in collection.get_queryable_fields():
+        program.import_(convert_func_import, True)
+        queryable_fields.append(
+            (expr.ExprConstant(propname), expr.ExprIdent(convert_func))
+        )
+
     program.root(
         stmt.LineBreak(),
         lambda ast00: ast00.assign(
             DeferredVar.simple("QUERYABLE_FIELDS"),
-            expr.ExprConstant(collection.get_queryable_fields()),
+            PredefinedFn.dict(queryable_fields),
         ),
         stmt.PythonDecoratorStatement(
             expr.ExprFuncCall(

{sera_2-1.19.3 → sera_2-1.20.12}/sera/make/make_python_model.py

@@ -1296,6 +1296,10 @@ def make_python_relational_object_property(
         idprop = prop.target.get_id_property()
         assert idprop is not None
         idprop_pytype = idprop.datatype.get_sqlalchemy_type()
+
+        if prop.is_optional:
+            idprop_pytype = idprop_pytype.as_optional_type()
+
         for dep in idprop_pytype.deps:
             program.import_(dep, True)
 

{sera_2-1.19.3 → sera_2-1.20.12}/sera/make/make_typescript_model.py

@@ -29,6 +29,8 @@ from sera.typing import is_set
 TS_GLOBAL_IDENTS = {
     "normalizers.normalizeNumber": "sera-db.normalizers",
     "normalizers.normalizeOptionalNumber": "sera-db.normalizers",
+    "normalizers.normalizeDate": "sera-db.normalizers",
+    "normalizers.normalizeOptionalDate": "sera-db.normalizers",
 }
 
 
@@ -50,6 +52,74 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
         # the original object, then it's okay.
         return value
 
+    def get_normal_deser_args(
+        prop: DataProperty | ObjectProperty,
+    ) -> expr.Expr:
+        """Extract the value from the data record from the server response to set to the class property in the client."""
+        handle_optional = lambda value: expr.ExprTernary(
+            expr.ExprNotEqual(value, expr.ExprConstant(None)),
+            value,
+            expr.ExprConstant("undefined"),
+        )
+
+        if isinstance(prop, DataProperty):
+            value = PredefinedFn.attr_getter(
+                expr.ExprIdent("data"), expr.ExprIdent(prop.name)
+            )
+            if prop.is_optional:
+                value = handle_optional(value)
+                value.true_expr = (
+                    prop.datatype.get_typescript_type().get_json_deser_func(
+                        value.true_expr
+                    )
+                )
+            else:
+                value = prop.datatype.get_typescript_type().get_json_deser_func(value)
+
+            return value
+
+        assert isinstance(prop, ObjectProperty)
+        if prop.target.db is not None:
+            value = PredefinedFn.attr_getter(
+                expr.ExprIdent("data"), expr.ExprIdent(prop.name + "_id")
+            )
+            if prop.is_optional:
+                value = handle_optional(value)
+            return value
+        else:
+            if prop.cardinality.is_star_to_many():
+                # optional type for a list is simply an empty list, we don't need to check for None
+                value = PredefinedFn.map_list(
+                    PredefinedFn.attr_getter(
+                        expr.ExprIdent("data"),
+                        expr.ExprIdent(prop.name),
+                    ),
+                    lambda item: expr.ExprMethodCall(
+                        expr.ExprIdent(
+                            assert_isinstance(prop, ObjectProperty).target.name
+                        ),
+                        "deser",
+                        [item],
+                    ),
+                )
+                return value
+            else:
+                value = expr.ExprFuncCall(
+                    PredefinedFn.attr_getter(
+                        expr.ExprIdent(prop.target.name),
+                        expr.ExprIdent("deser"),
+                    ),
+                    [
+                        PredefinedFn.attr_getter(
+                            expr.ExprIdent("data"),
+                            expr.ExprIdent(prop.name),
+                        )
+                    ],
+                )
+                if prop.is_optional:
+                    value = handle_optional(value)
+                return value
+
     def make_normal(cls: Class, pkg: Package):
         """Make a data model for the normal Python data model"""
         if not cls.is_public:
@@ -80,7 +150,9 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
 
                 if idprop is not None and prop.name == idprop.name:
                     # use id type alias
-                    tstype = TsTypeWithDep(f"{cls.name}Id")
+                    tstype = TsTypeWithDep(
+                        type=f"{cls.name}Id", spectype=tstype.spectype
+                    )
 
                 if prop.is_optional:
                     # convert type to optional
@@ -89,9 +161,7 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                 deser_args.append(
                     (
                         expr.ExprIdent(propname),
-                        PredefinedFn.attr_getter(
-                            expr.ExprIdent("data"), expr.ExprIdent(prop.name)
-                        ),
+                        get_normal_deser_args(prop),
                     )
                 )
             else:
@@ -100,10 +170,18 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                     # this class is stored in the database, we store the id instead
                     propname = propname + "Id"
                     tstype = TsTypeWithDep(
-                        f"{prop.target.name}Id",
-                        [
-                            f"@.models.{prop.target.get_tsmodule_name()}.{prop.target.name}.{prop.target.name}Id"
-                        ],
+                        type=f"{prop.target.name}Id",
+                        spectype=assert_not_null(prop.target.get_id_property())
+                        .get_data_model_datatype()
+                        .get_typescript_type()
+                        .spectype,
+                        deps=(
+                            [
+                                f"@.models.{prop.target.get_tsmodule_name()}.{prop.target.name}.{prop.target.name}Id"
+                            ]
+                            if prop.target.name != cls.name
+                            else []
+                        ),
                     )
                     if prop.cardinality.is_star_to_many():
                         tstype = tstype.as_list_type()
@@ -113,17 +191,15 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                     deser_args.append(
                         (
                             expr.ExprIdent(propname),
-                            PredefinedFn.attr_getter(
-                                expr.ExprIdent("data"),
-                                expr.ExprIdent(prop.name + "_id"),
-                            ),
+                            get_normal_deser_args(prop),
                         )
                     )
                 else:
                     # we are going to store the whole object
                     tstype = TsTypeWithDep(
-                        prop.target.name,
-                        [
+                        type=prop.target.name,
+                        spectype=prop.target.name,
+                        deps=[
                             f"@.models.{prop.target.get_tsmodule_name()}.{prop.target.name}.{prop.target.name}"
                         ],
                     )
@@ -132,21 +208,7 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                         deser_args.append(
                             (
                                 expr.ExprIdent(propname),
-                                PredefinedFn.map_list(
-                                    PredefinedFn.attr_getter(
-                                        expr.ExprIdent("data"),
-                                        expr.ExprIdent(prop.name),
-                                    ),
-                                    lambda item: expr.ExprMethodCall(
-                                        expr.ExprIdent(
-                                            assert_isinstance(
-                                                prop, ObjectProperty
-                                            ).target.name
-                                        ),
-                                        "deser",
-                                        [item],
-                                    ),
-                                ),
+                                get_normal_deser_args(prop),
                             )
                         )
                     else:
@@ -156,18 +218,7 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                         deser_args.append(
                             (
                                 expr.ExprIdent(propname),
-                                expr.ExprFuncCall(
-                                    PredefinedFn.attr_getter(
-                                        expr.ExprIdent(prop.target.name),
-                                        expr.ExprIdent("deser"),
-                                    ),
-                                    [
-                                        PredefinedFn.attr_getter(
-                                            expr.ExprIdent("data"),
-                                            expr.ExprIdent(prop.name),
-                                        )
-                                    ],
-                                ),
+                                get_normal_deser_args(prop),
                             )
                         )
 
@@ -215,6 +266,17 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                     ],
                 )(*prop_constructor_assigns),
                 stmt.LineBreak(),
+                lambda ast12: ast12.func(
+                    "className",
+                    [],
+                    expr.ExprIdent("string"),
+                    is_static=True,
+                    modifiers=["get"],
+                    comment="Name of the class in the Schema",
+                )(
+                    stmt.ReturnStatement(expr.ExprConstant(cls.name)),
+                ),
+                stmt.LineBreak(),
                 lambda ast12: ast12.func(
                     "deser",
                     [
@@ -348,10 +410,10 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                 if idprop is not None and prop.name == idprop.name:
                     # use id type alias
                     tstype = TsTypeWithDep(
-                        f"{cls.name}Id",
+                        type=f"{cls.name}Id",
+                        spectype=tstype.spectype,
                         deps=[f"@.models.{pkg.dir.name}.{cls.name}.{cls.name}Id"],
                     )
-                    original_tstype = tstype
                 elif tstype.type not in schema.enums:
                     # for none id & none enum properties, we need to include a type for "invalid" value
                     tstype = _inject_type_for_invalid_value(tstype)
@@ -378,8 +440,13 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                     if tstype.type in idprop_aliases:
                         create_propvalue = idprop_aliases[tstype.type].get_default()
                     elif tstype.type in schema.enums:
-                        create_propvalue = expr.ExprConstant(
-                            next(iter(schema.enums[tstype.type].values.values())).value
+                        enum_value = next(
+                            iter(schema.enums[tstype.type].values.values())
+                        ).value
+                        # TODO: handle enum value integer
+                        assert isinstance(enum_value, str)
+                        create_propvalue = expr.ExprIdent(
+                            tstype.type + "." + enum_value
                         )
                     else:
                         create_propvalue = tstype.get_default()
@@ -416,7 +483,10 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                         expr.ExprIdent("record"), expr.ExprIdent(propname)
                     )
 
-                if original_tstype.type != tstype.type:
+                if (
+                    original_tstype.type != tstype.type
+                    and tstype.type != f"{cls.name}Id"
+                ):
                     norm_func = get_norm_func(original_tstype, import_helper)
                 else:
                     norm_func = identity
@@ -441,17 +511,22 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                                     ),
                                     expr.ExprIdent("isValid"),
                                 ),
-                                norm_func(
-                                    PredefinedFn.attr_getter(
-                                        expr.ExprIdent("this"), expr.ExprIdent(propname)
+                                original_tstype.get_json_ser_func(
+                                    norm_func(
+                                        PredefinedFn.attr_getter(
+                                            expr.ExprIdent("this"),
+                                            expr.ExprIdent(propname),
+                                        )
                                     )
                                 ),
                                 expr.ExprIdent("undefined"),
                             )
                             if prop.is_optional
-                            else norm_func(
-                                PredefinedFn.attr_getter(
-                                    expr.ExprIdent("this"), expr.ExprIdent(propname)
+                            else original_tstype.get_json_ser_func(
+                                norm_func(
+                                    PredefinedFn.attr_getter(
+                                        expr.ExprIdent("this"), expr.ExprIdent(propname)
+                                    )
                                 )
                             )
                         ),
@@ -516,8 +591,12 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                 if prop.target.db is not None:
                     # this class is stored in the database, we store the id instead
                     tstype = TsTypeWithDep(
-                        f"{prop.target.name}Id",
-                        [
+                        type=f"{prop.target.name}Id",
+                        spectype=assert_not_null(prop.target.get_id_property())
+                        .get_data_model_datatype()
+                        .get_typescript_type()
+                        .spectype,
+                        deps=[
                             f"@.models.{prop.target.get_tsmodule_name()}.{prop.target.name}.{prop.target.name}Id"
                         ],
                     )
@@ -568,8 +647,9 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                 else:
                     # we are going to store the whole object
                     tstype = TsTypeWithDep(
-                        f"Draft{prop.target.name}",
-                        [
+                        type=f"Draft{prop.target.name}",
+                        spectype=f"Draft{prop.target.name}",
+                        deps=[
                             f"@.models.{prop.target.get_tsmodule_name()}.Draft{prop.target.name}.Draft{prop.target.name}"
                         ],
                     )
@@ -1131,7 +1211,13 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
             # if prop.data.is_private:
             #     # skip private fields as this is for APIs exchange
             #     continue
-            propname = to_camel_case(prop.name)
+            tspropname = to_camel_case(prop.name)
+            pypropname = prop.name
+            if isinstance(prop, ObjectProperty) and prop.target.db is not None:
+                # this is a database object, we append id to the property name
+                tspropname = tspropname + "Id"
+                pypropname = prop.name + "_id"
+
             tsprop = {}
 
             if isinstance(prop, DataProperty):
@@ -1146,7 +1232,7 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                     (
                         expr.ExprIdent("datatype"),
                         (
-                            expr.ExprConstant(tstype.type)
+                            expr.ExprConstant(tstype.spectype)
                             if tstype.type not in schema.enums
                             else expr.ExprConstant("enum")
                         ),
@@ -1169,7 +1255,7 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                 norm_func = get_normalizer(tstype, import_helper)
                 if norm_func is not None:
                     # we have a normalizer for this type
-                    prop_normalizers.append((expr.ExprIdent(propname), norm_func))
+                    prop_normalizers.append((expr.ExprIdent(tspropname), norm_func))
             else:
                 assert isinstance(prop, ObjectProperty)
                 if prop.target.db is not None:
@@ -1182,8 +1268,9 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                 else:
                     # we are going to store the whole object
                     tstype = TsTypeWithDep(
-                        prop.target.name,
-                        [
+                        type=prop.target.name,
+                        spectype=prop.target.name,
+                        deps=[
                             f"@.models.{prop.target.get_tsmodule_name()}.{prop.target.name}.{prop.target.name}"
                         ],
                     )
@@ -1204,7 +1291,9 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                     (
                         expr.ExprIdent("datatype"),
                         expr.ExprConstant(
-                            tstype.type if prop.target.db is not None else "undefined"
+                            tstype.spectype
+                            if prop.target.db is not None
+                            else "undefined"
                         ),
                     ),
                     (
@@ -1224,11 +1313,11 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
             prop_defs.append(
                 (
                     prop,
-                    expr.ExprIdent(propname),
+                    expr.ExprIdent(tspropname),
                     PredefinedFn.dict(
                         [
-                            (expr.ExprIdent("name"), expr.ExprConstant(prop.name)),
-                            (expr.ExprIdent("tsName"), expr.ExprConstant(propname)),
+                            (expr.ExprIdent("name"), expr.ExprConstant(pypropname)),
+                            (expr.ExprIdent("tsName"), expr.ExprConstant(tspropname)),
                             (
                                 expr.ExprIdent("updateFuncName"),
                                 expr.ExprConstant(f"update{to_pascal_case(prop.name)}"),
@@ -1298,7 +1387,10 @@ def make_typescript_data_model(schema: Schema, target_pkg: Package):
                                 " | ".join(
                                     [
                                         expr.ExprConstant(
-                                            to_camel_case(prop.name)
+                                            to_camel_case(prop.name) + "Id"
+                                            if isinstance(prop, ObjectProperty)
+                                            and prop.target.db is not None
+                                            else to_camel_case(prop.name)
                                         ).to_typescript()
                                         for prop in cls.properties.values()
                                         if not prop.data.is_private
@@ -1557,6 +1649,10 @@ def get_normalizer(
         return import_helper.use("normalizers.normalizeNumber")
     if tstype.type == "number | undefined":
         return import_helper.use("normalizers.normalizeOptionalNumber")
+    if tstype.type == "Date":
+        return import_helper.use("normalizers.normalizeDate")
+    if tstype.type == "Date | undefined":
+        return import_helper.use("normalizers.normalizeOptionalDate")
 
     assert "number" not in tstype.type, tstype.type
     return None

{sera_2-1.19.3 → sera_2-1.20.12}/sera/misc/__init__.py

@@ -1,6 +1,7 @@
 from sera.misc._formatter import File, Formatter
 from sera.misc._utils import (
     LoadTableDataArgs,
+    RelTableIndex,
     assert_isinstance,
     assert_not_null,
     filter_duplication,
@@ -26,4 +27,5 @@ __all__ = [
     "identity",
     "get_classpath",
     "LoadTableDataArgs",
+    "RelTableIndex",
 ]

{sera_2-1.19.3 → sera_2-1.20.12}/sera/misc/_utils.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import inspect
 import re
+from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Callable, Iterable, Optional, Sequence, Type, TypedDict, TypeVar
@@ -120,8 +122,38 @@ class LoadTableDataArgs(TypedDict, total=False):
     tables: Sequence[type]
     file: Path
     files: Sequence[Path]
-    file_deser: Callable[[Path], list[dict]]
-    record_deser: Callable[[dict], Any | list[Any]]
+    file_deser: Callable[[Path], list[Any]]
+    record_deser: (
+        Callable[[dict], Any | list[Any]]
+        | Callable[[dict, RelTableIndex], Any | list[Any]]
+    )
+    table_unique_index: dict[type, list[str]]
+
+
+class RelTableIndex:
+    """An index of relational tables to find a record by its unique property."""
+
+    def __init__(self, cls2index: Optional[dict[str, list[str]]] = None):
+        self.table2rows: dict[str, dict[str, Any]] = defaultdict(dict)
+        self.table2uniqindex2id: dict[str, dict[str, int]] = defaultdict(dict)
+        self.cls2index = cls2index or {}
+
+    def set_index(self, clsname: str, props: list[str]):
+        """Set the unique index for a class."""
+        self.cls2index[clsname] = props
+
+    def add(self, record: Any):
+        clsname = record.__class__.__name__
+        self.table2rows[clsname][record.id] = record
+        if clsname in self.cls2index:
+            for prop in self.cls2index[clsname]:
+                self.table2uniqindex2id[clsname][getattr(record, prop)] = record.id
+
+    def get_record(self, clsname: str, uniq_prop: str) -> Optional[Any]:
+        tbl = self.table2uniqindex2id[clsname]
+        if uniq_prop not in tbl:
+            return None
+        return self.table2rows[clsname][tbl[uniq_prop]]
 
 
 def load_data(
@@ -143,6 +175,8 @@ def load_data(
     with Session(engine) as session:
         create_db_and_tables()
 
+        reltable_index = RelTableIndex()
+
         for args in tqdm(table_data, disable=not verbose, desc="Loading data"):
             if "table" in args:
                 tbls = [args["table"]]
@@ -162,6 +196,12 @@ def load_data(
             else:
                 raise ValueError("Either 'file' or 'files' must be provided in args.")
 
+            if "table_unique_index" in args:
+                for tbl in tbls:
+                    reltable_index.set_index(
+                        tbl.__name__, args["table_unique_index"].get(tbl, [])
+                    )
+
             raw_records = []
             if "file_deser" not in args:
                 for file in files:
@@ -175,8 +215,17 @@ def load_data(
                 for file in files:
                     raw_records.extend(args["file_deser"](file))
 
+            assert "record_deser" in args
             deser = args["record_deser"]
-            records = [deser(row) for row in raw_records]
+
+            sig = inspect.signature(deser)
+            param_count = len(sig.parameters)
+            if param_count == 1:
+                records = [deser(row) for row in raw_records]
+            else:
+                assert param_count == 2
+                records = [deser(row, reltable_index) for row in raw_records]
+
             for r in tqdm(
                 records,
                 desc=f"load {', '.join(tbl.__name__ for tbl in tbls)}",
@@ -185,8 +234,11 @@ def load_data(
                 if isinstance(r, Sequence):
                     for x in r:
                         session.merge(x)
+                        reltable_index.add(x)
                 else:
                     session.merge(r)
+                    reltable_index.add(r)
+
             session.flush()
 
             # Reset the sequence for each table

{sera_2-1.19.3 → sera_2-1.20.12}/sera/models/_collection.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 
+from codegen.models import ImportHelper
+
 from sera.models._class import Class
 from sera.models._property import DataProperty, ObjectProperty
 
@@ -21,9 +23,9 @@ class DataCollection:
         """Get the python module name of this collection as if there is a python module created to store this collection only."""
         return self.cls.get_pymodule_name()
 
-    def get_queryable_fields(self) -> set[str]:
+    def get_queryable_fields(self) -> list[tuple[str, tuple[str, str]]]:
         """Get the fields of this collection that can be used in a queries."""
-        field_names = set()
+        output = []
         for prop in self.cls.properties.values():
             if prop.db is None or prop.data.is_private:
                 # This property is not stored in the database or it's private, so we skip it
@@ -45,8 +47,19 @@ class DataCollection:
             else:
                 # This property is a data property or an object property not stored in the database, so we use its name
                 propname = prop.name
-            field_names.add(propname)
-        return field_names
+
+            if isinstance(prop, DataProperty):
+                convert_func = prop.datatype.pytype.get_string_conversion_func()
+            else:
+                assert isinstance(prop, ObjectProperty) and prop.target.db is not None
+                target_idprop = prop.target.get_id_property()
+                assert target_idprop is not None
+                convert_func = (
+                    target_idprop.datatype.pytype.get_string_conversion_func()
+                )
+
+            output.append((propname, convert_func))
+        return output
 
     def get_service_name(self):
         return f"{self.name}Service"

{sera_2-1.19.3 → sera_2-1.20.12}/sera/models/_datatype.py

@@ -2,10 +2,12 @@ from __future__ import annotations
 
 import datetime
 from dataclasses import dataclass, field
-from typing import Literal
+from typing import Callable, Literal
 
 from codegen.models import expr
 
+from sera.misc import identity
+
 PyDataType = Literal["str", "int", "datetime", "float", "bool", "bytes", "dict"]
 TypescriptDataType = Literal["string", "number", "boolean"]
 SQLAlchemyDataType = Literal[
@@ -67,10 +69,25 @@ class PyTypeWithDep:
         """Clone the type with the same dependencies."""
         return PyTypeWithDep(type=self.type, deps=list(self.deps))
 
+    def get_string_conversion_func(self) -> tuple[str, str]:
+        if self.type == "str":
+            return ("identity", "sera.misc.identity")
+        if self.type == "int":
+            return ("TypeConversion.to_int", "sera.libs.api_helper.TypeConversion")
+        if self.type == "float":
+            return ("TypeConversion.to_float", "sera.libs.api_helper.TypeConversion")
+        if self.type == "bool":
+            return ("TypeConversion.to_bool", "sera.libs.api_helper.TypeConversion")
+        else:
+            raise NotImplementedError()
+
 
 @dataclass
 class TsTypeWithDep:
     type: str
+    # the specific type of the value, to provide more details for the type because typescript use
+    # number for both int and float, date for both date and datetime.
+    spectype: str
     deps: list[str] = field(default_factory=list)
 
     def get_default(self) -> expr.ExprConstant:
@@ -82,6 +99,8 @@ class TsTypeWithDep:
             return expr.ExprConstant(0)
         if self.type == "boolean":
             return expr.ExprConstant(False)
+        if self.type == "Date":
+            return expr.ExprConstant("new Date()")
         if self.type.endswith("| undefined"):
             return expr.ExprConstant("undefined")
         if self.type.endswith("| string)") or self.type.endswith("| string"):
@@ -96,16 +115,54 @@ class TsTypeWithDep:
         if not all(c.isalnum() or c == "_" for c in self.type.strip()):
             # Type contains special chars like | or spaces, wrap in parentheses
             list_type = f"({self.type})[]"
+            list_spectype = f"({self.spectype})[]"
         else:
             list_type = f"{self.type}[]"
-        return TsTypeWithDep(type=list_type, deps=self.deps)
+            list_spectype = f"{self.spectype}[]"
+        return TsTypeWithDep(type=list_type, spectype=list_spectype, deps=self.deps)
 
     def as_optional_type(self) -> TsTypeWithDep:
         if "undefined" in self.type:
             raise NotImplementedError(
                 f"Have not handle nested optional yet: {self.type}"
             )
-        return TsTypeWithDep(type=f"{self.type} | undefined", deps=self.deps)
+        return TsTypeWithDep(
+            type=f"{self.type} | undefined",
+            # not changing the spectype because we convert to optional when the value is missing
+            # spectype is used to tell the main type of the value when it is present.
+            spectype=self.spectype,
+            deps=self.deps,
+        )
+
+    def get_json_deser_func(self, value: expr.Expr) -> expr.Expr:
+        """Get the typescript expression to convert the value from json format to the correct type."""
+        if self.type in {"string", "number", "boolean", "string[]"}:
+            return value
+        if self.type == "Date":
+            return expr.ExprRawTypescript(f"new Date({value.to_typescript()})")
+        if any(x.startswith("@.models.enum") for x in self.deps):
+            # enum type, we don't need to do anything as we use strings for enums
+            return value
+        raise ValueError(f"Unknown type: {self.type}")
+
+    def get_json_ser_func(self, value: expr.Expr) -> expr.Expr:
+        """Get the typescript expression to convert the value to json format."""
+        if self.type in {
+            "string",
+            "number",
+            "boolean",
+            "string[]",
+            "number | undefined",
+            "boolean | undefined",
+            "string | undefined",
+        }:
+            return value
+        if self.type == "Date":
+            return expr.ExprRawTypescript(f"{value.to_typescript()}.toISOString()")
+        if any(x.startswith("@.models.enum") for x in self.deps):
+            # enum type, we don't need to do anything as we use strings for enums
+            return value
+        raise ValueError(f"Unknown type: {self.type}")
 
 
 @dataclass
@@ -122,6 +179,23 @@ class SQLTypeWithDep:
             mapped_pytype=f"list[{self.mapped_pytype}]",
         )
 
+    def as_optional_type(self) -> SQLTypeWithDep:
+        """Convert the type to an optional type."""
+        if "typing.Optional" not in self.deps:
+            deps = self.deps + ["typing.Optional"]
+        else:
+            deps = self.deps
+
+        if "Optional[" in self.mapped_pytype:
+            raise NotImplementedError(
+                f"Have not handle nested optional yet: {self.mapped_pytype}"
+            )
+        return SQLTypeWithDep(
+            type=self.type,
+            mapped_pytype=f"Optional[{self.mapped_pytype}]",
+            deps=deps,
+        )
+
 
 @dataclass
 class DataType:
@@ -156,7 +230,7 @@ predefined_datatypes = {
         sqltype=SQLTypeWithDep(
             type="String", mapped_pytype="str", deps=["sqlalchemy.String"]
         ),
-        tstype=TsTypeWithDep(type="string"),
+        tstype=TsTypeWithDep(type="string", spectype="string"),
         is_list=False,
     ),
     "integer": DataType(
@@ -164,7 +238,7 @@ predefined_datatypes = {
         sqltype=SQLTypeWithDep(
             type="Integer", mapped_pytype="int", deps=["sqlalchemy.Integer"]
         ),
-        tstype=TsTypeWithDep(type="number"),
+        tstype=TsTypeWithDep(type="number", spectype="integer"),
         is_list=False,
     ),
     "date": DataType(
@@ -174,7 +248,7 @@ predefined_datatypes = {
             mapped_pytype="date",
             deps=["sqlalchemy.Date", "datetime.date"],
         ),
-        tstype=TsTypeWithDep(type="string"),
+        tstype=TsTypeWithDep(type="Date", spectype="date"),
         is_list=False,
     ),
     "datetime": DataType(
@@ -184,7 +258,7 @@ predefined_datatypes = {
             mapped_pytype="datetime",
             deps=["sqlalchemy.DateTime", "datetime.datetime"],
         ),
-        tstype=TsTypeWithDep(type="string"),
+        tstype=TsTypeWithDep(type="Date", spectype="datetime"),
         is_list=False,
     ),
     "float": DataType(
@@ -192,7 +266,7 @@ predefined_datatypes = {
         sqltype=SQLTypeWithDep(
             type="Float", mapped_pytype="float", deps=["sqlalchemy.Float"]
         ),
-        tstype=TsTypeWithDep(type="number"),
+        tstype=TsTypeWithDep(type="number", spectype="float"),
         is_list=False,
     ),
     "boolean": DataType(
@@ -200,7 +274,7 @@ predefined_datatypes = {
         sqltype=SQLTypeWithDep(
             type="Boolean", mapped_pytype="bool", deps=["sqlalchemy.Boolean"]
         ),
-        tstype=TsTypeWithDep(type="boolean"),
+        tstype=TsTypeWithDep(type="boolean", spectype="boolean"),
         is_list=False,
     ),
     "bytes": DataType(
@@ -208,7 +282,7 @@ predefined_datatypes = {
         sqltype=SQLTypeWithDep(
             type="LargeBinary", mapped_pytype="bytes", deps=["sqlalchemy.LargeBinary"]
         ),
-        tstype=TsTypeWithDep(type="string"),
+        tstype=TsTypeWithDep(type="string", spectype="bytes"),
         is_list=False,
     ),
     "dict": DataType(
@@ -216,7 +290,7 @@ predefined_datatypes = {
         sqltype=SQLTypeWithDep(
             type="JSON", mapped_pytype="dict", deps=["sqlalchemy.JSON"]
         ),
-        tstype=TsTypeWithDep(type="string"),
+        tstype=TsTypeWithDep(type="string", spectype="dict"),
         is_list=False,
     ),
 }
@@ -228,5 +302,5 @@ predefined_sql_datatypes = {
     ),
 }
 predefined_ts_datatypes = {
-    "string": TsTypeWithDep(type="string"),
+    "string": TsTypeWithDep(type="string", spectype="string"),
 }

{sera_2-1.19.3 → sera_2-1.20.12}/sera/models/_parse.py

@@ -238,10 +238,17 @@ def _parse_datatype(schema: Schema, datatype: dict | str) -> DataType:
                     ],
                 ),
                 sqltype=SQLTypeWithDep(
-                    type="String", mapped_pytype="str", deps=["sqlalchemy.String"]
+                    type=f"Enum({enum.name})",
+                    mapped_pytype=enum.name,
+                    deps=[
+                        "sqlalchemy.Enum",
+                        f"{schema.name}.models.enums.{enum.get_pymodule_name()}.{enum.name}",
+                    ],
                 ),
                 tstype=TsTypeWithDep(
-                    type=enum.name, deps=[f"@.models.enums.{enum.name}"]
+                    type=enum.name,
+                    spectype=enum.name,
+                    deps=[f"@.models.enums.{enum.name}"],
                 ),
                 is_list=is_list,
             )