sera-2 1.23.0__py3-none-any.whl → 1.25.0__py3-none-any.whl

sera/libs/base_service.py

@@ -10,8 +10,8 @@ from sqlalchemy.orm import contains_eager, load_only
 
 from sera.libs.base_orm import BaseORM
 from sera.libs.search_helper import Query, QueryOp
-from sera.misc import assert_not_null, to_snake_case
-from sera.models import Cardinality, Class, DataProperty, ObjectProperty
+from sera.misc import assert_isinstance, assert_not_null, to_snake_case
+from sera.models import Cardinality, Class, DataProperty, IndexType, ObjectProperty
 
 R = TypeVar("R", bound=BaseORM)
 ID = TypeVar("ID")  # ID of a class
@@ -20,6 +20,7 @@ SqlResult = TypeVar("SqlResult", bound=Result)
 
 class QueryResult(NamedTuple, Generic[R]):
     records: Sequence[R]
+    extra_columns: Mapping[str, Sequence]
     total: Optional[int]
 
 
@@ -36,6 +37,7 @@ class BaseAsyncService(Generic[ID, R]):
         self._cls_id_prop = getattr(self.orm_cls, self.id_prop.name)
         self.is_id_auto_increment = assert_not_null(self.id_prop.db).is_auto_increment
 
+        # mapping from property name to ORM class for object properties
         self.prop2orm: dict[str, type] = {
             prop.name: orm_classes[prop.target.name]
             for prop in cls.properties.values()
@@ -103,15 +105,15 @@ class BaseAsyncService(Generic[ID, R]):
                 # A -> B is either 1:1 or N:1, we will store the foreign key is in A
                 # .join(B, A.<foreign_key> == B.id)
                 self.join_clauses[prop.name] = [
-                    {
-                        "class": target_tbl,
-                        "condition": getattr(
-                            target_tbl,
-                            assert_not_null(target_cls.get_id_property()).name,
-                        )
-                        == getattr(self.orm_cls, source_fk),
-                        "contains_eager": getattr(self.orm_cls, source_relprop),
-                    },
+                    #     {
+                    #         "class": target_tbl,
+                    #         "condition": getattr(
+                    #             target_tbl,
+                    #             assert_not_null(target_cls.get_id_property()).name,
+                    #         )
+                    #         == getattr(self.orm_cls, source_fk),
+                    #         "contains_eager": getattr(self.orm_cls, source_relprop),
+                    #     },
                 ]
 
     @classmethod
@@ -135,6 +137,7 @@ class BaseAsyncService(Generic[ID, R]):
             session: The database session
         """
         q = self._select()
+        extra_cols = []
 
         if len(query.fields) > 0:
             q = q.options(
@@ -195,12 +198,60 @@ class BaseAsyncService(Generic[ID, R]):
                 q = q.where(~getattr(self.orm_cls, clause.field).in_(clause.value))
             else:
                 assert clause.op == QueryOp.fuzzy
-                # Assuming fuzzy search is implemented as a full-text search
-                q = q.where(
-                    func.to_tsvector(getattr(self.orm_cls, clause.field)).match(
-                        clause.value
-                    )
+                clause_prop = self.cls.properties[clause.field]
+                assert (
+                    isinstance(clause_prop, DataProperty) and clause_prop.db is not None
                 )
+                clause_orm_field = getattr(self.orm_cls, clause.field)
+                extra_cols.append(f"{clause.field}_score")
+
+                if clause_prop.db.index_type == IndexType.POSTGRES_FTS_SEVI:
+                    # fuzzy search is implemented using Postgres Full-Text Search
+                    # sevi is a custom text search configuration that we defined in `configs/postgres-fts.sql`
+                    q = q.where(
+                        func.to_tsvector("sevi", clause_orm_field).bool_op("@@")(
+                            func.plainto_tsquery("sevi", clause.value)
+                        )
+                    )
+                    # TODO: figure out which rank function is better
+                    # https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-RANKING
+                    q = q.order_by(
+                        func.ts_rank_cd(
+                            func.to_tsvector("sevi", clause_orm_field),
+                            func.plainto_tsquery("sevi", clause.value),
+                        ).desc()
+                    )
+                    q = q.add_columns(
+                        func.ts_rank_cd(
+                            func.to_tsvector("sevi", clause_orm_field),
+                            func.plainto_tsquery("sevi", clause.value),
+                        ).label(f"{clause.field}_score")
+                    )
+                elif clause_prop.db.index_type == IndexType.POSTGRES_TRIGRAM:
+                    # fuzzy search is implemented using Postgres trigram index
+                    # using a custom function f_unaccent to ignore accents -- see `configs/postgres-fts.sql`
+                    q = q.where(
+                        func.f_unaccent(clause_orm_field).bool_op("%>")(
+                            func.f_unaccent(clause.value)
+                        )
+                    )
+                    q = q.order_by(
+                        func.f_unaccent(clause_orm_field).op("<->>")(
+                            func.f_unaccent(clause.value)
+                        )
+                    )
+                    q = q.add_columns(
+                        (
+                            1
+                            - func.f_unaccent(clause_orm_field).op("<->>")(
+                                func.f_unaccent(clause.value)
+                            )
+                        ).label(f"{clause.field}_score")
+                    )
+                else:
+                    raise NotImplementedError(
+                        f"Fuzzy search is not implemented for index type {clause_prop.db.index_type}"
+                    )
 
         for join_condition in query.join_conditions:
             for join_clause in self.join_clauses[join_condition.prop]:
@@ -211,16 +262,29 @@ class BaseAsyncService(Generic[ID, R]):
                     full=join_condition.join_type == "full",
                 ).options(contains_eager(join_clause["contains_eager"]))
 
-                print(">>>", join_clause)
-
-        cq = select(func.count()).select_from(q.subquery())
+        # Create count query without order_by clause to improve performance
+        cq = select(func.count()).select_from(q.order_by(None).subquery())
         rq = q.limit(query.limit).offset(query.offset)
-        records = self._process_result(await session.execute(rq)).scalars().all()
+
+        if len(extra_cols) == 0:
+            records = self._process_result(await session.execute(rq)).scalars().all()
+            extra_columns = {}
+        else:
+            records = []
+            raw_extra_columns = [[] for col in extra_cols]
+            for row in self._process_result(await session.execute(rq)):
+                records.append(row[0])
+                for i in range(len(extra_cols)):
+                    raw_extra_columns[i].append(row[i + 1])
+            extra_columns = {
+                col: raw_extra_columns[i] for i, col in enumerate(extra_cols)
+            }
+
         if query.return_total:
             total = (await session.execute(cq)).scalar_one()
         else:
             total = None
-        return QueryResult(records, total)
+        return QueryResult(records, extra_columns, total)
 
     async def get_by_id(self, id: ID, session: AsyncSession) -> Optional[R]:
         """Retrieving a record by ID."""

sera/libs/search_helper.py

@@ -253,9 +253,9 @@ class Query(msgspec.Struct):
             else:
                 if join_clause.join_type != "inner":
                     output[target_name].extend(
-                        deser_func((val := getattr(record, source_relprop)))
+                        deser_func(val)
                         for record in result.records
-                        if val is not None
+                        if (val := getattr(record, source_relprop)) is not None
                     )
                 else:
                     output[target_name].extend(
@@ -265,6 +265,8 @@ class Query(msgspec.Struct):
 
         deser_func = dataschema[cls.name].from_db
         output[cls.name] = [deser_func(record) for record in result.records]
+        # include extra columns such as fuzzy search scores
+        output.update(result.extra_columns)
 
         return output
 

sera/make/make_app.py

@@ -7,6 +7,7 @@ from typing import Annotated
 
 from codegen.models import DeferredVar, PredefinedFn, Program, expr, stmt
 from loguru import logger
+
 from sera.make.make_python_api import make_python_api
 from sera.make.make_python_model import (
     make_python_data_model,
@@ -14,10 +15,8 @@ from sera.make.make_python_model import (
     make_python_relational_model,
 )
 from sera.make.make_python_services import make_python_service_structure
-from sera.make.make_typescript_model import (
-    make_typescript_data_model,
-    make_typescript_enum,
-)
+from sera.make.make_typescript_model import make_typescript_data_model
+from sera.make.ts_frontend.make_enums import make_typescript_enums
 from sera.misc import Formatter
 from sera.models import App, DataCollection, parse_schema
 from sera.typing import Language
@@ -170,7 +169,7 @@ def make_app(
         # generate services
         make_python_service_structure(app, collections)
     elif language == Language.Typescript:
-        make_typescript_enum(schema, app.models)
+        make_typescript_enums(schema, app.models)
         make_typescript_data_model(schema, app.models)
 
     Formatter.get_instance().process()

sera/make/make_python_model.py

@@ -22,6 +22,7 @@ from sera.models import (
     Cardinality,
     Class,
     DataProperty,
+    IndexType,
     ObjectProperty,
     Package,
     PyTypeWithDep,
@@ -1207,14 +1208,79 @@ def make_python_relational_model(
         )
 
         index_stmts = []
-        if len(cls.db.indices) > 0:
+
+        if len(cls.db.indices) > 0 or any(
+            isinstance(prop, DataProperty)
+            and prop.db is not None
+            and prop.db.is_indexed
+            and (
+                prop.db.index_type == IndexType.POSTGRES_FTS_SEVI
+                or prop.db.index_type == IndexType.POSTGRES_TRIGRAM
+            )
+            for prop in cls.properties.values()
+        ):
             program.import_("sqlalchemy.Index", True)
+
+            fts_index = []
+            for prop in cls.properties.values():
+                if (
+                    not isinstance(prop, DataProperty)
+                    or prop.db is None
+                    or not prop.db.is_indexed
+                ):
+                    continue
+                if prop.db.index_type == IndexType.POSTGRES_FTS_SEVI:
+                    fts_index.append(
+                        expr.ExprFuncCall(
+                            expr.ExprIdent("Index"),
+                            [
+                                expr.ExprConstant(
+                                    f"ix_{cls.db.table_name}_{get_property_name(prop)}_gin"
+                                ),
+                                expr.ExprFuncCall(
+                                    ident_manager.use("text"),
+                                    [
+                                        expr.ExprConstant(
+                                            f"to_tsvector('sevi', {get_property_name(prop)})"
+                                        )
+                                    ],
+                                ),
+                                PredefinedFn.keyword_assignment(
+                                    "postgresql_using", expr.ExprConstant("gin")
+                                ),
+                            ],
+                        )
+                    )
+                if prop.db.index_type == IndexType.POSTGRES_TRIGRAM:
+                    fts_index.append(
+                        expr.ExprFuncCall(
+                            expr.ExprIdent("Index"),
+                            [
+                                expr.ExprConstant(
+                                    f"ix_{cls.db.table_name}_{get_property_name(prop)}_gist"
+                                ),
+                                expr.ExprFuncCall(
+                                    expr.ExprIdent("text"),
+                                    [
+                                        expr.ExprConstant(
+                                            f"f_unaccent({get_property_name(prop)}) gist_trgm_ops(siglen=256)"
+                                        )
+                                    ],
+                                ),
+                                PredefinedFn.keyword_assignment(
+                                    "postgresql_using", expr.ExprConstant("gist")
+                                ),
+                            ],
+                        )
+                    )
+
             index_stmts.append(
                 stmt.DefClassVarStatement(
                     "_table_args__",
                     None,
                     PredefinedFn.tuple(
-                        [
+                        fts_index
+                        + [
                             expr.ExprFuncCall(
                                 expr.ExprIdent("Index"),
                                 [expr.ExprConstant(index.name)]
@@ -1317,7 +1383,8 @@ def make_python_relational_model(
                                 "unique", expr.ExprConstant(True)
                             )
                         )
-                    elif prop.db.is_indexed:
+                    elif prop.db.is_indexed and prop.db.index_type == IndexType.DEFAULT:
+                        # only add index=True for default index type
                         propvalargs.append(
                             PredefinedFn.keyword_assignment(
                                 "index", expr.ExprConstant(True)