sera-2 1.1.0__py3-none-any.whl

sera/make/make_python_model.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+from codegen.models import DeferredVar, PredefinedFn, Program, expr, stmt
+from sera.misc import assert_isinstance, filter_duplication
+from sera.models import DataProperty, ObjectProperty, Package, Schema
+
+
+def make_python_data_model(schema: Schema, target_pkg: Package):
+    """Generate public classes for the API from the schema."""
+    for cls in schema.topological_sort():
+        program = Program()
+        program.import_("__future__.annotations", True)
+        program.import_("msgspec", False)
+        # program.import_("dataclasses.dataclass", True)
+
+        # program.root(stmt.PythonStatement("@dataclass"))
+        cls_ast = program.root.class_(cls.name, [expr.ExprIdent("msgspec.Struct")])
+        for prop in cls.properties.values():
+            if prop.is_private:
+                # skip private fields as this is for APIs exchange
+                continue
+
+            if isinstance(prop, DataProperty):
+                pytype = prop.datatype.get_python_type()
+                if pytype.dep is not None:
+                    program.import_(pytype.dep, True)
+                cls_ast(stmt.DefClassVarStatement(prop.name, pytype.type))
+            elif isinstance(prop, ObjectProperty):
+                program.import_(
+                    f"{target_pkg.module(prop.target.get_pymodule_name()).path}.{prop.target.name}",
+                    is_import_attr=True,
+                )
+                pytype = prop.target.name
+                cls_ast(stmt.DefClassVarStatement(prop.name, pytype))
+
+        target_pkg.module(cls.get_pymodule_name()).write(program)
+
+
+def make_python_relational_model(
+    schema: Schema, target_pkg: Package, target_data_pkg: Package
+):
+    """Make python classes for relational database using SQLAlchemy.
+
+    The new classes is going be compatible with SQLAlchemy 2.
+    """
+    app = target_pkg.app
+
+    def make_base(custom_types: Sequence[ObjectProperty]):
+        """Make a base class for our database."""
+        program = Program()
+        program.import_("__future__.annotations", True)
+        program.import_("sera.libs.base_orm.BaseORM", True)
+        program.import_("sera.libs.base_orm.create_engine", True)
+        program.import_("contextlib.contextmanager", True)
+        program.import_("sqlalchemy.orm.Session", True)
+
+        # assume configuration for the app at the top level
+        program.import_(f"{app.config.path}.DB_CONNECTION", True)
+        program.import_(f"{app.config.path}.DB_DEBUG", True)
+
+        program.root.linebreak()
+
+        type_map = []
+        for custom_type in custom_types:
+            program.import_(
+                f"{target_data_pkg.module(custom_type.target.get_pymodule_name()).path}.{custom_type.target.name}",
+                is_import_attr=True,
+            )
+
+            if custom_type.cardinality.is_star_to_many():
+                if custom_type.is_map:
+                    program.import_("typing.Mapping", True)
+                    program.import_("sera.libs.baseorm.DictDataClassType", True)
+                    type = f"Mapping[str, {custom_type.target.name}]"
+                    maptype = f"DictDataClassType({custom_type.target.name})"
+                else:
+                    program.import_("typing.Sequence", True)
+                    program.import_("sera.libs.baseorm.ListDataClassType", True)
+                    type = f"Sequence[str, {custom_type.target.name}]"
+                    maptype = f"ListDataClassType({custom_type.target.name})"
+            else:
+                program.import_("sera.libs.baseorm.DataClassType", True)
+                type = custom_type.target.name
+                maptype = f"DataClassType({custom_type.target.name})"
+
+            if custom_type.is_optional:
+                program.import_("typing.Optional", True)
+                type = f"Optional[{type}]"
+
+            type_map.append((expr.ExprIdent(type), expr.ExprIdent(maptype)))
+
+        cls_ast = program.root.class_("Base", [expr.ExprIdent("BaseORM")])(
+            stmt.DefClassVarStatement(
+                "type_annotation_map", "dict", PredefinedFn.dict(type_map)
+            ),
+            return_self=True,
+        )
+
+        program.root.linebreak()
+        program.root.assign(
+            DeferredVar("engine", force_name="engine"),
+            expr.ExprFuncCall(
+                expr.ExprIdent("create_engine"),
+                [
+                    expr.ExprIdent("DB_CONNECTION"),
+                    PredefinedFn.keyword_assignment(
+                        "debug", expr.ExprIdent("DB_DEBUG")
+                    ),
+                ],
+            ),
+        )
+
+        program.root.linebreak()
+        program.root.func("create_db_and_tables", [])(
+            stmt.PythonStatement("Base.metadata.create_all(engine)"),
+        )
+
+        program.root.linebreak()
+        program.root.python_stmt("@contextmanager")
+        program.root.func("get_session", [])(
+            lambda ast00: ast00.python_stmt("with Session(engine) as session:")(
+                lambda ast01: ast01.python_stmt("yield session")
+            )
+        )
+
+        target_pkg.module("base").write(program)
+
+    custom_types: list[ObjectProperty] = []
+
+    for cls in schema.topological_sort():
+        if cls.db is None:
+            # skip classes that are not stored in the database
+            continue
+
+        program = Program()
+        program.import_("__future__.annotations", True)
+        program.import_("sqlalchemy.orm.MappedAsDataclass", True)
+        program.import_("sqlalchemy.orm.mapped_column", True)
+        program.import_("sqlalchemy.orm.Mapped", True)
+        program.import_("typing.ClassVar", True)
+        program.import_(f"{target_pkg.path}.base.Base", True)
+
+        cls_ast = program.root.class_(
+            cls.name, [expr.ExprIdent("MappedAsDataclass"), expr.ExprIdent("Base")]
+        )
+        cls_ast(
+            stmt.DefClassVarStatement(
+                "__tablename__",
+                type="ClassVar[str]",
+                value=expr.ExprConstant(cls.db.table_name),
+            ),
+            stmt.LineBreak(),
+        )
+
+        for prop in cls.properties.values():
+            if prop.db is None:
+                # skip properties that are not stored in the database
+                continue
+
+            if isinstance(prop, DataProperty):
+                pytype = prop.datatype.get_sqlalchemy_type()
+                if pytype.dep is not None:
+                    program.import_(pytype.dep, True)
+
+                propname = prop.name
+                proptype = f"Mapped[{pytype.type}]"
+
+                propvalargs = []
+                if prop.db.is_primary_key:
+                    propvalargs.append(
+                        PredefinedFn.keyword_assignment(
+                            "primary_key", expr.ExprConstant(True)
+                        )
+                    )
+                    if prop.db.is_auto_increment:
+                        propvalargs.append(
+                            PredefinedFn.keyword_assignment(
+                                "autoincrement", expr.ExprConstant("auto")
+                            )
+                        )
+                    if prop.db.is_unique:
+                        propvalargs.append(
+                            PredefinedFn.keyword_assignment(
+                                "unique", expr.ExprConstant(True)
+                            )
+                        )
+                propval = expr.ExprFuncCall(
+                    expr.ExprIdent("mapped_column"), propvalargs
+                )
+            else:
+                assert isinstance(prop, ObjectProperty)
+                if prop.target.db is not None:
+                    # if the target class is in the database, we generate a foreign key for it.
+                    program.import_("sqlalchemy.ForeignKey", True)
+
+                    # we store this class in the database
+                    propname = f"{prop.name}_id"
+                    idprop = prop.target.get_id_property()
+                    assert idprop is not None
+                    idprop_pytype = idprop.datatype.get_sqlalchemy_type()
+                    if idprop_pytype.dep is not None:
+                        program.import_(idprop_pytype.dep, True)
+
+                    proptype = f"Mapped[{idprop_pytype.type}]"
+
+                    propvalargs: list[expr.Expr] = [
+                        expr.ExprConstant(f"{prop.target.db.table_name}.{idprop.name}")
+                    ]
+                    propvalargs.append(
+                        PredefinedFn.keyword_assignment(
+                            "ondelete",
+                            expr.ExprConstant(prop.db.on_delete.to_sqlalchemy()),
+                        )
+                    )
+                    propvalargs.append(
+                        PredefinedFn.keyword_assignment(
+                            "onupdate",
+                            expr.ExprConstant(prop.db.on_update.to_sqlalchemy()),
+                        )
+                    )
+
+                    propval = expr.ExprFuncCall(
+                        expr.ExprIdent("mapped_column"),
+                        [
+                            expr.ExprFuncCall(
+                                expr.ExprIdent("ForeignKey"),
+                                propvalargs,
+                            ),
+                        ],
+                    )
+                else:
+                    # if the target class is not in the database,
+                    program.import_(
+                        f"{target_pkg.module(prop.target.get_pymodule_name()).path}.{prop.target.name}",
+                        is_import_attr=True,
+                    )
+                    propname = prop.name
+                    proptype = f"Mapped[{prop.target.name}]"
+
+                    # we have two choices, one is to create a composite class, one is to create a custom field
+                    if prop.db.is_embedded == "composite":
+                        # for a class to be composite, it must have only data properties
+                        program.import_("sqlalchemy.orm.composite", True)
+                        propvalargs = [expr.ExprIdent(prop.target.name)]
+                        for p in prop.target.properties.values():
+                            propvalargs.append(
+                                expr.ExprFuncCall(
+                                    expr.ExprIdent("mapped_column"),
+                                    [
+                                        expr.ExprIdent(f"{prop.name}_{p.name}"),
+                                        expr.ExprIdent(
+                                            assert_isinstance(p, DataProperty)
+                                            .datatype.get_sqlalchemy_type()
+                                            .type
+                                        ),
+                                        expr.PredefinedFn.keyword_assignment(
+                                            "nullable",
+                                            expr.ExprConstant(prop.is_optional),
+                                        ),
+                                    ],
+                                )
+                            )
+                        propval = expr.ExprFuncCall(
+                            expr.ExprIdent("composite"),
+                            propvalargs,
+                        )
+                    else:
+                        # we create a custom field, the custom field mapping need to be defined in the base
+                        propval = expr.ExprFuncCall(expr.ExprIdent("mapped_column"), [])
+                        custom_types.append(prop)
+
+            cls_ast(stmt.DefClassVarStatement(propname, proptype, propval))
+
+        target_pkg.module(cls.get_pymodule_name()).write(program)
+
+    # make a base class that implements the mapping for custom types
+    custom_types = filter_duplication(
+        custom_types, lambda p: (p.target.name, p.cardinality, p.is_optional, p.is_map)
+    )
+    make_base(custom_types)

sera/make/make_python_services.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+from codegen.models import DeferredVar, Program, expr, stmt
+from loguru import logger
+from sera.misc import assert_not_null
+from sera.models import App, DataCollection, Package
+
+
+def make_python_service_structure(app: App, collections: Sequence[DataCollection]):
+    """Make the basic structure for the service."""
+    app.services.ensure_exists()
+
+    for collection in collections:
+        make_python_service(collection, app.services)
+
+
+def make_python_service(collection: DataCollection, target_pkg: Package):
+    app = target_pkg.app
+
+    outmod = target_pkg.module(collection.get_pymodule_name())
+    if outmod.exists():
+        logger.info("`{}` already exists. Skip generation.", outmod.path)
+        return
+
+    # assuming the collection has only one class
+    cls = collection.cls
+    id_type = assert_not_null(cls.get_id_property()).datatype.get_python_type().type
+
+    program = Program()
+    program.import_("__future__.annotations", True)
+    program.import_(
+        app.models.db.path + f".{collection.get_pymodule_name()}.{collection.name}",
+        True,
+    )
+    program.import_(app.config.path + f".schema", True)
+    program.import_("sera.libs.base_service.BaseService", True)
+
+    program.root(
+        stmt.LineBreak(),
+        lambda ast00: ast00.class_(
+            collection.get_service_name(),
+            [expr.ExprIdent(f"BaseService[{id_type}, {cls.name}]")],
+        )(
+            lambda ast01: ast01.func(
+                "__init__",
+                [
+                    DeferredVar.simple("self"),
+                ],
+            )(
+                lambda ast02: ast02.expr(
+                    expr.ExprFuncCall(
+                        expr.ExprIdent("super().__init__"),
+                        [
+                            expr.ExprRawPython(f"schema.classes['{cls.name}']"),
+                            expr.ExprIdent(cls.name),
+                        ],
+                    )
+                ),
+            ),
+        ),
+    )
+    outmod.write(program)

sera/make/make_typescript_model.py

@@ -0,0 +1 @@
+from __future__ import annotations

sera/misc/__init__.py

@@ -0,0 +1,16 @@
+from sera.misc._rdf import SingleNS, Term
+from sera.misc._utils import (
+    assert_isinstance,
+    assert_not_null,
+    filter_duplication,
+    to_snake_case,
+)
+
+__all__ = [
+    "SingleNS",
+    "Term",
+    "to_snake_case",
+    "assert_isinstance",
+    "filter_duplication",
+    "assert_not_null",
+]

sera/misc/_rdf.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from functools import cached_property
+from typing import Iterable
+
+from rdflib import OWL, RDF, RDFS, SKOS, XSD, Graph, URIRef
+from rdflib.namespace import NamespaceManager
+from sm.typing import IRI, InternalID, RelIRI
+
+
+@dataclass
+class SingleNS:
+    alias: str
+    namespace: str
+
+    def __post_init__(self):
+        assert self.namespace.endswith("/") or self.namespace.endswith(
+            "#"
+        ), f"Namespace {self.namespace} should end with / or #"
+
+    def term(self, name: str) -> Term:
+        return Term(self, name)
+
+    def id(self, uri: IRI | URIRef) -> str:
+        assert uri.startswith(self.namespace), (uri, self.namespace)
+        return uri[len(self.namespace) :]
+
+    def uri(self, name: InternalID) -> URIRef:
+        return URIRef(self.namespace + name)
+
+    def uristr(self, name: InternalID) -> IRI:
+        return self.namespace + name
+
+    def __getattr__(self, name: InternalID):
+        return self.alias + ":" + name
+
+    def __getitem__(self, name: InternalID):
+        return self.alias + ":" + name
+
+    def __contains__(self, uri: IRI | URIRef) -> bool:
+        return uri.startswith(self.namespace)
+
+    def rel2abs(self, reluri: RelIRI) -> URIRef:
+        return URIRef(self.namespace + reluri.split(":")[1])
+
+    def abs2rel(self, uri: IRI | URIRef) -> RelIRI:
+        return self.alias + ":" + self.id(uri)
+
+
+@dataclass
+class Term:
+    ns: SingleNS
+    name: str
+    reluri: str = field(init=False)
+    uri: URIRef = field(init=False)
+
+    def __post_init__(self):
+        self.reluri = self.ns[self.name]
+        self.uri = self.ns.uri(self.name)

sera/misc/_utils.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import re
+from typing import Any, Callable, Iterable, Optional, TypeVar
+
+T = TypeVar("T")
+
+
+def to_snake_case(camelcase: str) -> str:
+    """Convert camelCase to snake_case."""
+    snake = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", camelcase)
+    snake = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", snake)
+    return snake.lower()
+
+
+def assert_isinstance(x: Any, cls: type[T]) -> T:
+    if not isinstance(x, cls):
+        raise Exception(f"{type(x)} doesn't match with {type(cls)}")
+    return x
+
+
+def assert_not_null(x: Optional[T]) -> T:
+    assert x is not None
+    return x
+
+
+def filter_duplication(
+    lst: Iterable[T], key_fn: Optional[Callable[[T], Any]] = None
+) -> list[T]:
+    keys = set()
+    new_lst = []
+    if key_fn is not None:
+        for item in lst:
+            k = key_fn(item)
+            if k in keys:
+                continue
+
+            keys.add(k)
+            new_lst.append(item)
+    else:
+        for k in lst:
+            if k in keys:
+                continue
+            keys.add(k)
+            new_lst.append(k)
+    return new_lst

sera/models/__init__.py

@@ -0,0 +1,24 @@
+from sera.models._class import Class
+from sera.models._collection import DataCollection
+from sera.models._datatype import DataType
+from sera.models._module import App, Module, Package
+from sera.models._multi_lingual_string import MultiLingualString
+from sera.models._parse import parse_schema
+from sera.models._property import Cardinality, DataProperty, ObjectProperty, Property
+from sera.models._schema import Schema
+
+__all__ = [
+    "parse_schema",
+    "Schema",
+    "Property",
+    "DataProperty",
+    "ObjectProperty",
+    "Class",
+    "Cardinality",
+    "DataType",
+    "MultiLingualString",
+    "Package",
+    "DataCollection",
+    "Module",
+    "App",
+]

sera/models/_class.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+from sera.misc import to_snake_case
+from sera.models._multi_lingual_string import MultiLingualString
+from sera.models._property import DataProperty, ObjectProperty
+
+
+@dataclass(kw_only=True)
+class ClassDBMapInfo:
+    """Represent database information for a class."""
+
+    # name of a corresponding table in the database for this class
+    table_name: str
+
+
+@dataclass(kw_only=True)
+class Class:
+    """Represent a class in the schema."""
+
+    # name of the class in the application layer
+    name: str = field(
+        metadata={
+            "description": "Name of the property in the application layer, so it must be a valid Python identifier"
+        }
+    )
+    # human-readable name of the class
+    label: MultiLingualString
+    # human-readable description of the class
+    description: MultiLingualString
+    # properties of the class
+    properties: dict[str, DataProperty | ObjectProperty]
+
+    # whether to store this class in a table in the database
+    db: Optional[ClassDBMapInfo]
+
+    def get_id_property(self) -> Optional[DataProperty]:
+        """
+        Get the ID property of this class.
+        The ID property is the one tagged with is_primary_key
+        """
+        assert self.db is not None, "This class is not stored in the database"
+        for prop in self.properties.values():
+            if (
+                isinstance(prop, DataProperty)
+                and prop.db is not None
+                and prop.db.is_primary_key
+            ):
+                return prop
+        return None
+
+    def get_pymodule_name(self) -> str:
+        """Get the python module name of this class as if there is a python module created to store this class only."""
+        return to_snake_case(self.name)

sera/models/_collection.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from sera.models._class import Class
+
+
+@dataclass
+class DataCollection:
+    """Represent a data collection, which can be a class or a data product created via some transformation."""
+
+    cls: Class
+
+    @property
+    def name(self) -> str:
+        """Get the name of the collection."""
+        return self.cls.name
+
+    def get_pymodule_name(self) -> str:
+        """Get the python module name of this collection as if there is a python module created to store this collection only."""
+        return self.cls.get_pymodule_name()
+
+    def get_queryable_fields(self) -> set[str]:
+        """Get the fields of this collection that can be used in a queries."""
+        field_names = set()
+        for prop in self.cls.properties.values():
+            if prop.db is None:
+                # This property is not stored in the database, so we skip it
+                continue
+            field_names.add(prop.name)
+        return field_names
+
+    def get_service_name(self):
+        return f"{self.name}Service"

sera/models/_datatype.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import datetime
+from dataclasses import dataclass
+from enum import Enum
+from typing import Literal
+
+
+@dataclass
+class TypeWithDep:
+    type: str
+    dep: str | None = None
+
+    def get_python_type(self) -> type:
+        """Get the Python type from the type string for typing annotation in Python."""
+        if self.type == "str":
+            return str
+        elif self.type == "int":
+            return int
+        elif self.type == "float":
+            return float
+        elif self.type == "bool":
+            return bool
+        elif self.type == "bytes":
+            return bytes
+        elif self.type == "dict":
+            return dict
+        elif self.type == "datetime":
+            return datetime.datetime
+        else:
+            raise ValueError(f"Unknown type: {self.type}")
+
+
+@dataclass
+class DataType:
+    type: Literal["str", "int", "datetime", "float", "bool", "bytes", "dict"]
+    is_list: bool = False
+    parent: DataType | None = None
+
+    def get_python_type(self) -> TypeWithDep:
+        if self.type in ["str", "int", "float", "bool", "bytes", "dict"]:
+            return TypeWithDep(type=self.type)
+        if self.type == "datetime":
+            return TypeWithDep(type="datetime", dep="datetime.datetime")
+        raise NotImplementedError(self.type)
+
+    def get_sqlalchemy_type(self) -> TypeWithDep:
+        if self.type in ["str", "int", "float", "bool", "bytes"]:
+            return TypeWithDep(type=self.type)
+        if self.type == "dict":
+            return TypeWithDep(type="JSON")
+        if self.type == "datetime":
+            return TypeWithDep(type="datetime", dep="datetime.datetime")
+        raise NotImplementedError(self.type)