lomas-core 0.3.4__tar.gz

lomas_core-0.3.4/PKG-INFO

@@ -0,0 +1,43 @@
+Metadata-Version: 2.1
+Name: lomas-core
+Version: 0.3.4
+Summary: Lomas core.
+Home-page: https://github.com/dscc-admin-ch/lomas/
+Author: Data Science Competence Center, Swiss Federal Statistical Office
+Author-email: dscc@bfs.admin.ch
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Security
+Requires-Python: >=3.11, <3.13
+Description-Content-Type: text/markdown
+Requires-Dist: diffprivlib==0.6.5
+Requires-Dist: diffprivlib_logger>=0.0.3
+Requires-Dist: fastapi>=0.111.1
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: opendp==0.10.0
+Requires-Dist: opendp_logger>=0.3.0
+Requires-Dist: pandas>=2.2.2
+Requires-Dist: pymongo>=4.6.3
+Requires-Dist: scikit-learn>=1.4.2
+Requires-Dist: smartnoise-synth>=1.0.4
+Requires-Dist: smartnoise_synth_logger>=0.0.3
+
+<h1 align="center">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_darkmode_txt.png"  width="300">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_txt.png"  width="300">
+  <img alt="This is what is displayed on Pypi" src="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_txt.png">
+</picture>
+</h1><br>
+
+# Core
+See the technical documentation of core: https://dscc-admin-ch.github.io/lomas-docs/core_api.html.

lomas_core-0.3.4/README.md

@@ -0,0 +1,10 @@
+<h1 align="center">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_darkmode_txt.png"  width="300">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_txt.png"  width="300">
+  <img alt="This is what is displayed on Pypi" src="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_txt.png">
+</picture>
+</h1><br>
+
+# Core
+See the technical documentation of core: https://dscc-admin-ch.github.io/lomas-docs/core_api.html.

lomas_core-0.3.4/lomas_core/constants.py

@@ -0,0 +1,34 @@
+from enum import StrEnum
+
+# Server error messages
+INTERNAL_SERVER_ERROR = (
+    "Internal server error. Please contact the administrator of this service."
+)
+
+
+class DPLibraries(StrEnum):
+    """Name of DP Library used in the query."""
+
+    SMARTNOISE_SQL = "smartnoise_sql"
+    SMARTNOISE_SYNTH = "smartnoise_synth"
+    OPENDP = "opendp"
+    DIFFPRIVLIB = "diffprivlib"
+
+
+# Smartnoise synth
+class SSynthMarginalSynthesizer(StrEnum):
+    """Marginal Synthesizer models for smartnoise synth."""
+
+    AIM = "aim"
+    MWEM = "mwem"
+    MST = "mst"
+    PAC_SYNTH = "pacsynth"
+
+
+class SSynthGanSynthesizer(StrEnum):
+    """GAN Synthesizer models for smartnoise synth."""
+
+    DP_CTGAN = "dpctgan"
+    PATE_CTGAN = "patectgan"
+    PATE_GAN = "pategan"
+    DP_GAN = "dpgan"

lomas_core-0.3.4/lomas_core/error_handler.py

@@ -0,0 +1,125 @@
+from typing import Type
+
+from fastapi import FastAPI, Request, status
+from fastapi.responses import JSONResponse
+from pymongo.errors import WriteConcernError
+
+from lomas_core.constants import INTERNAL_SERVER_ERROR
+from lomas_core.logger import LOG
+
+
+class InvalidQueryException(Exception):
+    """
+    Custom exception for invalid queries.
+
+    For example, this exception will occur when the query:
+        - is not an opendp measurement
+        - cannot be reconstructed properly (for opendp and diffprivlib)
+    """
+
+    def __init__(self, error_message: str) -> None:
+        """Invalid Query Exception initialisation.
+
+        Args:
+            error_message (str): _description_
+        """
+        self.error_message = error_message
+
+
+class ExternalLibraryException(Exception):
+    """
+    Custom exception for issues within external libraries.
+
+    This exception will occur when the processes fail within the
+    external libraries (smartnoise-sql, opendp, diffprivlib)
+    """
+
+    def __init__(self, library: str, error_message: str) -> None:
+        """External Query Exception initialisation.
+
+        Args:
+            library (str): _description_
+            error_message (str): _description_
+        """
+        self.library = library
+        self.error_message = error_message
+
+
+class UnauthorizedAccessException(Exception):
+    """
+    Custom exception for unauthorized access:
+
+    (unknown user, no access to dataset, etc)
+    """
+
+    def __init__(self, error_message: str) -> None:
+        self.error_message = error_message
+
+
+class InternalServerException(Exception):
+    """Custom exception for issues within server internal functionalities."""
+
+    def __init__(self, error_message: str) -> None:
+        self.error_message = error_message
+
+
+KNOWN_EXCEPTIONS: tuple[Type[BaseException], ...] = (
+    ExternalLibraryException,
+    InternalServerException,
+    InvalidQueryException,
+    UnauthorizedAccessException,
+    WriteConcernError,
+)
+
+
+# Custom exception handlers
+def add_exception_handlers(app: FastAPI) -> None:
+    """
+    Translates custom exceptions to JSONResponses.
+
+    Args:
+        app (FastAPI): A fastapi App.
+    """
+
+    @app.exception_handler(InvalidQueryException)
+    async def invalid_query_exception_handler(
+        _: Request, exc: InvalidQueryException
+    ) -> JSONResponse:
+        LOG.info(f"InvalidQueryException raised: {exc.error_message}")
+        return JSONResponse(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            content={"InvalidQueryException": exc.error_message},
+        )
+
+    @app.exception_handler(ExternalLibraryException)
+    async def external_library_exception_handler(
+        _: Request, exc: ExternalLibraryException
+    ) -> JSONResponse:
+        LOG.info(f"ExternalLibraryException raised: {exc.error_message}")
+        return JSONResponse(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            content={
+                "ExternalLibraryException": exc.error_message,
+                "library": exc.library,
+            },
+        )
+
+    @app.exception_handler(UnauthorizedAccessException)
+    async def unauthorized_access_exception_handler(
+        _: Request, exc: UnauthorizedAccessException
+    ) -> JSONResponse:
+        LOG.info(f"UnauthorizedAccessException raised: {exc.error_message}")
+        return JSONResponse(
+            status_code=status.HTTP_403_FORBIDDEN,
+            content={"UnauthorizedAccessException": exc.error_message},
+        )
+
+    @app.exception_handler(InternalServerException)
+    async def internal_server_exception_handler(
+        _: Request, exc: InternalServerException
+    ) -> JSONResponse:
+        LOG.info(f"InternalServerException raised: {exc.error_message}")
+        return JSONResponse(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            content={"InternalServerException": INTERNAL_SERVER_ERROR},
+        )

lomas_core-0.3.4/lomas_core/logger.py

@@ -0,0 +1,29 @@
+import logging
+import logging.config
+
+LOGGING_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "standard": {
+            "format": "%(asctime)s - %(levelname)s - \
+                [%(filename)s:%(lineno)s - %(funcName)s()] - %(message)s"
+        }
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
+            "stream": "ext://sys.stdout",
+            "level": "INFO",
+        }
+    },
+    "root": {
+        "level": "INFO",
+        "handlers": ["console"],
+        "propagate": True,
+    },
+}
+logging.config.dictConfig(LOGGING_CONFIG)
+
+LOG = logging.getLogger("")

lomas_core-0.3.4/lomas_core/models/collections.py

@@ -0,0 +1,249 @@
+from datetime import datetime
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, Discriminator, Field, Tag, model_validator
+
+from lomas_core.models.constants import (
+    CARDINALITY_FIELD,
+    CATEGORICAL_TYPE_PREFIX,
+    DB_TYPE_FIELD,
+    TYPE_FIELD,
+    MetadataColumnType,
+    Precision,
+    PrivateDatabaseType,
+)
+
+# Dataset of User
+# -----------------------------------------------------------------------------
+
+
+class DatasetOfUser(BaseModel):
+    """BaseModel for informations of a user on a dataset."""
+
+    dataset_name: str
+    initial_epsilon: float
+    initial_delta: float
+    total_spent_epsilon: float
+    total_spent_delta: float
+
+
+# User
+# -----------------------------------------------------------------------------
+
+
+class User(BaseModel):
+    """BaseModel for a user in a user collection."""
+
+    user_name: str
+    may_query: bool
+    datasets_list: List[DatasetOfUser]
+
+
+class UserCollection(BaseModel):
+    """BaseModel for users collection."""
+
+    users: List[User]
+
+
+# Dataset Access Data
+# -----------------------------------------------------------------------------
+
+
+class DSAccess(BaseModel):
+    """BaseModel for access info to a private dataset."""
+
+    database_type: str
+
+
+class DSPathAccess(DSAccess):
+    """BaseModel for a local dataset."""
+
+    database_type: Literal[PrivateDatabaseType.PATH]  # type: ignore
+    path: str
+
+
+class DSS3Access(DSAccess):
+    """BaseModel for a dataset on S3."""
+
+    database_type: Literal[PrivateDatabaseType.S3]  # type: ignore
+    endpoint_url: str
+    bucket: str
+    key: str
+    access_key_id: Optional[str] = None
+    secret_access_key: Optional[str] = None
+    credentials_name: str
+
+
+class DSInfo(BaseModel):
+    """BaseModel for a dataset."""
+
+    dataset_name: str
+    dataset_access: Annotated[
+        Union[DSPathAccess, DSS3Access], Field(discriminator=DB_TYPE_FIELD)
+    ]
+    metadata_access: Annotated[
+        Union[DSPathAccess, DSS3Access], Field(discriminator=DB_TYPE_FIELD)
+    ]
+
+
+class DatasetsCollection(BaseModel):
+    """BaseModel for datasets collection."""
+
+    datasets: List[DSInfo]
+
+
+# Metadata
+# -----------------------------------------------------------------------------
+
+
+class ColumnMetadata(BaseModel):
+    """Base model for column metadata."""
+
+    private_id: bool = False
+    nullable: bool = False
+    # See issue #323 for checking this and validating.
+
+    max_partition_length: Optional[Annotated[int, Field(gt=0)]] = None
+    max_influenced_partitions: Optional[Annotated[int, Field(gt=0)]] = None
+    max_partition_contributions: Optional[Annotated[int, Field(gt=0)]] = None
+
+
+class StrMetadata(ColumnMetadata):
+    """Model for string metadata."""
+
+    type: Literal[MetadataColumnType.STRING]
+
+
+class CategoricalColumnMetadata(ColumnMetadata):
+    """Model for categorical column metadata."""
+
+    @model_validator(mode="after")
+    def validate_categories(self):
+        """Makes sure number of categories matches cardinality."""
+        if len(self.categories) != self.cardinality:
+            raise ValueError("Number of categories should be equal to cardinality.")
+        return self
+
+
+class StrCategoricalMetadata(CategoricalColumnMetadata):
+    """Model for categorical string metadata."""
+
+    type: Literal[MetadataColumnType.STRING]
+    cardinality: int
+    categories: List[str]
+
+
+class BoundedColumnMetadata(ColumnMetadata):
+    """Model for columns with bounded data."""
+
+    @model_validator(mode="after")
+    def validate_bounds(self):
+        """Validates column bounds."""
+        if (
+            self.lower is not None
+            and self.upper is not None
+            and self.lower > self.upper
+        ):
+            raise ValueError("Lower bound cannot be larger than upper bound.")
+
+        return self
+
+
+class IntMetadata(BoundedColumnMetadata):
+    """Model for integer column metadata."""
+
+    type: Literal[MetadataColumnType.INT]
+    precision: Precision
+    lower: int
+    upper: int
+
+
+class IntCategoricalMetadata(CategoricalColumnMetadata):
+    """Model for integer categorical column metadata."""
+
+    type: Literal[MetadataColumnType.INT]
+    precision: Precision
+    cardinality: int
+    categories: List[int]
+
+
+class FloatMetadata(BoundedColumnMetadata):
+    """Model for float column metadata."""
+
+    type: Literal[MetadataColumnType.FLOAT]
+    precision: Precision
+    lower: float
+    upper: float
+
+
+class BooleanMetadata(ColumnMetadata):
+    """Model for boolean column metadata."""
+
+    type: Literal[MetadataColumnType.BOOLEAN]
+
+
+class DatetimeMetadata(BoundedColumnMetadata):
+    """Model for datetime column metadata."""
+
+    type: Literal[MetadataColumnType.DATETIME]
+    lower: datetime
+    upper: datetime
+
+
+def get_column_metadata_discriminator(v: Any) -> str:
+    """Discriminator function for determining the type of column metadata.
+
+    Args:
+        v (Any): The unparsed column metadata (either dict or class object)
+
+    Raises:
+        ValueError: If the column type cannot be found.
+
+    Returns:
+        str: The metadata string type.
+    """
+    if isinstance(v, dict):
+        col_type = v.get(TYPE_FIELD)
+    else:
+        col_type = getattr(v, TYPE_FIELD)
+
+    if (
+        col_type
+        in (
+            MetadataColumnType.STRING,
+            MetadataColumnType.INT,
+        )
+    ) and (
+        ((isinstance(v, dict)) and CARDINALITY_FIELD in v)
+        or (hasattr(v, CARDINALITY_FIELD))
+    ):
+        col_type = f"{CATEGORICAL_TYPE_PREFIX}{col_type}"
+
+    if not isinstance(col_type, str):
+        raise ValueError("Could not find column type.")
+
+    return col_type
+
+
+class Metadata(BaseModel):
+    """BaseModel for a metadata format."""
+
+    max_ids: Annotated[int, Field(gt=0)]
+    rows: Annotated[int, Field(gt=0)]
+    row_privacy: bool
+    censor_dims: Optional[bool] = False
+    columns: Dict[
+        str,
+        Annotated[
+            Union[
+                Annotated[StrMetadata, Tag(MetadataColumnType.STRING)],
+                Annotated[StrCategoricalMetadata, Tag(MetadataColumnType.CAT_STRING)],
+                Annotated[IntMetadata, Tag(MetadataColumnType.INT)],
+                Annotated[IntCategoricalMetadata, Tag(MetadataColumnType.CAT_INT)],
+                Annotated[FloatMetadata, Tag(MetadataColumnType.FLOAT)],
+                Annotated[BooleanMetadata, Tag(MetadataColumnType.BOOLEAN)],
+                Annotated[DatetimeMetadata, Tag(MetadataColumnType.DATETIME)],
+            ],
+            Discriminator(get_column_metadata_discriminator),
+        ],
+    ]

lomas_core-0.3.4/lomas_core/models/config.py

@@ -0,0 +1,104 @@
+from typing import Annotated, List, Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from lomas_core.models.constants import (
+    AdminDBType,
+    PrivateDatabaseType,
+    TimeAttackMethod,
+)
+
+
+class TimeAttack(BaseModel):
+    """BaseModel for configs to prevent timing attacks."""
+
+    method: TimeAttackMethod
+    magnitude: float
+
+
+class Server(BaseModel):
+    """BaseModel for uvicorn server configs."""
+
+    time_attack: TimeAttack
+    host_ip: str
+    host_port: int
+    log_level: str
+    reload: bool
+    workers: int
+
+
+class DBConfig(BaseModel):
+    """BaseModel for database type config."""
+
+
+class YamlDBConfig(DBConfig):
+    """BaseModel for dataset store configs  in case of a Yaml database."""
+
+    db_type: Literal[AdminDBType.YAML]  # type: ignore
+    db_file: str
+
+
+class MongoDBConfig(DBConfig):
+    """BaseModel for dataset store configs  in case of a  MongoDB database."""
+
+    db_type: Literal[AdminDBType.MONGODB]  # type: ignore
+    address: str
+    port: int
+    username: str
+    password: str
+    db_name: str
+    max_pool_size: int
+    min_pool_size: int
+    max_connecting: int
+
+
+class PrivateDBCredentials(BaseModel):
+    """BaseModel for private database credentials."""
+
+
+class S3CredentialsConfig(PrivateDBCredentials):
+    """BaseModel for S3 database credentials."""
+
+    model_config = ConfigDict(extra="allow")
+
+    db_type: Literal[PrivateDatabaseType.S3]  # type: ignore
+    credentials_name: str
+    access_key_id: str
+    secret_access_key: str
+
+
+class OpenDPConfig(BaseModel):
+    """BaseModel for openDP librairy config."""
+
+    contrib: bool
+    floating_point: bool
+    honest_but_curious: bool
+
+
+class DPLibraryConfig(BaseModel):
+    """BaseModel for DP librairies config."""
+
+    opendp: OpenDPConfig
+
+
+class Config(BaseModel):
+    """Server runtime config."""
+
+    # Develop mode
+    develop_mode: bool
+
+    # Server configs
+    server: Server
+
+    # A limit on the rate which users can submit answers
+    submit_limit: float
+
+    admin_database: Annotated[
+        Union[MongoDBConfig, YamlDBConfig], Field(discriminator="db_type")
+    ]
+
+    private_db_credentials: List[Union[S3CredentialsConfig]] = Field(
+        ..., discriminator="db_type"
+    )
+
+    dp_libraries: DPLibraryConfig

lomas_core-0.3.4/lomas_core/models/constants.py

@@ -0,0 +1,67 @@
+from enum import IntEnum, StrEnum
+
+# Field names
+# -----------------------------------------------------------------------------
+
+DB_TYPE_FIELD = "database_type"
+TYPE_FIELD = "type"
+CARDINALITY_FIELD = "cardinality"
+
+
+# Metadata
+# -----------------------------------------------------------------------------
+
+
+class MetadataColumnType(StrEnum):
+    """Column types for metadata."""
+
+    STRING = "string"
+    CAT_STRING = "categorical_string"
+    INT = "int"
+    CAT_INT = "categorical_int"
+    FLOAT = "float"
+    BOOLEAN = "boolean"
+    DATETIME = "datetime"
+
+
+CATEGORICAL_TYPE_PREFIX = "categorical_"
+
+
+class Precision(IntEnum):
+    """Precision of integer and float data."""
+
+    SINGLE = 32
+    DOUBLE = 64
+
+
+# Config / Dataset Connectors
+# -----------------------------------------------------------------------------
+
+
+class ConfigKeys(StrEnum):
+    """Keys of the configuration file."""
+
+    RUNTIME_ARGS: str = "runtime_args"
+    SETTINGS: str = "settings"
+
+
+class AdminDBType(StrEnum):
+    """Types of administration databases."""
+
+    YAML: str = "yaml"
+    MONGODB: str = "mongodb"
+
+
+class TimeAttackMethod(StrEnum):
+    """Possible methods against timing attacks."""
+
+    JITTER = "jitter"
+    STALL = "stall"
+
+
+# Private Databases
+class PrivateDatabaseType(StrEnum):
+    """Type of Private Database for the private data."""
+
+    PATH = "PATH_DB"
+    S3 = "S3_DB"

lomas_core-0.3.4/lomas_core/models/requests.py

@@ -0,0 +1,170 @@
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from lomas_core.constants import (
+    DPLibraries,
+    SSynthGanSynthesizer,
+    SSynthMarginalSynthesizer,
+)
+from lomas_core.error_handler import InternalServerException
+
+
+class LomasRequestModel(BaseModel):
+    """Base class for all types of requests to the lomas server.
+
+    We differentiate between requests and queries:
+        - a request does not necessarily require an algorithm
+          to be executed on the private dataset (e.g. some cost requests).
+        - a query requires executing an algorithm on a private
+          dataset (or a potentially a dummy).
+    """
+
+    dataset_name: str
+
+
+class GetDummyDataset(LomasRequestModel):
+    """Model input to get a dummy dataset."""
+
+    dummy_nb_rows: int = Field(..., gt=0)
+    dummy_seed: int
+
+
+class QueryModel(LomasRequestModel):
+    """
+    Base input model for any query on a dataset.
+
+    We differentiate between requests and queries:
+        - a request does not necessarily require an algorithm
+          to be executed on the private dataset (e.g. some cost requests).
+        - a query requires executing an algorithm on a private
+          dataset (or a potentially a dummy).
+    """
+
+
+class DummyQueryModel(QueryModel):
+    """Input model for a query on a dummy dataset."""
+
+    dummy_nb_rows: int = Field(..., gt=0)
+    dummy_seed: int
+
+
+# SmartnoiseSQL
+# ----------------------------------------------------------------------------
+class SmartnoiseSQLRequestModel(LomasRequestModel):
+    """Base input model for a smarnoise-sql request."""
+
+    query_str: str
+    epsilon: float = Field(..., gt=0)
+    delta: float = Field(..., gt=0)
+    mechanisms: dict
+
+
+class SmartnoiseSQLQueryModel(SmartnoiseSQLRequestModel, QueryModel):
+    """Base input model for a smartnoise-sql query."""
+
+    postprocess: bool
+
+
+class SmartnoiseSQLDummyQueryModel(SmartnoiseSQLQueryModel, DummyQueryModel):
+    """Input model for a smartnoise-sql query on a dummy dataset."""
+
+
+# SmartnoiseSynth
+# ----------------------------------------------------------------------------
+class SmartnoiseSynthRequestModel(LomasRequestModel):
+    """Base input model for a SmartnoiseSynth request."""
+
+    synth_name: Union[SSynthMarginalSynthesizer, SSynthGanSynthesizer]
+    epsilon: float = Field(..., gt=0)
+    delta: Optional[float] = None
+    select_cols: List
+    synth_params: dict
+    nullable: bool
+    constraints: str
+
+
+class SmartnoiseSynthQueryModel(SmartnoiseSynthRequestModel, QueryModel):
+    """Base input model for a smarnoise-synth query."""
+
+    return_model: bool
+    condition: str
+    nb_samples: int
+
+
+class SmartnoiseSynthDummyQueryModel(SmartnoiseSynthQueryModel, DummyQueryModel):
+    """Input model for a smarnoise-synth query on a dummy dataset."""
+
+    # Same as normal query.
+    return_model: bool
+    condition: str
+    nb_samples: int
+
+
+# OpenDP
+# ----------------------------------------------------------------------------
+class OpenDPRequestModel(LomasRequestModel):
+    """Base input model for an opendp request."""
+
+    model_config = ConfigDict(use_attribute_docstrings=True)
+    opendp_json: str
+    """Opendp pipeline."""
+    fixed_delta: Optional[float] = None
+
+
+class OpenDPQueryModel(OpenDPRequestModel, QueryModel):
+    """Base input model for an opendp query."""
+
+
+class OpenDPDummyQueryModel(OpenDPRequestModel, DummyQueryModel):
+    """Input model for an opendp query on a dummy dataset."""
+
+
+# DiffPrivLib
+# ----------------------------------------------------------------------------
+class DiffPrivLibRequestModel(LomasRequestModel):
+    """Base input model for a diffprivlib request."""
+
+    diffprivlib_json: str
+    feature_columns: list
+    target_columns: Optional[list]
+    test_size: float = Field(..., gt=0.0, lt=1.0)
+    test_train_split_seed: int
+    imputer_strategy: str
+
+
+class DiffPrivLibQueryModel(DiffPrivLibRequestModel, QueryModel):
+    """Base input model for a diffprivlib query."""
+
+
+class DiffPrivLibDummyQueryModel(DiffPrivLibQueryModel, DummyQueryModel):
+    """Input model for a DiffPrivLib query on a dummy dataset."""
+
+
+# Utils
+# ----------------------------------------------------------------------------
+
+
+def model_input_to_lib(request: LomasRequestModel) -> DPLibraries:
+    """Return the type of DP library given a LomasRequestModel.
+
+    Args:
+        request (LomasRequestModel): The user request
+
+    Raises:
+        InternalServerException: If the library type cannot be determined.
+
+    Returns:
+        DPLibraries: The type of library for the request.
+    """
+    match request:
+        case SmartnoiseSQLRequestModel():
+            return DPLibraries.SMARTNOISE_SQL
+        case SmartnoiseSynthRequestModel():
+            return DPLibraries.SMARTNOISE_SYNTH
+        case OpenDPRequestModel():
+            return DPLibraries.OPENDP
+        case DiffPrivLibRequestModel():
+            return DPLibraries.DIFFPRIVLIB
+        case _:
+            raise InternalServerException("Cannot find library type for given model.")

lomas_core-0.3.4/lomas_core/models/responses.py

@@ -0,0 +1,171 @@
+from typing import Annotated, Dict, List, Literal, Union
+
+import pandas as pd
+from diffprivlib.validation import DiffprivlibMixin
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Discriminator,
+    PlainSerializer,
+    PlainValidator,
+    ValidationInfo,
+    field_validator,
+)
+from snsynth import Synthesizer
+
+from lomas_core.constants import DPLibraries
+from lomas_core.models.utils import (
+    dataframe_from_dict,
+    dataframe_to_dict,
+    deserialize_model,
+    serialize_model,
+)
+
+
+class ResponseModel(BaseModel):
+    """Base model for any response from the server."""
+
+
+class InitialBudgetResponse(ResponseModel):
+    """Model for responses to initial budget queries."""
+
+    initial_epsilon: float
+    initial_delta: float
+
+
+class SpentBudgetResponse(ResponseModel):
+    """Model for responses to spent budget queries."""
+
+    total_spent_epsilon: float
+    total_spent_delta: float
+
+
+class RemainingBudgetResponse(ResponseModel):
+    """Model for responses to remaining budget queries."""
+
+    remaining_epsilon: float
+    remaining_delta: float
+
+
+class DummyDsResponse(ResponseModel):
+    """Model for responses to dummy dataset requests."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    dtypes: Dict[str, str]
+    datetime_columns: List[str]
+    dummy_df: Annotated[pd.DataFrame, PlainSerializer(dataframe_to_dict)]
+
+    @field_validator("dummy_df", mode="before")
+    @classmethod
+    def deserialize_dummy_df(
+        cls, v: pd.DataFrame | dict, info: ValidationInfo
+    ) -> pd.DataFrame:
+        """Decodes the dict representation of the dummy df with correct types.
+
+        Only does so if the input value is not already a dataframe.
+        Args:
+            v (pd.DataFrame | dict): The dataframe to decode.
+            info (ValidationInfo): Validation info to access other model fields.
+
+        Returns:
+            pd.DataFrame: The decoded dataframe.
+        """
+        if isinstance(v, pd.DataFrame):
+            return v
+
+        dtypes = info.data["dtypes"]
+        datetime_columns = info.data["datetime_columns"]
+        dummy_df = dataframe_from_dict(v)
+        dummy_df = dummy_df.astype(dtypes)
+        for col in datetime_columns:
+            dummy_df[col] = pd.to_datetime(dummy_df[col])
+        return dummy_df
+
+
+class CostResponse(ResponseModel):
+    """Model for responses to cost estimation requests."""
+
+    epsilon: float
+    delta: float
+
+
+# Query Responses
+# -----------------------------------------------------------------------------
+
+
+# DiffPrivLib
+class DiffPrivLibQueryResult(BaseModel):
+    """Model for diffprivlib query result."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    res_type: Literal[DPLibraries.DIFFPRIVLIB] = DPLibraries.DIFFPRIVLIB
+    score: float
+    model: Annotated[
+        DiffprivlibMixin,
+        PlainSerializer(serialize_model),
+        PlainValidator(deserialize_model),
+    ]
+
+
+# SmartnoiseSQL
+class SmartnoiseSQLQueryResult(BaseModel):
+    """Type for smartnoise_sql result type."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    res_type: Literal[DPLibraries.SMARTNOISE_SQL] = DPLibraries.SMARTNOISE_SQL
+    df: Annotated[
+        pd.DataFrame,
+        PlainSerializer(dataframe_to_dict),
+        PlainValidator(dataframe_from_dict),
+    ]
+
+
+# SmartnoiseSynth
+class SmartnoiseSynthModel(BaseModel):
+    """Type for smartnoise_synth result when it is a pickled model."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    res_type: Literal[DPLibraries.SMARTNOISE_SYNTH] = DPLibraries.SMARTNOISE_SYNTH
+    model: Annotated[
+        Synthesizer, PlainSerializer(serialize_model), PlainValidator(deserialize_model)
+    ]
+
+
+class SmartnoiseSynthSamples(BaseModel):
+    """Type for smartnoise_synth result when it is a dataframe of samples."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    res_type: Literal["sn_synth_samples"] = "sn_synth_samples"
+    df_samples: Annotated[
+        pd.DataFrame,
+        PlainSerializer(dataframe_to_dict),
+        PlainValidator(dataframe_from_dict),
+    ]
+
+
+# OpenDP
+class OpenDPQueryResult(BaseModel):
+    """Type for opendp result."""
+
+    res_type: Literal[DPLibraries.OPENDP] = DPLibraries.OPENDP
+    value: Union[int, float, List[Union[int, float]]]
+
+
+# Response object
+QueryResultTypeAlias = Union[
+    DiffPrivLibQueryResult,
+    SmartnoiseSQLQueryResult,
+    SmartnoiseSynthModel,
+    SmartnoiseSynthSamples,
+    OpenDPQueryResult,
+]
+
+
+class QueryResponse(CostResponse):
+    """Model for responses to queries."""
+
+    requested_by: str
+    result: Annotated[
+        QueryResultTypeAlias,
+        Discriminator("res_type"),
+    ]

lomas_core-0.3.4/lomas_core/models/utils.py

@@ -0,0 +1,69 @@
+import pickle
+from base64 import b64decode, b64encode
+from typing import Any
+
+import pandas as pd
+
+PANDAS_SERIALIZATION_ORIENT = "tight"
+
+
+def dataframe_to_dict(df: pd.DataFrame) -> dict:
+    """Transforms pandas dataframe into a dictionary.
+
+    Args:
+        df (pd.DataFrame): The dataframe to "serialize".
+
+    Returns:
+        dict: The pandas dataframe in dictionary format.
+    """
+    return df.to_dict(orient=PANDAS_SERIALIZATION_ORIENT)
+
+
+def dataframe_from_dict(serialized_df: pd.DataFrame | dict) -> pd.DataFrame:
+    """Transforms input dict into pandas dataframe.
+
+    If the input is already a dataframe, it is simply returned unmodified.
+
+    Args:
+        serialized_df (pd.DataFrame | dict): Dataframe in dict format.
+            Or pd.Dataframe.
+
+    Returns:
+        pd.DataFrame: The transformed dataframe.
+    """
+    if isinstance(serialized_df, pd.DataFrame):
+        return serialized_df
+
+    return pd.DataFrame.from_dict(serialized_df, orient=PANDAS_SERIALIZATION_ORIENT)
+
+
+def serialize_model(model: Any) -> str:
+    """
+    Serialise a python object into an utf-8 string.
+
+    Fitted Smartnoise Synth synthesizer or fitted DiffPrivLib pipeline.
+
+    Args:
+        model (Any): An object to serialise
+
+    Returns:
+        str: string of serialised model
+    """
+    serialized = b64encode(pickle.dumps(model))
+    return serialized.decode("utf-8")
+
+
+def deserialize_model(serialized_model: Any) -> Any:
+    """Deserialize a base64 encoded byte string into a python object.
+
+    Args:
+        serialized_model (Any): Encoded python object.
+
+    Returns:
+        Any: Deserialized python object.
+    """
+    if isinstance(serialized_model, str):
+        raw_bytes = b64decode(serialized_model)
+        return pickle.loads(raw_bytes)
+
+    return serialized_model

lomas_core-0.3.4/lomas_core.egg-info/PKG-INFO

@@ -0,0 +1,43 @@
+Metadata-Version: 2.1
+Name: lomas-core
+Version: 0.3.4
+Summary: Lomas core.
+Home-page: https://github.com/dscc-admin-ch/lomas/
+Author: Data Science Competence Center, Swiss Federal Statistical Office
+Author-email: dscc@bfs.admin.ch
+License: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Security
+Requires-Python: >=3.11, <3.13
+Description-Content-Type: text/markdown
+Requires-Dist: diffprivlib==0.6.5
+Requires-Dist: diffprivlib_logger>=0.0.3
+Requires-Dist: fastapi>=0.111.1
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: opendp==0.10.0
+Requires-Dist: opendp_logger>=0.3.0
+Requires-Dist: pandas>=2.2.2
+Requires-Dist: pymongo>=4.6.3
+Requires-Dist: scikit-learn>=1.4.2
+Requires-Dist: smartnoise-synth>=1.0.4
+Requires-Dist: smartnoise_synth_logger>=0.0.3
+
+<h1 align="center">
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_darkmode_txt.png"  width="300">
+  <source media="(prefers-color-scheme: light)" srcset="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_txt.png"  width="300">
+  <img alt="This is what is displayed on Pypi" src="https://github.com/dscc-admin-ch/lomas/blob/wip_322_darkmode-logo/images/lomas_logo_txt.png">
+</picture>
+</h1><br>
+
+# Core
+See the technical documentation of core: https://dscc-admin-ch.github.io/lomas-docs/core_api.html.

lomas_core-0.3.4/lomas_core.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+README.md
+pyproject.toml
+setup.py
+lomas_core/__init__.py
+lomas_core/constants.py
+lomas_core/error_handler.py
+lomas_core/logger.py
+lomas_core.egg-info/PKG-INFO
+lomas_core.egg-info/SOURCES.txt
+lomas_core.egg-info/dependency_links.txt
+lomas_core.egg-info/requires.txt
+lomas_core.egg-info/top_level.txt
+lomas_core/models/__init__.py
+lomas_core/models/collections.py
+lomas_core/models/config.py
+lomas_core/models/constants.py
+lomas_core/models/requests.py
+lomas_core/models/responses.py
+lomas_core/models/utils.py

lomas_core-0.3.4/lomas_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

lomas_core-0.3.4/lomas_core.egg-info/requires.txt

@@ -0,0 +1,11 @@
+diffprivlib==0.6.5
+diffprivlib_logger>=0.0.3
+fastapi>=0.111.1
+numpy>=1.26.4
+opendp==0.10.0
+opendp_logger>=0.3.0
+pandas>=2.2.2
+pymongo>=4.6.3
+scikit-learn>=1.4.2
+smartnoise-synth>=1.0.4
+smartnoise_synth_logger>=0.0.3

lomas_core-0.3.4/lomas_core.egg-info/top_level.txt

@@ -0,0 +1 @@
+lomas_core

lomas_core-0.3.4/pyproject.toml

@@ -0,0 +1,34 @@
+[tool.black]
+line-length = 88
+include = '\.pyi?$'
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+line_length = 88
+split_on_trailing_comma = true
+src_paths = ["lomas_core"]
+
+[tool.flake8]
+max-line-length = 88
+
+[tool.mypy]
+disable_error_code = ["import-untyped", "import-not-found", "attr-defined"]
+
+[tool.pylint.MASTER]
+max-line-length = 88
+
+[tool.pylint."MESSAGES CONTROL"]
+disable = [
+    "E0401",  # import-error
+    "C0114",  # missing-module-docstring
+    "W1203",  # use lazy % formatting in logging functions,
+    "R0903",  # too-few-public-methods
+]
+
+[tool.pydocstringformatter]
+write = true
+max-line-length = 88

lomas_core-0.3.4/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

lomas_core-0.3.4/setup.py

@@ -0,0 +1,49 @@
+import pathlib
+
+from setuptools import find_packages, setup
+
+here = pathlib.Path(__file__).parent.resolve()
+
+this_directory = pathlib.Path(__file__).parent
+long_description = (this_directory / "README.md").read_text()
+
+
+setup(
+    name="lomas-core",
+    packages=find_packages(),
+    version="0.3.4",
+    description="Lomas core.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/dscc-admin-ch/lomas/",
+    author="Data Science Competence Center, Swiss Federal Statistical Office",
+    author_email="dscc@bfs.admin.ch",
+    license="MIT",
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Science/Research",
+        "License :: OSI Approved :: MIT License",
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Security",
+    ],
+    python_requires=">=3.11, <3.13",
+    install_requires=[
+        "diffprivlib==0.6.5",
+        "diffprivlib_logger>=0.0.3",
+        "fastapi>=0.111.1",
+        "numpy>=1.26.4",
+        "opendp==0.10.0",
+        "opendp_logger>=0.3.0",
+        "pandas>=2.2.2",
+        "pymongo>=4.6.3",
+        "scikit-learn>=1.4.2",
+        "smartnoise-synth>=1.0.4",
+        "smartnoise_synth_logger>=0.0.3"
+    ]
+)