mlrun 1.3.2rc1__py3-none-any.whl → 1.3.2rc2__py3-none-any.whl

mlrun/api/crud/pipelines.py

@@ -47,9 +47,9 @@ class Pipelines(
         format_: mlrun.api.schemas.PipelinesFormat = mlrun.api.schemas.PipelinesFormat.metadata_only,
         page_size: typing.Optional[int] = None,
     ) -> typing.Tuple[int, typing.Optional[int], typing.List[dict]]:
-        if project != "*" and (page_token or page_size):
+        if project != "*" and (page_token or page_size or sort_by):
             raise mlrun.errors.MLRunInvalidArgumentError(
-                "Filtering by project can not be used together with pagination"
+                "Filtering by project can not be used together with pagination, or sorting"
             )
         if format_ == mlrun.api.schemas.PipelinesFormat.summary:
             # we don't support summary format in list pipelines since the returned runs doesn't include the workflow
@@ -73,7 +73,6 @@ class Pipelines(
                 response = kfp_client._run_api.list_runs(
                     page_token=page_token,
                     page_size=mlrun.api.schemas.PipelinesPagination.max_page_size,
-                    sort_by=sort_by,
                     filter=filter_ if page_token == "" else "",
                 )
                 run_dicts.extend([run.to_dict() for run in response.runs or []])

mlrun/api/db/sqldb/models/models_mysql.py

@@ -30,9 +30,8 @@ from sqlalchemy import (
     UniqueConstraint,
 )
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import class_mapper, relationship
 
-import mlrun.utils.db
 from mlrun.api import schemas
 from mlrun.api.utils.db.sql_collation import SQLCollationUtil
 
@@ -41,8 +40,42 @@ NULL = None  # Avoid flake8 issuing warnings when comparing in filter
 run_time_fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
 
 
+class BaseModel:
+    def to_dict(self, exclude=None):
+        """
+        NOTE - this function (currently) does not handle serializing relationships
+        """
+        exclude = exclude or []
+        mapper = class_mapper(self.__class__)
+        columns = [column.key for column in mapper.columns if column.key not in exclude]
+        get_key_value = (
+            lambda c: (c, getattr(self, c).isoformat())
+            if isinstance(getattr(self, c), datetime)
+            else (c, getattr(self, c))
+        )
+        return dict(map(get_key_value, columns))
+
+
+class HasStruct(BaseModel):
+    @property
+    def struct(self):
+        return pickle.loads(self.body)
+
+    @struct.setter
+    def struct(self, value):
+        self.body = pickle.dumps(value)
+
+    def to_dict(self, exclude=None):
+        """
+        NOTE - this function (currently) does not handle serializing relationships
+        """
+        exclude = exclude or []
+        exclude.append("body")
+        return super().to_dict(exclude)
+
+
 def make_label(table):
-    class Label(Base, mlrun.utils.db.BaseModel):
+    class Label(Base, BaseModel):
         __tablename__ = f"{table}_labels"
         __table_args__ = (
             UniqueConstraint("name", "parent", name=f"_{table}_labels_uc"),
@@ -57,7 +90,7 @@ def make_label(table):
 
 
 def make_tag(table):
-    class Tag(Base, mlrun.utils.db.BaseModel):
+    class Tag(Base, BaseModel):
         __tablename__ = f"{table}_tags"
         __table_args__ = (
             UniqueConstraint("project", "name", "obj_id", name=f"_{table}_tags_uc"),
@@ -74,7 +107,7 @@ def make_tag(table):
 # TODO: don't want to refactor everything in one PR so splitting this function to 2 versions - eventually only this one
 #  should be used
 def make_tag_v2(table):
-    class Tag(Base, mlrun.utils.db.BaseModel):
+    class Tag(Base, BaseModel):
         __tablename__ = f"{table}_tags"
         __table_args__ = (
             UniqueConstraint("project", "name", "obj_name", name=f"_{table}_tags_uc"),
@@ -93,7 +126,7 @@ def make_tag_v2(table):
 with warnings.catch_warnings():
     warnings.simplefilter("ignore")
 
-    class Artifact(Base, mlrun.utils.db.HasStruct):
+    class Artifact(Base, HasStruct):
         __tablename__ = "artifacts"
         __table_args__ = (
             UniqueConstraint("uid", "project", "key", name="_artifacts_uc"),
@@ -116,7 +149,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.key}/{self.uid}"
 
-    class Function(Base, mlrun.utils.db.HasStruct):
+    class Function(Base, HasStruct):
         __tablename__ = "functions"
         __table_args__ = (
             UniqueConstraint("name", "project", "uid", name="_functions_uc"),
@@ -139,7 +172,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.name}/{self.uid}"
 
-    class Log(Base, mlrun.utils.db.BaseModel):
+    class Log(Base, BaseModel):
         __tablename__ = "logs"
 
         id = Column(Integer, primary_key=True)
@@ -151,7 +184,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.uid}"
 
-    class Run(Base, mlrun.utils.db.HasStruct):
+    class Run(Base, HasStruct):
         __tablename__ = "runs"
         __table_args__ = (
             UniqueConstraint("uid", "project", "iteration", name="_runs_uc"),
@@ -186,7 +219,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.uid}/{self.iteration}"
 
-    class BackgroundTask(Base, mlrun.utils.db.BaseModel):
+    class BackgroundTask(Base, BaseModel):
         __tablename__ = "background_tasks"
         __table_args__ = (
             UniqueConstraint("name", "project", name="_background_tasks_uc"),
@@ -210,7 +243,7 @@ with warnings.catch_warnings():
         state = Column(String(255, collation=SQLCollationUtil.collation()))
         timeout = Column(Integer)
 
-    class Schedule(Base, mlrun.utils.db.BaseModel):
+    class Schedule(Base, BaseModel):
         __tablename__ = "schedules_v2"
         __table_args__ = (UniqueConstraint("project", "name", name="_schedules_v2_uc"),)
 
@@ -262,14 +295,14 @@ with warnings.catch_warnings():
         Column("user_id", Integer, ForeignKey("users.id")),
     )
 
-    class User(Base, mlrun.utils.db.BaseModel):
+    class User(Base, BaseModel):
         __tablename__ = "users"
         __table_args__ = (UniqueConstraint("name", name="_users_uc"),)
 
         id = Column(Integer, primary_key=True)
         name = Column(String(255, collation=SQLCollationUtil.collation()))
 
-    class Project(Base, mlrun.utils.db.BaseModel):
+    class Project(Base, BaseModel):
         __tablename__ = "projects"
         # For now since we use project name a lot
         __table_args__ = (UniqueConstraint("name", name="_projects_uc"),)
@@ -305,7 +338,7 @@ with warnings.catch_warnings():
         def full_object(self, value):
             self._full_object = pickle.dumps(value)
 
-    class Feature(Base, mlrun.utils.db.BaseModel):
+    class Feature(Base, BaseModel):
         __tablename__ = "features"
         id = Column(Integer, primary_key=True)
         feature_set_id = Column(Integer, ForeignKey("feature_sets.id"))
@@ -319,7 +352,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.name}"
 
-    class Entity(Base, mlrun.utils.db.BaseModel):
+    class Entity(Base, BaseModel):
         __tablename__ = "entities"
         id = Column(Integer, primary_key=True)
         feature_set_id = Column(Integer, ForeignKey("feature_sets.id"))
@@ -333,7 +366,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.name}"
 
-    class FeatureSet(Base, mlrun.utils.db.BaseModel):
+    class FeatureSet(Base, BaseModel):
         __tablename__ = "feature_sets"
         __table_args__ = (
             UniqueConstraint("name", "project", "uid", name="_feature_set_uc"),
@@ -377,7 +410,7 @@ with warnings.catch_warnings():
             # TODO - convert to pickle, to avoid issues with non-json serializable fields such as datetime
             self._full_object = json.dumps(value, default=str)
 
-    class FeatureVector(Base, mlrun.utils.db.BaseModel):
+    class FeatureVector(Base, BaseModel):
         __tablename__ = "feature_vectors"
         __table_args__ = (
             UniqueConstraint("name", "project", "uid", name="_feature_vectors_uc"),
@@ -418,7 +451,7 @@ with warnings.catch_warnings():
             # TODO - convert to pickle, to avoid issues with non-json serializable fields such as datetime
             self._full_object = json.dumps(value, default=str)
 
-    class MarketplaceSource(Base, mlrun.utils.db.BaseModel):
+    class MarketplaceSource(Base, BaseModel):
         __tablename__ = "marketplace_sources"
         __table_args__ = (UniqueConstraint("name", name="_marketplace_sources_uc"),)
 
@@ -449,7 +482,7 @@ with warnings.catch_warnings():
             # TODO - convert to pickle, to avoid issues with non-json serializable fields such as datetime
             self._full_object = json.dumps(value, default=str)
 
-    class DataVersion(Base, mlrun.utils.db.BaseModel):
+    class DataVersion(Base, BaseModel):
         __tablename__ = "data_versions"
         __table_args__ = (UniqueConstraint("version", name="_versions_uc"),)
 

mlrun/api/db/sqldb/models/models_sqlite.py

@@ -31,9 +31,8 @@ from sqlalchemy import (
     UniqueConstraint,
 )
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import class_mapper, relationship
 
-import mlrun.utils.db
 from mlrun.api import schemas
 from mlrun.api.utils.db.sql_collation import SQLCollationUtil
 
@@ -42,8 +41,42 @@ NULL = None  # Avoid flake8 issuing warnings when comparing in filter
 run_time_fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
 
 
+class BaseModel:
+    def to_dict(self, exclude=None):
+        """
+        NOTE - this function (currently) does not handle serializing relationships
+        """
+        exclude = exclude or []
+        mapper = class_mapper(self.__class__)
+        columns = [column.key for column in mapper.columns if column.key not in exclude]
+        get_key_value = (
+            lambda c: (c, getattr(self, c).isoformat())
+            if isinstance(getattr(self, c), datetime)
+            else (c, getattr(self, c))
+        )
+        return dict(map(get_key_value, columns))
+
+
+class HasStruct(BaseModel):
+    @property
+    def struct(self):
+        return pickle.loads(self.body)
+
+    @struct.setter
+    def struct(self, value):
+        self.body = pickle.dumps(value)
+
+    def to_dict(self, exclude=None):
+        """
+        NOTE - this function (currently) does not handle serializing relationships
+        """
+        exclude = exclude or []
+        exclude.append("body")
+        return super().to_dict(exclude)
+
+
 def make_label(table):
-    class Label(Base, mlrun.utils.db.BaseModel):
+    class Label(Base, BaseModel):
         __tablename__ = f"{table}_labels"
         __table_args__ = (
             UniqueConstraint("name", "parent", name=f"_{table}_labels_uc"),
@@ -58,7 +91,7 @@ def make_label(table):
 
 
 def make_tag(table):
-    class Tag(Base, mlrun.utils.db.BaseModel):
+    class Tag(Base, BaseModel):
         __tablename__ = f"{table}_tags"
         __table_args__ = (
             UniqueConstraint("project", "name", "obj_id", name=f"_{table}_tags_uc"),
@@ -75,7 +108,7 @@ def make_tag(table):
 # TODO: don't want to refactor everything in one PR so splitting this function to 2 versions - eventually only this one
 #  should be used
 def make_tag_v2(table):
-    class Tag(Base, mlrun.utils.db.BaseModel):
+    class Tag(Base, BaseModel):
         __tablename__ = f"{table}_tags"
         __table_args__ = (
             UniqueConstraint("project", "name", "obj_name", name=f"_{table}_tags_uc"),
@@ -97,7 +130,7 @@ def make_tag_v2(table):
 with warnings.catch_warnings():
     warnings.simplefilter("ignore")
 
-    class Artifact(Base, mlrun.utils.db.HasStruct):
+    class Artifact(Base, HasStruct):
         __tablename__ = "artifacts"
         __table_args__ = (
             UniqueConstraint("uid", "project", "key", name="_artifacts_uc"),
@@ -118,7 +151,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.key}/{self.uid}"
 
-    class Function(Base, mlrun.utils.db.HasStruct):
+    class Function(Base, HasStruct):
         __tablename__ = "functions"
         __table_args__ = (
             UniqueConstraint("name", "project", "uid", name="_functions_uc"),
@@ -139,7 +172,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.name}/{self.uid}"
 
-    class Log(Base, mlrun.utils.db.BaseModel):
+    class Log(Base, BaseModel):
         __tablename__ = "logs"
 
         id = Column(Integer, primary_key=True)
@@ -151,7 +184,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.uid}"
 
-    class Run(Base, mlrun.utils.db.HasStruct):
+    class Run(Base, HasStruct):
         __tablename__ = "runs"
         __table_args__ = (
             UniqueConstraint("uid", "project", "iteration", name="_runs_uc"),
@@ -182,7 +215,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.uid}/{self.iteration}"
 
-    class BackgroundTask(Base, mlrun.utils.db.BaseModel):
+    class BackgroundTask(Base, BaseModel):
         __tablename__ = "background_tasks"
         __table_args__ = (
             UniqueConstraint("name", "project", name="_background_tasks_uc"),
@@ -200,7 +233,7 @@ with warnings.catch_warnings():
         state = Column(String(255, collation=SQLCollationUtil.collation()))
         timeout = Column(Integer)
 
-    class Schedule(Base, mlrun.utils.db.BaseModel):
+    class Schedule(Base, BaseModel):
         __tablename__ = "schedules_v2"
         __table_args__ = (UniqueConstraint("project", "name", name="_schedules_v2_uc"),)
 
@@ -252,14 +285,14 @@ with warnings.catch_warnings():
         Column("user_id", Integer, ForeignKey("users.id")),
     )
 
-    class User(Base, mlrun.utils.db.BaseModel):
+    class User(Base, BaseModel):
         __tablename__ = "users"
         __table_args__ = (UniqueConstraint("name", name="_users_uc"),)
 
         id = Column(Integer, primary_key=True)
         name = Column(String(255, collation=SQLCollationUtil.collation()))
 
-    class Project(Base, mlrun.utils.db.BaseModel):
+    class Project(Base, BaseModel):
         __tablename__ = "projects"
         # For now since we use project name a lot
         __table_args__ = (UniqueConstraint("name", name="_projects_uc"),)
@@ -293,7 +326,7 @@ with warnings.catch_warnings():
         def full_object(self, value):
             self._full_object = pickle.dumps(value)
 
-    class Feature(Base, mlrun.utils.db.BaseModel):
+    class Feature(Base, BaseModel):
         __tablename__ = "features"
         id = Column(Integer, primary_key=True)
         feature_set_id = Column(Integer, ForeignKey("feature_sets.id"))
@@ -307,7 +340,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.name}"
 
-    class Entity(Base, mlrun.utils.db.BaseModel):
+    class Entity(Base, BaseModel):
         __tablename__ = "entities"
         id = Column(Integer, primary_key=True)
         feature_set_id = Column(Integer, ForeignKey("feature_sets.id"))
@@ -321,7 +354,7 @@ with warnings.catch_warnings():
         def get_identifier_string(self) -> str:
             return f"{self.project}/{self.name}"
 
-    class FeatureSet(Base, mlrun.utils.db.BaseModel):
+    class FeatureSet(Base, BaseModel):
         __tablename__ = "feature_sets"
         __table_args__ = (
             UniqueConstraint("name", "project", "uid", name="_feature_set_uc"),
@@ -357,7 +390,7 @@ with warnings.catch_warnings():
         def full_object(self, value):
             self._full_object = json.dumps(value, default=str)
 
-    class FeatureVector(Base, mlrun.utils.db.BaseModel):
+    class FeatureVector(Base, BaseModel):
         __tablename__ = "feature_vectors"
         __table_args__ = (
             UniqueConstraint("name", "project", "uid", name="_feature_vectors_uc"),
@@ -390,7 +423,7 @@ with warnings.catch_warnings():
         def full_object(self, value):
             self._full_object = json.dumps(value, default=str)
 
-    class MarketplaceSource(Base, mlrun.utils.db.BaseModel):
+    class MarketplaceSource(Base, BaseModel):
         __tablename__ = "marketplace_sources"
         __table_args__ = (UniqueConstraint("name", name="_marketplace_sources_uc"),)
 
@@ -414,7 +447,7 @@ with warnings.catch_warnings():
         def full_object(self, value):
             self._full_object = json.dumps(value, default=str)
 
-    class DataVersion(Base, mlrun.utils.db.BaseModel):
+    class DataVersion(Base, BaseModel):
         __tablename__ = "data_versions"
         __table_args__ = (UniqueConstraint("version", name="_versions_uc"),)
 

mlrun/api/db/sqldb/session.py

@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-import typing
-
 from sqlalchemy import create_engine
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import Session
@@ -22,38 +19,35 @@ from sqlalchemy.orm import sessionmaker as SessionMaker
 
 from mlrun.config import config
 
-# TODO: wrap the following functions in a singleton class
-_engines: typing.Dict[str, Engine] = {}
-_session_makers: typing.Dict[str, SessionMaker] = {}
+engine: Engine = None
+_session_maker: SessionMaker = None
 
 
 # doing lazy load to allow tests to initialize the engine
-def get_engine(dsn=None) -> Engine:
-    global _engines
-    dsn = dsn or config.httpdb.dsn
-    if dsn not in _engines:
-        _init_engine(dsn=dsn)
-    return _engines[dsn]
+def get_engine() -> Engine:
+    global engine
+    if engine is None:
+        _init_engine()
+    return engine
 
 
-def create_session(dsn=None) -> Session:
-    session_maker = _get_session_maker(dsn=dsn)
+def create_session() -> Session:
+    session_maker = _get_session_maker()
     return session_maker()
 
 
 # doing lazy load to allow tests to initialize the engine
-def _get_session_maker(dsn) -> SessionMaker:
-    global _session_makers
-    dsn = dsn or config.httpdb.dsn
-    if dsn not in _session_makers:
-        _init_session_maker(dsn=dsn)
-    return _session_makers[dsn]
+def _get_session_maker() -> SessionMaker:
+    global _session_maker
+    if _session_maker is None:
+        _init_session_maker()
+    return _session_maker
 
 
 # TODO: we accept the dsn here to enable tests to override it, the "right" thing will be that config will be easily
 #  overridable by tests (today when you import the config it is already being initialized.. should be lazy load)
 def _init_engine(dsn=None):
-    global _engines
+    global engine
     dsn = dsn or config.httpdb.dsn
     kwargs = {}
     if "mysql" in dsn:
@@ -68,10 +62,9 @@ def _init_engine(dsn=None):
             "max_overflow": max_overflow,
         }
     engine = create_engine(dsn, **kwargs)
-    _engines[dsn] = engine
-    _init_session_maker(dsn=dsn)
+    _init_session_maker()
 
 
-def _init_session_maker(dsn):
-    global _session_makers
-    _session_makers[dsn] = SessionMaker(bind=get_engine(dsn=dsn))
+def _init_session_maker():
+    global _session_maker
+    _session_maker = SessionMaker(bind=get_engine())

mlrun/api/schemas/__init__.py

@@ -100,11 +100,13 @@ from .model_endpoints import (
     GrafanaStringColumn,
     GrafanaTable,
     GrafanaTimeSeriesTarget,
+    Metric,
     ModelEndpoint,
     ModelEndpointList,
     ModelEndpointMetadata,
     ModelEndpointSpec,
     ModelEndpointStatus,
+    ModelMonitoringMode,
     ModelMonitoringStoreKinds,
 )
 from .object import ObjectKind, ObjectMetadata, ObjectSpec, ObjectStatus

mlrun/api/schemas/constants.py

@@ -170,19 +170,6 @@ class APIStates:
     def terminal_states():
         return [APIStates.online, APIStates.offline]
 
-    @staticmethod
-    def description(state: str):
-        return {
-            APIStates.online: "API is online",
-            APIStates.waiting_for_migrations: "API is waiting for migrations to be triggered. "
-            "Send POST request to /api/operations/migrations to trigger it",
-            APIStates.migrations_in_progress: "Migrations are in progress",
-            APIStates.migrations_failed: "Migrations failed, API can't be started",
-            APIStates.migrations_completed: "Migrations completed, API is waiting to become online",
-            APIStates.offline: "API is offline",
-            APIStates.waiting_for_chief: "API is waiting for chief to be ready",
-        }.get(state, f"Unknown API state '{state}'")
-
 
 class ClusterizationRole:
     chief = "chief"

mlrun/api/schemas/frontend_spec.py

@@ -67,7 +67,6 @@ class FrontendSpec(pydantic.BaseModel):
     default_function_pod_resources: Resources = Resources()
     default_function_preemption_mode: str
     feature_store_data_prefixes: typing.Optional[typing.Dict[str, str]]
-    allowed_artifact_path_prefixes_list: typing.List[str]
 
     # ce_mode is deprecated, we will use the full ce config instead and ce_mode will be removed in 1.6.0
     ce_mode: typing.Optional[str]