dtlpy 1.90.39__py3-none-any.whl → 1.92.18__py3-none-any.whl

tests/features/environment.py

@@ -27,6 +27,7 @@ def before_all(context):
     # Get index driver from env var
     context.index_driver_var = os.environ.get("INDEX_DRIVER_VAR", None)
 
+
 @fixture
 def after_feature(context, feature):
     print_feature_filename(context, feature)
@@ -57,6 +58,13 @@ def after_feature(context, feature):
                     for page in apps:
                         for app in page:
                             app.uninstall()
+                    models = dl.models.list(
+                        filters=dl.Filters(use_defaults=False, resource=dl.FiltersResource.MODEL,
+                                           field="app.dpkName",
+                                           values=dpk.name))
+                    for page in models:
+                        for model in page:
+                            model.delete()
                     dpk.delete()
                 except:
                     logging.exception('Failed to delete dpk')
@@ -148,6 +156,26 @@ def after_scenario(context, scenario):
         context.dl = dl
 
 
+def get_step_key(step):
+    return '{}: line {}. {}'.format(step.location.filename, step.location.line, step.name)
+
+
+@fixture
+def before_step(context, step):
+    key = get_step_key(step)
+    setattr(context, key, time.time())
+
+
+@fixture
+def after_step(context, step):
+    key = get_step_key(step)
+    start_time = getattr(context, key, None)
+    total_time = time.time() - start_time
+    if total_time > 3:
+        print("######## {}\nStep Duration: {}".format(key, total_time))
+    delattr(context, key)
+
+
 @fixture
 def before_feature(context, feature):
     if 'rc_only' in context.tags and 'rc' not in os.environ.get("DLP_ENV_NAME"):
@@ -168,6 +196,9 @@ def fix_project_with_frozen_datasets(project):
 @fixture
 def before_tag(context, tag):
     if "skip_test" in tag:
+        """
+        For example: @skip_test_DAT-99999
+        """
         dat = tag.split("_")[-1] if "DAT" in tag else ""
         if hasattr(context, "scenario"):
             context.scenario.skip(f"Test mark as SKIPPED, Should be merged after {dat}")
@@ -225,6 +256,11 @@ def after_tag(context, tag):
             use_fixture(drivers_delete, context)
         except Exception:
             logging.exception('Failed to delete driver')
+    elif tag == 'models.delete':
+        try:
+            use_fixture(models_delete, context)
+        except Exception:
+            logging.exception('Failed to delete model')
     elif tag == 'setenv.reset':
         try:
             use_fixture(reset_setenv, context)
@@ -335,6 +371,11 @@ def delete_pipeline(context):
     while context.to_delete_pipelines_ids:
         pipeline_id = context.to_delete_pipelines_ids.pop(0)
         try:
+            filters = context.dl.Filters(resource=context.dl.FiltersResource.EXECUTION, field='latestStatus.status', values=['created', 'in-progress'], operator='in')
+            filters.add(field='pipeline.id', values=pipeline_id)
+            executions = context.dl.executions.list(filters=filters)
+            for execution in executions.items:
+                execution.terminate()
             context.dl.pipelines.delete(pipeline_id=pipeline_id)
         except context.dl.exceptions.NotFound:
             pass
@@ -418,4 +459,29 @@ def print_feature_filename(context, feature):
     s_r = SummaryReporter(context.config)
     stream = getattr(sys, s_r.output_stream_name, sys.stderr)
     p_stream = StreamOpener.ensure_stream_with_encoder(stream)
-    p_stream.write("{}\n".format(feature.filename.split('/')[-1]))
+    p_stream.write(f"Feature Finished : {feature.filename.split('/')[-1]}\n")
+    p_stream.write(f"Status: {str(feature.status).split('.')[-1]} - Duration: {feature.duration:.2f} seconds\n")
+
+
+@fixture
+def models_delete(context):
+    all_deleted = True
+    if hasattr(context, 'to_delete_model_ids'):
+        for model_id in context.to_delete_model_ids:
+            try:
+                context.project.models.delete(model_id=model_id)
+            except context.dl.exceptions.NotFound:
+                pass
+            except:
+                all_deleted = False
+                logging.exception('Failed deleting model: {}'.format(model_id))
+
+    for model in context.project.models.list().all():
+        try:
+            model.delete()
+        except context.dl.exceptions.NotFound:
+            pass
+        except:
+            all_deleted = False
+            logging.exception('Failed deleting model: {}'.format(model.id))
+    assert all_deleted

dtlpy/callbacks/__init__.py

@@ -1,16 +0,0 @@
-#! /usr/bin/env python3
-# This file is part of DTLPY.
-#
-# DTLPY is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# DTLPY is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with DTLPY.  If not, see <http://www.gnu.org/licenses/>.
-# from .progress_viewer import ProgressViewer

dtlpy/callbacks/piper_progress_reporter.py

@@ -1,29 +0,0 @@
-def get_callback(progress):
-    from keras.callbacks import Callback
-    import numpy as np
-    import logging
-    import time
-
-    logger = logging.getLogger(name='dtlpy')
-
-    class PiperProgressReporter(Callback):
-        def __init__(self, progress):
-            super(PiperProgressReporter, self).__init__()
-            self.progress = progress
-            self.results = dict()
-            self.epoch_time_start = None
-
-        def on_train_begin(self, logs=None):
-            self.results = dict()
-
-        def on_epoch_begin(self, batch, logs=None):
-            self.epoch_time_start = time.time()
-
-        def on_epoch_end(self, epoch, logs=None):
-            logs_dict = dict(zip(list(logs.keys()), [float(num) for num in np.array(list(logs.values())).tolist()]))
-            self.results[epoch] = logs_dict
-            self.results[epoch]['runtime'] = time.time() - self.epoch_time_start
-            self.progress.report_output(output={'epoch': epoch,
-                                                'logs': logs_dict})
-
-    return PiperProgressReporter(progress)

dtlpy/callbacks/progress_viewer.py

@@ -1,54 +0,0 @@
-def main():
-    from keras.callbacks import Callback
-    import dtlpy as dl
-    import numpy as np
-    import logging
-    import time
-    import json
-    import os
-
-    logger = logging.getLogger(name='dtlpy')
-
-    class ProgressViewer(Callback):
-        def __init__(self, session_id, directory=None):
-            super(ProgressViewer, self).__init__()
-            # init Dataloop instance
-            # get sessions artifact
-            self.session = dl.sessions.get(session_id=session_id)
-            artifacts = self.session.artifacts.list()
-            self.artifact = None
-            for artifact in artifacts:
-                if artifact.type == 'progress':
-                    self.artifact = artifact
-                    logger.info('Progress artifact found. overwriting. artifact_id: %s' % self.artifact.id)
-                    break
-            if self.artifact is None:
-                self.artifact = self.session.artifacts.create(artifact_name='progress.yml',
-                                                              artifact_type='progress',
-                                                              description='update progress on each epoch')
-
-                logger.info('[INFO] Creating progress artifact. artifact_id: %s' % self.artifact.id)
-            if directory is None:
-                directory = './results'
-            if not os.path.isdir(directory):
-                os.makedirs(directory)
-            self.filename = os.path.join(directory, 'progress.yml')
-            self.results = dict()
-            self.epoch_time_start = None
-
-        def on_train_begin(self, logs=None):
-            self.results = dict()
-
-        def on_epoch_begin(self, batch, logs=None):
-            self.epoch_time_start = time.time()
-
-        def on_epoch_end(self, epoch, logs=None):
-            self.results[epoch] = dict(zip(list(logs.keys()), np.array(list(logs.values())).tolist()))
-            self.results[epoch]['runtime'] = time.time() - self.epoch_time_start
-            with open(self.filename, 'w') as f:
-                json.dump(self.results, f)
-            self.session.artifacts.upload(filepath=self.filename,
-                                          artifact_name='progress.yml',
-                                          artifact_type='progress')
-
-    return ProgressViewer