dtlpy 1.115.44__py3-none-any.whl → 1.116.6__py3-none-any.whl

dtlpy/ml/summary_writer.py

@@ -1,57 +1,57 @@
-# from torch.utils.tensorboard import SummaryWriter, FileWriter
-# from tensorboard.compat.proto.event_pb2 import Event, SessionLog
-# import time
-#
-#
-# class DtlpyFileWrite(FileWriter):
-#     def __init__(self, snapshot, *args, **kwargs):
-#         self.snapshot = snapshot
-#         super(DtlpyFileWrite, self).__init__(*args, **kwargs)
-#
-#     def add_event(self, event, step=None, walltime=None):
-#         event: Event
-#         event.wall_time = time.time() if walltime is None else walltime
-#         if step is not None:
-#             # Make sure step is converted from numpy or other formats
-#             # since protobuf might not convert depending on version
-#             event.step = int(step)
-#         self.event_writer.add_event(event)
-#         # print(event)
-#         # print(event.step)
-#         # print(event.SerializeToString())
-#         dtlpy_records = list()
-#         for v in event.summary.value:
-#             dtlpy_records.append({'time': event.wall_time,
-#                                   'step': event.step,
-#                                   'tag': v.tag,
-#                                   'value': v.simple_value,
-#                                   'image': v.image.encoded_image_string,
-#                                   'string': v.tensor.string_val,
-#                                   'plugin_name': v.metadata.plugin_data.plugin_name})
-#         print(dtlpy_records)
-#         # self.snapshot.add_event(event)
-#
-#
-# class DtlpySummaryWriter(SummaryWriter):
-#     def __init__(self, snapshot, *args, **kwargs):
-#         self.snapshot = snapshot
-#         super(DtlpySummaryWriter, self).__init__(*args, **kwargs)
-#
-#     def _get_file_writer(self):
-#         """Returns the default FileWriter instance. Recreates it if closed."""
-#         print(self.snapshot)
-#         if self.all_writers is None or self.file_writer is None:
-#             self.file_writer = DtlpyFileWrite(snapshot=self.snapshot,
-#                                               log_dir=self.log_dir,
-#                                               max_queue=self.max_queue,
-#                                               flush_secs=self.flush_secs,
-#                                               filename_suffix=self.filename_suffix)
-#             self.all_writers = {self.file_writer.get_logdir(): self.file_writer}
-#             if self.purge_step is not None:
-#                 most_recent_step = self.purge_step
-#                 self.file_writer.add_event(
-#                     Event(step=most_recent_step, file_version='brain.Event:2'))
-#                 self.file_writer.add_event(
-#                     Event(step=most_recent_step, session_log=SessionLog(status=SessionLog.START)))
-#                 self.purge_step = None
-#         return self.file_writer
+# from torch.utils.tensorboard import SummaryWriter, FileWriter
+# from tensorboard.compat.proto.event_pb2 import Event, SessionLog
+# import time
+#
+#
+# class DtlpyFileWrite(FileWriter):
+#     def __init__(self, snapshot, *args, **kwargs):
+#         self.snapshot = snapshot
+#         super(DtlpyFileWrite, self).__init__(*args, **kwargs)
+#
+#     def add_event(self, event, step=None, walltime=None):
+#         event: Event
+#         event.wall_time = time.time() if walltime is None else walltime
+#         if step is not None:
+#             # Make sure step is converted from numpy or other formats
+#             # since protobuf might not convert depending on version
+#             event.step = int(step)
+#         self.event_writer.add_event(event)
+#         # print(event)
+#         # print(event.step)
+#         # print(event.SerializeToString())
+#         dtlpy_records = list()
+#         for v in event.summary.value:
+#             dtlpy_records.append({'time': event.wall_time,
+#                                   'step': event.step,
+#                                   'tag': v.tag,
+#                                   'value': v.simple_value,
+#                                   'image': v.image.encoded_image_string,
+#                                   'string': v.tensor.string_val,
+#                                   'plugin_name': v.metadata.plugin_data.plugin_name})
+#         print(dtlpy_records)
+#         # self.snapshot.add_event(event)
+#
+#
+# class DtlpySummaryWriter(SummaryWriter):
+#     def __init__(self, snapshot, *args, **kwargs):
+#         self.snapshot = snapshot
+#         super(DtlpySummaryWriter, self).__init__(*args, **kwargs)
+#
+#     def _get_file_writer(self):
+#         """Returns the default FileWriter instance. Recreates it if closed."""
+#         print(self.snapshot)
+#         if self.all_writers is None or self.file_writer is None:
+#             self.file_writer = DtlpyFileWrite(snapshot=self.snapshot,
+#                                               log_dir=self.log_dir,
+#                                               max_queue=self.max_queue,
+#                                               flush_secs=self.flush_secs,
+#                                               filename_suffix=self.filename_suffix)
+#             self.all_writers = {self.file_writer.get_logdir(): self.file_writer}
+#             if self.purge_step is not None:
+#                 most_recent_step = self.purge_step
+#                 self.file_writer.add_event(
+#                     Event(step=most_recent_step, file_version='brain.Event:2'))
+#                 self.file_writer.add_event(
+#                     Event(step=most_recent_step, session_log=SessionLog(status=SessionLog.START)))
+#                 self.purge_step = None
+#         return self.file_writer

dtlpy/ml/train_utils.py

@@ -1,60 +1,60 @@
-import datetime
-import logging
-import json
-import time
-
-import numpy as np
-
-from .. import entities, exceptions
-
-logger = logging.getLogger(name='dtlpy')
-
-
-def prepare_dataset(dataset: entities.Dataset,
-                    subsets: dict,
-                    filters: entities.Filters = None,
-                    async_clone: bool = False):
-    """
-    clones the given dataset and locks it to be readonly
-
-    :param dataset:  `dl.Dataset` to clone
-    :param dtlpy.entities.filters.Filters filters: `dl.Filters` to use when cloning - which items to clone
-    :param subsets: dictionary to set subsets from the data.
-                        key is subset name, value is the filter to set, eg.
-                                    {dl.DatasetSubsetType.TEST: dl.Filters(field='dir', values='/test')}
-    :param async_clone: `bool` if True (default) - waits until the sync is complete
-                                     False - return before the cloned dataset is fully populated
-    :return: `dl.Dataset` which is readonly
-    """
-
-    project = dataset.project
-    now = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='minutes', sep='T')  # This serves as an id
-    today = datetime.datetime.now(datetime.timezone.utc).strftime('%F')
-
-    # CLONE
-    clone_name = 'cloned-{ds_name}-{date_str}'.format(ds_name=dataset.name, date_str=today)
-    try:
-        cloned_dataset = project.datasets.get(clone_name)
-        logger.warning("Cloned dataset already exist. Using it...")
-        return cloned_dataset
-    except exceptions.NotFound:
-        pass
-
-    tic = time.time()
-    cloned_dataset = dataset.clone(clone_name=clone_name,
-                                   filters=filters)
-    toc = time.time()
-
-    orig_recipe = dataset.recipes.list()[0]
-    cloned_recipe = orig_recipe.clone(shallow=True)
-    cloned_dataset.metadata['system']['recipes'] = [cloned_recipe.id]
-
-    logger.info("clone complete: {!r} in {:1.1f}[s]".format(cloned_dataset.name, toc - tic))
-
-    assert cloned_dataset.name is not None, ('unable to get new ds {}'.format(clone_name))
-    cloned_dataset.metadata['system']['clone_info'] = {'date': now,
-                                                       'originalDatasetId': dataset.id}
-    if filters is not None:
-        cloned_dataset.metadata['system']['clone_info'].update({'filters': json.dumps(filters.prepare())})
-    cloned_dataset.update(system_metadata=True)
-    return cloned_dataset
+import datetime
+import logging
+import json
+import time
+
+import numpy as np
+
+from .. import entities, exceptions
+
+logger = logging.getLogger(name='dtlpy')
+
+
+def prepare_dataset(dataset: entities.Dataset,
+                    subsets: dict,
+                    filters: entities.Filters = None,
+                    async_clone: bool = False):
+    """
+    clones the given dataset and locks it to be readonly
+
+    :param dataset:  `dl.Dataset` to clone
+    :param dtlpy.entities.filters.Filters filters: `dl.Filters` to use when cloning - which items to clone
+    :param subsets: dictionary to set subsets from the data.
+                        key is subset name, value is the filter to set, eg.
+                                    {dl.DatasetSubsetType.TEST: dl.Filters(field='dir', values='/test')}
+    :param async_clone: `bool` if True (default) - waits until the sync is complete
+                                     False - return before the cloned dataset is fully populated
+    :return: `dl.Dataset` which is readonly
+    """
+
+    project = dataset.project
+    now = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='minutes', sep='T')  # This serves as an id
+    today = datetime.datetime.now(datetime.timezone.utc).strftime('%F')
+
+    # CLONE
+    clone_name = 'cloned-{ds_name}-{date_str}'.format(ds_name=dataset.name, date_str=today)
+    try:
+        cloned_dataset = project.datasets.get(clone_name)
+        logger.warning("Cloned dataset already exist. Using it...")
+        return cloned_dataset
+    except exceptions.NotFound:
+        pass
+
+    tic = time.time()
+    cloned_dataset = dataset.clone(clone_name=clone_name,
+                                   filters=filters)
+    toc = time.time()
+
+    orig_recipe = dataset.recipes.list()[0]
+    cloned_recipe = orig_recipe.clone(shallow=True)
+    cloned_dataset.metadata['system']['recipes'] = [cloned_recipe.id]
+
+    logger.info("clone complete: {!r} in {:1.1f}[s]".format(cloned_dataset.name, toc - tic))
+
+    assert cloned_dataset.name is not None, ('unable to get new ds {}'.format(clone_name))
+    cloned_dataset.metadata['system']['clone_info'] = {'date': now,
+                                                       'originalDatasetId': dataset.id}
+    if filters is not None:
+        cloned_dataset.metadata['system']['clone_info'].update({'filters': json.dumps(filters.prepare())})
+    cloned_dataset.update(system_metadata=True)
+    return cloned_dataset