evalscope 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/logger.py

@@ -0,0 +1,188 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import datetime
+import logging
+import time
+import torch
+import torch.distributed as dist
+from collections import defaultdict, deque
+
+from . import dist_utils
+
+
+class SmoothedValue(object):
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = '{median:.4f} ({global_avg:.4f})'
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, n=1):
+        self.deque.append(value)
+        self.count += n
+        self.total += value * n
+
+    def synchronize_between_processes(self):
+        """
+        Warning: does not synchronize the deque!
+        """
+        if not dist_utils.is_dist_avail_and_initialized():
+            return
+        t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda')
+        dist.barrier()
+        dist.all_reduce(t)
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median,
+            avg=self.avg,
+            global_avg=self.global_avg,
+            max=self.max,
+            value=self.value,
+        )
+
+
+class MetricLogger(object):
+
+    def __init__(self, delimiter='\t'):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr))
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append('{}: {}'.format(name, str(meter)))
+        return self.delimiter.join(loss_str)
+
+    def global_avg(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append('{}: {:.4f}'.format(name, meter.global_avg))
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ''
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt='{avg:.4f}')
+        data_time = SmoothedValue(fmt='{avg:.4f}')
+        space_fmt = ':' + str(len(str(len(iterable)))) + 'd'
+        log_msg = [
+            header,
+            '[{0' + space_fmt + '}/{1}]',
+            'eta: {eta}',
+            '{meters}',
+            'time: {time}',
+            'data: {data}',
+        ]
+        if torch.cuda.is_available():
+            log_msg.append('max mem: {memory:.0f}')
+        log_msg = self.delimiter.join(log_msg)
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0 or i == len(iterable) - 1:
+                eta_seconds = iter_time.global_avg * (len(iterable) - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(
+                        log_msg.format(
+                            i,
+                            len(iterable),
+                            eta=eta_string,
+                            meters=str(self),
+                            time=str(iter_time),
+                            data=str(data_time),
+                            memory=torch.cuda.max_memory_allocated() / MB,
+                        ))
+                else:
+                    print(
+                        log_msg.format(
+                            i,
+                            len(iterable),
+                            eta=eta_string,
+                            meters=str(self),
+                            time=str(iter_time),
+                            data=str(data_time),
+                        ))
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print('{} Total time: {} ({:.4f} s / it)'.format(header, total_time_str, total_time / len(iterable)))
+
+
+class AttrDict(dict):
+
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def setup_logger():
+    logging.basicConfig(
+        level=logging.INFO if dist_utils.is_main_process() else logging.WARN,
+        format='%(asctime)s [%(levelname)s] %(message)s',
+        handlers=[logging.StreamHandler()],
+    )

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/optims.py

@@ -0,0 +1,106 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import math
+
+from . import registry
+
+
+@registry.register_lr_scheduler('linear_warmup_step_lr')
+class LinearWarmupStepLRScheduler:
+
+    def __init__(self,
+                 optimizer,
+                 max_epoch,
+                 min_lr,
+                 init_lr,
+                 decay_rate=1,
+                 warmup_start_lr=-1,
+                 warmup_steps=0,
+                 **kwargs):
+        self.optimizer = optimizer
+
+        self.max_epoch = max_epoch
+        self.min_lr = min_lr
+
+        self.decay_rate = decay_rate
+
+        self.init_lr = init_lr
+        self.warmup_steps = warmup_steps
+        self.warmup_start_lr = warmup_start_lr if warmup_start_lr >= 0 else init_lr
+
+    def step(self, cur_epoch, cur_step):
+        if cur_epoch == 0:
+            warmup_lr_schedule(
+                step=cur_step,
+                optimizer=self.optimizer,
+                max_step=self.warmup_steps,
+                init_lr=self.warmup_start_lr,
+                max_lr=self.init_lr,
+            )
+        else:
+            step_lr_schedule(
+                epoch=cur_epoch,
+                optimizer=self.optimizer,
+                init_lr=self.init_lr,
+                min_lr=self.min_lr,
+                decay_rate=self.decay_rate,
+            )
+
+
+@registry.register_lr_scheduler('linear_warmup_cosine_lr')
+class LinearWarmupCosineLRScheduler:
+
+    def __init__(self, optimizer, max_epoch, min_lr, init_lr, warmup_steps=0, warmup_start_lr=-1, **kwargs):
+        self.optimizer = optimizer
+
+        self.max_epoch = max_epoch
+        self.min_lr = min_lr
+
+        self.init_lr = init_lr
+        self.warmup_steps = warmup_steps
+        self.warmup_start_lr = warmup_start_lr if warmup_start_lr >= 0 else init_lr
+
+    def step(self, cur_epoch, cur_step):
+        # assuming the warmup iters less than one epoch
+        if cur_epoch == 0:
+            warmup_lr_schedule(
+                step=cur_step,
+                optimizer=self.optimizer,
+                max_step=self.warmup_steps,
+                init_lr=self.warmup_start_lr,
+                max_lr=self.init_lr,
+            )
+        else:
+            cosine_lr_schedule(
+                epoch=cur_epoch,
+                optimizer=self.optimizer,
+                max_epoch=self.max_epoch,
+                init_lr=self.init_lr,
+                min_lr=self.min_lr,
+            )
+
+
+def cosine_lr_schedule(optimizer, epoch, max_epoch, init_lr, min_lr):
+    """Decay the learning rate"""
+    lr = (init_lr - min_lr) * 0.5 * (1.0 + math.cos(math.pi * epoch / max_epoch)) + min_lr
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = lr
+
+
+def warmup_lr_schedule(optimizer, step, max_step, init_lr, max_lr):
+    """Warmup the learning rate"""
+    lr = min(max_lr, init_lr + (max_lr - init_lr) * step / max(max_step, 1))
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = lr
+
+
+def step_lr_schedule(optimizer, epoch, init_lr, min_lr, decay_rate):
+    """Decay the learning rate"""
+    lr = max(min_lr, init_lr * (decay_rate**epoch))
+    for param_group in optimizer.param_groups:
+        param_group['lr'] = lr

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/common/registry.py

@@ -0,0 +1,307 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+
+class Registry:
+    mapping = {
+        'builder_name_mapping': {},
+        'task_name_mapping': {},
+        'processor_name_mapping': {},
+        'model_name_mapping': {},
+        'lr_scheduler_name_mapping': {},
+        'runner_name_mapping': {},
+        'state': {},
+        'paths': {},
+    }
+
+    # @classmethod
+    # def register_builder(cls, name):
+    #     r"""Register a dataset builder to registry with key 'name'
+
+    #     Args:
+    #         name: Key with which the builder will be registered.
+
+    #     Usage:
+
+    #         from lavis.common.registry import registry
+    #         from lavis.datasets.base_dataset_builder import BaseDatasetBuilder
+    #     """
+
+    #     def wrap(builder_cls):
+    #         from lavis.datasets.builders.base_dataset_builder import BaseDatasetBuilder
+
+    #         assert issubclass(
+    #             builder_cls, BaseDatasetBuilder
+    #         ), "All builders must inherit BaseDatasetBuilder class, found {}".format(
+    #             builder_cls
+    #         )
+    #         if name in cls.mapping["builder_name_mapping"]:
+    #             raise KeyError(
+    #                 "Name '{}' already registered for {}.".format(
+    #                     name, cls.mapping["builder_name_mapping"][name]
+    #                 )
+    #             )
+    #         cls.mapping["builder_name_mapping"][name] = builder_cls
+    #         return builder_cls
+
+    #     return wrap
+
+    # @classmethod
+    # def register_task(cls, name):
+    #     r"""Register a task to registry with key 'name'
+
+    #     Args:
+    #         name: Key with which the task will be registered.
+
+    #     Usage:
+
+    #         from lavis.common.registry import registry
+    #     """
+
+    #     def wrap(task_cls):
+    #         from lavis.tasks.base_task import BaseTask
+
+    #         assert issubclass(
+    #             task_cls, BaseTask
+    #         ), "All tasks must inherit BaseTask class"
+    #         if name in cls.mapping["task_name_mapping"]:
+    #             raise KeyError(
+    #                 "Name '{}' already registered for {}.".format(
+    #                     name, cls.mapping["task_name_mapping"][name]
+    #                 )
+    #             )
+    #         cls.mapping["task_name_mapping"][name] = task_cls
+    #         return task_cls
+
+    #     return wrap
+
+    @classmethod
+    def register_model(cls, name):
+        r"""Register a task to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from lavis.common.registry import registry
+        """
+
+        def wrap(model_cls):
+            from ..models import BaseModel
+
+            assert issubclass(model_cls, BaseModel), 'All models must inherit BaseModel class'
+            if name in cls.mapping['model_name_mapping']:
+                raise KeyError("Name '{}' already registered for {}.".format(name,
+                                                                             cls.mapping['model_name_mapping'][name]))
+            cls.mapping['model_name_mapping'][name] = model_cls
+            return model_cls
+
+        return wrap
+
+    @classmethod
+    def register_processor(cls, name):
+        r"""Register a processor to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from lavis.common.registry import registry
+        """
+
+        def wrap(processor_cls):
+            from ..processors import BaseProcessor
+
+            assert issubclass(processor_cls, BaseProcessor), 'All processors must inherit BaseProcessor class'
+            if name in cls.mapping['processor_name_mapping']:
+                raise KeyError("Name '{}' already registered for {}.".format(
+                    name, cls.mapping['processor_name_mapping'][name]))
+            cls.mapping['processor_name_mapping'][name] = processor_cls
+            return processor_cls
+
+        return wrap
+
+    @classmethod
+    def register_lr_scheduler(cls, name):
+        r"""Register a model to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from lavis.common.registry import registry
+        """
+
+        def wrap(lr_sched_cls):
+            if name in cls.mapping['lr_scheduler_name_mapping']:
+                raise KeyError("Name '{}' already registered for {}.".format(
+                    name, cls.mapping['lr_scheduler_name_mapping'][name]))
+            cls.mapping['lr_scheduler_name_mapping'][name] = lr_sched_cls
+            return lr_sched_cls
+
+        return wrap
+
+    @classmethod
+    def register_runner(cls, name):
+        r"""Register a model to registry with key 'name'
+
+        Args:
+            name: Key with which the task will be registered.
+
+        Usage:
+
+            from lavis.common.registry import registry
+        """
+
+        def wrap(runner_cls):
+            if name in cls.mapping['runner_name_mapping']:
+                raise KeyError("Name '{}' already registered for {}.".format(name,
+                                                                             cls.mapping['runner_name_mapping'][name]))
+            cls.mapping['runner_name_mapping'][name] = runner_cls
+            return runner_cls
+
+        return wrap
+
+    @classmethod
+    def register_path(cls, name, path):
+        r"""Register a path to registry with key 'name'
+
+        Args:
+            name: Key with which the path will be registered.
+
+        Usage:
+
+            from lavis.common.registry import registry
+        """
+        assert isinstance(path, str), 'All path must be str.'
+        if name in cls.mapping['paths']:
+            raise KeyError("Name '{}' already registered.".format(name))
+        cls.mapping['paths'][name] = path
+
+    @classmethod
+    def register(cls, name, obj):
+        r"""Register an item to registry with key 'name'
+
+        Args:
+            name: Key with which the item will be registered.
+
+        Usage::
+
+            from lavis.common.registry import registry
+
+            registry.register("config", {})
+        """
+        path = name.split('.')
+        current = cls.mapping['state']
+
+        for part in path[:-1]:
+            if part not in current:
+                current[part] = {}
+            current = current[part]
+
+        current[path[-1]] = obj
+
+    # @classmethod
+    # def get_trainer_class(cls, name):
+    #     return cls.mapping["trainer_name_mapping"].get(name, None)
+
+    @classmethod
+    def get_builder_class(cls, name):
+        return cls.mapping['builder_name_mapping'].get(name, None)
+
+    @classmethod
+    def get_model_class(cls, name):
+        return cls.mapping['model_name_mapping'].get(name, None)
+
+    @classmethod
+    def get_task_class(cls, name):
+        return cls.mapping['task_name_mapping'].get(name, None)
+
+    @classmethod
+    def get_processor_class(cls, name):
+        return cls.mapping['processor_name_mapping'].get(name, None)
+
+    @classmethod
+    def get_lr_scheduler_class(cls, name):
+        return cls.mapping['lr_scheduler_name_mapping'].get(name, None)
+
+    @classmethod
+    def get_runner_class(cls, name):
+        return cls.mapping['runner_name_mapping'].get(name, None)
+
+    @classmethod
+    def list_runners(cls):
+        return sorted(cls.mapping['runner_name_mapping'].keys())
+
+    @classmethod
+    def list_models(cls):
+        return sorted(cls.mapping['model_name_mapping'].keys())
+
+    @classmethod
+    def list_tasks(cls):
+        return sorted(cls.mapping['task_name_mapping'].keys())
+
+    @classmethod
+    def list_processors(cls):
+        return sorted(cls.mapping['processor_name_mapping'].keys())
+
+    @classmethod
+    def list_lr_schedulers(cls):
+        return sorted(cls.mapping['lr_scheduler_name_mapping'].keys())
+
+    @classmethod
+    def list_datasets(cls):
+        return sorted(cls.mapping['builder_name_mapping'].keys())
+
+    @classmethod
+    def get_path(cls, name):
+        return cls.mapping['paths'].get(name, None)
+
+    @classmethod
+    def get(cls, name, default=None, no_warning=False):
+        r"""Get an item from registry with key 'name'
+
+        Args:
+            name (string): Key whose value needs to be retrieved.
+            default: If passed and key is not in registry, default value will
+                     be returned with a warning. Default: None
+            no_warning (bool): If passed as True, warning when key doesn't exist
+                               will not be generated. Useful for MMF's
+                               internal operations. Default: False
+        """
+        original_name = name
+        name = name.split('.')
+        value = cls.mapping['state']
+        for subname in name:
+            value = value.get(subname, default)
+            if value is default:
+                break
+
+        if ('writer' in cls.mapping['state'] and value == default and no_warning is False):
+            cls.mapping['state']['writer'].warning('Key {} is not present in registry, returning default value '
+                                                   'of {}'.format(original_name, default))
+        return value
+
+    @classmethod
+    def unregister(cls, name):
+        r"""Remove an item from registry with key 'name'
+
+        Args:
+            name: Key which needs to be removed.
+        Usage::
+
+            from mmf.common.registry import registry
+
+            config = registry.unregister("config")
+        """
+        return cls.mapping['state'].pop(name, None)
+
+
+registry = Registry()