mlperf-logging 3.0.0__py3-none-any.whl

mlperf_logging/benchmark_meta.py

@@ -0,0 +1,184 @@
+# benchmark dictionary
+_ALL_RESULT_FILE_COUNTS = {
+    'training': {
+        'bert': 10,
+        'dlrm': 5,
+        'dlrm_dcnv2': 10,
+        'gnmt': 10,
+        'gpt3': 3,
+        'maskrcnn': 5,
+        'minigo': 10,
+        'resnet': 5,
+        'ssd': 5,
+        'stable_diffusion': 10,
+        'transformer': 10,
+        'ncf': 10,
+        'rnnt': 10,
+        'unet3d': 40,
+        'gnn' : 10,  
+        'llama2_70b_lora': 10,
+    },
+    
+    'hpc' : {
+        'deepcam': 5,
+        'cosmoflow': 10,
+        'oc20': 5,
+        'openfold': 10,
+    }
+}
+
+
+_ALL_ALLOWED_BENCHMARKS = {
+    'training': {
+        '0.6': [
+            'resnet',
+            'ssd',
+            'maskrcnn',
+            'gnmt',
+            'transformer',
+            'ncf',
+            'minigo',
+        ],
+        
+    '0.7': [
+        'bert',
+        'dlrm',
+        'gnmt',
+        'maskrcnn',
+        'minigo',
+        'resnet',
+        'ssd',
+        'transformer'
+    ],
+    '1.0': [
+        'bert',
+        'dlrm',
+        'maskrcnn',
+        'minigo',
+        'resnet',
+        'ssd',
+        'rnnt',
+        'unet3d',
+    ],
+    '1.1': [
+        'bert',
+        'dlrm',
+        'maskrcnn',
+        'minigo',
+        'resnet',
+        'ssd',
+        'rnnt',
+        'unet3d',
+    ],
+    '2.0': [
+        'bert',
+        'dlrm',
+        'maskrcnn',
+        'minigo',
+        'resnet',
+        'ssd',
+        'rnnt',
+        'unet3d',
+    ],
+    '2.1': [
+        'bert',
+        'dlrm',
+        'maskrcnn',
+        'minigo',
+        'resnet',
+        'ssd',
+        'rnnt',
+        'unet3d',
+    ],
+    '3.0': [
+        'bert',
+        'dlrm_dcnv2',
+        'gpt3',
+        'maskrcnn',
+        'resnet',
+        'ssd',
+        'rnnt',
+        'unet3d',
+    ],
+    '3.1': [
+        'bert',
+        'dlrm_dcnv2',
+        'gpt3',
+        'maskrcnn',
+        'resnet',
+        'ssd',
+        'rnnt',
+        'unet3d',
+        'stable_diffusion'
+    ],
+    '4.0': [
+        'bert',
+        'dlrm_dcnv2',
+        'gpt3',
+        'resnet',
+        'ssd',
+        'unet3d',
+        'stable_diffusion',
+        'llama2_70b_lora',
+        'stable_diffusion',
+        'gnn'
+    ],
+    '4.1': [
+        'bert',
+        'dlrm_dcnv2',
+        'gpt3',        
+        'ssd',        
+        'stable_diffusion',
+        'llama2_70b_lora',
+        'gnn'
+    ]    
+    },
+    
+    'hpc': {
+        '0.7': [
+            'cosmoflow',
+            'deepcam',
+        ],
+        
+        '1.0': [
+            'cosmoflow',
+            'deepcam',
+            'oc20',
+        ],
+        '2.0': [
+            'cosmoflow',
+            'deepcam',
+            'oc20',
+        ],
+        '3.0': [
+            'cosmoflow',
+            'deepcam',
+            'oc20',
+            'openfold',
+        ],
+    }
+}
+
+
+def get_allowed_benchmarks(usage, ruleset):
+    # check usage
+    if usage not in _ALL_ALLOWED_BENCHMARKS:
+        raise ValueError('usage {} not supported!'.format(usage))
+
+    # check ruleset
+    if ruleset not in _ALL_ALLOWED_BENCHMARKS[usage]:
+        # try short version:
+        ruleset_short = ".".join(ruleset.split(".")[:-1])
+        if ruleset_short not in _ALL_ALLOWED_BENCHMARKS[usage]:
+            raise ValueError('ruleset {} is not supported in {}'.format(ruleset, usage))
+        allowed_benchmarks = _ALL_ALLOWED_BENCHMARKS[usage][ruleset_short]
+    else:
+        allowed_benchmarks = _ALL_ALLOWED_BENCHMARKS[usage][ruleset]
+
+    return allowed_benchmarks
+
+
+def get_result_file_counts(usage):
+    if usage not in _ALL_RESULT_FILE_COUNTS:
+        raise ValueError('usage {} not supported!'.format(usage))
+    return _ALL_RESULT_FILE_COUNTS[usage]

mlperf_logging/compliance_checker/__main__.py

@@ -0,0 +1,32 @@
+import sys
+import logging
+
+from . import mlp_compliance
+
+parser = mlp_compliance.get_parser()
+args = parser.parse_args()
+
+logging.basicConfig(filename=args.log_output, level=logging.INFO)
+logging.getLogger().addHandler(logging.StreamHandler())
+formatter = logging.Formatter("%(levelname)s - %(message)s")
+logging.getLogger().handlers[0].setFormatter(formatter)
+logging.getLogger().handlers[1].setFormatter(formatter)
+
+config_file = args.config or f'{args.usage}_{args.ruleset}/common.yaml'
+
+checker = mlp_compliance.make_checker(
+    args.usage,
+    args.ruleset,
+    args.quiet,
+    args.werror,
+)
+
+valid, system_id, benchmark, result = mlp_compliance.main(args.filename, config_file, checker)
+
+if not valid:
+    logging.error('FAILED')
+    print('** Logging output also at', args.log_output)
+    sys.exit(1)
+else:
+    print('** Logging output also at', args.log_output)
+    logging.info('SUCCESS')

mlperf_logging/compliance_checker/hpc_1.0.0/closed_common.yaml

@@ -0,0 +1,24 @@
+
+- KEY:
+    NAME:  submission_benchmark
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['deepcam', 'cosmoflow', 'oc20'] "
+    POST:  " enqueue_config('hpc_1.0.0/closed_{}.yaml'.format(v['value'])) "
+
+- KEY:
+    NAME:  gradient_accumulation_steps
+    CHECK: " v['value'] > 0 "
+
+- KEY:
+    NAME:  gradient_accumulation_frequency
+    CHECK: " v['value'] > 0 "
+
+- KEY:
+    NAME:  number_of_nodes
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0"
+
+- KEY:
+    NAME:  accelerators_per_node
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0"

mlperf_logging/compliance_checker/hpc_1.0.0/closed_cosmoflow.yaml

@@ -0,0 +1,47 @@
+- KEY:
+    NAME:  global_batch_size
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0"
+
+- KEY:
+    NAME:  opt_name
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['sgd', 'SGD'] "
+
+- KEY:
+    NAME:  opt_base_learning_rate
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0."
+
+- KEY:
+    NAME:  opt_learning_rate_warmup_epochs
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0"
+
+- KEY:
+    NAME:  opt_learning_rate_warmup_factor
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0."
+
+- KEY:
+    NAME:  opt_learning_rate_decay_boundary_epochs
+    REQ:   EXACTLY_ONE
+
+- KEY:
+    NAME:  opt_learning_rate_decay_factor
+    REQ:   EXACTLY_ONE
+
+- KEY:
+    NAME:  dropout
+    CHECK: " v['value'] >= 0. and v['value'] < 1."
+
+- KEY:
+    NAME: opt_weight_decay
+    CHECK: " v['value'] >= 0."
+
+- KEY:
+    NAME:  eval_error
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+    ATLEAST_ONE_CHECK: "v['value'] <= 0.124 and v['value'] > 0."

mlperf_logging/compliance_checker/hpc_1.0.0/closed_deepcam.yaml

@@ -0,0 +1,80 @@
+# General Settings
+- KEY:
+    NAME:  gradient_accumulation_frequency
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0 "
+
+- KEY:
+    NAME:  seed
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0"
+
+- KEY:
+    NAME:  global_batch_size
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0"
+    
+- KEY:
+    NAME:  batchnorm_group_size
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0"
+
+
+# Optimizer Parameters
+- KEY:
+    NAME:  opt_name
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['Adam', 'AdamW', 'LAMB']"
+    POST:  " if (v['value'] == 'LAMB'): enqueue_config('hpc_1.0.0/closed_deepcam_lamb.yaml') "
+
+- KEY:
+    NAME:  opt_lr
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >0."
+
+- KEY:
+    NAME:  opt_betas
+    REQ:   EXACTLY_ONE
+    CHECK: " len(v['value']) == 2"
+
+- KEY:
+    NAME:  opt_eps
+    REQ:   EXACTLY_ONE
+    CHECK: " math.isclose(v['value'], 1e-6)"
+
+
+# Scheduler Parameters
+- KEY:
+    NAME:  scheduler_type
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['multistep', 'cosine_annealing']"
+    POST:  " enqueue_config('hpc_1.0.0/closed_deepcam_{}.yaml'.format(v['value'].lower())) "
+
+- KEY:
+    NAME:  scheduler_lr_warmup_steps
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0 "
+
+- KEY:
+    NAME:  scheduler_lr_warmup_factor
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 1. "
+
+# Dataset Properties
+- KEY:
+    NAME:  train_samples
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] == 121266"
+
+- KEY:
+    NAME:  eval_samples
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] == 15158"
+
+# Convergence Properties
+- KEY:
+    NAME:  eval_accuracy
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+    ATLEAST_ONE_CHECK: "v['value'] >= 0.82 and v['value'] <= 1."

mlperf_logging/compliance_checker/hpc_1.0.0/closed_deepcam_cosine_annealing.yaml

@@ -0,0 +1,10 @@
+# Scheduler Parameters
+- KEY:
+    NAME:  scheduler_t_max
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 1. "
+
+- KEY:
+    NAME:  scheduler_eta_min
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0. "

mlperf_logging/compliance_checker/hpc_1.0.0/closed_deepcam_lamb.yaml

@@ -0,0 +1,16 @@
+# Optimizer Parameters
+- KEY:
+    NAME:  opt_bias_correction
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] "
+
+- KEY:
+    NAME:  opt_grad_averaging
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] "
+
+- KEY:
+    NAME:  opt_max_grad_norm
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] == 1."
+

mlperf_logging/compliance_checker/hpc_1.0.0/closed_deepcam_multistep.yaml

@@ -0,0 +1,10 @@
+# Scheduler Parameters
+- KEY:
+    NAME:  scheduler_milestones
+    REQ:   EXACTLY_ONE
+    CHECK: " len(v['value']) >= 0 "
+
+- KEY:
+    NAME:  scheduler_decay_rate
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] <= 1. "

mlperf_logging/compliance_checker/hpc_1.0.0/closed_oc20.yaml

@@ -0,0 +1,39 @@
+- KEY:
+    NAME:  global_batch_size
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0"
+
+- KEY:
+    NAME:  opt_name
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] == 'AdamW'"
+
+- KEY:
+    NAME:  opt_base_learning_rate
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0."
+
+- KEY:
+    NAME:  opt_learning_rate_warmup_steps
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0"
+
+- KEY:
+    NAME:  opt_learning_rate_warmup_factor
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0."
+
+- KEY:
+    NAME:  opt_learning_rate_decay_boundary_steps
+    REQ:   EXACTLY_ONE
+
+- KEY:
+    NAME:  opt_learning_rate_decay_factor
+    REQ:   EXACTLY_ONE
+
+- KEY:
+    NAME:  eval_error
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+    ATLEAST_ONE_CHECK: "v['value'] <= 0.036 and v['value'] > 0."

mlperf_logging/compliance_checker/hpc_1.0.0/common.yaml

@@ -0,0 +1,156 @@
+# This file lists all the KEYs to be checked. Every line that matches mlperf logging regex (::MLL...) will be checked against these rules.
+# In the order of the appearance in the log, for each line will execute the code specified under CHECK for the KEY in that line.
+# The code will be launched using local state 'v' which is the content of value field in log line, and global state 's'.
+# Global state 's' exists to allow cross-line checks, like start/stop pairs etc. To initialize 's' use BEGIN record which CODE will
+# be executed before any checks.
+# In addition, occurrence of each key will be counted and at the end if a requirement regarding the number of occurrences is defined it will
+# be confirmed. This could be implemented using global state, but since this is a common thing to do it is natively supported.
+#
+# KEY record:
+# NAME
+# REQ   - optional - {EXACTLY_ONE, AT_LEAST_ONE}
+# PRE   - optional - code to be executed before CHECK
+# CHECK - optional - expression to be evaluated to verify correctness
+# POST  - optional - code to be executed after CHECK
+
+- BEGIN:
+    CODE: >
+        s.update({
+            'init_started': False,
+            'init_stopped' : False,
+            'run_started' : False,
+            'run_stopped' : False,
+            'in_epoch' : False,
+            'last_epoch' : 0,
+            'in_block' : False,
+            'block_first_epoch' : -1,
+            'first_init_start': 9e99,
+        })
+
+- KEY:
+    NAME:  submission_org
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] != '' "
+
+- KEY:
+    NAME:  submission_platform
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] != '' "
+
+- KEY:
+    NAME:  submission_division
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['closed', 'open'] "
+    POST:  " enqueue_config('hpc_1.0.0/{}_common.yaml'.format(v['value'])) "
+
+- KEY:
+    NAME:  submission_status
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['cloud', 'onprem', 'preview', 'research'] "
+
+# at least one record should be found, but any found records must pass the test
+- KEY:
+    NAME:  cache_clear
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "'value' in v"
+
+# frequency not checked
+- KEY:
+    NAME:  init_start
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "not s['init_stopped']"
+        - "not s['run_started']"
+    POST:  " s['init_started'] = True; s['first_init_start']=min(s['first_init_start'], ll.timestamp) "
+
+# confirm less than 20min since the very first init_start
+- KEY:
+    NAME:  init_stop
+    REQ:   EXACTLY_ONE
+    CHECK:
+        - "s['init_started']"
+        - "not s['run_started']"
+        - "ll.timestamp - s['first_init_start'] < (30*60*1e3)"
+    POST:  " s['init_stopped'] = True"
+
+# HPC requires data staging to be included in run timing
+- KEY:
+    NAME: staging_start
+    CHECK:
+        - "s['run_started']"
+    POST: " s['staging_started'] = True "
+
+- KEY:
+    NAME: staging_stop
+    CHECK:
+        - "s['staging_started']"
+
+# run start and run stop
+- KEY:
+    NAME:  run_start
+    REQ:   EXACTLY_ONE
+    CHECK: " ( s['init_stopped'] == True )"
+    POST:  " s['run_started'] = True "
+
+# status can also be aborted, but not allowing it here for now
+# if eval is inside epoch and we decide to terminate, we can lack epoch_stop, it is ok
+- KEY:
+    NAME:  run_stop
+    REQ:   EXACTLY_ONE
+    CHECK:
+        - "s['run_started']"
+        - "'status' in v['metadata']"
+    POST:  " s['run_stopped'] = True "
+
+# FIXME: check epoch_count value match
+- KEY:
+    NAME:  block_start
+    REQ:   AT_LEAST_ONE_OR(epoch_start)
+    CHECK:
+        - "s['run_started']"
+        - "'epoch_count' in v['metadata']"
+        - "'first_epoch_num' in v['metadata']"
+        - "v['metadata']['epoch_count'] > 0"
+
+- KEY:
+    NAME:  block_stop
+    REQ:   AT_LEAST_ONE_OR(epoch_stop)
+    CHECK:
+        - "'first_epoch_num' in v['metadata']"
+
+- KEY:
+    NAME:  epoch_start
+    REQ:   AT_LEAST_ONE_OR(block_start)
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+
+- KEY:
+    NAME:  epoch_stop
+    REQ:   AT_LEAST_ONE_OR(block_stop)
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+
+# making sure previous eval did print it's accuracy result
+- KEY:
+    NAME:  eval_start
+    REQ:   AT_LEAST_ONE_OR(block_start)
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+
+- KEY:
+    NAME:  eval_stop
+    REQ:   AT_LEAST_ONE_OR(block_stop)
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+
+- KEY:
+    NAME:  train_samples
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] != '' "
+
+- KEY:
+    NAME:  eval_samples
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] != '' "
+

mlperf_logging/compliance_checker/hpc_1.0.0/open_common.yaml

@@ -0,0 +1,5 @@
+- KEY:
+    NAME:  submission_benchmark
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['deepcam', 'cosmoflow', 'oc20'] "
+    POST:  " enqueue_config('hpc_1.0.0/open_{}.yaml'.format(v['value'])) "

mlperf_logging/compliance_checker/hpc_1.0.0/open_cosmoflow.yaml

@@ -0,0 +1,6 @@
+- KEY:
+    NAME:  eval_error
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+    ATLEAST_ONE_CHECK: "v['value'] <= 0.124 and v['value'] > 0."

mlperf_logging/compliance_checker/hpc_1.0.0/open_deepcam.yaml

@@ -0,0 +1,6 @@
+- KEY:
+    NAME:  eval_accuracy
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+    ATLEAST_ONE_CHECK: "v['value'] >= 0.82 and v['value'] <= 1."

mlperf_logging/compliance_checker/hpc_1.0.0/open_oc20.yaml

@@ -0,0 +1,6 @@
+- KEY:
+    NAME:  eval_error
+    REQ:   AT_LEAST_ONE
+    CHECK:
+        - "'epoch_num' in v['metadata']"
+    ATLEAST_ONE_CHECK: "v['value'] <= 0.036 and v['value'] > 0."

mlperf_logging/compliance_checker/hpc_2.0.0/closed_common.yaml

@@ -0,0 +1,24 @@
+
+- KEY:
+    NAME:  submission_benchmark
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] in ['deepcam', 'cosmoflow', 'oc20'] "
+    POST:  " enqueue_config('hpc_2.0.0/closed_{}.yaml'.format(v['value'])) "
+
+- KEY:
+    NAME:  gradient_accumulation_steps
+    CHECK: " v['value'] > 0 "
+
+- KEY:
+    NAME:  gradient_accumulation_frequency
+    CHECK: " v['value'] > 0 "
+
+- KEY:
+    NAME:  number_of_nodes
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] > 0"
+
+- KEY:
+    NAME:  accelerators_per_node
+    REQ:   EXACTLY_ONE
+    CHECK: " v['value'] >= 0"