SinaTools 0.1.40__py2.py3-none-any.whl → 1.0.1__py2.py3-none-any.whl

sinatools/arabert/araelectra/finetune/feature_spec.py

@@ -1,56 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Google Research Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Defines the inputs used when fine-tuning a model."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-import tensorflow as tf
-
-import configure_finetuning
-
-
-def get_shared_feature_specs(config: configure_finetuning.FinetuningConfig):
-  """Non-task-specific model inputs."""
-  return [
-      FeatureSpec("input_ids", [config.max_seq_length]),
-      FeatureSpec("input_mask", [config.max_seq_length]),
-      FeatureSpec("segment_ids", [config.max_seq_length]),
-      FeatureSpec("task_id", []),
-  ]
-
-
-class FeatureSpec(object):
-  """Defines a feature passed as input to the model."""
-
-  def __init__(self, name, shape, default_value_fn=None, is_int_feature=True):
-    self.name = name
-    self.shape = shape
-    self.default_value_fn = default_value_fn
-    self.is_int_feature = is_int_feature
-
-  def get_parsing_spec(self):
-    return tf.io.FixedLenFeature(
-        self.shape, tf.int64 if self.is_int_feature else tf.float32)
-
-  def get_default_values(self):
-    if self.default_value_fn:
-      return self.default_value_fn(self.shape)
-    else:
-      return np.zeros(
-          self.shape, np.int64 if self.is_int_feature else np.float32)

sinatools/arabert/araelectra/finetune/preprocessing.py

@@ -1,173 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Google Research Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Code for serializing raw fine-tuning data into tfrecords"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import os
-import random
-import numpy as np
-import tensorflow as tf
-
-import configure_finetuning
-from finetune import feature_spec
-from util import utils
-
-
-class Preprocessor(object):
-  """Class for loading, preprocessing, and serializing fine-tuning datasets."""
-
-  def __init__(self, config: configure_finetuning.FinetuningConfig, tasks):
-    self._config = config
-    self._tasks = tasks
-    self._name_to_task = {task.name: task for task in tasks}
-
-    self._feature_specs = feature_spec.get_shared_feature_specs(config)
-    for task in tasks:
-      self._feature_specs += task.get_feature_specs()
-    self._name_to_feature_config = {
-        spec.name: spec.get_parsing_spec()
-        for spec in self._feature_specs
-    }
-    assert len(self._name_to_feature_config) == len(self._feature_specs)
-
-  def prepare_train(self):
-    return self._serialize_dataset(self._tasks, True, "train")
-
-  def prepare_predict(self, tasks, split):
-    return self._serialize_dataset(tasks, False, split)
-
-  def _serialize_dataset(self, tasks, is_training, split):
-    """Write out the dataset as tfrecords."""
-    dataset_name = "_".join(sorted([task.name for task in tasks]))
-    dataset_name += "_" + split
-    dataset_prefix = os.path.join(
-        self._config.preprocessed_data_dir, dataset_name)
-    tfrecords_path = dataset_prefix + ".tfrecord"
-    metadata_path = dataset_prefix + ".metadata"
-    batch_size = (self._config.train_batch_size if is_training else
-                  self._config.eval_batch_size)
-
-    utils.log("Loading dataset", dataset_name)
-    n_examples = None
-    if (self._config.use_tfrecords_if_existing and
-        tf.io.gfile.exists(metadata_path)):
-      n_examples = utils.load_json(metadata_path)["n_examples"]
-
-    if n_examples is None:
-      utils.log("Existing tfrecords not found so creating")
-      examples = []
-      for task in tasks:
-        task_examples = task.get_examples(split)
-        examples += task_examples
-      if is_training:
-        random.shuffle(examples)
-      utils.mkdir(tfrecords_path.rsplit("/", 1)[0])
-      n_examples = self.serialize_examples(
-          examples, is_training, tfrecords_path, batch_size)
-      utils.write_json({"n_examples": n_examples}, metadata_path)
-
-    input_fn = self._input_fn_builder(tfrecords_path, is_training)
-    if is_training:
-      steps = int(n_examples // batch_size * self._config.num_train_epochs)
-    else:
-      steps = n_examples // batch_size
-
-    return input_fn, steps
-
-  def serialize_examples(self, examples, is_training, output_file, batch_size):
-    """Convert a set of `InputExample`s to a TFRecord file."""
-    n_examples = 0
-    with tf.io.TFRecordWriter(output_file) as writer:
-      for (ex_index, example) in enumerate(examples):
-        if ex_index % 2000 == 0:
-          utils.log("Writing example {:} of {:}".format(
-              ex_index, len(examples)))
-        for tf_example in self._example_to_tf_example(
-            example, is_training,
-            log=self._config.log_examples and ex_index < 1):
-          writer.write(tf_example.SerializeToString())
-          n_examples += 1
-      # add padding so the dataset is a multiple of batch_size
-      while n_examples % batch_size != 0:
-        writer.write(self._make_tf_example(task_id=len(self._config.task_names))
-                     .SerializeToString())
-        n_examples += 1
-    return n_examples
-
-  def _example_to_tf_example(self, example, is_training, log=False):
-    examples = self._name_to_task[example.task_name].featurize(
-        example, is_training, log)
-    if not isinstance(examples, list):
-      examples = [examples]
-    for example in examples:
-      yield self._make_tf_example(**example)
-
-  def _make_tf_example(self, **kwargs):
-    """Make a tf.train.Example from the provided features."""
-    for k in kwargs:
-      if k not in self._name_to_feature_config:
-        raise ValueError("Unknown feature", k)
-    features = collections.OrderedDict()
-    for spec in self._feature_specs:
-      if spec.name in kwargs:
-        values = kwargs[spec.name]
-      else:
-        values = spec.get_default_values()
-      if (isinstance(values, int) or isinstance(values, bool) or
-          isinstance(values, float) or isinstance(values, np.float32) or
-          (isinstance(values, np.ndarray) and values.size == 1)):
-        values = [values]
-      if spec.is_int_feature:
-        feature = tf.train.Feature(int64_list=tf.train.Int64List(
-            value=list(values)))
-      else:
-        feature = tf.train.Feature(float_list=tf.train.FloatList(
-            value=list(values)))
-      features[spec.name] = feature
-    return tf.train.Example(features=tf.train.Features(feature=features))
-
-  def _input_fn_builder(self, input_file, is_training):
-    """Creates an `input_fn` closure to be passed to TPUEstimator."""
-
-    def input_fn(params):
-      """The actual input function."""
-      d = tf.data.TFRecordDataset(input_file)
-      if is_training:
-        d = d.repeat()
-        d = d.shuffle(buffer_size=100)
-      return d.apply(
-          tf.data.experimental.map_and_batch(
-              self._decode_tfrecord,
-              batch_size=params["batch_size"],
-              drop_remainder=True))
-
-    return input_fn
-
-  def _decode_tfrecord(self, record):
-    """Decodes a record to a TensorFlow example."""
-    example = tf.io.parse_single_example(record, self._name_to_feature_config)
-    # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
-    # So cast all int64 to int32.
-    for name, tensor in example.items():
-      if tensor.dtype == tf.int64:
-        example[name] = tf.cast(tensor, tf.int32)
-      else:
-        example[name] = tensor
-    return example

sinatools/arabert/araelectra/finetune/scorer.py

@@ -1,54 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Google Research Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Base class for evaluation metrics."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import abc
-
-
-class Scorer(object):
-  """Abstract base class for computing evaluation metrics."""
-
-  __metaclass__ = abc.ABCMeta
-
-  def __init__(self):
-    self._updated = False
-    self._cached_results = {}
-
-  @abc.abstractmethod
-  def update(self, results):
-    self._updated = True
-
-  @abc.abstractmethod
-  def get_loss(self):
-    pass
-
-  @abc.abstractmethod
-  def _get_results(self):
-    return []
-
-  def get_results(self, prefix=""):
-    results = self._get_results() if self._updated else self._cached_results
-    self._cached_results = results
-    self._updated = False
-    return [(prefix + k, v) for k, v in results]
-
-  def results_str(self):
-    return " - ".join(["{:}: {:.2f}".format(k, v)
-                       for k, v in self.get_results()])

sinatools/arabert/araelectra/finetune/task.py

@@ -1,74 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Google Research Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Defines a supervised NLP task."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import abc
-from typing import List, Tuple
-
-import configure_finetuning
-from finetune import feature_spec
-from finetune import scorer
-from model import modeling
-
-
-class Example(object):
-  __metaclass__ = abc.ABCMeta
-
-  def __init__(self, task_name):
-    self.task_name = task_name
-
-
-class Task(object):
-  """Override this class to add a new fine-tuning task."""
-
-  __metaclass__ = abc.ABCMeta
-
-  def __init__(self, config: configure_finetuning.FinetuningConfig, name):
-    self.config = config
-    self.name = name
-
-  def get_test_splits(self):
-    return ["test"]
-
-  @abc.abstractmethod
-  def get_examples(self, split):
-    pass
-
-  @abc.abstractmethod
-  def get_scorer(self) -> scorer.Scorer:
-    pass
-
-  @abc.abstractmethod
-  def get_feature_specs(self) -> List[feature_spec.FeatureSpec]:
-    pass
-
-  @abc.abstractmethod
-  def featurize(self, example: Example, is_training: bool,
-                log: bool=False):
-    pass
-
-  @abc.abstractmethod
-  def get_prediction_module(
-      self, bert_model: modeling.BertModel, features: dict, is_training: bool,
-      percent_done: float) -> Tuple:
-    pass
-
-  def __repr__(self):
-    return "Task(" + self.name + ")"

sinatools/arabert/araelectra/finetune/task_builder.py

@@ -1,70 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Google Research Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Returns task instances given the task name."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import configure_finetuning
-from finetune.classification import classification_tasks
-from finetune.qa import qa_tasks
-from finetune.tagging import tagging_tasks
-from model import tokenization
-
-
-def get_tasks(config: configure_finetuning.FinetuningConfig):
-  tokenizer = tokenization.FullTokenizer(vocab_file=config.vocab_file,
-                                         do_lower_case=config.do_lower_case)
-  return [get_task(config, task_name, tokenizer)
-          for task_name in config.task_names]
-
-
-def get_task(config: configure_finetuning.FinetuningConfig, task_name,
-             tokenizer):
-  """Get an instance of a task based on its name."""
-  if task_name == "cola":
-    return classification_tasks.CoLA(config, tokenizer)
-  elif task_name == "mrpc":
-    return classification_tasks.MRPC(config, tokenizer)
-  elif task_name == "mnli":
-    return classification_tasks.MNLI(config, tokenizer)
-  elif task_name == "sst":
-    return classification_tasks.SST(config, tokenizer)
-  elif task_name == "rte":
-    return classification_tasks.RTE(config, tokenizer)
-  elif task_name == "qnli":
-    return classification_tasks.QNLI(config, tokenizer)
-  elif task_name == "qqp":
-    return classification_tasks.QQP(config, tokenizer)
-  elif task_name == "sts":
-    return classification_tasks.STS(config, tokenizer)
-  elif task_name == "squad":
-    return qa_tasks.SQuAD(config, tokenizer)
-  elif task_name == "squadv1":
-    return qa_tasks.SQuADv1(config, tokenizer)
-  elif task_name == "newsqa":
-    return qa_tasks.NewsQA(config, tokenizer)
-  elif task_name == "naturalqs":
-    return qa_tasks.NaturalQuestions(config, tokenizer)
-  elif task_name == "triviaqa":
-    return qa_tasks.TriviaQA(config, tokenizer)
-  elif task_name == "searchqa":
-    return qa_tasks.SearchQA(config, tokenizer)
-  elif task_name == "chunk":
-    return tagging_tasks.Chunking(config, tokenizer)
-  else:
-    raise ValueError("Unknown task " + task_name)

sinatools/arabert/araelectra/flops_computation.py

@@ -1,215 +0,0 @@
-"""Computes the flops needed for training/running transformer networks."""
-
-import collections
-
-# We checked this code with TensorFlow"s FLOPs counting, although we had to
-# correct for this issue: https://github.com/tensorflow/tensorflow/issues/22071
-# Assumptions going into the FLOPs counting
-#   - An "operation" is a mathematical operation, not a machine instruction. So
-#     an "exp" takes one opp like and add, even though in practice an exp
-#     might be slower. This is not too bad an assumption because
-#     matrix-multiplies dominate the compute for most models, so minor details
-#     about activation functions don"t matter too much. Similarly, we count
-#     matrix-multiplies as 2*m*n flops instead of m*n, as one might if
-#     if considering fused multiply-add ops.
-#   - Backward pass takes the same number of FLOPs as forward pass. No exactly
-#     right (e.g., for softmax cross entropy loss the backward pass is faster).
-#     Importantly, it really is the same for matrix-multiplies, which is most of
-#     the compute anyway.
-#   - We assume "dense" embedding lookups (i.e., multiplication by a one-hot
-#     vector). On some hardware accelerators, these dense operations are
-#     actually faster than sparse lookups.
-# Please open a github issue if you spot a problem with this code!
-
-# I am not sure if the below constants are 100% right, but they are only applied
-# to O(hidden_size) activations, which is generally a lot less compute than the
-# matrix-multiplies, which are O(hidden_size^2), so they don't affect the total
-# number of FLOPs much.
-
-# random number, >=, multiply activations by dropout mask, multiply activations
-# by correction (1 / (1 - dropout_rate))
-DROPOUT_FLOPS = 4
-
-# compute mean activation (sum), computate variance of activation
-# (square and sum), bias (add), scale (multiply)
-LAYER_NORM_FLOPS = 5
-
-# GELU: 0.5 * x * (1 + tanh(sqrt(2 / np.pi) * (x + 0.044715 * pow(x, 3))))
-ACTIVATION_FLOPS = 8
-
-# max/substract (for stability), exp, sum, divide
-SOFTMAX_FLOPS = 5
-
-
-class TransformerHparams(object):
-  """Computes the train/inference FLOPs for transformers."""
-
-  def __init__(self, h, l, s=512, v=30522, e=None, i=None, heads=None,
-      head_size=None, output_frac=0.15625, sparse_embed_lookup=False,
-      decoder=False):
-    self.h = h  # hidden size
-    self.l = l  # number of layers
-    self.s = s  # sequence length
-    self.v = v  # vocab size
-    self.e = h if e is None else e  # embedding size
-    self.i = h * 4 if i is None else i  # intermediate size
-    self.kqv = h if head_size is None else head_size * heads  # attn proj sizes
-    self.heads = max(h // 64, 1) if heads is None else heads  # attention heads
-    self.output_frac = output_frac  # percent of tokens using an output softmax
-    self.sparse_embed_lookup = sparse_embed_lookup  # sparse embedding lookups
-    self.decoder = decoder  # decoder has extra attn to encoder states
-
-  def get_block_flops(self):
-    """Get the forward-pass FLOPs for a single transformer block."""
-    attn_mul = 2 if self.decoder else 1
-    block_flops = dict(
-        kqv=3 * 2 * self.h * self.kqv * attn_mul,
-        kqv_bias=3 * self.kqv * attn_mul,
-        attention_scores=2 * self.kqv * self.s * attn_mul,
-        attn_softmax=SOFTMAX_FLOPS * self.s * self.heads * attn_mul,
-        attention_dropout=DROPOUT_FLOPS * self.s * self.heads * attn_mul,
-        attention_scale=self.s * self.heads * attn_mul,
-        attention_weighted_avg_values=2 * self.h * self.s * attn_mul,
-        attn_output=2 * self.h * self.h * attn_mul,
-        attn_output_bias=self.h * attn_mul,
-        attn_output_dropout=DROPOUT_FLOPS * self.h * attn_mul,
-        attn_output_residual=self.h * attn_mul,
-        attn_output_layer_norm=LAYER_NORM_FLOPS * attn_mul,
-        intermediate=2 * self.h * self.i,
-        intermediate_act=ACTIVATION_FLOPS * self.i,
-        intermediate_bias=self.i,
-        output=2 * self.h * self.i,
-        output_bias=self.h,
-        output_dropout=DROPOUT_FLOPS * self.h,
-        output_residual=self.h,
-        output_layer_norm=LAYER_NORM_FLOPS * self.h,
-    )
-    return sum(block_flops.values()) * self.s
-
-  def get_embedding_flops(self, output=False):
-    """Get the forward-pass FLOPs the transformer inputs or output softmax."""
-    embedding_flops = {}
-    if output or (not self.sparse_embed_lookup):
-      embedding_flops["main_multiply"] = 2 * self.e * self.v
-    # input embedding post-processing
-    if not output:
-      embedding_flops.update(dict(
-          tok_type_and_position=2 * self.e * (self.s + 2),
-          add_tok_type_and_position=2 * self.e,
-          emb_layer_norm=LAYER_NORM_FLOPS * self.e,
-          emb_dropout=DROPOUT_FLOPS * self.e
-      ))
-    # projection layer if e != h
-    if self.e != self.h or output:
-      embedding_flops.update(dict(
-          hidden_kernel=2 * self.h * self.e,
-          hidden_bias=self.e if output else self.h
-      ))
-      # extra hidden layer and output softmax
-      if output:
-        embedding_flops.update(dict(
-            hidden_activation=ACTIVATION_FLOPS * self.e,
-            hidden_layernorm=LAYER_NORM_FLOPS * self.e,
-            output_softmax=SOFTMAX_FLOPS * self.v,
-            output_target_word=2 * self.v
-        ))
-        return self.output_frac * sum(embedding_flops.values()) * self.s
-    return sum(embedding_flops.values()) * self.s
-
-  def get_binary_classification_flops(self):
-    classification_flops = dict(
-        hidden=2 * self.h * self.h,
-        hidden_bias=self.h,
-        hidden_act=ACTIVATION_FLOPS * self.h,
-        logits=2 * self.h
-    )
-    return sum(classification_flops.values()) * self.s
-
-  def get_train_flops(self, batch_size, train_steps, discriminator=False):
-    """Get the FLOPs for pre-training the transformer."""
-    # 2* for forward/backward pass
-    return 2 * batch_size * train_steps * (
-        (self.l * self.get_block_flops()) +
-        self.get_embedding_flops(output=False) +
-        (self.get_binary_classification_flops() if discriminator else
-         self.get_embedding_flops(output=True))
-    )
-
-  def get_infer_flops(self):
-    """Get the FLOPs for running inference with the transformer on a
-    classification task."""
-    return ((self.l * self.get_block_flops()) +
-            self.get_embedding_flops(output=False) +
-            self.get_binary_classification_flops())
-
-
-def get_electra_train_flops(
-    h_d, l_d, h_g, l_g, batch_size, train_steps, tied_embeddings,
-    e=None, s=512, output_frac=0.15625):
-  """Get the FLOPs needed for  pre-training ELECTRA."""
-  if e is None:
-    e = h_d
-  disc = TransformerHparams(
-      h_d, l_d, s=s, e=e,
-      output_frac=output_frac).get_train_flops(batch_size, train_steps, True)
-  gen = TransformerHparams(
-      h_g, l_g, s=s, e=e if tied_embeddings else None,
-      output_frac=output_frac).get_train_flops(batch_size, train_steps)
-  return disc + gen
-
-
-MODEL_FLOPS = collections.OrderedDict([
-    # These runtimes were computed with tensorflow FLOPs counting instead of the
-    # script, as the neural architectures are quite different.
-    # 768648884 words in LM1b benchmark, 10 epochs with batch size 20,
-    # seq length 128, 568093262680 FLOPs per example.
-    ("elmo", 2 * 10 * 768648884 * 568093262680 / (20.0 * 128)),
-    # 15064773691518 is FLOPs for forward pass on 32 examples.
-    # Therefore 2 * steps * batch_size * 15064773691518 / 32 is XLNet compute
-    ("xlnet", 2 * 500000 * 8192 * 15064773691518 / 32.0),
-
-    # Runtimes computed with the script
-    ("gpt", TransformerHparams(768, 12, v=40000, output_frac=1.0).get_train_flops(
-        128, 960800)),
-    ("bert_small", TransformerHparams(256, 12, e=128, s=128).get_train_flops(128, 1.45e6)),
-    ("bert_base", TransformerHparams(768, 12).get_train_flops(256, 1e6)),
-    ("bert_large", TransformerHparams(1024, 24).get_train_flops(256, 1e6)),
-    ("electra_small", get_electra_train_flops(256, 12, 64, 12, 128, 1e6, True, s=128, e=128)),
-    ("electra_base", get_electra_train_flops(768, 12, 256, 12, 256, 766000, True)),
-    ("electra_400k", get_electra_train_flops(1024, 24, 256, 24, 2048, 400000, True)),
-    ("electra_1.75M", get_electra_train_flops(1024, 24, 256, 24, 2048, 1750000, True)),
-
-    # RoBERTa, ALBERT, and T5 have  minor architectural differences from
-    # BERT/ELECTRA, but I believe they don't significantly effect the runtime,
-    # so we use this script for those models as well.
-    ("roberta", TransformerHparams(1024, 24, v=50265).get_train_flops(8000, 500000)),
-    ("albert", TransformerHparams(4096, 12, v=30000, e=128).get_train_flops(
-        4096, 1.5e6)),
-    ("t5_11b", TransformerHparams(
-        1024,  # hidden size
-        24,  # layers
-        v=32000,  # vocab size
-        i=65536,  # ff intermediate hidden size
-        heads=128, head_size=128,  # heads/head size
-        output_frac=0.0  # encoder has no output softmax
-    ).get_train_flops(2048, 1e6) +  # 1M steps with batch size 2048
-     TransformerHparams(
-         1024,
-         24,
-         v=32000,
-         i=65536,
-         heads=128, head_size=128,
-         output_frac=1.0,  # decoder has output softmax for all positions
-         decoder=True
-     ).get_train_flops(2048, 1e6))
-])
-
-
-def main():
-  for k, v in MODEL_FLOPS.items():
-    print(k, v)
-
-
-if __name__ == "__main__":
-  main()
-

sinatools/arabert/araelectra/model/__init__.py

@@ -1,14 +0,0 @@
-# coding=utf-8
-# Copyright 2020 The Google Research Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.