SinaTools 0.1.1__py2.py3-none-any.whl

nlptools/arabiner/bin/__init__.py

@@ -0,0 +1,14 @@
+from nlptools.DataDownload import downloader
+import os
+from nlptools.arabiner.utils.helpers import load_checkpoint
+import nlptools
+
+nlptools.tagger = None
+nlptools.tag_vocab = None
+nlptools.train_config = None
+
+filename = 'Wj27012000.tar'
+path =downloader.get_appdatadir()
+model_path = os.path.join(path, filename)
+print('1',model_path)
+nlptools.tagger, nlptools.tag_vocab, nlptools.train_config = load_checkpoint(model_path)

nlptools/arabiner/bin/eval.py

@@ -0,0 +1,87 @@
+import os
+import logging
+import argparse
+from collections import namedtuple
+from nlptools.arabiner.utils.helpers import load_checkpoint, make_output_dirs, logging_config
+from nlptools.arabiner.utils.data import get_dataloaders, parse_conll_files
+from nlptools.arabiner.utils.metrics import compute_single_label_metrics, compute_nested_metrics
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--output_path",
+        type=str,
+        required=True,
+        help="Path to save results",
+    )
+
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        required=True,
+        help="Model path",
+    )
+
+    parser.add_argument(
+        "--data_paths",
+        nargs="+",
+        type=str,
+        required=True,
+        help="Text or sequence to tag, this is in same format as training data with 'O' tag for all tokens",
+    )
+
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=32,
+        help="Batch size",
+    )
+
+    args = parser.parse_args()
+
+    return args
+
+
+def main(args):
+    # Create directory to save predictions
+    make_output_dirs(args.output_path, overwrite=True)
+    logging_config(log_file=os.path.join(args.output_path, "eval.log"))
+
+    # Load tagger
+    tagger, tag_vocab, train_config = load_checkpoint(args.model_path)
+
+    # Convert text to a tagger dataset and index the tokens in args.text
+    datasets, vocab = parse_conll_files(args.data_paths)
+
+    vocabs = namedtuple("Vocab", ["tags", "tokens"])
+    vocab = vocabs(tokens=vocab.tokens, tags=tag_vocab)
+
+    # From the datasets generate the dataloaders
+    dataloaders = get_dataloaders(
+        datasets, vocab,
+        train_config.data_config,
+        batch_size=args.batch_size,
+        shuffle=[False] * len(datasets)
+    )
+
+    # Evaluate the model on each dataloader
+    for dataloader, input_file in zip(dataloaders, args.data_paths):
+        filename = os.path.basename(input_file)
+        predictions_file = os.path.join(args.output_path, f"predictions_{filename}")
+        _, segments, _, _ = tagger.eval(dataloader)
+        tagger.segments_to_file(segments, predictions_file)
+
+        if "Nested" in train_config.trainer_config["fn"]:
+            compute_nested_metrics(segments, vocab.tags[1:])
+        else:
+            compute_single_label_metrics(segments)
+
+
+if __name__ == "__main__":
+    main(parse_args())

nlptools/arabiner/bin/infer.py

@@ -0,0 +1,91 @@
+import os
+from collections import namedtuple
+from nlptools.arabiner.utils.helpers import load_checkpoint
+from nlptools.arabiner.utils.data import get_dataloaders, text2segments
+from nlptools.DataDownload import downloader
+import nlptools
+def ner(text, batch_size=32):
+    """
+    This method takes a text as input, and a batch size, then performs named entity recognition (NER) on the input text and returns a list of tagged mentions.
+
+    Args:
+        text (str): The input text to perform NER on.
+        batch_size (int, optional): Batch size for inference. Default is 32.
+
+    Returns:
+        list: A list of lists containing token and label pairs for each segment.
+              Each inner list has the format ['token', 'label1 label2 ...'].
+    **Example:**
+
+     .. highlight:: python
+     .. code-block:: python
+
+            from nlptools.arabiner.bin import infer
+            infer.ner('ذهب محمد الى جامعة بيرزيت')
+
+            #the output   
+            [['ذهب', 'O'],
+            ['محمد', 'B-PERS'],
+            ['الى', 'O'],
+            ['جامعة', 'B-ORG'],
+            ['بيرزيت', 'B-GPE I-ORG']]
+    """    
+    # Load tagger
+    # filename = 'Wj27012000.tar'
+    # path =downloader.get_appdatadir()
+    # model_path = os.path.join(path, filename)
+    # print('1',model_path)
+    # tagger, tag_vocab, train_config = load_checkpoint(model_path)
+    
+
+    # Convert text to a tagger dataset and index the tokens in args.text
+    dataset, token_vocab = text2segments(text)
+
+    vocabs = namedtuple("Vocab", ["tags", "tokens"])
+    vocab = vocabs(tokens=token_vocab, tags=nlptools.tag_vocab)
+
+    # From the datasets generate the dataloaders
+    dataloader = get_dataloaders(
+        (dataset,),
+        vocab,
+        nlptools.train_config.data_config,
+        batch_size=batch_size,
+        shuffle=(False,),
+    )[0]
+
+    # Perform inference on the text and get back the tagged segments
+    segments = nlptools.tagger.infer(dataloader)
+    segments_lists = []
+    # Print results
+    for segment in segments:
+        for token in segment:
+            segments_list = []
+            segments_list.append(token.text)
+            #print([t['tag'] for t in token.pred_tag])
+            list_of_tags = [t['tag'] for t in token.pred_tag]
+            list_of_tags = [i for i in list_of_tags if i not in('O',' ','')]
+            #print(list_of_tags)
+            if list_of_tags == []:
+               segments_list.append(' '.join(['O']))
+            else:
+               segments_list.append(' '.join(list_of_tags))
+            segments_lists.append(segments_list)         
+    return segments_lists
+
+#Print results
+    # for segment in segments:
+    #     s = [
+    #         (token.text, token.pred_tag[0]['tag'])
+    #         for token in segment
+    #         if token.pred_tag[0]['tag'] != 'O'
+    #     ]
+    #     print(", ".join([f"({token}, {tag})" for token, tag in s]))
+
+def extract_tags(text):
+    tags = []
+    tokens = text.split()
+    for token in tokens:
+        tag = token.split("(")[-1].split(")")[0]
+        if tag != "O":
+            tags.append(tag)
+    return " ".join(tags)

nlptools/arabiner/bin/process.py

@@ -0,0 +1,140 @@
+import os
+import argparse
+import csv
+import logging
+import numpy as np
+from nlptools.arabiner.utils.helpers import logging_config
+from nlptools.arabiner.utils.data import conll_to_segments
+
+logger = logging.getLogger(__name__)
+
+
+def to_conll_format(input_files, output_path, multi_label=False):
+    """
+    Parse data files and convert them into CoNLL format
+    :param input_files: List[str] - list of filenames
+    :param output_path: str - output path
+    :param multi_label: boolean - True to process data with mutli-class/multi-label
+    :return:
+    """
+    for input_file in input_files:
+        tokens = list()
+        prev_sent_id = None
+
+        with open(input_file, "r") as fh:
+            r = csv.reader(fh, delimiter="\t", quotechar=" ")
+            next(r)
+
+            for row in r:
+                sent_id, token, labels = row[1], row[3], row[4].split()
+                valid_labels = sum([1 for l in labels if "-" in l or l == "O"]) == len(labels)
+
+                if not valid_labels:
+                    logging.warning("Invalid labels found %s", str(row))
+                    continue
+                if not labels:
+                    logging.warning("Token %s has no label", str(row))
+                    continue
+                if not token:
+                    logging.warning("Token %s is missing", str(row))
+                    continue
+                if len(token.split()) > 1:
+                    logging.warning("Token %s has multiple tokens", str(row))
+                    continue
+
+                if prev_sent_id is not None and sent_id != prev_sent_id:
+                    tokens.append([])
+
+                if multi_label:
+                    tokens.append([token] + labels)
+                else:
+                    tokens.append([token, labels[0]])
+
+                prev_sent_id = sent_id
+
+        num_segments = sum([1 for token in tokens if not token])
+        logging.info("Found %d segments and %d tokens in %s", num_segments + 1, len(tokens) - num_segments, input_file)
+
+        filename = os.path.basename(input_file)
+        output_file = os.path.join(output_path, filename)
+
+        with open(output_file, "w") as fh:
+            fh.write("\n".join(" ".join(token) for token in tokens))
+            logging.info("Output file %s", output_file)
+
+
+def train_dev_test_split(input_files, output_path, train_ratio, dev_ratio):
+    segments = list()
+    filenames = ["train.txt", "val.txt", "test.txt"]
+
+    for input_file in input_files:
+        segments += conll_to_segments(input_file)
+
+    n = len(segments)
+    np.random.shuffle(segments)
+    datasets = np.split(segments, [int(train_ratio*n), int((train_ratio+dev_ratio)*n)])
+
+    # write data to files
+    for i in range(len(datasets)):
+        filename = os.path.join(output_path, filenames[i])
+
+        with open(filename, "w") as fh:
+            text = "\n\n".join(["\n".join([f"{token.text} {' '.join(token.gold_tag)}" for token in segment]) for segment in datasets[i]])
+            fh.write(text)
+            logging.info("Output file %s", filename)
+
+
+def main(args):
+    if args.task == "to_conll_format":
+        to_conll_format(args.input_files, args.output_path, multi_label=args.multi_label)
+    if args.task == "train_dev_test_split":
+        train_dev_test_split(args.input_files, args.output_path, args.train_ratio, args.dev_ratio)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--input_files",
+        type=str,
+        nargs="+",
+        required=True,
+        help="List of input files",
+    )
+
+    parser.add_argument(
+        "--output_path",
+        type=str,
+        required=True,
+        help="Output path",
+    )
+
+    parser.add_argument(
+        "--train_ratio",
+        type=float,
+        required=False,
+        help="Training data ratio (percent of segments). Required with the task train_dev_test_split. "
+             "Files must in ConLL format",
+    )
+
+    parser.add_argument(
+        "--dev_ratio",
+        type=float,
+        required=False,
+        help="Dev/val data ratio (percent of segments). Required with the task train_dev_test_split. "
+             "Files must in ConLL format",
+    )
+
+    parser.add_argument(
+        "--task", required=True, choices=["to_conll_format", "train_dev_test_split"]
+    )
+
+    parser.add_argument(
+        "--multi_label", action='store_true'
+    )
+
+    args = parser.parse_args()
+    logging_config(os.path.join(args.output_path, "process.log"))
+    main(args)

nlptools/arabiner/bin/train.py

@@ -0,0 +1,221 @@
+import os
+import logging
+import json
+import argparse
+import torch.utils.tensorboard
+from torchvision import *
+import pickle
+from nlptools.arabiner.utils.data import get_dataloaders, parse_conll_files
+from nlptools.arabiner.utils.helpers import logging_config, load_object, make_output_dirs, set_seed
+
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--output_path",
+        type=str,
+        required=True,
+        help="Output path",
+    )
+
+    parser.add_argument(
+        "--train_path",
+        type=str,
+        required=True,
+        help="Path to training data",
+    )
+
+    parser.add_argument(
+        "--val_path",
+        type=str,
+        required=True,
+        help="Path to training data",
+    )
+
+    parser.add_argument(
+        "--test_path",
+        type=str,
+        required=True,
+        help="Path to training data",
+    )
+
+    parser.add_argument(
+        "--bert_model",
+        type=str,
+        default="aubmindlab/bert-base-arabertv2",
+        help="BERT model",
+    )
+
+    parser.add_argument(
+        "--gpus",
+        type=int,
+        nargs="+",
+        default=[0],
+        help="GPU IDs to train on",
+    )
+
+    parser.add_argument(
+        "--log_interval",
+        type=int,
+        default=10,
+        help="Log results every that many timesteps",
+    )
+
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=32,
+        help="Batch size",
+    )
+
+    parser.add_argument(
+        "--num_workers",
+        type=int,
+        default=0,
+        help="Dataloader number of workers",
+    )
+
+    parser.add_argument(
+        "--data_config",
+        type=json.loads,
+        default='{"fn": "arabiner.data.datasets.DefaultDataset", "kwargs": {"max_seq_len": 512}}',
+        help="Dataset configurations",
+    )
+
+    parser.add_argument(
+        "--trainer_config",
+        type=json.loads,
+        default='{"fn": "arabiner.trainers.BertTrainer", "kwargs": {"max_epochs": 50}}',
+        help="Trainer configurations",
+    )
+
+    parser.add_argument(
+        "--network_config",
+        type=json.loads,
+        default='{"fn": "arabiner.nn.BertSeqTagger", "kwargs": '
+                '{"dropout": 0.1, "bert_model": "aubmindlab/bert-base-arabertv2"}}',
+        help="Network configurations",
+    )
+
+    parser.add_argument(
+        "--optimizer",
+        type=json.loads,
+        default='{"fn": "torch.optim.AdamW", "kwargs": {"lr": 0.0001}}',
+        help="Optimizer configurations",
+    )
+
+    parser.add_argument(
+        "--lr_scheduler",
+        type=json.loads,
+        default='{"fn": "torch.optim.lr_scheduler.ExponentialLR", "kwargs": {"gamma": 1}}',
+        help="Learning rate scheduler configurations",
+    )
+
+    parser.add_argument(
+        "--loss",
+        type=json.loads,
+        default='{"fn": "torch.nn.CrossEntropyLoss", "kwargs": {}}',
+        help="Loss function configurations",
+    )
+
+    parser.add_argument(
+        "--overwrite",
+        action="store_true",
+        help="Overwrite output directory",
+    )
+
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=1,
+        help="Seed for random initialization",
+    )
+
+    args = parser.parse_args()
+
+    return args
+
+
+def main(args):
+    make_output_dirs(
+        args.output_path,
+        subdirs=("tensorboard", "checkpoints"),
+        overwrite=args.overwrite,
+    )
+
+    # Set the seed for randomization
+    set_seed(args.seed)
+
+    logging_config(os.path.join(args.output_path, "train.log"))
+    summary_writer = torch.utils.tensorboard.SummaryWriter(
+        os.path.join(args.output_path, "tensorboard")
+    )
+    os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(gpu) for gpu in args.gpus])
+
+    # Get the datasets and vocab for tags and tokens
+    datasets, vocab = parse_conll_files((args.train_path, args.val_path, args.test_path))
+
+    if "Nested" in args.network_config["fn"]:
+        args.network_config["kwargs"]["num_labels"] = [len(v) for v in vocab.tags[1:]]
+    else:
+        args.network_config["kwargs"]["num_labels"] = len(vocab.tags[0])
+
+    # Save tag vocab to desk
+    with open(os.path.join(args.output_path, "tag_vocab.pkl"), "wb") as fh:
+        pickle.dump(vocab.tags, fh)
+
+    # Write config to file
+    args_file = os.path.join(args.output_path, "args.json")
+    with open(args_file, "w") as fh:
+        logger.info("Writing config to %s", args_file)
+        json.dump(args.__dict__, fh, indent=4)
+
+    # From the datasets generate the dataloaders
+    args.data_config["kwargs"]["bert_model"] = args.network_config["kwargs"]["bert_model"]
+    train_dataloader, val_dataloader, test_dataloader = get_dataloaders(
+        datasets, vocab, args.data_config, args.batch_size, args.num_workers
+    )
+
+    model = load_object(args.network_config["fn"], args.network_config["kwargs"])
+    model = torch.nn.DataParallel(model, device_ids=range(len(args.gpus)))
+
+    if torch.cuda.is_available():
+        model = model.cuda()
+
+    args.optimizer["kwargs"]["params"] = model.parameters()
+    optimizer = load_object(args.optimizer["fn"], args.optimizer["kwargs"])
+
+    args.lr_scheduler["kwargs"]["optimizer"] = optimizer
+    if "num_training_steps" in args.lr_scheduler["kwargs"]:
+        args.lr_scheduler["kwargs"]["num_training_steps"] = args.max_epochs * len(
+            train_dataloader
+        )
+
+    scheduler = load_object(args.lr_scheduler["fn"], args.lr_scheduler["kwargs"])
+    loss = load_object(args.loss["fn"], args.loss["kwargs"])
+
+    args.trainer_config["kwargs"].update({
+        "model": model,
+        "optimizer": optimizer,
+        "scheduler": scheduler,
+        "loss": loss,
+        "train_dataloader": train_dataloader,
+        "val_dataloader": val_dataloader,
+        "test_dataloader": test_dataloader,
+        "log_interval": args.log_interval,
+        "summary_writer": summary_writer,
+        "output_path": args.output_path
+    })
+
+    trainer = load_object(args.trainer_config["fn"], args.trainer_config["kwargs"])
+    trainer.train()
+    return
+
+
+if __name__ == "__main__":
+    main(parse_args())

nlptools/arabiner/data/__init__.py

@@ -0,0 +1 @@
+from nlptools.arabiner.data.datasets import NestedTagsDataset