InvokeAI 2.2.4.5__py3-none-any.whl

vendored/clip/clip/simple_tokenizer.py

@@ -0,0 +1,132 @@
+import gzip
+import html
+import os
+from functools import lru_cache
+
+import ftfy
+import regex as re
+
+
+@lru_cache()
+def default_bpe():
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+class SimpleTokenizer(object):
+    def __init__(self, bpe_path: str = default_bpe()):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
+        merges = merges[1:49152-256-2+1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v+'</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        vocab.extend(['<|startoftext|>', '<|endoftext|>'])
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}
+        self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token+'</w>'
+
+        while True:
+            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word)-1 and word[i+1] == second:
+                    new_word.append(first+second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+        return text

vendored/clip/hubconf.py

@@ -0,0 +1,42 @@
+from clip.clip import tokenize as _tokenize, load as _load, available_models as _available_models
+import re
+import string
+
+dependencies = ["torch", "torchvision", "ftfy", "regex", "tqdm"]
+
+# For compatibility (cannot include special characters in function name)
+model_functions = { model: re.sub(f'[{string.punctuation}]', '_', model) for model in _available_models()}
+
+def _create_hub_entrypoint(model):
+    def entrypoint(**kwargs):      
+        return _load(model, **kwargs)
+    
+    entrypoint.__doc__ = f"""Loads the {model} CLIP model
+
+        Parameters
+        ----------
+        device : Union[str, torch.device]
+            The device to put the loaded model
+
+        jit : bool
+            Whether to load the optimized JIT model or more hackable non-JIT model (default).
+
+        download_root: str
+            path to download the model files; by default, it uses "~/.cache/clip"
+
+        Returns
+        -------
+        model : torch.nn.Module
+            The {model} CLIP model
+
+        preprocess : Callable[[PIL.Image], torch.Tensor]
+            A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
+        """
+    return entrypoint
+
+def tokenize():
+    return _tokenize
+
+_entrypoints = {model_functions[model]: _create_hub_entrypoint(model) for model in _available_models()}
+
+globals().update(_entrypoints)

vendored/clip/setup.py

@@ -0,0 +1,21 @@
+import os
+
+import pkg_resources
+from setuptools import setup, find_packages
+
+setup(
+    name="clip",
+    py_modules=["clip"],
+    version="1.0",
+    description="",
+    author="OpenAI",
+    packages=find_packages(exclude=["tests*"]),
+    install_requires=[
+        str(r)
+        for r in pkg_resources.parse_requirements(
+            open(os.path.join(os.path.dirname(__file__), "requirements.txt"))
+        )
+    ],
+    include_package_data=True,
+    extras_require={'dev': ['pytest']},
+)

vendored/clip/tests/test_consistency.py

@@ -0,0 +1,25 @@
+import numpy as np
+import pytest
+import torch
+from PIL import Image
+
+import clip
+
+
+@pytest.mark.parametrize('model_name', clip.available_models())
+def test_consistency(model_name):
+    device = "cpu"
+    jit_model, transform = clip.load(model_name, device=device, jit=True)
+    py_model, _ = clip.load(model_name, device=device, jit=False)
+
+    image = transform(Image.open("CLIP.png")).unsqueeze(0).to(device)
+    text = clip.tokenize(["a diagram", "a dog", "a cat"]).to(device)
+
+    with torch.no_grad():
+        logits_per_image, _ = jit_model(image, text)
+        jit_probs = logits_per_image.softmax(dim=-1).cpu().numpy()
+
+        logits_per_image, _ = py_model(image, text)
+        py_probs = logits_per_image.softmax(dim=-1).cpu().numpy()
+
+    assert np.allclose(jit_probs, py_probs, atol=0.01, rtol=0.1)

vendored/clipseg/datasets/coco_wrapper.py

@@ -0,0 +1,99 @@
+import pickle
+from types import new_class
+import torch
+import numpy as np
+import os
+import json
+
+from os.path import join, dirname, isdir, isfile, expanduser, realpath, basename
+from random import shuffle, seed as set_seed
+from PIL import Image
+
+from itertools import combinations
+from torchvision import transforms
+from torchvision.transforms.transforms import Resize
+
+from datasets.utils import blend_image_segmentation
+from general_utils import get_from_repository
+
+COCO_CLASSES = {0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 4: 'airplane', 5: 'bus', 6: 'train', 7: 'truck', 8: 'boat', 9: 'traffic light', 10: 'fire hydrant', 11: 'stop sign', 12: 'parking meter', 13: 'bench', 14: 'bird', 15: 'cat', 16: 'dog', 17: 'horse', 18: 'sheep', 19: 'cow', 20: 'elephant', 21: 'bear', 22: 'zebra', 23: 'giraffe', 24: 'backpack', 25: 'umbrella', 26: 'handbag', 27: 'tie', 28: 'suitcase', 29: 'frisbee', 30: 'skis', 31: 'snowboard', 32: 'sports ball', 33: 'kite', 34: 'baseball bat', 35: 'baseball glove', 36: 'skateboard', 37: 'surfboard', 38: 'tennis racket', 39: 'bottle', 40: 'wine glass', 41: 'cup', 42: 'fork', 43: 'knife', 44: 'spoon', 45: 'bowl', 46: 'banana', 47: 'apple', 48: 'sandwich', 49: 'orange', 50: 'broccoli', 51: 'carrot', 52: 'hot dog', 53: 'pizza', 54: 'donut', 55: 'cake', 56: 'chair', 57: 'couch', 58: 'potted plant', 59: 'bed', 60: 'dining table', 61: 'toilet', 62: 'tv', 63: 'laptop', 64: 'mouse', 65: 'remote', 66: 'keyboard', 67: 'cell phone', 68: 'microwave', 69: 'oven', 70: 'toaster', 71: 'sink', 72: 'refrigerator', 73: 'book', 74: 'clock', 75: 'vase', 76: 'scissors', 77: 'teddy bear', 78: 'hair drier', 79: 'toothbrush'}
+
+class COCOWrapper(object):
+
+    def __init__(self, split, fold=0, image_size=400, aug=None, mask='separate', negative_prob=0,
+                 with_class_label=False):
+        super().__init__()
+
+        self.mask = mask
+        self.with_class_label = with_class_label
+        self.negative_prob = negative_prob
+
+        from third_party.hsnet.data.coco import DatasetCOCO
+
+        get_from_repository('COCO-20i', ['COCO-20i.tar'])
+
+        foldpath = join(dirname(__file__), '../third_party/hsnet/data/splits/coco/%s/fold%d.pkl')
+
+        def build_img_metadata_classwise(self):
+            with open(foldpath % (self.split, self.fold), 'rb') as f:
+                img_metadata_classwise = pickle.load(f)
+            return img_metadata_classwise
+
+
+        DatasetCOCO.build_img_metadata_classwise = build_img_metadata_classwise
+        # DatasetCOCO.read_mask = read_mask
+        
+        mean = [0.485, 0.456, 0.406]
+        std = [0.229, 0.224, 0.225]
+        transform = transforms.Compose([
+            transforms.Resize((image_size, image_size)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean, std)
+        ])
+
+        self.coco = DatasetCOCO(expanduser('~/datasets/COCO-20i/'), fold, transform, split, 1, False)
+    
+        self.all_classes = [self.coco.class_ids]
+        self.coco.base_path = join(expanduser('~/datasets/COCO-20i'))
+
+    def __len__(self):
+        return len(self.coco)
+
+    def __getitem__(self, i):
+        sample = self.coco[i]
+
+        label_name = COCO_CLASSES[int(sample['class_id'])]
+
+        img_s, seg_s = sample['support_imgs'][0], sample['support_masks'][0]
+
+        if self.negative_prob > 0 and torch.rand(1).item() < self.negative_prob:
+            new_class_id = sample['class_id']
+            while new_class_id == sample['class_id']:
+                sample2 = self.coco[torch.randint(0, len(self), (1,)).item()]
+                new_class_id = sample2['class_id']
+            img_s = sample2['support_imgs'][0]
+            seg_s = torch.zeros_like(seg_s)
+
+        mask = self.mask
+        if mask == 'separate':
+            supp = (img_s, seg_s)
+        elif mask == 'text_label':
+            # DEPRECATED
+            supp = [int(sample['class_id'])]
+        elif mask == 'text':
+            supp = [label_name]      
+        else:
+            if mask.startswith('text_and_'):
+                mask = mask[9:]
+                label_add = [label_name]
+            else:
+                label_add = []
+
+            supp = label_add + blend_image_segmentation(img_s, seg_s, mode=mask)
+
+        if self.with_class_label:
+            label = (torch.zeros(0), sample['class_id'],)
+        else:
+            label = (torch.zeros(0), )
+
+        return (sample['query_img'],) + tuple(supp), (sample['query_mask'].unsqueeze(0),) + label

vendored/clipseg/datasets/pascal_zeroshot.py

@@ -0,0 +1,60 @@
+from os.path import expanduser
+import torch
+import json
+import torchvision
+from general_utils import get_from_repository
+from general_utils import log
+from torchvision import transforms
+
+PASCAL_VOC_CLASSES_ZS = [['cattle.n.01', 'motorcycle.n.01'], ['aeroplane.n.01', 'sofa.n.01'], 
+                         ['cat.n.01', 'television.n.03'], ['train.n.01', 'bottle.n.01'],
+                          ['chair.n.01', 'pot_plant.n.01']]
+
+
+class PascalZeroShot(object):
+
+    def __init__(self, split, n_unseen, image_size=224) -> None:
+        super().__init__()
+
+        import sys
+        sys.path.append('third_party/JoEm')
+        from third_party.JoEm.data_loader.dataset import VOCSegmentation
+        from third_party.JoEm.data_loader import get_seen_idx, get_unseen_idx, VOC
+
+        self.pascal_classes = VOC
+        self.image_size = image_size
+
+        self.transform = transforms.Compose([
+            transforms.Resize((image_size, image_size)),
+        ])
+
+        if split == 'train':
+            self.voc = VOCSegmentation(get_unseen_idx(n_unseen), get_seen_idx(n_unseen), 
+                                       split=split, transform=True, transform_args=dict(base_size=312, crop_size=312), 
+                                       ignore_bg=False, ignore_unseen=False, remv_unseen_img=True)
+        elif split == 'val':
+            self.voc = VOCSegmentation(get_unseen_idx(n_unseen), get_seen_idx(n_unseen), 
+                                       split=split, transform=False, 
+                                       ignore_bg=False, ignore_unseen=False)
+
+        self.unseen_idx = get_unseen_idx(n_unseen)
+
+    def __len__(self):
+        return len(self.voc)
+
+    def __getitem__(self, i):
+
+        sample = self.voc[i]
+        label = sample['label'].long()
+        all_labels = [l for l in torch.where(torch.bincount(label.flatten())>0)[0].numpy().tolist() if l != 255]
+        class_indices = [l for l in all_labels]
+        class_names = [self.pascal_classes[l] for l in all_labels]
+
+        image = self.transform(sample['image'])
+
+        label = transforms.Resize((self.image_size, self.image_size), 
+            interpolation=torchvision.transforms.InterpolationMode.NEAREST)(label.unsqueeze(0))[0]
+
+        return (image,), (label, )
+
+

vendored/clipseg/datasets/pfe_dataset.py

@@ -0,0 +1,129 @@
+from os.path import expanduser
+import torch
+import json
+from general_utils import get_from_repository
+from datasets.utils import blend_image_segmentation
+from general_utils import log
+
+PASCAL_CLASSES = {a['id']: a['synonyms'] for a in json.load(open('datasets/pascal_classes.json'))}
+
+
+class PFEPascalWrapper(object):
+
+    def __init__(self, mode, split, mask='separate', image_size=473, label_support=None, size=None, p_negative=0, aug=None):
+        import sys
+        # sys.path.append(expanduser('~/projects/new_one_shot'))
+        from third_party.PFENet.util.dataset import SemData
+
+        get_from_repository('PascalVOC2012', ['Pascal5i.tar'])
+
+        self.p_negative = p_negative
+        self.size = size
+        self.mode = mode
+        self.image_size = image_size
+        
+        if label_support in {True, False}:
+            log.warning('label_support argument is deprecated. Use mask instead.')
+            #raise ValueError()
+
+        self.mask = mask
+
+        value_scale = 255
+        mean = [0.485, 0.456, 0.406]
+        mean = [item * value_scale for item in mean]
+        std = [0.229, 0.224, 0.225]
+        std = [item * value_scale for item in std]
+
+        import third_party.PFENet.util.transform as transform
+
+        if mode == 'val':
+            data_list = expanduser('~/projects/old_one_shot/PFENet/lists/pascal/val.txt')
+
+            data_transform = [transform.test_Resize(size=image_size)] if image_size != 'original' else []
+            data_transform += [
+                transform.ToTensor(),
+                transform.Normalize(mean=mean, std=std)
+            ]   
+
+
+        elif mode == 'train':
+            data_list =  expanduser('~/projects/old_one_shot/PFENet/lists/pascal/voc_sbd_merge_noduplicate.txt')
+
+            assert image_size != 'original'
+
+            data_transform = [
+                transform.RandScale([0.9, 1.1]),
+                transform.RandRotate([-10, 10], padding=mean, ignore_label=255),
+                transform.RandomGaussianBlur(),
+                transform.RandomHorizontalFlip(),
+                transform.Crop((image_size, image_size), crop_type='rand', padding=mean, ignore_label=255),
+                transform.ToTensor(),
+                transform.Normalize(mean=mean, std=std)
+            ]
+
+        data_transform = transform.Compose(data_transform)
+
+        self.dataset = SemData(split=split, mode=mode, data_root=expanduser('~/datasets/PascalVOC2012/VOC2012'), 
+                               data_list=data_list, shot=1, transform=data_transform, use_coco=False, use_split_coco=False)
+
+        self.class_list = self.dataset.sub_val_list if mode == 'val' else self.dataset.sub_list
+
+        # verify that subcls_list always has length 1
+        # assert len(set([len(d[4]) for d in self.dataset])) == 1
+
+        print('actual length', len(self.dataset.data_list))
+
+    def __len__(self):
+        if self.mode == 'val':
+            return len(self.dataset.data_list)
+        else:
+            return len(self.dataset.data_list)
+
+    def __getitem__(self, index):
+        if self.dataset.mode == 'train':
+            image, label, s_x, s_y, subcls_list = self.dataset[index % len(self.dataset.data_list)]
+        elif self.dataset.mode == 'val':
+            image, label, s_x, s_y, subcls_list, ori_label = self.dataset[index % len(self.dataset.data_list)]
+            ori_label = torch.from_numpy(ori_label).unsqueeze(0)
+            
+            if self.image_size != 'original':
+                longerside = max(ori_label.size(1), ori_label.size(2))
+                backmask = torch.ones(ori_label.size(0), longerside, longerside).cuda()*255
+                backmask[0, :ori_label.size(1), :ori_label.size(2)] = ori_label
+                label = backmask.clone().long()      
+            else:
+                label = label.unsqueeze(0) 
+
+            # assert label.shape == (473, 473)
+
+        if self.p_negative > 0:
+            if torch.rand(1).item() < self.p_negative:
+                while True:
+                    idx = torch.randint(0, len(self.dataset.data_list), (1,)).item()
+                    _, _, s_x, s_y, subcls_list_tmp, _ = self.dataset[idx]
+                    if subcls_list[0] != subcls_list_tmp[0]:
+                        break
+
+        s_x = s_x[0]
+        s_y = (s_y == 1)[0]
+        label_fg = (label == 1).float()
+        val_mask = (label != 255).float()
+
+        class_id = self.class_list[subcls_list[0]]
+
+        label_name = PASCAL_CLASSES[class_id][0]
+        label_add = ()
+        mask = self.mask
+
+        if mask == 'text':
+            support = ('a photo of a ' + label_name + '.',)
+        elif mask == 'separate':
+            support = (s_x, s_y)
+        else:
+            if mask.startswith('text_and_'):
+                label_add = (label_name,)
+                mask = mask[9:]
+
+            support = (blend_image_segmentation(s_x, s_y.float(), mask)[0],)
+
+        return (image,) + label_add + support, (label_fg.unsqueeze(0), val_mask.unsqueeze(0), subcls_list[0])