flaxdiff 0.1.32__py3-none-any.whl → 0.1.34__py3-none-any.whl

flaxdiff/data/online_loader.py

@@ -21,11 +21,12 @@ import urllib
 
 import PIL.Image
 import cv2
-import traceback 
+import traceback
 
 USER_AGENT = get_datasets_user_agent()
 
 data_queue = Queue(16*2000)
+error_queue = Queue(16*2000)
 
 
 def fetch_single_image(image_url, timeout=None, retries=0):
@@ -45,7 +46,7 @@ def fetch_single_image(image_url, timeout=None, retries=0):
 
 
 def default_image_processor(
-    image, image_shape, 
+    image, image_shape,
     min_image_shape=(128, 128),
     upscale_interpolation=cv2.INTER_CUBIC,
     downscale_interpolation=cv2.INTER_AREA,
@@ -77,8 +78,15 @@ def default_image_processor(
     return image, original_height, original_width
 
 
+def default_feature_extractor(sample):
+    return {
+        "url": sample["url"],
+        "caption": sample["caption"],
+    }
+
+
 def map_sample(
-    url, caption,
+    sample,
     image_shape=(256, 256),
     min_image_shape=(128, 128),
     timeout=15,
@@ -86,8 +94,11 @@ def map_sample(
     upscale_interpolation=cv2.INTER_CUBIC,
     downscale_interpolation=cv2.INTER_AREA,
     image_processor=default_image_processor,
+    feature_extractor=default_feature_extractor,
 ):
     try:
+        features = feature_extractor(sample)
+        url, caption = features["url"], features["caption"]
         # Assuming fetch_single_image is defined elsewhere
         image = fetch_single_image(url, timeout=timeout, retries=retries)
         if image is None:
@@ -96,11 +107,12 @@ def map_sample(
         image, original_height, original_width = image_processor(
             image, image_shape, min_image_shape=min_image_shape,
             upscale_interpolation=upscale_interpolation,
-            downscale_interpolation=downscale_interpolation,)
-        
+            downscale_interpolation=downscale_interpolation,
+        )
+
         if image is None:
             return
-        
+
         data_queue.put({
             "url": url,
             "caption": caption,
@@ -110,22 +122,17 @@ def map_sample(
         })
     except Exception as e:
         print(f"Error maping sample {url}", e)
-        traceback.print_exc() 
+        traceback.print_exc()
         # error_queue.put_nowait({
         #     "url": url,
         #     "caption": caption,
         #     "error": str(e)
         # })
         pass
-    
-def default_feature_extractor(sample):
-    return {
-        "url": sample["url"],
-        "caption": sample["caption"],
-    }
+
 
 def map_batch(
-    batch, num_threads=256, image_shape=(256, 256), 
+    batch, num_threads=256, image_shape=(256, 256),
     min_image_shape=(128, 128),
     timeout=15, retries=3, image_processor=default_image_processor,
     upscale_interpolation=cv2.INTER_CUBIC,
@@ -133,40 +140,76 @@ def map_batch(
     feature_extractor=default_feature_extractor,
 ):
     try:
-        map_sample_fn = partial(map_sample, image_shape=image_shape, min_image_shape=min_image_shape,
-                                timeout=timeout, retries=retries, image_processor=image_processor,
-                                upscale_interpolation=upscale_interpolation,
-                                downscale_interpolation=downscale_interpolation)
+        map_sample_fn = partial(
+            map_sample, image_shape=image_shape, min_image_shape=min_image_shape,
+            timeout=timeout, retries=retries, image_processor=image_processor,
+            upscale_interpolation=upscale_interpolation,
+            downscale_interpolation=downscale_interpolation,
+            feature_extractor=feature_extractor
+        )
         with ThreadPoolExecutor(max_workers=num_threads) as executor:
-            features = feature_extractor(batch)
-            url, caption = features["url"], features["caption"]
-            executor.map(map_sample_fn, url, caption)
+            executor.map(map_sample_fn, batch)
     except Exception as e:
         print(f"Error maping batch", e)
-        traceback.print_exc() 
-        # error_queue.put_nowait({
-        #     "batch": batch,
-        #     "error": str(e)
-        # })
+        traceback.print_exc()
+        error_queue.put_nowait({
+            "batch": batch,
+            "error": str(e)
+        })
         pass
 
 
+def map_batch_repeat_forever(
+    batch, num_threads=256, image_shape=(256, 256),
+    min_image_shape=(128, 128),
+    timeout=15, retries=3, image_processor=default_image_processor,
+    upscale_interpolation=cv2.INTER_CUBIC,
+    downscale_interpolation=cv2.INTER_AREA,
+    feature_extractor=default_feature_extractor,
+):
+    while True:  # Repeat forever
+        try:
+            map_sample_fn = partial(
+                map_sample, image_shape=image_shape, min_image_shape=min_image_shape,
+                timeout=timeout, retries=retries, image_processor=image_processor,
+                upscale_interpolation=upscale_interpolation,
+                downscale_interpolation=downscale_interpolation,
+                feature_extractor=feature_extractor
+            )
+            with ThreadPoolExecutor(max_workers=num_threads) as executor:
+                executor.map(map_sample_fn, batch)
+            # Shuffle the batch
+            batch = batch.shuffle(seed=np.random.randint(0, 1000000))
+        except Exception as e:
+            print(f"Error maping batch", e)
+            traceback.print_exc()
+            error_queue.put_nowait({
+                "batch": batch,
+                "error": str(e)
+            })
+            pass
+
+
 def parallel_image_loader(
-    dataset: Dataset, num_workers: int = 8, image_shape=(256, 256), 
+    dataset: Dataset, num_workers: int = 8, image_shape=(256, 256),
     min_image_shape=(128, 128),
     num_threads=256, timeout=15, retries=3, image_processor=default_image_processor,
     upscale_interpolation=cv2.INTER_CUBIC,
     downscale_interpolation=cv2.INTER_AREA,
     feature_extractor=default_feature_extractor,
+    map_batch_fn=map_batch,
+
 ):
-    map_batch_fn = partial(map_batch, num_threads=num_threads, image_shape=image_shape, 
-                           min_image_shape=min_image_shape,
-                           timeout=timeout, retries=retries, image_processor=image_processor,
-                           upscale_interpolation=upscale_interpolation,
-                           downscale_interpolation=downscale_interpolation,
-                           feature_extractor=feature_extractor)
+    map_batch_fn = partial(
+        map_batch_fn, num_threads=num_threads, image_shape=image_shape,
+        min_image_shape=min_image_shape,
+        timeout=timeout, retries=retries, image_processor=image_processor,
+        upscale_interpolation=upscale_interpolation,
+        downscale_interpolation=downscale_interpolation,
+        feature_extractor=feature_extractor
+    )
     shard_len = len(dataset) // num_workers
-    print(f"Local Shard lengths: {shard_len}")
+    print(f"Local Shard lengths: {shard_len}, workers: {num_workers}")
     with multiprocessing.Pool(num_workers) as pool:
         iteration = 0
         while True:
@@ -178,6 +221,7 @@ def parallel_image_loader(
             iteration += 1
             print(f"Shuffling dataset with seed {iteration}")
             dataset = dataset.shuffle(seed=iteration)
+            print(f"Dataset shuffled")
             # Clear the error queue
             # while not error_queue.empty():
             #     error_queue.get_nowait()
@@ -185,27 +229,44 @@ def parallel_image_loader(
 
 class ImageBatchIterator:
     def __init__(
-        self, dataset: Dataset, batch_size: int = 64, image_shape=(256, 256), 
+        self, dataset: Dataset, batch_size: int = 64, image_shape=(256, 256),
         min_image_shape=(128, 128),
-        num_workers: int = 8, num_threads=256, timeout=15, retries=3, 
+        num_workers: int = 8, num_threads=256, timeout=15, retries=3,
         image_processor=default_image_processor,
         upscale_interpolation=cv2.INTER_CUBIC,
         downscale_interpolation=cv2.INTER_AREA,
         feature_extractor=default_feature_extractor,
+        map_batch_fn=map_batch,
     ):
         self.dataset = dataset
         self.num_workers = num_workers
         self.batch_size = batch_size
-        loader = partial(parallel_image_loader, num_threads=num_threads,
-                         image_shape=image_shape,
-                         min_image_shape=min_image_shape, 
-                         num_workers=num_workers, 
-                         timeout=timeout, retries=retries, image_processor=image_processor,
-                         upscale_interpolation=upscale_interpolation,
-                         downscale_interpolation=downscale_interpolation,
-                         feature_extractor=feature_extractor)
+        loader = partial(
+            parallel_image_loader,
+            num_threads=num_threads,
+            image_shape=image_shape,
+            min_image_shape=min_image_shape,
+            num_workers=num_workers,
+            timeout=timeout, retries=retries,
+            image_processor=image_processor,
+            upscale_interpolation=upscale_interpolation,
+            downscale_interpolation=downscale_interpolation,
+            feature_extractor=feature_extractor,
+            map_batch_fn=map_batch_fn,
+        )
         self.thread = threading.Thread(target=loader, args=(dataset,))
         self.thread.start()
+        self.error_queue = queue.Queue()
+
+        def error_fetcher():
+            while True:
+                error = error_queue.get()
+                self.error_queue.put(error)
+        self.error_thread = threading.Thread(target=error_fetcher)
+        self.error_thread.start()
+
+    def get_error(self):
+        yield self.error_queue.get()
 
     def __iter__(self):
         return self
@@ -269,6 +330,7 @@ class OnlineStreamingDataLoader():
         upscale_interpolation=cv2.INTER_CUBIC,
         downscale_interpolation=cv2.INTER_AREA,
         feature_extractor=default_feature_extractor,
+        map_batch_fn=map_batch,
     ):
         if isinstance(dataset, str):
             dataset_path = dataset
@@ -289,13 +351,16 @@ class OnlineStreamingDataLoader():
         self.dataset = dataset.shard(
             num_shards=global_process_count, index=global_process_index)
         print(f"Dataset length: {len(dataset)}")
-        self.iterator = ImageBatchIterator(self.dataset, image_shape=image_shape,
-                                           min_image_shape=min_image_shape,
-                                           num_workers=num_workers, batch_size=batch_size, num_threads=num_threads,
-                                            timeout=timeout, retries=retries, image_processor=image_processor,
-                                             upscale_interpolation=upscale_interpolation,
-                                             downscale_interpolation=downscale_interpolation,
-                                             feature_extractor=feature_extractor)
+        self.iterator = ImageBatchIterator(
+            self.dataset, image_shape=image_shape,
+            min_image_shape=min_image_shape,
+            num_workers=num_workers, batch_size=batch_size, num_threads=num_threads,
+            timeout=timeout, retries=retries, image_processor=image_processor,
+            upscale_interpolation=upscale_interpolation,
+            downscale_interpolation=downscale_interpolation,
+            feature_extractor=feature_extractor,
+            map_batch_fn=map_batch_fn,
+        )
         self.batch_size = batch_size
 
         # Launch a thread to load batches in the background
@@ -306,7 +371,7 @@ class OnlineStreamingDataLoader():
                 try:
                     self.batch_queue.put(collate_fn(batch))
                 except Exception as e:
-                    print("Error processing batch", e)
+                    print("Error collating batch", e)
 
         self.loader_thread = threading.Thread(target=batch_loader)
         self.loader_thread.start()
@@ -319,4 +384,4 @@ class OnlineStreamingDataLoader():
         # return self.collate_fn(next(self.iterator))
 
     def __len__(self):
-        return len(self.dataset)
+        return len(self.dataset)

{flaxdiff-0.1.32.dist-info → flaxdiff-0.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flaxdiff
-Version: 0.1.32
+Version: 0.1.34
 Summary: A versatile and easy to understand Diffusion library
 Author: Ashish Kumar Singh
 Author-email: ashishkmr472@gmail.com

{flaxdiff-0.1.32.dist-info → flaxdiff-0.1.34.dist-info}/RECORD

@@ -1,7 +1,7 @@
 flaxdiff/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 flaxdiff/utils.py,sha256=B0GcHlzlVYDNEIdh2v5qmP4u0neIT-FqexNohuyuCvg,2452
 flaxdiff/data/__init__.py,sha256=PM3PkHihyohT5SHVYKc8vQ4IeVfGPpCktkSVwvqMjQ4,52
-flaxdiff/data/online_loader.py,sha256=Yi1thc2izJoXeB9UPTAkfWhWp8m4UYWi6cDwW6I6zUc,11661
+flaxdiff/data/online_loader.py,sha256=VpwTivqTYXM4Eu3jLWoDkYRcl7KsgJ5A6t17zDSDsDc,13226
 flaxdiff/models/__init__.py,sha256=FAivVYXxM2JrCFIXf-C3374RB2Hth25dBrzOeNFhH1U,26
 flaxdiff/models/attention.py,sha256=ZbDGIb5Q6FRqJ6qRY660cqw4WvF9IwCnhEuYdTpLPdM,13023
 flaxdiff/models/common.py,sha256=hWsSs2BP2J-JN1s4qLRr-h-KYkcVyl2hOp1Wsm_L-h8,10994
@@ -34,7 +34,7 @@ flaxdiff/trainer/__init__.py,sha256=T-vUVq4zHcMK6kpCsG4Gu8vn71q6lZD-lg-Ul7yKfEk,
 flaxdiff/trainer/autoencoder_trainer.py,sha256=al7AsZ7yeDMEiDD-gbcXf0ADq_xfk1VMxvg24GfA-XQ,7008
 flaxdiff/trainer/diffusion_trainer.py,sha256=wKkg63DWZjx2MoM3VQNCDIr40rWN8fUGxH9jWWxfZao,9373
 flaxdiff/trainer/simple_trainer.py,sha256=Z77zRS5viJpd2Mpl6sonJk5WcnEWi2Cd4gl4u5tIX2M,18206
-flaxdiff-0.1.32.dist-info/METADATA,sha256=yqODOUEAlcE_60tOOfSP7XoH0VTBm3dY9J1ali36VFY,22083
-flaxdiff-0.1.32.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-flaxdiff-0.1.32.dist-info/top_level.txt,sha256=-2-nXnfkJgSfkki1tjm5Faw6Dso7vhtdn2szwCdX5CQ,9
-flaxdiff-0.1.32.dist-info/RECORD,,
+flaxdiff-0.1.34.dist-info/METADATA,sha256=5x3QvLbZ3AgBjghAxnWQXQD5-cJ9jZ0ifPhGxdTeuEc,22083
+flaxdiff-0.1.34.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+flaxdiff-0.1.34.dist-info/top_level.txt,sha256=-2-nXnfkJgSfkki1tjm5Faw6Dso7vhtdn2szwCdX5CQ,9
+flaxdiff-0.1.34.dist-info/RECORD,,