lost_ds 1.2.0a4__py3-none-any.whl

lost_ds/__init__.py

@@ -0,0 +1,63 @@
+from lost_ds.geometry.api import LOSTGeometries
+
+from lost_ds.functional.api import(remove_empty,
+                                   split_by_empty,
+                                   split_by_img_path,
+                                   split_train_test,
+                                   split_multilabels,
+                                   remap_labels,
+                                   remap_img_path,
+                                   ignore_labels,
+                                   img_selection,
+                                   is_multilabel,
+                                   label_selection,
+                                   polygon_to_bbox,
+                                   selection_mask,
+                                   to_abs,
+                                   to_rel,
+                                   transform_bbox_style,
+                                   to_coco,
+                                   unique_labels,
+                                   validate_empty_images,
+                                   validate_geometries,
+                                   validate_img_paths,
+                                   validate_unique_annos,
+                                   validate_single_labels)
+
+from lost_ds.cropping.api import (DSCropper,
+                                  crop_anno,
+                                  crop_img,
+                                  crop_dataset,
+                                  crop_components)
+
+from lost_ds.copy import (copy_imgs,
+                          pack_ds)
+
+from lost_ds.im_util import (get_imagesize,
+                             pad_image)
+
+from lost_ds.vis.api import (vis_sample,
+                             vis_and_store,
+                             vis_semantic_segmentation,
+                             draw_polygons,
+                             draw_boxes,
+                             draw_lines,
+                             draw_points,
+                             draw_text)
+
+from lost_ds.segmentation.api import (semantic_segmentation, 
+                                      segmentation_to_lost)
+
+from lost_ds.detection.api import (detection_dataset, bbox_nms, coco_eval, 
+                                   voc_eval, voc_score_iou_multiplex)
+
+from lost_ds.masking.api import mask_dataset 
+
+from lost_ds.util import (get_fs, 
+                          to_parquet)
+
+from lost_ds.core import LOSTDataset
+
+from lost_ds.compat import old_lds_to_new_lds
+
+__version__='1.2.0-alpha.4'

lost_ds/compat.py

@@ -0,0 +1,104 @@
+
+from ast import literal_eval
+import json
+import pandas as pd
+from lost_ds.core import LOSTDataset
+
+
+def old_lds_to_new_lds(df):
+    # load data (unstring)
+    def literal_eval_parse(entry):
+        if not pd.isnull(entry):
+            if isinstance(entry, str):
+                return literal_eval(entry.replace('nan', 'None'))
+            else:
+                return literal_eval(entry)
+        else:
+            return None
+
+    def parse_col(col):
+        try:
+            return col.map(lambda entry: literal_eval_parse(entry))
+        except Exception as e:
+            try:
+                return col.map(lambda entry: json.loads(entry))
+            except Exception as e:
+                return col
+            
+    df = df.apply(lambda x: parse_col(x), axis=0)
+    # map keys
+    mapping = { 'anno.idx': 'anno_uid',
+                'anno.timestamp': 'anno_timestamp',
+                'anno.state': 'anno_state',
+                'anno.dtype':'anno_dtype',
+                'anno.sim_class':'anno_sim_class',
+                'anno.iteration':'anno_iteration',
+                'anno.user_id':'anno_user_id',
+                'anno.user':'anno_user',
+                'anno.confidence':'anno_confidence',
+                'anno.anno_time':'anno_anno_time',
+                'anno.data': 'anno_data',
+                'anno.lbl.name':'anno_lbl',
+                'img.idx': 'img_uid',
+                'img.timestamp': 'img_timestamp',
+                'img.state': 'img_state',
+                'img.sim_class': 'img_sim_class',
+                'img.frame_n': 'img_frame_n',
+                'img.img_path': 'img_path',
+                'img.iteration': 'img_iteration',
+                'img.user_id': 'img_user_id',
+                'img.anno_time': 'img_anno_time',
+                'img.lbl.name': 'img_lbl',
+                'img.annotator': 'img_user',
+                'img.is_junk': 'img_is_junk'
+                }
+    df_map = df.rename(columns=mapping)
+    new_keys = ['anno_style', 'anno_format']
+    df_map[new_keys] = None
+
+    # # transform timestamp data
+    # all_keys = list(mapping.values())
+    # for k in all_keys:
+    #     if 'timestamp' in k:
+    #         df_map[k] = df_map[k].map(lambda x: str(x))
+    
+    # transform anno data
+    def parse_data(row):
+        dtype = row['anno_dtype']
+        data = None
+        style = None
+        frmt = 'rel'
+        if dtype == 'bbox':
+            data = list(row['anno_data'].values())
+            style = 'xcycwh'
+        elif dtype in ['polygon', 'line']:
+            data = [list(p.values()) for p in row['anno_data']]
+            style = 'xy'
+        elif dtype == 'point':
+            data = list(row['anno_data'].values())
+            style = 'xy'
+        return data, style, frmt
+    new_dat = df_map.apply(lambda x: parse_data(x), axis=1)
+    dic = {'anno_data': [],
+           'anno_style': [],
+           'anno_format': []}
+    
+    indexes = []
+    for i, dat in new_dat.iteritems():
+        indexes.append(i)
+        dic['anno_data'].append(dat[0])
+        dic['anno_style'].append(dat[1])
+        dic['anno_format'].append(dat[2])
+    df_dat = pd.DataFrame(dic, index=indexes)
+    df_map[['anno_data', 'anno_style', 'anno_format']] = df_dat[['anno_data', 'anno_style', 'anno_format']]
+    
+    drop_keys = [k for k in list(df_map.keys()) if '.' in k]
+    df_map.drop(labels=drop_keys, axis='columns', inplace=True)
+    
+    def _lbl_parser(lbl):
+        if isinstance(lbl, str): return lbl.lower()
+        else: return lbl
+    df_map['anno_lbl'] = df_map['anno_lbl'].map(lambda x: [_lbl_parser(lbl) for lbl in x] if isinstance(x, list) else x)
+    
+    return LOSTDataset(df_map)
+

lost_ds/copy.py

@@ -0,0 +1,117 @@
+from tqdm import tqdm
+import fsspec
+import os
+from joblib import Parallel, delayed, cpu_count
+from zipfile import ZipFile
+
+from lost_ds.functional.mapping import remap_img_path
+from lost_ds.util import get_fs
+
+
+def copy_imgs(df, out_dir, col='img_path', force_overwrite=False, 
+              filesystem=None, parallel=-1):
+    '''Copy all images of dataset into out_dir
+
+    Args:
+        df (pd.DataFrame): dataframe to copy
+        out_dir (str): Destination folder to store images
+        col (str): column containing paths to files
+        filesystem (fsspec.filesystem, FileMan): filesystem to use. Use local
+            if not initialized
+    '''
+    fs = get_fs(filesystem)
+    def copy_file(src_path, dst_dir):
+        dst_path = os.path.join(dst_dir, os.path.basename(src_path))
+        if fs.exists(dst_path) and not force_overwrite:
+            return
+        fs.copy(src_path, dst_path)
+        
+    fs.makedirs(out_dir, exist_ok=True)
+    img_paths = list(df[col].unique())
+    if parallel:
+        Parallel(n_jobs=parallel)(delayed(copy_file)(path, out_dir) 
+                            for path in tqdm(img_paths, desc='copy imgs'))
+    else:
+        for path in tqdm(img_paths, desc='copy imgs'):
+            copy_file(path, out_dir)
+        
+
+def copy_to_zip(zip_file, df, zip_dir, col='img_path', 
+              filesystem=None, progress_callback=None):
+    '''Copy all images of dataset into zip archive
+
+    Args:
+        df (pd.DataFrame): dataframe to copy
+        zip_root (str): Root path in zip archive
+        col (str): column containing paths to files
+        filesystem (fsspec.filesystem, FileMan): filesystem to use. Use local
+            if not initialized
+        progress_callback (function): Will be called on progress
+            callback definition -> progress_callback(progress), where progress 
+            value is between 0...100 
+    '''
+
+    fs = get_fs(filesystem).fs
+    def copy_file_to_zip(src_path, zip_file):
+        dst_path = os.path.join(zip_dir, os.path.basename(src_path))
+        try:
+            fs.ls('')
+        except:
+            pass
+        with fs.open(src_path, 'rb') as f:
+            zip_file.writestr(dst_path, f.read())
+        
+    img_paths = list(df[col].unique())
+    total = len(img_paths)
+    next_pg = 0
+    for idx, path in enumerate(img_paths):
+        copy_file_to_zip(path, zip_file)
+        if progress_callback is not None:
+            pg = (idx+1) *100 / total
+            if pg == 100:
+                progress_callback(pg)
+            elif pg >= next_pg:
+                progress_callback(pg)
+                next_pg += 5
+    
+def pack_ds(df, out_dir, cols=['img_path', 'mask_path', 'crop_path'],
+            dirs = ['imgs', 'masks', 'crops'], filesystem=None, zip_file=None, 
+            progress_callback=None):
+    '''Copy all images from dataset to a new place and update the dataframe 
+    
+    Args:
+        df (pd.DataFrame): Dataframe to copy
+        out_dir (str): Name of the directory to store the information
+        executor (Client, ThreadPoolExecutor): executor for parallelization
+            if None a new ThreadPoolExecutor will be initialized
+        filesystem (fsspec.filesystem, FileMan): filesystem to use. Use local
+            if not initialized
+        cols (list of string): column names containing file-paths
+        dirs (list of string): name of new directories according to cols. The 
+            dirs will contain the copied data
+        zip_file (zipfile.ZipFile or None): If not None, a ZipFile object will 
+            be used to pack dataset to zip archive  
+        progress_callback (function): Will be called on progress
+            callback definition -> progress_callback(progress), where progress 
+            value is between 0...100 
+        
+    Returns:
+        pd.DataFrame with new image paths
+    '''
+    fs = get_fs(filesystem)
+    for col, _dir in zip(cols, dirs):
+        if col in df.keys():
+            dout = os.path.join(out_dir, _dir)
+            if zip_file is None:
+                if progress_callback is not None:
+                    raise Exception('progress_callback is only implement for packing to zip files yet!')
+                fs.makedirs(dout, exist_ok=True)
+                copy_imgs(df=df, out_dir=dout, col=col, filesystem=fs)
+            else:
+                out_base = os.path.basename(out_dir)
+                out_base = os.path.splitext(out_base)[0]
+                zip_dir = os.path.join(out_base, _dir)
+                copy_to_zip(zip_file, df, zip_dir=zip_dir, col=col, filesystem=fs, 
+                            progress_callback=progress_callback)
+            df = remap_img_path(df, dout, col)
+    return df