py-cmdtabs 1.2.0__tar.gz → 1.2.1__tar.gz

{py_cmdtabs-1.2.0 → py_cmdtabs-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py_cmdtabs
-Version: 1.2.0
+Version: 1.2.1
 Summary: Set of tools to facilitate parsing of tabulated files
 Home-page: https://github.com/seoanezonjic/py_cmdtabs
 Author: seoanezonjic

{py_cmdtabs-1.2.0 → py_cmdtabs-1.2.1}/setup.cfg

@@ -39,6 +39,8 @@ testing =
 
 [options.entry_points]
 console_scripts = 
+	cmdtabs = py_cmdtabs.cli_manager:cmdtabs
+	cmdtabs_merge = py_cmdtabs.cli_manager:cmdtabs_merge
 	aggregate_column_data = py_cmdtabs.cli_manager:aggregate_column_data
 	column_filter = py_cmdtabs.cli_manager:column_filter
 	create_metric_table = py_cmdtabs.cli_manager:create_metric_table

py_cmdtabs-1.2.1/src/py_cmdtabs/__init__.py

@@ -0,0 +1,16 @@
+import sys
+
+if sys.version_info[:2] >= (3, 8):
+    # TODO: Import directly (no need for conditional) when `python_requires = >= 3.8`
+    from importlib.metadata import PackageNotFoundError, version  # pragma: no cover
+else:
+    from importlib_metadata import PackageNotFoundError, version  # pragma: no cover
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = __name__
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError

{py_cmdtabs-1.2.0 → py_cmdtabs-1.2.1}/src/py_cmdtabs/cli_manager.py

@@ -1,16 +1,14 @@
-import argparse
-import sys
-import os
-import codecs
-
-from py_cmdtabs.cmdtabs import CmdTabs
+import argparse, sys, os, codecs
 from py_cmdtabs.main_modules import *
 
 ## TYPES
 def based_0(string): return int(string) - 1
-def list_based_0(string): return CmdTabs.parse_column_indices(",", string)
+def list_based_0(string): 
+    from py_cmdtabs.cmdtabs import CmdTabs
+    return CmdTabs.parse_column_indices(string.split(','))
 def list_str(values): return values.split(',')
 def unescaped_str(arg_str): return codecs.decode(str(arg_str), 'unicode_escape') 
+def nested_int_list(arg_str): return [ [int(col) - 1 for col in str_cols.split(',')] for str_cols in  arg_str.split(';')]
 
 ## Common options
 def add_common_options(parser, flags_to_skip = [], help_replacer={}):
@@ -82,7 +80,7 @@ def create_metric_table(args=None):
     add_common_options(parser, flags_to_skip=["--input_file"])
     parser.add_argument("metric_file", metavar='M',
       help="File with tabulated metrics")
-    parser.add_argument("attributes", metavar='A',
+    parser.add_argument("attributes", metavar='A', type=list_str,
       help="String with comma separated attributes")
     parser.add_argument("output_file", metavar='O',
       help="Output file path")
@@ -110,10 +108,10 @@ def excel_to_tabular(args=None):
     add_common_options(parser, flags_to_skip=["--compressed_in"], help_replacer={"--input_file": "Input xlsx file"})
     parser.add_argument("-o", "--output_file", dest="output_file",
       help="Path to output file")
-    parser.add_argument("-c", "--columns2extract", dest="columns2extract", default=[0], type=list_based_0,
-      help="Column position to extract (1 based). Default 1. Use 0 to extract all columns")
-    parser.add_argument("-r", "--rows2extract", dest="rows2extract", default=[-1], type=list_based_0,
-      help="Row positions to extract (1 based). Default 0, which means all rows will be extracted")  
+    parser.add_argument("-c", "--columns2extract", dest="columns2extract", default=[], type=list_based_0,
+      help="Comma-separated column position(s) to extract (1 based). If not used, all columns will be extracted")
+    parser.add_argument("-r", "--rows2extract", dest="rows2extract", default=[], type=list_based_0,
+      help="Comma-separated row position(s) to extract (1 based). If not used, all rows will be extracted")  
     parser.add_argument("-s", "--sheet_number", dest="sheet_number", default=0, type=based_0,
       help="Sheet number to work with. Default 1")
 
@@ -292,4 +290,129 @@ def transpose_table(args=None):
     parser.add_argument("-o", "--output_file", dest="output_file", default=None, 
                         help="Path to output file")
     opts = parser.parse_args(args)
-    main_transpose_table(opts)
+    main_transpose_table(opts)
+
+def cmdtabs(args=None):
+    if args == None: args = sys.argv[1:]
+    parser = argparse.ArgumentParser(description=f'Usage: {os.path.basename(__file__)} [options]')
+    add_common_options(parser)
+    parser.add_argument("-c", "--columns", dest="columns", default=[0], type=list_based_0,
+                        help="Index of columns in base 1 to compare")
+    parser.add_argument("-H", "--header", dest="header", default=False, action='store_true',
+                     	  help="Indicate if files have header")
+    parser.add_argument("-I", "--index_file", dest="index_file", default=None,
+                        help="Path to index file")
+    parser.add_argument("-o", "--output_file", dest="output_file", default=None, 
+                        help="Path to output file")
+    parser.add_argument("-s", "--separator", dest="separator", default="\t", type=unescaped_str,
+                        help="Input table column character separator")
+    parser.add_argument("-t", "--tags", dest="tags", default= [],
+                        help="Strings or files (only first line will be used) sepparated by commas", type=list_str)
+    parser.add_argument("-u", "--remove_untranslated", dest="remove_untranslated", default=False, action='store_true',
+                        help="Activate this flag for outputting the untranslated entries")
+    parser.add_argument("-w", "--whole", dest="whole", default=False, action='store_true',
+                        help="Indicate if you want the whole table and tabular environment to be returned with the table transformed")
+
+    parser.add_argument("--aggregate", dest="aggregate", default=False, action='store_true',
+                        help="Aggregate table columns")
+    parser.add_argument("--agg_ref_col_index", dest="agg_col_index",
+                  	    help="Column index (1 based) to use as reference", type=list_based_0)
+    parser.add_argument("--agg_col", dest="col_aggregate",
+                        help="Column(s) index (1 based) to extract data and join for each id in column index (if more than one, comma separated)", type=list_based_0)
+    parser.add_argument("--agg_mode", dest="agg_mode", default="concatenate",
+                        help="Mode to perform aggregation. Current available: max,min,mean,median,sum,std,var,IQR,PC25,PC75,count & concatenate. Default (concatenate) is string concatenation by defined separator. More than one aggregation mode can be used separated by commas")
+    parser.add_argument("--agg_sep", dest="agg_sep", default=",",
+                        help="Character separator when collapse data")
+    parser.add_argument("--count-cols", dest="count_cols", default=None, type=list_based_0,
+                        help="Index of columns in base 1 to count")
+    parser.add_argument("--desaggregate", dest="desaggregate", default=False, action='store_true',
+                        help="Desaggregate table columns")
+    parser.add_argument("--desagg_col", dest="desagg_col",
+  	                    help="Column index (1 based) to use as reference", type=list_based_0)
+    parser.add_argument("--desagg_sep", dest="desagg_sep", default=",",
+                        help="Character separator when to split string column")
+    parser.add_argument("--excel_cols", dest="excColumns2extract", default=[], type=list_based_0,
+                        help="Comma-separated column position(s) to extract (1 based). If not used, all columns will be extracted")
+    parser.add_argument("--excel_rows", dest="excRows2extract", default=[], type=list_based_0,
+                        help="Comma-separated row position(s) to extract (1 based). If not used, all rows will be extracted")  
+    parser.add_argument("--excel_sheet_number", dest="excSheet_number", default=0, type=based_0,
+                        help="Sheet number to work with. Default 1")
+    parser.add_argument("--file_type", dest="file_type", default='text',
+                        help="Default text. Other options;excel")
+    parser.add_argument("--from", dest="frm", default=0, type=based_0,
+                        help="Column in index file to take reference value. Default 1. Numeration is 1 based")
+    parser.add_argument("--to", dest="to", default=1, type=based_0,
+                        help="Column in index file to take the value that will be used in substitution. Default 2. Numeration is 1 based")
+    parser.add_argument("--latex", dest="to_latex", default=False, action='store_true',
+                        help="Write table in latex code")
+    parser.add_argument("--sample_attributes", dest="sample_attributes", default=[], type=list_str,
+                        help="Define sample atributtes (comma separated list) to colapse a long table in a wide metric table")
+    parser.add_argument("--corrupted", dest="corrupted",
+                        help="File where corrupted metrics are stored")
+    parser.add_argument("--offset", dest="offset", default=[], type=list_str,
+                        help="To subset N rows from table. Indicate as 'start_row,number_row' where start_row es the line to begin the extraction (1 based) and number_row is the amount of lines to extract")
+    parser.add_argument("--split_out", dest="split_out", default=False, action='store_true',
+                        help="Split output table in several files")
+    parser.add_argument("--sp_chunk_size", dest="sp_chunk_size", default= 0, type=int,
+                        help="To split the ouput table in chunks on K lines in different files.")
+    parser.add_argument("--sp_file_number", dest="sp_file_number", default= 0, type=int,
+                        help="To split the output table in N files with the same number of lines.")
+    parser.add_argument("--extract_cols", dest="extract_cols", default=[], type=list_str,
+                        help="Columns to extract from table, comma separated (based 1)")
+    parser.add_argument("--ext_col_match", dest="ext_col_match", type=list_based_0,
+                        help="Select columns where search keywords. Format: x,y,z..")
+    parser.add_argument("--ext_keywords", dest="ext_keywords",
+                        help="Keywords for select rows. Format: key1_col1&key2_col1%%key1_col2&key2_col2")
+    parser.add_argument("--ext_keyword_file", dest="ext_keyword_file",
+                        help="File with one keyword per line. They will be used to search in all the specified columns in --ext_col_match")
+    parser.add_argument("--ext_search_mode", dest="ext_search_mode", default='c', choices=['c', 's'],
+                        help="c for match in every columns set, s some match in some column. Default c")
+    parser.add_argument("--ext_match_mode", dest="ext_match_mode", default='i', choices=['i', 'c'],
+                        help="i string must include the keyword, c for fullmatch. Default i")
+    parser.add_argument("--ext_reverse", dest="ext_reverse", default=False, action='store_true',
+                        help="Select not matching")
+    parser.add_argument("--ext_stats", dest="ext_stats", default=False, action='store_true',
+                        help="Print row extraction statistics")                        
+    parser.add_argument("--uniq", dest="uniq", default=False, action='store_true',
+                        help="Make rows unique")
+    opts = parser.parse_args(args)
+    main_cmdtabs(opts)
+
+
+def cmdtabs_merge(args=None):
+    if args == None: args = sys.argv[1:]
+    parser = argparse.ArgumentParser(description=f'Usage: {os.path.basename(__file__)} [options]')
+    add_common_options(parser)
+    parser.add_argument("-H", "--header", dest="header", default=False, action='store_true',
+                     	  help="Indicate if files have header")
+    parser.add_argument("-o", "--output_file", dest="output_file", default=None, 
+                        help="Path to output file")
+    parser.add_argument("-s", "--separator", dest="separator", default="\t", type=unescaped_str,
+                        help="Input table column character separator")
+    parser.add_argument("-a", "--a_file", dest="a_file",
+                        help="Path to input file")
+    parser.add_argument("-b", "--b_file", dest="b_file",
+                        help="Path to input file")
+    parser.add_argument("-A", "--a_cols", dest="a_cols", default=[0], type=list_based_0,
+                        help="Index of columns in base 1 to compare")
+    parser.add_argument("-B", "--b_cols", dest="b_cols", default=[0], type=list_based_0,
+                        help="Index of columns in base 1 to compare")
+    parser.add_argument("-c", "--count", dest="count", default=False, action='store_true',
+                        help="Only compute number of matches")
+    parser.add_argument("--full", dest="full", default=False, action='store_true',
+                        help="Give full record")
+    parser.add_argument("-k", "--keep", dest="keep", default='c', choices=['a', 'b', 'c', 'ab'],
+                        help="Keep records. c for common, 'a' for specific of file a, 'b' for specific of file b and 'ab' for specific of file a AND b")
+
+    parser.add_argument("--tables", dest="tables", default=None, type=list_str,
+                        help="Path to tables, comma separated. Paths could include wildcards")
+    parser.add_argument("--fill_character", dest="fill_character", default="-",
+                        help="Character to fill when a field is empty")
+    parser.add_argument("--columns2add", dest="columns2add", default=[], type=nested_int_list,
+                        help="For each file to add to the first file, column indexes 1 based comma separated. To separate each column set, use semicolon ;")
+    parser.add_argument("--union", dest="union", default=False, action='store_true',
+                     	  help="If a row in a file has not match in the merging table is added to it")
+    opts = parser.parse_args(args)
+    main_cmdtabs_merge(opts)
+
+                      

{py_cmdtabs-1.2.0 → py_cmdtabs-1.2.1}/src/py_cmdtabs/cmdtabs.py

@@ -1,25 +1,21 @@
-import sys
-import gzip
-import os
-import warnings
-import copy
+import sys, gzip, os, warnings, copy
 import os.path
-import numpy as np
 from collections import defaultdict
-import openpyxl
-import py_exp_calc.exp_calc as pxc
 
 class CmdTabs:
 	transposed = False
 	compressed_input = False
 	compressed_output = False
 
-	def load_input_data(input_path, sep="\t", limit=-1, first_only=False, add_empty_fields=True):
-		open_file = gzip.open if CmdTabs.compressed_input else open
+	def load_input_data(input_path, sep="\t", limit=-1, first_only=False, add_empty_fields=True, fill_character = "", autodetect_compression=False):
+		is_compressed = CmdTabs.compressed_input
+		if autodetect_compression: is_compressed = input_path.endswith('.gz')
+		
+		open_file = gzip.open if is_compressed else open
 		if limit > 0: # THis is due to ruby compute de cuts in other way and this fix enables ruby mode. Think if adapt to python way
 			limit -= 1
 		if input_path == '-':
-			if CmdTabs.compressed_input: 
+			if is_compressed: 
 				input_data = gzip.decompress(sys.stdin.buffer.read()).decode().strip().split('\n')
 			else: 
 				input_data = sys.stdin
@@ -32,9 +28,10 @@ class CmdTabs:
 			#print('---', file=sys.stderr)
 			#print(repr(), file=sys.stderr)
 			fields = line.rstrip().split(sep, limit)
+			fields = [ fill_character if field == "" else field for field in fields]
 			if add_empty_fields:
 				limit_fields = 0 if limit <= 0 else limit 
-				fields = fields + ( [""] * (fields_number - len(fields) - limit_fields) )
+				fields = fields + ( [fill_character] * (fields_number - len(fields) - limit_fields) )
 			input_data_arr.append(fields)
 			if first_only:
 				break
@@ -54,40 +51,40 @@ class CmdTabs:
 			loaded_files[key_id] = CmdTabs.load_input_data(file, sep, limit, add_empty_fields=add_empty_fields)
 		return loaded_files
 
-	def load_files(files_path): #NO TEST # Cleaning an filling behaviour could be sent to load_input_data as options and use on load_several_files
+	def load_files(files_path, fill_character = '-'): #NO TEST # Cleaning an filling behaviour could be sent to load_input_data as options and use on load_several_files
 		files = {}
 		for file_name in files_path:
 			input_table = CmdTabs.load_input_data(file_name)
 			file = []
 			for fields in input_table:
 				if fields.count('') == len(fields): continue #skip blank records
-				file.append([ '-' if field == "" else field for field in fields])
+				file.append([ fill_character if field == "" else field for field in fields])
 			files[file_name] = file
 		return files
 
 	def build_pattern(col_filter, keywords):
 		pattern = defaultdict(lambda: False	)
 		if col_filter != None and keywords != None:
-			keys_per_col = keywords.split('%')
-			if len(keys_per_col) != len(col_filter): os.abort('Number of keywords not equal to number of filtering columns') 
-			i = 0
-			for col in col_filter:
-					pattern[col] = keys_per_col[i].split('&')
-					i += 1
+			if isinstance(keywords, str): # search keywords from given string as species the % and & characters. % is delimiter of keywrod per column and & us delimiter of keywrods in th same column
+				keys_per_col = keywords.split('%')
+				if len(keys_per_col) != len(col_filter): os.abort('Number of keywords not equal to number of filtering columns') 
+				for i, col in enumerate(col_filter): pattern[col] = keys_per_col[i].split('&')
+			elif isinstance(keywords, list): # search keywords from given list (file) in any column
+				for i, col in enumerate(col_filter): pattern[col] = keywords
 		return pattern
 
-	def index_array(array, col_from=0, col_to=1, header=False):
-		indexed_array = defaultdict(lambda: False)
+	def index_array(table, col_from=0, col_to=1, header=False):
+		index = defaultdict(lambda: False)
 		if type(col_to) != list:
 			f = lambda x: x[col_to]
 		else: 
 			f = lambda x: [x[c] for c in col_to]
 		if header: 
-			head = array.pop(0)
-			indexed_array['header'] = f(head)
-		for elements in array:
-			indexed_array[elements[col_from]] = f(elements)
-		return indexed_array
+			head = table.pop(0)
+			index['header'] = f(head)
+		for row in table:
+			index[row[col_from]] = f(row)
+		return index
 
 	def index_metrics(input_data, attributes):
 		n_attrib = len(attributes)
@@ -115,16 +112,43 @@ class CmdTabs:
 				query[metric_name] = metric
 		return metric_names, indexed_metrics
 
-	def parse_column_indices(sep, col_string):
+	def parse_column_indices(col_str_idxs, has_header= False, table=None):
+		if has_header: 
+			col_str_idxs = CmdTabs.get_name_to_index_equivalences(col_str_idxs, table[0], idx_offset=1)
+	
 		cols = []
-		for col in col_string.split(sep):
+		for col in col_str_idxs:
 			if "-" in col: # Range of columns (both ends included)
 				start, end = col.split("-")
-				range_cols = [int(i) - 1 for i in range(int(start), int(end) + 1)]
+				range_cols = [i - 1 for i in range(int(start), int(end) + 1)]
 				cols.extend(range_cols)
 			else: # Single column
 				cols.append(int(col) - 1)
 		return cols
+	
+	def get_name_to_index_equivalences(string_list_to_convert, referece_string_list, idx_offset=1):
+		cols_header_dict = dict([(col_name, str(idx+idx_offset)) for idx, col_name in enumerate(referece_string_list)])
+		cols_to_get_processed = []
+		try:
+			for col in string_list_to_convert:
+				if col.isdigit(): #Single numeric column case: If the column specifier is a number, we leave as it is
+					cols_to_get_processed.append(col)
+				elif "-" in col and not "%-%" in col: #Range of numeric column case or named column with hyphen
+					start_col, end_col = col.split("-")
+					if start_col.isdigit() and end_col.isdigit(): #We check if the column specifier is a range of numeric columns, if so we leave as it is
+						cols_to_get_processed.append(col)
+					else: #This is an edge of a named column with a hyphen in the name but not numeric range, we convert to number
+						cols_to_get_processed.append(cols_header_dict[col])
+				elif "%-%" not in col: # Single named column case: This is the case of a single column name, we convert to number
+					cols_to_get_processed.append(cols_header_dict[col])
+				else: # Range of named columns case: This is the case of a range of column names, we convert to numbers and keep the range format
+					start_col, end_col = col.split("%-%")
+					start_col = cols_header_dict[start_col]
+					end_col = cols_header_dict[end_col]
+					cols_to_get_processed.append(f"{start_col}-{end_col}")
+		except KeyError as e:
+			raise KeyError(f"Column '{e.args[0]}' not found in header. Available columns: {', '.join(cols_header_dict.keys())}")
+		return cols_to_get_processed		
 
 	def load_and_parse_tags(tags, sep):
 		parsed_tags = []
@@ -146,6 +170,7 @@ class CmdTabs:
 
 
 	def aggregate_column(input_table, col_index, cols_agg, sep, agg_mode="concatenate"):
+		import py_exp_calc.exp_calc as pxc
 		aggregated_data = defaultdict(lambda: False	)
 		if type(cols_agg) == int: cols_agg = [cols_agg]
 		if type(col_index) == int: col_index = [col_index]
@@ -161,6 +186,7 @@ class CmdTabs:
 		return aggregated_data_arr
 	
 	def _make_one_or_several_aggregation(aggregated_column, aggregation_modes, sep):
+		import numpy as np
 		make_aggregation = {"concatenate": lambda agg_col: sep.join(agg_col), 
 					  		"mean": np.mean, "median": np.median, "max": np.max, "min": np.min, "sum": np.sum, "std": np.std, "var": np.var,
 					  		"count": lambda agg_col: len(agg_col), "IQR": lambda agg_col: np.percentile(agg_col, 75) - np.percentile(agg_col, 25),
@@ -251,6 +277,34 @@ class CmdTabs:
 				if not drop_line: linked_table.append(fields) 
 		return linked_table
 
+	def merge_tables2mainTab(main_table, table_paths, column_idxs, sep = "\t", fill_character='-', header = False, union= False):
+		table_length = len(main_table[0])
+		for i, table_path in enumerate(table_paths):
+			supp_file = CmdTabs.load_input_data(table_path, sep=sep)
+			id_col, data_cols = column_idxs[i]
+			supp_file_idx = CmdTabs.index_array(supp_file, id_col, data_cols, header = header)
+			for i,row in enumerate(main_table):
+				if i == 0 and header: # if we're in first row an is header, search for the header record in the index
+					id = 'header'
+				else:
+					id = row[0]
+				supp_data = supp_file_idx.get(id)
+				if supp_data:
+					if type(supp_data) == list:# list with several row fields
+						row.extend(supp_data)
+					else:
+						row.append(supp_data) # one single field as one single value
+				CmdTabs.row_start_fill(row, fill_character, table_length) # When there is no match, fill the gap
+			if union: # Add rows present in supp file but not in main_table
+				for item in (set(supp_file_idx.keys()) - set([r[0] for r in main_table])):
+					new_row = [item]
+					CmdTabs.row_start_fill(new_row, fill_character, table_length)
+					new_row.extend(supp_file_idx[item])
+					main_table.append(new_row)
+			table_length += len(supp_file_idx[list(supp_file_idx.keys())[0]])
+			
+		for row in main_table: CmdTabs.row_end_fill(row, fill_character, table_length) # When there is no match, fill the gap
+
 	def tag_file(input_file, tags, header):
 		taged_file = []
 		n_row = 0
@@ -264,23 +318,23 @@ class CmdTabs:
 			n_row += 1
 		return taged_file
 
-	def filter(line, all_patterns, search_mode, match_mode, reverse = False	):
-		filter = False	
+	def match_pattern(line, all_patterns, search_mode, match_mode, reverse = False	):
+		match_pat = False	
 		for col, patterns in all_patterns.items():
 			is_match = False	
 			for pattern in patterns:
 				is_match = CmdTabs.expanded_match(line[col], pattern, match_mode)
 				if is_match: break	  
 			if is_match and search_mode == 's':
-				filter = False	
+				match_pat = False	
 				break
 			elif not is_match and search_mode == 'c':
-				filter = True
+				match_pat = True
 				break
 			elif not is_match:
-				filter = True
-		if reverse: filter = not filter
-		return filter
+				match_pat = True
+		if reverse: match_pat = not match_pat
+		return match_pat
 
 	def expanded_match(string, pattern, match_mode):
 		is_match = False
@@ -288,16 +342,16 @@ class CmdTabs:
 		if string == pattern and match_mode == 'c': is_match = True 
 		return is_match
 
-	def filter_columns(input_table, options):
-		pattern = CmdTabs.build_pattern(options['col_filter'], options['keywords'])
+	def filter_columns(input_table, cols_to_show, cols_to_match = None, keywords = None, search_mode = None, match_mode = None, reverse = False):
+		pattern = CmdTabs.build_pattern(cols_to_match, keywords)
 		filtered_table = []
 		for line in input_table:
-			if not pattern or not CmdTabs.filter(line, pattern, options['search_mode'], options['match_mode'], options['reverse']):
-				filtered_table.append(CmdTabs.extract_fields(line, options['cols_to_show']) )
+			if not pattern or not CmdTabs.match_pattern(line, pattern, search_mode, match_mode, reverse):
+				filtered_table.append(CmdTabs.extract_fields(line, cols_to_show) )
 		return filtered_table
 
 	def extract_fields(arr_sub, indexes):
-		if indexes == [-1]:
+		if indexes == []:
 			return arr_sub
 		else:
 			return [ str(arr_sub[idx]) for idx in indexes] # The str instruction is used to ensure that always we have string data (i.e: when this function is used with excel objecb it could extrac numerical data)
@@ -310,23 +364,38 @@ class CmdTabs:
 			for fields in file:
 				id = fields.pop(0) 
 				local_length = len(fields)
-				if not parent_table.get(id):
-					parent_table[id] = [fill_character] * table_length
-				elif len(parent_table[id]) < table_length:
-					diference = table_length - len(parent_table[id])
-					parent_table[id].extend( [fill_character] * diference)
+				CmdTabs.row_start_fill_dict(parent_table, id, fill_character, table_length)
 				parent_table[id].extend(fields)				
 			table_length += local_length
-			
-		parent_table_arr = []
+
+		parent_table_arr = [] # Fill rows that have not full length (the sum of the row lengths of the all merged tables)
 		for id, fields in parent_table.items():
-			diference = table_length - len(fields)
-			if diference > 0: fields.extend([fill_character] * diference)
+			CmdTabs.row_end_fill(fields, fill_character, table_length)
 			record = [id]
 			record.extend(fields)
 			parent_table_arr.append(record)
 		return parent_table_arr
 
+	def row_start_fill_dict(indexed_table, row_id, fill_character, table_length):
+		if not indexed_table.get(row_id):
+			indexed_table[row_id] = [fill_character] * table_length
+		elif len(indexed_table[row_id]) < table_length:
+			diference = table_length - len(indexed_table[row_id])
+			indexed_table[row_id].extend( [fill_character] * diference)
+
+	def row_start_fill(row, fill_character, table_length):
+		n_fields = len(row)
+		if n_fields == 0:
+			row = [fill_character] * table_length
+		elif n_fields < table_length:
+			diference = table_length - n_fields
+			row.extend( [fill_character] * diference)
+		return row
+
+	def row_end_fill(row, fill_character, table_length):
+		diference = table_length - len(row)
+		if diference > 0: row.extend([fill_character] * diference)
+
 	def merge_and_filter_tables(input_files, options):
 		header = []
 		filtered_table = []
@@ -337,7 +406,8 @@ class CmdTabs:
 					header = file.pop(0)
 				else:
 					file.pop(0)
-			fields = CmdTabs.filter_columns(file, options)
+			
+			fields = CmdTabs.filter_columns(file, options['cols_to_show'], options['col_filter'], options['keywords'], options['search_mode'], options['match_mode'], options['reverse'])
 			filtered_table.extend(fields) 
 		if options.get('uniq') != None and options['uniq']: filtered_table = CmdTabs.get_uniq(filtered_table)
 		if len(header) > 0:
@@ -374,12 +444,13 @@ class CmdTabs:
 		return storage
 	
 	def extract_rows(table, rows2extract):
-		if rows2extract == [-1]:
+		if rows2extract == []:
 			return table
 		else:
 			return [ row for idx, row in enumerate(table) if idx in rows2extract ]
 
 	def get_table_from_excel(file, sheet_number):
+		import openpyxl
 		x = openpyxl.load_workbook(file)
 		sheets = x.sheetnames # list excel sheets by name
 		ws = x[sheets[sheet_number]] #select sheet by index (so, we select by sheet order)
@@ -406,25 +477,23 @@ class CmdTabs:
 		common = [[r] for r in common_set]
 		return common, a_only, b_only
 
-	def write_output_data(output_data, output_path=None, sep="\t"):
-		open_file = gzip.open if CmdTabs.compressed_output else open
-		if CmdTabs.transposed:
-			output_data = CmdTabs.transpose(output_data)
-			
-		if output_path != None:
-			with open_file(output_path, 'wt') as out_file:
-				for line in output_data:
-					out_file.write(sep.join([str(l) for l in line]) + "\n")
-		else:
-			if CmdTabs.compressed_output:
-				columns_joined = []
-				for line in output_data:
-					columns_joined.append(sep.join([str(l) for l in line]))
-				all_joined = "\n".join(columns_joined) + "\n"
-				sys.stdout.buffer.write(gzip.compress(bytes(all_joined, 'utf-8')))
-			else:
-				for line in output_data:
-					print(sep.join([str(l) for l in line]))
+	def get_subset(common, a_only, b_only, full_a_rec, full_b_rec, keep_subset='c', full=False):
+      # As the groups are list with nested list with only one element: [['str1'], ['str2']..] the full mode need to access to 0 element to be use as key in full_X_rec
+		if keep_subset == 'c':
+			result = common
+			if full: result = [full_a_rec[r[0]] + full_b_rec[r[0]] for r in common]
+		elif keep_subset == 'a':
+			result = a_only
+			if full: result = [full_a_rec[r[0]] for r in a_only]
+		elif keep_subset == 'b':
+			result = b_only
+			if full: result = [full_b_rec[r[0]] for r in b_only]
+		elif keep_subset == 'ab':
+			if full:
+				a_only = [full_a_rec[r[0]] for r in a_only]
+				b_only = [full_b_rec[r[0]] for r in b_only]
+			result = a_only + b_only
+		return result
 
 	def transpose(table):
 		transposed_table = list(map(list, zip(*table)))
@@ -454,4 +523,40 @@ class CmdTabs:
 			final_table += [f"\\end{{tabular}}\n\\label{{table:{name}}}\n\\end{{table}}"]
 		else:
 			final_table = latex_table
-		return final_table
+		return final_table
+
+	def write_output_data(output_data, output_path=None, sep="\t"):
+		open_file = gzip.open if CmdTabs.compressed_output else open
+		if CmdTabs.transposed:
+			output_data = CmdTabs.transpose(output_data)
+			
+		if output_path != None:
+			with open_file(output_path, 'wt') as out_file:
+				for line in output_data:
+					out_file.write(sep.join([str(l) for l in line]) + "\n")
+		else:
+			if CmdTabs.compressed_output:
+				columns_joined = []
+				for line in output_data:
+					columns_joined.append(sep.join([str(l) for l in line]))
+				all_joined = "\n".join(columns_joined) + "\n"
+				sys.stdout.buffer.write(gzip.compress(bytes(all_joined, 'utf-8')))
+			else:
+				for line in output_data:
+					print(sep.join([str(l) for l in line]))
+
+	def split_by_nFiles(input_table, file_number, output_folder, file_name='table', header = []):
+		chunk_size = len(input_table) // file_number
+		init_line = 1 if len(header) > 0 else 0
+		if len(input_table) % file_number > 0: chunk_size += 1
+		chunk_counter = 0
+		for idx in range(init_line, len(input_table), chunk_size):
+			CmdTabs.write_output_data(header+input_table[idx:idx+chunk_size], os.path.join(output_folder, f"{file_name}_chunk{chunk_counter}"))
+			chunk_counter += 1
+
+	def split_by_chunk(input_table, chunk_size, output_folder, file_name='table', header = []):
+		init_line = 1 if len(header) > 0 else 0
+		chunk_counter = 0
+		for idx in range(init_line, len(input_table), chunk_size):
+			CmdTabs.write_output_data(header+input_table[idx:idx+chunk_size], os.path.join(output_folder, f"{file_name}_chunk{chunk_counter}"))
+			chunk_counter += 1