opex-manifest-generator 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

opex_manifest_generator/cli.py

@@ -5,18 +5,21 @@ author: Christopher Prince
 license: Apache License 2.0"
 """
 
-import argparse, os, inspect, time
+import argparse, os, inspect, time, logging
 from opex_manifest_generator.opex_manifest import OpexManifestGenerator
 import importlib.metadata
+from datetime import datetime
+from opex_manifest_generator.common import running_time 
+
+logger = logging.getLogger(__name__)
 
 def parse_args():
     parser = argparse.ArgumentParser(description = "OPEX Manifest Generator for Preservica Uploads")
     parser.add_argument('root', nargs='?', default = os.getcwd(),
                         help = """The root path to generate Opexes for, will recursively traverse all sub-directories.
                         Generates an Opex for each folder & (depending on options) file in the directory tree.""")
-    
     parser.add_argument("-fx", "--fixity", required = False, nargs = '*', default = None,
-                        choices = ['SHA-1', 'MD5', 'SHA-256', 'SHA-512', 'SHA1', 'SHA256', 'SHA512'], type = str.upper, action=EmptyIsTrueFixity,
+                        choices = ['SHA-1', 'MD5', 'SHA-256', 'SHA-512'], type = fixity_helper, action=EmptyIsTrueFixity,
                         help="Generates a hash for each file and adds it to the opex, can select one or more algorithms to utilise. -fx SHA-1 MD5")
     parser.add_argument("--pax-fixity", required = False, action = 'store_true', default = False,
                         help="""Enables use of PAX fixity generation, in line with Preservica's Recommendation.
@@ -24,13 +27,20 @@ def parse_args():
     parser.add_argument("--fixity-export", required = False, action = 'store_false', default = True,
                         help="""Set whether to export the generated fixity list to a text file in the meta directory.
                         Enabled by default, disable with this flag.""")
+    parser.add_argument("--prevent-all-exports", required = False, action = 'store_true', default = False,
+                        help="Set to prevent all exports (Fixity, Removal, Empty) from being created in the meta directory.")
     parser.add_argument("-z", "--zip", required = False, action = 'store_true',
                         help="Set to zip files")
+    parser.add_argument("--zip-remove-files", required = False, action = 'store_true',
+                        help="Set to remove the files that have been zipped")
     parser.add_argument("--remove-empty", required = False, action = 'store_true', default = False,
                         help = "Remove and log empty directories from root. Log will be exported to 'meta' / output folder.")
+    parser.add_argument("--empty-export", required = False, action = 'store_false', default = True,
+                        help="""Set whether to export the generated empty list to a text file in the meta directory.
+                        Enabled by default, disable with this flag.""")
     parser.add_argument("--hidden", required = False, action = 'store_true', default = False,
                         help="Set whether to include hidden files and folders")
-    parser.add_argument("-o", "--output", required = False, nargs = 1,
+    parser.add_argument("-o", "--output", required = False, nargs = '?',
                         help = "Sets the output to send any generated files (Remove Empty, Fixity List, Autoref Export) to. Will not affect creation of a meta dir.")
     parser.add_argument("-clr", "--clear-opex", required = False, action = 'store_true', default = False,
                         help = """Clears existing opex files from a directory. If set with no further options will only clear opexes; 
@@ -47,24 +57,28 @@ def parse_args():
                         help="Set to utilise a CSV / XLSX spreadsheet to import data from")
     parser.add_argument("-rm", "--remove", required = False, action = "store_true", default = False,
                         help="Set whether to enable removals of files and folders from a directory. ***Currently in testing")    
+    parser.add_argument("--removal-export", required = False, action = 'store_false', default = True,
+                        help="""Set whether to export the generated removals list to a text file in the meta directory.
+                        Enabled by default, disable with this flag.""")
     parser.add_argument("-mdir","--metadata-dir", required=False, nargs= '?',
                         default = os.path.join(os.path.dirname(os.path.realpath(__file__)), "metadata"),
                         help="Specify the metadata directory to pull XML files from")
-    parser.add_argument("-m", "--metadata", required = False, const = 'e', default = 'none',
-                        nargs = '?', choices = ['none', 'n', 'exact', 'e', 'flat', 'f'], type = str.lower,
+    parser.add_argument("-m", "--metadata", required = False, const = 'e', default = None,
+                        nargs = '?', choices = ['exact', 'e', 'flat', 'f'], type = str.lower,
                         help="Set whether to include xml metadata fields in the generation of the Opex")
     parser.add_argument("--print-xmls", required = False, action = "store_true", default = False,
                         help="Prints the elements from your xmls to the consoles")
+    parser.add_argument("--convert-xmls", required=False, action ='store_true', default = False,
+                         help="Convert XMLs templates files in mdir to spreadsheets/csv files")
     
     # Auto Reference Options
     parser.add_argument("-r", "--autoref", required = False,
                         choices = ['catalog', 'c', 'accession', 'a', 'both', 'b', 'generic', 'g', 'catalog-generic', 'cg', "accession-generic", "ag", "both-generic", "bg"],
                         type = str.lower,
                         help="""Toggles whether to utilise the auto_reference_generator 
-                        to generate an on the fly Reference listing.
-                        
+                        to generate an on the fly Reference listing.\n
                         There are several options, {catalog} will generate
-                        a Archival Reference following an ISAD(G) structure.
+                        a Archival Reference following an ISAD(G) structure.\n
                         {accession} will create a running number of files.
                         {both} will do both at the same time!
                         {generic} will populate the title and description fields with the folder/file's name,
@@ -82,7 +96,7 @@ def parse_args():
     parser.add_argument("--accession-mode", nargs = '?', required=False, const='file', default=None, choices=["file",'directory','both'],
                         help="""Set the mode when utilising the Accession option in autoref.
                         file - only adds on files, folder - only adds on folders, both - adds on files and folders""")
-    parser.add_argument("-str", "--start-ref", required = False, nargs = '?', default = 1, 
+    parser.add_argument("-str", "--start-ref", required = False, type=int, nargs = '?', default = 1, 
                         help="Set a custom Starting reference for the Auto Reference Generator. The generated reference will")
     parser.add_argument("-ex", "--export-autoref", required = False, action = 'store_true', default = False,
                         help="Set whether to export the generated references to an AutoRef spreadsheet")
@@ -102,61 +116,101 @@ def parse_args():
                         help = "Set to set the number of letters to abbreviate for 'firstletters' mode, does not impact 'initialise' mode.")
     parser.add_argument("--sort-by", required=False, nargs = '?', default = 'folders_first', choices = ['folders_first','alphabetical'], type=str.lower,
                         help = "Set the sorting method, 'folders_first' sorts folders first then files alphabetically; 'alphabetically' sorts alphabetically (ignoring folder distinction)")
-    
+    parser.add_argument("--log-level", required=False, nargs='?', choices=['DEBUG','INFO','WARNING','ERROR'], default=None, type=str.upper,
+                        help="Set the logging level (default: INFO)")
+    parser.add_argument("--log-file", required=False, nargs='?', default=None,
+                        help="Optional path to write logs to a file (default: stdout)")
     parser.add_argument("-v", "--version", action = 'version', version = '%(prog)s {version}'.format(version = importlib.metadata.version("opex_manifest_generator")))
 
     args = parser.parse_args()
     return args
 
 def run_cli():
-    args = parse_args()    
-    print(f"Running Opex Generation on: {args.root}")
+    args = parse_args()
+
+    # Configure logging early so other modules inherit the settings
+    try:
+        log_level = getattr(logging, args.log_level.upper()) if args.log_level else logging.INFO
+    except Exception:
+        log_level = logging.INFO
+    log_format = '%(asctime)s %(levelname)-8s [%(name)s] %(message)s'
+    if args.log_file:
+        logging.basicConfig(level=log_level, filename=args.log_file, filemode='a', format=log_format)
+    else:
+        logging.basicConfig(level=log_level, format=log_format)
+    logger.debug(f'Logging configured (level={logging.getLevelName(log_level)}, file={args.log_file or "stdout"})')
+
+    if not os.path.exists(args.root):
+        logger.error(f'Please ensure that root path {args.root} exists. \n' \
+        'If you are utilising Windows ensure that the path does not end with \\\' or \\"')
+        raise FileNotFoundError(f'Please ensure that root path {args.root} exists. \n' \
+        'If you are utilising Windows ensure that the path does not end with \\\' or \\" ')
+    
     if isinstance(args.root, str):
         args.root = args.root.strip("\"").rstrip("\\")
+    logger.info(f"Running Opex Generation on: {args.root}")
+
     if not args.output:
         args.output = os.path.abspath(args.root)
-        print(f'Output path defaulting to root directory: {args.output}')
+        logger.debug(f'Output path set to root directory: {args.output}')        
     else:
-        args.output = os.path.abspath(args.output[0])
-        print(f'Output path set to: {args.output}')    
+        args.output = os.path.abspath(args.output)
+        logger.info(f'Output path set to {args.output}')
+
     if args.input and args.autoref:
-        print(f'Both Input and Auto ref options have been selected, please use only one...')
-        time.sleep(5); raise SystemExit()
+        logger.error(f'Both Input and Auto ref options have been selected, please use only one...')
+        raise ValueError('Both Input and Auto ref options have been selected, please use only one...')
     if args.remove and not args.input:
-        print('Removal flag has been given without input, please ensure an input file is utilised when using this option.')
-        time.sleep(5); raise SystemExit()
-    if not args.metadata in {'none', 'n'} and not args.input:
-        print(f'Warning: Metadata Flag has been given without Input. Metadata won\'t be generated.')
-        time.sleep(5)
+        logger.error('Removal flag has been given without input, please ensure an input file is utilised when using this option.')
+        raise ValueError('Removal flag has been given without input, please ensure an input file is utilised when using this option.')
+    if args.metadata is not None and not args.input:
+        logger.warning(f'Warning: Metadata Flag has been given without Input. Metadata won\'t be generated.')
+  
     if args.print_xmls:
+        logger.info(f'Printing xmls in {args.metadata_dir} then ending')
         OpexManifestGenerator(root = args.root, metadata_dir=args.metadata_dir).print_descriptive_xmls()
+        raise SystemExit()
+    if args.convert_xmls:
+        logger.info(f'Converting XMLs in {args.metadata_dir} then ending')
+        OpexManifestGenerator(root = args.root, output_format=args.output_format, metadata_dir=args.metadata_dir).convert_descriptive_xmls()
+        raise SystemExit()
+
+    if args.prevent_all_exports:
+        args.fixity_export_flag = False
+        args.removal_export_flag = False
+        args.empty_export_flag = False
+        logger.info('All exports have been prevented via --prevent-all-exports flag.')
+
     acc_prefix = None
     if args.autoref in {"accession", "a", "accession-generic", "ag", "both", "b", "both-generic", "bg"} and args.accession_mode is None:
             args.accession_mode = "file"
+            logger.debug(f'Accession mode not set, defaulting to "file" mode for accession generation.')
+
     if args.prefix:
         if args.autoref in {"both", "b", "both-generic", "bg"}:
             if len(args.prefix) < 2 or len(args.prefix) > 2:
-                print('"Both" option is selected, please pass only two prefixes: [-p CATALOG_PREFIX ACCESSION_PREFIX]');
-                time.sleep(3); raise SystemExit
+                logger.error('"Both" option is selected, please pass only two prefixes: [-p CATALOG_PREFIX ACCESSION_PREFIX]'); 
+                raise ValueError('"Both" option is selected, please pass only two prefixes: [-p CATALOG_PREFIX ACCESSION_PREFIX]')
             for n, a in enumerate(args.prefix):
                 if n == 0:
                     args.prefix = str(a)
                 elif n == 1:
                     acc_prefix = str(a)
-            print(f"Prefixes are set as: \t Catalog: {args.prefix} \t Acc: {acc_prefix}")
+            logger.info(f"Prefixes are set as: \t Catalog: {args.prefix} \t Acc: {acc_prefix}")
         elif args.autoref in {"accession", "a", "accession-generic", "ag"}:
             for a in args.prefix:
                 acc_prefix = str(a)
-            print('Prefix is set as: ' + acc_prefix)                        
+            logger.info('Prefix is set as: ' + acc_prefix)                        
         elif args.autoref in {"catalog", "c", "catalog-generic", "cg"}:
             acc_prefix = None
             for a in args.prefix: 
                 args.prefix = str(a)
-            print('Prefix is set as: ' + args.prefix)
+            logger.info('Prefix is set as: ' + args.prefix)
         elif args.autoref in {"generic", "g"}:
+            logger.info('Using Generic mode')
             pass
         else:
-            print('''An invalid option has been selected, please select a valid option: 
+            logger.error('''An invalid option has been selected, please select a valid option: 
                   {c, catalog
                    a, accession
                    b, both
@@ -164,27 +218,47 @@ def run_cli():
                    cg, catalog-generic
                    ag, accession-generic
                    bg, both-generic}''')    
-            time.sleep(3)
-            raise SystemExit               
+            raise ValueError('An invalid option has been selected, please select a valid option.')   
+    
     if args.fixity:
-        print(f'Fixity is activated, using {args.fixity} algorithm')
+        logger.info(f'Fixity is activated, using {args.fixity} algorithm')
+
+    sort_key = None
     if args.sort_by:
         if args.sort_by == "folders_first":
+            logger.debug('Sorting by folders_first')
             sort_key = lambda x: (os.path.isfile(x), str.casefold(x))
         elif args.sort_by == "alphabetical":
+            logger.debug('Sorting by alphabetical')
             sort_key = str.casefold
+
     if args.remove:
-        print(inspect.cleandoc("""****
-                                    You have enabled the remove functionality of the program. This action will remove all files and folders listed for removal and any sub-files/sub-folders.
-                            
-                                    This process will permanently delete the selected items, with no way recover the items. 
-                                
-                                    ****"""))
-        time.sleep(2)
+        logger.warning(inspect.cleandoc("\n***WARNING***" \
+                                "\nYou have enabled the remove functionality of the program. " \
+                                "This action will remove all files and folders listed for removal and any sub-files/sub-folders." \
+                                "\nThis process will permanently delete the selected items, with no way recover the items." \
+                                "\n***"))
+        i = input(inspect.cleandoc("Please type Y if you wish to proceed, otherwise the program will close: "))
+        if not i.lower() == "y":
+            logger.info("Y not typed, safetly aborted...")
+            raise SystemExit()
+        else:
+            logger.info("Confirmation recieved proceeding to remove files")
+
+    if args.remove_empty:
+        logger.warning(inspect.cleandoc("\n***WARNING***" \
+                                "\nYou have enabled the remove empty folders functionality of the program. " \
+                                "This action will remove all empty folders." \
+                                "\nThis process will permanently delete all empty folders, with no way recover the items." \
+                                "\n***"))
         i = input(inspect.cleandoc("Please type Y if you wish to proceed, otherwise the program will close: "))
         if not i.lower() == "y":
-            print("Closing program..."); time.sleep(3); raise SystemExit()
-    time.sleep(3)
+            logger.info("Y not typed, safetly aborted...")
+            raise SystemExit()
+        else:
+            logger.info("Confirmation recieved proceeding to remove empty folders...")
+
+    start_time = datetime.now()
     OpexManifestGenerator(root = args.root, 
                           output_path = args.output, 
                           autoref_flag = args.autoref, 
@@ -192,7 +266,9 @@ def run_cli():
                           accession_mode=args.accession_mode,
                           acc_prefix = acc_prefix, 
                           empty_flag = args.remove_empty, 
+                          empty_export_flag = args.empty_export,
                           removal_flag = args.remove, 
+                          removal_export_flag = args.removal_export,
                           clear_opex_flag = args.clear_opex, 
                           algorithm = args.fixity,
                           pax_fixity= args.pax_fixity,
@@ -204,17 +280,29 @@ def run_cli():
                           metadata_dir = args.metadata_dir,
                           hidden_flag= args.hidden,
                           zip_flag = args.zip, 
+                          zip_file_removal= args.zip_remove_files,
                           input = args.input, 
                           output_format = args.output_format,
                           options_file=args.options_file,
                           keywords = args.keywords,
                           keywords_mode = args.keywords_mode,
                           keywords_retain_order = args.keywords_retain_order,
-                          keywords_case_sensitive = args.keywords_case_sensitivity,
+                          keywords_case_sensitivity = args.keywords_case_sensitivity,
                           delimiter = args.delimiter,
                           keywords_abbreviation_number = args.keywords_abbreviation_number,
                           sort_key = sort_key,
                           ).main()
+    logger.info(f"Run Complete! Ran for: {running_time(start_time)}")    
+
+def fixity_helper(x: str):
+    x = x.upper()
+    if x == 'SHA1':
+        x = 'SHA-1'
+    if x == 'SHA256':
+        x = 'SHA-256'
+    if x == 'SHA512':
+        x = 'SHA-512'
+    return x.upper()
 
 class EmptyIsTrueFixity(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
@@ -223,4 +311,7 @@ class EmptyIsTrueFixity(argparse.Action):
         setattr(namespace, self.dest, values)
 
 if __name__ == "__main__":
-    run_cli()
+    try:
+        run_cli()
+    except KeyboardInterrupt:
+        logger.warning("Process interrupted by user, exiting...")

opex_manifest_generator/common.py

@@ -5,40 +5,51 @@ author: Christopher Prince
 license: Apache License 2.0"
 """
 
-import zipfile, os, sys, time, stat, datetime, shutil
-from lxml import etree
+import zipfile, os, sys, stat, shutil, logging, lxml
+from datetime import datetime, timedelta
 
-def zip_opex(file_path,opex_path):
+logger = logging.getLogger(__name__)
+
+def zip_opex(file_path,opex_path = None) -> str:
     zip_file = f"{file_path}.zip"
     if not os.path.exists(zip_file):
         with zipfile.ZipFile(zip_file,'w') as z:
-            z.write(file_path,os.path.basename(file_path))
-            z.write(opex_path,os.path.basename(opex_path))
-    else: print(f'A zip file already exists for: {zip_file}')
+            if os.path.exists(file_path):
+                z.write(file_path,os.path.basename(file_path))
+            if opex_path is not None and os.path.exists(opex_path):
+                z.write(opex_path,os.path.basename(opex_path))
+        logger.debug(f'File has been zipped to: {zip_file}')
+    else:
+        logger.warning(f'A Zip file already exists for: {zip_file}')
+    return zip_file
 
-def remove_tree(path: str, removed_list: list):
+def remove_tree(path: str, removed_list: list) -> None:
     removed_list.append(path)
-    print(f"Removing: {path}")
+    logger.info(f"Removing: {path}")
     if os.path.isdir(path):
         for dp,d,f in os.walk(path):
             for fn in f:
                 removed_list.append(win_256_check(dp+win_path_delimiter()+fn))
+                logger.info(f'Removing {dp + win_path_delimiter() + fn}')            
             for dn in d:
                 removed_list.append(win_256_check(dp+win_path_delimiter()+dn))
+                logger.info(f'Removing {dp + win_path_delimiter() + dn}')
         shutil.rmtree(path)
     else:
         if os.path.exists(path):
             os.remove(path)
+            logger.info (f'Removing File: {path}')
 
-def win_256_check(path):
+def win_256_check(path) -> str:
     if len(path) > 255 and sys.platform == "win32":
+        logger.debug(f'Path: {path} is greater than 255 Characters')
         if path.startswith(u"\\\\?\\"):
             path = path 
         else:
             path = u"\\\\?\\" + path
     return path
 
-def filter_win_hidden(path: str):
+def filter_win_hidden(path: str) -> bool:
     if sys.platform =="win32":
         if bool(os.stat(path).st_file_attributes & stat.FILE_ATTRIBUTE_HIDDEN) is True:
             return True
@@ -47,33 +58,33 @@ def filter_win_hidden(path: str):
     else:
         return False
     
-def win_path_delimiter():
+def win_path_delimiter() -> str:
     if sys.platform == "win32":
         return "\\"
     else:
         return "/"
 
-def check_nan(value):
+def check_nan(value) -> str | None:
     if str(value).lower() in {"nan","nat"}:
         value = None
     return value
 
-def check_opex(opex_path:str):
+def check_opex(opex_path:str) -> bool:
     opex_path = opex_path + ".opex" 
     if os.path.exists(win_256_check(opex_path)):
         return False
     else: 
         return True
 
-def write_opex(path: str, opexxml: etree.Element):
+def write_opex(path: str, opexxml: lxml.etree.Element) -> str:
     opex_path = win_256_check(str(path) + ".opex")
-    opex = etree.indent(opexxml, "  ")
-    opex = etree.tostring(opexxml, pretty_print=True, xml_declaration=True, encoding="UTF-8", standalone=True)
+    opex = lxml.etree.indent(opexxml, "  ")
+    opex = lxml.etree.tostring(opexxml, pretty_print=True, xml_declaration=True, encoding="UTF-8", standalone=True)
     with open(f'{opex_path}', 'w', encoding="UTF-8") as writer:
         writer.write(opex.decode('UTF-8'))
-        print('Saved Opex File to: ' + opex_path)
+        logger.info('Saved Opex File to: ' + opex_path)
     return opex_path
 
-def print_running_time(start_time):
-    print(f'Running time: {datetime.datetime.now() - start_time}')
-    time.sleep(5)
+def running_time(start_time) -> timedelta:
+    running_time = datetime.now() - start_time 
+    return running_time

opex_manifest_generator/hash.py

@@ -5,9 +5,11 @@ author: Christopher Prince
 license: Apache License 2.0"
 """
 
-import hashlib
+import hashlib, logging
 from opex_manifest_generator.common import win_256_check 
 
+logger = logging.getLogger(__name__)
+
 class HashGenerator():
     def __init__(self, algorithm: str = "SHA-1", buffer: int = 4096):
         self.algorithm = algorithm
@@ -15,17 +17,17 @@ class HashGenerator():
 
     def hash_generator(self, file_path: str):
         file_path = win_256_check(file_path)
-        if self.algorithm in ("SHA-1","SHA1"):
+        if "SHA-1" in self.algorithm:
             hash = hashlib.sha1()
-        elif self.algorithm == "MD5":
+        elif "MD5" in self.algorithm:
             hash = hashlib.md5()
-        elif self.algorithm in ("SHA-256","SHA256"):
+        elif "SHA-256" in self.algorithm:
             hash = hashlib.sha256()
-        elif self.algorithm in ("SHA-512","SHA512"):
+        elif "SHA-512" in self.algorithm:
             hash = hashlib.sha512()
         else:
             hash = hashlib.sha1()
-        print(f'Generating Fixity using {self.algorithm} for: {file_path}')
+        logger.info(f'Generating Fixity using {self.algorithm} for: {file_path}')
         try:
             with open(file_path, 'rb', buffering = 0) as f:
                 while True:
@@ -34,12 +36,19 @@ class HashGenerator():
                         break
                     hash.update(buff)
                 f.close()
+            logger.debug(f'Generated Hash: {hash.hexdigest().upper()}')
+            return hash.hexdigest().upper()
+        except FileNotFoundError as e:
+            logger.exception(f'File Not Found generating Hash: {e}')
+            raise
+        except IOError as e:
+            logger.exception(f'I/O Error generating Hash: {e}')
+            raise
         except Exception as e:
-            print(e)
-            raise SystemError()
-        return hash.hexdigest().upper()
+            logger.exception(f'Error Generating Hash: {e}')
+            raise
         
-    def hash_generator_pax_zip(self, filename, z):
+    def hash_generator_pax_zip(self, filename: str, z):
         if self.algorithm in ("SHA1","SHA-1"):
             hash = hashlib.sha1()
         elif self.algorithm == "MD5":
@@ -50,7 +59,7 @@ class HashGenerator():
             hash = hashlib.sha512()
         else:
             hash = hashlib.sha1()
-        print(f'Generating Fixity using {self.algorithm} for: {filename}')
+        logger.info(f'Generating Fixity using {self.algorithm} for: {filename}')
         try:
             with z.open(filename, 'r') as data:            
                 while True:
@@ -58,7 +67,14 @@ class HashGenerator():
                     if not buff:
                         break
                     hash.update(buff)
+                data.close()
+        except FileNotFoundError as e:
+            logger.exception(f'File Not Found generating Hash: {e}')
+            raise
+        except IOError as e:
+            logger.exception(f'I/O Error generating Hash: {e}')
+            raise
         except Exception as e:
-            print(e)
-            raise SystemError()
-        return hash.hexdigest().upper()
+            logger.exception(f'Error Generating Hash: {e}')
+            raise
+        return str(hash.hexdigest().upper())