opex-manifest-generator 1.1.9__py3-none-any.whl → 1.1.11__py3-none-any.whl

opex_manifest_generator/cli.py

@@ -13,24 +13,66 @@ import importlib.metadata
 
 def parse_args():
     parser = argparse.ArgumentParser(description = "OPEX Manifest Generator for Preservica Uploads")
-    parser.add_argument('root', default = os.getcwd())
-    parser.add_argument("-c", "--autoclass", required = False, choices = ['catalog', 'c', 'accession', 'a', 'both', 'b', 'generic', 'g', 'catalog-generic', 'cg', "accession-generic", "ag", "both-generic", "bg"], type = str.lower)
-    parser.add_argument("-p", "--prefix", required = False, nargs = '+')
-    parser.add_argument("-fx", "--fixity", required = False, const = "SHA-1", default = None, nargs = '?', choices = ['NONE', 'SHA-1', 'MD5', 'SHA-256', 'SHA-512'], type = str.upper)
-    parser.add_argument("-rme", "--remove-empty", required = False, action = 'store_true', default = False)
-    parser.add_argument("-o", "--output", required = False, nargs = 1)
-    parser.add_argument("-clr", "--clear-opex", required = False, action = 'store_true', default = False)
-    parser.add_argument("-s", "--start-ref", required = False, nargs = '?', default = 1)
-    parser.add_argument("-m", "--metadata", required = False, const = 'e', default = 'none', nargs = '?', choices = ['none', 'n', 'exact', 'e', 'flat', 'f'], type = str.lower)
-    parser.add_argument("-dmd", "--disable-meta-dir", required = False, action = 'store_false')
-    parser.add_argument("-ex", "--export", required = False, action = 'store_true', default = False)
-    parser.add_argument("-i", "--input", required = False)
-    parser.add_argument("-rm", "--remove", required = False, action = "store_true", default = False)
-    parser.add_argument("-z", "--zip", required = False, action = 'store_true')
-    parser.add_argument("-fmt", "--output-format", required = False, default = "xlsx", choices = ['xlsx', 'csv'])
+    parser.add_argument('root', default = os.getcwd(), help = "The root path to generate Opexes for")
+    parser.add_argument("-c", "--autoclass", required = False,
+                        choices = ['catalog', 'c', 'accession', 'a', 'both', 'b', 'generic', 'g', 'catalog-generic', 'cg', "accession-generic", "ag", "both-generic", "bg"],
+                        type = str.lower,
+                        help="""Toggles whether to utilise the auto_classification_generator 
+                        to generate an on the fly Reference listing.
+                        
+                        There are several options, {catalog} will generate
+                        a Archival Reference following an ISAD(G) sturcutre.
+                        {accession} will create a running number of files.
+                        {both} will do both at the same time!
+                        {generic} will populate the title and description fields with the folder/file's name,
+                        if used in conjunction with one of the above options:
+                        {generic-catalog,generic-accession, generic-both} it will do both simultaneously.
+                        """)
+    parser.add_argument("-p", "--prefix", required = False, nargs = '+',
+                        help= """Assign a prefix when utilising the --autoclass option. Prefix will append any text before all generated text.
+                        When utilising the {both} option fill in like: [catalog-prefix, accession-prefix] without square brackets.                        
+                        """)
+    parser.add_argument("-fx", "--fixity", required = False, const = "SHA-1", default = None,
+                        nargs = '?', choices = ['NONE', 'SHA-1', 'MD5', 'SHA-256', 'SHA-512'], type = str.upper,
+                        help="Generates a hash for each file and adds it to the opex, can select the algorithm to utilise.")
+    parser.add_argument("-rme", "--remove-empty", required = False, action = 'store_true', default = False,
+                        help = "Remove and log empty directories from root. Log will be exported to 'meta' / output folder.")
+    parser.add_argument("-o", "--output", required = False, nargs = 1,
+                        help = "Sets the output to send any generated files to. Will not affect creation of a meta dir.")
+    parser.add_argument("-dmd", "--disable-meta-dir", required = False, action = 'store_false',
+                        help = """Set whether to disable the creation of a 'meta' directory for generated files,
+                        default behaviour is to always generate this directory""")
+    parser.add_argument("-clr", "--clear-opex", required = False, action = 'store_true', default = False,
+                        help = """Clears existing opex files from a directory. If set with no further options will only clear opexes; 
+                        if multiple options are set will clear opexes and then run the program""")
+    parser.add_argument("-opt","--options-file", required = False, default=os.path.join(os.path.dirname(__file__),'options.properties'),
+                        help="Specify a custom Options file, changing the set presets for column headers (Title,Description,etc)")
+    parser.add_argument("-s", "--start-ref", required = False, nargs = '?', default = 1, 
+                        help="Set a custom Starting reference for the Auto Classification generator. The generated reference will")
+    parser.add_argument("-mdir","--metadata-dir", required=False, nargs= '?',
+                        default = os.path.join(os.path.dirname(os.path.realpath(__file__)), "metadata"),
+                        help="Specify the metadata directory to pull XML files from")
+    parser.add_argument("-m", "--metadata", required = False, const = 'e', default = 'none',
+                        nargs = '?', choices = ['none', 'n', 'exact', 'e', 'flat', 'f'], type = str.lower,
+                        help="Set whether to include xml metadata fields in the generation of the Opex")
+    parser.add_argument("-ex", "--export", required = False, action = 'store_true', default = False,
+                        help="Set whether to export the generated auto classification references to an AutoClass spreadsheet")
+    parser.add_argument("-i", "--input", required = False, nargs='?', 
+                        help="Set to utilise a CSV / XLSX spreadsheet to import data from")
+    parser.add_argument("-rm", "--remove", required = False, action = "store_true", default = False,
+                        help="Set whether to enable removals of files and folders from a directory. ***Currently in testing")
+    parser.add_argument("-z", "--zip", required = False, action = 'store_true',
+                        help="Set to zip files")
+    parser.add_argument("-fmt", "--output-format", required = False, default = "xlsx", choices = ['xlsx', 'csv'],
+                        help="Set whether to output to an xlsx or csv format")
     parser.add_argument("-v", "--version", action = 'version', version = '%(prog)s {version}'.format(version = importlib.metadata.version("opex_manifest_generator")))
-    parser.add_argument("--hidden", required = False, action = 'store_true', default = False)
-    parser.add_argument("--print-xmls", required = False, action = "store_true", default = False)
+    parser.add_argument("--accession-mode", required=False, choices=["file",'directory','both'],
+                        help="""Set the mode when utilising the Accession option in autoclass.
+                        file - only adds on files, folder - only adds on folders, both - adds on files and folders""")
+    parser.add_argument("--hidden", required = False, action = 'store_true', default = False,
+                        help="Set whether to include hidden files and folders")
+    parser.add_argument("--print-xmls", required = False, action = "store_true", default = False,
+                        help="Prints the elements from your xmls to the consoles")
     args = parser.parse_args()
     return args
 
@@ -51,11 +93,6 @@ def run_cli():
         time.sleep(5)
     if args.print_xmls:
         OpexManifestGenerator.print_descriptive_xmls()
-    if args.autoclass:
-        pass
-        # if not args.prefix:
-        #     print('A prefix must be set when using Auto-Classification, stopping operation')
-        #     time.sleep(3); raise SystemExit()
     acc_prefix = None
     if args.prefix:
         if args.autoclass in {"both", "b", "both-generic", "bg"}:
@@ -113,7 +150,11 @@ def run_cli():
                           export_flag = args.export, 
                           meta_dir_flag = args.disable_meta_dir, 
                           metadata_flag = args.metadata,
+                          metadata_dir = args.metadata_dir,
                           hidden_flag= args.hidden,
                           zip_flag = args.zip, 
                           input = args.input, 
                           output_format = args.output_format).main()
+    
+if __name__ == "__main__":
+    run_cli()

opex_manifest_generator/common.py

@@ -5,7 +5,9 @@ author: Christopher Prince
 license: Apache License 2.0"
 """
 
-import zipfile, os, sys
+import zipfile, os, sys, time, stat
+import datetime
+from lxml import etree
 
 def zip_opex(file_path,opex_path):
     zip_file = f"{file_path}.zip"
@@ -19,4 +21,38 @@ def win_256_check(path: str):
     if len(path) > 255 and sys.platform == "win32":
         if path.startswith(u'\\\\?\\'): path = path
         else: path = u"\\\\?\\" + path
-    return path
+    return path
+
+def filter_win_hidden(path: str):
+    if sys.platform =="win32":
+        if bool(os.stat(path).st_file_attribute & stat.FILE_ATTRIBUTE_HIDDEN) is True:
+            return True
+        else:
+            return False
+    else:
+        return False
+
+def check_nan(value):
+    if str(value).lower() in {"nan","nat"}:
+        value = None
+    return value
+
+def check_opex(opex_path:str):
+    opex_path = opex_path + ".opex" 
+    if os.path.exists(win_256_check(opex_path)):
+        return False
+    else: 
+        return True
+
+def write_opex(path: str, opexxml: etree.Element):
+    opex_path = win_256_check(str(path) + ".opex")
+    opex = etree.indent(opexxml, "  ")
+    opex = etree.tostring(opexxml, pretty_print=True, xml_declaration=True, encoding="UTF-8", standalone=True)
+    with open(f'{opex_path}', 'w', encoding="UTF-8") as writer:
+        writer.write(opex.decode('UTF-8'))
+        print('Saved Opex File to: ' + opex_path)
+    return opex_path
+
+def print_running_time(start_time):
+    print(f'Running time: {datetime.datetime.now() - start_time}')
+    time.sleep(5)

opex_manifest_generator/opex_manifest.py

@@ -8,24 +8,22 @@ license: Apache License 2.0"
 """
 
 import lxml.etree as ET
-import os
+import os, time, shutil
 from auto_classification_generator import ClassificationGenerator
 from auto_classification_generator.common import export_list_txt, export_xl, export_csv, define_output_file
-from datetime import datetime
-import time
+import datetime
 import pandas as pd
 from pandas.api.types import is_datetime64_any_dtype
 from opex_manifest_generator.hash import HashGenerator
 from opex_manifest_generator.common import *
-import stat
-import shutil
+import configparser
 
 class OpexManifestGenerator():
     def __init__(self,
                  root: str,
                  output_path: os.path = os.getcwd(),
                  meta_dir_flag: bool = True,
-                 metadata_dir: bool = os.path.join(os.path.dirname(os.path.realpath(__file__)), "metadata"),
+                 metadata_dir: os = os.path.join(os.path.dirname(os.path.realpath(__file__)), "metadata"),
                  metadata_flag: str = 'none',
                  autoclass_flag: str = None,
                  prefix: str = None,
@@ -40,13 +38,14 @@ class OpexManifestGenerator():
                  zip_flag: bool = False,
                  hidden_flag: bool = False,
                  output_format: str = "xlsx",
-                 print_xmls_flag: bool = False):
+                 print_xmls_flag: bool = False,
+                 options_file: str = os.path.join(os.path.dirname(__file__),'options.properties')):
         
         self.root = os.path.abspath(root)
         self.opexns = "http://www.openpreservationexchange.org/opex/v1.2"        
         self.list_path = []
         self.list_fixity = []
-        self.start_time = datetime.now()
+        self.start_time = datetime.datetime.now()
         self.algorithm = algorithm
         self.empty_flag = empty_flag
         self.remove_flag = remove_flag
@@ -59,52 +58,131 @@ class OpexManifestGenerator():
         self.prefix = prefix
         self.acc_prefix = acc_prefix
         self.input = input
-        self.title_flag = False
-        self.description_flag = False
-        self.security_flag = False
-        self.ignore_flag = False
-        self.sourceid_flag = False
-        self.hash_from_spread = False        
         self.hidden_flag = hidden_flag
         self.zip_flag = zip_flag
         self.output_format = output_format
         self.metadata_flag = metadata_flag
         self.metadata_dir = metadata_dir
         self.print_xmls_flag = print_xmls_flag
-        
-    def print_running_time(self):
-        print(f'Running time: {datetime.now() - self.start_time}')
-        time.sleep(1)
+        self.parse_config(options_file=os.path.abspath(options_file))
+
+        self.title_flag = False
+        self.description_flag = False
+        self.security_flag = False
+        self.ignore_flag = False
+        self.sourceid_flag = False
+        self.hash_from_spread = False        
+    
+    def parse_config(self, options_file: str = 'options.properties'):
+        config = configparser.ConfigParser()
+        config.read(options_file, encoding='utf-8')
+        global INDEX_FIELD
+        INDEX_FIELD = config['options']['INDEX_FIELD']
+        global TITLE_FIELD
+        TITLE_FIELD = config['options']['TITLE_FIELD']
+        global DESCRIPTION_FIELD
+        DESCRIPTION_FIELD = config['options']['DESCRIPTION_FIELD']
+        global SECUIRTY_FIELD
+        SECUIRTY_FIELD = config['options']['SECUIRTY_FIELD']
+        global IDENTIFIER_FIELD
+        IDENTIFIER_FIELD = config['options']['IDENTIFIER_FIELD']
+        global IDENTIFIER_DEFAULT
+        IDENTIFIER_DEFAULT = config['options']['IDENTIFIER_DEFAULT']
+        global REMOVAL_FIELD
+        REMOVAL_FIELD = config['options']['REMOVAL_FIELD']
+        global IGNORE_FIELD
+        IGNORE_FIELD = config['options']['IGNORE_FIELD']
+        global SOURCEID_FIELD
+        SOURCEID_FIELD = config['options']['SOURCEID_FIELD']
+        global HASH_FIELD
+        HASH_FIELD = config['options']['HASH_FIELD']
+        global ALGORITHM_FIELD
+        ALGORITHM_FIELD = config['options']['ALGORITHM_FIELD']
+
+    def print_descriptive_xmls(self):
+        for file in os.scandir(self.metadata_dir):
+            path = os.path.join(self.metadata_dir, file.name)
+            print(path)
+            xml_file = ET.parse(path)
+            root_element = ET.QName(xml_file.find('.'))
+            root_element_ln = root_element.localname
+            for elem in xml_file.findall(".//"):
+                elem_path = xml_file.getelementpath(elem)
+                elem = ET.QName(elem)
+                elem_lnpath = elem_path.replace(f"{{{elem.namespace}}}", root_element_ln + ":")
+                print(elem_lnpath)
+    
+    def set_input_flags(self):
+        if 'Title' in self.column_headers:
+            self.title_flag = True
+        if 'Description' in self.column_headers:
+            self.description_flag = True
+        if 'Security' in self.column_headers:
+            self.security_flag = True
+        if 'SourceID' in self.column_headers:
+            self.sourceid_flag = True
+        if 'Ignore' in self.column_headers:
+            self.ignore_flag = True
+        if 'Hash' in self.column_headers and 'Algorithm' in self.column_headers:
+            self.hash_from_spread = True
+            print("Hash detected in Spreadsheet; taking hashes from spreadsheet")
+            time.sleep(3)
+
+    def init_df(self):
+        if self.autoclass_flag:
+            if self.autoclass_flag in {"catalog", "c", "catalog-generic", "cg"}:
+                ac = ClassificationGenerator(self.root, output_path = self.output_path, prefix = self.prefix, start_ref = self.startref, empty_flag = self.empty_flag, accession_flag = False)
+            elif self.autoclass_flag in {"accession", "a", "accession-generic", "ag", "both", "b", "both-generic", "bg"}:
+                ac = ClassificationGenerator(self.root, output_path = self.output_path, prefix = self.prefix, accprefix = self.acc_prefix, start_ref = self.startref, empty_flag = self.empty_flag, accession_flag="File")
+            self.df = ac.init_dataframe()
+            if self.autoclass_flag in {"accession", "a", "accesion-generic", "ag"}:
+                self.df = self.df.drop('Archive_Reference', axis=1)
+            self.column_headers = self.df.columns.values.tolist()
+            self.set_input_flags()
+            if self.export_flag:
+                output_path = define_output_file(self.output_path, self.root, meta_dir_flag = self.meta_dir_flag, output_format = self.output_format)                
+                if self.output_format == "xlsx":
+                    export_xl(self.df, output_path)
+                elif self.output_format == "csv":
+                    export_csv(self.df, output_path)
+        elif self.input:
+            if self.input.endswith('xlsx'):
+                self.df = pd.read_excel(self.input)
+            elif self.input.endswith('csv'):
+                self.df = pd.read_csv(self.input)
+            self.column_headers = self.df.columns.values.tolist()
+            self.set_input_flags()
+        else:
+            self.df = None
+            self.column_headers = None
+                
+    def clear_opex(self):
+        walk = list(os.walk(self.root))
+        for dir, _, files in walk[::-1]:
+            for file in files:
+                file_path = win_256_check(os.path.join(dir, file))   
+                if str(file_path).endswith('.opex'):
+                    os.remove(file_path)
+                    print(f'Cleared Opex: {file_path}')
     
     def index_df_lookup(self, path: str):
-        idx = self.df['FullName'].index[self.df['FullName'] == path]
+        idx = self.df[INDEX_FIELD].index[self.df[INDEX_FIELD] == path]
         return idx
 
-    def meta_df_lookup(self, idx: pd.Index):
+    def xip_df_lookup(self, idx: pd.Index):
         try:
+            title = None
+            description = None
+            security = None
             if idx.empty:
-                title = None
-                description = None
-                security = None
+                pass
             else:
-                    if self.title_flag:
-                        title = self.df['Title'].loc[idx].item()
-                        if str(title).lower() in {"nan","nat"}:
-                            title = None
-                    else:
-                        title = None
-                    if self.description_flag:
-                        description = self.df['Description'].loc[idx].item()
-                        if str(description).lower() in {"nan","nat"}:
-                            description = None
-                    else:
-                        description = None
-                    if self.security_flag:
-                        security = self.df['Security'].loc[idx].item()
-                        if str(security).lower() in {"nan","nat"}:
-                            security = None
-                    else:
-                        security = None
+                if self.title_flag:
+                    title = check_nan(self.df[TITLE_FIELD].loc[idx].item())
+                if self.description_flag:
+                    description = check_nan(self.df[DESCRIPTION_FIELD].loc[idx].item())
+                if self.security_flag:
+                    security = check_nan(self.df[SECUIRTY_FIELD].loc[idx].item())
             return title,description,security
         except Exception as e:
             print('Error Looking up XIP Metadata')
@@ -115,10 +193,8 @@ class OpexManifestGenerator():
             if idx.empty:
                 return False
             else:
-                remove = self.df['Removals'].loc[idx].item()
-                if str(remove).lower() in {"nan","nat"}:
-                    return False
-                elif bool(remove):
+                remove = check_nan(self.df[REMOVAL_FIELD].loc[idx].item())
+                if remove:                                  
                     print(f"Removing: {path}")
                     # Not functioning correctly
                     if os.path.isdir(path):
@@ -137,15 +213,10 @@ class OpexManifestGenerator():
             if idx.empty:
                 return False
             else:
-                ignore = self.df['Ignore'].loc[idx].item()
-            if str(ignore).lower() in {"nan","nat"}:
-                return False
-            elif str(ignore).lower() in {"true", "1.0"}:
-                return True
-            elif str(ignore).lower() in {"false", "0.0"}:
-                return False
+                ignore = check_nan(self.df[IGNORE_FIELD].loc[idx].item())
+            return bool(ignore)
         except Exception as e:
-            print('Error looking up Removals')
+            print('Error looking up Ignore')
             print(e)
 
     def sourceid_df_lookup(self, xml_element: ET.SubElement, idx: pd.Index):
@@ -153,149 +224,54 @@ class OpexManifestGenerator():
             if idx.empty:
                 pass
             else:
-                sourceid = self.df['SourceID'].loc[idx].item()
-                if str(sourceid) in {"nan","nat"}:
-                    pass
-                else:
+                sourceid = check_nan(self.df[SOURCEID_FIELD].loc[idx].item())
+                if sourceid:
                     source_xml = ET.SubElement(xml_element,f"{{{self.opexns}}}SourceID")
                     source_xml.text = str(sourceid)
         except Exception as e:
             print('Error looking up SourceID')
             print(e)
 
-    def hash_df_lookup(self, file_path: str, xml_fixities: ET.SubElement, idx: pd.Index):
+    def hash_df_lookup(self, xml_fixities: ET.SubElement, idx: pd.Index):
         try:
             if idx.empty:
                 pass
             else:
                 self.fixity = ET.SubElement(xml_fixities,f"{{{self.opexns}}}Fixity")  
-                self.hash = self.df["Hash"].loc[idx].item()
-                self.algorithm = self.df["Algorithm"].loc[idx].item()
-                #self.hash = HashGenerator(algorithm=self.algorithm).hash_generator(file_path)
-                self.fixity.set("type", self.algorithm)
-                self.fixity.set("value",self.hash)
+                self.hash = self.df[HASH_FIELD].loc[idx].item()
+                self.algorithm = self.df[ALGORITHM_FIELD].loc[idx].item()
+                self.fixity.set('type', self.algorithm)
+                self.fixity.set('value',self.hash)
         except Exception as e:
-            print('Error looking up Removals')
+            print('Error looking up Hash')
             print(e)
 
-    def ident_df_lookup(self, idx: pd.Index, key_override: str = None):
+    def ident_df_lookup(self, idx: pd.Index, default_key: str = None):
         try:
             if idx.empty:
-                ident = "ERROR"
-                self.identifier = ET.SubElement(self.identifiers,f"{{{self.opexns}}}Identifier") 
-                if key_override is None:
-                    key_name = "code"
-                else:
-                    key_name = key_override
-                self.identifier.set("type",key_name)
-                self.identifier.text = ident
+                pass
             else:
                 for header in self.column_headers:
-                    if any(s in header for s in {'Identifier','Archive_Reference','Accession_Reference'}):
-                        ident = self.df[header].loc[idx].item()
-                        if 'Identifier:' in header:
-                            key_name = str(header).rsplit(':')[-1]
-                        elif key_override is None:
-                            if 'Archive_Reference' in header:
-                                key_name = "code"
-                            elif 'Accession_Reference' in header:
-                                key_name = "accref"
-                            elif 'Identifier' in header:
-                                key_name = "code"
+                    ident = None
+                    if any(s in header for s in {IDENTIFIER_FIELD,'Archive_Reference','Accession_Reference'}):
+                        if f'{IDENTIFIER_FIELD}:' in header:
+                            key_name = str(header).split(':',1)[-1]
+                        elif IDENTIFIER_FIELD in header:
+                            key_name = IDENTIFIER_DEFAULT    
+                        elif 'Archive_Reference' in header:
+                            key_name = IDENTIFIER_DEFAULT
+                        elif 'Accession_Reference' in header:
+                            key_name = "accref"
                         else:
-                            key_name = key_override
-                    else:
-                        ident = None
-                    if str(ident).lower() in {"nan", "nat"} or not ident:
-                        pass
-                    else:
-                        self.identifier = ET.SubElement(self.identifiers, f"{{{self.opexns}}}Identifier") 
-                        self.identifier.set("type", key_name)
-                        self.identifier.text = str(ident)
+                            key_name = IDENTIFIER_DEFAULT
+                        ident = check_nan(self.df[header].loc[idx].item())                    
+                        if ident:
+                            self.identifier = ET.SubElement(self.identifiers, f"{{{self.opexns}}}Identifier") 
+                            self.identifier.set("type", key_name)
+                            self.identifier.text = str(ident)
         except Exception as e:
             print('Error looking up Identifiers')
             print(e)            
-    
-    def check_opex(self, opex_path: str):
-        opex_path = opex_path + ".opex" 
-        if os.path.exists(win_256_check(opex_path)):
-            return False
-        else:
-            return True
-
-    def write_opex(self, path: str, opexxml: ET.Element):
-        opex_path = win_256_check(str(path) + ".opex")
-        opex = ET.indent(opexxml, "  ")
-        opex = ET.tostring(opexxml, pretty_print=True, xml_declaration=True, encoding="UTF-8", standalone=True)
-        with open(f'{opex_path}', 'w', encoding="UTF-8") as writer:
-            writer.write(opex.decode('UTF-8'))
-            print('Saved Opex File to: ' + opex_path)
-        return opex_path
-
-    def init_df(self):
-        if self.autoclass_flag:
-            if self.autoclass_flag in {"catalog", "c", "catalog-generic", "cg"}:
-                ac = ClassificationGenerator(self.root, output_path = self.output_path, prefix = self.prefix, start_ref = self.startref, empty_flag = self.empty_flag, accession_flag = False)
-            elif self.autoclass_flag in {"accession", "a", "accession-generic", "ag", "both", "b", "both-generic", "bg"}:
-                ac = ClassificationGenerator(self.root, output_path = self.output_path, prefix = self.prefix, accprefix = self.acc_prefix, start_ref = self.startref, empty_flag = self.empty_flag, accession_flag="File")
-            self.df = ac.init_dataframe()
-            if self.autoclass_flag in {"accession", "a", "accesion-generic", "ag"}:
-                self.df = self.df.drop('Archive_Reference', axis=1)
-            self.column_headers = self.df.columns.values.tolist()
-            if self.export_flag:
-                output_path = define_output_file(self.output_path, self.root, meta_dir_flag = self.meta_dir_flag, output_format = self.output_format)                
-                if self.output_format == "xlsx":
-                    export_xl(self.df, output_path)
-                elif self.output_format == "csv":
-                    export_csv(self.df, output_path)
-        elif self.input:
-            if self.input.endswith('xlsx'):
-                self.df = pd.read_excel(self.input)
-            elif self.input.endswith('csv'):
-                self.df = pd.read_csv(self.input)
-            self.column_headers = self.df.columns.values.tolist()
-            self.set_input_flags()
-        else:
-            self.df = None
-            self.column_headers = None
-                
-    def clear_opex(self):
-        walk = list(os.walk(self.root))
-        for dir, _, files in walk[::-1]:
-            for file in files:
-                file_path = win_256_check(os.path.join(dir, file))   
-                if str(file_path).endswith('.opex'):
-                    os.remove(file_path)
-                    print(f'Cleared Opex: {file_path}')
-    
-    def set_input_flags(self):
-        if 'Title' in self.column_headers:
-            self.title_flag = True
-        if 'Description' in self.column_headers:
-            self.description_flag = True
-        if 'Security' in self.column_headers:
-            self.security_flag = True
-        if 'SourceID' in self.column_headers:
-            self.sourceid_flag = True
-        if 'Ignore' in self.column_headers:
-            self.ignore_flag = True
-        if 'Hash' in self.column_headers and 'Algorithm' in self.column_headers:
-            self.hash_from_spread = True
-            print("Hash detected in Spreadsheet; taking hashes from spreadsheet")
-            time.sleep(3)
-
-    def print_descriptive_xmls(self):
-        for file in os.scandir(self.metadata_dir):
-            path = os.path.join(self.metadata_dir, file.name)
-            print(path)
-            xml_file = ET.parse(path)
-            root_element = ET.QName(xml_file.find('.'))
-            root_element_ln = root_element.localname
-            for elem in xml_file.findall(".//"):
-                elem_path = xml_file.getelementpath(elem)
-                elem = ET.QName(elem)
-                elem_lnpath = elem_path.replace(f"{{{elem.namespace}}}", root_element_ln + ":")
-                print(elem_lnpath)
 
     def init_generate_descriptive_metadata(self):
         self.xml_files = []
@@ -329,15 +305,19 @@ class OpexManifestGenerator():
             if len(list_xml) > 0:
                 self.xml_files.append({'data': list_xml, 'localname': root_element_ln, 'xmlfile': path})
 
-    def generate_descriptive_metadata(self, xml_desc: ET.Element, idx: int):
+    def generate_descriptive_metadata(self, xml_desc_elem: ET.Element, idx: pd.Index):
+        """
+        Composes the data into an xml file.
+        """
         for xml_file in self.xml_files:
             list_xml = xml_file.get('data')
             localname = xml_file.get('localname')
-            """
-            Composes the data into an xml file.
-            """
-            if len(list_xml):
-                if not idx.empty:
+            if len(list_xml) == 0:
+                pass
+            else:
+                if idx.empty:
+                    pass
+                else:
                     xml_new = ET.parse(xml_file.get('xmlfile'))
                     for elem_dict in list_xml:
                         name = elem_dict.get('Name')
@@ -345,21 +325,28 @@ class OpexManifestGenerator():
                         ns = elem_dict.get('Namespace')
                         try:
                             if self.metadata_flag in {'e', 'exact'}:
-                                val = self.df.loc[idx, path].values[0]
+                                val = check_nan(self.df[path].loc[idx].item())
                             elif self.metadata_flag in {'f', 'flat'}:
-                                val = self.df.loc[idx, name].values[0]
-                            if pd.isnull(val):
+                                val = check_nan(self.df[name].loc[idx].item())
+                            if val is None:
                                 continue
                             else:
                                 if is_datetime64_any_dtype(str(val)):
                                     val = pd.to_datetime(val)
-                                    val = datetime.strftime(val, "%Y-%m-%dT%H-%M-%S.00Z")
+                                    val = datetime.datetime.strftime(val, "%Y-%m-%dT%H:%M:%S.000Z")
+                            if self.metadata_flag in {'e','exact'}:
+                                n = path.replace(localname + ":", f"{{{ns}}}")
+                                elem = xml_new.find(f'/{n}')
+                            elif self.metadata_flag in {'f', 'flat'}:
+                                n = name.split(':')[-1]
+                                elem = xml_new.find(f'//{{{ns}}}{n}')
+                            elem.text = str(val)
                         except KeyError as e:
                             print('Key Error: please ensure column header\'s are an exact match...')
                             print(f'Missing Column: {e}')
                             print('Alternatively use flat mode...')
                             time.sleep(3)
-                            raise SystemError()
+                            raise SystemExit()
                         except IndexError as e:
                             print("""Index Error; it is likely you have removed or added a file/folder to the directory \
                                 after generating the spreadsheet. An opex will still be generated but with no xml metadata. \
@@ -367,25 +354,11 @@ class OpexManifestGenerator():
                             print(f'Error: {e}')
                             time.sleep(3)
                             break
-                        if str(val).lower() in {"nan", "nat"}:
-                            continue
-                        if self.metadata_flag in {'e','exact'}:
-                            n = path.replace(localname + ":", f"{{{ns}}}")
-                            elem = xml_new.find(f'/{n}')
-                        elif self.metadata_flag in {'f', 'flat'}:
-                            n = name.split(':')[-1]
-                            elem = xml_new.find(f'//{{{ns}}}{n}')
-                        elem.text = str(val)    
-                    xml_desc.append(xml_new.find('.'))
-                else:
-                    pass
-            else:
-                pass
+                    xml_desc_elem.append(xml_new.find('.'))
 
-    def generate_opex_properties(self, xmlroot: ET.Element, idx: int, 
-                                 title: str = None, description: str = None, security: str = None):
+    def generate_opex_properties(self, xmlroot: ET.Element, idx: int, title: str = None,
+                                  description: str = None, security: str = None):
         self.properties = ET.SubElement(xmlroot, f"{{{self.opexns}}}Properties")
-        self.identifiers = ET.SubElement(self.properties, f"{{{self.opexns}}}Identifiers")
         if title:
             self.titlexml = ET.SubElement(self.properties, f"{{{self.opexns}}}Title")
             self.titlexml.text = str(title)
@@ -395,12 +368,9 @@ class OpexManifestGenerator():
         if security:
             self.securityxml = ET.SubElement(self.properties, f"{{{self.opexns}}}SecurityDescriptor")
             self.securityxml.text = str(security)
-        if self.autoclass_flag in {"generic", "g"}:
-            self.properties.remove(self.identifiers)
-        elif self.autoclass_flag not in {"generic", "g"} or self.input:
+        if self.autoclass_flag not in {"generic", "g"} or self.input:
+            self.identifiers = ET.SubElement(self.properties, f"{{{self.opexns}}}Identifiers")
             self.ident_df_lookup(idx)
-        if self.identifiers is None:
-            self.properties.remove(self.identifiers)
         if self.properties is None:
             xmlroot.remove(self.properties)
 
@@ -423,7 +393,7 @@ class OpexManifestGenerator():
             if self.autoclass_flag or self.algorithm or self.input:
                 pass
             else:
-                self.print_running_time()
+                print_running_time(self.start_time)
                 print('Cleared OPEXES. No additional arguments passed, so ending program.'); time.sleep(3)
                 raise SystemExit()
         if self.empty_flag:
@@ -437,7 +407,7 @@ class OpexManifestGenerator():
         if self.algorithm:
             output_path = define_output_file(self.output_path, self.root, self.meta_dir_flag, output_suffix = "_Fixities", output_format = "txt")
             export_list_txt(self.list_fixity, output_path)
-        self.print_running_time()
+        print_running_time(self.start_time)
 
 class OpexDir(OpexManifestGenerator):
     def __init__(self, OMG: OpexManifestGenerator, folder_path: str, title: str = None, description: str = None, security: str = None):
@@ -448,19 +418,17 @@ class OpexDir(OpexManifestGenerator):
             self.folder_path = folder_path.replace(u'\\\\?\\', "")
         else:
             self.folder_path = folder_path
-        if self.OMG.input or self.OMG.autoclass_flag in {"c", "catalog", "a", "accession", "b", "both", "cg", "catalog-generic", "ag", "accession-generic", "bg", "both-generic"} \
-             or self.OMG.ignore_flag or self.OMG.remove_flag or self.OMG.sourceid_flag \
-             or self.OMG.title_flag or self.OMG.description_flag or self.OMG.security_flag:
-                self.index = self.OMG.index_df_lookup(self.folder_path)
-        elif self.OMG.autoclass_flag is None or self.OMG.autoclass_flag in {"g", "generic"}:
-            self.index = None
+        if self.OMG.autoclass_flag not in {None, "g","generic"}:
+            index = self.OMG.index_df_lookup(self.folder_path)
+        else:
+            index = None            
         if self.OMG.ignore_flag or self.OMG.remove_flag:
             if self.OMG.ignore_flag:
-                self.ignore = self.OMG.ignore_df_lookup(self.index)
+                self.ignore = self.OMG.ignore_df_lookup(index)
                 if self.ignore:
                     return
             if self.OMG.remove_flag:
-                self.removal = self.OMG.remove_df_lookup(self.folder_path, self.index)
+                self.removal = self.OMG.remove_df_lookup(self.folder_path, index)
                 if self.removal:
                     return
         else:
@@ -474,7 +442,7 @@ class OpexDir(OpexManifestGenerator):
         self.files = ET.SubElement(self.manifest, f"{{{self.opexns}}}Files")
 
         if self.OMG.title_flag or self.OMG.description_flag or self.OMG.security_flag:
-            self.title, self.description, self.security = self.OMG.meta_df_lookup(self.index) 
+            self.title, self.description, self.security = self.OMG.xip_df_lookup(index) 
         elif self.OMG.autoclass_flag in {"generic", "g", "catalog-generic", "cg", "accession-generic", "ag", "both-generic", "bg"}:
             self.title = os.path.basename(self.folder_path)
             self.description = os.path.basename(self.folder_path)
@@ -484,29 +452,36 @@ class OpexDir(OpexManifestGenerator):
             self.description = description
             self.security = security
         if self.OMG.sourceid_flag:
-            self.OMG.sourceid_df_lookup(self.transfer, self.folder_path, self.index)
+            self.OMG.sourceid_df_lookup(self.transfer, self.folder_path, index)
         if self.OMG.autoclass_flag or self.OMG.input:
-            self.OMG.generate_opex_properties(self.xmlroot, self.index, 
+            self.OMG.generate_opex_properties(self.xmlroot, index, 
                                               title = self.title,
                                               description = self.description,
                                               security = self.security)
             if not self.OMG.metadata_flag in {'none', 'n'}:
                 self.xml_descmeta = ET.SubElement(self.xmlroot,f"{{{self.opexns}}}DescriptiveMetadata")
-                self.OMG.generate_descriptive_metadata(self.xmlroot, idx = self.index)
+                self.OMG.generate_descriptive_metadata(self.xmlroot, idx = index)
 
-    def filter_directories(self, directory: str):
-        if self.OMG.hidden_flag is False:
-            list_directories = sorted([win_256_check(os.path.join(directory, f.name)) for f in os.scandir(directory)
-                                       if not f.name.startswith('.')
-                                       and not bool(os.stat(win_256_check(os.path.join(directory, f.name))).st_file_attributes & stat.FILE_ATTRIBUTE_HIDDEN)
-                                       and f.name != 'meta'
-                                       and f.name != os.path.basename(__file__)]
-                                       , key=str.casefold)
-        elif self.OMG.hidden_flag is True:
-            list_directories = sorted([os.path.join(directory, f.name) for f in os.scandir(directory) \
-                                       if f.name != 'meta' \
-                                       and f.name != os.path.basename(__file__)], key=str.casefold)
-        return list_directories
+    def filter_directories(self, directory: str, sort_key: str = str.casefold):
+        try:
+            if self.OMG.hidden_flag is False:
+                list_directories = sorted([win_256_check(os.path.join(directory, f.name)) for f in os.scandir(directory)
+                                        if not f.name.startswith('.')
+                                        and filter_win_hidden(win_256_check(os.path.join(directory, f.name))) is False
+                                        and f.name != 'meta'
+                                        and f.name != os.path.basename(__file__)],
+                                        key=sort_key)
+            elif self.OMG.hidden_flag is True:
+                list_directories = sorted([os.path.join(directory, f.name) for f in os.scandir(directory) \
+                                        if f.name != 'meta'
+                                        and f.name != os.path.basename(__file__)],
+                                        key=sort_key)
+            return list_directories
+        except Exception as e:
+            print('Failed to Filter')
+            print(e)
+            raise SystemError()
+    
         
     def generate_opex_dirs(self, path: str):
         self = OpexDir(self.OMG, path)
@@ -521,7 +496,7 @@ class OpexDir(OpexManifestGenerator):
                 self.generate_opex_dirs(f_path)
             else:
                 OpexFile(self.OMG, f_path, self.OMG.algorithm)
-        if self.OMG.check_opex(opex_path):
+        if check_opex(opex_path):
             if not self.ignore:
                 for f_path in self.filter_directories(path):
                     if os.path.isfile(f_path):
@@ -532,7 +507,7 @@ class OpexDir(OpexManifestGenerator):
                             file.set("type", "content")
                             file.set("size", str(os.path.getsize(f_path)))
                         file.text = str(os.path.basename(f_path))
-                self.OMG.write_opex(opex_path, self.xmlroot)
+                write_opex(opex_path, self.xmlroot)
         else:
             print(f"Avoiding override, Opex exists at: {opex_path}")
 
@@ -544,26 +519,32 @@ class OpexFile(OpexManifestGenerator):
             self.file_path = file_path.replace(u'\\\\?\\', "")
         else:
             self.file_path = file_path
-        if self.OMG.check_opex(self.file_path):
-            if self.OMG.input or self.OMG.autoclass_flag in {"c","catalog","a","accession","b","both","cg","catalog-generic","ag","accession-generic","bg","both-generic"} \
-                or self.OMG.ignore_flag or self.OMG.remove_flag or self.OMG.sourceid_flag \
-                or self.OMG.title_flag or self.OMG.description_flag or self.OMG.security_flag:
-                    self.index = self.OMG.index_df_lookup(self.file_path)
+        if check_opex(self.file_path):
+            if any([self.OMG.input,
+                    self.OMG.autoclass_flag in {"c","catalog","a","accession","b","both","cg","catalog-generic","ag","accession-generic","bg","both-generic"},
+                    self.OMG.ignore_flag,
+                    self.OMG.remove_flag,
+                    self.OMG.sourceid_flag,
+                    self.OMG.title_flag,
+                    self.OMG.description_flag,
+                    self.OMG.security_flag]):
+                    index = self.OMG.index_df_lookup(self.file_path)
             elif self.OMG.autoclass_flag is None or self.OMG.autoclass_flag in {"g","generic"}:
-                self.index = None
+                index = None
             if self.OMG.ignore_flag:
-                self.ignore = self.OMG.ignore_df_lookup(self.index)
+                self.ignore = self.OMG.ignore_df_lookup(index)
                 if self.ignore:
+                    #WTF is this?
                     return
             if self.OMG.remove_flag:
-                removal = self.OMG.remove_df_lookup(self.file_path, self.index)
+                removal = self.OMG.remove_df_lookup(self.file_path, index)
                 if removal:
                     return                
             else:
                 self.ignore = False
             self.algorithm = algorithm
             if self.OMG.title_flag or self.OMG.description_flag or self.OMG.security_flag:
-                self.title, self.description, self.security = self.OMG.meta_df_lookup(self.index) 
+                self.title, self.description, self.security = self.OMG.xip_df_lookup(index) 
             elif self.OMG.autoclass_flag in {"generic", "g", "catalog-generic", "cg", "accession-generic", "ag", "both-generic", "bg"}:
                 self.title = os.path.splitext(os.path.basename(self.file_path))[0]
                 self.description = os.path.splitext(os.path.basename(self.file_path))[0]
@@ -580,20 +561,20 @@ class OpexFile(OpexManifestGenerator):
                 if self.OMG.algorithm:
                     self.fixities = ET.SubElement(self.transfer, f"{{{self.opexns}}}Fixities")
                     if self.OMG.hash_from_spread:
-                        self.OMG.hash_df_lookup(self.file_path, self.fixities, self.index)  
+                        self.OMG.hash_df_lookup(self.fixities, index)  
                     else:
                         self.genererate_opex_fixity(self.file_path)
                 if self.transfer is None:
                     self.xmlroot.remove(self.transfer)
                 if self.OMG.autoclass_flag or self.OMG.input:
-                    self.OMG.generate_opex_properties(self.xmlroot, self.index,
+                    self.OMG.generate_opex_properties(self.xmlroot, index,
                                                       title = self.title,
                                                       description = self.description,
                                                       security = self.security)
                     if not self.OMG.metadata_flag in {'none','n'}:
                         self.xml_descmeta = ET.SubElement(self.xmlroot, f"{{{self.opexns}}}DescriptiveMetadata")
-                        self.OMG.generate_descriptive_metadata(self.xml_descmeta, self.index)
-                opex_path = self.OMG.write_opex(self.file_path, self.xmlroot)
+                        self.OMG.generate_descriptive_metadata(self.xml_descmeta, index)
+                opex_path = write_opex(self.file_path, self.xmlroot)
                 if self.OMG.zip_flag:
                     zip_opex(self.file_path, opex_path)
         else:

opex_manifest_generator/options.properties

@@ -0,0 +1,13 @@
+[options]
+
+INDEX_FIELD = "FullName"
+TITLE_FIELD = "Title"
+DESCRIPTION_FIELD = "Description"
+SECUIRTY_FIELD = "Security"
+IDENTIFIER_FIELD = "Identifier"
+IDENTIFIER_DEFAULT = "code"
+REMOVAL_FIELD = "Removals"
+IGNORE_FIELD = "Ignore"
+SOURCEID_FIELD = "SourceID"
+HASH_FIELD = "Hash"
+ALGORITHM_FIELD = "Algorithm"

{opex_manifest_generator-1.1.9.dist-info → opex_manifest_generator-1.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opex_manifest_generator
-Version: 1.1.9
+Version: 1.1.11
 Summary: Opex Manifest Generator Tool for use with Opex / Preservica
 Author-email: Christopher Prince <c.pj.prince@gmail.com>
 Project-URL: Homepage, https://github.com/CPJPRINCE/opex_manifest_generator

{opex_manifest_generator-1.1.9.dist-info → opex_manifest_generator-1.1.11.dist-info}/RECORD

@@ -1,9 +1,9 @@
 opex_manifest_generator/__init__.py,sha256=fsN-dLjNMn-AfjeG2jkryzIa69bY09B5xLTqWedOv_w,461
-opex_manifest_generator/cli.py,sha256=EC-TtCrrIlesgd1KmilCDjm6hANdfalTGuiQ1R4DKBE,6418
-opex_manifest_generator/common.py,sha256=Xa6aC9lPWpgs9tOvxEvHUr6f6j1xoHn07jF4IWSr1I4,655
+opex_manifest_generator/cli.py,sha256=6-ZbFZQT82aUYA5aNYLV4xteEw-pMgXsQr_yOCDklYM,10121
+opex_manifest_generator/common.py,sha256=dJGRQicA7B9n5WXpG5pJlV0fzO0itF6GyDXWTJ-5mxs,1782
 opex_manifest_generator/hash.py,sha256=mpLP-BqqDC7BuQQ5TEc4jaIBGabW-qVr128JaZq54mY,992
-opex_manifest_generator/opex_manifest.py,sha256=9mFOwJvOer0yMoGdLfKusGFumvBL_GS4lpvQnwoaRGo,30448
-opex_manifest_generator/test_cli.py,sha256=qLjANnpoGhbIBMAE9FTI2aTB7tt_-KiFYT9IxxaAMU8,58
+opex_manifest_generator/opex_manifest.py,sha256=NkvgVR2C5QIJPq3kIcjjAURh3uaO4P2CGpEiIP76N6Q,29209
+opex_manifest_generator/options.properties,sha256=wLa_PjlOjzH1t88edz18wgTFC6F0it8ImPK9xUrQo9g,308
 opex_manifest_generator/metadata/DublinCore Template.xml,sha256=csNGXzSH27Whs4BQNuwMZl8nLSdDq7Y_OblTfzeBqWQ,775
 opex_manifest_generator/metadata/EAD Template.xml,sha256=OsWjUfKiLAsc1zqm56auyFSDYMuZpu6a49AFgqZNzlU,2167
 opex_manifest_generator/metadata/GDPR Template.xml,sha256=r6WTATRVt1sr7VCdaXOwz4vLgU3IgFFx7oRmPnORBWc,475
@@ -14,9 +14,9 @@ opex_manifest_generator/samples/spreads/dctemplate.xlsx,sha256=R33CNcBK3s-KFYEhm
 opex_manifest_generator/samples/spreads/eadtemplate.xlsx,sha256=4lrP0LLZXwv73fl3fvqr7yqpdDJuRj2D4ZADQ4OS6Ps,19299
 opex_manifest_generator/samples/spreads/gdprtemplate.xlsx,sha256=3k6FpN6n83yF5wYd64Yy8Rxv2b1Z497icIB8UBGDW-M,18662
 opex_manifest_generator/samples/spreads/modstemplate.xlsx,sha256=5kNpp4Cju_POvnTrgFk34OJVe5yc6o3R4ZNX2TT8zAc,19509
-opex_manifest_generator-1.1.9.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-opex_manifest_generator-1.1.9.dist-info/METADATA,sha256=EGGFaGs76vTrVgA_8PQXEHQTV4sOD3RAcx_TgB82bBs,686
-opex_manifest_generator-1.1.9.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
-opex_manifest_generator-1.1.9.dist-info/entry_points.txt,sha256=WGMc3hWlqOsQ8DcTuy8-DyBbJKkWNImT4J1FasVDHts,70
-opex_manifest_generator-1.1.9.dist-info/top_level.txt,sha256=K48eGnaDLVO6YDJdAZLqbeoZvJHBGX25cvYT-i8gWt0,24
-opex_manifest_generator-1.1.9.dist-info/RECORD,,
+opex_manifest_generator-1.1.11.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+opex_manifest_generator-1.1.11.dist-info/METADATA,sha256=qYpxKL4BOjXcYip-W6Z_jDA-FXs22O65G9IYTcU6J9I,687
+opex_manifest_generator-1.1.11.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+opex_manifest_generator-1.1.11.dist-info/entry_points.txt,sha256=WGMc3hWlqOsQ8DcTuy8-DyBbJKkWNImT4J1FasVDHts,70
+opex_manifest_generator-1.1.11.dist-info/top_level.txt,sha256=K48eGnaDLVO6YDJdAZLqbeoZvJHBGX25cvYT-i8gWt0,24
+opex_manifest_generator-1.1.11.dist-info/RECORD,,

{opex_manifest_generator-1.1.9.dist-info → opex_manifest_generator-1.1.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.2.0)
+Generator: setuptools (75.5.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

opex_manifest_generator/test_cli.py

@@ -1,2 +0,0 @@
-from opex_manifest_generator.cli import run_cli
-run_cli()