opex-manifest-generator 1.1.12__py3-none-any.whl → 1.2.0__py3-none-any.whl

opex_manifest_generator/cli.py

@@ -5,9 +5,7 @@ author: Christopher Prince
 license: Apache License 2.0"
 """
 
-import argparse
-import os
-import time
+import argparse, os, inspect, time
 from opex_manifest_generator.opex_manifest import OpexManifestGenerator
 import importlib.metadata
 
@@ -39,7 +37,7 @@ def parse_args():
                         help = "Remove and log empty directories from root. Log will be exported to 'meta' / output folder.")
     parser.add_argument("-o", "--output", required = False, nargs = 1,
                         help = "Sets the output to send any generated files to. Will not affect creation of a meta dir.")
-    parser.add_argument("-dmd", "--disable-meta-dir", required = False, action = 'store_false',
+    parser.add_argument("--disable-meta-dir", required = False, action = 'store_false',
                         help = """Set whether to disable the creation of a 'meta' directory for generated files,
                         default behaviour is to always generate this directory""")
     parser.add_argument("-clr", "--clear-opex", required = False, action = 'store_true', default = False,
@@ -66,13 +64,19 @@ def parse_args():
     parser.add_argument("-fmt", "--output-format", required = False, default = "xlsx", choices = ['xlsx', 'csv'],
                         help="Set whether to output to an xlsx or csv format")
     parser.add_argument("-v", "--version", action = 'version', version = '%(prog)s {version}'.format(version = importlib.metadata.version("opex_manifest_generator")))
-    parser.add_argument("--accession-mode", required=False, choices=["file",'directory','both'],
+    parser.add_argument("--accession-mode", nargs = '?', required=False, const='file', default=None, choices=["file",'directory','both'],
                         help="""Set the mode when utilising the Accession option in autoclass.
                         file - only adds on files, folder - only adds on folders, both - adds on files and folders""")
     parser.add_argument("--hidden", required = False, action = 'store_true', default = False,
                         help="Set whether to include hidden files and folders")
     parser.add_argument("--print-xmls", required = False, action = "store_true", default = False,
                         help="Prints the elements from your xmls to the consoles")
+    parser.add_argument("-key","--keywords", nargs = '*', default = None)
+    parser.add_argument("-keym","--keywords-mode", nargs = '?', const = "intialise", default = "intialise", choices = ['intialise','firstletters'])
+    parser.add_argument("--keywords-retain-order", required = False, default = False, action = 'store_true')
+    parser.add_argument("--keywords-abbreviation-number", required = False, nargs='?', default = -3, type = int)
+    parser.add_argument("--sort-by", required=False, nargs = '?', default = 'foldersfirst', choices = ['foldersfirst','alphabetical'], type=str.lower)
+    parser.add_argument("-dlm", "--delimiter", required=False,nargs = '?', type = str)
     args = parser.parse_args()
     return args
 
@@ -94,6 +98,8 @@ def run_cli():
     if args.print_xmls:
         OpexManifestGenerator.print_descriptive_xmls()
     acc_prefix = None
+    if args.autoclass in {"accession", "a", "accession-generic", "ag", "both", "b", "both-generic", "bg"} and args.accession_mode is None:
+            args.accession_mode = "file"
     if args.prefix:
         if args.autoclass in {"both", "b", "both-generic", "bg"}:
             if len(args.prefix) < 2 or len(args.prefix) > 2:
@@ -129,11 +135,20 @@ def run_cli():
             raise SystemExit               
     if args.fixity:
         print(f'Fixity is activated, using {args.fixity} algorithm')
+    if args.sort_by:
+        if args.sort_by == "foldersfirst":
+            sort_key = lambda x: (os.path.isfile(x), str.casefold(x))
+        elif args.sort_by == "alphabetical":
+            sort_key = str.casefold
     if args.remove:
-        i = "y"
-        # i = input(inspect.cleandoc("""You have enabled the remove functionality.
-        #                         This action will remove files listed for removal, it is irreversible.
-        #                         Please type Y to confirm, otherwise program will close: """))
+        print(inspect.cleandoc("""****
+                                    You have enabled the remove functionality of the program. This action will remove all files and folders listed for removal and any sub-files/sub-folders.
+                            
+                                    This process will permanently delete the selected items, with no way recover the items. 
+                                
+                                    ****"""))
+        time.sleep(2)
+        i = input(inspect.cleandoc("Please type Y if you wish to proceed, otherwise the program will close: "))
         if not i.lower() == "y":
             print("Closing program..."); time.sleep(3); raise SystemExit()
     time.sleep(3)
@@ -141,6 +156,7 @@ def run_cli():
                           output_path = args.output, 
                           autoclass_flag = args.autoclass, 
                           prefix = args.prefix, 
+                          accession_mode=args.accession_mode,
                           acc_prefix = acc_prefix, 
                           empty_flag = args.remove_empty, 
                           remove_flag = args.remove, 
@@ -155,7 +171,13 @@ def run_cli():
                           zip_flag = args.zip, 
                           input = args.input, 
                           output_format = args.output_format,
-                          options_file=args.options_file).main()
+                          options_file=args.options_file,
+                          keywords = args.keywords,
+                          keywords_mode = args.keywords_mode,
+                          keywords_retain_order = args.keywords_retain_order,
+                          sort_key = sort_key,
+                          delimiter = args.delimiter,
+                          keywords_abbreviation_number = args.keywords_abbreviation_number).main()
     
 if __name__ == "__main__":
     run_cli()

opex_manifest_generator/common.py

@@ -25,12 +25,18 @@ def win_256_check(path: str):
 
 def filter_win_hidden(path: str):
     if sys.platform =="win32":
-        if bool(os.stat(path).st_file_attribute & stat.FILE_ATTRIBUTE_HIDDEN) is True:
+        if bool(os.stat(path).st_file_attributes & stat.FILE_ATTRIBUTE_HIDDEN) is True:
             return True
         else:
             return False
     else:
         return False
+    
+def win_path_delimiter():
+    if sys.platform == "win32":
+        return "\\"
+    else:
+        return "/"
 
 def check_nan(value):
     if str(value).lower() in {"nan","nat"}:

opex_manifest_generator/opex_manifest.py

@@ -21,13 +21,14 @@ import configparser
 class OpexManifestGenerator():
     def __init__(self,
                  root: str,
-                 output_path: os.path = os.getcwd(),
+                 output_path: str = os.getcwd(),
                  meta_dir_flag: bool = True,
-                 metadata_dir: os = os.path.join(os.path.dirname(os.path.realpath(__file__)), "metadata"),
+                 metadata_dir: str = os.path.join(os.path.dirname(os.path.realpath(__file__)), "metadata"),
                  metadata_flag: str = 'none',
                  autoclass_flag: str = None,
                  prefix: str = None,
                  acc_prefix: str = None,
+                 accession_mode: str = False,
                  startref: int = 1,
                  algorithm: str = None,
                  empty_flag: bool = False,
@@ -39,7 +40,13 @@ class OpexManifestGenerator():
                  hidden_flag: bool = False,
                  output_format: str = "xlsx",
                  print_xmls_flag: bool = False,
-                 options_file: str = os.path.join(os.path.dirname(__file__),'options.properties')):
+                 options_file: str = os.path.join(os.path.dirname(__file__),'options.properties'),
+                 keywords: list = None,
+                 keywords_mode: str = "intitalise",
+                 keywords_retain_order: bool = False,
+                 sort_key = lambda x: (os.path.isfile(x), str.casefold(x)),
+                 keywords_abbreviation_number: int = 3,
+                 delimiter = "/"):
         
         self.root = os.path.abspath(root)
         self.opexns = "http://www.openpreservationexchange.org/opex/v1.2"        
@@ -49,6 +56,8 @@ class OpexManifestGenerator():
         self.algorithm = algorithm
         self.empty_flag = empty_flag
         self.remove_flag = remove_flag
+        if self.remove_flag:
+            self.remove_list = []
         self.export_flag = export_flag
         self.startref = startref
         self.autoclass_flag = autoclass_flag
@@ -57,6 +66,7 @@ class OpexManifestGenerator():
         self.meta_dir_flag = meta_dir_flag
         self.prefix = prefix
         self.acc_prefix = acc_prefix
+        self.accession_mode = accession_mode
         self.input = input
         self.hidden_flag = hidden_flag
         self.zip_flag = zip_flag
@@ -65,13 +75,18 @@ class OpexManifestGenerator():
         self.metadata_dir = metadata_dir
         self.print_xmls_flag = print_xmls_flag
         self.parse_config(options_file=os.path.abspath(options_file))
+        self.keywords_list = keywords
+        self.keywords_mode = keywords_mode
+        self.keywords_retain_order = keywords_retain_order
+        self.sort_key = sort_key
+        self.keywords_abbreviation_number = keywords_abbreviation_number
 
         self.title_flag = False
         self.description_flag = False
         self.security_flag = False
         self.ignore_flag = False
         self.sourceid_flag = False
-        self.hash_from_spread = False        
+        self.hash_from_spread = False
     
     def parse_config(self, options_file: str = 'options.properties'):
         config = configparser.ConfigParser()
@@ -130,10 +145,19 @@ class OpexManifestGenerator():
 
     def init_df(self):
         if self.autoclass_flag:
-            if self.autoclass_flag in {"catalog", "c", "catalog-generic", "cg"}:
-                ac = ClassificationGenerator(self.root, output_path = self.output_path, prefix = self.prefix, start_ref = self.startref, empty_flag = self.empty_flag, accession_flag = False)
-            elif self.autoclass_flag in {"accession", "a", "accession-generic", "ag", "both", "b", "both-generic", "bg"}:
-                ac = ClassificationGenerator(self.root, output_path = self.output_path, prefix = self.prefix, accprefix = self.acc_prefix, start_ref = self.startref, empty_flag = self.empty_flag, accession_flag="File")
+            ac = ClassificationGenerator(self.root,
+                                            output_path = self.output_path,
+                                            prefix = self.prefix,
+                                            accprefix = self.acc_prefix,
+                                            start_ref = self.startref,
+                                            empty_flag = self.empty_flag,
+                                            accession_flag=self.accession_mode,
+                                            keywords = self.keywords_list,
+                                            keywords_mode = self.keywords_mode,
+                                            keywords_retain_order = self.keywords_retain_order,
+                                            sort_key = self.sort_key,
+                                            keywords_abbreviation_number = self.keywords_abbreviation_number,
+                                            delimiter = self.delimiter)
             self.df = ac.init_dataframe()
             if self.autoclass_flag in {"accession", "a", "accesion-generic", "ag"}:
                 self.df = self.df.drop('Archive_Reference', axis=1)
@@ -188,19 +212,25 @@ class OpexManifestGenerator():
             print('Error Looking up XIP Metadata')
             print(e)
     
-    def remove_df_lookup(self, path: str, idx: pd.Index):
+    def remove_df_lookup(self, path: str, removed_list: list, idx: pd.Index):
         try:
             if idx.empty:
                 return False
             else:
                 remove = check_nan(self.df[REMOVAL_FIELD].loc[idx].item())
-                if remove:                                  
+                if remove is True:                  
+                    removed_list.append(path)
                     print(f"Removing: {path}")
-                    # Not functioning correctly
                     if os.path.isdir(path):
+                        for dp,d,f in os.walk(path):
+                            for fn in f:
+                                removed_list.append(win_256_check(dp+win_path_delimiter()+fn))
+                            for dn in d:
+                                removed_list.append(win_256_check(dp+win_path_delimiter()+dn))
                         shutil.rmtree(path)
                     else:
-                        os.remove(path)
+                        if os.path.exists(path):
+                            os.remove(path)
                     return True
                 else:
                     return False
@@ -285,7 +315,7 @@ class OpexManifestGenerator():
                 xml_file = ET.parse(path)
                 root_element = ET.QName(xml_file.find('.'))
                 root_element_ln = root_element.localname
-                root_element_ns = root_element.namespace
+                #root_element_ns = root_element.namespace
                 elements_list = []
                 for elem in xml_file.findall('.//'):
                     elem_path = xml_file.getelementpath(elem)
@@ -336,10 +366,10 @@ class OpexManifestGenerator():
                                     val = datetime.datetime.strftime(val, "%Y-%m-%dT%H:%M:%S.000Z")
                             if self.metadata_flag in {'e','exact'}:
                                 n = path.replace(localname + ":", f"{{{ns}}}")
-                                elem = xml_new.find(f'/{n}')
+                                elem = xml_new.find(f'./{n}')
                             elif self.metadata_flag in {'f', 'flat'}:
                                 n = name.split(':')[-1]
-                                elem = xml_new.find(f'//{{{ns}}}{n}')
+                                elem = xml_new.find(f'.//{{{ns}}}{n}')
                             elem.text = str(val)
                         except KeyError as e:
                             print('Key Error: please ensure column header\'s are an exact match...')
@@ -407,6 +437,9 @@ class OpexManifestGenerator():
         if self.algorithm:
             output_path = define_output_file(self.output_path, self.root, self.meta_dir_flag, output_suffix = "_Fixities", output_format = "txt")
             export_list_txt(self.list_fixity, output_path)
+        if self.remove_flag:
+            output_path = define_output_file(self.output_path, self.root, self.meta_dir_flag, output_suffix = "_Removed", output_format = "txt")
+            export_list_txt(self.remove_list, output_path)
         print_running_time(self.start_time)
 
 class OpexDir(OpexManifestGenerator):
@@ -418,23 +451,35 @@ class OpexDir(OpexManifestGenerator):
             self.folder_path = folder_path.replace(u'\\\\?\\', "")
         else:
             self.folder_path = folder_path
-        if self.OMG.autoclass_flag not in {None, "g","generic"}:
-            index = self.OMG.index_df_lookup(self.folder_path)
+        if any([self.OMG.input,
+                self.OMG.autoclass_flag in {"c","catalog","a","accession","b","both","cg","catalog-generic","ag","accession-generic","bg","both-generic"},
+                self.OMG.ignore_flag,
+                self.OMG.remove_flag,
+                self.OMG.sourceid_flag,
+                self.OMG.title_flag,
+                self.OMG.description_flag,
+                self.OMG.security_flag]):
+                index = self.OMG.index_df_lookup(self.folder_path)
+        elif self.OMG.autoclass_flag in {None, "g","generic"}:
+            index = None
         else:
-            index = None            
+            index = None
         if self.OMG.ignore_flag or self.OMG.remove_flag:
             if self.OMG.ignore_flag:
                 self.ignore = self.OMG.ignore_df_lookup(index)
                 if self.ignore:
                     return
+            else:
+                self.ignore = False
             if self.OMG.remove_flag:
-                self.removal = self.OMG.remove_df_lookup(self.folder_path, index)
+                self.removal = self.OMG.remove_df_lookup(self.folder_path, self.OMG.remove_list, index)
                 if self.removal:
                     return
+            else: 
+                self.removal = False
         else:
             self.ignore = False
             self.removal = False
-
         self.xmlroot = ET.Element(f"{{{self.opexns}}}OPEXMetadata", nsmap={"opex":self.opexns})
         self.transfer = ET.SubElement(self.xmlroot, f"{{{self.opexns}}}Transfer")
         self.manifest = ET.SubElement(self.transfer, f"{{{self.opexns}}}Manifest")
@@ -481,23 +526,29 @@ class OpexDir(OpexManifestGenerator):
             print('Failed to Filter')
             print(e)
             raise SystemError()
-    
         
     def generate_opex_dirs(self, path: str):
         self = OpexDir(self.OMG, path)
         opex_path = os.path.join(os.path.abspath(self.folder_path), os.path.basename(self.folder_path))
-        for f_path in self.filter_directories(path):
-            if f_path.endswith('.opex'):
-                pass
-            elif os.path.isdir(f_path):
-                if not self.ignore:
-                    self.folder = ET.SubElement(self.folders, f"{{{self.opexns}}}Folder")
-                    self.folder.text = str(os.path.basename(f_path))
-                self.generate_opex_dirs(f_path)
-            else:
-                OpexFile(self.OMG, f_path, self.OMG.algorithm)
-        if check_opex(opex_path):
-            if not self.ignore:
+        if self.removal is True:
+            pass
+        else:
+            for f_path in self.filter_directories(path):
+                if f_path.endswith('.opex'):
+                    pass
+                elif os.path.isdir(f_path):
+                    if self.ignore is True:
+                        pass
+                    else:
+                        self.folder = ET.SubElement(self.folders, f"{{{self.opexns}}}Folder")
+                        self.folder.text = str(os.path.basename(f_path))
+                    self.generate_opex_dirs(f_path)
+                else:
+                    OpexFile(self.OMG, f_path, self.OMG.algorithm)
+        if self.removal is True or self.ignore is True:
+            pass
+        else:
+            if check_opex(opex_path):
                 for f_path in self.filter_directories(path):
                     if os.path.isfile(f_path):
                         file = ET.SubElement(self.files, f"{{{self.opexns}}}File")
@@ -508,8 +559,8 @@ class OpexDir(OpexManifestGenerator):
                             file.set("size", str(os.path.getsize(f_path)))
                         file.text = str(os.path.basename(f_path))
                 write_opex(opex_path, self.xmlroot)
-        else:
-            print(f"Avoiding override, Opex exists at: {opex_path}")
+            else:
+                print(f"Avoiding override, Opex exists at: {opex_path}")
 
 class OpexFile(OpexManifestGenerator):
     def __init__(self, OMG: OpexManifestGenerator, file_path: str, algorithm: str = None, title: str = None, description: str = None, security: str = None):
@@ -531,17 +582,24 @@ class OpexFile(OpexManifestGenerator):
                     index = self.OMG.index_df_lookup(self.file_path)
             elif self.OMG.autoclass_flag is None or self.OMG.autoclass_flag in {"g","generic"}:
                 index = None
-            if self.OMG.ignore_flag:
-                self.ignore = self.OMG.ignore_df_lookup(index)
-                if self.ignore:
-                    #WTF is this?
-                    return
-            if self.OMG.remove_flag:
-                removal = self.OMG.remove_df_lookup(self.file_path, index)
-                if removal:
-                    return                
+            else:
+                index = None
+            if self.OMG.ignore_flag or self.OMG.remove_flag:
+                if self.OMG.ignore_flag:
+                    self.ignore = self.OMG.ignore_df_lookup(index)
+                    if self.ignore:
+                        return
+                else:
+                    self.ignore = False
+                if self.OMG.remove_flag:
+                    self.removal = self.OMG.remove_df_lookup(self.file_path, self.OMG.remove_list, index)
+                    if self.removal:
+                        return
+                else: 
+                    self.removal = False
             else:
                 self.ignore = False
+                self.removal = False
             self.algorithm = algorithm
             if self.OMG.title_flag or self.OMG.description_flag or self.OMG.security_flag:
                 self.title, self.description, self.security = self.OMG.xip_df_lookup(index) 

{opex_manifest_generator-1.1.12.dist-info → opex_manifest_generator-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opex_manifest_generator
-Version: 1.1.12
+Version: 1.2.0
 Summary: Opex Manifest Generator Tool for use with Opex / Preservica
 Author-email: Christopher Prince <c.pj.prince@gmail.com>
 Project-URL: Homepage, https://github.com/CPJPRINCE/opex_manifest_generator
@@ -10,8 +10,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: System :: Archiving
 License-File: LICENSE.md
-Requires-Dist: auto-classification-generator
+Requires-Dist: auto_classification_generator
 Requires-Dist: pandas
 Requires-Dist: openpyxl
 Requires-Dist: lxml
-

{opex_manifest_generator-1.1.12.dist-info → opex_manifest_generator-1.2.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
 opex_manifest_generator/__init__.py,sha256=fsN-dLjNMn-AfjeG2jkryzIa69bY09B5xLTqWedOv_w,461
-opex_manifest_generator/cli.py,sha256=SvSEAzVXqL-L5NgM7kd5TZ18f47-uMo90g_2zzWVJHY,10179
-opex_manifest_generator/common.py,sha256=dJGRQicA7B9n5WXpG5pJlV0fzO0itF6GyDXWTJ-5mxs,1782
+opex_manifest_generator/cli.py,sha256=Y8oo1CZzx_AXfXmjtBbhGgn3j53t2KgeBlB-5Pc_ZZI,12030
+opex_manifest_generator/common.py,sha256=vVDNlERDoI2WxEhpDYqnJC7oC9N1xQC2yxHjwuNus4Y,1895
 opex_manifest_generator/hash.py,sha256=mpLP-BqqDC7BuQQ5TEc4jaIBGabW-qVr128JaZq54mY,992
-opex_manifest_generator/opex_manifest.py,sha256=NkvgVR2C5QIJPq3kIcjjAURh3uaO4P2CGpEiIP76N6Q,29209
+opex_manifest_generator/opex_manifest.py,sha256=VBbArkAfDRmTuf59RDS5MZtPfrW2gcMetrLNxXwnCcs,32055
 opex_manifest_generator/options.properties,sha256=UKERpUH6M6SWXbXN2XitbPd1FvT4LlCbLfoHbdWuDAQ,286
 opex_manifest_generator/metadata/DublinCore Template.xml,sha256=csNGXzSH27Whs4BQNuwMZl8nLSdDq7Y_OblTfzeBqWQ,775
 opex_manifest_generator/metadata/EAD Template.xml,sha256=OsWjUfKiLAsc1zqm56auyFSDYMuZpu6a49AFgqZNzlU,2167
@@ -14,9 +14,9 @@ opex_manifest_generator/samples/spreads/dctemplate.xlsx,sha256=R33CNcBK3s-KFYEhm
 opex_manifest_generator/samples/spreads/eadtemplate.xlsx,sha256=4lrP0LLZXwv73fl3fvqr7yqpdDJuRj2D4ZADQ4OS6Ps,19299
 opex_manifest_generator/samples/spreads/gdprtemplate.xlsx,sha256=3k6FpN6n83yF5wYd64Yy8Rxv2b1Z497icIB8UBGDW-M,18662
 opex_manifest_generator/samples/spreads/modstemplate.xlsx,sha256=5kNpp4Cju_POvnTrgFk34OJVe5yc6o3R4ZNX2TT8zAc,19509
-opex_manifest_generator-1.1.12.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-opex_manifest_generator-1.1.12.dist-info/METADATA,sha256=3-2EQByfb-0uIK_MYB081jqy6D7jxQ7Vzj-EWNsjBIM,687
-opex_manifest_generator-1.1.12.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-opex_manifest_generator-1.1.12.dist-info/entry_points.txt,sha256=WGMc3hWlqOsQ8DcTuy8-DyBbJKkWNImT4J1FasVDHts,70
-opex_manifest_generator-1.1.12.dist-info/top_level.txt,sha256=K48eGnaDLVO6YDJdAZLqbeoZvJHBGX25cvYT-i8gWt0,24
-opex_manifest_generator-1.1.12.dist-info/RECORD,,
+opex_manifest_generator-1.2.0.dist-info/LICENSE.md,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+opex_manifest_generator-1.2.0.dist-info/METADATA,sha256=1q9TcPKrkgUQMiloGdUIBH09N2DfFAVSFkxGIJ2JGeI,685
+opex_manifest_generator-1.2.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+opex_manifest_generator-1.2.0.dist-info/entry_points.txt,sha256=WGMc3hWlqOsQ8DcTuy8-DyBbJKkWNImT4J1FasVDHts,70
+opex_manifest_generator-1.2.0.dist-info/top_level.txt,sha256=K48eGnaDLVO6YDJdAZLqbeoZvJHBGX25cvYT-i8gWt0,24
+opex_manifest_generator-1.2.0.dist-info/RECORD,,

{opex_manifest_generator-1.1.12.dist-info → opex_manifest_generator-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any