opex-manifest-generator 1.3.3__tar.gz → 1.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opex_manifest_generator-1.3.4/.github/workflows/python-tests.yml +31 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/PKG-INFO +3 -1
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/cli.py +134 -43
- opex_manifest_generator-1.3.4/opex_manifest_generator/common.py +90 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/hash.py +30 -14
- opex_manifest_generator-1.3.4/opex_manifest_generator/opex_manifest.py +1004 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator.egg-info/PKG-INFO +3 -1
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator.egg-info/SOURCES.txt +4 -1
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator.egg-info/requires.txt +3 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/pyproject.toml +8 -2
- opex_manifest_generator-1.3.4/pytest.ini +19 -0
- opex_manifest_generator-1.3.4/tests/test_opex_manifest.py +316 -0
- opex_manifest_generator-1.3.3/opex_manifest_generator/common.py +0 -79
- opex_manifest_generator-1.3.3/opex_manifest_generator/opex_manifest.py +0 -782
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/.github/workflows/codeql.yml +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/.github/workflows/pypi-publish.yml +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/.gitignore +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/LICENSE.md +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/README.md +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/assets/Column Headers.png +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/assets/FullName Column.png +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/assets/Hash Headers.png +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/assets/Identifiers Headers.png +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/assets/XML Headers.png +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/__init__.py +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/metadata/DublinCore Template.xml +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/metadata/EAD Template.xml +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/metadata/GDPR Template.xml +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/metadata/MODS Template.xml +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/options/options.properties +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator.egg-info/dependency_links.txt +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator.egg-info/entry_points.txt +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator.egg-info/top_level.txt +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/Sample File 1.jpg.opex +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/Sample File 2.gif.opex +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/Sample Folder Manifest.opex +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/opex_manifest_generator_AutoClass.xlsx +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/spreads/dctemplate.xlsx +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/spreads/eadtemplate.xlsx +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/spreads/gdprtemplate.xlsx +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/samples/spreads/modstemplate.xlsx +0 -0
- {opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/setup.cfg +0 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ master ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ master ]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ['3.10', '3.11', '3.12', '3.13']
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
20
|
+
uses: actions/setup-python@v4
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
|
|
24
|
+
- name: Install dependencies
|
|
25
|
+
run: |
|
|
26
|
+
python -m pip install --upgrade pip
|
|
27
|
+
python -m pip install -e ".[dev,addex]"
|
|
28
|
+
|
|
29
|
+
- name: Run tests
|
|
30
|
+
run: |
|
|
31
|
+
python -m pytest -v
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opex_manifest_generator
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.4
|
|
4
4
|
Summary: An Opex Manifest Generator tool for use with OPEX Files, as designed by Preservica
|
|
5
5
|
Author-email: Christopher Prince <c.pj.prince@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -18,6 +18,8 @@ Requires-Dist: openpyxl
|
|
|
18
18
|
Requires-Dist: lxml
|
|
19
19
|
Provides-Extra: addex
|
|
20
20
|
Requires-Dist: odfpy; extra == "addex"
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
21
23
|
Dynamic: license-file
|
|
22
24
|
|
|
23
25
|
# Opex Manifest Generator Tool
|
{opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/cli.py
RENAMED
|
@@ -5,18 +5,21 @@ author: Christopher Prince
|
|
|
5
5
|
license: Apache License 2.0"
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
import argparse, os, inspect, time
|
|
8
|
+
import argparse, os, inspect, time, logging
|
|
9
9
|
from opex_manifest_generator.opex_manifest import OpexManifestGenerator
|
|
10
10
|
import importlib.metadata
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from opex_manifest_generator.common import running_time
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
11
15
|
|
|
12
16
|
def parse_args():
|
|
13
17
|
parser = argparse.ArgumentParser(description = "OPEX Manifest Generator for Preservica Uploads")
|
|
14
18
|
parser.add_argument('root', nargs='?', default = os.getcwd(),
|
|
15
19
|
help = """The root path to generate Opexes for, will recursively traverse all sub-directories.
|
|
16
20
|
Generates an Opex for each folder & (depending on options) file in the directory tree.""")
|
|
17
|
-
|
|
18
21
|
parser.add_argument("-fx", "--fixity", required = False, nargs = '*', default = None,
|
|
19
|
-
choices = ['SHA-1', 'MD5', 'SHA-256', 'SHA-512'
|
|
22
|
+
choices = ['SHA-1', 'MD5', 'SHA-256', 'SHA-512'], type = fixity_helper, action=EmptyIsTrueFixity,
|
|
20
23
|
help="Generates a hash for each file and adds it to the opex, can select one or more algorithms to utilise. -fx SHA-1 MD5")
|
|
21
24
|
parser.add_argument("--pax-fixity", required = False, action = 'store_true', default = False,
|
|
22
25
|
help="""Enables use of PAX fixity generation, in line with Preservica's Recommendation.
|
|
@@ -24,13 +27,20 @@ def parse_args():
|
|
|
24
27
|
parser.add_argument("--fixity-export", required = False, action = 'store_false', default = True,
|
|
25
28
|
help="""Set whether to export the generated fixity list to a text file in the meta directory.
|
|
26
29
|
Enabled by default, disable with this flag.""")
|
|
30
|
+
parser.add_argument("--prevent-all-exports", required = False, action = 'store_true', default = False,
|
|
31
|
+
help="Set to prevent all exports (Fixity, Removal, Empty) from being created in the meta directory.")
|
|
27
32
|
parser.add_argument("-z", "--zip", required = False, action = 'store_true',
|
|
28
33
|
help="Set to zip files")
|
|
34
|
+
parser.add_argument("--zip-remove-files", required = False, action = 'store_true',
|
|
35
|
+
help="Set to remove the files that have been zipped")
|
|
29
36
|
parser.add_argument("--remove-empty", required = False, action = 'store_true', default = False,
|
|
30
37
|
help = "Remove and log empty directories from root. Log will be exported to 'meta' / output folder.")
|
|
38
|
+
parser.add_argument("--empty-export", required = False, action = 'store_false', default = True,
|
|
39
|
+
help="""Set whether to export the generated empty list to a text file in the meta directory.
|
|
40
|
+
Enabled by default, disable with this flag.""")
|
|
31
41
|
parser.add_argument("--hidden", required = False, action = 'store_true', default = False,
|
|
32
42
|
help="Set whether to include hidden files and folders")
|
|
33
|
-
parser.add_argument("-o", "--output", required = False, nargs =
|
|
43
|
+
parser.add_argument("-o", "--output", required = False, nargs = '?',
|
|
34
44
|
help = "Sets the output to send any generated files (Remove Empty, Fixity List, Autoref Export) to. Will not affect creation of a meta dir.")
|
|
35
45
|
parser.add_argument("-clr", "--clear-opex", required = False, action = 'store_true', default = False,
|
|
36
46
|
help = """Clears existing opex files from a directory. If set with no further options will only clear opexes;
|
|
@@ -47,24 +57,28 @@ def parse_args():
|
|
|
47
57
|
help="Set to utilise a CSV / XLSX spreadsheet to import data from")
|
|
48
58
|
parser.add_argument("-rm", "--remove", required = False, action = "store_true", default = False,
|
|
49
59
|
help="Set whether to enable removals of files and folders from a directory. ***Currently in testing")
|
|
60
|
+
parser.add_argument("--removal-export", required = False, action = 'store_false', default = True,
|
|
61
|
+
help="""Set whether to export the generated removals list to a text file in the meta directory.
|
|
62
|
+
Enabled by default, disable with this flag.""")
|
|
50
63
|
parser.add_argument("-mdir","--metadata-dir", required=False, nargs= '?',
|
|
51
64
|
default = os.path.join(os.path.dirname(os.path.realpath(__file__)), "metadata"),
|
|
52
65
|
help="Specify the metadata directory to pull XML files from")
|
|
53
|
-
parser.add_argument("-m", "--metadata", required = False, const = 'e', default =
|
|
54
|
-
nargs = '?', choices = ['
|
|
66
|
+
parser.add_argument("-m", "--metadata", required = False, const = 'e', default = None,
|
|
67
|
+
nargs = '?', choices = ['exact', 'e', 'flat', 'f'], type = str.lower,
|
|
55
68
|
help="Set whether to include xml metadata fields in the generation of the Opex")
|
|
56
69
|
parser.add_argument("--print-xmls", required = False, action = "store_true", default = False,
|
|
57
70
|
help="Prints the elements from your xmls to the consoles")
|
|
71
|
+
parser.add_argument("--convert-xmls", required=False, action ='store_true', default = False,
|
|
72
|
+
help="Convert XMLs templates files in mdir to spreadsheets/csv files")
|
|
58
73
|
|
|
59
74
|
# Auto Reference Options
|
|
60
75
|
parser.add_argument("-r", "--autoref", required = False,
|
|
61
76
|
choices = ['catalog', 'c', 'accession', 'a', 'both', 'b', 'generic', 'g', 'catalog-generic', 'cg', "accession-generic", "ag", "both-generic", "bg"],
|
|
62
77
|
type = str.lower,
|
|
63
78
|
help="""Toggles whether to utilise the auto_reference_generator
|
|
64
|
-
to generate an on the fly Reference listing
|
|
65
|
-
|
|
79
|
+
to generate an on the fly Reference listing.\n
|
|
66
80
|
There are several options, {catalog} will generate
|
|
67
|
-
a Archival Reference following an ISAD(G) structure
|
|
81
|
+
a Archival Reference following an ISAD(G) structure.\n
|
|
68
82
|
{accession} will create a running number of files.
|
|
69
83
|
{both} will do both at the same time!
|
|
70
84
|
{generic} will populate the title and description fields with the folder/file's name,
|
|
@@ -82,7 +96,7 @@ def parse_args():
|
|
|
82
96
|
parser.add_argument("--accession-mode", nargs = '?', required=False, const='file', default=None, choices=["file",'directory','both'],
|
|
83
97
|
help="""Set the mode when utilising the Accession option in autoref.
|
|
84
98
|
file - only adds on files, folder - only adds on folders, both - adds on files and folders""")
|
|
85
|
-
parser.add_argument("-str", "--start-ref", required = False, nargs = '?', default = 1,
|
|
99
|
+
parser.add_argument("-str", "--start-ref", required = False, type=int, nargs = '?', default = 1,
|
|
86
100
|
help="Set a custom Starting reference for the Auto Reference Generator. The generated reference will")
|
|
87
101
|
parser.add_argument("-ex", "--export-autoref", required = False, action = 'store_true', default = False,
|
|
88
102
|
help="Set whether to export the generated references to an AutoRef spreadsheet")
|
|
@@ -102,61 +116,101 @@ def parse_args():
|
|
|
102
116
|
help = "Set to set the number of letters to abbreviate for 'firstletters' mode, does not impact 'initialise' mode.")
|
|
103
117
|
parser.add_argument("--sort-by", required=False, nargs = '?', default = 'folders_first', choices = ['folders_first','alphabetical'], type=str.lower,
|
|
104
118
|
help = "Set the sorting method, 'folders_first' sorts folders first then files alphabetically; 'alphabetically' sorts alphabetically (ignoring folder distinction)")
|
|
105
|
-
|
|
119
|
+
parser.add_argument("--log-level", required=False, nargs='?', choices=['DEBUG','INFO','WARNING','ERROR'], default=None, type=str.upper,
|
|
120
|
+
help="Set the logging level (default: INFO)")
|
|
121
|
+
parser.add_argument("--log-file", required=False, nargs='?', default=None,
|
|
122
|
+
help="Optional path to write logs to a file (default: stdout)")
|
|
106
123
|
parser.add_argument("-v", "--version", action = 'version', version = '%(prog)s {version}'.format(version = importlib.metadata.version("opex_manifest_generator")))
|
|
107
124
|
|
|
108
125
|
args = parser.parse_args()
|
|
109
126
|
return args
|
|
110
127
|
|
|
111
128
|
def run_cli():
|
|
112
|
-
args = parse_args()
|
|
113
|
-
|
|
129
|
+
args = parse_args()
|
|
130
|
+
|
|
131
|
+
# Configure logging early so other modules inherit the settings
|
|
132
|
+
try:
|
|
133
|
+
log_level = getattr(logging, args.log_level.upper()) if args.log_level else logging.INFO
|
|
134
|
+
except Exception:
|
|
135
|
+
log_level = logging.INFO
|
|
136
|
+
log_format = '%(asctime)s %(levelname)-8s [%(name)s] %(message)s'
|
|
137
|
+
if args.log_file:
|
|
138
|
+
logging.basicConfig(level=log_level, filename=args.log_file, filemode='a', format=log_format)
|
|
139
|
+
else:
|
|
140
|
+
logging.basicConfig(level=log_level, format=log_format)
|
|
141
|
+
logger.debug(f'Logging configured (level={logging.getLevelName(log_level)}, file={args.log_file or "stdout"})')
|
|
142
|
+
|
|
143
|
+
if not os.path.exists(args.root):
|
|
144
|
+
logger.error(f'Please ensure that root path {args.root} exists. \n' \
|
|
145
|
+
'If you are utilising Windows ensure that the path does not end with \\\' or \\"')
|
|
146
|
+
raise FileNotFoundError(f'Please ensure that root path {args.root} exists. \n' \
|
|
147
|
+
'If you are utilising Windows ensure that the path does not end with \\\' or \\" ')
|
|
148
|
+
|
|
114
149
|
if isinstance(args.root, str):
|
|
115
150
|
args.root = args.root.strip("\"").rstrip("\\")
|
|
151
|
+
logger.info(f"Running Opex Generation on: {args.root}")
|
|
152
|
+
|
|
116
153
|
if not args.output:
|
|
117
154
|
args.output = os.path.abspath(args.root)
|
|
118
|
-
|
|
155
|
+
logger.debug(f'Output path set to root directory: {args.output}')
|
|
119
156
|
else:
|
|
120
|
-
args.output = os.path.abspath(args.output
|
|
121
|
-
|
|
157
|
+
args.output = os.path.abspath(args.output)
|
|
158
|
+
logger.info(f'Output path set to {args.output}')
|
|
159
|
+
|
|
122
160
|
if args.input and args.autoref:
|
|
123
|
-
|
|
124
|
-
|
|
161
|
+
logger.error(f'Both Input and Auto ref options have been selected, please use only one...')
|
|
162
|
+
raise ValueError('Both Input and Auto ref options have been selected, please use only one...')
|
|
125
163
|
if args.remove and not args.input:
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
if
|
|
129
|
-
|
|
130
|
-
|
|
164
|
+
logger.error('Removal flag has been given without input, please ensure an input file is utilised when using this option.')
|
|
165
|
+
raise ValueError('Removal flag has been given without input, please ensure an input file is utilised when using this option.')
|
|
166
|
+
if args.metadata is not None and not args.input:
|
|
167
|
+
logger.warning(f'Warning: Metadata Flag has been given without Input. Metadata won\'t be generated.')
|
|
168
|
+
|
|
131
169
|
if args.print_xmls:
|
|
170
|
+
logger.info(f'Printing xmls in {args.metadata_dir} then ending')
|
|
132
171
|
OpexManifestGenerator(root = args.root, metadata_dir=args.metadata_dir).print_descriptive_xmls()
|
|
172
|
+
raise SystemExit()
|
|
173
|
+
if args.convert_xmls:
|
|
174
|
+
logger.info(f'Converting XMLs in {args.metadata_dir} then ending')
|
|
175
|
+
OpexManifestGenerator(root = args.root, output_format=args.output_format, metadata_dir=args.metadata_dir).convert_descriptive_xmls()
|
|
176
|
+
raise SystemExit()
|
|
177
|
+
|
|
178
|
+
if args.prevent_all_exports:
|
|
179
|
+
args.fixity_export_flag = False
|
|
180
|
+
args.removal_export_flag = False
|
|
181
|
+
args.empty_export_flag = False
|
|
182
|
+
logger.info('All exports have been prevented via --prevent-all-exports flag.')
|
|
183
|
+
|
|
133
184
|
acc_prefix = None
|
|
134
185
|
if args.autoref in {"accession", "a", "accession-generic", "ag", "both", "b", "both-generic", "bg"} and args.accession_mode is None:
|
|
135
186
|
args.accession_mode = "file"
|
|
187
|
+
logger.debug(f'Accession mode not set, defaulting to "file" mode for accession generation.')
|
|
188
|
+
|
|
136
189
|
if args.prefix:
|
|
137
190
|
if args.autoref in {"both", "b", "both-generic", "bg"}:
|
|
138
191
|
if len(args.prefix) < 2 or len(args.prefix) > 2:
|
|
139
|
-
|
|
140
|
-
|
|
192
|
+
logger.error('"Both" option is selected, please pass only two prefixes: [-p CATALOG_PREFIX ACCESSION_PREFIX]');
|
|
193
|
+
raise ValueError('"Both" option is selected, please pass only two prefixes: [-p CATALOG_PREFIX ACCESSION_PREFIX]')
|
|
141
194
|
for n, a in enumerate(args.prefix):
|
|
142
195
|
if n == 0:
|
|
143
196
|
args.prefix = str(a)
|
|
144
197
|
elif n == 1:
|
|
145
198
|
acc_prefix = str(a)
|
|
146
|
-
|
|
199
|
+
logger.info(f"Prefixes are set as: \t Catalog: {args.prefix} \t Acc: {acc_prefix}")
|
|
147
200
|
elif args.autoref in {"accession", "a", "accession-generic", "ag"}:
|
|
148
201
|
for a in args.prefix:
|
|
149
202
|
acc_prefix = str(a)
|
|
150
|
-
|
|
203
|
+
logger.info('Prefix is set as: ' + acc_prefix)
|
|
151
204
|
elif args.autoref in {"catalog", "c", "catalog-generic", "cg"}:
|
|
152
205
|
acc_prefix = None
|
|
153
206
|
for a in args.prefix:
|
|
154
207
|
args.prefix = str(a)
|
|
155
|
-
|
|
208
|
+
logger.info('Prefix is set as: ' + args.prefix)
|
|
156
209
|
elif args.autoref in {"generic", "g"}:
|
|
210
|
+
logger.info('Using Generic mode')
|
|
157
211
|
pass
|
|
158
212
|
else:
|
|
159
|
-
|
|
213
|
+
logger.error('''An invalid option has been selected, please select a valid option:
|
|
160
214
|
{c, catalog
|
|
161
215
|
a, accession
|
|
162
216
|
b, both
|
|
@@ -164,27 +218,47 @@ def run_cli():
|
|
|
164
218
|
cg, catalog-generic
|
|
165
219
|
ag, accession-generic
|
|
166
220
|
bg, both-generic}''')
|
|
167
|
-
|
|
168
|
-
|
|
221
|
+
raise ValueError('An invalid option has been selected, please select a valid option.')
|
|
222
|
+
|
|
169
223
|
if args.fixity:
|
|
170
|
-
|
|
224
|
+
logger.info(f'Fixity is activated, using {args.fixity} algorithm')
|
|
225
|
+
|
|
226
|
+
sort_key = None
|
|
171
227
|
if args.sort_by:
|
|
172
228
|
if args.sort_by == "folders_first":
|
|
229
|
+
logger.debug('Sorting by folders_first')
|
|
173
230
|
sort_key = lambda x: (os.path.isfile(x), str.casefold(x))
|
|
174
231
|
elif args.sort_by == "alphabetical":
|
|
232
|
+
logger.debug('Sorting by alphabetical')
|
|
175
233
|
sort_key = str.casefold
|
|
234
|
+
|
|
176
235
|
if args.remove:
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
236
|
+
logger.warning(inspect.cleandoc("\n***WARNING***" \
|
|
237
|
+
"\nYou have enabled the remove functionality of the program. " \
|
|
238
|
+
"This action will remove all files and folders listed for removal and any sub-files/sub-folders." \
|
|
239
|
+
"\nThis process will permanently delete the selected items, with no way recover the items." \
|
|
240
|
+
"\n***"))
|
|
241
|
+
i = input(inspect.cleandoc("Please type Y if you wish to proceed, otherwise the program will close: "))
|
|
242
|
+
if not i.lower() == "y":
|
|
243
|
+
logger.info("Y not typed, safetly aborted...")
|
|
244
|
+
raise SystemExit()
|
|
245
|
+
else:
|
|
246
|
+
logger.info("Confirmation recieved proceeding to remove files")
|
|
247
|
+
|
|
248
|
+
if args.remove_empty:
|
|
249
|
+
logger.warning(inspect.cleandoc("\n***WARNING***" \
|
|
250
|
+
"\nYou have enabled the remove empty folders functionality of the program. " \
|
|
251
|
+
"This action will remove all empty folders." \
|
|
252
|
+
"\nThis process will permanently delete all empty folders, with no way recover the items." \
|
|
253
|
+
"\n***"))
|
|
184
254
|
i = input(inspect.cleandoc("Please type Y if you wish to proceed, otherwise the program will close: "))
|
|
185
255
|
if not i.lower() == "y":
|
|
186
|
-
|
|
187
|
-
|
|
256
|
+
logger.info("Y not typed, safetly aborted...")
|
|
257
|
+
raise SystemExit()
|
|
258
|
+
else:
|
|
259
|
+
logger.info("Confirmation recieved proceeding to remove empty folders...")
|
|
260
|
+
|
|
261
|
+
start_time = datetime.now()
|
|
188
262
|
OpexManifestGenerator(root = args.root,
|
|
189
263
|
output_path = args.output,
|
|
190
264
|
autoref_flag = args.autoref,
|
|
@@ -192,7 +266,9 @@ def run_cli():
|
|
|
192
266
|
accession_mode=args.accession_mode,
|
|
193
267
|
acc_prefix = acc_prefix,
|
|
194
268
|
empty_flag = args.remove_empty,
|
|
269
|
+
empty_export_flag = args.empty_export,
|
|
195
270
|
removal_flag = args.remove,
|
|
271
|
+
removal_export_flag = args.removal_export,
|
|
196
272
|
clear_opex_flag = args.clear_opex,
|
|
197
273
|
algorithm = args.fixity,
|
|
198
274
|
pax_fixity= args.pax_fixity,
|
|
@@ -204,17 +280,29 @@ def run_cli():
|
|
|
204
280
|
metadata_dir = args.metadata_dir,
|
|
205
281
|
hidden_flag= args.hidden,
|
|
206
282
|
zip_flag = args.zip,
|
|
283
|
+
zip_file_removal= args.zip_remove_files,
|
|
207
284
|
input = args.input,
|
|
208
285
|
output_format = args.output_format,
|
|
209
286
|
options_file=args.options_file,
|
|
210
287
|
keywords = args.keywords,
|
|
211
288
|
keywords_mode = args.keywords_mode,
|
|
212
289
|
keywords_retain_order = args.keywords_retain_order,
|
|
213
|
-
|
|
290
|
+
keywords_case_sensitivity = args.keywords_case_sensitivity,
|
|
214
291
|
delimiter = args.delimiter,
|
|
215
292
|
keywords_abbreviation_number = args.keywords_abbreviation_number,
|
|
216
293
|
sort_key = sort_key,
|
|
217
294
|
).main()
|
|
295
|
+
logger.info(f"Run Complete! Ran for: {running_time(start_time)}")
|
|
296
|
+
|
|
297
|
+
def fixity_helper(x: str):
|
|
298
|
+
x = x.upper()
|
|
299
|
+
if x == 'SHA1':
|
|
300
|
+
x = 'SHA-1'
|
|
301
|
+
if x == 'SHA256':
|
|
302
|
+
x = 'SHA-256'
|
|
303
|
+
if x == 'SHA512':
|
|
304
|
+
x = 'SHA-512'
|
|
305
|
+
return x.upper()
|
|
218
306
|
|
|
219
307
|
class EmptyIsTrueFixity(argparse.Action):
|
|
220
308
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
@@ -223,4 +311,7 @@ class EmptyIsTrueFixity(argparse.Action):
|
|
|
223
311
|
setattr(namespace, self.dest, values)
|
|
224
312
|
|
|
225
313
|
if __name__ == "__main__":
|
|
226
|
-
|
|
314
|
+
try:
|
|
315
|
+
run_cli()
|
|
316
|
+
except KeyboardInterrupt:
|
|
317
|
+
logger.warning("Process interrupted by user, exiting...")
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Common tools for use throghout program.
|
|
3
|
+
|
|
4
|
+
author: Christopher Prince
|
|
5
|
+
license: Apache License 2.0"
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import zipfile, os, sys, stat, shutil, logging, lxml
|
|
9
|
+
from datetime import datetime, timedelta
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
def zip_opex(file_path,opex_path = None) -> str:
|
|
14
|
+
zip_file = f"{file_path}.zip"
|
|
15
|
+
if not os.path.exists(zip_file):
|
|
16
|
+
with zipfile.ZipFile(zip_file,'w') as z:
|
|
17
|
+
if os.path.exists(file_path):
|
|
18
|
+
z.write(file_path,os.path.basename(file_path))
|
|
19
|
+
if opex_path is not None and os.path.exists(opex_path):
|
|
20
|
+
z.write(opex_path,os.path.basename(opex_path))
|
|
21
|
+
logger.debug(f'File has been zipped to: {zip_file}')
|
|
22
|
+
else:
|
|
23
|
+
logger.warning(f'A Zip file already exists for: {zip_file}')
|
|
24
|
+
return zip_file
|
|
25
|
+
|
|
26
|
+
def remove_tree(path: str, removed_list: list) -> None:
|
|
27
|
+
removed_list.append(path)
|
|
28
|
+
logger.info(f"Removing: {path}")
|
|
29
|
+
if os.path.isdir(path):
|
|
30
|
+
for dp,d,f in os.walk(path):
|
|
31
|
+
for fn in f:
|
|
32
|
+
removed_list.append(win_256_check(dp+win_path_delimiter()+fn))
|
|
33
|
+
logger.info(f'Removing {dp + win_path_delimiter() + fn}')
|
|
34
|
+
for dn in d:
|
|
35
|
+
removed_list.append(win_256_check(dp+win_path_delimiter()+dn))
|
|
36
|
+
logger.info(f'Removing {dp + win_path_delimiter() + dn}')
|
|
37
|
+
shutil.rmtree(path)
|
|
38
|
+
else:
|
|
39
|
+
if os.path.exists(path):
|
|
40
|
+
os.remove(path)
|
|
41
|
+
logger.info (f'Removing File: {path}')
|
|
42
|
+
|
|
43
|
+
def win_256_check(path) -> str:
|
|
44
|
+
if len(path) > 255 and sys.platform == "win32":
|
|
45
|
+
logger.debug(f'Path: {path} is greater than 255 Characters')
|
|
46
|
+
if path.startswith(u"\\\\?\\"):
|
|
47
|
+
path = path
|
|
48
|
+
else:
|
|
49
|
+
path = u"\\\\?\\" + path
|
|
50
|
+
return path
|
|
51
|
+
|
|
52
|
+
def filter_win_hidden(path: str) -> bool:
|
|
53
|
+
if sys.platform =="win32":
|
|
54
|
+
if bool(os.stat(path).st_file_attributes & stat.FILE_ATTRIBUTE_HIDDEN) is True:
|
|
55
|
+
return True
|
|
56
|
+
else:
|
|
57
|
+
return False
|
|
58
|
+
else:
|
|
59
|
+
return False
|
|
60
|
+
|
|
61
|
+
def win_path_delimiter() -> str:
|
|
62
|
+
if sys.platform == "win32":
|
|
63
|
+
return "\\"
|
|
64
|
+
else:
|
|
65
|
+
return "/"
|
|
66
|
+
|
|
67
|
+
def check_nan(value) -> str | None:
|
|
68
|
+
if str(value).lower() in {"nan","nat"}:
|
|
69
|
+
value = None
|
|
70
|
+
return value
|
|
71
|
+
|
|
72
|
+
def check_opex(opex_path:str) -> bool:
|
|
73
|
+
opex_path = opex_path + ".opex"
|
|
74
|
+
if os.path.exists(win_256_check(opex_path)):
|
|
75
|
+
return False
|
|
76
|
+
else:
|
|
77
|
+
return True
|
|
78
|
+
|
|
79
|
+
def write_opex(path: str, opexxml: lxml.etree.Element) -> str:
|
|
80
|
+
opex_path = win_256_check(str(path) + ".opex")
|
|
81
|
+
opex = lxml.etree.indent(opexxml, " ")
|
|
82
|
+
opex = lxml.etree.tostring(opexxml, pretty_print=True, xml_declaration=True, encoding="UTF-8", standalone=True)
|
|
83
|
+
with open(f'{opex_path}', 'w', encoding="UTF-8") as writer:
|
|
84
|
+
writer.write(opex.decode('UTF-8'))
|
|
85
|
+
logger.info('Saved Opex File to: ' + opex_path)
|
|
86
|
+
return opex_path
|
|
87
|
+
|
|
88
|
+
def running_time(start_time) -> timedelta:
|
|
89
|
+
running_time = datetime.now() - start_time
|
|
90
|
+
return running_time
|
{opex_manifest_generator-1.3.3 → opex_manifest_generator-1.3.4}/opex_manifest_generator/hash.py
RENAMED
|
@@ -5,9 +5,11 @@ author: Christopher Prince
|
|
|
5
5
|
license: Apache License 2.0"
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
import hashlib
|
|
8
|
+
import hashlib, logging
|
|
9
9
|
from opex_manifest_generator.common import win_256_check
|
|
10
10
|
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
11
13
|
class HashGenerator():
|
|
12
14
|
def __init__(self, algorithm: str = "SHA-1", buffer: int = 4096):
|
|
13
15
|
self.algorithm = algorithm
|
|
@@ -15,17 +17,17 @@ class HashGenerator():
|
|
|
15
17
|
|
|
16
18
|
def hash_generator(self, file_path: str):
|
|
17
19
|
file_path = win_256_check(file_path)
|
|
18
|
-
if
|
|
20
|
+
if "SHA-1" in self.algorithm:
|
|
19
21
|
hash = hashlib.sha1()
|
|
20
|
-
elif self.algorithm
|
|
22
|
+
elif "MD5" in self.algorithm:
|
|
21
23
|
hash = hashlib.md5()
|
|
22
|
-
elif
|
|
24
|
+
elif "SHA-256" in self.algorithm:
|
|
23
25
|
hash = hashlib.sha256()
|
|
24
|
-
elif
|
|
26
|
+
elif "SHA-512" in self.algorithm:
|
|
25
27
|
hash = hashlib.sha512()
|
|
26
28
|
else:
|
|
27
29
|
hash = hashlib.sha1()
|
|
28
|
-
|
|
30
|
+
logger.info(f'Generating Fixity using {self.algorithm} for: {file_path}')
|
|
29
31
|
try:
|
|
30
32
|
with open(file_path, 'rb', buffering = 0) as f:
|
|
31
33
|
while True:
|
|
@@ -34,12 +36,19 @@ class HashGenerator():
|
|
|
34
36
|
break
|
|
35
37
|
hash.update(buff)
|
|
36
38
|
f.close()
|
|
39
|
+
logger.debug(f'Generated Hash: {hash.hexdigest().upper()}')
|
|
40
|
+
return hash.hexdigest().upper()
|
|
41
|
+
except FileNotFoundError as e:
|
|
42
|
+
logger.exception(f'File Not Found generating Hash: {e}')
|
|
43
|
+
raise
|
|
44
|
+
except IOError as e:
|
|
45
|
+
logger.exception(f'I/O Error generating Hash: {e}')
|
|
46
|
+
raise
|
|
37
47
|
except Exception as e:
|
|
38
|
-
|
|
39
|
-
raise
|
|
40
|
-
return hash.hexdigest().upper()
|
|
48
|
+
logger.exception(f'Error Generating Hash: {e}')
|
|
49
|
+
raise
|
|
41
50
|
|
|
42
|
-
def hash_generator_pax_zip(self, filename, z):
|
|
51
|
+
def hash_generator_pax_zip(self, filename: str, z):
|
|
43
52
|
if self.algorithm in ("SHA1","SHA-1"):
|
|
44
53
|
hash = hashlib.sha1()
|
|
45
54
|
elif self.algorithm == "MD5":
|
|
@@ -50,7 +59,7 @@ class HashGenerator():
|
|
|
50
59
|
hash = hashlib.sha512()
|
|
51
60
|
else:
|
|
52
61
|
hash = hashlib.sha1()
|
|
53
|
-
|
|
62
|
+
logger.info(f'Generating Fixity using {self.algorithm} for: {filename}')
|
|
54
63
|
try:
|
|
55
64
|
with z.open(filename, 'r') as data:
|
|
56
65
|
while True:
|
|
@@ -58,7 +67,14 @@ class HashGenerator():
|
|
|
58
67
|
if not buff:
|
|
59
68
|
break
|
|
60
69
|
hash.update(buff)
|
|
70
|
+
data.close()
|
|
71
|
+
except FileNotFoundError as e:
|
|
72
|
+
logger.exception(f'File Not Found generating Hash: {e}')
|
|
73
|
+
raise
|
|
74
|
+
except IOError as e:
|
|
75
|
+
logger.exception(f'I/O Error generating Hash: {e}')
|
|
76
|
+
raise
|
|
61
77
|
except Exception as e:
|
|
62
|
-
|
|
63
|
-
raise
|
|
64
|
-
return hash.hexdigest().upper()
|
|
78
|
+
logger.exception(f'Error Generating Hash: {e}')
|
|
79
|
+
raise
|
|
80
|
+
return str(hash.hexdigest().upper())
|