refscan 0.1.7__tar.gz → 0.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {refscan-0.1.7 → refscan-0.1.8}/PKG-INFO +33 -5
- {refscan-0.1.7 → refscan-0.1.8}/README.md +32 -4
- {refscan-0.1.7 → refscan-0.1.8}/pyproject.toml +1 -1
- {refscan-0.1.7 → refscan-0.1.8}/refscan/lib/ReferenceList.py +6 -0
- {refscan-0.1.7 → refscan-0.1.8}/refscan/lib/helpers.py +12 -1
- {refscan-0.1.7 → refscan-0.1.8}/refscan/refscan.py +36 -10
- {refscan-0.1.7 → refscan-0.1.8}/refscan/__init__.py +0 -0
- {refscan-0.1.7 → refscan-0.1.8}/refscan/lib/Finder.py +0 -0
- {refscan-0.1.7 → refscan-0.1.8}/refscan/lib/Reference.py +0 -0
- {refscan-0.1.7 → refscan-0.1.8}/refscan/lib/Violation.py +0 -0
- {refscan-0.1.7 → refscan-0.1.8}/refscan/lib/ViolationList.py +0 -0
- {refscan-0.1.7 → refscan-0.1.8}/refscan/lib/constants.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: refscan
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.8
|
|
4
4
|
Summary: Command-line program that scans the NMDC MongoDB database for referential integrity violations
|
|
5
5
|
Home-page: https://github.com/microbiomedata/refscan
|
|
6
6
|
Keywords: mongodb,mongo,relationships,reference,database,data,referential integrity,scan
|
|
@@ -40,6 +40,34 @@ graph LR
|
|
|
40
40
|
classDef dashed_border stroke-dasharray: 5 5
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
## Table of contents
|
|
44
|
+
|
|
45
|
+
<!-- TOC -->
|
|
46
|
+
* [refscan](#refscan)
|
|
47
|
+
* [Table of contents](#table-of-contents)
|
|
48
|
+
* [How it works](#how-it-works)
|
|
49
|
+
* [Limitations](#limitations)
|
|
50
|
+
* [Usage](#usage)
|
|
51
|
+
* [Install](#install)
|
|
52
|
+
* [Run](#run)
|
|
53
|
+
* [The MongoDB connection string (`--mongo-uri`)](#the-mongodb-connection-string---mongo-uri)
|
|
54
|
+
* [The schema (`--schema`)](#the-schema---schema)
|
|
55
|
+
* [Output](#output)
|
|
56
|
+
* [Update](#update)
|
|
57
|
+
* [Uninstall](#uninstall)
|
|
58
|
+
* [Development](#development)
|
|
59
|
+
* [Clone repository](#clone-repository)
|
|
60
|
+
* [Create virtual environment](#create-virtual-environment)
|
|
61
|
+
* [Install dependencies](#install-dependencies)
|
|
62
|
+
* [Make changes](#make-changes)
|
|
63
|
+
* [Run tests](#run-tests)
|
|
64
|
+
* [Format code](#format-code)
|
|
65
|
+
* [Check format](#check-format)
|
|
66
|
+
* [Building and publishing](#building-and-publishing)
|
|
67
|
+
* [Build for production](#build-for-production)
|
|
68
|
+
* [Test the build process locally](#test-the-build-process-locally)
|
|
69
|
+
<!-- TOC -->
|
|
70
|
+
|
|
43
71
|
## How it works
|
|
44
72
|
|
|
45
73
|
`refscan` does its job in two stages:
|
|
@@ -60,7 +88,7 @@ which it, in turn, uses to determine _which fields_ of that document can contain
|
|
|
60
88
|
|
|
61
89
|
## Usage
|
|
62
90
|
|
|
63
|
-
###
|
|
91
|
+
### Install
|
|
64
92
|
|
|
65
93
|
Assuming you have `pipx` installed, you can install the tool by running the following command:
|
|
66
94
|
|
|
@@ -73,7 +101,7 @@ pipx install refscan
|
|
|
73
101
|
> Python scripts that are hosted on PyPI.
|
|
74
102
|
> You can [install `pipx`](https://pipx.pypa.io/stable/installation/) by running `$ python -m pip install pipx`.
|
|
75
103
|
|
|
76
|
-
###
|
|
104
|
+
### Run
|
|
77
105
|
|
|
78
106
|
You can display the tool's `--help` snippet by running:
|
|
79
107
|
|
|
@@ -176,7 +204,7 @@ While `refscan` is running, it will display console output indicating what it's
|
|
|
176
204
|
Once the scan is complete, the reference report (TSV file) and violation report (TSV file) will be available
|
|
177
205
|
in the current directory (or in custom directories, if any were specified via CLI options).
|
|
178
206
|
|
|
179
|
-
###
|
|
207
|
+
### Update
|
|
180
208
|
|
|
181
209
|
You can update the tool to [the latest version available on PyPI](https://pypi.org/project/refscan/) by running:
|
|
182
210
|
|
|
@@ -184,7 +212,7 @@ You can update the tool to [the latest version available on PyPI](https://pypi.o
|
|
|
184
212
|
pipx upgrade refscan
|
|
185
213
|
```
|
|
186
214
|
|
|
187
|
-
###
|
|
215
|
+
### Uninstall
|
|
188
216
|
|
|
189
217
|
You can uninstall the tool from your computer by running:
|
|
190
218
|
|
|
@@ -21,6 +21,34 @@ graph LR
|
|
|
21
21
|
classDef dashed_border stroke-dasharray: 5 5
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
+
## Table of contents
|
|
25
|
+
|
|
26
|
+
<!-- TOC -->
|
|
27
|
+
* [refscan](#refscan)
|
|
28
|
+
* [Table of contents](#table-of-contents)
|
|
29
|
+
* [How it works](#how-it-works)
|
|
30
|
+
* [Limitations](#limitations)
|
|
31
|
+
* [Usage](#usage)
|
|
32
|
+
* [Install](#install)
|
|
33
|
+
* [Run](#run)
|
|
34
|
+
* [The MongoDB connection string (`--mongo-uri`)](#the-mongodb-connection-string---mongo-uri)
|
|
35
|
+
* [The schema (`--schema`)](#the-schema---schema)
|
|
36
|
+
* [Output](#output)
|
|
37
|
+
* [Update](#update)
|
|
38
|
+
* [Uninstall](#uninstall)
|
|
39
|
+
* [Development](#development)
|
|
40
|
+
* [Clone repository](#clone-repository)
|
|
41
|
+
* [Create virtual environment](#create-virtual-environment)
|
|
42
|
+
* [Install dependencies](#install-dependencies)
|
|
43
|
+
* [Make changes](#make-changes)
|
|
44
|
+
* [Run tests](#run-tests)
|
|
45
|
+
* [Format code](#format-code)
|
|
46
|
+
* [Check format](#check-format)
|
|
47
|
+
* [Building and publishing](#building-and-publishing)
|
|
48
|
+
* [Build for production](#build-for-production)
|
|
49
|
+
* [Test the build process locally](#test-the-build-process-locally)
|
|
50
|
+
<!-- TOC -->
|
|
51
|
+
|
|
24
52
|
## How it works
|
|
25
53
|
|
|
26
54
|
`refscan` does its job in two stages:
|
|
@@ -41,7 +69,7 @@ which it, in turn, uses to determine _which fields_ of that document can contain
|
|
|
41
69
|
|
|
42
70
|
## Usage
|
|
43
71
|
|
|
44
|
-
###
|
|
72
|
+
### Install
|
|
45
73
|
|
|
46
74
|
Assuming you have `pipx` installed, you can install the tool by running the following command:
|
|
47
75
|
|
|
@@ -54,7 +82,7 @@ pipx install refscan
|
|
|
54
82
|
> Python scripts that are hosted on PyPI.
|
|
55
83
|
> You can [install `pipx`](https://pipx.pypa.io/stable/installation/) by running `$ python -m pip install pipx`.
|
|
56
84
|
|
|
57
|
-
###
|
|
85
|
+
### Run
|
|
58
86
|
|
|
59
87
|
You can display the tool's `--help` snippet by running:
|
|
60
88
|
|
|
@@ -157,7 +185,7 @@ While `refscan` is running, it will display console output indicating what it's
|
|
|
157
185
|
Once the scan is complete, the reference report (TSV file) and violation report (TSV file) will be available
|
|
158
186
|
in the current directory (or in custom directories, if any were specified via CLI options).
|
|
159
187
|
|
|
160
|
-
###
|
|
188
|
+
### Update
|
|
161
189
|
|
|
162
190
|
You can update the tool to [the latest version available on PyPI](https://pypi.org/project/refscan/) by running:
|
|
163
191
|
|
|
@@ -165,7 +193,7 @@ You can update the tool to [the latest version available on PyPI](https://pypi.o
|
|
|
165
193
|
pipx upgrade refscan
|
|
166
194
|
```
|
|
167
195
|
|
|
168
|
-
###
|
|
196
|
+
### Uninstall
|
|
169
197
|
|
|
170
198
|
You can uninstall the tool from your computer by running:
|
|
171
199
|
|
|
@@ -28,6 +28,12 @@ class ReferenceList(UserList):
|
|
|
28
28
|
distinct_source_collection_names.append(reference.source_collection_name)
|
|
29
29
|
return distinct_source_collection_names
|
|
30
30
|
|
|
31
|
+
def count_source_collections(self) -> int:
|
|
32
|
+
r"""
|
|
33
|
+
Returns the number of distinct source collection names among all references in the list.
|
|
34
|
+
"""
|
|
35
|
+
return len(self.get_source_collection_names())
|
|
36
|
+
|
|
31
37
|
def get_source_field_names_of_source_collection(self, collection_name: str) -> list[str]:
|
|
32
38
|
"""
|
|
33
39
|
Returns the distinct source field names of the specified source collection.
|
|
@@ -51,12 +51,23 @@ def get_collection_names_from_schema(schema_view: SchemaView) -> list[str]:
|
|
|
51
51
|
return collection_names
|
|
52
52
|
|
|
53
53
|
|
|
54
|
+
def get_names_of_classes_eligible_for_collection(schema_view: SchemaView, collection_name: str) -> list[str]:
|
|
55
|
+
r"""
|
|
56
|
+
Returns a list of the names of the classes whose instances can be stored in the specified collection,
|
|
57
|
+
according to the specified `SchemaView`.
|
|
58
|
+
"""
|
|
59
|
+
slot_definition = schema_view.induced_slot(collection_name, DATABASE_CLASS_NAME)
|
|
60
|
+
name_of_eligible_class = slot_definition.range
|
|
61
|
+
names_of_eligible_classes = schema_view.class_descendants(name_of_eligible_class) # includes own class name
|
|
62
|
+
return names_of_eligible_classes
|
|
63
|
+
|
|
64
|
+
|
|
54
65
|
@cache # memoizes the decorated function
|
|
55
66
|
def translate_class_uri_into_schema_class_name(schema_view: SchemaView, class_uri: str) -> Optional[str]:
|
|
56
67
|
r"""
|
|
57
68
|
Returns the name of the schema class that has the specified value as its `class_uri`.
|
|
58
69
|
|
|
59
|
-
Example
|
|
70
|
+
Example: "nmdc:Biosample" (a `class_uri` value) -> "Biosample" (a class name)
|
|
60
71
|
|
|
61
72
|
References:
|
|
62
73
|
- https://linkml.io/linkml/developers/schemaview.html#linkml_runtime.utils.schemaview.SchemaView.all_classes
|
|
@@ -6,7 +6,7 @@ import typer
|
|
|
6
6
|
import linkml_runtime
|
|
7
7
|
|
|
8
8
|
from refscan.lib.Finder import Finder
|
|
9
|
-
from refscan.lib.constants import
|
|
9
|
+
from refscan.lib.constants import console
|
|
10
10
|
from refscan.lib.helpers import (
|
|
11
11
|
connect_to_database,
|
|
12
12
|
get_collection_names_from_schema,
|
|
@@ -14,6 +14,7 @@ from refscan.lib.helpers import (
|
|
|
14
14
|
init_progress_bar,
|
|
15
15
|
get_lowercase_key,
|
|
16
16
|
print_section_header,
|
|
17
|
+
get_names_of_classes_eligible_for_collection,
|
|
17
18
|
)
|
|
18
19
|
from refscan.lib.Reference import Reference
|
|
19
20
|
from refscan.lib.ReferenceList import ReferenceList
|
|
@@ -32,7 +33,7 @@ app_version = get_package_metadata("Version")
|
|
|
32
33
|
|
|
33
34
|
def display_app_version_and_exit(is_active: bool = False) -> None:
|
|
34
35
|
r"""
|
|
35
|
-
Displays the app's version number, then
|
|
36
|
+
Displays the app's version number, then exits.
|
|
36
37
|
|
|
37
38
|
Note: The `is_active` flag will be `True` if the program was
|
|
38
39
|
invoked with the associated CLI option.
|
|
@@ -148,10 +149,10 @@ def scan(
|
|
|
148
149
|
# For each collection, determine the names of the classes whose instances can be stored in that collection.
|
|
149
150
|
collection_name_to_class_names = {} # example: { "study_set": ["Study"] }
|
|
150
151
|
for collection_name in sorted(collection_names):
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
152
|
+
collection_name_to_class_names[collection_name] = get_names_of_classes_eligible_for_collection(
|
|
153
|
+
schema_view=schema_view,
|
|
154
|
+
collection_name=collection_name,
|
|
155
|
+
)
|
|
155
156
|
|
|
156
157
|
# Initialize the list of references. A reference is effectively a "foreign key" (i.e. a pointer).
|
|
157
158
|
references = ReferenceList()
|
|
@@ -205,6 +206,10 @@ def scan(
|
|
|
205
206
|
|
|
206
207
|
console.print(f"References described by schema: {len(references)}")
|
|
207
208
|
|
|
209
|
+
num_collections_having_references = references.count_source_collections()
|
|
210
|
+
console.print(f"Collections containing references: {num_collections_having_references}")
|
|
211
|
+
console.print() # newline
|
|
212
|
+
|
|
208
213
|
# Create a reference report in TSV format.
|
|
209
214
|
console.print(f"Writing reference report: {reference_report_file_path}")
|
|
210
215
|
references.dump_to_tsv_file(file_path=reference_report_file_path)
|
|
@@ -241,6 +246,13 @@ def scan(
|
|
|
241
246
|
else:
|
|
242
247
|
source_collection_names_in_db.append(collection_name)
|
|
243
248
|
|
|
249
|
+
# Print a message about each collection being skipped.
|
|
250
|
+
for i, collection_name in enumerate(names_of_source_collections_to_skip):
|
|
251
|
+
if i == 0:
|
|
252
|
+
console.print() # leading newline
|
|
253
|
+
console.print(f"⚠️ [orange][bold]Skipping source collection:[/bold][/orange] {collection_name}")
|
|
254
|
+
console.print() # newline
|
|
255
|
+
|
|
244
256
|
# Process each collection, checking for referential integrity violations;
|
|
245
257
|
# using the reference catalog created earlier to know which collections can
|
|
246
258
|
# contain "referrers" (documents), which of their slots can contain references (fields),
|
|
@@ -249,7 +261,6 @@ def scan(
|
|
|
249
261
|
|
|
250
262
|
# If this source collection is one of the ones the user wanted to skip, skip it now.
|
|
251
263
|
if source_collection_name in names_of_source_collections_to_skip:
|
|
252
|
-
console.print(f"⚠️ [orange][bold]Skipping source collection:[/bold][/orange] {source_collection_name}")
|
|
253
264
|
continue
|
|
254
265
|
|
|
255
266
|
collection = db.get_collection(source_collection_name)
|
|
@@ -357,17 +368,32 @@ def scan(
|
|
|
357
368
|
|
|
358
369
|
print_section_header(console, text="Summarizing results")
|
|
359
370
|
|
|
360
|
-
#
|
|
371
|
+
# Make a list of all violations among all collections.
|
|
361
372
|
all_violations = ViolationList()
|
|
362
373
|
for collection_name, violations in sorted(source_collections_and_their_violations.items(), key=get_lowercase_key):
|
|
363
|
-
console.print(f"Number of violations in {collection_name}: {len(violations)}")
|
|
364
374
|
all_violations.extend(violations)
|
|
375
|
+
|
|
376
|
+
# Print a message indicating the number of violations in this collection.
|
|
377
|
+
num_violations = len(violations)
|
|
378
|
+
color_name = "white" if num_violations == 0 else "red"
|
|
379
|
+
console.print(
|
|
380
|
+
f"Number of violations in {collection_name}: "
|
|
381
|
+
f"[{color_name} not bold]{num_violations}[/{color_name} not bold]"
|
|
382
|
+
)
|
|
383
|
+
|
|
365
384
|
if verbose:
|
|
366
385
|
console.print(violations)
|
|
367
386
|
|
|
368
|
-
|
|
387
|
+
num_all_violations = len(all_violations)
|
|
388
|
+
color_name = "white" if num_all_violations == 0 else "red"
|
|
389
|
+
console.print() # newline
|
|
390
|
+
console.print(f"Total violations: " f"[{color_name}]{num_all_violations}[/{color_name}]")
|
|
391
|
+
console.print() # newline
|
|
392
|
+
|
|
393
|
+
# Create a TSV-formatted violation report that lists all violations among all collections.
|
|
369
394
|
console.print(f"Writing violation report: {violation_report_file_path}")
|
|
370
395
|
all_violations.dump_to_tsv_file(file_path=violation_report_file_path)
|
|
396
|
+
console.print() # newline
|
|
371
397
|
|
|
372
398
|
|
|
373
399
|
if __name__ == "__main__":
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|