refscan 0.1.22__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {refscan-0.1.22 → refscan-0.2.0}/PKG-INFO +88 -67
- {refscan-0.1.22 → refscan-0.2.0}/README.md +84 -63
- {refscan-0.1.22 → refscan-0.2.0}/pyproject.toml +9 -8
- refscan-0.2.0/refscan/cli/cli.py +27 -0
- refscan-0.2.0/refscan/cli/graph.py +80 -0
- refscan-0.2.0/refscan/cli/scan.py +217 -0
- refscan-0.2.0/refscan/cli/version.py +14 -0
- refscan-0.1.22/refscan/refgraph.py → refscan-0.2.0/refscan/grapher.py +6 -81
- {refscan-0.1.22 → refscan-0.2.0}/refscan/lib/Reference.py +1 -1
- {refscan-0.1.22 → refscan-0.2.0}/refscan/lib/Violation.py +1 -0
- refscan-0.2.0/refscan/scanner.py +251 -0
- {refscan-0.1.22 → refscan-0.2.0}/refscan/templates/graph.template.html +2 -2
- refscan-0.1.22/refscan/refscan.py +0 -395
- {refscan-0.1.22 → refscan-0.2.0}/refscan/__init__.py +0 -0
- {refscan-0.1.22 → refscan-0.2.0}/refscan/lib/Finder.py +0 -0
- {refscan-0.1.22 → refscan-0.2.0}/refscan/lib/ReferenceList.py +0 -0
- {refscan-0.1.22 → refscan-0.2.0}/refscan/lib/ViolationList.py +0 -0
- {refscan-0.1.22 → refscan-0.2.0}/refscan/lib/constants.py +0 -0
- {refscan-0.1.22 → refscan-0.2.0}/refscan/lib/helpers.py +0 -0
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: refscan
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Command-line program that scans the NMDC MongoDB database for referential integrity violations
|
|
5
|
-
Home-page: https://github.com/microbiomedata/refscan
|
|
6
5
|
Keywords: mongodb,mongo,relationships,reference,database,data,referential integrity,scan
|
|
7
6
|
Requires-Python: >=3.9,<4.0
|
|
8
7
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -14,14 +13,15 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
14
13
|
Requires-Dist: linkml-runtime (>=1.7.5,<2.0.0)
|
|
15
14
|
Requires-Dist: pymongo (>=4.7.1,<5.0.0)
|
|
16
15
|
Requires-Dist: rich (>=13.7.1,<14.0.0)
|
|
17
|
-
Requires-Dist: typer (>=0.
|
|
16
|
+
Requires-Dist: typer (>=0.15.1,<0.16.0)
|
|
18
17
|
Project-URL: Documentation, https://github.com/microbiomedata/refscan
|
|
18
|
+
Project-URL: Homepage, https://github.com/microbiomedata/refscan
|
|
19
19
|
Project-URL: Repository, https://github.com/microbiomedata/refscan
|
|
20
20
|
Description-Content-Type: text/markdown
|
|
21
21
|
|
|
22
22
|
# refscan
|
|
23
23
|
|
|
24
|
-
`refscan` is a command-line tool people can use to scan the [NMDC](https://microbiomedata.org/) MongoDB database
|
|
24
|
+
`refscan` is a command-line tool people can use to **scan** the [NMDC](https://microbiomedata.org/) MongoDB database
|
|
25
25
|
for referential integrity violations.
|
|
26
26
|
|
|
27
27
|
```mermaid
|
|
@@ -31,7 +31,7 @@ for referential integrity violations.
|
|
|
31
31
|
graph LR
|
|
32
32
|
schema[LinkML<br>schema]
|
|
33
33
|
database[(MongoDB<br>database)]
|
|
34
|
-
script[["refscan
|
|
34
|
+
script[["refscan"]]
|
|
35
35
|
violations["List of<br>violations"]
|
|
36
36
|
references["List of<br>references"]:::dashed_border
|
|
37
37
|
schema --> script
|
|
@@ -42,39 +42,20 @@ graph LR
|
|
|
42
42
|
classDef dashed_border stroke-dasharray: 5 5
|
|
43
43
|
```
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
* [Limitations](#limitations)
|
|
52
|
-
* [Usage](#usage)
|
|
53
|
-
* [Install](#install)
|
|
54
|
-
* [Run](#run)
|
|
55
|
-
* [The MongoDB connection string (`--mongo-uri`)](#the-mongodb-connection-string---mongo-uri)
|
|
56
|
-
* [The schema (`--schema`)](#the-schema---schema)
|
|
57
|
-
* [Output](#output)
|
|
58
|
-
* [Update](#update)
|
|
59
|
-
* [Uninstall](#uninstall)
|
|
60
|
-
* [Development](#development)
|
|
61
|
-
* [Clone repository](#clone-repository)
|
|
62
|
-
* [Create virtual environment](#create-virtual-environment)
|
|
63
|
-
* [Install dependencies](#install-dependencies)
|
|
64
|
-
* [Make changes](#make-changes)
|
|
65
|
-
* [Run tests](#run-tests)
|
|
66
|
-
* [Format code](#format-code)
|
|
67
|
-
* [Check format](#check-format)
|
|
68
|
-
* [Building and publishing](#building-and-publishing)
|
|
69
|
-
* [Build for production](#build-for-production)
|
|
70
|
-
* [Test the build process locally](#test-the-build-process-locally)
|
|
71
|
-
* [Appendix](#appendix)
|
|
72
|
-
* [refgraph](#refgraph)
|
|
73
|
-
<!-- TOC -->
|
|
45
|
+
In addition to using refscan to scan the NMDC MongoDB database for referential integrity violations,
|
|
46
|
+
people can use `refscan` to generate **graphs** (diagrams) depicting which collections' documents (or which classes'
|
|
47
|
+
instances) can contain references to which _other_ collections' documents (or classes' instances) while still being
|
|
48
|
+
schema compliant.
|
|
49
|
+
|
|
50
|
+
<!-- Note: We removed the hard-coded Table of Contents because—nowadays—GitHub automatically derives/presents one. -->
|
|
74
51
|
|
|
75
52
|
## How it works
|
|
76
53
|
|
|
77
|
-
`refscan`
|
|
54
|
+
Here is a summary of how each of `refscan`'s main functions works under the hood.
|
|
55
|
+
|
|
56
|
+
### Scan
|
|
57
|
+
|
|
58
|
+
`refscan` does this in two stages:
|
|
78
59
|
1. It uses the LinkML schema to determine where references _can_ exist in a MongoDB database that conforms to the schema.
|
|
79
60
|
> **Example:** The schema might say that, if a document in the `biosample_set` collection has a field named
|
|
80
61
|
> `associated_studies`, that field must contain a list of `id`s of documents in the `study_set` collection.
|
|
@@ -82,12 +63,19 @@ graph LR
|
|
|
82
63
|
> **Example:** For each document in the `biosample_set` collection that _has_ a field named `associated_studies`,
|
|
83
64
|
> for each value in that field, confirm there _is_ a document having that `id` in the `study_set` collection.
|
|
84
65
|
|
|
85
|
-
|
|
66
|
+
### Graph
|
|
67
|
+
|
|
68
|
+
`refscan` does this in three stages:
|
|
69
|
+
1. It uses the LinkML schema to determine where references _can_ exist in a MongoDB database that conforms to the schema.
|
|
70
|
+
2. It formats that list of references into a data structure compatible with [`Cytoscape.js`](https://js.cytoscape.org/).
|
|
71
|
+
3. It outputs an HTML document that uses `Cytoscape.js` to visualize that data structure as a graph.
|
|
72
|
+
|
|
73
|
+
## Assumptions
|
|
86
74
|
|
|
87
75
|
`refscan` was designed under the assumption that **every document** in **every collection described by the schema** has
|
|
88
76
|
a **field named `type`**, whose value is the [class_uri](https://linkml.io/linkml/code/metamodel.html#linkml_runtime.linkml_model.meta.ClassDefinition.class_uri) of the schema class the document represents an instance
|
|
89
|
-
of. `refscan` uses that `class_uri` value (in that `type` field) to determine the
|
|
90
|
-
|
|
77
|
+
of. `refscan` uses that `class_uri` value (in that `type` field) to determine the _name_ of that schema class,
|
|
78
|
+
whose definition `refscan` then uses to determine _which fields_ of that document can contain references.
|
|
91
79
|
|
|
92
80
|
## Usage
|
|
93
81
|
|
|
@@ -106,7 +94,7 @@ pipx install refscan
|
|
|
106
94
|
|
|
107
95
|
### Run
|
|
108
96
|
|
|
109
|
-
|
|
97
|
+
Once installed, you can display the tool's `--help` snippet by running:
|
|
110
98
|
|
|
111
99
|
```shell
|
|
112
100
|
refscan --help
|
|
@@ -115,9 +103,30 @@ refscan --help
|
|
|
115
103
|
At the time of this writing, the tool's `--help` snippet is:
|
|
116
104
|
|
|
117
105
|
```console
|
|
118
|
-
Usage: refscan [OPTIONS]
|
|
106
|
+
Usage: refscan [OPTIONS] COMMAND [ARGS]...
|
|
119
107
|
|
|
120
|
-
|
|
108
|
+
╭─ Options ──────────────────────────────────────────────────────────────────────────────╮
|
|
109
|
+
│ --help Show this message and exit. │
|
|
110
|
+
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
111
|
+
╭─ Commands ─────────────────────────────────────────────────────────────────────────────╮
|
|
112
|
+
│ version Show version number and exit. │
|
|
113
|
+
│ scan Scan the NMDC MongoDB database for referential integrity violations. │
|
|
114
|
+
│ graph Generate an interactive graph of the references described by a schema. │
|
|
115
|
+
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
<!-- Note: The above snippet was captured from a terminal window whose width was 90 characters. -->
|
|
119
|
+
|
|
120
|
+
Each command has its own `--help` snippet.
|
|
121
|
+
|
|
122
|
+
#### The `scan` command
|
|
123
|
+
|
|
124
|
+
At the time of this writing, the `--help` snippet for the `scan` command is:
|
|
125
|
+
|
|
126
|
+
```console
|
|
127
|
+
Usage: refscan scan [OPTIONS]
|
|
128
|
+
|
|
129
|
+
Scan the NMDC MongoDB database for referential integrity violations.
|
|
121
130
|
|
|
122
131
|
╭─ Options ──────────────────────────────────────────────────────────────────────────────╮
|
|
123
132
|
│ * --schema FILE Filesystem path at which the YAML file │
|
|
@@ -147,7 +156,6 @@ At the time of this writing, the tool's `--help` snippet is:
|
|
|
147
156
|
│ program to generate its violation │
|
|
148
157
|
│ report. │
|
|
149
158
|
│ [default: violations.tsv] │
|
|
150
|
-
│ --version Show version number and exit. │
|
|
151
159
|
│ --no-scan Generate a reference report, but do │
|
|
152
160
|
│ not scan the database for violations. │
|
|
153
161
|
│ --locate-misplaced-documents For each referenced document not found │
|
|
@@ -158,9 +166,9 @@ At the time of this writing, the tool's `--help` snippet is:
|
|
|
158
166
|
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
159
167
|
```
|
|
160
168
|
|
|
161
|
-
|
|
169
|
+
<!-- Note: The above snippet was captured from a terminal window whose width was 90 characters. -->
|
|
162
170
|
|
|
163
|
-
|
|
171
|
+
##### The MongoDB connection string (`--mongo-uri`)
|
|
164
172
|
|
|
165
173
|
As documented in the `--help` snippet above, you can provide the MongoDB connection string to the tool via either
|
|
166
174
|
(a) the `--mongo-uri` option; or (b) an environment variable named `MONGO_URI`. The latter can come in handy
|
|
@@ -173,7 +181,7 @@ Here's how you could create that environment variable:
|
|
|
173
181
|
export MONGO_URI='mongodb://username:password@localhost:27017'
|
|
174
182
|
```
|
|
175
183
|
|
|
176
|
-
|
|
184
|
+
##### The schema (`--schema`)
|
|
177
185
|
|
|
178
186
|
As documented in the `--help` snippet above, you can provide the path to a YAML-formatted LinkML schema file to the tool
|
|
179
187
|
via the `--schema` option.
|
|
@@ -197,14 +205,17 @@ curl -o schema.yaml https://raw.githubusercontent.com/{user_or_org}/{repo}/{bran
|
|
|
197
205
|
For example:
|
|
198
206
|
|
|
199
207
|
```shell
|
|
200
|
-
# Download the raw content of https://github.com/microbiomedata/
|
|
201
|
-
curl -o schema.yaml https://raw.githubusercontent.com/microbiomedata/
|
|
208
|
+
# Download the raw content of https://github.com/microbiomedata/nmdc-schema/blob/main/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
209
|
+
curl -o schema.yaml https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
210
|
+
|
|
211
|
+
# Download the raw content of https://github.com/microbiomedata/nmdc-schema/blob/v11.2.1/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
212
|
+
curl -o schema.yaml https://raw.githubusercontent.com/microbiomedata/nmdc-schema/v11.2.1/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
202
213
|
```
|
|
203
214
|
|
|
204
215
|
---
|
|
205
216
|
</details>
|
|
206
217
|
|
|
207
|
-
|
|
218
|
+
##### Output
|
|
208
219
|
|
|
209
220
|
While `refscan` is running, it will display console output indicating what it's currently doing.
|
|
210
221
|
|
|
@@ -213,6 +224,33 @@ While `refscan` is running, it will display console output indicating what it's
|
|
|
213
224
|
Once the scan is complete, the reference report (TSV file) and violation report (TSV file) will be available
|
|
214
225
|
in the current directory (or in custom directories, if any were specified via CLI options).
|
|
215
226
|
|
|
227
|
+
#### The `graph` command
|
|
228
|
+
|
|
229
|
+
At the time of this writing, the `--help` snippet for the `graph` command is:
|
|
230
|
+
|
|
231
|
+
```console
|
|
232
|
+
Usage: refscan graph [OPTIONS]
|
|
233
|
+
|
|
234
|
+
Generate an interactive graph of the references described by a schema.
|
|
235
|
+
|
|
236
|
+
╭─ Options ──────────────────────────────────────────────────────────────────────────────╮
|
|
237
|
+
│ * --schema FILE Filesystem path at which the YAML file │
|
|
238
|
+
│ representing the schema is located. │
|
|
239
|
+
│ [default: None] │
|
|
240
|
+
│ [required] │
|
|
241
|
+
│ --graph FILE Filesystem path at which you want refscan to │
|
|
242
|
+
│ generate the graph. │
|
|
243
|
+
│ [default: graph.html] │
|
|
244
|
+
│ --subject [collection|class] Whether you want each node of the graph to │
|
|
245
|
+
│ represent a collection or a class. │
|
|
246
|
+
│ [default: collection] │
|
|
247
|
+
│ --verbose Show verbose output. │
|
|
248
|
+
│ --help Show this message and exit. │
|
|
249
|
+
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
<!-- Note: The above snippet was captured from a terminal window whose width was 90 characters. -->
|
|
253
|
+
|
|
216
254
|
### Update
|
|
217
255
|
|
|
218
256
|
You can update the tool to [the latest version available on PyPI](https://pypi.org/project/refscan/) by running:
|
|
@@ -272,7 +310,7 @@ Edit the tool's source code and documentation however you want.
|
|
|
272
310
|
While editing the tool's source code, you can run the tool as you normally would in order to test things out.
|
|
273
311
|
|
|
274
312
|
```shell
|
|
275
|
-
refscan --help
|
|
313
|
+
poetry run refscan --help
|
|
276
314
|
```
|
|
277
315
|
|
|
278
316
|
### Run tests
|
|
@@ -332,20 +370,3 @@ poetry build
|
|
|
332
370
|
> [wheel](https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format)
|
|
333
371
|
> file (whose name ends with `.whl`) in the `dist` directory.
|
|
334
372
|
|
|
335
|
-
## Appendix
|
|
336
|
-
|
|
337
|
-
### refgraph
|
|
338
|
-
|
|
339
|
-
When `pipx` installs `refscan`, it also installs a program called `refgraph`. `refgraph` is a program you can use to
|
|
340
|
-
generate a web-based, interactive graph (network diagram) of the relationships that can exist between documents in a
|
|
341
|
-
database that conforms to a given schema. It can present the relationships in terms of either database collections
|
|
342
|
-
or schema classes.
|
|
343
|
-
|
|
344
|
-
You can learn more about `refgraph` by running:
|
|
345
|
-
|
|
346
|
-
```shell
|
|
347
|
-
refgraph --help
|
|
348
|
-
```
|
|
349
|
-
|
|
350
|
-
> Note: `refgraph` is still in early development and its features and CLI are subject to change.
|
|
351
|
-
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# refscan
|
|
2
2
|
|
|
3
|
-
`refscan` is a command-line tool people can use to scan the [NMDC](https://microbiomedata.org/) MongoDB database
|
|
3
|
+
`refscan` is a command-line tool people can use to **scan** the [NMDC](https://microbiomedata.org/) MongoDB database
|
|
4
4
|
for referential integrity violations.
|
|
5
5
|
|
|
6
6
|
```mermaid
|
|
@@ -10,7 +10,7 @@ for referential integrity violations.
|
|
|
10
10
|
graph LR
|
|
11
11
|
schema[LinkML<br>schema]
|
|
12
12
|
database[(MongoDB<br>database)]
|
|
13
|
-
script[["refscan
|
|
13
|
+
script[["refscan"]]
|
|
14
14
|
violations["List of<br>violations"]
|
|
15
15
|
references["List of<br>references"]:::dashed_border
|
|
16
16
|
schema --> script
|
|
@@ -21,39 +21,20 @@ graph LR
|
|
|
21
21
|
classDef dashed_border stroke-dasharray: 5 5
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
* [Limitations](#limitations)
|
|
31
|
-
* [Usage](#usage)
|
|
32
|
-
* [Install](#install)
|
|
33
|
-
* [Run](#run)
|
|
34
|
-
* [The MongoDB connection string (`--mongo-uri`)](#the-mongodb-connection-string---mongo-uri)
|
|
35
|
-
* [The schema (`--schema`)](#the-schema---schema)
|
|
36
|
-
* [Output](#output)
|
|
37
|
-
* [Update](#update)
|
|
38
|
-
* [Uninstall](#uninstall)
|
|
39
|
-
* [Development](#development)
|
|
40
|
-
* [Clone repository](#clone-repository)
|
|
41
|
-
* [Create virtual environment](#create-virtual-environment)
|
|
42
|
-
* [Install dependencies](#install-dependencies)
|
|
43
|
-
* [Make changes](#make-changes)
|
|
44
|
-
* [Run tests](#run-tests)
|
|
45
|
-
* [Format code](#format-code)
|
|
46
|
-
* [Check format](#check-format)
|
|
47
|
-
* [Building and publishing](#building-and-publishing)
|
|
48
|
-
* [Build for production](#build-for-production)
|
|
49
|
-
* [Test the build process locally](#test-the-build-process-locally)
|
|
50
|
-
* [Appendix](#appendix)
|
|
51
|
-
* [refgraph](#refgraph)
|
|
52
|
-
<!-- TOC -->
|
|
24
|
+
In addition to using refscan to scan the NMDC MongoDB database for referential integrity violations,
|
|
25
|
+
people can use `refscan` to generate **graphs** (diagrams) depicting which collections' documents (or which classes'
|
|
26
|
+
instances) can contain references to which _other_ collections' documents (or classes' instances) while still being
|
|
27
|
+
schema compliant.
|
|
28
|
+
|
|
29
|
+
<!-- Note: We removed the hard-coded Table of Contents because—nowadays—GitHub automatically derives/presents one. -->
|
|
53
30
|
|
|
54
31
|
## How it works
|
|
55
32
|
|
|
56
|
-
`refscan`
|
|
33
|
+
Here is a summary of how each of `refscan`'s main functions works under the hood.
|
|
34
|
+
|
|
35
|
+
### Scan
|
|
36
|
+
|
|
37
|
+
`refscan` does this in two stages:
|
|
57
38
|
1. It uses the LinkML schema to determine where references _can_ exist in a MongoDB database that conforms to the schema.
|
|
58
39
|
> **Example:** The schema might say that, if a document in the `biosample_set` collection has a field named
|
|
59
40
|
> `associated_studies`, that field must contain a list of `id`s of documents in the `study_set` collection.
|
|
@@ -61,12 +42,19 @@ graph LR
|
|
|
61
42
|
> **Example:** For each document in the `biosample_set` collection that _has_ a field named `associated_studies`,
|
|
62
43
|
> for each value in that field, confirm there _is_ a document having that `id` in the `study_set` collection.
|
|
63
44
|
|
|
64
|
-
|
|
45
|
+
### Graph
|
|
46
|
+
|
|
47
|
+
`refscan` does this in three stages:
|
|
48
|
+
1. It uses the LinkML schema to determine where references _can_ exist in a MongoDB database that conforms to the schema.
|
|
49
|
+
2. It formats that list of references into a data structure compatible with [`Cytoscape.js`](https://js.cytoscape.org/).
|
|
50
|
+
3. It outputs an HTML document that uses `Cytoscape.js` to visualize that data structure as a graph.
|
|
51
|
+
|
|
52
|
+
## Assumptions
|
|
65
53
|
|
|
66
54
|
`refscan` was designed under the assumption that **every document** in **every collection described by the schema** has
|
|
67
55
|
a **field named `type`**, whose value is the [class_uri](https://linkml.io/linkml/code/metamodel.html#linkml_runtime.linkml_model.meta.ClassDefinition.class_uri) of the schema class the document represents an instance
|
|
68
|
-
of. `refscan` uses that `class_uri` value (in that `type` field) to determine the
|
|
69
|
-
|
|
56
|
+
of. `refscan` uses that `class_uri` value (in that `type` field) to determine the _name_ of that schema class,
|
|
57
|
+
whose definition `refscan` then uses to determine _which fields_ of that document can contain references.
|
|
70
58
|
|
|
71
59
|
## Usage
|
|
72
60
|
|
|
@@ -85,7 +73,7 @@ pipx install refscan
|
|
|
85
73
|
|
|
86
74
|
### Run
|
|
87
75
|
|
|
88
|
-
|
|
76
|
+
Once installed, you can display the tool's `--help` snippet by running:
|
|
89
77
|
|
|
90
78
|
```shell
|
|
91
79
|
refscan --help
|
|
@@ -94,9 +82,30 @@ refscan --help
|
|
|
94
82
|
At the time of this writing, the tool's `--help` snippet is:
|
|
95
83
|
|
|
96
84
|
```console
|
|
97
|
-
Usage: refscan [OPTIONS]
|
|
85
|
+
Usage: refscan [OPTIONS] COMMAND [ARGS]...
|
|
98
86
|
|
|
99
|
-
|
|
87
|
+
╭─ Options ──────────────────────────────────────────────────────────────────────────────╮
|
|
88
|
+
│ --help Show this message and exit. │
|
|
89
|
+
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
90
|
+
╭─ Commands ─────────────────────────────────────────────────────────────────────────────╮
|
|
91
|
+
│ version Show version number and exit. │
|
|
92
|
+
│ scan Scan the NMDC MongoDB database for referential integrity violations. │
|
|
93
|
+
│ graph Generate an interactive graph of the references described by a schema. │
|
|
94
|
+
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
<!-- Note: The above snippet was captured from a terminal window whose width was 90 characters. -->
|
|
98
|
+
|
|
99
|
+
Each command has its own `--help` snippet.
|
|
100
|
+
|
|
101
|
+
#### The `scan` command
|
|
102
|
+
|
|
103
|
+
At the time of this writing, the `--help` snippet for the `scan` command is:
|
|
104
|
+
|
|
105
|
+
```console
|
|
106
|
+
Usage: refscan scan [OPTIONS]
|
|
107
|
+
|
|
108
|
+
Scan the NMDC MongoDB database for referential integrity violations.
|
|
100
109
|
|
|
101
110
|
╭─ Options ──────────────────────────────────────────────────────────────────────────────╮
|
|
102
111
|
│ * --schema FILE Filesystem path at which the YAML file │
|
|
@@ -126,7 +135,6 @@ At the time of this writing, the tool's `--help` snippet is:
|
|
|
126
135
|
│ program to generate its violation │
|
|
127
136
|
│ report. │
|
|
128
137
|
│ [default: violations.tsv] │
|
|
129
|
-
│ --version Show version number and exit. │
|
|
130
138
|
│ --no-scan Generate a reference report, but do │
|
|
131
139
|
│ not scan the database for violations. │
|
|
132
140
|
│ --locate-misplaced-documents For each referenced document not found │
|
|
@@ -137,9 +145,9 @@ At the time of this writing, the tool's `--help` snippet is:
|
|
|
137
145
|
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
138
146
|
```
|
|
139
147
|
|
|
140
|
-
|
|
148
|
+
<!-- Note: The above snippet was captured from a terminal window whose width was 90 characters. -->
|
|
141
149
|
|
|
142
|
-
|
|
150
|
+
##### The MongoDB connection string (`--mongo-uri`)
|
|
143
151
|
|
|
144
152
|
As documented in the `--help` snippet above, you can provide the MongoDB connection string to the tool via either
|
|
145
153
|
(a) the `--mongo-uri` option; or (b) an environment variable named `MONGO_URI`. The latter can come in handy
|
|
@@ -152,7 +160,7 @@ Here's how you could create that environment variable:
|
|
|
152
160
|
export MONGO_URI='mongodb://username:password@localhost:27017'
|
|
153
161
|
```
|
|
154
162
|
|
|
155
|
-
|
|
163
|
+
##### The schema (`--schema`)
|
|
156
164
|
|
|
157
165
|
As documented in the `--help` snippet above, you can provide the path to a YAML-formatted LinkML schema file to the tool
|
|
158
166
|
via the `--schema` option.
|
|
@@ -176,14 +184,17 @@ curl -o schema.yaml https://raw.githubusercontent.com/{user_or_org}/{repo}/{bran
|
|
|
176
184
|
For example:
|
|
177
185
|
|
|
178
186
|
```shell
|
|
179
|
-
# Download the raw content of https://github.com/microbiomedata/
|
|
180
|
-
curl -o schema.yaml https://raw.githubusercontent.com/microbiomedata/
|
|
187
|
+
# Download the raw content of https://github.com/microbiomedata/nmdc-schema/blob/main/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
188
|
+
curl -o schema.yaml https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
189
|
+
|
|
190
|
+
# Download the raw content of https://github.com/microbiomedata/nmdc-schema/blob/v11.2.1/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
191
|
+
curl -o schema.yaml https://raw.githubusercontent.com/microbiomedata/nmdc-schema/v11.2.1/nmdc_schema/nmdc_materialized_patterns.yaml
|
|
181
192
|
```
|
|
182
193
|
|
|
183
194
|
---
|
|
184
195
|
</details>
|
|
185
196
|
|
|
186
|
-
|
|
197
|
+
##### Output
|
|
187
198
|
|
|
188
199
|
While `refscan` is running, it will display console output indicating what it's currently doing.
|
|
189
200
|
|
|
@@ -192,6 +203,33 @@ While `refscan` is running, it will display console output indicating what it's
|
|
|
192
203
|
Once the scan is complete, the reference report (TSV file) and violation report (TSV file) will be available
|
|
193
204
|
in the current directory (or in custom directories, if any were specified via CLI options).
|
|
194
205
|
|
|
206
|
+
#### The `graph` command
|
|
207
|
+
|
|
208
|
+
At the time of this writing, the `--help` snippet for the `graph` command is:
|
|
209
|
+
|
|
210
|
+
```console
|
|
211
|
+
Usage: refscan graph [OPTIONS]
|
|
212
|
+
|
|
213
|
+
Generate an interactive graph of the references described by a schema.
|
|
214
|
+
|
|
215
|
+
╭─ Options ──────────────────────────────────────────────────────────────────────────────╮
|
|
216
|
+
│ * --schema FILE Filesystem path at which the YAML file │
|
|
217
|
+
│ representing the schema is located. │
|
|
218
|
+
│ [default: None] │
|
|
219
|
+
│ [required] │
|
|
220
|
+
│ --graph FILE Filesystem path at which you want refscan to │
|
|
221
|
+
│ generate the graph. │
|
|
222
|
+
│ [default: graph.html] │
|
|
223
|
+
│ --subject [collection|class] Whether you want each node of the graph to │
|
|
224
|
+
│ represent a collection or a class. │
|
|
225
|
+
│ [default: collection] │
|
|
226
|
+
│ --verbose Show verbose output. │
|
|
227
|
+
│ --help Show this message and exit. │
|
|
228
|
+
╰────────────────────────────────────────────────────────────────────────────────────────╯
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
<!-- Note: The above snippet was captured from a terminal window whose width was 90 characters. -->
|
|
232
|
+
|
|
195
233
|
### Update
|
|
196
234
|
|
|
197
235
|
You can update the tool to [the latest version available on PyPI](https://pypi.org/project/refscan/) by running:
|
|
@@ -251,7 +289,7 @@ Edit the tool's source code and documentation however you want.
|
|
|
251
289
|
While editing the tool's source code, you can run the tool as you normally would in order to test things out.
|
|
252
290
|
|
|
253
291
|
```shell
|
|
254
|
-
refscan --help
|
|
292
|
+
poetry run refscan --help
|
|
255
293
|
```
|
|
256
294
|
|
|
257
295
|
### Run tests
|
|
@@ -310,20 +348,3 @@ poetry build
|
|
|
310
348
|
> file (whose name ends with `.tar.gz`) and a
|
|
311
349
|
> [wheel](https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format)
|
|
312
350
|
> file (whose name ends with `.whl`) in the `dist` directory.
|
|
313
|
-
|
|
314
|
-
## Appendix
|
|
315
|
-
|
|
316
|
-
### refgraph
|
|
317
|
-
|
|
318
|
-
When `pipx` installs `refscan`, it also installs a program called `refgraph`. `refgraph` is a program you can use to
|
|
319
|
-
generate a web-based, interactive graph (network diagram) of the relationships that can exist between documents in a
|
|
320
|
-
database that conforms to a given schema. It can present the relationships in terms of either database collections
|
|
321
|
-
or schema classes.
|
|
322
|
-
|
|
323
|
-
You can learn more about `refgraph` by running:
|
|
324
|
-
|
|
325
|
-
```shell
|
|
326
|
-
refgraph --help
|
|
327
|
-
```
|
|
328
|
-
|
|
329
|
-
> Note: `refgraph` is still in early development and its features and CLI are subject to change.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[tool.poetry]
|
|
4
4
|
name = "refscan"
|
|
5
|
-
version = "v0.
|
|
5
|
+
version = "v0.2.0"
|
|
6
6
|
description = "Command-line program that scans the NMDC MongoDB database for referential integrity violations"
|
|
7
7
|
authors = []
|
|
8
8
|
readme = "README.md"
|
|
@@ -24,7 +24,7 @@ pymongo = "^4.7.1"
|
|
|
24
24
|
rich = "^13.7.1"
|
|
25
25
|
# We use `typer` as a CLI framework.
|
|
26
26
|
# Docs: https://typer.tiangolo.com/
|
|
27
|
-
typer = "^0.
|
|
27
|
+
typer = "^0.15.1"
|
|
28
28
|
|
|
29
29
|
[tool.poetry.group.dev.dependencies]
|
|
30
30
|
# We use `black` for code formatting.
|
|
@@ -43,14 +43,15 @@ pytest-cov = "^5.0.0"
|
|
|
43
43
|
|
|
44
44
|
[tool.poetry.scripts]
|
|
45
45
|
# Reference: https://python-poetry.org/docs/pyproject#scripts
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
# Reference: https://typer.tiangolo.com/tutorial/package/#add-a-script
|
|
47
|
+
refscan = "refscan.cli.cli:app"
|
|
48
48
|
|
|
49
49
|
[tool.pytest.ini_options]
|
|
50
|
-
# Make it so that, when someone runs `$ poetry run pytest`, in addition to running the tests, it
|
|
51
|
-
#
|
|
52
|
-
# Reference: https://pytest
|
|
53
|
-
|
|
50
|
+
# Make it so that, when someone runs `$ poetry run pytest`, in addition to running the tests, it also (a) runs doctests
|
|
51
|
+
# defined in Python modules, and (b) measures—via the `pytest-cov` plugin—test coverage within the `refscan` folder.
|
|
52
|
+
# Reference (a): https://docs.pytest.org/en/stable/how-to/doctest.html#how-to-run-doctests
|
|
53
|
+
# Reference (b): https://pytest-cov.readthedocs.io/en/latest/config.html#configuration
|
|
54
|
+
addopts = "--doctest-modules --cov=refscan"
|
|
54
55
|
|
|
55
56
|
[tool.black]
|
|
56
57
|
# Make it so that, when someone runs `$ poetry run black`, these default CLI options are used.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import typer
|
|
2
|
+
|
|
3
|
+
from refscan.cli.version import version
|
|
4
|
+
from refscan.cli.scan import scan
|
|
5
|
+
from refscan.cli.graph import graph
|
|
6
|
+
|
|
7
|
+
app = typer.Typer(
|
|
8
|
+
no_args_is_help=True, # treats the absence of args like the `--help` arg
|
|
9
|
+
add_completion=False, # hides the shell completion options from `--help` output
|
|
10
|
+
rich_markup_mode="markdown", # enables use of Markdown in docstrings and CLI help
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
# Add commands—implemented in other modules—to our Typer app.
|
|
14
|
+
# Reference: https://typer.tiangolo.com/tutorial/one-file-per-command/
|
|
15
|
+
#
|
|
16
|
+
# Note: Instead of annotating (with a decorator) a function defined in this module, we decorate a function
|
|
17
|
+
# that we import from a different module. While both approaches result in the "no-longer-bare" function
|
|
18
|
+
# being defined in this module, only the latter approach allows for the "bare" function to be defined
|
|
19
|
+
# in a different module (which I think will facilitate code maintenance).
|
|
20
|
+
# Reference: https://github.com/fastapi/typer/issues/178#issuecomment-738853400
|
|
21
|
+
#
|
|
22
|
+
app.command()(version)
|
|
23
|
+
app.command()(scan)
|
|
24
|
+
app.command()(graph)
|
|
25
|
+
|
|
26
|
+
if __name__ == "__main__":
|
|
27
|
+
app()
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Optional
|
|
3
|
+
from typing_extensions import Annotated
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
import linkml_runtime
|
|
7
|
+
|
|
8
|
+
from refscan import grapher
|
|
9
|
+
from refscan.lib.constants import console
|
|
10
|
+
from refscan.lib.helpers import print_section_header
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def graph(
|
|
14
|
+
# Reference: https://typer.tiangolo.com/tutorial/parameter-types/path/
|
|
15
|
+
schema_file_path: Annotated[
|
|
16
|
+
Path,
|
|
17
|
+
typer.Option(
|
|
18
|
+
"--schema",
|
|
19
|
+
dir_okay=False,
|
|
20
|
+
writable=False,
|
|
21
|
+
readable=True,
|
|
22
|
+
resolve_path=True,
|
|
23
|
+
help="Filesystem path at which the YAML file representing the schema is located.",
|
|
24
|
+
),
|
|
25
|
+
],
|
|
26
|
+
graph_file_path: Annotated[
|
|
27
|
+
Optional[Path],
|
|
28
|
+
typer.Option(
|
|
29
|
+
"--graph",
|
|
30
|
+
dir_okay=False,
|
|
31
|
+
writable=True,
|
|
32
|
+
readable=False,
|
|
33
|
+
resolve_path=True,
|
|
34
|
+
help="Filesystem path at which you want **refscan** to generate the graph.",
|
|
35
|
+
),
|
|
36
|
+
] = "graph.html",
|
|
37
|
+
subject: Annotated[
|
|
38
|
+
grapher.Subject,
|
|
39
|
+
typer.Option(
|
|
40
|
+
"--subject",
|
|
41
|
+
case_sensitive=False,
|
|
42
|
+
help="Whether you want each node of the graph to represent a collection or a class.",
|
|
43
|
+
),
|
|
44
|
+
] = grapher.Subject.collection,
|
|
45
|
+
verbose: Annotated[
|
|
46
|
+
bool,
|
|
47
|
+
typer.Option(
|
|
48
|
+
"--verbose",
|
|
49
|
+
help="Show verbose output.",
|
|
50
|
+
),
|
|
51
|
+
] = False,
|
|
52
|
+
):
|
|
53
|
+
r"""
|
|
54
|
+
Generate an interactive graph of the references described by a schema.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
print_section_header(console, text="Reading schema")
|
|
58
|
+
|
|
59
|
+
# Instantiate a `linkml_runtime.SchemaView` bound to the specified schema.
|
|
60
|
+
if verbose:
|
|
61
|
+
console.print(f"Schema YAML file: {schema_file_path}")
|
|
62
|
+
schema_view = linkml_runtime.SchemaView(schema_file_path)
|
|
63
|
+
|
|
64
|
+
# Show high-level information about the schema.
|
|
65
|
+
console.print(f"Schema version: {schema_view.schema.version}")
|
|
66
|
+
|
|
67
|
+
html_result = grapher.graph(
|
|
68
|
+
schema_view=schema_view,
|
|
69
|
+
subject=subject,
|
|
70
|
+
verbose=verbose,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
if verbose:
|
|
74
|
+
console.print(html_result)
|
|
75
|
+
|
|
76
|
+
with open(graph_file_path, "w") as f:
|
|
77
|
+
f.write(html_result)
|
|
78
|
+
|
|
79
|
+
console.print(f"Graph generated at: {graph_file_path}")
|
|
80
|
+
console.print()
|