refscan 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {refscan-0.3.2 → refscan-0.3.3}/PKG-INFO +52 -40
- {refscan-0.3.2 → refscan-0.3.3}/README.md +46 -27
- refscan-0.3.3/pyproject.toml +121 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/cli/graph.py +2 -3
- {refscan-0.3.2 → refscan-0.3.3}/refscan/cli/scan.py +5 -5
- {refscan-0.3.2 → refscan-0.3.3}/refscan/lib/Finder.py +0 -3
- {refscan-0.3.2 → refscan-0.3.3}/refscan/lib/ReferenceList.py +0 -1
- {refscan-0.3.2 → refscan-0.3.3}/refscan/lib/helpers.py +0 -1
- {refscan-0.3.2 → refscan-0.3.3}/refscan/scanner.py +1 -5
- refscan-0.3.2/LICENSE +0 -42
- refscan-0.3.2/pyproject.toml +0 -89
- {refscan-0.3.2 → refscan-0.3.3}/refscan/__init__.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/cli/cli.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/cli/version.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/grapher.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/lib/Reference.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/lib/Violation.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/lib/ViolationList.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/lib/constants.py +0 -0
- {refscan-0.3.2 → refscan-0.3.3}/refscan/templates/graph.template.html +0 -0
|
@@ -1,19 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: refscan
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Command-line program that scans the NMDC MongoDB database for referential integrity violations
|
|
5
5
|
Keywords: mongodb,mongo,relationships,reference,database,data,referential integrity,scan
|
|
6
|
-
Requires-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
-
Requires-Dist: linkml-runtime (>=1.7.5,<2.0.0)
|
|
14
|
-
Requires-Dist: pymongo (>=4.7.1,<5.0.0)
|
|
15
|
-
Requires-Dist: rich (>=13.7.1,<14.0.0)
|
|
16
|
-
Requires-Dist: typer (>=0.15.1,<0.16.0)
|
|
6
|
+
Requires-Dist: linkml-runtime>=1.7.5,<2
|
|
7
|
+
Requires-Dist: pymongo>=4.7.1,<5
|
|
8
|
+
Requires-Dist: rich>=13.7.1,<14
|
|
9
|
+
Requires-Dist: typer>=0.15.1,<0.16
|
|
10
|
+
Requires-Python: >=3.9, <3.10
|
|
17
11
|
Project-URL: Documentation, https://github.com/microbiomedata/refscan
|
|
18
12
|
Project-URL: Homepage, https://github.com/microbiomedata/refscan
|
|
19
13
|
Project-URL: Repository, https://github.com/microbiomedata/refscan
|
|
@@ -269,10 +263,12 @@ pipx uninstall refscan
|
|
|
269
263
|
|
|
270
264
|
## Development
|
|
271
265
|
|
|
272
|
-
We use [
|
|
266
|
+
We use [`uv`](https://docs.astral.sh/uv/) to both (a) manage dependencies and (b) build distributable packages that can be published to PyPI.
|
|
273
267
|
|
|
274
|
-
- `pyproject.toml`: Configuration file for
|
|
275
|
-
- `
|
|
268
|
+
- `pyproject.toml`: [Configuration file](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) for `uv` and other tools
|
|
269
|
+
- `uv.lock`: List of dependencies, both direct and [indirect/transitive](https://en.wikipedia.org/wiki/Transitive_dependency)
|
|
270
|
+
|
|
271
|
+
> Note: We initialized this repository using Poetry. We switched from Poetry to `uv` at around commit `#1449ceca`.
|
|
276
272
|
|
|
277
273
|
### Clone repository
|
|
278
274
|
|
|
@@ -281,28 +277,29 @@ git clone https://github.com/microbiomedata/refscan.git
|
|
|
281
277
|
cd refscan
|
|
282
278
|
```
|
|
283
279
|
|
|
284
|
-
###
|
|
280
|
+
### Set up Python virtual environment
|
|
285
281
|
|
|
286
|
-
|
|
282
|
+
You can set up a Python virtual environment by issuing the following command from the root directory of the repository:
|
|
287
283
|
|
|
288
284
|
```shell
|
|
289
|
-
|
|
285
|
+
uv sync
|
|
290
286
|
```
|
|
291
287
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
288
|
+
That command will:
|
|
289
|
+
1. **Create a Python virtual environment** at `.venv` (if one doesn't already exist there)
|
|
290
|
+
2. **Install all dependencies** described in `uv.lock` into that Python virtual environment
|
|
291
|
+
3. Uninstall all dependencies _not_ described in `uv.lock` from that Python virtual environment
|
|
295
292
|
|
|
296
|
-
|
|
293
|
+
### Activate Python virtual environment
|
|
297
294
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
At the Poetry shell, install the project's dependencies:
|
|
295
|
+
Now that you have set up a Python virtual environment, you can activate it by issuing the following command:
|
|
301
296
|
|
|
302
297
|
```shell
|
|
303
|
-
|
|
298
|
+
source .venv/bin/activate
|
|
304
299
|
```
|
|
305
300
|
|
|
301
|
+
> Note: Once you're ready to _deactivate_ the Python virtual environment, you can do so by running `$ deactivate`.
|
|
302
|
+
|
|
306
303
|
### Make changes
|
|
307
304
|
|
|
308
305
|
Edit the tool's source code and documentation however you want.
|
|
@@ -310,7 +307,7 @@ Edit the tool's source code and documentation however you want.
|
|
|
310
307
|
While editing the tool's source code, you can run the tool as you normally would in order to test things out.
|
|
311
308
|
|
|
312
309
|
```shell
|
|
313
|
-
|
|
310
|
+
uv run refscan --help
|
|
314
311
|
```
|
|
315
312
|
|
|
316
313
|
### Check types
|
|
@@ -320,7 +317,7 @@ We use [mypy](https://mypy.readthedocs.io/en/stable/) as the static type checker
|
|
|
320
317
|
You can perform static type checking by running the following command from the root directory of the repository:
|
|
321
318
|
|
|
322
319
|
```shell
|
|
323
|
-
|
|
320
|
+
uv run mypy
|
|
324
321
|
```
|
|
325
322
|
|
|
326
323
|
### Run tests
|
|
@@ -332,29 +329,45 @@ Tests are defined in the `tests` directory.
|
|
|
332
329
|
You can run the tests by running the following command from the root directory of the repository:
|
|
333
330
|
|
|
334
331
|
```shell
|
|
335
|
-
|
|
332
|
+
uv run pytest
|
|
336
333
|
```
|
|
337
334
|
|
|
338
335
|
### Format code
|
|
339
336
|
|
|
340
|
-
We use [`
|
|
337
|
+
We use [`ruff`](https://docs.astral.sh/ruff/formatter/) as the code _formatter_ for `refscan`.
|
|
341
338
|
|
|
342
|
-
We
|
|
343
|
-
|
|
339
|
+
We mostly use it with its default rules. All of the ways we deviate from those are listed
|
|
340
|
+
in the `[tool.ruff]` section of `pyproject.toml`.
|
|
344
341
|
|
|
345
|
-
You can
|
|
346
|
-
from the root directory of the repository:
|
|
342
|
+
You can _check_ the code's compliance with the "formatter rules" by running this command from the root directory of the repository:
|
|
347
343
|
|
|
348
344
|
```shell
|
|
349
|
-
|
|
345
|
+
uv run ruff format --check
|
|
350
346
|
```
|
|
351
347
|
|
|
352
|
-
|
|
348
|
+
That will output a _list_ of files that don't comply. To see the violations, themselves, you can run:
|
|
353
349
|
|
|
354
|
-
|
|
350
|
+
```shell
|
|
351
|
+
uv run ruff format --diff
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
You can _format_ the code by omitting the `--check` and `--diff` flags:
|
|
355
355
|
|
|
356
356
|
```shell
|
|
357
|
-
|
|
357
|
+
uv run ruff format
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
### Lint code
|
|
361
|
+
|
|
362
|
+
We also use [`ruff`](https://docs.astral.sh/ruff/linter/) as the code _linter_ for `refscan`.
|
|
363
|
+
|
|
364
|
+
We use it with its [default rules](https://docs.astral.sh/ruff/rules/), **plus** some additional ones,
|
|
365
|
+
all of which are listed in the `[tool.ruff.lint]` section of `pyproject.toml`.
|
|
366
|
+
|
|
367
|
+
You can _check_ the code's compliance with the "linter rules" by running this command from the root directory of the repository:
|
|
368
|
+
|
|
369
|
+
```shell
|
|
370
|
+
uv run ruff check
|
|
358
371
|
```
|
|
359
372
|
|
|
360
373
|
## Building and publishing
|
|
@@ -371,7 +384,7 @@ That package will have a version identifier that matches the name of the Git tag
|
|
|
371
384
|
In case you want to test the build process locally, you can do so by running:
|
|
372
385
|
|
|
373
386
|
```shell
|
|
374
|
-
|
|
387
|
+
uv build
|
|
375
388
|
```
|
|
376
389
|
|
|
377
390
|
> That will create both a
|
|
@@ -379,4 +392,3 @@ poetry build
|
|
|
379
392
|
> file (whose name ends with `.tar.gz`) and a
|
|
380
393
|
> [wheel](https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format)
|
|
381
394
|
> file (whose name ends with `.whl`) in the `dist` directory.
|
|
382
|
-
|
|
@@ -248,10 +248,12 @@ pipx uninstall refscan
|
|
|
248
248
|
|
|
249
249
|
## Development
|
|
250
250
|
|
|
251
|
-
We use [
|
|
251
|
+
We use [`uv`](https://docs.astral.sh/uv/) to both (a) manage dependencies and (b) build distributable packages that can be published to PyPI.
|
|
252
252
|
|
|
253
|
-
- `pyproject.toml`: Configuration file for
|
|
254
|
-
- `
|
|
253
|
+
- `pyproject.toml`: [Configuration file](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) for `uv` and other tools
|
|
254
|
+
- `uv.lock`: List of dependencies, both direct and [indirect/transitive](https://en.wikipedia.org/wiki/Transitive_dependency)
|
|
255
|
+
|
|
256
|
+
> Note: We initialized this repository using Poetry. We switched from Poetry to `uv` at around commit `#1449ceca`.
|
|
255
257
|
|
|
256
258
|
### Clone repository
|
|
257
259
|
|
|
@@ -260,28 +262,29 @@ git clone https://github.com/microbiomedata/refscan.git
|
|
|
260
262
|
cd refscan
|
|
261
263
|
```
|
|
262
264
|
|
|
263
|
-
###
|
|
265
|
+
### Set up Python virtual environment
|
|
264
266
|
|
|
265
|
-
|
|
267
|
+
You can set up a Python virtual environment by issuing the following command from the root directory of the repository:
|
|
266
268
|
|
|
267
269
|
```shell
|
|
268
|
-
|
|
270
|
+
uv sync
|
|
269
271
|
```
|
|
270
272
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
273
|
+
That command will:
|
|
274
|
+
1. **Create a Python virtual environment** at `.venv` (if one doesn't already exist there)
|
|
275
|
+
2. **Install all dependencies** described in `uv.lock` into that Python virtual environment
|
|
276
|
+
3. Uninstall all dependencies _not_ described in `uv.lock` from that Python virtual environment
|
|
274
277
|
|
|
275
|
-
|
|
278
|
+
### Activate Python virtual environment
|
|
276
279
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
At the Poetry shell, install the project's dependencies:
|
|
280
|
+
Now that you have set up a Python virtual environment, you can activate it by issuing the following command:
|
|
280
281
|
|
|
281
282
|
```shell
|
|
282
|
-
|
|
283
|
+
source .venv/bin/activate
|
|
283
284
|
```
|
|
284
285
|
|
|
286
|
+
> Note: Once you're ready to _deactivate_ the Python virtual environment, you can do so by running `$ deactivate`.
|
|
287
|
+
|
|
285
288
|
### Make changes
|
|
286
289
|
|
|
287
290
|
Edit the tool's source code and documentation however you want.
|
|
@@ -289,7 +292,7 @@ Edit the tool's source code and documentation however you want.
|
|
|
289
292
|
While editing the tool's source code, you can run the tool as you normally would in order to test things out.
|
|
290
293
|
|
|
291
294
|
```shell
|
|
292
|
-
|
|
295
|
+
uv run refscan --help
|
|
293
296
|
```
|
|
294
297
|
|
|
295
298
|
### Check types
|
|
@@ -299,7 +302,7 @@ We use [mypy](https://mypy.readthedocs.io/en/stable/) as the static type checker
|
|
|
299
302
|
You can perform static type checking by running the following command from the root directory of the repository:
|
|
300
303
|
|
|
301
304
|
```shell
|
|
302
|
-
|
|
305
|
+
uv run mypy
|
|
303
306
|
```
|
|
304
307
|
|
|
305
308
|
### Run tests
|
|
@@ -311,29 +314,45 @@ Tests are defined in the `tests` directory.
|
|
|
311
314
|
You can run the tests by running the following command from the root directory of the repository:
|
|
312
315
|
|
|
313
316
|
```shell
|
|
314
|
-
|
|
317
|
+
uv run pytest
|
|
315
318
|
```
|
|
316
319
|
|
|
317
320
|
### Format code
|
|
318
321
|
|
|
319
|
-
We use [`
|
|
322
|
+
We use [`ruff`](https://docs.astral.sh/ruff/formatter/) as the code _formatter_ for `refscan`.
|
|
323
|
+
|
|
324
|
+
We mostly use it with its default rules. All of the ways we deviate from those are listed
|
|
325
|
+
in the `[tool.ruff]` section of `pyproject.toml`.
|
|
326
|
+
|
|
327
|
+
You can _check_ the code's compliance with the "formatter rules" by running this command from the root directory of the repository:
|
|
328
|
+
|
|
329
|
+
```shell
|
|
330
|
+
uv run ruff format --check
|
|
331
|
+
```
|
|
320
332
|
|
|
321
|
-
|
|
322
|
-
instead of the default 88 characters. That option is defined in the `[tool.black]` section of `pyproject.toml`.
|
|
333
|
+
That will output a _list_ of files that don't comply. To see the violations, themselves, you can run:
|
|
323
334
|
|
|
324
|
-
|
|
325
|
-
|
|
335
|
+
```shell
|
|
336
|
+
uv run ruff format --diff
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
You can _format_ the code by omitting the `--check` and `--diff` flags:
|
|
326
340
|
|
|
327
341
|
```shell
|
|
328
|
-
|
|
342
|
+
uv run ruff format
|
|
329
343
|
```
|
|
330
344
|
|
|
331
|
-
|
|
345
|
+
### Lint code
|
|
346
|
+
|
|
347
|
+
We also use [`ruff`](https://docs.astral.sh/ruff/linter/) as the code _linter_ for `refscan`.
|
|
348
|
+
|
|
349
|
+
We use it with its [default rules](https://docs.astral.sh/ruff/rules/), **plus** some additional ones,
|
|
350
|
+
all of which are listed in the `[tool.ruff.lint]` section of `pyproject.toml`.
|
|
332
351
|
|
|
333
|
-
You can _check_ the
|
|
352
|
+
You can _check_ the code's compliance with the "linter rules" by running this command from the root directory of the repository:
|
|
334
353
|
|
|
335
354
|
```shell
|
|
336
|
-
|
|
355
|
+
uv run ruff check
|
|
337
356
|
```
|
|
338
357
|
|
|
339
358
|
## Building and publishing
|
|
@@ -350,7 +369,7 @@ That package will have a version identifier that matches the name of the Git tag
|
|
|
350
369
|
In case you want to test the build process locally, you can do so by running:
|
|
351
370
|
|
|
352
371
|
```shell
|
|
353
|
-
|
|
372
|
+
uv build
|
|
354
373
|
```
|
|
355
374
|
|
|
356
375
|
> That will create both a
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "refscan"
|
|
3
|
+
version = "0.3.3"
|
|
4
|
+
description = "Command-line program that scans the NMDC MongoDB database for referential integrity violations"
|
|
5
|
+
authors = []
|
|
6
|
+
# Note: We constrain the Python version to 3.9.* because we know that `nmdc-schema`, which depends upon this
|
|
7
|
+
# package, is also constrained to Python 3.9.* (according to its `pyproject.toml` file).
|
|
8
|
+
# Reference: https://github.com/microbiomedata/nmdc-schema/blob/main/pyproject.toml#L34
|
|
9
|
+
requires-python = ">= 3.9, < 3.10"
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
keywords = [
|
|
12
|
+
"mongodb",
|
|
13
|
+
"mongo",
|
|
14
|
+
"relationships",
|
|
15
|
+
"reference",
|
|
16
|
+
"database",
|
|
17
|
+
"data",
|
|
18
|
+
"referential integrity",
|
|
19
|
+
"scan",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
# We use `linkml-runtime` to facilitate using `nmdc-schema`.
|
|
23
|
+
# Docs: https://linkml.io/linkml/developers/schemaview.html#linkml_runtime.utils.schemaview.SchemaView
|
|
24
|
+
"linkml-runtime >= 1.7.5, < 2",
|
|
25
|
+
# We use `pymongo` to communicate with MongoDB servers.
|
|
26
|
+
# Docs: (https://pymongo.readthedocs.io/en/stable/
|
|
27
|
+
"pymongo >= 4.7.1, < 5",
|
|
28
|
+
# We use `rich` to format console output.
|
|
29
|
+
# Docs: https://rich.readthedocs.io/en/stable/index.html
|
|
30
|
+
"rich >= 13.7.1, < 14",
|
|
31
|
+
# We use `typer` as a CLI framework.
|
|
32
|
+
# Docs: https://typer.tiangolo.com/
|
|
33
|
+
"typer >= 0.15.1, < 0.16",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/microbiomedata/refscan"
|
|
38
|
+
Repository = "https://github.com/microbiomedata/refscan"
|
|
39
|
+
Documentation = "https://github.com/microbiomedata/refscan"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
refscan = "refscan.cli.cli:app"
|
|
43
|
+
|
|
44
|
+
[dependency-groups]
|
|
45
|
+
dev = [
|
|
46
|
+
# We use `pyinstrument` for performance profiling.
|
|
47
|
+
# Usage: $ uv run pyinstrument --from-path refscan --help
|
|
48
|
+
# Docs: https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-python-cli-command
|
|
49
|
+
"pyinstrument >= 4.6.2, < 5",
|
|
50
|
+
# We use `pytest` to run tests.
|
|
51
|
+
# Usage: $ uv run pytest
|
|
52
|
+
# Docs: https://docs.pytest.org/en/stable/contents.html
|
|
53
|
+
"pytest >= 8.2.2, < 9",
|
|
54
|
+
"pytest-cov >= 5.0.0, < 6",
|
|
55
|
+
# We use `mongomock` to test MongoDB interaction without involving a real MongoDB instance.
|
|
56
|
+
# Docs: https://github.com/mongomock/mongomock
|
|
57
|
+
"mongomock >= 4.3.0, < 5",
|
|
58
|
+
# We use `mypy` to perform static type checking.
|
|
59
|
+
# Usage: $ uv run mypy
|
|
60
|
+
# Docs: https://mypy.readthedocs.io/en/stable/
|
|
61
|
+
"mypy >= 1.16.0, < 2",
|
|
62
|
+
# We use `ruff` for both code formatting and linting.
|
|
63
|
+
# Docs: https://docs.astral.sh/ruff/
|
|
64
|
+
"ruff >= 0.12.11",
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
[build-system]
|
|
68
|
+
# Note: The `migrate-to-uv` tool (i.e. https://github.com/mkniewallner/migrate-to-uv), which we used to switch
|
|
69
|
+
# from Poetry to `uv`, set this to use `hatchling`. We then changed it to use `uv_build` instead.
|
|
70
|
+
# Reference: https://docs.astral.sh/uv/concepts/build-backend/#using-the-uv-build-backend
|
|
71
|
+
requires = ["uv_build >= 0.8.14, < 0.9.0"]
|
|
72
|
+
build-backend = "uv_build"
|
|
73
|
+
|
|
74
|
+
[tool.uv.build-backend]
|
|
75
|
+
# Note: We specify `module-name` here to tell the `uv_build` backend that our main module is in `refscan/`
|
|
76
|
+
# (as opposed to being in a `src/` directory, which is where the backend would look for it by default).
|
|
77
|
+
# Reference: https://docs.astral.sh/uv/concepts/build-backend/#modules
|
|
78
|
+
module-name = "refscan"
|
|
79
|
+
module-root = ""
|
|
80
|
+
|
|
81
|
+
[tool.pytest.ini_options]
|
|
82
|
+
# Make it so that, when someone runs `$ uv run pytest`, in addition to running the tests, it also (a) runs doctests
|
|
83
|
+
# defined in Python modules, and (b) measures—via the `pytest-cov` plugin—test coverage within the `refscan` folder.
|
|
84
|
+
# Reference (a): https://docs.pytest.org/en/stable/how-to/doctest.html#how-to-run-doctests
|
|
85
|
+
# Reference (b): https://pytest-cov.readthedocs.io/en/latest/config.html#configuration
|
|
86
|
+
addopts = "--doctest-modules --cov=refscan"
|
|
87
|
+
|
|
88
|
+
[tool.ruff]
|
|
89
|
+
line-length = 120
|
|
90
|
+
|
|
91
|
+
[tool.ruff.lint]
|
|
92
|
+
# Specify which linting rules will be used when someone runs `$ uv run ruff check`.
|
|
93
|
+
# Note: The default rules are `["E4", "E7", "E9", "F"]`.
|
|
94
|
+
# References:
|
|
95
|
+
# - https://docs.astral.sh/ruff/rules/ (the list of rules)
|
|
96
|
+
# - https://docs.astral.sh/ruff/settings/#lint_select (the `select` setting)
|
|
97
|
+
select = ["E4", "E7", "E9", "F", "D101", "D102", "D103"]
|
|
98
|
+
|
|
99
|
+
[tool.ruff.lint.per-file-ignores]
|
|
100
|
+
# Ignore some rules when linting files in specified directories.
|
|
101
|
+
# Reference: https://docs.astral.sh/ruff/settings/#lint_per-file-ignores
|
|
102
|
+
"tests/*.py" = ["D101", "D102", "D103"]
|
|
103
|
+
|
|
104
|
+
[tool.mypy]
|
|
105
|
+
# Make it so, when someone runs `$ uv run mypy`, these default CLI options are used.
|
|
106
|
+
# Reference: https://mypy.readthedocs.io/en/stable/config_file.html
|
|
107
|
+
files = "."
|
|
108
|
+
|
|
109
|
+
[[tool.mypy.overrides]]
|
|
110
|
+
# Silence the following error:
|
|
111
|
+
# > Skipping analyzing "linkml_runtime": module is installed, but missing library stubs or py.typed marker [import-untyped]
|
|
112
|
+
#
|
|
113
|
+
# Note: We first tried using the following, but it seemed to have no effect on the behavior of mypy when run via `$ poetry run mypy` (back when we used poetry instead of uv).
|
|
114
|
+
# ```
|
|
115
|
+
# [mypy-linkml_runtime]
|
|
116
|
+
# ignore_missing_imports = true
|
|
117
|
+
# ```
|
|
118
|
+
# So, we—instead—used the syntax shown on: https://stackoverflow.com/a/74198016
|
|
119
|
+
#
|
|
120
|
+
module = "linkml_runtime"
|
|
121
|
+
ignore_missing_imports = true
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
|
-
from typing import Optional
|
|
3
2
|
from typing_extensions import Annotated
|
|
4
3
|
|
|
5
4
|
import typer
|
|
@@ -24,7 +23,7 @@ def graph(
|
|
|
24
23
|
),
|
|
25
24
|
],
|
|
26
25
|
graph_file_path: Annotated[
|
|
27
|
-
|
|
26
|
+
Path,
|
|
28
27
|
typer.Option(
|
|
29
28
|
"--graph",
|
|
30
29
|
dir_okay=False,
|
|
@@ -33,7 +32,7 @@ def graph(
|
|
|
33
32
|
resolve_path=True,
|
|
34
33
|
help="Filesystem path at which you want **refscan** to generate the graph.",
|
|
35
34
|
),
|
|
36
|
-
] = "graph.html",
|
|
35
|
+
] = Path("graph.html"),
|
|
37
36
|
subject: Annotated[
|
|
38
37
|
grapher.Subject,
|
|
39
38
|
typer.Option(
|
|
@@ -71,7 +71,7 @@ def scan(
|
|
|
71
71
|
),
|
|
72
72
|
] = None,
|
|
73
73
|
reference_report_file_path: Annotated[
|
|
74
|
-
|
|
74
|
+
Path,
|
|
75
75
|
typer.Option(
|
|
76
76
|
"--reference-report",
|
|
77
77
|
dir_okay=False,
|
|
@@ -80,9 +80,9 @@ def scan(
|
|
|
80
80
|
resolve_path=True,
|
|
81
81
|
help="Filesystem path at which you want the program to generate its reference report.",
|
|
82
82
|
),
|
|
83
|
-
] = "references.tsv",
|
|
83
|
+
] = Path("references.tsv"),
|
|
84
84
|
violation_report_file_path: Annotated[
|
|
85
|
-
|
|
85
|
+
Path,
|
|
86
86
|
typer.Option(
|
|
87
87
|
"--violation-report",
|
|
88
88
|
dir_okay=False,
|
|
@@ -91,7 +91,7 @@ def scan(
|
|
|
91
91
|
resolve_path=True,
|
|
92
92
|
help="Filesystem path at which you want the program to generate its violation report.",
|
|
93
93
|
),
|
|
94
|
-
] = "violations.tsv",
|
|
94
|
+
] = Path("violations.tsv"),
|
|
95
95
|
user_wants_to_skip_scan: Annotated[
|
|
96
96
|
bool,
|
|
97
97
|
typer.Option(
|
|
@@ -202,7 +202,7 @@ def scan(
|
|
|
202
202
|
num_all_violations = len(all_violations)
|
|
203
203
|
color_name = "white" if num_all_violations == 0 else "red"
|
|
204
204
|
console.print() # newline
|
|
205
|
-
console.print(f"Total violations:
|
|
205
|
+
console.print(f"Total violations: [{color_name}]{num_all_violations}[/{color_name}]")
|
|
206
206
|
console.print() # newline
|
|
207
207
|
|
|
208
208
|
# Create a TSV-formatted violation report that lists all violations among all collections.
|
|
@@ -63,7 +63,6 @@ class Finder:
|
|
|
63
63
|
# Check whether either (a) the queue is empty or (b) this collection name is not at the front of the queue.
|
|
64
64
|
# Example: [] is empty, or "b_set" is not the first item in ["a_set", "b_set", "c_set"]
|
|
65
65
|
if len(queue) == 0 or queue[0] != collection_name:
|
|
66
|
-
|
|
67
66
|
# If this collection name is already (elsewhere) in the queue, remove it from that position, automatically
|
|
68
67
|
# updating the list indexes of subsequent list items (so there is no "gap" in the list).
|
|
69
68
|
# Example: Removing "b_set" from ["a_set", "b_set", "c_set"] → ["a_set", "c_set"]
|
|
@@ -86,7 +85,6 @@ class Finder:
|
|
|
86
85
|
|
|
87
86
|
# Iterate over the collection names in the queue, from back (oldest) to front (newest).
|
|
88
87
|
for cached_collection_name in reversed(self.cached_collection_names_where_recently_found):
|
|
89
|
-
|
|
90
88
|
# If this cached collection name is among the eligible ones, move it to the front of the list.
|
|
91
89
|
# Example: Since "b_set" is in ["a_set", "b_set", "c_set"] → ["b_set, "a_set", "c_set"]
|
|
92
90
|
if cached_collection_name in names_of_eligible_collections:
|
|
@@ -143,7 +141,6 @@ class Finder:
|
|
|
143
141
|
# Search the collections.
|
|
144
142
|
query_filter = dict(id=document_id)
|
|
145
143
|
for collection_name in ordered_collection_names:
|
|
146
|
-
|
|
147
144
|
# Before querying the database, check our cache of document `id` presences/absences per collection.
|
|
148
145
|
is_id_present: Optional[bool] = self._get_cached_id_presence_in_collection(
|
|
149
146
|
collection_name=collection_name, document_id=document_id
|
|
@@ -69,7 +69,6 @@ class ReferenceList(UserList):
|
|
|
69
69
|
"""
|
|
70
70
|
target_collection_names = []
|
|
71
71
|
for reference in self.data: # note: in a `UserList`, `self.data` refers to the underlying list data structure
|
|
72
|
-
|
|
73
72
|
# If this reference's source describes the specified source, record the reference's target collection name.
|
|
74
73
|
if reference.source_class_name == source_class_name and reference.source_field_name == source_field_name:
|
|
75
74
|
target_collection_names.append(reference.target_collection_name)
|
|
@@ -241,7 +241,6 @@ def identify_references(
|
|
|
241
241
|
for collection_name, class_names in sorted_collection_names_to_class_names:
|
|
242
242
|
for class_name in class_names:
|
|
243
243
|
for slot_name in schema_view.class_slots(class_name):
|
|
244
|
-
|
|
245
244
|
# Get the slot definition in the context of its use on this particular class.
|
|
246
245
|
slot_definition = schema_view.induced_slot(slot_name=slot_name, class_name=class_name)
|
|
247
246
|
|
|
@@ -50,7 +50,7 @@ def identify_referring_documents(
|
|
|
50
50
|
"""
|
|
51
51
|
|
|
52
52
|
# Initialize a list of descriptors of documents that reference the specified document.
|
|
53
|
-
referring_document_descriptors = []
|
|
53
|
+
referring_document_descriptors: List[Dict] = []
|
|
54
54
|
|
|
55
55
|
# If the specified document does not have an `id` field, we know it cannot be referenced.
|
|
56
56
|
if "id" not in document:
|
|
@@ -97,7 +97,6 @@ def identify_referring_documents(
|
|
|
97
97
|
|
|
98
98
|
# For each referring document, store a descriptor of it.
|
|
99
99
|
for referring_document in referring_documents:
|
|
100
|
-
|
|
101
100
|
# Handle the case where the source document does not have an `id`.
|
|
102
101
|
#
|
|
103
102
|
# Note: As a reminder, according to the NMDC Schema (as of version v11.8.0), documents in the
|
|
@@ -195,7 +194,6 @@ def scan_outgoing_references(
|
|
|
195
194
|
)
|
|
196
195
|
)
|
|
197
196
|
if name_of_collection_containing_target_document is None:
|
|
198
|
-
|
|
199
197
|
# If the user wants to locate misplaced documents,
|
|
200
198
|
# search all illegal collections for the misplaced document.
|
|
201
199
|
if user_wants_to_locate_misplaced_documents:
|
|
@@ -268,7 +266,6 @@ def scan(
|
|
|
268
266
|
|
|
269
267
|
source_collections_and_their_violations: dict[str, ViolationList] = {}
|
|
270
268
|
with custom_progress as progress:
|
|
271
|
-
|
|
272
269
|
# Filter out the collections that the schema says can contain references, but that don't exist in the database.
|
|
273
270
|
source_collection_names_in_db = []
|
|
274
271
|
for collection_name in references.get_source_collection_names():
|
|
@@ -289,7 +286,6 @@ def scan(
|
|
|
289
286
|
# contain "referrers" (documents), which of their slots can contain references (fields),
|
|
290
287
|
# and which collections can contain the referred-to "referees" (documents).
|
|
291
288
|
for source_collection_name in sorted(source_collection_names_in_db):
|
|
292
|
-
|
|
293
289
|
# If this source collection is one of the ones the user wanted to skip, skip it now.
|
|
294
290
|
if source_collection_name in names_of_source_collections_to_skip:
|
|
295
291
|
continue
|
refscan-0.3.2/LICENSE
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
Refscan Copyright (c) 2024, The Regents of the University of California,
|
|
2
|
-
through Lawrence Berkeley National Laboratory (subject to receipt of any
|
|
3
|
-
required approvals from the U.S. Dept. of Energy). All rights reserved.
|
|
4
|
-
|
|
5
|
-
Redistribution and use in source and binary forms, with or without
|
|
6
|
-
modification, are permitted provided that the following conditions are met:
|
|
7
|
-
|
|
8
|
-
(1) Redistributions of source code must retain the above copyright notice,
|
|
9
|
-
this list of conditions and the following disclaimer.
|
|
10
|
-
|
|
11
|
-
(2) Redistributions in binary form must reproduce the above copyright
|
|
12
|
-
notice, this list of conditions and the following disclaimer in the
|
|
13
|
-
documentation and/or other materials provided with the distribution.
|
|
14
|
-
|
|
15
|
-
(3) Neither the name of the University of California, Lawrence Berkeley
|
|
16
|
-
National Laboratory, U.S. Dept. of Energy nor the names of its contributors
|
|
17
|
-
may be used to endorse or promote products derived from this software
|
|
18
|
-
without specific prior written permission.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
22
|
-
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
23
|
-
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
24
|
-
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
|
25
|
-
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
26
|
-
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
27
|
-
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
28
|
-
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
29
|
-
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
30
|
-
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
31
|
-
POSSIBILITY OF SUCH DAMAGE.
|
|
32
|
-
|
|
33
|
-
You are under no obligation whatsoever to provide any bug fixes, patches,
|
|
34
|
-
or upgrades to the features, functionality or performance of the source
|
|
35
|
-
code ("Enhancements") to anyone; however, if you choose to make your
|
|
36
|
-
Enhancements available either publicly, or directly to Lawrence Berkeley
|
|
37
|
-
National Laboratory, without imposing a separate written license agreement
|
|
38
|
-
for such Enhancements, then you hereby grant the following license: a
|
|
39
|
-
non-exclusive, royalty-free perpetual license to install, use, modify,
|
|
40
|
-
prepare derivative works, incorporate into other computer software,
|
|
41
|
-
distribute, and sublicense such enhancements or derivative works thereof,
|
|
42
|
-
in binary and source code form.
|
refscan-0.3.2/pyproject.toml
DELETED
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
# Reference: https://python-poetry.org/docs/pyproject/
|
|
2
|
-
|
|
3
|
-
[tool.poetry]
|
|
4
|
-
name = "refscan"
|
|
5
|
-
version = "v0.3.2"
|
|
6
|
-
description = "Command-line program that scans the NMDC MongoDB database for referential integrity violations"
|
|
7
|
-
authors = []
|
|
8
|
-
readme = "README.md"
|
|
9
|
-
repository = "https://github.com/microbiomedata/refscan"
|
|
10
|
-
homepage = "https://github.com/microbiomedata/refscan"
|
|
11
|
-
documentation = "https://github.com/microbiomedata/refscan"
|
|
12
|
-
keywords = ["mongodb", "mongo", "relationships", "reference", "database", "data", "referential integrity", "scan"]
|
|
13
|
-
|
|
14
|
-
[tool.poetry.dependencies]
|
|
15
|
-
python = "^3.9"
|
|
16
|
-
# We use `linkml-runtime` to facilitate using `nmdc-schema`.
|
|
17
|
-
# Docs: https://linkml.io/linkml/developers/schemaview.html#linkml_runtime.utils.schemaview.SchemaView
|
|
18
|
-
linkml-runtime = "^1.7.5"
|
|
19
|
-
# We use `pymongo` to communicate with MongoDB servers.
|
|
20
|
-
# Docs: (https://pymongo.readthedocs.io/en/stable/
|
|
21
|
-
pymongo = "^4.7.1"
|
|
22
|
-
# We use `rich` to format console output.
|
|
23
|
-
# Docs: https://rich.readthedocs.io/en/stable/index.html
|
|
24
|
-
rich = "^13.7.1"
|
|
25
|
-
# We use `typer` as a CLI framework.
|
|
26
|
-
# Docs: https://typer.tiangolo.com/
|
|
27
|
-
typer = "^0.15.1"
|
|
28
|
-
|
|
29
|
-
[tool.poetry.group.dev.dependencies]
|
|
30
|
-
# We use `black` for code formatting.
|
|
31
|
-
# Usage: $ poetry run black --line-length 120 .
|
|
32
|
-
# Docs: https://black.readthedocs.io/en/stable/
|
|
33
|
-
black = "^24.4.2"
|
|
34
|
-
# We use `pyinstrument` for performance profiling.
|
|
35
|
-
# Usage: $ pyinstrument --from-path refscan --help
|
|
36
|
-
# Docs: https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-python-cli-command
|
|
37
|
-
pyinstrument = "^4.6.2"
|
|
38
|
-
# We use `pytest` to run tests.
|
|
39
|
-
# Usage: $ poetry run pytest
|
|
40
|
-
# Docs: https://docs.pytest.org/en/stable/contents.html
|
|
41
|
-
pytest = "^8.2.2"
|
|
42
|
-
pytest-cov = "^5.0.0"
|
|
43
|
-
# We use `mongomock` to test MongoDB interaction without involving a real MongoDB instance.
|
|
44
|
-
# Docs: https://github.com/mongomock/mongomock
|
|
45
|
-
mongomock = "^4.3.0"
|
|
46
|
-
# We use `mypy` to perform static type checking.
|
|
47
|
-
# Usage: $ poetry run mypy
|
|
48
|
-
# Docs: https://mypy.readthedocs.io/en/stable/
|
|
49
|
-
mypy = "^1.16.0"
|
|
50
|
-
|
|
51
|
-
[tool.poetry.scripts]
|
|
52
|
-
# Reference: https://python-poetry.org/docs/pyproject#scripts
|
|
53
|
-
# Reference: https://typer.tiangolo.com/tutorial/package/#add-a-script
|
|
54
|
-
refscan = "refscan.cli.cli:app"
|
|
55
|
-
|
|
56
|
-
[tool.pytest.ini_options]
|
|
57
|
-
# Make it so that, when someone runs `$ poetry run pytest`, in addition to running the tests, it also (a) runs doctests
|
|
58
|
-
# defined in Python modules, and (b) measures—via the `pytest-cov` plugin—test coverage within the `refscan` folder.
|
|
59
|
-
# Reference (a): https://docs.pytest.org/en/stable/how-to/doctest.html#how-to-run-doctests
|
|
60
|
-
# Reference (b): https://pytest-cov.readthedocs.io/en/latest/config.html#configuration
|
|
61
|
-
addopts = "--doctest-modules --cov=refscan"
|
|
62
|
-
|
|
63
|
-
[tool.black]
|
|
64
|
-
# Make it so that, when someone runs `$ poetry run black`, these default CLI options are used.
|
|
65
|
-
# Reference: https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#configuration-format
|
|
66
|
-
line-length = 120
|
|
67
|
-
|
|
68
|
-
[tool.mypy]
|
|
69
|
-
# Make it so, when someone runs `$ poetry run mypy`, these default CLI options are used.
|
|
70
|
-
# Reference: https://mypy.readthedocs.io/en/stable/config_file.html
|
|
71
|
-
files = "."
|
|
72
|
-
|
|
73
|
-
[[tool.mypy.overrides]]
|
|
74
|
-
# Silence the following error:
|
|
75
|
-
# > Skipping analyzing "linkml_runtime": module is installed, but missing library stubs or py.typed marker [import-untyped]
|
|
76
|
-
#
|
|
77
|
-
# Note: We first tried using the following, but it seemed to have no effect on the behavior of mypy when run via `$ poetry run mypy`.
|
|
78
|
-
# ```
|
|
79
|
-
# [mypy-linkml_runtime]
|
|
80
|
-
# ignore_missing_imports = true
|
|
81
|
-
# ```
|
|
82
|
-
# So, we—instead—used the syntax shown on: https://stackoverflow.com/a/74198016
|
|
83
|
-
#
|
|
84
|
-
module = "linkml_runtime"
|
|
85
|
-
ignore_missing_imports = true
|
|
86
|
-
|
|
87
|
-
[build-system]
|
|
88
|
-
requires = ["poetry-core"]
|
|
89
|
-
build-backend = "poetry.core.masonry.api"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|