wwpdb.apps.val-rel 0.27.dev1__tar.gz

wwpdb_apps_val_rel-0.27.dev1/.gitignore

@@ -0,0 +1,113 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# vscode
+.idea
+.vscode
+
+# Emacs backup
+*~
+
+tests/test-output

wwpdb_apps_val_rel-0.27.dev1/PKG-INFO

@@ -0,0 +1,185 @@
+Metadata-Version: 2.5
+Name: wwpdb.apps.val-rel
+Version: 0.27.dev1
+Summary: wwPDB validation processing for weekly release
+Project-URL: Homepage, https://github.com/wwpdb/py-wwpdb_apps_val_rel
+Author-email: Ezra Peisach <ezra.peisach@rcsb.org>
+License-Expression: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Requires-Python: >=3.8
+Requires-Dist: filelock
+Requires-Dist: mmcif
+Requires-Dist: oslo-concurrency
+Requires-Dist: requests
+Requires-Dist: urllib3
+Requires-Dist: wwpdb-apps-validation~=2.49
+Requires-Dist: wwpdb-io<2.0,>=0.15
+Requires-Dist: wwpdb-utils-config>=0.24
+Requires-Dist: wwpdb-utils-detach
+Requires-Dist: wwpdb-utils-dp
+Requires-Dist: wwpdb-utils-message-queue~=0.6
+Requires-Dist: wwpdb-utils-nmr
+Requires-Dist: wwpdb-utils-session
+Requires-Dist: wwpdb-utils-ws-utils
+Provides-Extra: dev
+Requires-Dist: check-manifest; extra == 'dev'
+Provides-Extra: test
+Requires-Dist: coverage; extra == 'test'
+Description-Content-Type: text/markdown
+
+# py-wwpdb_apps_val_rel
+Service to prepare weekly validation reports
+
+## requirements
+In site-config the following must be set
+
+    VAL_REL_PROTOCOL = determines the protocol to use (ftp, http, https) if local ftp tree is not available.
+    VAL_REL_ADMIN_EMAIL = comma separated list of who to contact on failures
+    VAL_REL_DISABLE_MULTITHREAD = if set disable validation code from running in parallel fashion
+    SITE_PDB_FTP_ROOT_DIR = the root of the PDB FTP tree (before pdb/data) - this can be not set if its not available at the local site 
+    SITE_EMDB_FTP_ROOT_DIR = the root of the EMDB FTP tree (before emdb/structures) - this can be not set if its not available at the local site
+    SITE_FTP_SERVER = this will default to ftp.wwpdb.org if not set
+    SITE_FTP_SERVER_PREFIX = this is the prefix in the URL at the FTP server until you get to the PDB or EMDB data - i.e. on ftp.wwpdb.org this is "pub"
+    SITE_HTTP_SERVER = this will default to files.wwpdb.org if not set
+    SITE_HTTP_SERVER_PREFIX = this is the prefix in the URL at the HTTP server until you get to the PDB or EMDB data - i.e. on files.wwpdb.org this is "pub"
+    SITE_RBMQ_SERVER_HOST = this is the host that runs the rabbitMQ server
+
+## to setup rabbitMQ
+
+See
+
+    https://github.com/wwPDB/onedep-maintenance/blob/master/pdbe_redhat7/server_scripts/setup_rabbitMQ.sh
+    
+## usage
+
+This process has two parts
+1) finding entries to run validation reports and adding to a rabbitMQ queue
+2) workers which take work from the rabbitMQ queue and create validation reports
+
+
+
+### 1) Finding entries
+
+Entries are found by a script which searches the for_release folder 
+
+    python -m wwpdb.apps.val_rel.PopulateValidateRelease
+    usage options are
+        --pdb_release - find PDB entries in the for_release/{added/modified} folders  
+        --emdb_release - find EMDB entries in the for_release/emd folder
+        --skip_emdb - if only PDB entry reports with no visual analysis is required.
+        --entry_list - comma separated list of entries
+        --entry_file - file containing entries, one per line
+        --output_root - to output to a path of your choice
+        --help for other options
+
+This should be run on a cron every few hours.
+
+If a priority queue is required, add the option --priority.
+
+### 2) validation report workers
+    
+workers pick up work from the rabbitMQ queue and generate validation reports.
+To start one worker
+
+    python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --start
+    
+To read from a priority queue, add the option --priority.
+
+To start several workers - below example starts 50 workers 
+
+    for i in {1..50}
+    do
+        echo ${i}
+        python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --start --instance ${i}
+    done
+ 
+ To stop workers
+ 
+    python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --stop
+    or
+    python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --stop --instance ${i}
+
+### 3) To find missing entries and queue for the following week
+    
+To find missing entries and write them out to a temporary file 
+
+    python -m wwpdb.apps.val_rel.utils.find_and_run_missing --write_missing
+
+then to read the temporary file and load the entries into the missing queue
+    
+    python -m wwpdb.apps.val_rel.utils.find_and_run_missing --read_missing
+
+For priority queues, add --priority along with --read_missing.
+
+### to run on a single entry without the rabbitMQ queue
+
+    python -m wwpdb.apps.val_rel.ValidateRelease
+    usage options are
+        --pdbid - a single pdb id
+        --emdbid - a single emdb id
+        --output_root - to output to a path of your choice
+
+### 4) Publishing and consuming from a priority queue
+
+By default, the queues are not priority queues.
+
+To make priority queues, pass an argument of --priority to PopulateValidateRelease, ValidationReleaseServiceHandler, and find_and_run_missing.
+
+Priority queues cannot write to or read from non-priority queues, and an error is thrown if you attempt to mix them.
+
+Therefore, an argument of --priority is always specified for priority queues.
+
+The priority values are determined automatically, with higher values having higher priority.
+
+- 10 missing
+- 8 new pdb
+- 6 new emdb
+- 4 modified pdb
+- 2 modified emdb
+- 1 default
+
+
+### 5) Subscription queues
+
+By default, the queues are persistent queues that store messages regardless of the existence of producers or consumers.
+
+However, even with priority queues, it's not possible to read a subset of messages with particular attributes, for example only emdb messages.
+
+Therefore, subscription queues have been built which, if necessary, enable higher specificity.
+
+Subscription queues publish to a larger number of exchanges and then subscribe to exchanges of choice.
+
+To make subscription queues, pass an argument of --subscribe to PopulateValidateRelease and ValidationReleaseServiceHandler, followed by an exchange name which should be the same for both.
+
+You must invoke the consumer before the producer or else the messages will be lost.
+
+A subscription queue is only temporary until the consumer closes, therefore they are not intended to be relied upon exclusively, rather as a supplement to already existing queues.
+
+Once the consumer is stopped, the queue and all of its contents will be deleted.
+
+Subscription queues may have advantages during service peaks.
+
+For example, to relieve congestion in the emdb folder, stop the regular producer crons, then start selective consumers with exchange names 'pdb_production_exchange' and 'emdb_production_exchange', then start corresponding producers with the same exchange names, one with option '--pdb_release' and one with option '--emdb_release'.
+
+In the present system, subscription queues must not be run at the same time as the regular queues, at least not with the same options, or else they will both publish/consume the same files.
+
+### 6) Debugging
+
+ValidationReleaseServiceHandler runs a background process which is difficult to get debugging information from.
+
+If errors occur from mixing of priority and non-priority queues, ValidationReleaseServiceHandler will not report them.
+
+One possible side effect is that consumers will not stop when attempting to stop them with ValidationReleaseServiceHandler.
+
+If consumers will not stop, pass an argument of --list to ValidationReleaseServiceHandler.
+
+The running consumer process ids will be listed.
+
+If it's possible to detect the processes that are responsible, delete all or some of the running process ids and their associated pid files.
+
+The path to the pids file directory is printed out with each invocation of ValidationReleaseServiceHandler.

wwpdb_apps_val_rel-0.27.dev1/README.md

@@ -0,0 +1,151 @@
+# py-wwpdb_apps_val_rel
+Service to prepare weekly validation reports
+
+## requirements
+In site-config the following must be set
+
+    VAL_REL_PROTOCOL = determines the protocol to use (ftp, http, https) if local ftp tree is not available.
+    VAL_REL_ADMIN_EMAIL = comma separated list of who to contact on failures
+    VAL_REL_DISABLE_MULTITHREAD = if set disable validation code from running in parallel fashion
+    SITE_PDB_FTP_ROOT_DIR = the root of the PDB FTP tree (before pdb/data) - this can be not set if its not available at the local site 
+    SITE_EMDB_FTP_ROOT_DIR = the root of the EMDB FTP tree (before emdb/structures) - this can be not set if its not available at the local site
+    SITE_FTP_SERVER = this will default to ftp.wwpdb.org if not set
+    SITE_FTP_SERVER_PREFIX = this is the prefix in the URL at the FTP server until you get to the PDB or EMDB data - i.e. on ftp.wwpdb.org this is "pub"
+    SITE_HTTP_SERVER = this will default to files.wwpdb.org if not set
+    SITE_HTTP_SERVER_PREFIX = this is the prefix in the URL at the HTTP server until you get to the PDB or EMDB data - i.e. on files.wwpdb.org this is "pub"
+    SITE_RBMQ_SERVER_HOST = this is the host that runs the rabbitMQ server
+
+## to setup rabbitMQ
+
+See
+
+    https://github.com/wwPDB/onedep-maintenance/blob/master/pdbe_redhat7/server_scripts/setup_rabbitMQ.sh
+    
+## usage
+
+This process has two parts
+1) finding entries to run validation reports and adding to a rabbitMQ queue
+2) workers which take work from the rabbitMQ queue and create validation reports
+
+
+
+### 1) Finding entries
+
+Entries are found by a script which searches the for_release folder 
+
+    python -m wwpdb.apps.val_rel.PopulateValidateRelease
+    usage options are
+        --pdb_release - find PDB entries in the for_release/{added/modified} folders  
+        --emdb_release - find EMDB entries in the for_release/emd folder
+        --skip_emdb - if only PDB entry reports with no visual analysis is required.
+        --entry_list - comma separated list of entries
+        --entry_file - file containing entries, one per line
+        --output_root - to output to a path of your choice
+        --help for other options
+
+This should be run on a cron every few hours.
+
+If a priority queue is required, add the option --priority.
+
+### 2) validation report workers
+    
+workers pick up work from the rabbitMQ queue and generate validation reports.
+To start one worker
+
+    python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --start
+    
+To read from a priority queue, add the option --priority.
+
+To start several workers - below example starts 50 workers 
+
+    for i in {1..50}
+    do
+        echo ${i}
+        python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --start --instance ${i}
+    done
+ 
+ To stop workers
+ 
+    python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --stop
+    or
+    python -m wwpdb.apps.val_rel.service.ValidationReleaseServiceHandler --stop --instance ${i}
+
+### 3) To find missing entries and queue for the following week
+    
+To find missing entries and write them out to a temporary file 
+
+    python -m wwpdb.apps.val_rel.utils.find_and_run_missing --write_missing
+
+then to read the temporary file and load the entries into the missing queue
+    
+    python -m wwpdb.apps.val_rel.utils.find_and_run_missing --read_missing
+
+For priority queues, add --priority along with --read_missing.
+
+### to run on a single entry without the rabbitMQ queue
+
+    python -m wwpdb.apps.val_rel.ValidateRelease
+    usage options are
+        --pdbid - a single pdb id
+        --emdbid - a single emdb id
+        --output_root - to output to a path of your choice
+
+### 4) Publishing and consuming from a priority queue
+
+By default, the queues are not priority queues.
+
+To make priority queues, pass an argument of --priority to PopulateValidateRelease, ValidationReleaseServiceHandler, and find_and_run_missing.
+
+Priority queues cannot write to or read from non-priority queues, and an error is thrown if you attempt to mix them.
+
+Therefore, an argument of --priority is always specified for priority queues.
+
+The priority values are determined automatically, with higher values having higher priority.
+
+- 10 missing
+- 8 new pdb
+- 6 new emdb
+- 4 modified pdb
+- 2 modified emdb
+- 1 default
+
+
+### 5) Subscription queues
+
+By default, the queues are persistent queues that store messages regardless of the existence of producers or consumers.
+
+However, even with priority queues, it's not possible to read a subset of messages with particular attributes, for example only emdb messages.
+
+Therefore, subscription queues have been built which, if necessary, enable higher specificity.
+
+Subscription queues publish to a larger number of exchanges and then subscribe to exchanges of choice.
+
+To make subscription queues, pass an argument of --subscribe to PopulateValidateRelease and ValidationReleaseServiceHandler, followed by an exchange name which should be the same for both.
+
+You must invoke the consumer before the producer or else the messages will be lost.
+
+A subscription queue is only temporary until the consumer closes, therefore they are not intended to be relied upon exclusively, rather as a supplement to already existing queues.
+
+Once the consumer is stopped, the queue and all of its contents will be deleted.
+
+Subscription queues may have advantages during service peaks.
+
+For example, to relieve congestion in the emdb folder, stop the regular producer crons, then start selective consumers with exchange names 'pdb_production_exchange' and 'emdb_production_exchange', then start corresponding producers with the same exchange names, one with option '--pdb_release' and one with option '--emdb_release'.
+
+In the present system, subscription queues must not be run at the same time as the regular queues, at least not with the same options, or else they will both publish/consume the same files.
+
+### 6) Debugging
+
+ValidationReleaseServiceHandler runs a background process which is difficult to get debugging information from.
+
+If errors occur from mixing of priority and non-priority queues, ValidationReleaseServiceHandler will not report them.
+
+One possible side effect is that consumers will not stop when attempting to stop them with ValidationReleaseServiceHandler.
+
+If consumers will not stop, pass an argument of --list to ValidationReleaseServiceHandler.
+
+The running consumer process ids will be listed.
+
+If it's possible to detect the processes that are responsible, delete all or some of the running process ids and their associated pid files.
+
+The path to the pids file directory is printed out with each invocation of ValidationReleaseServiceHandler.

wwpdb_apps_val_rel-0.27.dev1/pyproject.toml

@@ -0,0 +1,175 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "wwpdb.apps.val-rel"
+dynamic = ["version"]
+description = "wwPDB validation processing for weekly release"
+readme = "README.md"
+license = "Apache-2.0"
+authors = [
+    { name = "Ezra Peisach", email = "ezra.peisach@rcsb.org" },
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+]
+dependencies = [
+    "filelock",
+    "mmcif",
+    "oslo_concurrency",
+    "requests",
+    "urllib3",
+    "wwpdb.apps.validation ~= 2.49",
+    "wwpdb.io >=0.15, < 2.0",
+    "wwpdb.utils.config >= 0.24",
+    "wwpdb.utils.detach",
+    "wwpdb.utils.dp",
+    "wwpdb.utils.message_queue ~= 0.6",
+    "wwpdb.utils.nmr",
+    "wwpdb.utils.session",
+    "wwpdb.utils.ws_utils",
+]
+
+requires-python = ">=3.8"
+
+
+[project.optional-dependencies]
+dev = [
+    "check-manifest",
+]
+test = [
+    "coverage",
+]
+
+[project.urls]
+Homepage = "https://github.com/wwpdb/py-wwpdb_apps_val_rel"
+
+[tool.hatch.version]
+path = "wwpdb/apps/val_rel/__init__.py"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "/wwpdb",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["wwpdb"]
+
+#=================================== Ruff basics =============================#
+[tool.ruff]
+target-version = "py38"
+output-format = "concise"
+
+line-length = 120 # 120 is hatch's default via 'hatch init'
+include = [
+  "*.py",           # Source
+  "pyproject.toml", # This file (RUF rules only)
+  "*.ipynb",        # Jupyter notebooks
+]
+
+#
+#=================================== Ruff lint ==============================#
+
+[tool.ruff.lint]
+
+#
+# ------------------------- Ruff misc. options ----------------------
+preview = false
+explicit-preview-rules = true
+
+ignore = [
+  "C402",     #  Unnecessary generator (rewrite as a `dict` comprehension)
+  "EXE001",   # Shebang is present but file is not executabl
+  "FA100",    # Add `from __future__ import annotations` to simplify `typing.Optional`
+  "FBT002",   # Boolean default positional argument in function definition
+  "FBT001",   # Boolean-typed positional argument in function definition
+  "FURB113",  # Use `stPL.extend(...)` instead of repeatedly calling `stPL.append()`
+  "INP001",   # File `xx` is part of an implicit namespace package. Add an `__init__.py
+  "N801",     # Class name `markdownTests` should use CapWords convention
+  "N802",     # Function name should be lowercase
+  "N803",     # Argument name should be lowercase
+  "N806",     # in function should be lowercase
+  "N815",     # Caiable... in class shope should not be mixedCase
+  "N816",     # Variable in global scope should not be mixedCase
+  "N999",     # Invalid module name,
+  "PERF401",  # Use a list comprehension to create a transformed list
+  "PERF403",  #  Use a dictionary comprehension instead of a for-loop
+  "PLR1714",  # Consider merging multiple comparisons
+  "PLR2004",  # Magic value used in comparison, consider replacing
+  "PLR6201",  # Use a `set` literal when testing for membership
+  "PLR6301",  # Method `` could be a function, class method, or static method
+  "PLW1514",  # `open` in text mode without explicit `encoding` argument
+  "PT009",    # Use a regular `assert` instead of unittest-style `assertGreater`
+  "RET504",   # Unnecessary assignment to `X` before `return` statement
+  "PT027",    # Use `pytest.raises` instead of unittest-style `assertRaises`
+  "RUF100",   # RUF REMOVES FLAKE8 noqa!
+  "S110",     # `try`-`except`-`pass` detected, consider logging the exception
+  "SIM102",   # Use a single `if` statement instead of nested `if` statements
+  "SIM103",   #  Return the condition directly
+  "SIM108",   # Use ternary operator
+  "SIM115",   # Use context handler for opening files
+  "TRY300",   # Consider moving this statement to an `else` block
+  "TRY400",   # Use `logging.exception` instead of `logging.error`
+  "TRY401",   # Redundant exception object included in `logging.exception` call
+  "UP008",    # Use `super()` instead of `super(__class__, self)`
+  "UP031",    # Use format specifiers instead of percent format
+  "UP036"     # Version block is outdated for minimum Python version
+]
+
+# -------------------- default environment scripts ------------------
+[tool.hatch.envs.default]
+skip-install = true
+dependencies = ["mypy", "ruff >=0.6.9", "wwpdb.utils.config", "wwpdb.utils.message_queue", "wwpdb.io", "wwpdb.utils.dp", "wwpdb.utils.session", "wwpdb.utils.detach"]
+
+
+[tool.hatch.envs.default.scripts]
+
+# 'format' formats and fixes issues but does not complain
+format = [
+  "ruff format",
+  "ruff check --fix-only",
+]
+
+# 'format-python' just excludes Prettier
+format-python = ["ruff format", "ruff check --fix-only"]
+
+# 'check' only checks but never fixes
+check = ["ruff check --no-fix .", "mypy --non-interactive --install-types -p wwpdb -p tests --strict"]
+
+# only checks with ruff (mypy has a lot of false positives)
+check-ruff = "ruff check --no-fix . {args}"
+
+# to complement 'check-ruff'
+check-mypy = "mypy --non-interactive --install-types -p wwpdb -p tests {args}"
+
+# only check security via Bandit; we require these rules to pass
+check-security = "ruff check --select S --no-fix ."
+
+# -----------------------  testing environment ----------------
+[tool.hatch.envs.hatch-test]
+dependencies = ["tox"]
+
+[tool.hatch.envs.hatch-test.scripts]
+run = ["tox"]
+
+
+# -------------------------  Mypy disable ---------
+[[tool.mypy.overrides]]
+module = ["oslo_concurrency.*", "wwpdb.io.*", "mmcif.*", "wwpdb.utils.ws_utils.*", "wwpdb.utils.dp.*", "wwpdb.apps.validation.*", "wwpdb.utils.detach.*","wwpdb.utils.message_queue.*", "wwpdb.utils.session.*", "filelock"]
+ignore_missing_imports = true
+
+[tool.mypy]
+exclude = [
+	"test-output",
+]
+
+# ------------------------ ruff disable until typing added
+[tool.ruff.lint.per-file-ignores]
+"wwpdb/apps/val_rel/utils/check_results.py" = ["T201", "T203"]  # Command line script
+"tests/test_get_files_release_ftp_emdb.py" = ["T201"]