linkml-reference-validator 0.3.0rc2__tar.gz → 0.3.1rc1__tar.gz

{linkml_reference_validator-0.3.0rc2 → linkml_reference_validator-0.3.1rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: linkml_reference_validator
-Version: 0.3.0rc2
+Version: 0.3.1rc1
 Summary: Validation of supporting text from references and publications
 Author-email: Chris Mungall <cjmungall@lbl.gov>
 License-Expression: Apache-2.0
@@ -10,7 +10,7 @@ Requires-Dist: beautifulsoup4>=4.9.0
 Requires-Dist: biopython>=1.80
 Requires-Dist: jsonpath-ng>=1.6.0
 Requires-Dist: linkml-runtime>=1.9.4
-Requires-Dist: lxml>=4.9.0
+Requires-Dist: lxml>=5.0.0
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pypdf>=4.0.0
 Requires-Dist: rapidfuzz>=3.14.3

linkml_reference_validator-0.3.1rc1/benchmarks/repeated_quotes.py

@@ -0,0 +1,49 @@
+"""Time repeated matching against a local UTF-8 document, without network access.
+
+Run the same script and input with each checkout's package on PYTHONPATH:
+    uv run python benchmarks/repeated_quotes.py /path/to/cached/publication.md
+
+Only successful matching is timed; this is a CPU microbenchmark, not end-to-end
+CLI validation. It includes each validator's first normalization of the body.
+"""
+
+import argparse
+import json
+import re
+from pathlib import Path
+from statistics import median
+from time import perf_counter
+
+from linkml_reference_validator.models import ReferenceContent, ReferenceValidationConfig
+from linkml_reference_validator.validation.supporting_text_validator import SupportingTextValidator
+
+
+def main() -> None:
+    """Report median time to match evenly spaced literal excerpts from a document."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("document", type=Path)
+    parser.add_argument("--quotes", type=int, default=100)
+    parser.add_argument("--rounds", type=int, default=3)
+    args = parser.parse_args()
+    content = args.document.read_text(encoding="utf-8")
+    # Each query is taken verbatim from the source; avoid editorial-bracket and
+    # ellipsis syntax, which would change what the matching API searches for.
+    candidates = [m.group() for m in re.finditer(r"[\w ,;:-]{30,100}", content)]
+    if args.quotes < 1 or args.rounds < 1 or not candidates:
+        parser.error("positive --quotes/--rounds and a document with prose are required")
+    quotes = [candidates[i * len(candidates) // args.quotes] for i in range(args.quotes)]
+    reference = ReferenceContent(reference_id="TEST:benchmark", content=content)
+    elapsed = []
+    for _ in range(args.rounds):
+        validator = SupportingTextValidator(ReferenceValidationConfig())
+        start = perf_counter()
+        for quote in quotes:
+            if not validator.find_text_in_reference(quote, reference).found:
+                raise SystemExit(f"Excerpt unexpectedly failed validation: {quote!r}")
+        elapsed.append(perf_counter() - start)
+    print(json.dumps({"characters": len(content), "quotes": len(quotes),
+                      "rounds_seconds": elapsed, "median_seconds": median(elapsed)}))
+
+
+if __name__ == "__main__":
+    main()

{linkml_reference_validator-0.3.0rc2 → linkml_reference_validator-0.3.1rc1}/docs/concepts/content-types.md

@@ -11,6 +11,7 @@ When fetching references, the validator tracks what type of content was retrieve
 | `abstract_only` | PMID, DOI | Only the abstract was available |
 | `summary` | GEO, BioProject, BioSample, ClinicalTrials | Database summary/description field |
 | `local_file` | file: | Content read from local filesystem |
+| `local_svg` | file: (`.svg`) | Text an SVG figure renders, a blank line between labels; a quote must fall within one label |
 | `url` | url: | Content fetched from web URL |
 | `unavailable` | Any | No content could be retrieved |
 

{linkml_reference_validator-0.3.0rc2 → linkml_reference_validator-0.3.1rc1}/docs/how-to/use-local-files-and-urls.md

@@ -89,7 +89,7 @@ URLs are cached the same way as PMID and DOI references:
 
 ### Title Extraction
 
-For HTML pages, the title is extracted from the `<title>` tag. For other content types, the URL itself is used as the title.
+For HTML pages, the title is the `citation_title` meta tag when present, then the `<title>` tag. For PDFs, the validator tries a publisher landing page's `citation_title` and then the PDF's embedded title. When neither is found, and for other content types, the URL itself is used as the title. See [Validating URLs](validate-urls.md#titles).
 
 ### Example: Validating Against a Web Page
 
@@ -105,6 +105,51 @@ linkml-reference-validator validate text \
   url:https://example.com/pangram-examples.html
 ```
 
+## Diagrams as References (SVG)
+
+A `file:` reference can point at an `.svg` figure, such as a pathway diagram.
+The quote is then matched against the text the figure **renders**, not its
+markup:
+
+```bash
+linkml-reference-validator validate text \
+  "loss of PTEN activates AKT" \
+  file:./figures/pathway.svg
+```
+
+- Each `<text>` element is one label. A label split over several `<tspan>`
+  lines can be quoted across the break.
+- A quote must fall within one label. A figure with a `PTEN` label and a
+  separate `activates AKT` label does not support `"PTEN activates AKT"`,
+  however close together they sit: neither label says it. To cite both, quote
+  them as separate parts, `"PTEN ... activates AKT"`.
+- HTML labels inside `<foreignObject>`, which is how drawio's **Export as SVG**
+  writes them, are read too.
+- Text the reader cannot see is never quotable. That covers `<desc>`, `<title>`,
+  `<metadata>`, comments, attribute values (`id`, `fill`, `class`, ...),
+  `<defs>`/`<symbol>` content that no `<use>` draws, and text hidden by
+  `display:none`, `visibility:hidden`, zero `opacity`, `fill="none"` or a zero
+  `font-size`, whether set on the element, inherited from a group, or set by a
+  `<style>` rule.
+- If the extractor cannot tell what the figure renders, the reference is
+  `unavailable` rather than read as raw markup. That happens when a `<style>`
+  rule hides or shows text through a selector other than a type, class or id
+  selector (`g > text`, `:first-child`, `@media`), when visibility depends on
+  `var()` or `calc()`, when a stylesheet is imported or linked, when the SVG
+  contains a `<script>` or animates visibility, or when something changes the
+  characters drawn: CSS `content`, a string `list-style-type` or `quotes`,
+  `-webkit-text-security`, a `unicode-bidi` override, `<bdo>`, or a bidi
+  control character in the text.
+
+Known limits: a label drawn as several `<text>` elements, one per line, is
+read as several labels, so quote its lines with `...` between them. Text
+clipped by `clip-path` or `mask`, placed outside the
+`viewBox`, scaled to nothing, covered by another shape, drawn in the
+background colour, or cut short by CSS `text-overflow` or `-webkit-line-clamp`
+is still read. Text converted to outlines is not text and
+cannot be quoted. Native `.drawio` files and connector (edge) labels between
+nodes are not yet supported.
+
 ## Using in Data Files
 
 Both file and URL references work in LinkML data files:
@@ -145,6 +190,8 @@ Both file and URL references work in LinkML data files:
 
 ## Limitations
 
-- **PDF files**: Not yet supported (planned for future)
+- **Local files are read as text**, apart from `.svg` figures (see above).
+  An `.html` file is matched against its raw markup.
+- **Local PDF files**: Not yet supported as `file:` references. PDFs fetched by `url:` are supported.
 - **Authentication**: URLs requiring login are not supported
 - **Dynamic content**: JavaScript-rendered pages may not work

{linkml_reference_validator-0.3.0rc2 → linkml_reference_validator-0.3.1rc1}/docs/how-to/validate-urls.md

@@ -15,8 +15,8 @@ The linkml-reference-validator supports validating references that point to web
 When a reference field contains a URL, the validator:
 
 1. Fetches the web page content
-2. Extracts the page title from `<title>` tag (for HTML)
-3. Caches the content for future validations
+2. Extracts the page title (see [Titles](#titles))
+3. Converts HTML to readable text and caches it for future validations
 4. Validates your supporting text against the page content
 
 ## URL Format
@@ -79,14 +79,54 @@ When the validator encounters a URL reference, it:
 
 The fetcher stores:
 
-- **Title**: Extracted from the `<title>` tag (for HTML pages)
-- **Content**: The raw page content as received
-- **Content type**: Marked as `url` to distinguish from other reference types
-
-Note: The validator stores raw page content without HTML-to-text conversion.
-HTML tags remain in the cached file, and tag names can surface during
-normalization. If validation fails because tags interrupt the text, consider
-extracting plain text and validating against a local `file:` reference instead.
+- **Title**: See [Titles](#titles)
+- **Content**: Readable text for HTML pages; plain text and XML as received;
+  extracted text for PDFs
+- **Content type**: `url` for pages, `full_text_pdf` for PDFs with text
+
+HTML is converted to readable text before it is cached. A quote copied from
+the page then matches even when it runs through a link, bold or italic text,
+and a cached page can be read, and diffed, line by line:
+
+- Inline tags (`<a>`, `<b>`, `<em>`, `<span>`, ...) are removed and their
+  text is kept in place.
+- Block-level tags (`<p>`, `<li>`, `<h1>`-`<h6>`, `<div>`, `<br>`, ...) start
+  a new line.
+- A table row is one line, its cells separated by ` | `. Normalization drops
+  the `|`, so a quote copied from a rendered row with its cells separated by
+  spaces still matches.
+- `<pre>` keeps its whitespace. Elsewhere whitespace collapses to one space.
+- Entities such as `&amp;` are unescaped.
+
+The whole page is kept, including navigation, sidebars and footers; it is not
+narrowed to the article body. Because caches are often committed to public
+repositories and page scripts routinely carry signed asset URLs and API keys,
+`<script>`, `<style>`, `<noscript>` and `<template>` elements and HTML comments
+are removed with their text, and no tag attribute reaches the cache.
+
+#### Titles
+
+For HTML pages the title is the `citation_title` meta tag when present (the
+Highwire / Google Scholar convention, which names the article rather than the
+site), then the `<title>` tag.
+
+For PDFs the validator tries, in order:
+
+1. The `citation_title` of the publisher's landing page, where a known rule
+   maps the PDF URL to it. Currently J-STAGE: `.../_pdf` becomes
+   `.../_article`.
+2. The PDF's embedded `/Title` metadata, ignoring placeholders such as
+   `Microsoft Word - draft.doc`, bare filenames and `Untitled`.
+3. The URL itself.
+
+So a PDF entry whose `title` equals its URL is one for which no title was
+found.
+
+The landing-page rules and the placeholder titles are kept in
+`src/linkml_reference_validator/etl/rules.py`, with the other publisher and
+site rules. Each rule sits beside examples of what it must and must not match.
+To support another publisher, add a rule there with at least one example;
+`tests/test_rules.py` fails for a rule without one.
 
 ### 3. Caching
 
@@ -104,9 +144,9 @@ content_type: url
 ## Content
 
 <html>
-  <head>
-    <title>Chapter 3: Cell Structure and Function</title>
-  </head>
+<head>
+<title>Chapter 3: Cell Structure and Function</title>
+</head>
   ...
 ```
 
@@ -145,10 +185,11 @@ URL validation is designed for static web pages. It may not work well with:
 
 ### Raw Content
 
-The validator stores raw page content. For HTML pages:
+The validator stores the page's text, not its markup. For HTML pages:
 
-- HTML tags are preserved in the cache
-- The text normalization during validation handles most cases
+- Text that only appears in an attribute, such as an image's `alt` text, is
+  not kept
+- Text inside `<noscript>` or `<template>` is not kept
 - Complex HTML layouts may require careful text extraction
 
 ### No Rendering

linkml_reference_validator-0.3.1rc1/docs/reference/cache-format.md

@@ -0,0 +1,150 @@
+# Reference cache format and validation
+
+The library owns the contract for Markdown cache files. Use its public model and
+checkers instead of maintaining a downstream copy of the frontmatter fields:
+
+```python
+from linkml_reference_validator import (
+    CacheFrontmatter,
+    scan_cache_dir,
+    validate_cache_file,
+)
+
+result = validate_cache_file("references_cache/PMID_12345.md")
+for finding in result.findings:
+    print(result.path, finding.code, finding.field, finding.message)
+
+results = scan_cache_dir("references_cache")
+invalid = [result for result in results if not result.is_valid]
+if invalid:
+    raise SystemExit(f"{len(invalid)} invalid cache files")
+```
+
+These functions read files without fetching references, refreshing extraction
+stamps, or rewriting the cache. They return `CacheValidationResult` objects with
+`path`, `frontmatter`, `findings`, and an `is_valid` property. `frontmatter` is a
+`CacheFrontmatter` instance when the header passes validation, including when the
+filename fails its separate check; otherwise it is `None`.
+
+`scan_cache_dir(directory, recursive=False)` returns a list in path order. It
+checks immediate `*.md` files by default; pass `recursive=True` to include nested
+directories. It skips symlinked directories during recursion but reads symlinked
+Markdown files. Non-Markdown files, including legacy `.txt` caches and downloaded
+XML/PDF files, are outside this contract and are ignored. An existing empty directory returns `[]`. Missing,
+unreadable, or non-directory scan roots raise `OSError` (including its subclasses)
+so a mistaken path cannot silently pass a gate. Individual file errors are
+reported as findings and do not stop the scan.
+
+## Frontmatter contract
+
+Files contain UTF-8 text without a byte-order mark (BOM), with a YAML mapping
+between opening and closing `---` lines, beginning on the first line. The remaining
+text is an unrestricted Markdown body. Duplicate YAML keys are invalid.
+
+Only `reference_id` and `content_type` are required; both must be nonblank strings.
+All other known fields are optional and may be null:
+
+- Bibliography: `title`, `authors`, `journal`, `year`, `doi`, `keywords`, and
+  `publication_types`.
+- Extraction stamps: `extractor_version`, `html_full_text_version`,
+  `xml_extraction_version`, `url_source_version`, `svg_extraction_version`, and
+  `absent_content_version`.
+- Publication status: `is_preprint` and `peer_review_status`.
+- Full-text metadata: `full_text_attempted`, `full_text_declined`,
+  `full_text_provider`, `full_text_url`, `oa_status`, `license`, `local_pdf_path`,
+  `full_text_access_type`, and `full_text_source_item_id`.
+- Captured fields and attachments: `extra_fields_captured` and
+  `supplementary_files`.
+
+Known fields are checked without coercion. Extraction stamps and supplementary
+file sizes are integers (booleans do not count); `is_preprint` and
+`full_text_attempted` are booleans. Authors, keywords, publication types, and
+captured field names are lists of strings. `year` accepts a string or an integer
+for compatibility with older unquoted YAML years. Other scalar metadata is text.
+Each supplementary file requires a nonempty string `filename`; its optional fields
+are `download_url`, `content_type`, `size_bytes`, `checksum`, `description`, and
+`local_path`. Older cache files may contain unquoted numeric values in fields
+such as `checksum` that require strings. These are reported as `invalid_field`;
+quote the intended text to repair them. The lenient fetcher also recovers numeric
+and boolean scalars in string-valued metadata fields (including attachment
+metadata) by converting their parsed values to strings. The field types come
+from the public model; reference IDs, version stamps, booleans, lists, and
+unknown extension fields are not coerced.
+A cache rewrite then quotes these strings and retains consumer extensions.
+This compatibility recovery applies only to disk reads, not source plugin output
+or the public validator. It cannot reconstruct leading zeros or original
+scientific notation already lost during YAML parsing; check the source when the
+exact spelling matters.
+
+**Unknown keys are allowed and preserved**, both in the header and in
+supplementary-file metadata. No extension namespace is required: a consumer's
+`database: Orphanet` is valid. String vocabularies such as `content_type`,
+`peer_review_status`, and `full_text_access_type` are open, so future values do not
+break older consumers. Extraction stamps may be absent, old, or newer than this
+library; format validity does not imply freshness.
+
+When rewriting an existing cache with a valid header for the same reference,
+the fetcher carries over unknown fields, including explicit null values. Unknown
+fields are retained as-is, not recalculated or certified current on refresh;
+this also applies to fields written by a newer version of the library. Known
+fields always come from the new reference. Attachment extensions follow filenames
+that are unique in both the old and new attachment lists; removed or ambiguous
+attachments do not transfer their extensions. Preservation reads only the file
+being overwritten, so metadata is not copied between public and private caches
+or between different references whose sanitized filenames collide. A damaged
+header that remains invalid after legacy scalar recovery cannot supply validated
+extensions: the fetcher logs a warning and permits
+a fresh fetch to repair the file.
+
+The writer omits empty optional attachment strings, retaining the historical
+serialization policy. Invalid known metadata supplied by a source or carried
+through a lenient read raises `pydantic.ValidationError` before a cache write;
+`fetch()` propagates that error. An existing cache file is left intact. This
+makes a source's type-contract error visible instead of silently coercing values
+or reporting a successful cache write that never happened.
+
+`CacheFrontmatter` is a public Pydantic model released with the library. The writer
+serializes this same model. To inspect the complete, machine-readable contract:
+
+```python
+schema = CacheFrontmatter.model_json_schema()
+header = CacheFrontmatter.model_validate({
+    "reference_id": "PMID:12345",
+    "content_type": "abstract_only",
+    "database": "Orphanet",
+})
+assert header.model_dump()["database"] == "Orphanet"
+```
+
+The checker derives the filename from the stored `reference_id` by replacing
+`:`, `/`, `?`, and `=` with `_` and appending `.md`. DOI filenames are compared
+case-insensitively, matching the fetcher's existing behavior. Other filenames must
+match exactly. This uses the ID already stored in the file, without applying
+configured prefix aliases or requiring a currently registered reference source.
+
+## Findings and local policy
+
+Each `CacheFinding` has a stable `code`, a human-readable `message`, and a `field`
+when applicable. Field locations use dotted paths, for example `authors.0` or
+`supplementary_files.0.size_bytes`.
+
+| Code | Meaning |
+| --- | --- |
+| `read_error` | The file cannot be read or is not valid UTF-8. |
+| `invalid_frontmatter` | Delimiters are missing/misplaced or the YAML is not a mapping. |
+| `invalid_yaml` | YAML parsing failed, including duplicate keys. |
+| `invalid_field` | A required field is absent or a known field has an invalid type/value. |
+| `filename_mismatch` | The filename does not match the stored reference ID. |
+
+Apply local requirements after `result.is_valid`, using `result.frontmatter`.
+Requiring authors or a journal for particular reference sources, for example,
+is a consumer policy rather than a format rule. The ordinary fetcher's reader
+remains lenient for recovering old cache data; the checker reports malformed
+known fields instead of silently recovering them. In particular, the reader
+accepts text before the opening frontmatter delimiter; the checker requires the
+opening delimiter on the first line.
+
+Validation cannot prove that a file was never hand-edited, that its text agrees
+with the publication, or that its bibliography is complete. A well-formed edit
+still passes; use version-control review or an external integrity record to
+track changes when needed.

{linkml_reference_validator-0.3.0rc2 → linkml_reference_validator-0.3.1rc1}/docs/troubleshooting.md

@@ -902,6 +902,51 @@ rather than as `unavailable`, so it falls outside this stamp and is not
 re-tested. Clear such entries by hand if you were running that setting before
 this version.
 
+`url:` entries carry their own stamp, `url_source_version`. Version 1 is
+two URLSource changes: HTML is sanitized before caching, so page scripts,
+comments and attributes stop reaching the cache (issue #92), and a PDF's title
+is recovered from a publisher landing page or its embedded metadata rather than
+set to its URL (issue #93). Version 2 caches an HTML page as readable text
+rather than markup, so a quote that runs through a link or bold text matches
+(issue #102). Every quote that matched the version 1 markup still matches the
+text, provided the page itself has not changed. A `url:` entry with a missing or older stamp is
+re-fetched once on the next validation fetch. If the page cannot be reached,
+the old entry is still served and is not rewritten, so a later run retries.
+Other sources are unaffected. With `trust_cached_entries` set, old entries are
+served as they are and are not refreshed.
+
+`file:` references to `.svg` figures carry their own stamp,
+`svg_extraction_version`. Version 1 reads a figure as the text it renders
+(issue #104); before it, the file was cached as raw markup, so a quote could
+match a `<desc>`, an attribute value or hidden text. Such an entry is re-read
+once on the next validation fetch. **A quote that only matched the markup will
+then fail**, which is the point: cite text the figure actually shows. Unlike
+other stale entries, an old SVG entry is never served in place of a fresh
+read: not when the file has moved, not when the new read finds no rendered
+text, and not with `trust_cached_entries`. The file is the source, and its raw
+markup is not evidence. Version 2 separates a figure's labels, so a quote is
+matched within one label; version 1 entries are re-read once for it.
+
+**A quote that verified against a `url:` page can fail after that refresh.**
+Sanitizing removes `<noscript>` and `<template>` elements with their text.
+Some publisher pages put a fallback copy of the abstract inside `<noscript>`,
+for readers without JavaScript. A quote that only ever matched that copy will
+stop matching once the entry is re-fetched. If the same text appears elsewhere
+on the page, nothing changes. If it does not, cite a source that carries it,
+such as the `PMID:` or `DOI:` of the article.
+
+**The version 2 refresh rewrites every cached HTML page.** Each `url:` HTML
+entry changes from one line of markup to many lines of text, so expect a large
+diff in a committed cache. The refresh fetches each page again, so a page that
+has changed or gone since it was cached can stop supporting a quote; an
+unreachable page keeps its old entry, as above. To keep that diff reviewable,
+refresh in a commit of its own: right after upgrading, run your usual
+validation on an otherwise unchanged checkout and commit only the rewritten
+cache files, before any curation changes. Entries are refreshed only as a
+validation uses them (see
+[Cached references re-fetched after upgrading](#cached-references-re-fetched-after-upgrading)),
+so a page no validation cites keeps its old entry until one does.
+
 **A cache-wide refresh adds one line to every enriched entry.** Both index
 providers now set `access_type` where they previously left it unset, so a
 refreshed public entry gains a `full_text_access_type: open` line. It means the

{linkml_reference_validator-0.3.0rc2 → linkml_reference_validator-0.3.1rc1}/mkdocs.yml

@@ -52,6 +52,7 @@ nav:
       - Editorial Conventions: concepts/editorial-conventions.md
   - Reference:
       - CLI Reference: reference/cli.md
+      - Cache Format and Validation: reference/cache-format.md
       - Supporting Text Splitting: reference/splitting.md
   - Troubleshooting: troubleshooting.md
   - Roadmap: todo.md

{linkml_reference_validator-0.3.0rc2 → linkml_reference_validator-0.3.1rc1}/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
   "linkml-runtime >=1.9.4",
   "biopython >= 1.80",
   "beautifulsoup4 >= 4.9.0",
-  "lxml >= 4.9.0",
+  "lxml >= 5.0.0",
   "requests >= 2.28.0",
   "pydantic >= 2.0.0",
   "ruamel-yaml >= 0.18.0",

linkml_reference_validator-0.3.1rc1/src/linkml_reference_validator/__init__.py

@@ -0,0 +1,23 @@
+try:
+    from linkml_reference_validator._version import __version__, __version_tuple__
+except ImportError:  # pragma: no cover
+    __version__ = "0.0.0"
+    __version_tuple__ = (0, 0, 0)
+
+from linkml_reference_validator.cache import (
+    CacheFinding,
+    CacheFrontmatter,
+    CacheSupplementaryFile,
+    CacheValidationResult,
+    scan_cache_dir,
+    validate_cache_file,
+)
+
+__all__ = [
+    "CacheFinding",
+    "CacheFrontmatter",
+    "CacheSupplementaryFile",
+    "CacheValidationResult",
+    "scan_cache_dir",
+    "validate_cache_file",
+]