refsource 0.1.0__py3-none-any.whl

refsource/__main__.py

@@ -0,0 +1,213 @@
+"""Command line for refsource: look something up without writing any code.
+
+    refsource datasets loan limit
+    refsource fields conforming-loan-limits
+    refsource lookup conforming-loan-limits state=AL county_name="AUTAUGA COUNTY"
+    refsource search ai-model-deprecation-and-retirement gpt-4
+    refsource show conforming-loan-limits 01001
+
+Output leads with the value and follows it with the source and the quote,
+because a value without those is the thing this package exists not to be.
+Add --json for machine-readable output.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import warnings
+
+import refsource
+from refsource import _clip
+
+
+def _host(url):
+    return (url or "").split("://", 1)[-1].split("/", 1)[0].lower()
+
+
+def _print_record(rec, show_all_fields):
+    print("\n{0}".format(rec.id or rec.url))
+    for name, val in rec.values.items():
+        line = "  {0}: {1}".format(name, val)
+        if val.derived:
+            line += "   [our reading, not the page's words]"
+        elif not val.confirmed:
+            line += "   [not confirmed word-for-word in the quote]"
+        print(line)
+        # A field a second publisher reported must not be read off the
+        # record's own source line below it. Say whose value it is, here,
+        # next to the number.
+        if val.source and val.source != rec.source_url:
+            print("      from {0}".format(val.source))
+            if val.quote:
+                print('      quoted: "{0}"'.format(_clip(val.quote, 160)))
+        if show_all_fields and val.disagreement:
+            for alt in val.disagreement:
+                same = _host(alt.get("source")) == _host(val.source)
+                print("      {0} — {1} says: {2}".format(
+                    "the same publisher states this differently" if same
+                    else "another source disagrees",
+                    alt.get("source"), _clip(alt.get("value"), 200)))
+    if rec.source_url:
+        print("  source: {0}".format(rec.source_url))
+    if rec.source_quote:
+        print('  quoted: "{0}"'.format(rec.source_quote.strip()[:300]))
+    print("  page:   {0}".format(rec.url))
+    print("  verified {0}{1}".format(
+        rec.verified,
+        " — PAST ITS RE-CHECK DATE ({0})".format(rec.stale_after)
+        if rec.dataset.stale else ""))
+
+
+def cmd_datasets(args):
+    hits = refsource.datasets(" ".join(args.query) if args.query else None)
+    if args.json:
+        print(json.dumps([{
+            "slug": d.slug, "title": d.title, "records": d.record_count,
+            "last_verified": d.last_verified, "url": d.url, "fields": d.fields,
+        } for d in hits], indent=2))
+        return 0
+    for d in sorted(hits, key=lambda x: x.slug):
+        print("{0}\n    {1}\n    {2} records, verified {3} — {4}".format(
+            d.slug, d.title, d.record_count, d.last_verified, d.url))
+    print("\n{0} dataset(s).".format(len(hits)))
+    return 0
+
+
+def cmd_fields(args):
+    ds = refsource.dataset(args.slug)
+    if args.json:
+        print(json.dumps(ds.fields, indent=2))
+        return 0
+    print("{0} — {1}".format(ds.slug, ds.title))
+    for f in ds.fields:
+        print("  {0}".format(f))
+    return 0
+
+
+def _parse_filters(pairs):
+    filters = {}
+    for p in pairs:
+        if "=" not in p:
+            raise SystemExit(
+                "filters look like field=value (got {0!r}). "
+                "Run `refsource fields <slug>` for the field names.".format(p))
+        k, v = p.split("=", 1)
+        filters.setdefault(k.strip(), []).append(v.strip())
+    return {k: (v if len(v) > 1 else v[0]) for k, v in filters.items()}
+
+
+def cmd_lookup(args):
+    filters = _parse_filters(args.filters)
+    rows = refsource.lookup(args.slug, **filters)
+    if not rows and filters and not args.json:
+        # Nothing matched. Almost always the spelling: one dataset writes a
+        # state as "TX", the next as "Texas". Show what is actually there
+        # rather than leaving it looking like the data is missing.
+        ds = refsource.dataset(args.slug)
+        print("no matching records — these are the values present:")
+        for field in filters:
+            vals = ds.values_of(field, limit=8)
+            print("  {0}: {1}{2}".format(
+                field, ", ".join(vals) if vals else "(no values)",
+                " ..." if len(vals) == 8 else ""))
+        return 1
+    return _emit(rows, args)
+
+
+def cmd_search(args):
+    rows = refsource.search(args.slug, " ".join(args.text), limit=args.limit)
+    return _emit(rows, args)
+
+
+def cmd_show(args):
+    rec = refsource.get(args.slug, args.id)
+    if rec is None:
+        print("no record {0!r} in {1}".format(args.id, args.slug), file=sys.stderr)
+        return 1
+    return _emit([rec], args)
+
+
+def _emit(rows, args):
+    if args.json:
+        print(json.dumps([r.to_dict() for r in rows[: args.limit]], indent=2))
+        return 0
+    if not rows:
+        print("no matching records")
+        return 1
+    for rec in rows[: args.limit]:
+        _print_record(rec, show_all_fields=True)
+    if len(rows) > args.limit:
+        print("\n... and {0} more (use --limit)".format(len(rows) - args.limit))
+    return 0
+
+
+def main(argv=None):
+    # The shared flags hang off every subcommand as well as the top level, so
+    # both `refsource --json lookup ...` and `refsource lookup ... --json` work
+    # — people type the second one. SUPPRESS is what makes that safe: without
+    # it a subparser's default would overwrite a flag given before the command.
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--json", action="store_true",
+                        default=argparse.SUPPRESS, help="machine-readable output")
+    common.add_argument("--limit", type=int, default=argparse.SUPPRESS,
+                        help="max records shown (default 20)")
+    common.add_argument("--offline", action="store_true",
+                        default=argparse.SUPPRESS,
+                        help="use only what is already cached")
+    common.add_argument("--base-url", default=argparse.SUPPRESS,
+                        help="fetch bundles from here instead")
+
+    ap = argparse.ArgumentParser(
+        prog="refsource", parents=[common],
+        description="Look up reference data that carries its own source.",
+        epilog="Catalogue and method: https://referencesource.org/")
+    sub = ap.add_subparsers(dest="cmd")
+
+    p = sub.add_parser("datasets", parents=[common],
+                       help="list or search the catalogue (no network)")
+    p.add_argument("query", nargs="*")
+    p.set_defaults(func=cmd_datasets)
+
+    p = sub.add_parser("fields", parents=[common],
+                       help="field names of one dataset (no network)")
+    p.add_argument("slug")
+    p.set_defaults(func=cmd_fields)
+
+    p = sub.add_parser("lookup", parents=[common],
+                       help="records matching field=value filters")
+    p.add_argument("slug")
+    p.add_argument("filters", nargs="*")
+    p.set_defaults(func=cmd_lookup)
+
+    p = sub.add_parser("search", parents=[common],
+                       help="records containing some text")
+    p.add_argument("slug")
+    p.add_argument("text", nargs="+")
+    p.set_defaults(func=cmd_search)
+
+    p = sub.add_parser("show", parents=[common], help="one record by id")
+    p.add_argument("slug")
+    p.add_argument("id")
+    p.set_defaults(func=cmd_show)
+
+    args = ap.parse_args(argv)
+    if not getattr(args, "func", None):
+        ap.print_help()
+        return 2
+    args.json = getattr(args, "json", False)
+    args.limit = getattr(args, "limit", 20)
+    args.offline = getattr(args, "offline", False)
+    args.base_url = getattr(args, "base_url", None)
+
+    refsource.configure(base_url=args.base_url, offline=args.offline or None)
+    warnings.simplefilter("always", refsource.StaleDataWarning)
+    try:
+        return args.func(args)
+    except refsource.RefsourceError as exc:
+        print("refsource: {0}".format(exc), file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())