mirrorcommons 0.1.0__py3-none-any.whl

artifactgrid/__init__.py

@@ -0,0 +1,3 @@
+"""ArtifactGrid continuity manifest tools."""
+
+__version__ = "0.1.0"

artifactgrid/cli.py

@@ -0,0 +1,380 @@
+"""Command-line interface for the local ArtifactGrid validation workflow."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .manifest import (
+    build_manifest,
+    build_publisher_mirror_manifest,
+    protect_snapshot,
+    protect_snapshot_to_s3,
+    verify_manifest,
+    verify_s3_manifest,
+)
+from .continuity import (
+    check_huggingface_source,
+    discard_temporary_snapshot,
+    download_huggingface_snapshot,
+    restore_s3_snapshot,
+    restore_snapshot,
+    write_private_continuity_report,
+    write_private_s3_continuity_report,
+)
+from .publisher_api import complete_upload, create_publisher_draft, create_upload_session, decide_review, issue_source_challenge, publish_mirror, publisher_draft_status, publisher_token, put_multipart_file, put_staged_file, refresh_upload_grant, tombstone_mirror, verify_source_challenge
+
+
+def _add_snapshot_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--source-url", required=True, help="Public Hugging Face model URL.")
+    parser.add_argument("--revision", required=True, help="Full 40-character source commit hash.")
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument("--snapshot-dir", type=Path, help="Existing local snapshot directory.")
+    source.add_argument("--download", action="store_true", help="Download the exact revision directly from Hugging Face.")
+
+
+def _snapshot_directory(args: argparse.Namespace) -> Path:
+    if args.download:
+        return download_huggingface_snapshot(args.source_url, args.revision)
+    return args.snapshot_dir
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="artifactgrid", description="Create and verify local continuity evidence.")
+    subcommands = parser.add_subparsers(dest="command", required=True)
+
+    inspect_parser = subcommands.add_parser("inspect", help="Build a manifest from an existing local snapshot.")
+    _add_snapshot_arguments(inspect_parser)
+    inspect_parser.add_argument("--output", type=Path, help="Optional path for the JSON manifest. Prints to stdout when omitted.")
+
+    source_check_parser = subcommands.add_parser("source-check", help="Check whether the exact public source revision is reachable.")
+    source_check_parser.add_argument("--source-url", required=True, help="Public Hugging Face model URL.")
+    source_check_parser.add_argument("--revision", required=True, help="Full 40-character source commit hash.")
+    source_check_parser.add_argument("--output", type=Path, help="Optional path for the JSON observation. Prints to stdout when omitted.")
+
+    protect_parser = subcommands.add_parser("protect", help="Copy a supported snapshot into customer-controlled local storage.")
+    _add_snapshot_arguments(protect_parser)
+    protect_target = protect_parser.add_mutually_exclusive_group(required=True)
+    protect_target.add_argument("--backup-dir", type=Path, help="Customer-controlled local destination directory.")
+    protect_target.add_argument("--s3-bucket", help="Customer-controlled S3-compatible bucket.")
+    protect_parser.add_argument("--s3-prefix", default="artifactgrid", help="S3 object prefix when using --s3-bucket.")
+    protect_parser.add_argument("--s3-endpoint-url", help="Optional S3-compatible endpoint, for example an R2 endpoint.")
+    protect_parser.add_argument("--output", type=Path, help="Optional path for a portable local copy of the manifest.")
+
+    verify_parser = subcommands.add_parser("verify", help="Verify a previously protected local backup.")
+    verify_parser.add_argument("--manifest", type=Path, required=True, help="Path to artifactgrid-manifest.json.")
+    verify_target = verify_parser.add_mutually_exclusive_group(required=True)
+    verify_target.add_argument("--backup-dir", type=Path, help="Directory containing the protected files.")
+    verify_target.add_argument("--s3-bucket", help="Customer-controlled S3-compatible bucket.")
+    verify_parser.add_argument("--s3-prefix", default="artifactgrid", help="S3 object prefix when using --s3-bucket.")
+    verify_parser.add_argument("--s3-endpoint-url", help="Optional S3-compatible endpoint, for example an R2 endpoint.")
+
+    restore_parser = subcommands.add_parser("restore", help="Run a clean restore drill from a verified customer backup.")
+    restore_parser.add_argument("--manifest", type=Path, required=True, help="Path to artifactgrid-manifest.json.")
+    restore_target = restore_parser.add_mutually_exclusive_group(required=True)
+    restore_target.add_argument("--backup-dir", type=Path, help="Verified customer-controlled local backup directory.")
+    restore_target.add_argument("--s3-bucket", help="Verified customer-controlled S3-compatible bucket.")
+    restore_parser.add_argument("--s3-prefix", default="artifactgrid", help="S3 object prefix when using --s3-bucket.")
+    restore_parser.add_argument("--s3-endpoint-url", help="Optional S3-compatible endpoint, for example an R2 endpoint.")
+    restore_parser.add_argument("--restore-dir", type=Path, required=True, help="Empty destination for the restore drill.")
+
+    report_parser = subcommands.add_parser("report", help="Write a private Consumer Continuity Report.")
+    report_parser.add_argument("--manifest", type=Path, required=True, help="Path to artifactgrid-manifest.json.")
+    report_target = report_parser.add_mutually_exclusive_group(required=True)
+    report_target.add_argument("--backup-dir", type=Path, help="Verified customer-controlled local backup directory.")
+    report_target.add_argument("--s3-bucket", help="Verified customer-controlled S3-compatible bucket.")
+    report_parser.add_argument("--s3-prefix", default="artifactgrid", help="S3 object prefix when using --s3-bucket.")
+    report_parser.add_argument("--s3-endpoint-url", help="Optional S3-compatible endpoint, for example an R2 endpoint.")
+    report_parser.add_argument("--output", type=Path, required=True, help="Destination for the private HTML report.")
+    report_parser.add_argument("--restore-result", type=Path, help="Optional artifactgrid-restore-result.json from a drill.")
+    report_parser.add_argument("--source-observation", type=Path, help="Optional source-check JSON for this exact pinned revision.")
+
+    publish_parser = subcommands.add_parser("publish", help="Create and manage a Commons publisher draft through the shared control plane.")
+    publish_commands = publish_parser.add_subparsers(dest="publish_command", required=True)
+    publish_create = publish_commands.add_parser("create", help="Create a staged publisher draft; no files are uploaded by this command.")
+    publish_create.add_argument("--slug", required=True, help="Public ArtifactGrid Commons slug.")
+    publish_create.add_argument("--display-name", required=True, help="Public display name.")
+    publish_create.add_argument("--declared-license", required=True, help="License identifier or exact declared license text.")
+    publish_create.add_argument("--origin", choices=("community_mirror", "source_mirror", "original_creator_submission"), required=True)
+    publish_create.add_argument("--source-url", help="Public Hugging Face source URL for a community or source mirror.")
+    publish_create.add_argument("--revision", help="Full 40-character source commit for a community or source mirror.")
+    publish_create.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    publish_status = publish_commands.add_parser("status", help="Read the state of one of your private publisher drafts.")
+    publish_status.add_argument("--draft", required=True, help="Draft ID returned by publish create.")
+    publish_status.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    publish_challenge = publish_commands.add_parser("source-challenge", help="Issue a repository-controller challenge for a source mirror.")
+    publish_challenge.add_argument("--draft", required=True, help="Draft ID returned by publish create.")
+    publish_challenge.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    publish_verify_source = publish_commands.add_parser("verify-source", help="Check the controller challenge at the pinned source revision.")
+    publish_verify_source.add_argument("--draft", required=True, help="Draft ID returned by publish create.")
+    publish_verify_source.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    publish_upload = publish_commands.add_parser("upload", help="Upload a declared publisher manifest through private R2 staging grants.")
+    publish_upload.add_argument("--draft", required=True, help="Draft ID returned by publish create.")
+    publish_upload.add_argument("--manifest", type=Path, required=True, help="Publisher mirror manifest JSON created for this exact snapshot.")
+    publish_upload.add_argument("--snapshot-dir", type=Path, required=True, help="Directory containing the declared files.")
+    publish_upload.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    publish_inspect = publish_commands.add_parser("inspect", help="Build the local file inventory required for a publisher upload.")
+    publish_inspect.add_argument("--snapshot-dir", type=Path, required=True, help="Directory containing the staged artifact files.")
+    publish_inspect.add_argument("--slug", required=True, help="Public ArtifactGrid Commons slug.")
+    publish_inspect.add_argument("--origin", choices=("community_mirror", "source_mirror", "original_creator_submission"), required=True)
+    publish_inspect.add_argument("--source-url", help="Public Hugging Face source URL for a community or source mirror.")
+    publish_inspect.add_argument("--revision", help="Full 40-character source commit for a community or source mirror.")
+    publish_inspect.add_argument("--source-control", choices=("official_source_controller_verified", "not_claimed", "not_applicable"), required=True)
+    publish_inspect.add_argument("--output", type=Path, required=True, help="Destination for publisher-mirror-manifest.json.")
+
+    review_parser = subcommands.add_parser("review", help="Reviewer-only ArtifactGrid Commons actions.")
+    review_commands = review_parser.add_subparsers(dest="review_command", required=True)
+    review_decision = review_commands.add_parser("decision", help="Approve, restrict, or reject a private review case.")
+    review_decision.add_argument("--case", required=True, help="Private review case ID.")
+    review_decision.add_argument("--decision", choices=("approved", "restricted", "rejected"), required=True)
+    review_decision.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    review_publish = review_commands.add_parser("publish", help="Publish an approved mirror into the public Commons prefix.")
+    review_publish.add_argument("--mirror", required=True, help="Approved mirror ID.")
+    review_publish.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    review_tombstone = review_commands.add_parser("tombstone", help="Remove a published mirror while retaining an operator audit record.")
+    review_tombstone.add_argument("--mirror", required=True, help="Published mirror ID.")
+    review_tombstone.add_argument("--reason", required=True, help="Bounded operator reason retained in the private review history.")
+    review_tombstone.add_argument("--api-url", help="Override the ArtifactGrid API URL; for local development only.")
+    return parser
+
+
+def _s3_client(endpoint_url: str | None):
+    try:
+        import boto3
+    except ImportError as error:
+        raise RuntimeError("S3 support requires the boto3 dependency. Reinstall ArtifactGrid with its project dependencies.") from error
+    return boto3.client("s3", endpoint_url=endpoint_url)
+
+
+def _write_or_print_manifest(manifest: dict, output: Path | None) -> None:
+    serialized = json.dumps(manifest, indent=2, sort_keys=True) + "\n"
+    if output:
+        destination = output.expanduser()
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(serialized, encoding="utf-8")
+    else:
+        sys.stdout.write(serialized)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    try:
+        if args.command == "review" and args.review_command == "decision":
+            _write_or_print_manifest(decide_review(token=publisher_token(), review_case_id=args.case, decision=args.decision, base_url=args.api_url), None)
+            return 0
+        if args.command == "review" and args.review_command == "publish":
+            _write_or_print_manifest(publish_mirror(token=publisher_token(), mirror_id=args.mirror, base_url=args.api_url), None)
+            return 0
+        if args.command == "review" and args.review_command == "tombstone":
+            _write_or_print_manifest(tombstone_mirror(token=publisher_token(), mirror_id=args.mirror, reason=args.reason, base_url=args.api_url), None)
+            return 0
+        if args.command == "publish" and args.publish_command == "create":
+            response = create_publisher_draft(
+                token=publisher_token(),
+                slug=args.slug,
+                display_name=args.display_name,
+                declared_license=args.declared_license,
+                origin=args.origin,
+                source_url=args.source_url,
+                source_revision=args.revision,
+                base_url=args.api_url,
+            )
+            _write_or_print_manifest(response, None)
+            return 0
+        if args.command == "publish" and args.publish_command == "status":
+            _write_or_print_manifest(
+                publisher_draft_status(token=publisher_token(), draft_id=args.draft, base_url=args.api_url), None
+            )
+            return 0
+        if args.command == "publish" and args.publish_command == "source-challenge":
+            _write_or_print_manifest(
+                issue_source_challenge(token=publisher_token(), draft_id=args.draft, base_url=args.api_url), None
+            )
+            return 0
+        if args.command == "publish" and args.publish_command == "verify-source":
+            _write_or_print_manifest(
+                verify_source_challenge(token=publisher_token(), draft_id=args.draft, base_url=args.api_url), None
+            )
+            return 0
+        if args.command == "publish" and args.publish_command == "upload":
+            manifest = json.loads(args.manifest.expanduser().read_text(encoding="utf-8"))
+            if not isinstance(manifest, dict) or manifest.get("spec") != "artifactgrid/publisher-mirror-manifest@0.1":
+                raise ValueError("manifest must be an ArtifactGrid publisher mirror manifest")
+            files = manifest.get("files")
+            if not isinstance(files, list) or not files:
+                raise ValueError("publisher manifest contains no files")
+            declared_files: list[dict[str, object]] = []
+            root = args.snapshot_dir.expanduser().resolve()
+            for entry in files:
+                if not isinstance(entry, dict) or not isinstance(entry.get("path"), str) or not isinstance(entry.get("bytes"), int) or not isinstance(entry.get("sha256"), str):
+                    raise ValueError("publisher manifest contains an invalid file entry")
+                declared_files.append({"path": entry["path"], "bytes": entry["bytes"], "sha256": entry["sha256"]})
+            session = create_upload_session(token=publisher_token(), draft_id=args.draft, files=declared_files, base_url=args.api_url)
+            upload = session.get("upload") if isinstance(session, dict) else None
+            grants = upload.get("files") if isinstance(upload, dict) else None
+            if not isinstance(grants, list) or len(grants) != len(declared_files):
+                raise RuntimeError("ArtifactGrid returned an invalid upload session")
+            upload_id = upload.get("id")
+            if not isinstance(upload_id, str):
+                raise RuntimeError("ArtifactGrid returned an upload without an ID")
+
+            def fresh_single_url(path: str) -> str:
+                grant = refresh_upload_grant(token=publisher_token(), draft_id=args.draft, upload_id=upload_id, path=path, base_url=args.api_url)
+                put_url = grant.get("grant", {}).get("putUrl")
+                if not isinstance(put_url, str):
+                    raise RuntimeError("ArtifactGrid returned an invalid refreshed upload grant")
+                return put_url
+
+            def fresh_part_url(path: str, part_number: int) -> str:
+                grant = refresh_upload_grant(token=publisher_token(), draft_id=args.draft, upload_id=upload_id, path=path, part_numbers=[part_number], base_url=args.api_url)
+                parts = grant.get("grant", {}).get("parts")
+                if not isinstance(parts, list) or not parts or not isinstance(parts[0].get("putUrl"), str):
+                    raise RuntimeError("ArtifactGrid returned an invalid refreshed multipart grant")
+                return parts[0]["putUrl"]
+
+            multipart_completions: list[dict[str, object]] = []
+            for entry, grant in zip(declared_files, grants, strict=True):
+                if not isinstance(grant, dict):
+                    raise RuntimeError("ArtifactGrid returned an invalid file upload grant")
+                path = str(entry["path"])
+                source = root / path
+                if grant.get("kind") == "multipart":
+                    parts = grant.get("parts")
+                    if not isinstance(parts, list):
+                        raise RuntimeError("ArtifactGrid returned an invalid multipart upload grant")
+                    completions = put_multipart_file(
+                        parts=parts, source=source, expected_bytes=int(entry["bytes"]),
+                        get_fresh_part_url=lambda part_number, path=path: fresh_part_url(path, part_number),
+                    )
+                    multipart_completions.append({"path": path, "parts": completions})
+                elif isinstance(grant.get("putUrl"), str):
+                    put_staged_file(
+                        put_url=grant["putUrl"], source=source, expected_bytes=int(entry["bytes"]),
+                        get_fresh_put_url=lambda path=path: fresh_single_url(path),
+                    )
+                else:
+                    raise RuntimeError("ArtifactGrid returned an invalid file upload grant")
+            completion = complete_upload(token=publisher_token(), draft_id=args.draft, upload_id=upload_id, multipart=multipart_completions or None, base_url=args.api_url)
+            _write_or_print_manifest(completion, None)
+            return 0
+        if args.command == "publish" and args.publish_command == "inspect":
+            _write_or_print_manifest(
+                build_publisher_mirror_manifest(
+                    args.snapshot_dir,
+                    slug=args.slug,
+                    origin=args.origin,
+                    source_control=args.source_control,
+                    source_url=args.source_url,
+                    source_revision=args.revision,
+                ),
+                args.output,
+            )
+            return 0
+        if args.command == "source-check":
+            _write_or_print_manifest(check_huggingface_source(args.source_url, args.revision), args.output)
+            return 0
+        if args.command == "inspect":
+            snapshot_directory = _snapshot_directory(args)
+            try:
+                manifest = build_manifest(snapshot_directory, args.source_url, args.revision)
+                _write_or_print_manifest(manifest, args.output)
+                return 0
+            finally:
+                if args.download:
+                    discard_temporary_snapshot(snapshot_directory)
+        if args.command == "protect":
+            snapshot_directory = _snapshot_directory(args)
+            try:
+                if args.s3_bucket:
+                    manifest = protect_snapshot_to_s3(
+                        snapshot_directory,
+                        _s3_client(args.s3_endpoint_url),
+                        args.s3_bucket,
+                        args.s3_prefix,
+                        args.source_url,
+                        args.revision,
+                    )
+                    _write_or_print_manifest(manifest, args.output)
+                    return 0
+                manifest_path = protect_snapshot(snapshot_directory, args.backup_dir, args.source_url, args.revision)
+                if args.output:
+                    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+                    if not isinstance(manifest, dict):
+                        raise RuntimeError("local backup wrote an invalid manifest")
+                    _write_or_print_manifest(manifest, args.output)
+                print(f"Customer-attested backup written to {manifest_path}")
+                return 0
+            finally:
+                if args.download:
+                    discard_temporary_snapshot(snapshot_directory)
+        if args.command == "restore":
+            if args.s3_bucket:
+                manifest = json.loads(args.manifest.expanduser().read_text(encoding="utf-8"))
+                if not isinstance(manifest, dict):
+                    raise ValueError("manifest must be a JSON object")
+                result = restore_s3_snapshot(
+                    manifest,
+                    _s3_client(args.s3_endpoint_url),
+                    args.s3_bucket,
+                    args.s3_prefix,
+                    args.restore_dir,
+                )
+            else:
+                result = restore_snapshot(args.manifest, args.backup_dir, args.restore_dir)
+            print(f"Restore drill passed in {result['recovery_time_ms']} ms. Evidence written to {args.restore_dir}")
+            return 0
+        if args.command == "report":
+            restore_result = None
+            if args.restore_result:
+                restore_result = json.loads(args.restore_result.expanduser().read_text(encoding="utf-8"))
+                if not isinstance(restore_result, dict):
+                    raise ValueError("restore result must be a JSON object")
+            source_observation = None
+            if args.source_observation:
+                source_observation = json.loads(args.source_observation.expanduser().read_text(encoding="utf-8"))
+                if not isinstance(source_observation, dict):
+                    raise ValueError("source observation must be a JSON object")
+            if args.s3_bucket:
+                manifest = json.loads(args.manifest.expanduser().read_text(encoding="utf-8"))
+                if not isinstance(manifest, dict):
+                    raise ValueError("manifest must be a JSON object")
+                report_path = write_private_s3_continuity_report(
+                    manifest,
+                    _s3_client(args.s3_endpoint_url),
+                    args.s3_bucket,
+                    args.s3_prefix,
+                    report_path=args.output,
+                    restore_result=restore_result,
+                    source_observation=source_observation,
+                )
+            else:
+                report_path = write_private_continuity_report(
+                    args.manifest,
+                    args.backup_dir,
+                    report_path=args.output,
+                    restore_result=restore_result,
+                    source_observation=source_observation,
+                )
+            print(f"Private Consumer Continuity Report written to {report_path}")
+            return 0
+        if args.s3_bucket:
+            manifest = json.loads(args.manifest.expanduser().read_text(encoding="utf-8"))
+            errors = verify_s3_manifest(manifest, _s3_client(args.s3_endpoint_url), args.s3_bucket, args.s3_prefix)
+        else:
+            errors = verify_manifest(args.manifest, args.backup_dir)
+        if errors:
+            print("Verification failed:", file=sys.stderr)
+            for error in errors:
+                print(f"- {error}", file=sys.stderr)
+            return 1
+        print("Verification passed: every declared file exists and matches its SHA-256 checksum.")
+        return 0
+    except (OSError, RuntimeError, ValueError) as error:
+        print(f"ArtifactGrid: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())