mostlyright-data 0.9.2__tar.gz → 0.9.3__tar.gz

{mostlyright_data-0.9.2 → mostlyright_data-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mostlyright-data
-Version: 0.9.2
+Version: 0.9.3
 Summary: Mostly Right CLI for reproducible, reviewed Parquet datasets
 Project-URL: Homepage, https://mostlyright.md/
 Project-URL: Documentation, https://mostlyright.md/docs/guides/cli/
@@ -81,9 +81,9 @@ Description-Content-Type: text/markdown
 
 # mostlyright-data
 
-Mostly Right turns public weather, economic, financial, and satellite data into datasets for
-analysis and prediction. It keeps source links, timestamps, and check results with each dataset
-version.
+Mostly Right turns historical and live public data—weather, economic releases, financial reports,
+and satellite data—into consistent datasets for analysis and prediction. Each dataset includes
+source links, timestamps, and check results.
 
 `mostlyright-data` is the command-line client. It starts hosted research and build work, follows
 run status, and downloads completed Parquet datasets.

{mostlyright_data-0.9.2 → mostlyright_data-0.9.3}/README.md

@@ -1,8 +1,8 @@
 # mostlyright-data
 
-Mostly Right turns public weather, economic, financial, and satellite data into datasets for
-analysis and prediction. It keeps source links, timestamps, and check results with each dataset
-version.
+Mostly Right turns historical and live public data—weather, economic releases, financial reports,
+and satellite data—into consistent datasets for analysis and prediction. Each dataset includes
+source links, timestamps, and check results.
 
 `mostlyright-data` is the command-line client. It starts hosted research and build work, follows
 run status, and downloads completed Parquet datasets.

{mostlyright_data-0.9.2 → mostlyright_data-0.9.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mostlyright-data"
-version = "0.9.2"
+version = "0.9.3"
 description = "Mostly Right CLI for reproducible, reviewed Parquet datasets"
 readme = "README.md"
 requires-python = ">=3.13,<3.14"

{mostlyright_data-0.9.2 → mostlyright_data-0.9.3}/src/mostlyright/data_harness/sources/catalog/authoring_shards.py

@@ -193,6 +193,7 @@ class AuthoringLimits:
 
     max_records: int
     max_input_bytes: int
+    max_conflicting_identifiers: int = 0
     max_shard_entries: int = MAX_AUTHORING_SHARD_ENTRIES
     max_shard_bytes: int = MAX_AUTHORING_SHARD_BYTES
     max_disk_bytes: int = 30 * 1024 * 1024 * 1024
@@ -212,6 +213,14 @@ class AuthoringLimits:
                 raise CatalogAuthoringRefused(
                     "AUTHOR_LIMIT", f"limits.{name}", "must be a positive integer"
                 )
+        if type(self.max_conflicting_identifiers) is not int or (
+            self.max_conflicting_identifiers < 0
+        ):
+            raise CatalogAuthoringRefused(
+                "AUTHOR_LIMIT",
+                "limits.max_conflicting_identifiers",
+                "must be a non-negative integer; zero admits none",
+            )
         if (
             self.max_shard_entries > MAX_AUTHORING_SHARD_ENTRIES
             or self.max_shard_bytes > MAX_AUTHORING_SHARD_BYTES
@@ -226,6 +235,7 @@ class AuthoringLimits:
         return {
             "max_records": self.max_records,
             "max_input_bytes": self.max_input_bytes,
+            "max_conflicting_identifiers": self.max_conflicting_identifiers,
             "max_shard_entries": self.max_shard_entries,
             "max_shard_bytes": self.max_shard_bytes,
             "max_disk_bytes": self.max_disk_bytes,
@@ -460,6 +470,9 @@ class _RecordIndex:
         self.count = 0
         self.selected = 0
         self.input_bytes = 0
+        #: Identifiers the provider published twice with different content, arbitrated the way
+        #: the sweep's own pass map arbitrates them.
+        self.conflicting_identifiers = 0
         self._bounded = False
         self._connection = sqlite3.connect(path)
         self._connection.execute("PRAGMA journal_mode=OFF")
@@ -567,11 +580,32 @@ class _RecordIndex:
             "SELECT semantic FROM record WHERE identifier = ?", (identifier,)
         ).fetchone()
         if existing is not None:
-            if existing[0] != record["normalized_sha256"]:
+            if existing[0] == record["normalized_sha256"]:
+                return
+            # The provider published this identifier twice with different content. The sweep's
+            # pass map already settled that case -- `_deduplicate_sorted` keeps the lexically
+            # smaller semantic digest -- and the authored corpus has to be the same corpus the
+            # sweep converged on, so the same rule is applied here rather than a refusal that
+            # would make the two stages disagree about what the catalogue contains.
+            self.conflicting_identifiers += 1
+            if self.conflicting_identifiers > limits.max_conflicting_identifiers:
                 raise CatalogAuthoringRefused(
                     "AUTHOR_RECORD_CONFLICT",
                     "authoring.records",
-                    "one provider identifier carries two different semantic digests",
+                    "more identifiers carry two different semantic digests than were admitted",
+                )
+            if record["normalized_sha256"] < existing[0]:
+                self._connection.execute(
+                    "UPDATE record SET semantic = ?, page = ?, observed = ?, rank = ?, "
+                    "payload = ? WHERE identifier = ?",
+                    (
+                        record["normalized_sha256"],
+                        page_sha256,
+                        row[0],
+                        _selection_rank(record),
+                        raw,
+                        identifier,
+                    ),
                 )
             return
         self._connection.execute(

{mostlyright_data-0.9.2 → mostlyright_data-0.9.3}/src/mostlyright/data_harness/ux/commands/catalog_author.py

@@ -97,6 +97,16 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "--max-input-bytes", type=int, required=True, help="maximum normalized input bytes read"
     )
+    parser.add_argument(
+        "--max-conflicting-identifiers",
+        type=int,
+        help=(
+            "how many identifiers the provider may publish twice with different content "
+            "(default 0). Each one is settled the way the sweep's own pass map settled it, by "
+            "keeping the lexically smaller semantic digest, so the authored corpus is the corpus "
+            "the sweep converged on. The harvest receipt states the count"
+        ),
+    )
     parser.add_argument(
         "--max-shard-entries",
         type=int,
@@ -207,6 +217,7 @@ def _run_locked(
         limits=AuthoringLimits(
             max_records=args.max_records,
             max_input_bytes=args.max_input_bytes,
+            max_conflicting_identifiers=getattr(args, "max_conflicting_identifiers", None) or 0,
             max_shard_entries=args.max_shard_entries,
             max_shard_bytes=args.max_shard_bytes,
             max_disk_bytes=args.max_disk_bytes,

{mostlyright_data-0.9.2 → mostlyright_data-0.9.3}/src/mostlyright/data_harness/ux/commands/catalog_publish.py

@@ -222,10 +222,14 @@ _AUTHOR_STAGING_MEMBERS = frozenset(
 _AUTHOR_COUNT_MEMBERS = frozenset(
     {"records", "authored", "flagged", "skipped", "failed", "layer_text_truncated"}
 )
+#: The one authoring bound that is meaningfully zero: zero admits no double-published identifier,
+#: which is what an authoring run does when the flag is left off entirely.
+_AUTHOR_ZERO_ALLOWED = frozenset({"max_conflicting_identifiers"})
 _AUTHOR_LIMIT_MEMBERS = frozenset(
     {
         "max_records",
         "max_input_bytes",
+        "max_conflicting_identifiers",
         "max_shard_entries",
         "max_shard_bytes",
         "max_disk_bytes",
@@ -262,6 +266,7 @@ _AUTHOR_JOURNAL_MEMBERS = frozenset(
 #:                       it, and this command does not pretend otherwise.
 AUTHOR_CLAIM_RECONCILIATION = {
     "journal_sha256": "bound",
+    "limits.max_conflicting_identifiers": "unverified",
     "limits.max_disk_bytes": "unverified",
     "limits.max_input_bytes": "unverified",
     "limits.max_records": "bounded_below",
@@ -1234,7 +1239,10 @@ def _verify_author_receipt(
     if (
         not isinstance(limits, dict)
         or set(limits) != _AUTHOR_LIMIT_MEMBERS
-        or any(type(value) is not int or value < 1 for value in limits.values())
+        or any(
+            type(value) is not int or value < (0 if name in _AUTHOR_ZERO_ALLOWED else 1)
+            for name, value in limits.items()
+        )
         or limits["max_shard_entries"] > MAX_AUTHORING_SHARD_ENTRIES
         or limits["max_shard_bytes"] > MAX_AUTHORING_SHARD_BYTES
     ):