file-observer 1.2.2__tar.gz → 1.2.4__tar.gz

{file_observer-1.2.2/src/file_observer.egg-info → file_observer-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: file-observer
-Version: 1.2.2
+Version: 1.2.4
 Summary: Not a file watcher — a one-shot, read-only scan that emits a deterministic JSON manifest of what's in a directory before you ingest it.
 Author-email: Russell Pfister <russalo@russalo.com>
 License: AGPL-3.0-or-later

{file_observer-1.2.2 → file_observer-1.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "file-observer"
-version = "1.2.2"
+version = "1.2.4"
 description = "Not a file watcher — a one-shot, read-only scan that emits a deterministic JSON manifest of what's in a directory before you ingest it."
 readme = "docs/README.md"
 requires-python = ">=3.12"

{file_observer-1.2.2 → file_observer-1.2.4}/src/file_observer/scanner.py

@@ -71,8 +71,8 @@ except ImportError:
     _defusedxml_available = False
 
 
-SCANNER_VERSION = "1.2.2"
-LOGIC_VERSION = "1.1.2"
+SCANNER_VERSION = "1.2.4"
+LOGIC_VERSION = "1.1.4"
 SCHEMA_VERSION = "1.2"
 
 
@@ -546,21 +546,21 @@ CHATLOG_TOOL = "chatlog_signals"
 # any detection regex or extraction algorithm requires bumping METHOD_VERSION
 # and updating the rules definition string.
 CHATLOG_VECTOR_ID = "chatlog"
-CHATLOG_METHOD_VERSION = 6  # v1.2.2: prose Rule 1 requires a recurring speaker; expanded stop-list (FP fixes)
+CHATLOG_METHOD_VERSION = 8  # v1.2.4: case-insensitive stop-list; embedded-dialogue parity with prose Rule 1
 CHATLOG_RULES_DEFINITION = (
-    "detect:speaker_label_re(distinct>=2,total>=3,recurring>=1,stop_list),"
+    "detect:speaker_label_re(distinct>=2,total>=3,recurring>=1,stop_list_ci),"
     "h3_header_re(5+)|section_divider_re(3+)[require speaker_cosignal_distinct>=2],"
     "json_conversation(role_keys{type,role,from,speaker,author}+content_keys{text,value,content,message,body},"
-    "line/array/tree,embedded_speaker_labels,require msgs>=3 AND distinct_speakers>=2);"
-    "extract:turn_count,speaker_labels(freq>=3,stop_list),section_markers,"
+    "line/array/tree,embedded_speaker_labels(distinct>=2,total>=3,recurring>=1),require msgs>=3 AND distinct_speakers>=2);"
+    "extract:turn_count,speaker_labels(freq>=3,stop_list_ci),section_markers,"
     "turn_char_stats,speaker_turn_counts,speaker_turn_chars,alternation,"
     "reference_tokens(at_mentions,wiki_links,code_fence_blocks,url_count),"
     "top_capitalized_tokens(freq>=3,top20),vocabulary_size_estimate;"
-    "stop_list:Allow,Arguments,Authorization,Bcc,CAUTION,Cc,Command,Commands,"
-    "Copyright,Date,Description,Disallow,Distribution,Documentation,Error,Example,"
-    "Examples,FIXME,Format,From,IMPORTANT,License,Lines,Message,NOTE,Newsgroups,"
-    "Note,Options,Organization,Parameters,Password,Path,References,Result,Returns,"
-    "Sender,Subject,Summary,Synopsis,TIP,TODO,To,Usage,Version,Warning"
+    "stop_list_ci:allow,answer,arguments,authorization,bcc,caution,cc,command,commands,"
+    "copyright,date,description,disallow,distribution,documentation,error,example,examples,"
+    "fixme,format,from,important,license,lines,message,newsgroups,note,options,organization,"
+    "parameters,password,path,question,references,result,returns,sender,subject,summary,"
+    "synopsis,tip,to,todo,usage,version,warning"
 )
 CHATLOG_STATIC_TUNING = {
     "detection_threshold": 3,
@@ -624,7 +624,20 @@ CHATLOG_SPEAKER_STOP_LIST: set[str] = {
     "Version", "Usage", "Options", "Command", "Commands", "Format", "Synopsis",
     "Description", "Arguments", "Parameters", "Returns", "Summary",
     "Authorization", "Documentation", "License", "Copyright", "Password",
+    # v1.2.3: FAQ labels — `Question:`/`Answer:` recur like speakers but aren't
+    # (stopgap for the most common recurring-Key:value FP; the real fix is a
+    # non-count signal — see scratch/review/v1_2_2_fp_findings.md). Deliberately
+    # NOT adding single letters `Q`/`A`: `A:`/`B:` is legitimate anonymized dialogue.
+    "Question", "Answer",
 }
+# v1.2.4: match case-INSENSITIVELY. The label regex matches all-caps, so the
+# case-sensitive set left holes (ALL-CAPS `FROM:`/`SUBJECT:` header dumps still
+# false-positived; only FAQ got dual-cased in v1.2.3). Folding once here closes
+# every all-caps hole AND removes the dual-listing maintenance burden — `User`
+# stays a legitimate speaker because it isn't in the set in any case.
+CHATLOG_SPEAKER_STOP_LIST_CF: frozenset[str] = frozenset(
+    w.casefold() for w in CHATLOG_SPEAKER_STOP_LIST
+)
 
 # v0.9: Reference tokens vector identity constants.
 REFERENCE_TOKENS_VECTOR_ID = "reference_tokens"
@@ -2544,7 +2557,7 @@ class Scanner:
         # sweep showed separates transcripts (recurring) from header blocks (one-shot).
         speaker_matches = [
             m.group(1) for m in CHATLOG_SPEAKER_LABEL_RE.finditer(text)
-            if m.group(1) not in CHATLOG_SPEAKER_STOP_LIST
+            if m.group(1).casefold() not in CHATLOG_SPEAKER_STOP_LIST_CF
         ]
         speaker_counts = Counter(speaker_matches)
         if (len(speaker_counts) >= 2 and sum(speaker_counts.values()) >= 3
@@ -2571,14 +2584,17 @@ class Scanner:
     @staticmethod
     def _string_has_speaker_dialogue(s: Any) -> bool:
         """Dialogue embedded in a JSON string value (e.g. hh-rlhf's
-        '\\n\\nHuman: ...\\n\\nAssistant: ...'): 3+ prose speaker labels."""
+        '\\n\\nHuman: ...\\n\\nAssistant: ...'). v1.2.4: applies the SAME predicate
+        as prose Rule 1 — >=2 distinct speakers, >=3 total, >=1 recurring — so the
+        same text isn't detected inside a JSON string yet rejected as prose."""
         if not isinstance(s, str) or len(s) < 20:
             return False
-        labels = [
+        labels = Counter(
             m.group(1) for m in CHATLOG_SPEAKER_LABEL_RE.finditer(s)
-            if m.group(1) not in CHATLOG_SPEAKER_STOP_LIST
-        ]
-        return len(labels) >= 3
+            if m.group(1).casefold() not in CHATLOG_SPEAKER_STOP_LIST_CF
+        )
+        return (len(labels) >= 2 and sum(labels.values()) >= 3
+                and any(c >= 2 for c in labels.values()))
 
     def _detect_conversational_json(self, text: str) -> bool:
         """Generalized conversational JSON/JSONL detection (v1.2; tightened v1.2.1).
@@ -2687,7 +2703,7 @@ class Scanner:
         (e.g. hh-rlhf's '\\n\\nHuman: ...\\n\\nAssistant: ...')."""
         pairs: list[tuple[str, str]] = []
         matches = [m for m in CHATLOG_SPEAKER_LABEL_RE.finditer(s)
-                   if m.group(1) not in CHATLOG_SPEAKER_STOP_LIST]
+                   if m.group(1).casefold() not in CHATLOG_SPEAKER_STOP_LIST_CF]
         for i, m in enumerate(matches):
             end = matches[i + 1].start() if i + 1 < len(matches) else len(s)
             pairs.append((m.group(1), s[m.end():end].strip()))
@@ -2811,7 +2827,7 @@ class Scanner:
             # v0.9.1: filter stop-list tokens from both detection and turn metrics
             raw_label_matches = [
                 m for m in CHATLOG_SPEAKER_LABEL_RE.finditer(text)
-                if m.group(1) not in CHATLOG_SPEAKER_STOP_LIST
+                if m.group(1).casefold() not in CHATLOG_SPEAKER_STOP_LIST_CF
             ]
             turn_count = len(raw_label_matches)
             label_counts = Counter(m.group(1) for m in raw_label_matches)

{file_observer-1.2.2 → file_observer-1.2.4/src/file_observer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: file-observer
-Version: 1.2.2
+Version: 1.2.4
 Summary: Not a file watcher — a one-shot, read-only scan that emits a deterministic JSON manifest of what's in a directory before you ingest it.
 Author-email: Russell Pfister <russalo@russalo.com>
 License: AGPL-3.0-or-later

{file_observer-1.2.2 → file_observer-1.2.4}/src/file_observer.egg-info/SOURCES.txt

@@ -20,4 +20,6 @@ tests/test_unit.py
 tests/test_v1_1.py
 tests/test_v1_2.py
 tests/test_v1_2_1.py
-tests/test_v1_2_2.py
+tests/test_v1_2_2.py
+tests/test_v1_2_3.py
+tests/test_v1_2_4.py

{file_observer-1.2.2 → file_observer-1.2.4}/tests/test_packaging.py

@@ -17,7 +17,7 @@ def test_canonical_top_level_api():
 
     import file_observer
 
-    assert file_observer.__version__ == "1.2.2"
+    assert file_observer.__version__ == "1.2.4"
     assert "Scanner" in file_observer.__all__
 
 
@@ -49,9 +49,9 @@ def test_canonical_submodule_constants_unchanged():
     """Constants and the manifest field they feed stay stable across the 1.0.x → 1.1 line (the 1.0.1 import-package rename left them intact)."""
     from file_observer.scanner import SCANNER_VERSION, LOGIC_VERSION, SCHEMA_VERSION
 
-    assert SCANNER_VERSION == "1.2.2"
+    assert SCANNER_VERSION == "1.2.4"
     assert SCHEMA_VERSION == "1.2"  # additive minor bump (v1.1 corpus-intelligence fields)
-    assert LOGIC_VERSION == "1.1.2"  # v1.2.2: prose chatlog detection tightened (recurrence + stop-list)
+    assert LOGIC_VERSION == "1.1.4"  # v1.2.4: case-insensitive stop-list + embedded-dialogue parity
 
 
 def test_legacy_scanner_import_warns():

{file_observer-1.2.2 → file_observer-1.2.4}/tests/test_unit.py

@@ -811,8 +811,8 @@ class TestScanContext:
         (tmp_path / "a.txt").write_text("hello")
         manifest = Scanner(source_dir=tmp_path).scan()
         ctx = manifest.context
-        assert ctx.scanner_version == "1.2.2"
-        assert ctx.logic_version == "1.1.2"
+        assert ctx.scanner_version == "1.2.4"
+        assert ctx.logic_version == "1.1.4"
         assert ctx.python_version  # non-empty
         assert ctx.platform  # non-empty
 
@@ -850,7 +850,7 @@ class TestScanContext:
         manifest = Scanner(source_dir=tmp_path).scan()
         data = json_mod.loads(manifest_to_json(manifest))
         assert "context" in data
-        assert data["context"]["scanner_version"] == "1.2.2"
+        assert data["context"]["scanner_version"] == "1.2.4"
 
 
 # ---------------------------------------------------------------------------
@@ -2042,8 +2042,8 @@ class TestSemanticToolNames:
 
     def test_version_is_current(self) -> None:
         from file_observer.scanner import SCANNER_VERSION, LOGIC_VERSION
-        assert SCANNER_VERSION == "1.2.2"
-        assert LOGIC_VERSION == "1.1.2"
+        assert SCANNER_VERSION == "1.2.4"
+        assert LOGIC_VERSION == "1.1.4"
 
 
 # ---------------------------------------------------------------------------
@@ -3530,4 +3530,4 @@ class TestMarkdownReport:
         from file_observer.scanner import manifest_to_markdown
         (tmp_path / "a.txt").write_text("hello")
         md = manifest_to_markdown(Scanner(source_dir=tmp_path).scan())
-        assert "1.2.2" in md
+        assert "1.2.4" in md

file_observer-1.2.4/tests/test_v1_2_3.py

@@ -0,0 +1,41 @@
+"""v1.2.3 — FAQ false-positive stopgap.
+
+Pro + flash both flagged that recurring `Question:`/`Answer:` labels (a FAQ doc)
+satisfy the v1.2.2 prose rule (≥2 distinct, ≥3 total, ≥1 recurring) and
+false-positive as a chatlog. v1.2.3 adds `Question`/`Answer` to the stop-list.
+
+This is a STOPGAP for the most common single case — NOT the real fix. The root
+issue (prose `Key:value` is ambiguous with dialogue) needs a non-count signal;
+see scratch/review/v1_2_2_fp_findings.md. We deliberately do NOT stop-list single
+letters like `Q`/`A` — `A:`/`B:` is a legitimate anonymized-dialogue pattern.
+
+Falsify-first: the FAQ test FAILS on v1.2.2, passes after the stop-list add.
+"""
+
+from file_observer.scanner import Scanner, ScannerConfig
+
+
+def _is_chatlog(tmp_path, name, content):
+    (tmp_path / name).write_text(content)
+    return Scanner(source_dir=tmp_path, config=ScannerConfig()).scan().files[0].is_chatlog
+
+
+def test_faq_question_answer_not_flagged(tmp_path):
+    c = ("Question: How do I install?\nAnswer: Use pip.\n"
+         "Question: Where are the docs?\nAnswer: In docs/.\n")
+    assert _is_chatlog(tmp_path, "FAQ.md", c) is False
+
+
+def test_faq_allcaps_not_flagged(tmp_path):
+    # case-sensitive stop-list + the label regex matches all-caps → ALL-CAPS FAQ
+    # style needs explicit coverage (PR #30 review: flash/codex/copilot flagged it).
+    c = ("QUESTION: How do I install?\nANSWER: Use pip.\n"
+         "QUESTION: Where are the docs?\nANSWER: In docs/.\n")
+    assert _is_chatlog(tmp_path, "FAQ.md", c) is False
+
+
+def test_anonymized_ab_dialogue_still_detected(tmp_path):
+    # `A:`/`B:` anonymized speakers are a LEGIT conversation — must stay detected
+    # (guards against over-stop-listing single letters while fixing FAQ).
+    c = "A: hey did you see this\nB: yeah just now\nA: what do you think\n"
+    assert _is_chatlog(tmp_path, "dialogue.txt", c) is True

file_observer-1.2.4/tests/test_v1_2_4.py

@@ -0,0 +1,56 @@
+"""v1.2.4 — two clean wins from the in-house multi-agent recall review (PR #31).
+
+The Claude multi-agent /code-review pass found two real issues the cross-model
+(Gemini) reviews missed, both verified live:
+
+  F9: the stop-list is case-sensitive, but the case-sensitivity fix was applied
+      only to FAQ (Question/QUESTION) — ALL-CAPS email/usenet headers (FROM:/
+      SUBJECT:) still false-positived. Fix: match the stop-list case-insensitively
+      (also kills the dual-casing maintenance burden).
+  F4: `_string_has_speaker_dialogue` (the embedded-JSON-string path) was never
+      hardened — still `len(labels) >= 3` with no distinct/recurrence — so the same
+      dialogue detected inside a JSON string was rejected as prose. Fix: apply the
+      prose Rule 1 predicate (>=2 distinct, >=3 total, >=1 recurring).
+
+Falsify-first: the two FP/asymmetry tests FAIL on v1.2.3, pass after the fix; the
+guard tests must stay green throughout. See scratch/review/v1_2_2_fp_findings.md.
+"""
+
+import json
+from file_observer.scanner import Scanner, ScannerConfig
+
+
+def _is_chatlog(tmp_path, name, content):
+    (tmp_path / name).write_text(content)
+    return Scanner(source_dir=tmp_path, config=ScannerConfig()).scan().files[0].is_chatlog
+
+
+class TestF9CaseInsensitiveStopList:
+    def test_allcaps_email_headers_not_flagged(self, tmp_path):
+        # ALL-CAPS recurring RFC822/usenet header dump — the FP class v1.2.2 targeted,
+        # but only title-case From/Subject were stop-listed.
+        c = "FROM: alice\nSUBJECT: re hi\nFROM: bob\nSUBJECT: re re hi\n"
+        assert _is_chatlog(tmp_path, "thread.txt", c) is False
+
+    def test_allcaps_faq_still_not_flagged(self, tmp_path):
+        c = "QUESTION: how?\nANSWER: like so.\nQUESTION: and then?\nANSWER: done.\n"
+        assert _is_chatlog(tmp_path, "FAQ.md", c) is False
+
+    def test_user_still_a_legit_speaker(self, tmp_path):
+        # `User` is intentionally NOT stop-listed; case-insensitive matching must
+        # not start suppressing it.
+        c = "User: hi there\nBob: hey\nUser: how are you\n"
+        assert _is_chatlog(tmp_path, "chat.txt", c) is True
+
+
+class TestF4EmbeddedDialogueParity:
+    def test_embedded_json_dialogue_requires_recurrence(self, tmp_path):
+        # 3 distinct one-shot speakers in a JSON string value: rejected as prose
+        # (Rule 1), so must also be rejected here — no container-format asymmetry.
+        c = json.dumps({"chosen": "Alice: hi\n\nBob: hello\n\nCarol: hey there friend"})
+        assert _is_chatlog(tmp_path, "e.jsonl", c) is False
+
+    def test_hh_rlhf_embedded_still_detected(self, tmp_path):
+        # 2 distinct, recurring (Human x2) — a real embedded conversation, must stay.
+        c = json.dumps({"chosen": "\n\nHuman: hi\n\nAssistant: hello\n\nHuman: bye"})
+        assert _is_chatlog(tmp_path, "hh.jsonl", c) is True