snifftest 0.1.0

package/rules/default.yaml

@@ -0,0 +1,458 @@
+# Sniff Test, the default ruleset.
+#
+# Fifteen rules, ten of which apply to most prose and five of which are
+# conventions of copy meant to sell or persuade. Five rules are countable and
+# run as regular expressions on your machine, with nothing sent anywhere. Ten
+# are judgment calls and are answered by the judgment model, one probability
+# per rule per paragraph. A flag names its rule and its probability.
+#
+# The rules tagged `marketing` encode a convention of marketing copy, and they
+# sit out an ordinary run because `off_by_default` names that tag below. Ask
+# for them on a landing page with `--only marketing`, or run everything else
+# and add them with a project file that extends this one and writes
+# `off_by_default: []`. Every rule here is one view of good writing. Disagree
+# with any of them by editing your own copy of the file; that is the feature.
+#
+# A countable rule may name the kinds of block it applies to with `chunks`.
+# The kinds are front_matter, heading, table, link_definition, html_comment,
+# list, block_quote and prose. A rule that names none applies to prose, block
+# quotes, lists and headings, which is everything a person would call writing.
+# The judgment rules are asked about prose, block quotes and lists long enough
+# to hold a sentence, and never about the rest: a heading is not a paragraph,
+# and a request about one costs what a request about a paragraph costs.
+#
+# A `.snifftest.yaml` in your project extends or replaces this file. Copy a
+# rule, change its words, keep its id unique. The `seed` on each rule is how
+# `snifftest eval` manufactures one defect of that kind inside your own clean
+# text, so that recall and false alarms can be measured on your prose rather
+# than ours. A transform seed is mechanical; a splice seed inserts one of the
+# listed sentences, at the start, at the end, or anywhere, per `position`.
+#
+# Each judgment rule carries four parts. `what` describes the defect.
+# `not_for` lists the near misses that must not be flagged. `examples` are
+# sentences that do trip the rule. `criteria` are the two statements the
+# model decides between. The contrast is what makes the judgment reliable;
+# a rule with a `what` and no `not_for` will flag things you did not mean.
+
+version: 1
+threshold: 0.7
+
+# Tags whose rules sit out a run unless they are asked for by name.
+off_by_default: [marketing]
+
+rules:
+  # --- countable rules: regular expressions, no network -------------------
+
+  - id: dash_present
+    kind: regex
+    # The built-in matches two characters: an en dash (U+2013) and an em dash
+    # (U+2014). A plain hyphen in a compound word does not count, and neither
+    # does an en dash between numerals, because "pages 10-20" written with the
+    # typographer's dash is a range and this rule is about a writer reaching
+    # for a dash instead of a full stop.
+    #
+    # A block quote is somebody else's text, so its dashes are theirs and the
+    # rule stays out of it. A quotation running inline inside a sentence cannot
+    # be told from the writer's own words without parsing quotation marks that
+    # mean five other things as well, so an inline quotation is still flagged.
+    builtin: dash_present
+    chunks: [prose, list, heading]
+    message: "A long dash. Give the sentence a full stop instead."
+    seed: { transform: insert_em_dash, count: 1 }
+
+  - id: colon_heavy
+    kind: regex
+    # The built-in counts the colon character (U+003A) in the paragraph and
+    # trips at `min` or more. A semicolon is not a colon. The colon in a URL,
+    # the one in a clock time and the ones inside an HTML comment are not
+    # punctuation a reader sees, so they are not counted.
+    #
+    # A list is not one paragraph: four bullets with a colon apiece are four
+    # sentences, and counting them together told a design note to pick one
+    # colon out of four.
+    builtin: colon_count
+    min: 3
+    chunks: [prose, block_quote]
+    message: "Three colons in one paragraph. Pick one."
+    seed: { transform: add_colons, count: 3 }
+
+  - id: sentence_rhythm
+    kind: regex
+    # The built-in splits on (?<=[.!?])\s+, counts words per sentence, and
+    # trips when the coefficient of variation (standard deviation over mean)
+    # is below `floor`. Paragraphs with fewer than `min_sentences` sentences
+    # or fewer than `min_words` words are skipped: four six-word sentences are
+    # a status note, and telling its author that every sentence runs to the
+    # same length is telling them their note is a note.
+    builtin: sentence_rhythm
+    floor: 0.25
+    min_sentences: 4
+    min_words: 60
+    chunks: [prose, block_quote]
+    message: "Every sentence runs to the same length. The reader nods off by the third."
+    seed: { transform: equalize_sentences }
+
+  - id: slop_vocab
+    kind: regex
+    # The built-in matches any listed word and its ordinary inflections: the
+    # stem as written plus s, es, ed, ing, ly, ment(s) and ion(s), with a final
+    # silent e dropped where English drops it, so "delve" catches "delves" and
+    # "delving", and "seamless" catches "seamlessly". Matching is
+    # case-insensitive and stops at a word boundary, so "landscape" does not
+    # catch "landscaped lawn" through some longer word. A derivation further
+    # from the stem than those suffixes, such as "intricacies", is a separate
+    # word: add it to the list if you want it.
+    #
+    # `except` lists phrases in which a listed word is doing its literal job.
+    # These are matched case-insensitively and take the word out of the count.
+    # Edit both lists freely; a word on this list is a tell in some registers
+    # and the plain choice in others.
+    builtin: slop_vocab
+    words:
+      - delve
+      - tapestry
+      - landscape
+      - unlock
+      - testament
+      - navigate
+      - realm
+      - myriad
+      - intricate
+      - pivotal
+      - seamless
+      - showcase
+    except:
+      - landscape architect
+      - landscape architecture
+      - landscape gardener
+      - landscape orientation
+      - landscape mode
+      - navigate to
+      - navigate by
+      - navigate the menu
+      - navigation bar
+      - navigation pane
+      - unlock the door
+      - unlock the screen
+      - unlock your phone
+    message: "A word that turns up far more often in generated text than in prose a person wrote. Try the plain one."
+    seed: { transform: insert_slop_word, count: 1 }
+
+  - id: banned_words
+    kind: regex
+    # Same matcher as slop_vocab, over your own list. Empty by default, so it
+    # flags nothing until you add a word. Put the words your house never uses.
+    builtin: banned_words
+    words: []
+    message: "A word from your own banned list."
+    seed: { transform: insert_banned_word, count: 1 }
+
+  # --- judgment rules: answered by the model, one probability each ---------
+
+  - id: not_x_but_y
+    kind: judgment
+    what: |
+      A sentence, or a pair of sentences, whose whole job is to swap one
+      label for another. The writer denies one description of a thing and
+      puts a replacement in its place, and the swap is the point. The shape
+      is "it is not X, it is Y", or "this was never X. It was Y.", or "not a
+      X but a Y", or the same swap in reverse, "it is Y, not X". Both halves
+      describe the same subject, and the sentence carries nothing else.
+    not_for: |
+      A plain negation with no replacement offered ("This is not a
+      guarantee.").
+      A denial followed by a separate fact rather than a relabelling ("It is
+      not finished, and two tests still fail.").
+      A comparison with "rather than" or "instead of" inside a sentence that
+      carries other information ("We ship on Tuesdays rather than Fridays so
+      the weekend stays quiet.").
+      A factual correction of a claim someone else made, with the evidence
+      for it.
+      Two sentences that each carry their own information, where the second
+      does not exist only to replace the first.
+    examples:
+      - "It is not a shortcut, it is the whole method."
+      - "The problem was never the tooling. It was the calendar."
+      - "That is a habit, not a strategy."
+    criteria:
+      true: "At least one sentence, or sentence pair, denies one description of a thing and substitutes another for effect, in either order, with the swap as its point."
+      false: "Every negation either stands alone, is followed by new information rather than a relabelling, or corrects a claim with evidence."
+    message: "Not X, it's Y. The reframe is doing the arguing for you."
+    seed:
+      splice:
+        - "Calling Tuesday an outage would be generous. Somebody chose to turn it off."
+        - "Whatever the Friday meeting has become, a review is no longer the word for it. A rehearsal is."
+        - "Nobody has a scheduling problem here. What everyone has is a saying-no problem."
+      position: any
+
+  - id: tricolon
+    kind: judgment
+    what: |
+      Judge sentence by sentence. One sentence is enough to trip this rule,
+      wherever it sits in the paragraph and however informative the rest of
+      the paragraph is.
+      A decorative rule of three. Three short parallel words or phrases in a
+      row whose job is the beat rather than the information, such as three
+      adjectives, three verbs, or a three-word fragment used as a flourish.
+    not_for: |
+      A list of three distinct things the reader needs, such as three named
+      steps, three options, or three findings, each adding something the
+      others do not.
+      A list of two, or of four or more.
+      Three items quoted from somewhere else and discussed.
+    examples:
+      - "The fix is cleaner, faster, and truer."
+      - "Plan, build, repeat."
+      - "It was slow, it was loud, it was late."
+    criteria:
+      true: "At least one sentence, anywhere in the paragraph, carries a three-item run whose purpose is rhythm rather than information. One such sentence is enough, and the informative lists around it do not excuse it."
+      false: "Every three-item run in the paragraph is an informative list, or there are none."
+    message: "Three items for the cadence. Keep the one that carries the weight."
+    seed:
+      splice:
+        - "The result was quieter, lighter, and kinder."
+        - "Read, mark, move on."
+        - "Small budget, small room, small hopes."
+      position: any
+
+  - id: stacked_hedging
+    kind: judgment
+    what: |
+      One claim carrying two or more softeners at once, so that the sentence
+      never commits. "Might perhaps", "arguably, in some sense, probably",
+      "it seems to me that it could possibly".
+    not_for: |
+      A single hedge used once and on purpose, such as one "I think" or one
+      "roughly".
+      Honest uncertainty stated plainly ("The cause is unknown.").
+      A precise qualifier on a number ("about 40 percent", "at least three").
+      A passage that quotes or discusses hedging as its subject.
+    examples:
+      - "It might perhaps, in some sense, arguably be the main reason."
+      - "It seems to me that this could possibly tend to help a little."
+    criteria:
+      true: "At least one claim carries two or more piled-up softeners in the same sentence."
+      false: "Hedges are absent, or each claim carries at most one, used deliberately."
+    message: "Three hedges on one claim. Either say it or leave it out."
+    seed:
+      splice:
+        - "That could perhaps, in a way, maybe explain part of the gap."
+        - "I would think it might arguably, more or less, be worth a try."
+      position: any
+
+  - id: rhetorical_opener
+    kind: judgment
+    what: |
+      The paragraph's first sentence is a question asked for effect, to set
+      up what the writer is about to say, rather than to get an answer from
+      anyone.
+    not_for: |
+      A question that appears later in the paragraph.
+      A first sentence that is a statement, even a curious-sounding one.
+      An opening question addressed to a named person, or one the paragraph
+      reports someone else asking.
+    examples:
+      - "What if the checker itself is the thing that is wrong?"
+      - "But what happens when nobody reads the draft twice?"
+    criteria:
+      true: "The first sentence is a question asked for rhetorical effect."
+      false: "The first sentence is a statement, or the only questions come later, or the opening question is one someone actually asked."
+    # About the paragraph's opening, so it is not asked of the later pieces of a
+    # paragraph that was too long to send in one request.
+    sentence: first
+    message: "Opens on a question nobody asked. Start with the answer."
+    seed:
+      splice:
+        - "Who decided that a draft has to be finished before anyone reads it?"
+        - "And what is a rule worth if nobody remembers it on a Friday?"
+      position: start
+
+  - id: restating_closer
+    kind: judgment
+    what: |
+      The paragraph's final sentence only says again what the paragraph has
+      already said. It often opens with "in short", "in other words", "to sum
+      up", or "put simply", or refers back to "everything above".
+    not_for: |
+      A closing sentence that lands something new: a consequence, a named
+      mechanism, a number, a source, a condition, or a turn the paragraph had
+      not yet made, even when it is short or emphatic.
+      A paragraph of a single sentence.
+    examples:
+      - "In short, everything above is what we said."
+      - "To sum up, that is how the process works, as this paragraph has described."
+    criteria:
+      true: "The last sentence is a summary of the paragraph and adds nothing."
+      false: "The last sentence advances the argument or adds information, or the paragraph is a single sentence."
+    # About the paragraph's ending, so it is not asked of the earlier pieces of
+    # a paragraph that was too long to send in one request.
+    sentence: last
+    message: "The last sentence says the paragraph again. End one sentence earlier."
+    seed:
+      splice:
+        - "In other words, this paragraph has just made the point it set out to make."
+        - "Put simply, that is the whole of what was said above."
+      position: end
+
+  - id: self_undercutting
+    kind: judgment
+    tags: [marketing]
+    # A page meant to sell has to make its offer. A page meant to document
+    # has to state its limits, which is why this rule sits out by default.
+    what: |
+      Judge sentence by sentence. One sentence is enough to trip this rule,
+      however confident and specific the rest of the paragraph is.
+      A line that talks the reader out of the thing being offered. It says
+      the thing may not work, that the writer does not claim it works, that
+      there is no guarantee, or that it may not be for them. On a page meant
+      to sell, describe, or recommend, this weakens the offer and tells the
+      reader nothing they can use.
+    not_for: |
+      A precise statement of scope or a known limit ("It checks Markdown and
+      plain text, not PDFs.").
+      A measured result with its range ("Recall was 0.85 to 0.94 across three
+      runs.").
+      A specific past failure that the paragraph then explains.
+      An honest statement of what a piece of work does and does not cover
+      ("We do not claim this catches every case."), which is a scope note and
+      belongs in good documentation.
+      Disputing a claim someone else made.
+    examples:
+      - "This may not work for you, and we are not saying it will."
+      - "There is no guarantee any of this helps."
+    criteria:
+      true: "At least one sentence, anywhere in the paragraph, tells the reader not to count on the thing working, with no specific limit named. One such sentence is enough."
+      false: "Limits are named precisely, or no self-undercutting line appears."
+    message: "The copy is talking the reader out of it. Name the limit and drop the shrug."
+    seed:
+      splice:
+        - "It might do nothing at all for your team, and that is fine."
+        - "Plenty of people try this once and shrug, so keep your expectations low."
+        - "Results vary and we make no promises about yours."
+      position: any
+
+  - id: first_x_that
+    kind: judgment
+    tags: [marketing]
+    # A first claim is a marketing habit; a technical note may need the word
+    # for a sequence, which `not_for` covers.
+    what: |
+      Judge sentence by sentence. One sentence is enough to trip this rule,
+      wherever it sits in the paragraph.
+      A claim that the writer, or their team, product, or company, is the
+      first to do something. "The first tool that", "the first team to", "we
+      were the first to".
+    not_for: |
+      A first-ever claim about someone else, with a source.
+      "First" as a position in a sequence, such as the first step, the first
+      draft, or the first of two things.
+      A claim of being different, deeper, or better that does not use
+      "first".
+      A dated historical fact with a source ("The first edition appeared in
+      1918.").
+    examples:
+      - "This is the first tool that reads a draft the way an editor does."
+      - "We are the first team to run this check on every save."
+    criteria:
+      true: "At least one sentence, anywhere in the paragraph, claims the writer, or their product, team, or company, was first to do something. One such sentence is enough."
+      false: "No first claim is made about the writer or their own work."
+    message: '"The first X that." Someone will check.'
+    seed:
+      splice:
+        - "Nobody had put a probability beside every flag until our team did."
+        - "Ours is the earliest field notebook a crew could carry down a mine with no signal."
+      position: any
+
+  - id: naked_cost_figure
+    kind: judgment
+    tags: [marketing]
+    # This is a house rule about marketing surfaces, not about writing.
+    what: |
+      The paragraph states what something cost to make or run, in model
+      spend, compute, hours, or "it cost us", without naming the price a
+      customer pays and the alternative beside it.
+    not_for: |
+      A cost quoted together with the customer price and a comparison ("It
+      costs us about a dollar to run, you pay $49, and an agency charges
+      $2,000.").
+      A cost in a technical appendix or a bench table that names what it
+      measures.
+      A customer price quoted as a price.
+      A revenue figure, a valuation, or a figure about someone else's
+      business.
+    examples:
+      - "This report cost us $1.30 in API calls."
+      - "Each run costs us about nine cents in model spend."
+    criteria:
+      true: "A production or running cost appears with no customer price and no alternative next to it."
+      false: "Every cost figure is accompanied by the price and an alternative, or no cost figure appears."
+    message: "A cost with no price next to it. Sniffs like a discount nobody asked for."
+    seed:
+      splice:
+        - "It cost us about four dollars of compute to make this."
+        - "Rendering the page costs us a little under two cents each time."
+      position: any
+
+  - id: jobs_claim
+    kind: judgment
+    tags: [marketing]
+    # A positioning rule for pages that sell. A report about employment
+    # says what it measures and is not covered by this.
+    what: |
+      Judge sentence by sentence. One sentence is enough to trip this rule,
+      wherever it sits in the paragraph and however carefully the rest of the
+      paragraph describes the division of work.
+      A claim about employment. That the thing creates jobs, that it
+      replaces the people who used to do the work, or that it would never
+      replace a person.
+    not_for: |
+      Describing what the tool does and what a person still decides, with no
+      claim about employment.
+      Naming who reviews or signs off.
+      Replacing a thing rather than a person ("It replaces the spreadsheet.").
+      A sourced statistic about employment, in a report whose subject is
+      employment.
+    examples:
+      - "We would never replace a person."
+      - "This replaces the people who used to do the checking by hand."
+      - "Four seats means we create jobs rather than removing them."
+    criteria:
+      true: "At least one sentence, anywhere in the paragraph, makes an employment claim of one of those kinds. One such sentence is enough, whatever the surrounding sentences say about what people still do."
+      false: "No sentence makes an employment claim; the paragraph describes the division of work and stops there."
+    message: "A claim about jobs. Say what the tool does and what people keep, then stop."
+    seed:
+      splice:
+        - "Nobody loses their job to this, and if anything it adds a few."
+        - "The tool now does the work that three people used to do."
+        - "We have promised the team that no role will ever be automated away."
+      position: any
+
+  - id: pullquote_fragment
+    kind: judgment
+    tags: [marketing]
+    # A typographic convention of published copy, not of ordinary prose.
+    what: |
+      Judge line by line. One display line is enough to trip this rule,
+      wherever it sits in the paragraph and however complete the other lines
+      around it are.
+      A pull quote, meaning a short line set apart in quotation marks or
+      italics as a display line, whose text is a fragment with no subject and
+      verb of its own.
+    not_for: |
+      A quotation from a named source running inline as part of a sentence,
+      however short.
+      A complete sentence used as a pull quote.
+      A heading or a title.
+      A paragraph with no pull quote at all.
+    examples:
+      - "*\"Built to be argued with.\"*"
+      - "*\"The part nobody reads twice.\"*"
+    criteria:
+      true: "At least one display line, anywhere in the paragraph, is set apart as a pull quote and is a fragment rather than a complete sentence. One such line is enough, and the complete ones beside it do not excuse it."
+      false: "There is no pull quote, or every pull quote in the paragraph is a complete sentence."
+    message: "A pull quote with no verb. Quote a whole sentence."
+    seed:
+      splice:
+        - "*\"Cheaper by the paragraph.\"*"
+        - "*\"A pen on a string.\"*"
+      position: any