markovka 0.1.0__py3-none-any.whl

markovka/chat.py

@@ -0,0 +1,26 @@
+import json
+
+
+def fix_encoding(text):
+    raw_bytes = text
+    fixed = raw_bytes.encode("latin1")
+    result = fixed.decode("utf-8")
+    return result
+
+
+def load_chat_messages(path, sender):
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    messages = list()
+    for message in data["messages"]:
+        if "sender_name" in message:
+            name = fix_encoding(message["sender_name"])
+            text = message.get("content")
+            if text != None:
+                text = fix_encoding(text)
+        else:
+            name = message.get("senderName")
+            text = message.get("text")
+        if name == sender and text:
+            messages.append(text)
+    return messages

markovka/experiments.py

@@ -0,0 +1,33 @@
+from markovka.markov import (
+    build_ngram_counts,
+    generate_text_backoff,
+    generate_text_ngram,
+    get_tables,
+)
+from markovka.stats import copy_score, share_single_options
+
+
+def print_result(words, max_n):
+    result = get_tables(words, max_n)
+    print(len(result[1]))
+    print(len(result[2]))
+    print(len(result[3]))
+
+
+def tune_min_options(tables, settings, runs, chunk_set):
+    for opt in settings:
+        totals = dict()
+        score_sum = 0
+        for _ in range(runs):
+            text, counts = generate_text_backoff(tables, 3, 30, min_options=opt)
+            score_sum += copy_score(text, chunk_set, 5)
+            for size in counts:
+                if size not in totals:
+                    totals[size] = 0
+                totals[size] += counts[size]
+        average = score_sum / runs
+
+        print(f"--- {opt} ---")
+        print(totals)
+        print(average)
+        print(text)

markovka/main.py

@@ -0,0 +1,184 @@
+import argparse
+
+from markovka.chat import load_chat_messages
+from markovka.experiments import print_result, tune_min_options
+from markovka.markov import (
+    generate_text_backoff,
+    get_tables,
+    get_tables_from_texts,
+    normalize,
+)
+from markovka.stats import book_chunks, copy_score
+from markovka.storage import load_tables, save_tables
+from markovka.utils import clean_text, get_book_text, remove_header
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("book", nargs="*", help="file name of the book in books/")
+    parser.add_argument(
+        "--length", type=int, default=30, help="How many words to generate"
+    )
+    parser.add_argument(
+        "--max-n",
+        type=int,
+        default=3,
+        help="Biggest memory size (order of markov chain). 3 is default and best for books. try 2 for chats.",
+    )
+    parser.add_argument(
+        "--min-options",
+        type=int,
+        default=2,
+        help="best leave at default, copies the source text at 1 and is more gibberish at more than 2",
+    )
+    parser.add_argument(
+        "--count", type=int, default=1, help="Amount of texts to generate"
+    )
+    parser.add_argument(
+        "--experiment",
+        action="store_true",
+        help="run the copy-score tests and the min_options experiment",
+    )
+    parser.add_argument(
+        "--override-sentences",
+        action="store_true",
+        help="don't force texts to start with a capital and end with . ! or ?",
+    )
+    parser.add_argument(
+        "--score",
+        action="store_true",
+        help="Show how much of each text is copied from the source material",
+    )
+    parser.add_argument("--save", help="Save the tables to this JSON file")
+    parser.add_argument(
+        "--load", help="load tables from this JSON file instead of reading books"
+    )
+    parser.add_argument(
+        "--chat", nargs="+", help="one or more chat export files in chats/"
+    )
+    parser.add_argument(
+        "--sender", help="whose messages to learn from (exact name as in the export)"
+    )
+    parser.add_argument(
+        "--score-k",
+        type=int,
+        default=5,
+        help="How many words in a row count as copied, try 3 for chats",
+    )
+    parser.add_argument("--talk", action="store_true", help="talk with the generator!")
+    args = parser.parse_args()
+    if not args.book and args.load is None and args.chat is None:
+        parser.error("Give at least one book, --chat FILES or --load FILE")
+    if args.chat and args.sender is None:
+        parser.error("--chat needs --sender NAME")
+    if args.sender and args.chat is None:
+        parser.error("Did you mean --chat?")
+
+    if args.load and (args.book or args.chat):
+        parser.error("Use either books or --load, not both")
+    if args.load and (args.score or args.experiment):
+        parser.error("--score and --experiment need the book text not just tables")
+    if args.load:
+        try:
+            tables = load_tables(args.load)
+        except FileNotFoundError:
+            parser.error(f"file {args.load} not found")
+    else:
+        book_words = list()
+        texts = list()
+        for book_name in args.book:
+            book_path = f"books/{book_name}"
+            try:
+                book_text = get_book_text(book_path)
+            except FileNotFoundError:
+                parser.error(f"Book {book_name} not found")
+            no_header = remove_header(book_text)
+            cleaned = clean_text(no_header)
+            book_words.extend(cleaned.split())
+            texts.append(cleaned.split())
+        if args.chat:
+            for chat_name in args.chat:
+                chat_path = f"chats/{chat_name}"
+                chat_messages = load_chat_messages(chat_path, args.sender)
+                for message in chat_messages:
+                    book_words.extend(message.split())
+                    texts.append(message.split())
+        tables = get_tables_from_texts(texts, args.max_n)
+    max_n = len(tables)
+    if args.save:
+        save_tables(tables, args.save)
+
+    if args.score or args.experiment:
+        book_chunk_set = book_chunks(book_words, args.score_k)
+        score_sum = 0
+    if args.talk:
+        known = set()
+        for key in tables[max_n]:
+            known.add(normalize(key[0]))
+        while True:
+            try:
+                user = input("you: ")
+            except (KeyboardInterrupt, EOFError):
+                print()
+                break
+            if user == "/quit":
+                break
+            candidates = list()
+            for word in user.split():
+                if normalize(word) in known:
+                    candidates.append(word)
+            if not candidates:
+                start = None
+            else:
+                start = max(candidates, key=len)
+            text, counts = generate_text_backoff(
+                tables,
+                max_n,
+                args.length,
+                min_options=args.min_options,
+                sentences=not args.override_sentences,
+                start_word=start,
+            )
+            print(f"bot: {text}")
+    else:
+        for _ in range(args.count):
+            text, counts = generate_text_backoff(
+                tables,
+                max_n,
+                args.length,
+                min_options=args.min_options,
+                sentences=not args.override_sentences,
+            )
+            print(text)
+            if (
+                not args.override_sentences
+                and text[-1].endswith((".", "!", "?")) == False
+            ):
+                print(
+                    "Error! Couldn't generate text ending with a sentence ending mark. Try again, or run with --override-sentences"
+                )
+                print("-------------")
+            if args.score:
+                score = copy_score(text, book_chunk_set, args.score_k)
+                score_sum += score
+                percent = round(score * 100)
+                print(f"copied: {percent}%")
+        if args.score:
+            print("average copied")
+            print(round(score_sum / args.count * 100))
+
+        if args.experiment:
+            print(len(book_chunk_set))
+            print(len(book_words))
+            test_text = """On an exceptionally hot evening early in July a young man came out of
+            the garret in which he lodged in S. Place and walked slowly, as though
+            in hesitation,"""
+            print("--- SANITY TEST ---")
+            print(copy_score(test_text, book_chunk_set, 5))
+            print("--- ACTUAL TEST --- ")
+            print(copy_score(text, book_chunk_set, 5))
+            tune_min_options(tables, range(1, 6), 30, book_chunk_set)
+
+
+if __name__ == "__main__":
+    main()

markovka/markov.py

@@ -0,0 +1,125 @@
+import random
+
+
+#  build the table
+def build_ngram_counts(words, n, counts=None):
+    if counts == None:
+        ngram_counts = {}
+    else:
+        ngram_counts = counts
+    for i in range(0, len(words) - n):
+        current_key = tuple(words[i : i + n])
+        next_word = words[i + n]
+        if current_key not in ngram_counts:
+            ngram_counts[current_key] = dict()
+        if next_word not in ngram_counts[current_key]:
+            ngram_counts[current_key][next_word] = 0
+
+        ngram_counts[current_key][next_word] += 1
+    return ngram_counts
+
+
+# generating text
+def generate_text_ngram(table, length):
+    all_keys = list(table.keys())
+    current = random.choice(all_keys)
+    generated = list(current)
+    n = len(current)
+
+    for i in range(length):
+        if current not in table:
+            current = random.choice(all_keys)
+        options = list(table[current].keys())
+        weights = list(table[current].values())
+        next_word = random.choices(options, weights=weights)[0]
+        generated.append(next_word)
+        current = tuple(generated[-n:])
+    result_seperator = " "
+    result = result_seperator.join(generated)
+    return result
+
+
+def generate_text_backoff(
+    tables, max_n, length, min_options=2, sentences=True, start_word=None
+):
+    all_keys = list(tables[max_n].keys())
+    start_keys = list()
+    for key in all_keys:
+        if key[0][0].isupper():
+            start_keys.append(key)
+    if len(start_keys) == 0:
+        start_keys = all_keys
+    if sentences == False:
+        start_keys = all_keys
+    if start_word != None:
+        matching = list()
+        for key in all_keys:
+            if key[0] == start_word:
+                matching.append(key)
+        if len(matching) > 0:
+            start_keys = matching
+    current = random.choice(start_keys)
+    generated = list(current)
+    sizes_used = dict()
+    if sentences == True:
+        max_rounds = length * 3
+    else:
+        max_rounds = length
+    for i in range(max_rounds):
+        picked = None
+        for n in range(max_n, 0, -1):
+            key = tuple(generated[-n:])
+            table = tables[n]
+            if key in table and (len(table[key]) >= min_options or n == 1):
+                options = list(table[key].keys())
+                weights = list(table[key].values())
+                picked = random.choices(options, weights=weights)[0]
+                if n not in sizes_used:
+                    sizes_used[n] = 0
+                sizes_used[n] += 1
+                break
+        if picked is None:
+            single_word_table = tables[1]
+            keys = single_word_table.keys()
+            key_list = list(keys)
+            random_key = random.choice(key_list)
+            picked = random_key[0]
+            if "jump" not in sizes_used:
+                sizes_used["jump"] = 0
+            sizes_used["jump"] += 1
+        generated.append(picked)
+        if sentences and i + 1 >= length and generated[-1].endswith((".", "!", "?")):
+            break
+    result_seperator = " "
+    result = result_seperator.join(generated)
+    return result, sizes_used
+
+
+def get_tables(words, max_n):
+    tables = dict()
+    for n in range(1, max_n + 1):
+        tables[n] = build_ngram_counts(words, n)
+    return tables
+
+
+def get_tables_from_texts(texts, max_n):
+    tables = dict()
+    for n in range(1, max_n + 1):
+        tables[n] = dict()
+    for words in texts:
+        for n in range(1, max_n + 1):
+            build_ngram_counts(words, n, tables[n])
+    return tables
+
+
+def print_result(words, max_n):
+    result = get_tables(words, max_n)
+    print(len(result[1]))
+    print(len(result[2]))
+    print(len(result[3]))
+
+
+def normalize(word):
+    lowercase = word.lower()
+    stripped = lowercase.strip(".,!?")
+    return stripped

markovka/stats.py

@@ -0,0 +1,30 @@
+def share_single_options(table):
+    single_count = 0
+    for followers in table.values():
+        if len(followers) == 1:
+            single_count += 1
+    return single_count / len(table)
+
+
+def book_chunks(words, k):
+    chunks = set()
+    last_start = len(words) - k
+    for i in range(last_start + 1):
+        chunk = tuple(words[i : i + k])
+        chunks.add(chunk)
+    return chunks
+
+
+def copy_score(text, chunk_set, k):
+    text_words = text.split()
+    all_chunks = 0
+    found_chunk = 0
+    last_start = len(text_words) - k
+    for i in range(last_start + 1):
+        chunk = tuple(text_words[i : i + k])
+        all_chunks += 1
+        if chunk in chunk_set:
+            found_chunk += 1
+    if all_chunks == 0:
+        return 0.0
+    return found_chunk / all_chunks

markovka/storage.py

@@ -0,0 +1,30 @@
+import json
+
+
+def save_tables(tables, path):
+    json_ready = dict()
+    key_seperator = " "
+    for n in tables:
+        table = tables[n]
+        string_table = dict()
+        for key in table:
+            string_key = key_seperator.join(key)
+            string_table[string_key] = table[key]
+        json_ready[n] = string_table
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(json_ready, f, ensure_ascii=False)
+
+
+def load_tables(path):
+    with open(path, encoding="utf-8") as f:
+        loaded = json.load(f)
+    tables = dict()
+    for n_string in loaded:
+        n = int(n_string)
+        string_table = loaded[n_string]
+        table = dict()
+        for string_key in string_table:
+            key = tuple(string_key.split())
+            table[key] = string_table[string_key]
+        tables[n] = table
+    return tables

markovka/utils.py

@@ -0,0 +1,30 @@
+import random
+
+
+def remove_header(text):
+    try:
+        start_book = text.index("START OF THE PROJECT GUTENBERG EBOOK")
+        marker_end = text.index("***", start_book)
+        end_book = text.index("*** END OF")
+        no_header = text[marker_end + 3 : end_book]
+        return no_header
+    except ValueError:
+        return text
+
+
+def get_book_text(path):
+    with open(path) as f:
+        return f.read()
+
+
+def clean_text(text):
+    lines = text.splitlines()
+    kept = list()
+    for line in lines:
+        stripped = line.strip()
+        if stripped.startswith("CHAPTER") or stripped.startswith("PART"):
+            continue
+        kept.append(stripped)
+    result = "\n".join(kept)
+    no_april_underscores = result.replace("_", "")
+    return no_april_underscores

markovka-0.1.0.dist-info/METADATA

@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: markovka
+Version: 0.1.0
+Summary: Markov chain text generator with option of importing messenger chats and books as source material.
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Markov chain generator
+
+[![forthebadge](https://forthebadge.com/badges/0-percent-optimized.svg)](https://forthebadge.com)
+
+A command-line Markov chain text generator, that learnsf rom a book and writes new text in its style.
+
+> The shy, unknown street? I shall be simply a puppy, and a pack of perspiration stood open. Peasants stood before his
+
+1. it reads a book and builds tables of which words follow which 1, 2 or 3-wordsequences
+2. generates text by picking the next word randomly, weighted by how often it appeared
+3. uses longest memory when there's a real choice and uses backoff to fall back to shorter memory when there's only one option to avoid plagiarizing the source text
+4. an originality checker that measures how much of the output is copied word for word
+
+## Requirements
+
+- Python 3 (no exterlan packages; only the standard library)
+
+## Usage
+
+Put a plain-text book from [Project Gutenberg](https://www.gutenberg.org) in the `books/` folder, then:
+
+```bash
+python3 main.py <book_file> [options]
+```
+
+| Option                 | Default | Description                                                                                    |
+| ---------------------- | ------- | ---------------------------------------------------------------------------------------------- |
+| `--length`             | 30      | How many words to generate                                                                     |
+| `--max-n`              | 3       | Biggest memory size (n-gram order)                                                             |
+| `--min-options`        | 2       | Backoff threshold: minimum number of possible next words before falling back to shorter memory |
+| `--count`              | 1       | How many texts to generate                                                                     |
+| `--experiment`         | off     | Run the copy-score tests and the min_options experiment                                        |
+| `--override-sentences` | off     | Don't force texts to start with a capital letter and end with . ! or ?                         |
+| `--score`              | off     | Show how much of each text is copied word for word from the source material                    |
+
+### Examples
+
+```bash
+python3 main.py crimeandpunishment.txt
+```
+
+```bash
+wretched little beast was still life and strength that moment without this three roubles; I may have had nothing to do with you. I do no exactly.” “Do tell you something more essential
+```
+
+```bash
+python3 main.py crimeandpunishment.txt --count 3 --length 50
+```
+
+```bash
+spoke rather loud noise and animated conversation with a sharp little nose off with my sister--the day before yesterday.’ ‘And where did you go, will all keep him on the score of the old, dirty, by this. Then he remembered quite different. “Yes, Mr. Captain, and I am an ass; forgive myself for
+why he went,” Raskolnikov by way will sing ‘My Village’ only, for the smart lady who was so unfortunate girl were asking your _fiancé_ in and weighed upon him. Here is some water. The feeling that it was like the latch on an empty tea-glass on a staircase, he mounted to tell you
+scripture, too,” she was growing up, moved a night cap with new mourning had done _that_, could never bear that hardly audibly and God has smitten you, has no skill in such attire. Suddenly he withdrew his short-sighted eyes. You might well not such a fool I was afraid of!” Porfiry Petrovitch, receiving
+```
+
+````bash
+python3 main.py crimeandpunishment.txt --experiment```
+````
+
+```bash
+away in her will, ha-ha!). And she would not overlook anything. For the family who wanted a handkerchief, had turned the pocket is honourable. The vast quantities he said that he would not
+202521
+203612
+--- SANITY TEST ---
+1.0
+--- ACTUAL TEST ---
+0.034482758620689655
+--- 1 ---
+{3: 900}
+0.9045977011494255
+corner was disproportionately obtuse. There was scarcely any furniture in the big room: in the corner and several chairs--all government furniture, of polished yellow wood. In the further wall there was a time....
+--- 2 ---
+{1: 380, 2: 361, 3: 159}
+0.08275862068965514
+They’ve taken them food... I spoke about her mistrustfully, but that probably from his face. “And here... however queer, almost sullen insistence. “You’re right: she put them all so fond of Sonia? Sonia
+--- 3 ---
+{1: 509, 2: 313, 3: 78}
+0.051724137931034475
+this hesitating question, Sonia....” He stopped for me...” He rang the glass, put a stop here was the gate. She read loudly, cold with her own account, hasn’t said suddenly. “He is himself,”
+--- 4 ---
+{1: 555, 2: 288, 3: 57}
+0.035632183908045977
+and went in. He looked carefully and exactly what the doctor says!” Katerina Ivanovna, pale, thin spare the bridge. He became at parting for a poor student, or hunchback, I forgot! A plain,
+--- 5 ---
+{1: 599, 2: 266, 3: 35}
+0.028735632183908046
+“but what help the habit he said this with some other general’s.... Only just getting home in my opinions.... You know my opinion, and describing a change it, the business, and then to
+```
+
+```bash
+ python3 main.py crimeandpunishment.txt --score
+```
+
+```bash
+After brief hesitation at the police office? Where’s the greatest suspicion of any folly.” They simply looked rather irregular for that moment when he stood with the axe in the noose, he tried to hang himself, and that I did not ask forgiveness, but he was in a fever of dust of the street.
+copied: 18%
+```
+
+## Results
+
+Tested on _Crime and Punishment_ (about 200k words).
+
+**Memory size vs copying** The longer the memory the more often there is one possible next word so the generator just copies the book:
+
+| Memory  | Keys with only one possible next word |
+| ------- | ------------------------------------- |
+| 1 word  | 59%                                   |
+| 2 words | 82%                                   |
+| 3 words | 94%                                   |
+
+**Backoff vs copying** Copy score = share of 5-0word chunks in the output that appear word for word in the book (average of 30 texts per setting):
+
+| min_options    | 3-word memory used | Copy score |
+| -------------- | ------------------ | ---------- |
+| 1 (no backoff) | 100%               | 91%        |
+| 2              | 16%                | 7.5%       |
+| 3              | 10%                | 6.1%       |
+| 4              | 5%                 | 2.6%       |
+| 5              | 6%                 | 2.6%       |
+
+backoff cuts the copying form 91% to about 7.5. above 2 copying barely drops and the text gets more mashed up. thats why 2 is the default
+
+## Running tests
+
+- Create and activate a .venv
+- install pytest
+- run pytest
+  Thet ests cover tables, copy score, header removal, cleaning and sentence boundiaries
+
+## Known Limitations
+
+- Sentence ends inside quotes like "Good-bye!" arent recognised as sentence ends
+- a sentence start is detected by a capital letter so names in the middle of a sentence can also be piocked as starts
+- in rare cases no sentence end is found within the limit and the program prints a warning
+- for text without capitals or punctuation like chats use --override-sentences
+
+## Roadmap
+
+- Generating text from other data sources like JSON
+- Cleaning the source text depending on the type
+- Saving tables to a file
+
+## Credits
+
+- Book source: Project gutenberg, shoutout :)

markovka-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+markovka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+markovka/chat.py,sha256=4_N1y2outTKD_nBU7AKbFCpjvibFExV5ivRMzuejT5A,716
+markovka/experiments.py,sha256=HY8d7sf2KbkJqboFAmFjYtEldghM6gkAg5HVfX47XJs,917
+markovka/main.py,sha256=0WO9taaFg5sd5SUKHHVcyj8AsL2PBm-WbydxpMfl6-0,6721
+markovka/markov.py,sha256=8435-xdR3F_7J8fioQAfYFwxEAYaTIW9Ft-AQ8bXjZg,3770
+markovka/stats.py,sha256=DWsaQBfoPZqEdgGVOdJfS4AQcPSi6b8yWjDVIzp9Kc0,778
+markovka/storage.py,sha256=qde38LiOxMNoEgaN1NV6Omq2waHl15JrSiQQHDX4R2M,848
+markovka/utils.py,sha256=Jg-hOBBGt407NY0ZFL1RZjgmLhXK9NlDwKpU70EITr4,778
+markovka-0.1.0.dist-info/licenses/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
+markovka-0.1.0.dist-info/METADATA,sha256=d67o7vcJGKvScxDilr3LGiT4wpTR-NK8VGg-iHq0W2w,7626
+markovka-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+markovka-0.1.0.dist-info/entry_points.txt,sha256=GBR5jWAcYF-xI2uxWvHPHEPF61KqGIA-fA_zkwnTU3A,48
+markovka-0.1.0.dist-info/top_level.txt,sha256=zqLraDoQWUOQXZl9nJ5tAQm60-EWRwSzctgELrtUXUU,9
+markovka-0.1.0.dist-info/RECORD,,

markovka-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

markovka-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+markovka = markovka.main:main

markovka-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>

markovka-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+markovka