epstein-files 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

epstein_files/util/doc_cfg.py

@@ -8,7 +8,7 @@ from dateutil.parser import parse
 
 from epstein_files.util.constant.names import *
 from epstein_files.util.constant.strings import *
-from epstein_files.util.data import without_nones
+from epstein_files.util.data import without_falsey
 
 DuplicateType = Literal['earlier', 'quoted', 'redacted', 'same']
 Metadata = dict[str, bool | datetime | int | str | list[str | None] |dict[str, bool | str]]
@@ -121,7 +121,7 @@ class DocCfg:
         elif self.category and self.author is None and self.description is None:
             return self.category
 
-        pieces = without_nones([self.author, self.description])
+        pieces = without_falsey([self.author, self.description])
         return ' '.join(pieces) if pieces else None
 
     def metadata(self) -> Metadata:
@@ -223,6 +223,7 @@ class EmailCfg(CommunicationCfg):
         recipients (list[str | None]): Who received the email
     """
     actual_text: str | None = None  # Override for the Email._actual_text() method for particularly broken emails
+    fwded_text_after: str | None = None  # If set, any text after this is a fwd of an article or similar
     is_fwded_article: bool = False
     recipients: list[str | None] = field(default_factory=list)
 
@@ -234,7 +235,7 @@ class EmailCfg(CommunicationCfg):
     def from_doc_cfg(cls, cfg: DocCfg) -> 'EmailCfg':
         return cls(**asdict(cfg))
 
-    # This is necessary for some dumb reason. @dataclass(repr=False) doesn't cut it
+    # This is necessary because for some dumb reason @dataclass(repr=False) doesn't cut it
     def __repr__(self) -> str:
         return super().__repr__()
 
@@ -245,6 +246,6 @@ class TextCfg(CommunicationCfg):
         super().__post_init__()
         self.category = TEXT_MESSAGE
 
-    # This is necessary for some dumb reason. @dataclass(repr=False) doesn't cut it
+    # This is necessary because for some dumb reason @dataclass(repr=False) doesn't cut it
     def __repr__(self) -> str:
         return super().__repr__()

epstein_files/util/env.py

@@ -6,8 +6,9 @@ from sys import argv
 
 from epstein_files.util.logging import datefinder_logger, env_log_level, logger
 
+COUNT_WORDS_SCRIPT = 'count_words.py'
 DEFAULT_WIDTH = 154
-HTML_SCRIPTS = ['epstein_generate', 'generate_html.py', 'count_words.py']
+HTML_SCRIPTS = ['epstein_generate', 'generate_html.py', COUNT_WORDS_SCRIPT]
 
 
 parser = ArgumentParser(description="Parse epstein OCR docs and generate HTML page.")
@@ -65,7 +66,7 @@ datefinder_logger.setLevel(logger.level)
 
 # Massage args that depend on other args to the appropriate state
 if not (args.json_metadata or args.output_texts or args.output_emails or args.output_other_files):
-    if is_html_script:
+    if is_html_script and current_script != COUNT_WORDS_SCRIPT and not args.make_clean:
         logger.warning(f"No output section chosen; outputting default of texts, selected emails, and other files...")
 
     args.output_texts = True

epstein_files/util/highlighted_group.py

@@ -2,7 +2,6 @@ import re
 from dataclasses import dataclass, field
 
 from rich.highlighter import RegexHighlighter
-from rich.text import Text
 
 from epstein_files.util.constant.names import *
 from epstein_files.util.constant.strings import *
@@ -10,7 +9,7 @@ from epstein_files.util.constant.urls import ARCHIVE_LINK_COLOR
 from epstein_files.util.constants import (EMAILER_ID_REGEXES, EPSTEIN_V_ROTHSTEIN_EDWARDS, HEADER_ABBREVIATIONS,
      OSBORNE_LLP, REPLY_REGEX, SENT_FROM_REGEX, VIRGIN_ISLANDS)
 from epstein_files.util.doc_cfg import *
-from epstein_files.util.data import extract_last_name, listify
+from epstein_files.util.data import extract_last_name, listify, without_falsey
 
 CIVIL_ATTORNEY = 'civil attorney'
 CRIMINAL_DEFENSE_ATTORNEY = 'criminal defense attorney'
@@ -48,7 +47,6 @@ class HighlightedText:
     label: str = ''
     pattern: str = ''
     style: str
-    # Computed fields
     regex: re.Pattern = field(init=False)
     theme_style_name: str = field(init=False)
     _capture_group_label: str = field(init=False)
@@ -76,7 +74,7 @@ class HighlightedNames(HighlightedText):
     Attributes:
         category (str): optional string to use as an override for self.label in some contexts
         emailers (dict[str, str | None]): optional names to construct regexes for (values are descriptions)
-        _pattern (str): complete regex pattern that combines 'pattern' with 'emailers'
+        _pattern (str): regex pattern combining 'pattern' with first & last names of all 'emailers'
     """
     category: str = ''
     emailers: dict[str, str | None] = field(default_factory=dict)
@@ -102,7 +100,7 @@ class HighlightedNames(HighlightedText):
             self.emailers.get(name),
         ]
 
-        info_pieces = [p for p in info_pieces if p is not None]
+        info_pieces = without_falsey(info_pieces)
         return ', '.join(info_pieces) if info_pieces else None
 
     def _emailer_pattern(self, name: str) -> str:
@@ -114,10 +112,10 @@ class HighlightedNames(HighlightedText):
         if name in EMAILER_ID_REGEXES:
             pattern = EMAILER_ID_REGEXES[name].pattern
 
-            # Include regex for last name
-            # TODO: handle word boundary issue for names that end in symbols
-            if SIMPLE_NAME_REGEX.match(last_name) and last_name.lower() not in NAMES_TO_NOT_HIGHLIGHT:
-                pattern += fr"|{last_name}"
+            # Include regex for first and last names
+            for partial_name in [first_name, last_name]:
+                if SIMPLE_NAME_REGEX.match(partial_name) and partial_name.lower() not in NAMES_TO_NOT_HIGHLIGHT:
+                    pattern += fr"|{partial_name}"
 
             return pattern
         elif ' ' not in name:
@@ -163,7 +161,7 @@ HIGHLIGHTED_NAMES = [
             ALIREZA_ITTIHADIEH: 'CEO Freestream Aircraft Limited',
             BARBRO_C_EHNBOM: 'Swedish pharmaceuticals',
             FRED_HADDAD: "co-founder of Heck's in West Virginia",
-            GERALD_BARTON: "Maryland property developer, fan of Trump's Irish golf course",
+            GERALD_BARTON: "Maryland property developer Landmark Land Company, fan of Trump's Irish golf course",
             GORDON_GETTY: 'heir of oil tycoon J. Paul Getty',
             NICHOLAS_RIBIS: 'Hilton CEO, former president of Trump Organization',
             'Philip Kafka': 'president of Prince Concepts (and son of Terry Kafka?)',
@@ -272,7 +270,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='europe',
         style='light_sky_blue3',
-        pattern=r'(Angela )?Merk(el|le)|Austria|(Benjamin\s*)?Harnwell|Berlin|Borge|Brexit(eers?)?|Brit(ain|ish)|Brussels|Cannes|(Caroline|Jack)?\s*Lang(, Caroline)?|Cypr(iot|us)|Davos|ECB|EU|Europe(an)?(\s*Union)?|France|Geneva|Germany?|Gillard|Gree(ce|k)|Ital(ian|y)|Jacques|(Kevin\s*)?Rudd|Le\s*Pen|London|Macron|Melusine|Munich|(Natalia\s*)?Veselnitskaya|(Nicholas\s*)?Sarkozy|Nigel(\s*Farage)?|Oslo|Paris|Polish|(Sebastian )?Kurz|(Vi(c|k)tor\s+)?Orbah?n|Edward Rod Larsen|Strasbourg|Strauss[- ]?Kahn|Swed(en|ish)(?![-\s]+America)|Switzerland|(Tony\s)?Blair|Ukrain(e|ian)|Vienna|(Vitaly\s*)?Churkin|Zug',
+        pattern=r'(Angela )?Merk(el|le)|Austria|(Benjamin\s*)?Harnwell|Berlin|Borge|Boris\s*Johnson|Brexit(eers?)?|Brit(ain|ish)|Brussels|Cannes|(Caroline|Jack)?\s*Lang(, Caroline)?|Cypr(iot|us)|Davos|ECB|England|EU|Europe(an)?(\s*Union)?|Fr(ance|ench)|Geneva|Germany?|Gillard|Gree(ce|k)|Ital(ian|y)|Jacques|(Kevin\s*)?Rudd|Le\s*Pen|London|Macron|Melusine|Munich|(Natalia\s*)?Veselnitskaya|(Nicholas\s*)?Sarkozy|Nigel(\s*Farage)?|Norw(ay|egian)|Oslo|Paris|Polish|(Sebastian )?Kurz|(Vi(c|k)tor\s+)?Orbah?n|Edward Rod Larsen|Strasbourg|Strauss[- ]?Kahn|Swed(en|ish)(?![-\s]+America)|Switzerland|(Tony\s)?Blair|Ukrain(e|ian)|Vienna|(Vitaly\s*)?Churkin|Zug',
         emailers = {
             ANDRZEJ_DUDA: 'former president of Poland',
             MIROSLAV_LAJCAK: 'Russia-friendly Slovakian politician, friend of Steve Bannon',
@@ -306,7 +304,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='finance',
         style='green',
-        pattern=r'Apollo|Ari\s*Glass|(Bernie\s*)?Madoff|Black(rock|stone)|BofA|Boothbay(\sFund\sManagement)?|Chase\s*Bank|Credit\s*Suisse|DB|Deutsche\s*(Asset|Bank)|Electron\s*Capital\s*(Partners)?|Fenner|FRBNY|Goldman(\s*Sachs)|HSBC|Invesco|(Janet\s*)?Yellen|(Jerome\s*)?Powell(?!M\. Cabot)|(Jimmy\s*)?Cayne|JPMC?|j\.?p\.?\s*morgan(\.?com|\s*Chase)?|Madoff|Merrill(\s*Lynch)?|(Michael\s*)?(Cembalest|Milken)|MLPF&S|(money\s+)?launder(s?|ers?|ing)?(\s+money)?|Morgan Stanley|(Peter L. )?Scher|(Ray\s*)?Dalio|Schwartz?man|Serageldin|UBS|us.gio@jpmorgan.com',
+        pattern=r'Apollo|Ari\s*Glass|(Bernie\s*)?Madoff|Black(rock|stone)|BofA|Boothbay(\sFund\sManagement)?|Chase\s*Bank|Credit\s*Suisse|DB|Deutsche\s*(Asset|Bank)|Electron\s*Capital\s*(Partners)?|Fenner|FRBNY|Goldman(\s*Sachs)|HSBC|Invesco|(Janet\s*)?Yellen|(Jerome\s*)?Powell(?!M\. Cabot)|(Jimmy\s*)?Cayne|JPMC?|j\.?p\.?\s*morgan(\.?com|\s*Chase)?|Madoff|Merrill(\s*Lynch)?|(Michael\s*)?(Cembalest|Milken)|Mizrahi\s*Bank|MLPF&S|(money\s+)?launder(s?|ers?|ing)?(\s+money)?|Morgan Stanley|(Peter L. )?Scher|(Ray\s*)?Dalio|Schwartz?man|Serageldin|UBS|us.gio@jpmorgan.com',
         emailers={
             AMANDA_ENS: 'Citigroup',
             DANIEL_SABBA: 'UBS Investment Bank',
@@ -342,7 +340,7 @@ HIGHLIGHTED_NAMES = [
         emailers = {
             ANIL_AMBANI: 'chairman of Reliance Group',
             VINIT_SAHNI: None,
-            ZUBAIR_KHAN: 'Tranchulas CEO, InsightsPod founder',
+            ZUBAIR_KHAN: 'cybersecurity firm Tranchulas CEO, InsightsPod founder, based in Islamabad and Dubai',
         }
     ),
     HighlightedNames(
@@ -391,7 +389,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='law enforcement',
         style='color(24) bold',
-        pattern=r'ag|(Alicia\s*)?Valle|((Bob|Robert)\s*)?Mueller|(Byung\s)?Pak|CFTC|CIA|CIS|CVRA|Dep(artmen)?t\.?\s*of\s*(the\s*)?(Justice|Treasury)|DHS|DOJ|FBI|FCPA|FDIC|Federal\s*Bureau\s*of\s*Investigation|FinCEN|FINRA|FOIA|FTC|IRS|(James\s*)?Comey|(Jennifer\s*Shasky\s*)?Calvery|((Judge|Mark)\s*)?(Carney|Filip)|(Kirk )?Blouin|KYC|NIH|NS(A|C)|OCC|OFAC|(Lann?a\s*)?Belohlavek|(Michael\s*)?Reiter|OGE|Office\s*of\s*Government\s*Ethics|Police Code Enforcement|(Preet\s*)?Bharara|SCOTUS|SD(FL|NY)|Southern\s*District\s*of\s*(Florida|New\s*York)|SEC|Securities\s*and\s*Exchange\s*Commission|State\s*Dep(artmen)?t|Strzok|Supreme\s*Court|Treasury\s*(Dep(artmen)?t|Secretary)|TSA|USAID|(William\s*J\.?\s*)?Zloch',
+        pattern=r'ag|(Alicia\s*)?Valle|attorney|((Bob|Robert)\s*)?Mueller|(Byung\s)?Pak|CFTC|CIA|CIS|CVRA|Dep(artmen)?t\.?\s*of\s*(the\s*)?(Justice|Treasury)|DHS|DOJ|FBI|FCPA|FDIC|Federal\s*Bureau\s*of\s*Investigation|FinCEN|FINRA|FOIA|FTC|IRS|(James\s*)?Comey|(Jennifer\s*Shasky\s*)?Calvery|((Judge|Mark)\s*)?(Carney|Filip)|(Kirk )?Blouin|KYC|NIH|NS(A|C)|OCC|OFAC|(Lann?a\s*)?Belohlavek|lawyer|(Michael\s*)?Reiter|OGE|Office\s*of\s*Government\s*Ethics|Police Code Enforcement|(Preet\s*)?Bharara|SCOTUS|SD(FL|NY)|Southern\s*District\s*of\s*(Florida|New\s*York)|SEC|Securities\s*and\s*Exchange\s*Commission|State\s*Dep(artmen)?t|Strzok|Supreme\s*Court|Treasury\s*(Dep(artmen)?t|Secretary)|TSA|USAID|(William\s*J\.?\s*)?Zloch',
         emailers = {
             ANN_MARIE_VILLAFANA: 'southern district of Florida U.S. Attorney',
             DANNY_FROST: 'Director of Communications at Manhattan DA',
@@ -426,7 +424,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='modeling',
         style='pale_violet_red1',
-        pattern=r'\w+@mc2mm.com|(Nicole\s*)?Junkerman',
+        pattern=r'\w+@mc2mm.com|model(ed|ing)|(Nicole\s*)?Junkerman',
         emailers = {
             'Abi Schwinck': 'MC2 Model Management (?)',
             DANIEL_SIAD: None,
@@ -458,7 +456,8 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='republicans',
         style='bold dark_red',
-        pattern=r'Alberto\sGonzale[sz]|(Alex\s*)?Acosta|(Bill\s*)?Barr|Bill\s*Shine|(Bob\s*)?Corker|(John\s*(R.?\s*)?)Bolton|Broidy|(Chris\s)?Christie|Devin\s*Nunes|(Don\s*)?McGa[hn]n|McMaster|(George\s*)?Nader|GOP|(Brett\s*)?Kavanaugh|Kissinger|Kobach|Koch\s*Brothers|Kolfage|Kudlow|Lewandowski|(Marco\s)?Rubio|(Mark\s*)Meadows|Mattis|(?<!Merwin Dela )Cruz|(Michael\s)?Hayden|((General|Mike)\s*)?(Flynn|Pence)|(Mitt\s*)?Romney|Mnuchin|Nikki|Haley|(Paul\s+)?Manafort|(Peter\s)?Navarro|Pompeo|Reagan|Republican|(?<!Cynthia )(Richard\s*)?Nixon|Sasse|(Rex\s*)?Tillerson',
+        pattern=r'Alberto\sGonzale[sz]|(Alex\s*)?Acosta|(Bill\s*)?Barr|Bill\s*Shine|(Bob\s*)?Corker|(John\s*(R.?\s*)?)Bolton|Broidy|(Chris\s)?Christie|Devin\s*Nunes|(Don\s*)?McGa[hn]n|McMaster|(George\s*)?Nader|GOP|(Brett\s*)?Kavanaugh|Kissinger|Kobach|Koch\s*Brothers|Kolfage|Kudlow|Lewandowski|(Marco\s)?Rubio|(Mark\s*)Meadows|Mattis|(?<!Merwin Dela )Cruz|(Michael\s)?Hayden|((General|Mike)\s*)?(Flynn|Pence)|(Mitt\s*)?Romney|Mnuchin|Nikki|Haley|(Paul\s+)?Manafort|(Peter\s)?Navarro|Pompeo|Reagan|Reince|Priebus|Republican|(?<!Cynthia )(Richard\s*)?Nixon|Sasse|(Rex\s*)?Tillerson',
+        # There's no emails from these people, they're just here to automate the regex creation for both first + last names
         emailers = {
             RUDY_GIULIANI: 'disbarred formed mayor of New York City',
             TULSI_GABBARD: None,
@@ -475,7 +474,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='russia',
         style='red bold',
-        pattern=r'Alfa\s*Bank|Anya\s*Rasulova|Chernobyl|Day\s+One\s+Ventures|(Dmitry\s)?(Kiselyov|(Lana\s*)?Pozhidaeva|Medvedev|Rybolo(o?l?ev|vlev))|Dmitry|FSB|GRU|KGB|Kislyak|Kremlin|Kuznetsova|Lavrov|Lukoil|Moscow|(Oleg\s*)?Deripaska|Oleksandr Vilkul|Rosneft|RT|St.?\s*?Petersburg|Russian?|Sberbank|Soviet(\s*Union)?|USSR|(Vladimir\s*)?(Putin|Yudashkin)|Women\s*Empowerment|Xitrans',
+        pattern=r'Alfa\s*Bank|Anya\s*Rasulova|Chernobyl|Day\s+One\s+Ventures|(Dmitry\s)?(Kiselyov|(Lana\s*)?Pozhidaeva|Medvedev|Rybolo(o?l?ev|vlev))|Dmitry|FSB|GRU|KGB|Kislyak|Kremlin|Kuznetsova|Lavrov|Lukoil|Moscow|(Oleg\s*)?Deripaska|Oleksandr Vilkul|Rosneft|RT|St.?\s*?Petersburg|Russian?|Sberbank|Soviet(\s*Union)?|USSR|Vladimir|(Vladimir\s*)?(Putin|Yudashkin)|Women\s*Empowerment|Xitrans',
         emailers = {
             MASHA_DROKOVA: 'silicon valley VC, former Putin Youth',
             RENATA_BOLOTOVA: 'former aspiring model, now fund manager at New York State Insurance Fund',
@@ -541,7 +540,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label=VIRGIN_ISLANDS,
         style='sea_green1',
-        pattern=r'Bahamas|Caribb?ean|Dominican\s*Republic|(Great|Little)\s*St.?\s*James|Haiti(an)?|(John\s*)deJongh(\s*Jr\.?)|(Kenneth E\. )?Mapp|Palm\s*Beach(?!\s*Post)|PBI|S(ain)?t.?\s*Thomas|USVI|VI|(The\s*)?Virgin\s*Islands(\s*Daily\s*News)?',  # TODO: VI Daily News should be yellow but it's hard bc Daily News xists
+        pattern=r'Antigua|Bahamas|Caribb?ean|Dominican\s*Republic|(Great|Little)\s*St.?\s*James|Haiti(an)?|(John\s*)deJongh(\s*Jr\.?)|(Kenneth E\. )?Mapp|Palm\s*Beach(?!\s*Post)|PBI|S(ain)?t.?\s*Thomas|USVI|VI|(The\s*)?Virgin\s*Islands(\s*Daily\s*News)?',  # TODO: VI Daily News should be yellow but it's hard bc Daily News xists
         emailers = {
             CECILE_DE_JONGH: f'First lady 2007-2015',
             STACEY_PLASKETT: 'non-voting member of Congress',
@@ -561,7 +560,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label=STEVE_BANNON,
         style='color(58)',
-        pattern=r'((Steve|Sean)\s*)?Bannon?',
+        pattern=r'((Steve|Sean)\s*)?Bannon?|(American\s*)?Dharma',
     ),
     HighlightedNames(
         emailers={STEVEN_HOFFENBERG: HEADER_ABBREVIATIONS['Hoffenberg']},
@@ -578,7 +577,18 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(emailers={PRINCE_ANDREW: 'British royal family'}, style='dodger_blue1'),
     HighlightedNames(emailers={SOON_YI_PREVIN: "wife of Woody Allen"}, style='hot_pink'),
     HighlightedNames(emailers={SULTAN_BIN_SULAYEM: 'CEO of DP World, chairman of ports in Dubai'}, style='green1'),
-    HighlightedText(label='unknown', style='cyan', pattern=r'\(unknown\)'),  # HighlightedText bc of word boundary issue
+
+    # HighlightedText not HighlightedNames bc of word boundary issue
+    HighlightedText(
+        label='unknown',
+        style='cyan',
+        pattern=r'\(unknown\)'
+    ),
+    HighlightedText(
+        label='phone_number',
+        style='bright_green',
+        pattern=r"\+?(1?\(?\d{3}\)?[- ]\d{3}[- ]\d{4}|\d{2}[- ]\(?0?\)?\d{2}[- ]\d{4}[- ]\d{4})|[\d+]{10,12}",
+    ),
 ]
 
 # Highlight regexes for things other than names, only used by RegexHighlighter pattern matching
@@ -593,11 +603,6 @@ HIGHLIGHTED_TEXTS = [
         style=f'{ARCHIVE_LINK_COLOR} underline',
         pattern=r"https?:[^\s]+",
     ),
-    HighlightedText(
-        label='phone_number',
-        style='bright_green',
-        pattern=r"\+?(1?\(?\d{3}\)?[- ]\d{3}[- ]\d{4}|\d{2}[- ]\(?0?\)?\d{2}[- ]\d{4}[- ]\d{4})|[\d+]{10,12}",
-    ),
     HighlightedText(
         label='quoted_reply_line',
         style='dim',

epstein_files/util/logging.py

@@ -29,6 +29,7 @@ LOG_THEME[f"{ReprHighlighter.base_style}epstein_filename"] = FILENAME_STYLE
 LOG_LEVEL_ENV_VAR = 'LOG_LEVEL'
 
 
+# Augment the standard log highlighter with 'epstein_filename' matcher
 class LogHighlighter(ReprHighlighter):
     highlights = ReprHighlighter.highlights + [
         *[fr"(?P<{doc_type}>{doc_type})" for doc_type in DOC_TYPE_STYLES.keys()],

epstein_files/util/output.py

@@ -102,7 +102,7 @@ def print_emails(epstein_files: EpsteinFiles) -> int:
         epstein_files.print_email_device_info()
 
     if args.all_emails:
-        _verify_all_emails_printed(epstein_files, already_printed_emails)
+        _verify_all_emails_were_printed(epstein_files, already_printed_emails)
 
     logger.warning(f"Rewrote {len(Email.rewritten_header_ids)} headers of {len(epstein_files.emails)} emails")
     return len(already_printed_emails)
@@ -146,31 +146,30 @@ def print_text_messages(epstein_files: EpsteinFiles) -> None:
 
 
 def write_urls() -> None:
+    """Write _URL style constant variables to a file bash scripts can load as env vars."""
     if args.output_file == 'index.html':
         logger.warning(f"Can't write env vars to '{args.output_file}', writing to '{URLS_ENV}' instead.\n")
         args.output_file = URLS_ENV
 
     url_vars = {
         k: v for k, v in vars(urls).items()
-        if isinstance(v, str) and \
-            k.split('_')[-1] in ['URL'] and \
-            'michelcrypt4d4mus' in v and \
-            'github.com' not in v and \
-            'BASE' not in v
+        if isinstance(v, str) and k.split('_')[-1] in ['URL'] and 'github.io' in v and 'BASE' not in k
     }
 
     with open(args.output_file, 'w') as f:
         for var_name, url in url_vars.items():
             key_value = f"{var_name}='{url}'"
-            console.print(key_value, style='dim')
+
+            if not args.suppress_output:
+                console.print(key_value, style='dim')
+
             f.write(f"{key_value}\n")
 
     console.line()
     logger.warning(f"Wrote {len(url_vars)} URL variables to '{args.output_file}'\n")
 
 
-
-def _verify_all_emails_printed(epstein_files: EpsteinFiles, already_printed_emails: list[Email]) -> None:
+def _verify_all_emails_were_printed(epstein_files: EpsteinFiles, already_printed_emails: list[Email]) -> None:
     """Log warnings if some emails were never printed."""
     email_ids_that_were_printed = set([email.file_id for email in already_printed_emails])
     logger.warning(f"Printed {len(already_printed_emails)} emails of {len(email_ids_that_were_printed)} unique file IDs.")

epstein_files/util/rich.py

@@ -148,7 +148,12 @@ def print_color_key() -> None:
 
 
 def print_header(epstein_files: 'EpsteinFiles') -> None:
-    console.print(f"This site isn't optimized for mobile but if you get past the header it should be readable.", style='dim')
+    not_optimized_msg = f"This site isn't optimized for mobile"
+
+    if not args.all_emails:
+        not_optimized_msg += f" but if you get past the header it should be readable"
+
+    console.print(f"{not_optimized_msg}.\n", style='dim')
     print_page_title(width=TITLE_WIDTH)
     print_other_site_link()
     _print_external_links()

{epstein_files-1.0.2.dist-info → epstein_files-1.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: epstein-files
-Version: 1.0.2
+Version: 1.0.4
 Summary: Tools for working with the Jeffrey Epstein documents released in November 2025.
 Home-page: https://michelcrypt4d4mus.github.io/epstein_text_messages/
 License: GPL-3.0-or-later
@@ -25,6 +25,7 @@ Requires-Dist: requests (>=2.32.5,<3.0.0)
 Requires-Dist: rich (>=14.2.0,<15.0.0)
 Project-URL: Emails, https://michelcrypt4d4mus.github.io/epstein_text_messages/all_emails_epstein_files_nov_2025.html
 Project-URL: Metadata, https://michelcrypt4d4mus.github.io/epstein_text_messages/file_metadata_epstein_files_nov_2025.json
+Project-URL: Repository, https://github.com/michelcrypt4d4mus/epstein_text_messages
 Project-URL: TextMessages, https://michelcrypt4d4mus.github.io/epstein_text_messages
 Project-URL: WordCounts, https://michelcrypt4d4mus.github.io/epstein_text_messages/communication_word_count_epstein_files_nov_2025.html
 Description-Content-Type: text/markdown
@@ -32,11 +33,11 @@ Description-Content-Type: text/markdown
 # I Made Epstein's Text Messages Great Again
 
 * [I Made Epstein's Text Messages Great Again (And You Should Read Them)](https://cryptadamus.substack.com/p/i-made-epsteins-text-messages-great) post on [Substack](https://cryptadamus.substack.com/p/i-made-epsteins-text-messages-great)
-* The Epstein text messages (and some of the emails along with summary counts of sent emails to/from Epstein) generated by this code can be viewed [here](https://michelcrypt4d4mus.github.io/epstein_text_messages/).
-* All of His Emails can be read at another page also generated by this code [here](https://michelcrypt4d4mus.github.io/epstein_text_messages/all_emails_epstein_files_nov_2025.html).
-* Word counts for the emails and text messages are [here](https://michelcrypt4d4mus.github.io/epstein_text_messages/communication_word_count_epstein_files_nov_2025.html).
+* The Epstein text messages (and some of the emails along with summary information) generated by this code can be viewed [here](https://michelcrypt4d4mus.github.io/epstein_text_messages/).
+* All of His Emails along with descriptions of the 496 files that were neither emails nor text messages can be read at [another page also generated by this code](https://michelcrypt4d4mus.github.io/epstein_text_messages/all_emails_epstein_files_nov_2025.html).
+* Word counts for the communications are [here](https://michelcrypt4d4mus.github.io/epstein_text_messages/communication_word_count_epstein_files_nov_2025.html).
 * Metadata containing what I have figured out about who sent or received the communications in a given file (and a brief explanation for how I figured it out for each file) is deployed [here](https://michelcrypt4d4mus.github.io/epstein_text_messages/file_metadata_epstein_files_nov_2025.json)
-* Configuration variables assigning specific `HOUSE_OVERSIGHT_XXXXXX.txt` file IDs (the `111111` part) as being emails to or from particular people based on various research and contributions can be found in [constants.py](./epstein_files/util/constants.py). Everything in `constants.py` should also appear in the JSON metadata.
+* Configuration variables assigning specific `HOUSE_OVERSIGHT_XXXXXX.txt` file IDs (the `111111` part) as being emails to or from particular people based on various research and contributions can be found in [constants.py](./epstein_files/util/constants.py). Everything in `constants.py` appears in the JSON metadata linked above.
 
 
 ### Usage
@@ -46,11 +47,20 @@ Description-Content-Type: text/markdown
 You need to set the `DOCS_DIR` environment variable with the path to the folder of files you just downloaded when running. You can either create a `.env` file modeled on [`.env.example`](./.env.example) (which will set it permanently) or you can run with:
 
 ```bash
-DOCS_DIR=/path/to/epstein/ocr_txt_files ./generate.py
+# Generate color highlighted texts/emails/other files
+DOCS_DIR=/path/to/epstein/ocr_txt_files epstein_generate
+
+# Search
+DOCS_DIR=/path/to/epstein/ocr_txt_files epstein_search Bannon
+
+# Show a color highlighted file
+DOCS_DIR=/path/to/epstein/ocr_txt_files epstein_show 030999
+# This also works
+DOCS_DIR=/path/to/epstein/ocr_txt_files epstein_show HOUSE_OVERSIGHT_030999
 ```
 
-Run `./generate.py --help` for command line option assistance. Look in the [scripts](./scripts/) folder for various scripts.
-The first time you run anything it will take a few minutes to fix all the data, attribute the redacted emails, etc. Once you've run things once you can run the `./generate.py --pickled` to load the cached fixed up data and things will be quick.
+Run `epstein_generate --help` for command line option assistance.
+The first time you run anything it will take a few minutes to fix all the data, attribute the redacted emails, etc. Once you've run things once you can run the `epstein_generate --pickled` to load the cached fixed up data and things will be quick.
 
 #### As A Library
 ```python

epstein_files-1.0.4.dist-info/RECORD

@@ -0,0 +1,33 @@
+epstein_files/__init__.py,sha256=ovNZSEnsLnGahLlDZIai9jC6ZWo7RFd79KstrOGwJak,4861
+epstein_files/documents/communication.py,sha256=SunZdjMhR9v6y8LlQ6jhIu8vYjSndaBK0Su1mKnhfj0,2060
+epstein_files/documents/document.py,sha256=s6k3qqZ9pnAWBqkm45o1T4nTIUth1No2jxhiYTF3jpI,16732
+epstein_files/documents/email.py,sha256=QRlOE3OIu_o4AU7o3XuBMzVTTV3OZCajEXpbIZUdsck,38211
+epstein_files/documents/emails/email_header.py,sha256=wkPfSLbmzkAeQwvhf0bAeFDLPbQT-EeG0v8vNNLYktM,7502
+epstein_files/documents/imessage/text_message.py,sha256=wfWPQhwGG5Yzyhbr1NAQAY0bzRjjqVZmh8SPl48XmAM,3025
+epstein_files/documents/json_file.py,sha256=Vf9iHiEhUqk-hEh6lGZX_mJNWZUUQWeT-UI3FhKW0hw,1101
+epstein_files/documents/messenger_log.py,sha256=rOSy5yMerYBQ2r_o4SvuJ7Oeu-KhZVIIlh_Csb6uar0,5860
+epstein_files/documents/other_file.py,sha256=aD1nFkwsQ1eKb9Li1xfqEGGbqhkQlpR-6vIt5GHMyk8,9087
+epstein_files/epstein_files.py,sha256=XLyit5LmICnP44XoQWHUgMpRLS1-QgJZDaxO15qUnsI,18349
+epstein_files/util/constant/common_words.py,sha256=aR0UjoWmxyR49XS-DtHECQ1CiA_bK8hNP6CQ1TS9yZA,3696
+epstein_files/util/constant/html.py,sha256=9U098TGzlghGg4WfxLYHyub5JGR17Dv7VP5i2MSu8Kk,1415
+epstein_files/util/constant/names.py,sha256=iwnFqIceffYyLj7AaCsh3uve4Y0WbsZvHukjA1aFJWY,10104
+epstein_files/util/constant/output_files.py,sha256=B2qEXfNI_gT2Vp5HGSld2xM0PfeZ27j65HNymSmyzX4,974
+epstein_files/util/constant/strings.py,sha256=RBWJQnOXqBba8TwmjpvtkY8Jt54aFt4LYajoR8RanDk,1917
+epstein_files/util/constant/urls.py,sha256=2AM7AUqJQcNjCLrB6bKdh2sMtBOn7u8a8mwNq6HC0Zk,5621
+epstein_files/util/constants.py,sha256=GxI7RcnRgR8rDH4-dPrLlUH7q6mnhg5oK7ks56ZmpkA,110796
+epstein_files/util/data.py,sha256=xwTqrbAi7ZDJM0iyFVOevnokP_oIQ2npkRjHzF1KGGY,2908
+epstein_files/util/doc_cfg.py,sha256=I-n7_77hcA5DHGcqXISEjXOiTjYnTCLS7E16PUYw2kw,9736
+epstein_files/util/env.py,sha256=mvYy3Kj576aULU6E7NPuN-5ZEVOn0Zb4HAwR8VOmUMo,4839
+epstein_files/util/file_helper.py,sha256=v_bE10MHEcXti9DVJo4WqyOsG83Xrv05S3Vc70cYJkk,3082
+epstein_files/util/highlighted_group.py,sha256=He6LDIhcT_YZyAkGHcOR-T_znACoEyNznKWOKH4lA40,35564
+epstein_files/util/logging.py,sha256=b9iCTQkpAxu_3HphM5wm7VuPbWj9tocTuyUvMUTaI5A,2137
+epstein_files/util/output.py,sha256=k4HrOKvP6Os0-LrxVRiNKk7Pw9M3RxrlYP2Eql7TFIA,7366
+epstein_files/util/rich.py,sha256=x81QievKngC0oxsCpHRxKhjFstZ5hBVx91nFy63g_RQ,13709
+epstein_files/util/search_result.py,sha256=1fxe0KPBQXBk4dLfu6m0QXIzYfZCzvaSkWqvghJGzxY,567
+epstein_files/util/timer.py,sha256=8hxW4Y1JcTUfnBrHh7sL2pM9xu1sL4HFQM4CmmzTarU,837
+epstein_files/util/word_count.py,sha256=XTINgLm01jFQlNgdiLCcVFCodXAIb1dNbaAvznoRb1o,6757
+epstein_files-1.0.4.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+epstein_files-1.0.4.dist-info/METADATA,sha256=0_21zSew4FH6uTlcNitaX9yU3jtXiscsdV0fOP98ccA,5042
+epstein_files-1.0.4.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+epstein_files-1.0.4.dist-info/entry_points.txt,sha256=EV9qTh_n9X_1MOiQnxG5hM6R5v0rfi8W4QE-bsZkw3o,238
+epstein_files-1.0.4.dist-info/RECORD,,

epstein_files-1.0.2.dist-info/RECORD

@@ -1,33 +0,0 @@
-epstein_files/__init__.py,sha256=N4-A81KlSNWXyliBsjooi6GZr9_2qIB08qTG5RE9GzA,4725
-epstein_files/documents/communication.py,sha256=SunZdjMhR9v6y8LlQ6jhIu8vYjSndaBK0Su1mKnhfj0,2060
-epstein_files/documents/document.py,sha256=MOTS6AZFNOqnzpvYLXvoJC05ZVL9lTZnsRc-qjAjcJ4,16515
-epstein_files/documents/email.py,sha256=IpKeOuLTmHWowBvUUEp-tyTC8pwEmXg3XgLJkplQZWg,37717
-epstein_files/documents/emails/email_header.py,sha256=wkPfSLbmzkAeQwvhf0bAeFDLPbQT-EeG0v8vNNLYktM,7502
-epstein_files/documents/imessage/text_message.py,sha256=DMdd__L7UPad0YS49MJf_3bTVyE1BLfWafQbDQierC8,3272
-epstein_files/documents/json_file.py,sha256=Vf9iHiEhUqk-hEh6lGZX_mJNWZUUQWeT-UI3FhKW0hw,1101
-epstein_files/documents/messenger_log.py,sha256=-hSMFH_dyrjGLdph4SU2xQK4RpXOhkR3R_nqRrz3e1w,5620
-epstein_files/documents/other_file.py,sha256=JbKDtNrQ1Ua5vGPlZayON5Kgy0oJ-gHcdO9l9Iv9wRo,8930
-epstein_files/epstein_files.py,sha256=NpgQaxM3cC8CsAbzCyysakMbPdASWAt-wOhPZ879ZyQ,18018
-epstein_files/util/constant/common_words.py,sha256=aR0UjoWmxyR49XS-DtHECQ1CiA_bK8hNP6CQ1TS9yZA,3696
-epstein_files/util/constant/html.py,sha256=9U098TGzlghGg4WfxLYHyub5JGR17Dv7VP5i2MSu8Kk,1415
-epstein_files/util/constant/names.py,sha256=g0s9NA4zIexDGG7u5x0KDpLB9fyrYaYatsBhf_lH688,10253
-epstein_files/util/constant/output_files.py,sha256=B2qEXfNI_gT2Vp5HGSld2xM0PfeZ27j65HNymSmyzX4,974
-epstein_files/util/constant/strings.py,sha256=3JTqD0luJrC3NbGXn4q6P-gIaaNVx36P1oCmp92gAoM,1750
-epstein_files/util/constant/urls.py,sha256=x7Lv8yNNNLGU3GCvG4YbJ4qX3s_hiXffYuYUCjXyfbg,5526
-epstein_files/util/constants.py,sha256=xME94iH9a4b95N4yABs20Gn3Tu3cwmx5kNE_fPhsJEM,103420
-epstein_files/util/data.py,sha256=P4D_ggNNyScpTnu9wow8-67BlZtAXFKulJ5zbGtBR9A,2907
-epstein_files/util/doc_cfg.py,sha256=6H5EFLxG0ABG4BJHIEL7PSMBVwkcyjH1vvmitSQRa48,9615
-epstein_files/util/env.py,sha256=A2hEVg1HYymGd2odrLLo6k7yIvu0hh1XZniUW2u21dM,4734
-epstein_files/util/file_helper.py,sha256=v_bE10MHEcXti9DVJo4WqyOsG83Xrv05S3Vc70cYJkk,3082
-epstein_files/util/highlighted_group.py,sha256=7MfES52q10eq35L93iTIr_v5v0bamdQKCf7hlDIy7O8,35196
-epstein_files/util/logging.py,sha256=GjmOYiWAF1R_0Dvb5kXHAgPH5UJs-_gGcRig7LEDDL0,2066
-epstein_files/util/output.py,sha256=YH_-1YyNxiO29N88jvSAePbK5n25tgEIDFyP3sNNnnI,7309
-epstein_files/util/rich.py,sha256=gvSbWc-PQC5cAUg8zn02yZQkU57ExG1ArRLMIPrVxOc,13597
-epstein_files/util/search_result.py,sha256=1fxe0KPBQXBk4dLfu6m0QXIzYfZCzvaSkWqvghJGzxY,567
-epstein_files/util/timer.py,sha256=8hxW4Y1JcTUfnBrHh7sL2pM9xu1sL4HFQM4CmmzTarU,837
-epstein_files/util/word_count.py,sha256=XTINgLm01jFQlNgdiLCcVFCodXAIb1dNbaAvznoRb1o,6757
-epstein_files-1.0.2.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-epstein_files-1.0.2.dist-info/METADATA,sha256=Wf4rPDzdGRxogbs0H8cZoFb7HsRTrkPvkYHdClbg74c,4653
-epstein_files-1.0.2.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-epstein_files-1.0.2.dist-info/entry_points.txt,sha256=EV9qTh_n9X_1MOiQnxG5hM6R5v0rfi8W4QE-bsZkw3o,238
-epstein_files-1.0.2.dist-info/RECORD,,