epstein-files 1.0.13__py3-none-any.whl → 1.0.15__py3-none-any.whl

epstein_files/util/env.py

@@ -8,10 +8,8 @@ from rich_argparse_plus import RichHelpFormatterPlus
 
 from epstein_files.util.logging import env_log_level, logger
 
-COUNT_WORDS_SCRIPT = 'epstein_word_count'
 DEFAULT_WIDTH = 145
-HTML_SCRIPTS = ['epstein_generate', COUNT_WORDS_SCRIPT]
-EPSTEIN_DOCS_DIR_ENV_VAR_NAME = 'EPSTEIN_DOCS_DIR'
+HTML_SCRIPTS = ['epstein_generate', 'epstein_word_count']
 
 
 RichHelpFormatterPlus.choose_theme('morning_glory')
@@ -31,8 +29,8 @@ output.add_argument('--output-other', '-oo', action='store_true', help='generate
 output.add_argument('--output-texts', '-ot', action='store_true', help='generate text messages section')
 output.add_argument('--sort-alphabetical', action='store_true', help='sort emailers alphabetically intead of by email count')
 output.add_argument('--suppress-output', action='store_true', help='no output to terminal (use with --build)')
+output.add_argument('--uninteresting', action='store_true', help='only output uninteresting other files')
 output.add_argument('--width', '-w', type=int, default=DEFAULT_WIDTH, help='screen width to use (in characters)')
-output.add_argument('--use-epstein-web', action='store_true', help='use epsteinweb.org links instead of epstein.media')
 
 scripts = parser.add_argument_group('SCRIPTS', 'Options used by epstein_search, epstein_show, and epstein_diff.')
 scripts.add_argument('positional_args', nargs='*', help='strings to searchs for, file IDs to show or diff, etc.')
@@ -50,6 +48,7 @@ args = parser.parse_args()
 
 
 # Verify Epstein docs can be found
+EPSTEIN_DOCS_DIR_ENV_VAR_NAME = 'EPSTEIN_DOCS_DIR'
 DOCS_DIR_ENV = environ.get(EPSTEIN_DOCS_DIR_ENV_VAR_NAME)
 DOCS_DIR = Path(DOCS_DIR_ENV or '').resolve()
 
@@ -65,13 +64,13 @@ is_env_var_set = lambda s: len(environ.get(s) or '') > 0
 is_html_script = current_script in HTML_SCRIPTS
 
 args.debug = args.deep_debug or args.debug or is_env_var_set('DEBUG')
+args.names = [None if n == 'None' else n for n in (args.names or [])]
 args.output_emails = args.output_emails or args.all_emails
-args.output_other = args.output_other or args.all_other_files
+args.output_other = args.output_other or args.all_other_files or args.uninteresting
 args.overwrite_pickle = args.overwrite_pickle or (is_env_var_set('OVERWRITE_PICKLE') and not is_env_var_set('PICKLED'))
 args.width = args.width if is_html_script else None
-is_output_selected = any([arg.startswith('output_') and value for arg, value in vars(args).items()])
-is_output_selected = is_output_selected or args.json_metadata or args.colors_only
-specified_names: list[str | None] = [None if n == 'None' else n for n in (args.names or [])]
+is_any_output_selected = any([arg.startswith('output_') and value for arg, value in vars(args).items()])
+is_any_output_selected = is_any_output_selected or args.json_metadata or args.colors_only
 
 # Log level args
 if args.deep_debug:
@@ -86,12 +85,12 @@ elif not env_log_level:
 logger.info(f'Log level set to {logger.level}...')
 
 # Massage args that depend on other args to the appropriate state
-if current_script == 'epstein_generate' and not (is_output_selected or args.make_clean):
+if current_script == 'epstein_generate' and not (is_any_output_selected or args.make_clean):
     logger.warning(f"No output section chosen; outputting default selection of texts, selected emails, and other files...")
     args.output_texts = args.output_emails = args.output_other = True
 
-if args.use_epstein_web:
-    logger.warning(f"Using links to epsteinweb.org links instead of epsteinify.com...")
-
 if args.debug:
-    logger.warning(f"Invocation args:\ncurrent_script={current_script}\nis_html_script={is_html_script},\nis_output_selected={is_output_selected}\nspecified_names={specified_names},\nargs={args}")
+    logger.warning(f"Invocation args:\ncurrent_script={current_script}\nis_html_script={is_html_script},\nis_output_selected={is_any_output_selected},\nargs={args}")
+
+if args.names:
+    logger.warning(f"Output restricted to {args.names}")

epstein_files/util/file_helper.py

@@ -12,7 +12,6 @@ KB = 1024
 MB = KB * KB
 
 file_size = lambda file_path: Path(file_path).stat().st_size
-file_size_str = lambda file_path: file_size_to_str(file_size(file_path))
 
 # Coerce methods handle both string and int arguments.
 coerce_file_name = lambda filename_or_id: coerce_file_stem(filename_or_id) + '.txt'
@@ -46,8 +45,12 @@ def extract_file_id(filename_or_id: int | str | Path) -> str:
     return file_match.group(1)
 
 
-def file_size_to_str(size: int) -> str:
-    digits = 2
+def file_size_str(file_path, digits: int | None = None):
+    return file_size_to_str(file_size(file_path), digits)
+
+
+def file_size_to_str(size: int, digits: int | None = None) -> str:
+    _digits = 2
 
     if size > MB:
         size_num = float(size) / MB
@@ -55,10 +58,11 @@ def file_size_to_str(size: int) -> str:
     elif size > KB:
         size_num = float(size) / KB
         size_str = 'kb'
-        digits = 1
+        _digits = 1
     else:
         return f"{size} b"
 
+    digits = _digits if digits is None else digits
     return f"{size_num:,.{digits}f} {size_str}"
 
 

epstein_files/util/highlighted_group.py

@@ -302,7 +302,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label=FINANCE,
         style='green',
-        pattern=r'Apollo|Ari\s*Glass|Bank|(Bernie\s*)?Madoff|Black(rock|stone)|B\s*of\s*A|Boothbay(\sFund\sManagement)?|Chase\s*Bank|Credit\s*Suisse|DB|Deutsche\s*(Asset|Bank)|Electron\s*Capital\s*(Partners)?|Fenner|FRBNY|Goldman(\s*Sachs)|HSBC|Invesco|(Janet\s*)?Yellen|(Jerome\s*)?Powell(?!M\. Cabot)|(Jimmy\s*)?Cayne|JPMC?|j\.?p\.?\s*morgan(\.?com|\s*Chase)?|Madoff|Merrill(\s*Lynch)?|(Michael\s*)?(Cembalest|Milken)|Mizrahi\s*Bank|MLPF&S|((anti.?)?money\s+)?launder(s?|ers?|ing)?(\s+money)?|Morgan Stanley|(Peter L. )?Scher|(Ray\s*)?Dalio|(Richard\s*)?LeFrak|Schwartz?man|Serageldin|UBS|us.gio@jpmorgan.com',
+        pattern=r'Apollo|Ari\s*Glass|Bank|(Bernie\s*)?Madoff|Black(rock|stone)|B\s*of\s*A|Boothbay(\sFund\sManagement)?|Chase\s*Bank|Credit\s*Suisse|DB|Deutsche?\s*(Asset|Bank)|Electron\s*Capital\s*(Partners)?|Fenner|FRBNY|Goldman(\s*Sachs)|HSBC|Invesco|(Janet\s*)?Yellen|(Jerome\s*)?Powell(?!M\. Cabot)|(Jimmy\s*)?Cayne|JPMC?|j\.?p\.?\s*morgan(\.?com|\s*Chase)?|Madoff|Merrill(\s*Lynch)?|(Michael\s*)?(Cembalest|Milken)|Mizrahi\s*Bank|MLPF&S|((anti.?)?money\s+)?launder(s?|ers?|ing)?(\s+money)?|Morgan Stanley|(Peter L. )?Scher|(Ray\s*)?Dalio|(Richard\s*)?LeFrak|Schwartz?man|Serageldin|UBS|us.gio@jpmorgan.com',
         emailers={
             AMANDA_ENS: 'Citigroup',
             BRAD_WECHSLER: f"head of {LEON_BLACK}'s personal investment vehicle according to FT",
@@ -373,14 +373,6 @@ HIGHLIGHTED_NAMES = [
         style='color(168)',
         pattern=r'BOJ|(Bank\s+of\s+)?Japan(ese)?|jpy?(?! Morgan)|SG|Singapore|Toky[op]',
     ),
-    HighlightedNames(
-        label='javanka',
-        style='medium_violet_red',
-        emailers = {
-            IVANKA: None,
-            JARED_KUSHNER: None,
-        }
-    ),
     HighlightedNames(
         label=JOURNALIST,
         style='bright_yellow',
@@ -404,7 +396,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='law enforcement',
         style='color(24) bold',
-        pattern=r'ag|(Alicia\s*)?Valle|AML|(Andrew\s*)?McCabe|attorney|((Bob|Robert)\s*)?Mueller|(Byung\s)?Pak|CFTC?|CIA|CIS|CVRA|Dep(artmen)?t\.?\s*of\s*(the\s*)?(Justice|Treasury)|DHS|DOJ|FBI|FCPA|FDIC|Federal\s*Bureau\s*of\s*Investigation|FinCEN|FINRA|FOIA|FTC|IRS|(James\s*)?Comey|(Jennifer\s*Shasky\s*)?Calvery|((Judge|Mark)\s*)?(Carney|Filip)|(Kirk )?Blouin|KYC|NIH|NS(A|C)|OCC|OFAC|(Lann?a\s*)?Belohlavek|lawyer|(Michael\s*)?Reiter|OGE|Office\s*of\s*Government\s*Ethics|Police Code Enforcement|(Preet\s*)?Bharara|SCOTUS|SD(FL|NY)|Southern\s*District\s*of\s*(Florida|New\s*York)|SEC|Secret\s*Service|Securities\s*and\s*Exchange\s*Commission|State\s*Dep(artmen)?t|Strzok|Supreme\s*Court|Treasury\s*(Dep(artmen)?t|Secretary)|TSA|USAID|(William\s*J\.?\s*)?Zloch',
+        pattern=r'ag|(Alicia\s*)?Valle|AML|(Andrew\s*)?McCabe|((Bob|Robert)\s*)?Mueller|(Byung\s)?Pak|CFTC?|CIA|CIS|CVRA|Dep(artmen)?t\.?\s*of\s*(the\s*)?(Justice|Treasury)|DHS|DOJ|FBI|FCPA|FDIC|Federal\s*Bureau\s*of\s*Investigation|FinCEN|FINRA|FOIA|FTC|IRS|(James\s*)?Comey|(Jennifer\s*Shasky\s*)?Calvery|((Judge|Mark)\s*)?(Carney|Filip)|(Kirk )?Blouin|KYC|NIH|NS(A|C)|OCC|OFAC|(Lann?a\s*)?Belohlavek|(Michael\s*)?Reiter|OGE|Office\s*of\s*Government\s*Ethics|Police Code Enforcement|(Preet\s*)?Bharara|SCOTUS|SD(FL|NY)|SEC|Secret\s*Service|Securities\s*and\s*Exchange\s*Commission|Southern\s*District\s*of\s*(Florida|New\s*York)|State\s*Dep(artmen)?t|Strzok|Supreme\s*Court|Treasury\s*(Dep(artmen)?t|Secretary)|TSA|USAID|(William\s*J\.?\s*)?Zloch',
         emailers = {
             ANN_MARIE_VILLAFANA: 'southern district of Florida U.S. Attorney',
             DANNY_FROST: 'Director of Communications at Manhattan DA',
@@ -476,7 +468,7 @@ HIGHLIGHTED_NAMES = [
         pattern=r'Alberto\sGonzale[sz]|(Alex\s*)?Acosta|(Bill\s*)?Barr|Bill\s*Shine|(Bob\s*)?Corker|(John\s*(R.?\s*)?)Bolton|Broidy|(Chris\s)?Christie|Devin\s*Nunes|(Don\s*)?McGa[hn]n|McMaster|(George\s*)?Nader|GOP|(Brett\s*)?Kavanaugh|Kissinger|Kobach|Koch\s*Brothers|Kolfage|Kudlow|Lewandowski|(Marco\s)?Rubio|(Mark\s*)Meadows|Mattis|McCain|(?<!Merwin Dela )Cruz|(Michael\s)?Hayden|((General|Mike)\s*)?(Flynn|Pence)|(Mitt\s*)?Romney|Mnuchin|Nikki|Haley|(Paul\s+)?(Manafort|Volcker)|(Peter\s)?Navarro|Pompeo|Reagan|Reince|Priebus|Republican|(Rex\s*)?Tillerson|(?<!Cynthia )(Richard\s*)?Nixon|Sasse|Tea\s*Party',
         # There's no emails from these people, they're just here to automate the regex creation for both first + last names
         emailers = {
-            RUDY_GIULIANI: 'disbarred formed mayor of New York City',
+            RUDY_GIULIANI: None,
             TULSI_GABBARD: None,
         },
     ),
@@ -484,7 +476,7 @@ HIGHLIGHTED_NAMES = [
         label='Rothschild family',
         style='indian_red',
         emailers={
-            ARIANE_DE_ROTHSCHILD: None,
+            ARIANE_DE_ROTHSCHILD: 'heiress',
             JOHNNY_EL_HACHEM: f'Works with {ARIANE_DE_ROTHSCHILD}',
         },
     ),
@@ -504,7 +496,7 @@ HIGHLIGHTED_NAMES = [
         style='light_goldenrod2',
         pattern=r'Alain Forget|Brotherton|Carl\s*Sagan|Columbia|David Grosof|J(ames|im)\s*Watson|(Lord\s*)?Martin\s*Rees|Massachusetts\s*Institute\s*of\s*Technology|MIT(\s*Media\s*Lab)?|Media\s*Lab|Minsky|((Noam|Valeria)\s*)?Chomsky|Norman\s*Finkelstein|Praluent|Regeneron|(Richard\s*)?Dawkins|Sanofi|Stanford|(Stephen\s*)?Hawking|(Steven?\s*)?Pinker|UCLA',
         emailers = {
-            DAVID_HAIG: None,
+            DAVID_HAIG: 'evolutionary geneticist?',
             JOSCHA_BACH: 'cognitive science / AI research',
             'Daniel Kahneman': 'Nobel economic sciences laureate and cognitivie psychologist (?)',
             'Ed Boyden': 'Associate Professor, MIT Media Lab neurobiology',
@@ -538,7 +530,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label='trump',
         style='red3 bold',
-        pattern=r"@?realDonaldTrump|(Alan\s*)?Weiss?elberg|\bDJ?T\b|Donald J. Tramp|(Donald\s+(J\.\s+)?)?Trump(ism|\s*Properties)?|Don(ald| *Jr)(?! Rubin)|Ivana|(Madeleine\s*)?Westerhout|Mar[-\s]*a[-\s]*Lago|(Marla\s*)?Maples|(Matt(hew)? )?Calamari|\bMatt C\b|Melania|(Michael (J.? )?)?Boccio|Rebekah\s*Mercer|Roger\s+Stone|rona|(The\s*)?Art\s*of\s*the\s*Deal",
+        pattern=r"@?realDonaldTrump|(Alan\s*)?Weiss?elberg|\bDJ?T\b|Donald J. Tramp|(Donald\s+(J\.\s+)?)?Trump(ism|\s*Properties)?|Don(ald| *Jr)(?! Rubin)|Ivank?a|Jared|Kushner|(Madeleine\s*)?Westerhout|Mar[-\s]*a[-\s]*Lago|(Marla\s*)?Maples|(Matt(hew)? )?Calamari|\bMatt C\b|Melania|(Michael (J.? )?)?Boccio|Rebekah\s*Mercer|Roger\s+Stone|rona|(The\s*)?Art\s*of\s*the\s*Deal",
         emailers = {
             'Bruce Moskowitz': "'Trump's health guy' according to Epstein",
         },
@@ -560,7 +552,7 @@ HIGHLIGHTED_NAMES = [
     HighlightedNames(
         label=VIRGIN_ISLANDS,
         style='sea_green1',
-        pattern=r'Antigua|Bahamas|Caribb?ean|Dominican\s*Republic|(Great|Little)\s*St.?\s*James|Haiti(an)?|(John\s*)deJongh(\s*Jr\.?)|(Kenneth E\. )?Mapp|Palm\s*Beach(?!\s*Post)|PBI|S(ain)?t.?\s*Thomas|USVI|VI|(The\s*)?Virgin\s*Islands(\s*Daily\s*News)?',  # TODO: VI Daily News should be yellow but it's hard bc Daily News xists
+        pattern=r'Antigua|Bahamas|Caribb?ean|Dominican\s*Republic|(Great|Little)\s*St.?\s*James|Haiti(an)?|(John\s*)deJongh(\s*Jr\.?)|(Kenneth E\. )?Mapp|Palm\s*Beach(?!\s*Post)|PBI|S(ain)?t.?\s*Thomas|USVI|(?<!Epstein )VI|(The\s*)?Virgin\s*Islands(\s*Daily\s*News)?',  # TODO: VI Daily News should be yellow but it's hard bc Daily News xists
         emailers = {
             CECILE_DE_JONGH: f'First lady 2007-2015',
             STACEY_PLASKETT: 'non-voting member of Congress',
@@ -669,7 +661,7 @@ def get_info_for_name(name: str) -> str | None:
 def get_style_for_category(category: str) -> str | None:
     if category in CATEGORY_STYLES:
         return CATEGORY_STYLES[category]
-    elif category in [CONFERENCE, SPEECH]:
+    elif category == CONFERENCE:
         return f"{get_style_for_category(ACADEMIA)} dim"
     elif category == SOCIAL:
         return get_style_for_category(PUBLICIST)

epstein_files/util/output.py

@@ -4,13 +4,14 @@ from rich.padding import Padding
 
 from epstein_files.documents.email import Email
 from epstein_files.documents.messenger_log import MessengerLog
+from epstein_files.documents.other_file import FIRST_FEW_LINES, OtherFile
 from epstein_files.epstein_files import EpsteinFiles, count_by_month
 from epstein_files.util.constant import output_files
 from epstein_files.util.constant.html import *
 from epstein_files.util.constant.names import *
 from epstein_files.util.constant.output_files import JSON_FILES_JSON_PATH, JSON_METADATA_PATH
 from epstein_files.util.data import dict_sets_to_lists
-from epstein_files.util.env import args, specified_names
+from epstein_files.util.env import args
 from epstein_files.util.file_helper import log_file_write
 from epstein_files.util.logging import logger
 from epstein_files.util.rich import *
@@ -20,20 +21,18 @@ PRINT_COLOR_KEY_EVERY_N_EMAILS = 150
 # Order matters. Default names to print emails for.
 DEFAULT_EMAILERS = [
     JEREMY_RUBIN,
-    AL_SECKEL,
     JOI_ITO,
     JABOR_Y,
     STEVEN_SINOFSKY,
+    AL_SECKEL,
     DANIEL_SIAD,
     JEAN_LUC_BRUNEL,
     STEVEN_HOFFENBERG,
+    RENATA_BOLOTOVA,
+    MASHA_DROKOVA,
     EHUD_BARAK,
     MARTIN_NOWAK,
-    MASHA_DROKOVA,
-    RENATA_BOLOTOVA,
     STEVE_BANNON,
-    OLIVIER_COLOM,
-    BORIS_NIKOLIC,
     PRINCE_ANDREW,
     JIDE_ZEITLIN,
     DAVID_STERN,
@@ -41,6 +40,7 @@ DEFAULT_EMAILERS = [
     JENNIFER_JACQUET,
     TYLER_SHEARS,
     CHRISTINA_GALBRAITH,
+    ZUBAIR_KHAN,
     None,
 ]
 
@@ -57,19 +57,19 @@ if len(set(DEFAULT_EMAILERS).intersection(set(DEFAULT_EMAILER_TABLES))) > 0:
     raise RuntimeError(f"Some names appear in both DEFAULT_EMAILERS and DEFAULT_EMAILER_TABLES")
 
 
-def print_emails(epstein_files: EpsteinFiles) -> int:
-    """Returns number of emails printed."""
+def print_emails_section(epstein_files: EpsteinFiles) -> list[Email]:
+    """Returns emails that were printed (may contain dupes if printed for both author and recipient)."""
     print_section_header(('Selections from ' if not args.all_emails else '') + 'His Emails')
-    print_other_site_link(is_header=False)
+    print_all_files_page_link(epstein_files)
     emailers_to_print: list[str | None]
     emailer_tables: list[str | None] = []
     already_printed_emails: list[Email] = []
     num_emails_printed_since_last_color_key = 0
 
-    if specified_names:
-        emailers_to_print = specified_names
+    if args.names:
+        emailers_to_print = args.names
     else:
-        epstein_files.print_emailer_counts_table()
+        print_centered(Padding(epstein_files.table_of_emailers(), (2, 0)))
 
         if args.all_emails:
             emailers_to_print = sorted(epstein_files.all_emailers(), key=lambda e: epstein_files.earliest_email_at(e))
@@ -99,16 +99,16 @@ def print_emails(epstein_files: EpsteinFiles) -> int:
         for name in DEFAULT_EMAILER_TABLES:
             epstein_files.print_emails_table_for(name)
 
-    if not specified_names:
+    if not args.names:
         epstein_files.print_email_device_info()
 
     if args.all_emails:
         _verify_all_emails_were_printed(epstein_files, already_printed_emails)
 
-    fwded_articles = [e for e in already_printed_emails if e.config and e.config.is_fwded_article]
-    log_msg = f"Rewrote {len(Email.rewritten_header_ids)} email headers (out of {len(already_printed_emails)})"
+    fwded_articles = [e for e in already_printed_emails if e.config and e.is_fwded_article()]
+    log_msg = f"Rewrote {len(Email.rewritten_header_ids)} of {len(already_printed_emails)} email headers"
     logger.warning(f"{log_msg}, {len(fwded_articles)} of the emails were forwarded articles.")
-    return len(already_printed_emails)
+    return already_printed_emails
 
 
 def print_json_files(epstein_files: EpsteinFiles):
@@ -125,17 +125,6 @@ def print_json_files(epstein_files: EpsteinFiles):
             console.print_json(json_file.json_str(), indent=4, sort_keys=False)
 
 
-def write_json_metadata(epstein_files: EpsteinFiles) -> None:
-    json_str = epstein_files.json_metadata()
-
-    if args.build:
-        with open(JSON_METADATA_PATH, 'w') as f:
-            f.write(json_str)
-            log_file_write(JSON_METADATA_PATH)
-    else:
-        console.print_json(json_str, indent=4, sort_keys=True)
-
-
 def print_json_stats(epstein_files: EpsteinFiles) -> None:
     console.line(5)
     console.print(Panel('JSON Stats Dump', expand=True, style='reverse bold'), '\n')
@@ -149,21 +138,52 @@ def print_json_stats(epstein_files: EpsteinFiles) -> None:
     print_json("count_by_month", count_by_month(epstein_files.all_documents()))
 
 
-def print_text_messages(epstein_files: EpsteinFiles) -> None:
-    print_section_header('Text Messages')
+def print_other_files_section(files: list[OtherFile], epstein_files: EpsteinFiles) -> None:
+    """Returns the OtherFile objects that were interesting enough to print."""
+    category_table = OtherFile.count_by_category_table(files)
+    other_files_preview_table = OtherFile.files_preview_table(files)
+    header_pfx = '' if args.all_other_files else 'Selected '
+    print_section_header(f"{FIRST_FEW_LINES} of {len(files)} {header_pfx}Files That Are Neither Emails Nor Text Messages")
+
+    if args.all_other_files:
+        console.line(1)
+    else:
+        print_all_files_page_link(epstein_files)
+        console.line(2)
+
+        for table in [category_table, other_files_preview_table]:
+            table.title = f"{header_pfx}{table.title}"
+
+    print_centered(category_table)
+    console.line(2)
+    console.print(other_files_preview_table)
+
+
+def print_text_messages_section(epstein_files: EpsteinFiles) -> None:
+    """Print summary table and stats for text messages."""
+    print_section_header('All of His Text Messages')
     print_centered("(conversations are sorted chronologically based on timestamp of first message)\n", style='gray30')
-    authors: list[str | None] = specified_names if specified_names else [JEFFREY_EPSTEIN]
-    log_files = epstein_files.imessage_logs_for(authors)
 
-    for log_file in log_files:
+    for log_file in epstein_files.imessage_logs:
         console.print(Padding(log_file))
         console.line(2)
 
-    epstein_files.print_imessage_summary()
+    print_centered(MessengerLog.summary_table(epstein_files.imessage_logs))
+
+
+def write_json_metadata(epstein_files: EpsteinFiles) -> None:
+    json_str = epstein_files.json_metadata()
+
+    if args.build:
+        with open(JSON_METADATA_PATH, 'w') as f:
+            f.write(json_str)
+            log_file_write(JSON_METADATA_PATH)
+    else:
+        console.print_json(json_str, indent=4, sort_keys=True)
 
 
 def write_urls() -> None:
-    """Write _URL style constant variables to a file bash scripts can load as env vars."""
+    """Write _URL style constant variables to URLS_ENV file so bash scripts can load as env vars."""
     url_vars = {k: v for k, v in vars(output_files).items() if k.endswith('URL') and not k.startswith('GH')}
 
     if not args.suppress_output:
@@ -190,8 +210,8 @@ def _verify_all_emails_were_printed(epstein_files: EpsteinFiles, already_printed
     logger.warning(f"Printed {len(already_printed_emails):,} emails of {len(email_ids_that_were_printed):,} unique file IDs.")
     missed_an_email = False
 
-    for email in epstein_files.emails:
-        if email.file_id not in email_ids_that_were_printed and not email.is_duplicate():
+    for email in epstein_files.non_duplicate_emails():
+        if email.file_id not in email_ids_that_were_printed:
             logger.warning(f"Failed to print {email.summary()}")
             missed_an_email = True
 

epstein_files/util/rich.py

@@ -102,8 +102,13 @@ def build_highlighter(pattern: str) -> EpsteinHighlighter:
     return TempHighlighter()
 
 
-def build_table(title: str | None, **kwargs) -> Table:
-    return Table(title=title, **{**DEFAULT_TABLE_KWARGS, **kwargs})
+def build_table(title: str | None, cols: list[str | dict] | None = None, **kwargs) -> Table:
+    table = Table(title=title, **{**DEFAULT_TABLE_KWARGS, **kwargs})
+
+    if cols:
+        add_cols_to_table(table, cols)
+
+    return table
 
 
 def join_texts(txts: list[Text], join: str = ' ', encloser: str = '', encloser_style: str = 'wheat4') -> Text:
@@ -138,6 +143,13 @@ def parenthesize(msg: str | Text, style: str = '') -> Text:
     return Text('(', style=style).append(txt).append(')')
 
 
+def print_all_files_page_link(epstein_files: 'EpsteinFiles') -> None:
+    markup_msg = link_markup(SITE_URLS[EMAIL], 'the other page', style='light_slate_grey bold')
+    txt = Text.from_markup(markup_msg).append(f' is uncurated and has all {len(epstein_files.other_files)}')
+    txt.append(f" unclassifiable files and {len(epstein_files.emails):,} emails")
+    print_centered(parenthesize(txt), style='dim')
+
+
 def print_author_header(msg: str, color: str | None, footer: str | None = None) -> None:
     txt = Text(msg, justify='center')
     color = color or 'white'
@@ -183,7 +195,20 @@ def print_header(epstein_files: 'EpsteinFiles') -> None:
 
     console.print(f"{not_optimized_msg}.\n", style='dim')
     print_page_title(width=TITLE_WIDTH)
-    print_other_site_link()
+    site_type: SiteType = EMAIL if args.all_emails else TEXT_MESSAGE
+    print_starred_header(f"This is the Epstein {site_type.title()}s site", num_spaces=4, num_stars=14)
+    other_site_type: SiteType = TEXT_MESSAGE if site_type == EMAIL else EMAIL
+    other_site_msg = "another site with" + (' all of' if other_site_type == EMAIL else '')
+    other_site_msg += f" Epstein's {other_site_type}s also generated by this code"
+
+    markup_msg = link_markup(SITE_URLS[other_site_type], other_site_msg, OTHER_SITE_LINK_STYLE)
+    print_centered(parenthesize(Text.from_markup(markup_msg)), style='bold')
+    word_count_link = link_text_obj(WORD_COUNT_URL, 'most frequently used words in the emails and texts', AUX_SITE_LINK_STYLE)
+    print_centered(parenthesize(word_count_link))
+    metadata_link = link_text_obj(JSON_METADATA_URL, 'author attribution explanations', AUX_SITE_LINK_STYLE)
+    print_centered(parenthesize(metadata_link))
+    json_link = link_text_obj(WORD_COUNT_URL, "epstein's json files", AUX_SITE_LINK_STYLE)
+    print_centered(parenthesize(json_link))
     _print_external_links()
     console.line()
     _print_abbreviations_table()
@@ -247,31 +272,6 @@ def print_numbered_list_of_emailers(_list: list[str | None], epstein_files = Non
     console.line()
 
 
-def print_other_site_link(is_header: bool = True) -> None:
-    """Print a link to the emails site if we're building text messages site and vice versa."""
-    site_type: SiteType = EMAIL if args.all_emails else TEXT_MESSAGE
-    link_style = OTHER_SITE_LINK_STYLE if is_header else 'light_slate_grey bold'
-
-    if is_header:
-        print_starred_header(f"This is the Epstein {site_type.title()}s site", num_spaces=4, num_stars=14)
-
-    other_site_type: SiteType = TEXT_MESSAGE if site_type == EMAIL else EMAIL
-    other_site_msg = "another site with" + (' all of' if other_site_type == EMAIL else '')
-    other_site_msg += f" Epstein's {other_site_type}s also generated by this code"
-    markup_msg = link_markup(SITE_URLS[other_site_type], other_site_msg, link_style)
-    print_centered(parenthesize(Text.from_markup(markup_msg)), style='bold')
-
-    if not is_header:
-        return
-
-    word_count_link = link_text_obj(WORD_COUNT_URL, 'most frequently used words in the emails and texts', AUX_SITE_LINK_STYLE)
-    print_centered(parenthesize(word_count_link))
-    metadata_link = link_text_obj(JSON_METADATA_URL, 'author attribution explanations', AUX_SITE_LINK_STYLE)
-    print_centered(parenthesize(metadata_link))
-    json_link = link_text_obj(WORD_COUNT_URL, "epstein's json files", AUX_SITE_LINK_STYLE)
-    print_centered(parenthesize(json_link))
-
-
 def print_page_title(expand: bool = True, width: int | None = None) -> None:
     title_panel = Panel(Text(PAGE_TITLE, justify='center'), expand=expand, style=TITLE_STYLE, width=width)
     console.print(Align.center(vertically_pad(title_panel)))
@@ -294,7 +294,7 @@ def print_panel(msg: str, style: str = 'black on white', padding: tuple | None =
 def print_section_header(msg: str, style: str = SECTION_HEADER_STYLE, is_centered: bool = False) -> None:
     panel = Panel(Text(msg, justify='center'), expand=True, padding=(1, 1), style=style)
     panel = Align.center(panel) if is_centered else panel
-    console.print(Padding(panel, (3, 0, 1, 0)))
+    console.print(Padding(panel, (3, 2, 1, 2)))
 
 
 def print_starred_header(msg: str, num_stars: int = 7, num_spaces: int = 2, style: str = TITLE_STYLE) -> None:

epstein_files/util/word_count.py

@@ -14,7 +14,7 @@ from epstein_files.util.constant.common_words import COMMON_WORDS_LIST, COMMON_W
 from epstein_files.util.constant.names import OTHER_NAMES
 from epstein_files.util.constant.output_files import WORD_COUNT_HTML_PATH
 from epstein_files.util.data import ALL_NAMES, flatten, sort_dict
-from epstein_files.util.env import args, specified_names
+from epstein_files.util.env import args
 from epstein_files.util.logging import logger
 from epstein_files.util.rich import (console, highlighter, print_centered, print_color_key, print_page_title,
      print_panel, print_starred_header, write_html)
@@ -198,10 +198,10 @@ def write_word_counts_html() -> None:
     word_count = WordCount()
 
     # Remove dupes, junk mail, and fwded articles from emails
-    emails = [e for e in epstein_files.emails if not (e.is_duplicate() or e.is_junk_mail() or e.is_fwded_article())]
+    emails = [e for e in epstein_files.non_duplicate_emails() if not (e.is_junk_mail() or e.is_fwded_article())]
 
     for email in emails:
-        if specified_names and email.author not in specified_names:
+        if args.names and email.author not in args.names:
             continue
 
         logger.info(f"Counting words in {email}\n  [SUBJECT] {email.subject()}")
@@ -218,14 +218,12 @@ def write_word_counts_html() -> None:
             for word in line.split():
                 word_count.tally_word(word, SearchResult(email, [MatchedLine(line, i)]))
 
-    # Add in iMessage conversation words
-    imessage_logs = epstein_files.imessage_logs_for(specified_names) if specified_names else epstein_files.imessage_logs
-
-    for imessage_log in imessage_logs:
+    # Add in iMessage conversations
+    for imessage_log in epstein_files.imessage_logs:
         logger.info(f"Counting words in {imessage_log}")
 
         for i, msg in enumerate(imessage_log.messages):
-            if specified_names and msg.author not in specified_names:
+            if args.names and msg.author not in args.names:
                 continue
             elif HTML_REGEX.search(line):
                 continue
@@ -234,7 +232,7 @@ def write_word_counts_html() -> None:
                 word_count.tally_word(word, SearchResult(imessage_log, [MatchedLine(msg.text, i)]))
 
     print_page_title(expand=False)
-    print_starred_header(f"Most Common Words in {len(emails):,} Emails and {len(imessage_logs)} iMessage Logs")
+    print_starred_header(f"Most Common Words in {len(emails):,} Emails and {len(epstein_files.imessage_logs)} iMessage Logs")
     print_centered(f"(excluding {len(COMMON_WORDS_LIST)} particularly common words at bottom)", style='dim')
     console.line()
     print_color_key()

{epstein_files-1.0.13.dist-info → epstein_files-1.0.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: epstein-files
-Version: 1.0.13
+Version: 1.0.15
 Summary: Tools for working with the Jeffrey Epstein documents released in November 2025.
 Home-page: https://michelcrypt4d4mus.github.io/epstein_text_messages/
 License: GPL-3.0-or-later
@@ -43,11 +43,12 @@ Description-Content-Type: text/markdown
 
 
 ## Usage
-
 #### Installation
 1. Requires you have a local copy of the OCR text files from the House Oversight document release in a directory `/path/to/epstein/ocr_txt_files`. You can download those OCR text files from [the Congressional Google Drive folder](https://drive.google.com/drive/folders/1ldncvdqIf6miiskDp_EDuGSDAaI_fJx8) (make sure you grab both the `001/` and `002/` folders).
 1. Use `poetry install` for easiest time installing. `pip install epstein-files` should also work, though `pipx install epstein-files` is usually better.
 
+
+#### Command Line Tools
 You need to set the `EPSTEIN_DOCS_DIR` environment variable with the path to the folder of files you just downloaded when running. You can either create a `.env` file modeled on [`.env.example`](./.env.example) (which will set it permanently) or you can run with:
 
 ```bash
@@ -63,7 +64,7 @@ epstein_generate
 # Search for a string:
 epstein_search Bannon
 # Or a regex:
-epstein_search '\bSteve\s*Bannon\b'
+epstein_search '\bSteve\s*Bannon|Jeffrey\s*Epstein\b'
 
 # Show a file with color highlighting of keywords:
 epstein_show 030999
@@ -82,6 +83,12 @@ epstein_diff 030999 020442
 The first time you run anything it will take a few minutes to fix all the janky OCR text, attribute the redacted emails, etc. After that things will be quick.
 Run `epstein_generate --help` for command line option assistance.
 
+**Optional:** There are a handful of emails that I extracted from the legal filings they were contained in. If you want to include these files in your local analysis you'll need to copy those files from the repo into your local document directory. Something like:
+
+```bash
+cp ./emails_extracted_from_legal_filings/*.txt "$EPSTEIN_DOCS_DIR"
+```
+
 
 #### As A Library
 ```python

epstein_files-1.0.15.dist-info/RECORD

@@ -0,0 +1,33 @@
+epstein_files/__init__.py,sha256=nOJ26LghS0sCdQ4qGwklVm18kbuFRn6Uu1-JDE8jwd4,5050
+epstein_files/documents/communication.py,sha256=oqNsSDWe-N0jSmchIHxpihihzIWha-foFqMwKZlxyng,2057
+epstein_files/documents/document.py,sha256=5GdM8grhaaefmXeV26XwYRZvMNqFETFnE7mFpVGGNik,17249
+epstein_files/documents/email.py,sha256=h8V-20JAw2U_7mIWfnt3phvnykf-ioCdML8dAYoFVMc,40937
+epstein_files/documents/emails/email_header.py,sha256=wkPfSLbmzkAeQwvhf0bAeFDLPbQT-EeG0v8vNNLYktM,7502
+epstein_files/documents/imessage/text_message.py,sha256=JPHlO-5Xgt5V5E9fDsgCEr3XOp7FyzlhIHmZhFNZcmg,2806
+epstein_files/documents/json_file.py,sha256=WcZW5NNqA67rHTdopbOGtup00muNaLlvrNgKb-K4zO8,1504
+epstein_files/documents/messenger_log.py,sha256=yFw5ScyP3KM8AoV9D36GUxLIT3pLsDdpi5F61Dw7fwQ,6510
+epstein_files/documents/other_file.py,sha256=CCwOYsipYTWZj8pSTl0kgUy_LRu7Z5ZuWygQEYhilNA,9778
+epstein_files/epstein_files.py,sha256=pBtrcEGZAMUDO_Xghxt-bd_RZp6JQlDj770EaM9oEZ4,17887
+epstein_files/util/constant/common_words.py,sha256=aR0UjoWmxyR49XS-DtHECQ1CiA_bK8hNP6CQ1TS9yZA,3696
+epstein_files/util/constant/html.py,sha256=9U098TGzlghGg4WfxLYHyub5JGR17Dv7VP5i2MSu8Kk,1415
+epstein_files/util/constant/names.py,sha256=nk_kuOtIo_UTl4R4klUGJRfCRNDZZkqKkjbJx6XCOzQ,10471
+epstein_files/util/constant/output_files.py,sha256=BkV4_gmdj46RfGy5SFYp6dgTty3FtlBth5YGmaGutls,1700
+epstein_files/util/constant/strings.py,sha256=rOKmYrGIZFPQbhCgd8z7mUF0CbQQBO4ij8kXvGTDkoE,1857
+epstein_files/util/constant/urls.py,sha256=Rpr9QQfjk2-8YGTXSQYg2oNA2ZxWrkedr5tL_53Z6ak,4896
+epstein_files/util/constants.py,sha256=cIFDxq7-t6UE_qs2RtcyY2R3BAZmVV3glgTgKVkrG0c,112348
+epstein_files/util/data.py,sha256=JccGFZGiCGm7XtwpQTocIjGYOr6hTUpEPwHhjyW9Xnc,3164
+epstein_files/util/doc_cfg.py,sha256=aBIm0hyxf-aeMsb8ZUNiQFVsPFimjVUIkrVdDrg1iQU,9105
+epstein_files/util/env.py,sha256=aSUC_TwdtKB8z6PCDaLeu3k5qZE3AgcsP9jU2p9eaP0,5640
+epstein_files/util/file_helper.py,sha256=uJzNHihuAi2-XMNbUdjdovgppcIt-fVIIrXHXVShA6Q,2904
+epstein_files/util/highlighted_group.py,sha256=ACDwLratRV9bemDyHY528JMFDf0ZDjm9HBfFJiqDgyY,36535
+epstein_files/util/logging.py,sha256=fuREq06xUUI3DfCV2JE-8QM-sQKxpLDj0_AYFO6qR1M,1983
+epstein_files/util/output.py,sha256=2YKpL2gGhS2SI5nC2yan0vNHvYMg1Ayq_ptzx9Gi9E4,9055
+epstein_files/util/rich.py,sha256=bQPlOEZ6qTpVF4yPnwmlBk4K11zH5RJjwbWeBIOUUGE,15191
+epstein_files/util/search_result.py,sha256=1fxe0KPBQXBk4dLfu6m0QXIzYfZCzvaSkWqvghJGzxY,567
+epstein_files/util/timer.py,sha256=8hxW4Y1JcTUfnBrHh7sL2pM9xu1sL4HFQM4CmmzTarU,837
+epstein_files/util/word_count.py,sha256=o_-HnfzHdPDPR8oA_dv6fjy1dbsHee8p_aoe62PEQHw,9213
+epstein_files-1.0.15.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+epstein_files-1.0.15.dist-info/METADATA,sha256=yELMsoWwHhLiDjU7OFOYDAmifBak0h6xGLEu-3nwiHc,5867
+epstein_files-1.0.15.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+epstein_files-1.0.15.dist-info/entry_points.txt,sha256=5qYgwAXpxegeAicD_rzda_trDRnUC51F5UVDpcZ7j6Q,240
+epstein_files-1.0.15.dist-info/RECORD,,

epstein_files-1.0.13.dist-info/RECORD

@@ -1,33 +0,0 @@
-epstein_files/__init__.py,sha256=qVFB7sS6XSlZX-ByyDwdbGSn2h06aoX2Mx8WcgRb-To,4710
-epstein_files/documents/communication.py,sha256=i6eob0ucnI7AY1sGuJlzrtU_C0mxEtrPEQZE4yNeM9w,2049
-epstein_files/documents/document.py,sha256=VsXIJhY7HTUlQrnTzYv5VLtYugdb-c2ClrfInPb5l2M,17095
-epstein_files/documents/email.py,sha256=P2c8Z23qaiWj6qWf4QKTuLvqKDoUJmu7SyzJ6FQCmPA,40727
-epstein_files/documents/emails/email_header.py,sha256=wkPfSLbmzkAeQwvhf0bAeFDLPbQT-EeG0v8vNNLYktM,7502
-epstein_files/documents/imessage/text_message.py,sha256=4gFvTfulj_Su10lNQl6Hq_p9ArTrSmn5pfC22YRJXjI,2794
-epstein_files/documents/json_file.py,sha256=tIYTwA3FYkwVZSpXvFYyUoH9m2sGYCD1U0ttamH6r1o,1306
-epstein_files/documents/messenger_log.py,sha256=BW3J52a9XxNVoK7SxtfJGydSNPoeKGE0PkdkMpf9sJ0,6294
-epstein_files/documents/other_file.py,sha256=pnl_q1o7ur3eeqGPwsYL2qbM3Y8O9LX6j6LbWnoxAiE,9939
-epstein_files/epstein_files.py,sha256=sbgS7GlrlWo1kuH4YXUSvbYX_SApgrwhpx6_qdHmrBM,19119
-epstein_files/util/constant/common_words.py,sha256=aR0UjoWmxyR49XS-DtHECQ1CiA_bK8hNP6CQ1TS9yZA,3696
-epstein_files/util/constant/html.py,sha256=9U098TGzlghGg4WfxLYHyub5JGR17Dv7VP5i2MSu8Kk,1415
-epstein_files/util/constant/names.py,sha256=ObLJCvit_C1cwLc-7Wcjh-C5oQrpww1HLk6DrEvnu8E,10368
-epstein_files/util/constant/output_files.py,sha256=BkV4_gmdj46RfGy5SFYp6dgTty3FtlBth5YGmaGutls,1700
-epstein_files/util/constant/strings.py,sha256=02DwbhAe8qBRq5HOUFx5FafXJ1P2-RJf9TCVu2b7UDQ,1932
-epstein_files/util/constant/urls.py,sha256=EKzpL1np3cdyYvUlgv1zl9IJU8Vrg8jXpAu6YpM0_Bk,5283
-epstein_files/util/constants.py,sha256=tSHrgzDOTx0KwGy4HI7ShIwaRMX1o7otW3CkKvRplZs,112121
-epstein_files/util/data.py,sha256=gOaETo4LvN3VgRvA1dB2_NRDL3wiL-xBICZ2jfD-knk,2997
-epstein_files/util/doc_cfg.py,sha256=irQljeGXlB9wbRbP05pWQZMS2-DbhdugfVGD8Q2UhSg,9006
-epstein_files/util/env.py,sha256=HnYcfHSNkwVJ_T75Woy43_OpDyxD0KHPj3GxcVx86N4,5751
-epstein_files/util/file_helper.py,sha256=tacTe1GcAnckPFvjMgxRRSLnFgr2aVIYsgfDR_C9uXk,2780
-epstein_files/util/highlighted_group.py,sha256=rKJsUQfPzQwMMu5--QWsxNFxW303RI8yq69sw4vi2RA,36720
-epstein_files/util/logging.py,sha256=fuREq06xUUI3DfCV2JE-8QM-sQKxpLDj0_AYFO6qR1M,1983
-epstein_files/util/output.py,sha256=XcflgSOlzUGj6FsFaK6j4Dljld8A0h_uVV7ERcI_EYw,8120
-epstein_files/util/rich.py,sha256=V8WDQPuCcubn46HW1cLZ6uT-CBAnBB17Hozsvv7Pd4g,14987
-epstein_files/util/search_result.py,sha256=1fxe0KPBQXBk4dLfu6m0QXIzYfZCzvaSkWqvghJGzxY,567
-epstein_files/util/timer.py,sha256=8hxW4Y1JcTUfnBrHh7sL2pM9xu1sL4HFQM4CmmzTarU,837
-epstein_files/util/word_count.py,sha256=8qBTuq3d0Q-3fwiuECKWi2RfL-KUiZD8TciwvfL0D_o,9353
-epstein_files-1.0.13.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-epstein_files-1.0.13.dist-info/METADATA,sha256=ZfGSbuEOmAhzDs_a6KiKjXfhLNRwcZoXklAK1KphX7I,5480
-epstein_files-1.0.13.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-epstein_files-1.0.13.dist-info/entry_points.txt,sha256=5qYgwAXpxegeAicD_rzda_trDRnUC51F5UVDpcZ7j6Q,240
-epstein_files-1.0.13.dist-info/RECORD,,