epstein-files 1.2.5__py3-none-any.whl → 1.5.0__py3-none-any.whl

epstein_files/util/data.py

@@ -19,6 +19,8 @@ MULTINEWLINE_REGEX = re.compile(r"\n{2,}")
 CONSTANT_VAR_REGEX = re.compile(r"^[A-Z_]+$")
 ALL_NAMES = [v for k, v in vars(names).items() if isinstance(v, str) and CONSTANT_VAR_REGEX.match(k)]
 
+AMERICAN_DATE_FORMAT = r"%m/%d/%y %I:%M:%S %p"
+AMERICAN_TIME_REGEX = re.compile(r"(\d{1,2}/\d{1,2}/\d{2,4}\s+\d{1,2}:\d{2}(?::\d{2})?\s*(?:AM|PM)?)")
 PACIFIC_TZ = tz.gettz("America/Los_Angeles")
 TIMEZONE_INFO = {"PDT": PACIFIC_TZ, "PST": PACIFIC_TZ}  # Suppresses annoying warnings from parse() calls
 

epstein_files/util/doc_cfg.py

@@ -1,3 +1,4 @@
+import json
 import re
 from copy import deepcopy
 from dataclasses import Field, asdict, dataclass, field, fields
@@ -9,20 +10,21 @@ from dateutil.parser import parse
 from epstein_files.util.constant.names import *
 from epstein_files.util.constant.strings import *
 from epstein_files.util.data import remove_zero_time, without_falsey
+from epstein_files.util.env import args
 
-DuplicateType = Literal['earlier', 'quoted', 'redacted', 'same']
+DuplicateType = Literal['bounced', 'earlier', 'quoted', 'redacted', 'same']
 Metadata = dict[str, bool | datetime | int | str | list[str | None] |dict[str, bool | str]]
 
 # Misc
-CONSTANTIZE_NAMES = False  # A flag set to True that causes repr() of these classes to return strings of usable code
 INDENT = '    '
 INDENT_NEWLINE = f'\n{INDENT}'
 INDENTED_JOIN = f',{INDENT_NEWLINE}'
-MAX_LINE_LENGTH = 150
+MAX_LINE_LENGTH = 135
 REPUTATION_MGMT = f'{REPUTATION} management'
 SAME = 'same'
 
 DUPE_TYPE_STRS: dict[DuplicateType, str] = {
+    'bounced': 'a bounced copy of',
     'earlier': 'an earlier draft of',
     'quoted': 'quoted in full in',
     'redacted': 'a redacted version of',
@@ -32,7 +34,10 @@ DUPE_TYPE_STRS: dict[DuplicateType, str] = {
 FIELD_SORT_KEY = {
     'id': 'a',
     'author': 'aa',
-    'attribution_reason': 'zz',
+    'comment': 'zz',
+    'duplicate_ids': 'dup',
+    'duplicate_of_id': 'dupe',
+    'recipients': 'aaa',
 }
 
 FINANCIAL_REPORTS_AUTHORS = [
@@ -49,7 +54,6 @@ FINANCIAL_REPORTS_AUTHORS = [
 # Fields like timestamp and author are better added from the Document object
 NON_METADATA_FIELDS = [
     'actual_text',
-    'date',
     'id',
     'is_synthetic',
 ]
@@ -64,18 +68,19 @@ class DocCfg:
         id (str): ID of file
         author (Name): Author of the document (if any)
         category (str | None): Type of file
-        date (str | None): If passed will be immediated parsed into the 'timestamp' field
+        date (str | None): Parsed to a datetime by timestamp() if it exists
         dupe_type (DuplicateType | None): The type of duplicate this file is or its 'duplicate_ids' are
         duplicate_ids (list[str]): IDs of *other* documents that are dupes of this document
         duplicate_of_id (str | None): If this is a dupe the ID of the duplicated file. This file will be suppressed
         is_interesting (bool | None): Override other considerations and always consider this file interesting (or not)
-        timestamp (datetime | None): Time this email was sent, file was created, article published, etc.
         is_synthetic (bool): True if this config was generated by the duplicate_cfgs() method
+        replace_text_with (bool): True if `description` should replace body of the document when printing.
     """
     id: str
     attached_to_email_id: str | None = None
     author: Name = None
     category: str | None = None
+    comment: str = ''
     date: str | None = None
     description: str | None = None
     dupe_type: DuplicateType | None = None
@@ -84,15 +89,9 @@ class DocCfg:
     is_attribution_uncertain: bool = False
     is_interesting: bool | None = None
     is_synthetic: bool = False
-    timestamp: datetime | None = None
-
-    def __post_init__(self):
-        if self.date:
-            self.timestamp = parse(self.date)
-
-        if self.duplicate_of_id or self.duplicate_ids:
-            self.dupe_type = self.dupe_type or SAME
+    replace_text_with: str = ''
 
+    @property
     def complete_description(self) -> str | None:
         """String that summarizes what is known about this document."""
         description = ''
@@ -130,6 +129,24 @@ class DocCfg:
 
         return description
 
+    @property
+    def metadata(self) -> Metadata:
+        metadata = {k: v for k, v in asdict(self).items() if k not in NON_METADATA_FIELDS and v}
+
+        if self.is_interesting is False:
+            metadata['is_interesting'] = False
+
+        return metadata
+
+    @property
+    def timestamp(self) -> datetime | None:
+        if self.date:
+            return parse(self.date)
+
+    def __post_init__(self):
+        if self.duplicate_of_id or self.duplicate_ids:
+            self.dupe_type = self.dupe_type or SAME
+
     def duplicate_cfgs(self) -> Generator['DocCfg', None, None]:
         """Create synthetic DocCfg objects that set the 'duplicate_of_id' field to point back to this object."""
         for id in self.duplicate_ids:
@@ -141,9 +158,6 @@ class DocCfg:
             dupe_cfg.is_synthetic = True
             yield dupe_cfg
 
-    def metadata(self) -> Metadata:
-        return {k: v for k, v in asdict(self).items() if k not in NON_METADATA_FIELDS and v}
-
     def _props_strs(self) -> list[str]:
         props = []
         add_prop = lambda f, value: props.append(f"{f.name}={value}")
@@ -151,20 +165,16 @@ class DocCfg:
         for _field in sorted(fields(self), key=lambda f: FIELD_SORT_KEY.get(f.name, f.name)):
             value = getattr(self, _field.name)
 
-            if value is None or value is False or (isinstance(value, list) and len(value) == 0):
+            if _field.name in ['actual_text', 'is_fwded_article', 'is_interesting']:  # fields can be False or None or ''
+                if value is not None:
+                    add_prop(_field, str(value))
+            elif not value or _field.name == 'dupe_type' and value == 'same':
                 continue
             elif _field.name == AUTHOR:
-                add_prop(_field, constantize_name(str(value)) if CONSTANTIZE_NAMES else f"'{value}'")
-            elif _field.name == 'category' and value in [EMAIL, TEXT_MESSAGE]:
-                continue
-            elif _field.name == 'recipients' and value:
-                recipients_str = str([constantize_name(r) if (CONSTANTIZE_NAMES and r) else r for r in value])
-                add_prop(_field, recipients_str.replace("'", '') if CONSTANTIZE_NAMES else recipients_str)
-            elif _field.name == 'timestamp' and self.date is not None:
-                continue  # Don't print both timestamp and date
-            elif isinstance(value, datetime):
-                value_str = remove_zero_time(value)
-                add_prop(_field, f"parse('{value_str}')" if CONSTANTIZE_NAMES else f"'{value}'")
+                add_prop(_field, constantize_name(str(value)) if args.constantize else f"'{value}'")
+            elif _field.name == 'recipients':
+                recipients_str = str([constantize_name(r) if (args.constantize and r) else r for r in value])
+                add_prop(_field, recipients_str.replace("'", '') if args.constantize else recipients_str)
             elif isinstance(value, str):
                 if "'" in value:
                     value = '"' + value.replace('"', r'\"') + '"'
@@ -182,14 +192,14 @@ class DocCfg:
         type_str = f"{type(self).__name__}("
         single_line_repr = type_str + ', '.join(props) + f')'
 
-        if len(single_line_repr) < MAX_LINE_LENGTH:
+        if len(single_line_repr) < MAX_LINE_LENGTH or (self.comment and getattr(self, 'is_fwded_article')):
             repr_str = single_line_repr
         else:
             repr_str = f"{type_str}{INDENT_NEWLINE}" + INDENTED_JOIN.join(props)
             repr_str += ',' if props else ''
             repr_str += '\n)'
 
-        if CONSTANTIZE_NAMES:
+        if args.constantize:
             repr_str = INDENT + INDENT_NEWLINE.join(repr_str.split('\n'))
             return repr_str.replace(',,', ',').replace(',),', '),').replace(',),', '),')
         else:
@@ -224,9 +234,10 @@ class EmailCfg(CommunicationCfg):
     """
     actual_text: str | None = None
     fwded_text_after: str | None = None
-    is_fwded_article: bool = False
+    is_fwded_article: bool | None = None
     recipients: list[Name] = field(default_factory=list)
     subject: str | None = None
+    truncate_to: int | None = None
 
     # This is necessary because for some dumb reason @dataclass(repr=False) doesn't cut it
     def __repr__(self) -> str:

epstein_files/util/env.py

@@ -5,23 +5,24 @@ from pathlib import Path
 
 from rich_argparse_plus import RichHelpFormatterPlus
 
-from epstein_files.util.constant.output_files import ALL_EMAILS_PATH, CHRONOLOGICAL_EMAILS_PATH, TEXT_MSGS_HTML_PATH
-from epstein_files.util.logging import env_log_level, exit_with_error, logger
+from epstein_files.util.constant.output_files import (ALL_EMAILS_PATH, CHRONOLOGICAL_EMAILS_PATH,
+     DOJ_2026_HTML_PATH, TEXT_MSGS_HTML_PATH)
+from epstein_files.util.helpers.env_helpers import get_env_dir
+from epstein_files.util.logging import env_log_level, exit_with_error, logger, set_log_level
 
 DEFAULT_WIDTH = 155
 DEFAULT_FILE = 'default_file'
 EPSTEIN_GENERATE = 'epstein_generate'
 HTML_SCRIPTS = [EPSTEIN_GENERATE, 'epstein_word_count']
 
-# Verify Epstein docs dir exists
-EPSTEIN_DOCS_DIR_ENV_VAR_NAME = 'EPSTEIN_DOCS_DIR'
-DOCS_DIR_ENV = environ.get(EPSTEIN_DOCS_DIR_ENV_VAR_NAME)
-DOCS_DIR = Path(DOCS_DIR_ENV or '').resolve()
+# Get dirs from Env vars
+DOCS_DIR_ENV_VAR = 'EPSTEIN_DOCS_DIR'
+DOJ_PDFS_20260130_DIR_ENV_VAR = 'EPSTEIN_DOJ_PDFS_20260130_DIR'
+DOJ_TXTS_20260130_DIR_ENV_VAR = 'EPSTEIN_DOJ_TXTS_20260130_DIR'
 
-if not DOCS_DIR_ENV:
-    exit_with_error(f"{EPSTEIN_DOCS_DIR_ENV_VAR_NAME} env var not set!\n")
-elif not DOCS_DIR.exists():
-    exit_with_error(f"{EPSTEIN_DOCS_DIR_ENV_VAR_NAME}='{DOCS_DIR}' does not exist!\n")
+DOCS_DIR: Path = get_env_dir(DOCS_DIR_ENV_VAR, must_exist=True)
+DOJ_PDFS_20260130_DIR: Path = get_env_dir(DOJ_PDFS_20260130_DIR_ENV_VAR, must_exist=False)
+DOJ_TXTS_20260130_DIR: Path = get_env_dir(DOJ_TXTS_20260130_DIR_ENV_VAR, must_exist=False)
 
 is_env_var_set = lambda s: len(environ.get(s) or '') > 0
 is_output_arg = lambda arg: any([arg.startswith(pfx) for pfx in ['colors_only', 'json', 'make_clean', 'output']])
@@ -41,6 +42,7 @@ output.add_argument('--email-timeline', action='store_true', help='print a table
 output.add_argument('--emailers-info', '-ei', action='store_true', help='write a .png of the eeailers info table')
 output.add_argument('--json-files', action='store_true', help='pretty print all the raw JSON data files in the collection and exit')
 output.add_argument('--json-metadata', action='store_true', help='dump JSON metadata for all files and exit')
+output.add_argument('--output-doj-files', '-od', action='store_true', help='generate the DOJ files from 2026-01-30')
 output.add_argument('--output-emails', '-oe', action='store_true', help='generate emails section')
 output.add_argument('--output-other', '-oo', action='store_true', help='generate other files section')
 output.add_argument('--output-texts', '-ot', action='store_true', help='generate text messages section')
@@ -49,16 +51,19 @@ output.add_argument('--suppress-output', action='store_true', help='no output to
 output.add_argument('--uninteresting', action='store_true', help='only output uninteresting other files')
 output.add_argument('--width', '-w', type=int, default=DEFAULT_WIDTH, help='screen width to use (in characters)')
 
-scripts = parser.add_argument_group('SCRIPTS', 'Options used by epstein_search, epstein_show, and epstein_diff.')
+scripts = parser.add_argument_group('SCRIPTS', 'Options used by epstein_grep, epstein_show, and epstein_diff.')
 scripts.add_argument('positional_args', nargs='*', help='strings to searchs for, file IDs to show or diff, etc.')
+scripts.add_argument('--email-body', action='store_true', help='epstein_grep but only for the body of the email')
+scripts.add_argument('--min-line-length', type=int, help='epstein_grep minimum length of a matched line')
 scripts.add_argument('--raw', '-r', action='store_true', help='show raw contents of file (used by epstein_show)')
 scripts.add_argument('--whole-file', '-wf', action='store_true', help='print whole files')
 
 debug = parser.add_argument_group('DEBUG')
 debug.add_argument('--colors-only', '-c', action='store_true', help='print header with color key table and links and exit')
+debug.add_argument('--constantize', action='store_true', help='constantize names when printing repr() of objects')
 debug.add_argument('--debug', '-d', action='store_true', help='set debug level to INFO')
 debug.add_argument('--deep-debug', '-dd', action='store_true', help='set debug level to DEBUG')
-debug.add_argument('--json-stats', '-j', action='store_true', help='print JSON formatted stats about the files')
+debug.add_argument('--stats', '-j', action='store_true', help='print JSON formatted stats about the files')
 debug.add_argument('--skip-other-files', '-sof', action='store_true', help='skip parsing non email/text files')
 debug.add_argument('--suppress-logs', '-sl', action='store_true', help='set debug level to FATAL')
 debug.add_argument('--truncate', '-t', type=int, help='truncate emails to this many characters')
@@ -66,7 +71,11 @@ debug.add_argument('--write-txt', '-wt', action='store_true', help='write a plai
 
 
 # Parse args
-args = parser.parse_args()
+if environ.get('INVOKED_BY_PYTEST'):
+    args = parser.parse_args([EPSTEIN_GENERATE])
+else:
+    args = parser.parse_args()
+
 is_html_script = parser.prog in HTML_SCRIPTS
 
 args.debug = args.deep_debug or args.debug or is_env_var_set('DEBUG')
@@ -78,7 +87,9 @@ args.width = args.width if is_html_script else None
 args.any_output_selected = any([is_output_arg(arg) and val for arg, val in vars(args).items()])
 
 if not (args.any_output_selected or args.email_timeline or args.emailers_info):
-    logger.warning(f"No output section chosen; outputting default selection of texts, selected emails, and other files...")
+    if is_html_script:
+        logger.warning(f"No output section chosen; outputting default selection of texts, selected emails, and other files...")
+
     args.output_emails = args.output_other = args.output_texts = True
 
 if is_html_script:
@@ -95,25 +106,29 @@ if is_html_script:
             args.build = ALL_EMAILS_PATH
         elif args.email_timeline:
             args.build = CHRONOLOGICAL_EMAILS_PATH
+        elif args.output_doj_files:
+            args.build = DOJ_2026_HTML_PATH
         else:
             args.build = TEXT_MSGS_HTML_PATH
-elif parser.prog.startswith('epstein_') and not args.positional_args:
+elif parser.prog.startswith('epstein_') and not args.positional_args and not args.names:
     exit_with_error(f"{parser.prog} requires positional arguments but got none!")
 
 if args.names:
     logger.warning(f"Output restricted to {args.names}")
     args.output_other = False
 
+if args.truncate and args.whole_file:
+    exit_with_error(f"--whole-file and --truncate are incompatible")
 
 # Log level args
 if args.deep_debug:
-    logger.setLevel(logging.DEBUG)
+    set_log_level(logging.DEBUG)
 elif args.debug:
-    logger.setLevel(logging.INFO)
+    set_log_level(logging.INFO)
 elif args.suppress_logs:
-    logger.setLevel(logging.FATAL)
+    set_log_level(logging.FATAL)
 elif not env_log_level:
-    logger.setLevel(logging.WARNING)
+    set_log_level(logging.WARNING)
 
 logger.debug(f'Log level set to {logger.level}...')
 args_str = ',\n'.join([f"{k}={v}" for k, v in vars(args).items() if v])

epstein_files/util/file_helper.py

@@ -1,43 +1,67 @@
 import re
 from pathlib import Path
 
-from epstein_files.util.constant.strings import FILE_NAME_REGEX, FILE_STEM_REGEX, HOUSE_OVERSIGHT_PREFIX
-from epstein_files.util.env import DOCS_DIR
+from epstein_files.util.constant.strings import (DOJ_FILE_NAME_REGEX, EFTA_PREFIX,
+     HOUSE_OVERSIGHT_NOV_2025_FILE_NAME_REGEX, HOUSE_OVERSIGHT_NOV_2025_FILE_STEM_REGEX,
+     HOUSE_OVERSIGHT_PREFIX)
+from epstein_files.util.env import DOCS_DIR, DOJ_TXTS_20260130_DIR
 from epstein_files.util.logging import logger
 
 EXTRACTED_EMAILS_DIR = Path('emails_extracted_from_legal_filings')
-FILE_ID_REGEX = re.compile(fr".*{FILE_NAME_REGEX.pattern}")
+FILE_ID_REGEX = re.compile(fr".*{HOUSE_OVERSIGHT_NOV_2025_FILE_NAME_REGEX.pattern}")
 FILENAME_LENGTH = len(HOUSE_OVERSIGHT_PREFIX) + 6
 KB = 1024
 MB = KB * KB
 
 # Coerce methods handle both string and int arguments.
 coerce_file_name = lambda filename_or_id: coerce_file_stem(filename_or_id) + '.txt'
-coerce_file_path = lambda filename_or_id: DOCS_DIR.joinpath(coerce_file_name(filename_or_id))
 file_size = lambda file_path: Path(file_path).stat().st_size
 id_str = lambda id: f"{int(id):06d}"
 
 
+def coerce_file_path(filename_or_id: int | str) -> Path:
+    """Returns the `Path` for the file with `filename_or_id` ID."""
+    filename = coerce_file_name(filename_or_id)
+
+    if isinstance(filename_or_id, str) and DOJ_FILE_NAME_REGEX.match(filename_or_id):
+        for txt_file in DOJ_TXTS_20260130_DIR.glob('**/*.txt'):
+            if txt_file.name == filename:
+                return txt_file
+
+        raise RuntimeError(f"'{filename_or_id}' looks like DOJ file but no file named {filename} in '{DOJ_TXTS_20260130_DIR}'")
+    else:
+        return DOCS_DIR.joinpath(filename)
+
+
 def coerce_file_stem(filename_or_id: int | str) -> str:
-    """Generate a valid file_stem no matter what form the argument comes in."""
+    """Generate a valid file stem no matter what form the argument comes in."""
+    if isinstance(filename_or_id, str) and DOJ_FILE_NAME_REGEX.search(filename_or_id):
+        return Path(filename_or_id).stem
+
     if isinstance(filename_or_id, str) and filename_or_id.startswith(HOUSE_OVERSIGHT_PREFIX):
         file_id = extract_file_id(filename_or_id)
         file_stem = file_stem_for_id(file_id)
     else:
         file_stem = file_stem_for_id(filename_or_id)
 
-    if not FILE_STEM_REGEX.match(file_stem):
+    if not HOUSE_OVERSIGHT_NOV_2025_FILE_STEM_REGEX.match(file_stem):
         raise RuntimeError(f"Invalid stem '{file_stem}' from '{filename_or_id}'")
 
     return file_stem
 
 
 def extract_file_id(filename_or_id: int | str | Path) -> str:
+    # DOJ 2026-01 files have different pattern
+    if isinstance(filename_or_id, str) and filename_or_id.startswith(EFTA_PREFIX):
+        return Path(filename_or_id).stem
+
     if isinstance(filename_or_id, str):
         filename_or_id = filename_or_id.removesuffix(',')
 
     if isinstance(filename_or_id, int) or (isinstance(filename_or_id, str) and len(filename_or_id) <= 6):
         return id_str(filename_or_id)
+    elif isinstance(filename_or_id, str) and len(filename_or_id) == 8:
+        return f"{HOUSE_OVERSIGHT_PREFIX}{filename_or_id}"
 
     file_match = FILE_ID_REGEX.match(str(filename_or_id).upper())
 

epstein_files/util/helpers/debugging_helper.py

@@ -0,0 +1,13 @@
+
+
+def _show_timestamps(epstein_files):
+    for doc in epstein_files.doj_files:
+        doc.warn(f"timestamp: {doc.timestamp}")
+
+
+def _verify_filenames(epstein_files):
+    doc_filenames = set([doc.file_path.name for doc in epstein_files.all_documents])
+
+    for file_path in epstein_files.all_files:
+        if file_path.name not in doc_filenames:
+            print(f"'{file_path}' is not in list of {len(doc_filenames)} Document obj filenames!")

epstein_files/util/helpers/env_helpers.py

@@ -0,0 +1,21 @@
+"""Helpers for dealing with environment variables."""
+from os import environ
+from pathlib import Path
+
+from epstein_files.util.logging import exit_with_error, logger
+
+
+def get_env_dir(env_var_name: str, must_exist: bool = True) -> Path | None:
+    if (dir := environ.get(env_var_name)):
+        dir = Path(dir)
+        error_msg = f"env var {env_var_name} set to '{dir}' but that's not a directory"
+
+        if dir.is_dir():
+            return dir.resolve()
+        elif must_exist:
+            exit_with_error(f"Required {error_msg}.\n")
+        else:
+            logger.warning(f"Optional {error_msg}. Some features will be unavailable.")
+            return None
+    else:
+        logger.warning(f"Optional env var {env_var_name} not set. Some features will be unavailable.")