wikiq 0.3.0__py3-none-any.whl

wikiq/__init__.py

@@ -0,0 +1,1590 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2015-2026 Benjamin Mako Hill, Nathan TeBlunthuis, Will
+# Beason, Sohyeon Hwang, Kaylea Champion, and other contributors
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# original wikiq headers are: title articleid revid date_time anon
+# editor editor_id minor text_size text_entropy text_md5 reversion
+# additions_size deletions_size
+import argparse
+import gc
+import json
+import os.path
+import re
+import signal
+import sys
+import threading
+import time
+from collections import deque, defaultdict
+from hashlib import sha1
+from io import TextIOWrapper
+from itertools import groupby
+from pathlib import Path
+from subprocess import PIPE, Popen
+from typing import IO, Any, Generator, TextIO, Union
+
+import mwpersistence
+import mwreverts
+import mwxml
+import pyarrow as pa
+import pyarrow.csv as pacsv
+from deltas import SegmentMatcher, SequenceMatcher
+from deltas.tokenizers import wikitext_split
+from more_itertools import peekable
+from mwxml import Dump
+
+import wikiq.tables as tables
+from wikiq.resume import get_resume_point
+from wikiq.tables import RevisionTable
+from wikiq.wikitext_parser import WikitextParser
+
+try:
+    from importlib.metadata import PackageNotFoundError
+    from importlib.metadata import version as _package_version
+    __version__ = _package_version("wikiq")
+except PackageNotFoundError:
+    __version__ = "unknown"
+
+TO_ENCODE = ("title", "editor")
+PERSISTENCE_RADIUS = 7
+DIFF_TIMEOUT_MS = 60000
+
+
+# Some dependencies serve a single feature and are awkward enough to install
+# that wikiq leaves them out of the base install. Each is imported inside the
+# code path that needs it, so importing wikiq works without them and a plain
+# `pip install` needs no C++ compiler. These helpers do that import and, when
+# it fails, explain what to install rather than raising an ImportError.
+
+def require_pywikidiff2(feature: str):
+    """Import and return pywikidiff2, or exit explaining how to install it.
+
+    feature names the wikiq option that needs it, so the message points at
+    whatever the user actually asked for.
+    """
+    try:
+        import pywikidiff2
+    except ImportError:
+        raise SystemExit(
+            f"{feature} requires pywikidiff2, which wikiq does not install by "
+            "default because it compiles a C++ extension.\n"
+            "Install it with:\n"
+            "    pip install 'pywikidiff2 @ git+"
+            "https://gitea.communitydata.science/groceryheist/pywikidiff2.git'\n"
+            "A C++ compiler and libthai must be available. Other persistence "
+            "methods (-p sequence, -p segment, -p legacy) do not need it."
+        )
+    return pywikidiff2
+
+
+def require_mw_persistence():
+    """Import and return mw.lib.persistence, or exit explaining how to get it."""
+    try:
+        from mw.lib import persistence
+    except ImportError:
+        raise SystemExit(
+            "-p legacy requires mediawiki-utilities, which wikiq no longer "
+            "installs by default.\n"
+            "Install it with:\n"
+            "    pip install 'wikiq[legacy]'\n"
+            "This method exists to reproduce results from older research "
+            "projects; -p sequence is the current equivalent."
+        )
+    return persistence
+
+
+def pyarrow_type_to_spark(pa_type):
+    """Convert a PyArrow type to Spark JSON schema format."""
+    if pa.types.is_int64(pa_type):
+        return "long"
+    elif pa.types.is_int32(pa_type):
+        return "integer"
+    elif pa.types.is_int8(pa_type):
+        return "byte"
+    elif pa.types.is_boolean(pa_type):
+        return "boolean"
+    elif pa.types.is_string(pa_type) or pa.types.is_large_string(pa_type):
+        return "string"
+    elif pa.types.is_timestamp(pa_type):
+        return "timestamp"
+    elif pa.types.is_list(pa_type):
+        return {
+            "type": "array",
+            "elementType": pyarrow_type_to_spark(pa_type.value_type),
+            "containsNull": True
+        }
+    elif pa.types.is_struct(pa_type):
+        return {
+            "type": "struct",
+            "fields": [
+                {
+                    "name": field.name,
+                    "type": pyarrow_type_to_spark(field.type),
+                    "nullable": field.nullable,
+                    "metadata": {}
+                }
+                for field in pa_type
+            ]
+        }
+    elif pa.types.is_map(pa_type):
+        return {
+            "type": "map",
+            "keyType": pyarrow_type_to_spark(pa_type.key_type),
+            "valueType": pyarrow_type_to_spark(pa_type.item_type),
+            "valueContainsNull": True
+        }
+    else:
+        return "string"
+
+
+def pyarrow_to_spark_schema(schema: pa.Schema) -> dict:
+    """Convert a PyArrow schema to Spark JSON schema format."""
+    return {
+        "type": "struct",
+        "fields": [
+            {
+                "name": field.name,
+                "type": pyarrow_type_to_spark(field.type),
+                "nullable": field.nullable,
+                "metadata": {}
+            }
+            for field in schema
+        ]
+    }
+
+
+def build_table(
+    text: bool = False,
+    collapse_user: bool = False,
+    external_links: bool = False,
+    citations: bool = False,
+    wikilinks: bool = False,
+    templates: bool = False,
+    headings: bool = False,
+    redirect_aliases: Union[list[str], None] = None,
+):
+    """Build the RevisionTable with appropriate columns based on flags.
+
+    Returns:
+        (table, reverts_column, wikitext_parser) - the table, a reference to
+        the reverts column (which process() needs for setting the revert
+        detector), and the wikitext parser (None unless a wikitext-parsing
+        column was requested).
+    """
+    reverts_column = tables.RevisionReverts()
+    redirect_detector = tables.RedirectDetector(redirect_aliases)
+
+    table = RevisionTable([
+        tables.RevisionId(),
+        tables.RevisionTimestamp(),
+        tables.RevisionArticleId(),
+        tables.RevisionPageTitle(),
+        tables.RevisionNamespace(),
+        tables.RevisionIsRedirect(redirect_detector),
+        tables.RevisionRedirectTarget(redirect_detector),
+        tables.RevisionDeleted(),
+        tables.RevisionEditorId(),
+        tables.RevisionEditSummary(),
+        tables.RevisionTextChars(),
+        reverts_column,
+        tables.RevisionSha1(),
+        tables.RevisionIsMinor(),
+        tables.RevisionEditorText(),
+        tables.RevisionIsAnon(),
+    ])
+
+    if text:
+        table.columns.append(tables.RevisionText())
+
+    if collapse_user:
+        table.columns.append(tables.RevisionCollapsed())
+
+    wikitext_parser = None
+    if external_links or citations or wikilinks or templates or headings:
+        wikitext_parser = WikitextParser()
+
+        if external_links:
+            table.columns.append(tables.RevisionExternalLinks(wikitext_parser))
+
+        if citations:
+            table.columns.append(tables.RevisionCitations(wikitext_parser))
+
+        if wikilinks:
+            table.columns.append(tables.RevisionWikilinks(wikitext_parser))
+
+        if templates:
+            table.columns.append(tables.RevisionTemplates(wikitext_parser))
+
+        if headings:
+            table.columns.append(tables.RevisionHeadings(wikitext_parser))
+
+        table.columns.append(tables.RevisionParserTimeout(wikitext_parser))
+
+    return table, reverts_column, wikitext_parser
+
+
+def build_schema(
+    table,
+    diff: bool = False,
+    persist: int = 0,
+    text: bool = False,
+    regex_revision_pairs: list = None,
+    regex_comment_pairs: list = None,
+) -> pa.Schema:
+    """Build the PyArrow schema from a table, adding output-only fields."""
+    schema = table.schema()
+    schema = schema.append(pa.field("revert", pa.bool_(), nullable=True))
+
+    if diff:
+        from wikiq.diff_pyarrow_schema import diff_field
+        schema = schema.append(diff_field)
+        schema = schema.append(pa.field("diff_timeout", pa.bool_()))
+
+    if regex_revision_pairs:
+        for pair in regex_revision_pairs:
+            for field in pair.get_pyarrow_fields():
+                schema = schema.append(field)
+
+    if regex_comment_pairs:
+        for pair in regex_comment_pairs:
+            for field in pair.get_pyarrow_fields():
+                schema = schema.append(field)
+
+    if persist != PersistMethod.none:
+        # RevisionText is added to the table for extraction, but not to schema
+        # (unless text=True, in which case it's already in the schema from build_table)
+        schema = schema.append(pa.field("token_revs", pa.int64(), nullable=True))
+        schema = schema.append(pa.field("tokens_added", pa.int64(), nullable=True))
+        schema = schema.append(pa.field("tokens_removed", pa.int64(), nullable=True))
+        schema = schema.append(pa.field("tokens_window", pa.int64(), nullable=True))
+
+    return schema
+
+
+def make_regex_pairs(patterns, labels, count_only=False) -> list:
+    """Create RegexPair objects from patterns and labels."""
+    if (patterns is not None and labels is not None) and (len(patterns) == len(labels)):
+        return [RegexPair(pattern, label, count_only) for pattern, label in zip(patterns, labels)]
+    elif patterns is None and labels is None:
+        return []
+    else:
+        sys.exit("Each regular expression *must* come with a corresponding label and vice versa.")
+
+
+class JSONLWriter:
+    """Write JSONL output with schema validation."""
+
+    def __init__(self, output_file: str, schema: pa.Schema, append: bool = False):
+        self.output_file = output_file
+        self.schema = schema
+        self.field_names = [field.name for field in schema]
+
+        if append and os.path.exists(output_file):
+            self._validate_and_fix_last_line(output_file)
+
+        mode = "a" if append else "w"
+        self._file = open(output_file, mode)
+
+    def _validate_and_fix_last_line(self, filepath: str):
+        """Validate the last line of JSONL file; truncate if corrupted.
+
+        If the previous run was interrupted mid-write, the last line may be
+        incomplete JSON. This detects and removes such corrupted lines.
+        """
+        with open(filepath, 'rb') as f:
+            f.seek(0, 2)
+            file_size = f.tell()
+            if file_size == 0:
+                return
+
+            # Read backwards to find the last newline
+            chunk_size = min(8192, file_size)
+            f.seek(-chunk_size, 2)
+            chunk = f.read(chunk_size)
+
+            # Find the last complete line
+            last_newline = chunk.rfind(b'\n')
+            if last_newline == -1:
+                # Entire file is one line (possibly corrupted)
+                last_line = chunk.decode('utf-8', errors='replace')
+                truncate_pos = 0
+            else:
+                last_line = chunk[last_newline + 1:].decode('utf-8', errors='replace')
+                truncate_pos = file_size - chunk_size + last_newline + 1
+
+            # If last line is empty, file ends with newline - that's fine
+            if not last_line.strip():
+                return
+
+            # Try to parse the last line as JSON
+            try:
+                json.loads(last_line)
+            except json.JSONDecodeError:
+                print(f"Warning: Last line of {filepath} is corrupted JSON, removing it",
+                      file=sys.stderr)
+                # Truncate the file to remove the corrupted last line
+                with open(filepath, 'r+b') as f:
+                    f.truncate(truncate_pos)
+
+    def write_batch(self, data: dict):
+        """Write a batch of rows as JSONL.
+
+        Args:
+            data: dict mapping column names to lists of values
+        """
+        if not data or not data.get(self.field_names[0]):
+            return
+
+        num_rows = len(data[self.field_names[0]])
+        for i in range(num_rows):
+            row = {}
+            for name in self.field_names:
+                if name in data:
+                    value = data[name][i]
+                    row[name] = self._convert_value(value)
+            self._file.write(json.dumps(row) + "\n")
+
+    def _convert_value(self, value):
+        """Convert a value to JSON-serializable format."""
+        if value is None:
+            return None
+        elif isinstance(value, (str, int, float, bool)):
+            return value
+        elif hasattr(value, "isoformat"):
+            return value.isoformat()
+        elif isinstance(value, (list, tuple)):
+            return [self._convert_value(v) for v in value]
+        elif isinstance(value, dict):
+            return {k: self._convert_value(v) for k, v in value.items()}
+        else:
+            return str(value)
+
+    def close(self):
+        self._file.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+        return False
+
+
+class PersistMethod:
+    none = 0
+    sequence = 1
+    segment = 2
+    legacy = 3
+    wikidiff2 = 4
+
+
+def diff_with_timeout(differ, last_text, text):
+    """Returns (result, timed_out) tuple using native pywikidiff2 timeout."""
+    result = differ.inline_json_diff(last_text, text, timeout_ms=DIFF_TIMEOUT_MS)
+    return result, differ.timed_out()
+
+
+def calculate_persistence(tokens_added):
+    return (sum([(len(x.revisions) - 1) for x in tokens_added]), len(tokens_added))
+
+
+def fix_hex_digests(revs: list[mwxml.Revision]) -> list[mwxml.Revision]:
+    i = 0
+    for rev in revs:
+        if rev.text is None:
+            rev.text = ""
+        if not rev.sha1 and not rev.deleted.text:
+            rev.sha1 = sha1(bytes(rev.text, "utf8")).hexdigest()
+        revs[i] = rev
+        i += 1
+    return revs
+
+
+class WikiqIterator:
+    def __init__(self, fh, collapse_user=False):
+        self.fh = fh
+        self.collapse_user = collapse_user
+        self.mwiterator = Dump.from_file(self.fh)
+        self.namespace_map = {
+            ns.id: ns.name for ns in self.mwiterator.site_info.namespaces
+        }
+        self.__pages: Generator[WikiqPage] = self.load_pages()
+
+    def load_pages(self):
+        for page in self.mwiterator:
+            yield WikiqPage(
+                page, namespace_map=self.namespace_map, collapse_user=self.collapse_user
+            )
+
+    def __iter__(self):
+        return self.__pages
+
+    def __next__(self):
+        return next(self.__pages)
+
+
+class WikiqPage:
+    __slots__ = (
+        "id",
+        "redirect",
+        "restrictions",
+        "mwpage",
+        "__revisions",
+        "collapse_user",
+    )
+
+    def __init__(self, page, namespace_map, collapse_user=False):
+        self.id = page.id
+        # following mwxml, we assume namespace 0 in cases where
+        # page.namespace is inconsistent with namespace_map
+        if page.namespace not in namespace_map:
+            page.namespace = 0
+        # mwxml >= 0.3.7 keeps the namespace prefix on the title when the dump
+        # has <ns> tags, but strips it on the no-<ns> fallback path
+        if page.namespace != 0:
+            ns_prefix = namespace_map[page.namespace] + ":"
+            if not page.title.startswith(ns_prefix):
+                page.title = ns_prefix + page.title
+        self.restrictions = page.restrictions
+        self.collapse_user = collapse_user
+        self.mwpage = page
+        self.__revisions: Generator[list[mwxml.Revision]] = self.rev_list()
+
+    @staticmethod
+    def user_text(rev) -> Union[str, None]:
+        return None if rev.deleted.user else rev.user.text
+
+    def rev_list(self):
+        # Outline for how we want to handle collapse_user=True
+        # iteration   rev.user   prev_rev.user   add prev_rev?
+        #         0          A            None           Never
+        #         1          A               A           False
+        #         2          B               A            True
+        #         3          A               B            True
+        #         4          A               A           False
+        # Post-loop                          A          Always
+
+        if not self.collapse_user:
+            for rev in self.mwpage:
+                yield [rev]
+            return
+
+        for _, revs in groupby(self.mwpage, self.user_text):
+            # All revisions are either from the same user, or this is a single
+            # revision where the user is missing.
+            yield list(revs)
+
+    def __iter__(self):
+        return self.__revisions
+
+    def __next__(self):
+        return next(self.__revisions)
+
+
+"""
+A RegexPair is defined by a regular expression (pattern) and a label.
+The pattern can include capture groups.  If it does then each capture group will have a resulting column in the output.
+If the pattern does not include a capture group, then only one output column will result.
+"""
+
+
+class RegexPair(object):
+    def __init__(self, pattern, label, count_only=False):
+        self.pattern = re.compile(pattern)
+        self.label = label
+        self.count_only = count_only
+        self.has_groups = bool(self.pattern.groupindex)
+        if self.has_groups:
+            self.capture_groups = list(self.pattern.groupindex.keys())
+
+    def get_pyarrow_fields(self):
+        value_type = pa.int64() if self.count_only else pa.string()
+        if self.has_groups:
+            fields = [
+                pa.field(self._make_key(cap_group), value_type)
+                for cap_group in self.capture_groups
+            ]
+        else:
+            fields = [pa.field(self.label, value_type)]
+
+        return fields
+
+    def _make_key(self, cap_group):
+        return "{}_{}".format(self.label, cap_group)
+
+    def matchmake(self, content: str) -> dict:
+        temp_dict = {}
+        # if there are named capture groups in the regex
+        if self.has_groups:
+            # if there are matches of some sort in this revision content, fill the lists for each cap_group
+            # content can be None when the text or comment was deleted/suppressed
+            if content is not None and (matchobjects := list(self.pattern.finditer(content))):
+                for cap_group in self.capture_groups:
+                    key = self._make_key(cap_group)
+                    temp_list = []
+                    for match in matchobjects:
+                        # we only want to add the match for the capture group if the match is not None
+                        if (group := match.group(cap_group)) is not None:
+                            temp_list.append(group)
+
+                    # if temp_list of matches is empty just make that column None (0 in count mode)
+                    if len(temp_list) == 0:
+                        if self.count_only:
+                            temp_dict[key] = 0
+                        else:
+                            temp_dict[key] = None
+                    # else we put in the list we made in the for-loop above
+                    else:
+                        if self.count_only:
+                            temp_dict[key] = len(temp_list)
+                        else:
+                            temp_dict[key] = ", ".join(temp_list)
+
+            # there are no matches at all in this revision content, we default values to None (0 in count mode)
+            else:
+                for cap_group in self.capture_groups:
+                    key = self._make_key(cap_group)
+                    if self.count_only:
+                        temp_dict[key] = 0
+                    else:
+                        temp_dict[key] = None
+
+        # there are no capture groups, we just search for all the matches of the regex
+        else:
+            # given that there are matches to be made
+            if content is not None and (matches := self.pattern.findall(content)):
+                if self.count_only:
+                    temp_dict[self.label] = len(matches)
+                else:
+                    temp_dict[self.label] = ", ".join(matches)
+            else:
+                if self.count_only:
+                    temp_dict[self.label] = 0
+                else:
+                    temp_dict[self.label] = None
+
+        return temp_dict
+
+
+class WikiqParser:
+    def __init__(
+        self,
+        input_file: Union[TextIOWrapper, IO[Any], IO[bytes]],
+        output_file: Union[TextIO, str],
+        regex_match_revision: list[str],
+        regex_match_comment: list[str],
+        regex_revision_label: list[str],
+        regex_comment_label: list[str],
+        regex_revision_output_count: bool = False,
+        regex_comment_output_count: bool = False,
+        text: bool = False,
+        diff: bool = False,
+        collapse_user: bool = False,
+        persist: int = None,
+        namespaces: Union[list[int], None] = None,
+        revert_radius: int = 15,
+        output_jsonl: bool = False,
+        output_jsonl_dir: bool = False,
+        batch_size: int = 1024,
+        resume_point: Union[tuple, None] = None,
+        external_links: bool = False,
+        citations: bool = False,
+        wikilinks: bool = False,
+        templates: bool = False,
+        headings: bool = False,
+        redirect_aliases: Union[list[str], None] = None,
+        time_limit_seconds: Union[float, None] = None,
+        input_filename: Union[str, None] = None,
+    ):
+        """
+        Parameters:
+           persist : what persistence method to use. Takes a PersistMethod value
+           resume_point : if set, a (pageid, revid) tuple; skip all revisions up
+                          to and including this point.
+           input_filename : original input filename (needed for .jsonl.d output to derive output filename)
+        """
+        self.input_file = input_file
+        self.input_filename = input_filename
+
+        self.collapse_user: bool = collapse_user
+        self.persist: int = persist
+        self.namespaces = []
+        self.revert_radius = revert_radius
+        self.diff = diff
+        self.text = text
+        self.resume_point = resume_point
+        self.external_links = external_links
+        self.citations = citations
+        self.wikilinks = wikilinks
+        self.templates = templates
+        self.headings = headings
+        self.redirect_aliases = redirect_aliases
+        self.shutdown_requested = False
+        self.time_limit_seconds = time_limit_seconds
+        if namespaces is not None:
+            self.namespace_filter = set(namespaces)
+        else:
+            self.namespace_filter = None
+
+        self.regex_schemas = []
+        self.regex_revision_pairs: list[RegexPair] = self.make_matchmake_pairs(
+            regex_match_revision, regex_revision_label, regex_revision_output_count
+        )
+        self.regex_comment_pairs: list[RegexPair] = self.make_matchmake_pairs(
+            regex_match_comment, regex_comment_label, regex_comment_output_count
+        )
+
+        # Initialize output
+        self.batch_size = batch_size
+        self.output_jsonl = output_jsonl
+        self.output_jsonl_dir = output_jsonl_dir
+        self.output_file = output_file
+
+        if output_jsonl:
+            pass  # JSONLWriter created in process()
+        else:
+            # TSV output
+            self.print_header = True
+            if output_file == sys.stdout.buffer:
+                pass
+            else:
+                self.output_file = open(output_file, "wb")
+
+    def request_shutdown(self):
+        """Request graceful shutdown. The process() method will exit after completing the current batch."""
+        self.shutdown_requested = True
+
+    def _time_limit_expired(self):
+        """Timer callback when time limit is reached."""
+        hours = self.time_limit_seconds / 3600
+        print(f"Time limit of {hours:.2f} hours reached, requesting shutdown...", file=sys.stderr)
+        self.request_shutdown()
+
+    def _start_time_limit_timer(self):
+        """Start a background timer to trigger shutdown when time limit is reached."""
+        if self.time_limit_seconds is None:
+            return None
+        timer = threading.Timer(self.time_limit_seconds, self._time_limit_expired)
+        timer.daemon = True
+        timer.start()
+        return timer
+
+    def _cancel_time_limit_timer(self, timer):
+        """Cancel the time limit timer if it's still running."""
+        if timer is not None:
+            timer.cancel()
+
+    def make_matchmake_pairs(self, patterns, labels, count_only=False) -> list[RegexPair]:
+        if (patterns is not None and labels is not None) and (
+            len(patterns) == len(labels)
+        ):
+            result: list[RegexPair] = []
+            for pattern, label in zip(patterns, labels):
+                rp = RegexPair(pattern, label, count_only)
+                result.append(rp)
+                self.regex_schemas = self.regex_schemas + rp.get_pyarrow_fields()
+            return result
+        elif (patterns is None) and (labels is None):
+            return []
+        else:
+            sys.exit(
+                "Each regular expression *must* come with a corresponding label and vice versa."
+            )
+
+    def matchmake_revision(self, rev: mwxml.Revision):
+        result = self.matchmake_text(rev.text)
+        for k, v in self.matchmake_comment(rev.comment).items():
+            result[k] = v
+        return result
+
+    def matchmake_text(self, text: str):
+        return self.matchmake_pairs(text, self.regex_revision_pairs)
+
+    def matchmake_comment(self, comment: str):
+        return self.matchmake_pairs(comment, self.regex_comment_pairs)
+
+    @staticmethod
+    def matchmake_pairs(text, pairs):
+        result = {}
+        for pair in pairs:
+            for k, v in pair.matchmake(text).items():
+                result[k] = v
+        return result
+
+    def __get_namespace_from_title(self, title):
+        default_ns = None
+
+        for ns in self.namespaces:
+            # skip if the namespace is not defined
+            if ns is None:
+                default_ns = self.namespaces[ns]
+                continue
+
+            if title.startswith(ns + ":"):
+                return self.namespaces[ns]
+
+        # if we've made it this far with no matches, we return the default namespace
+        return default_ns
+
+    def process(self):
+        # Start time limit timer if configured
+        time_limit_timer = self._start_time_limit_timer()
+
+        # Track whether we've passed the resume point
+        found_resume_point = self.resume_point is None
+
+        # Construct dump file iterator
+        dump = WikiqIterator(self.input_file, collapse_user=self.collapse_user)
+
+        table, reverts_column, wikitext_parser = build_table(
+            text=self.text,
+            collapse_user=self.collapse_user,
+            external_links=self.external_links,
+            citations=self.citations,
+            wikilinks=self.wikilinks,
+            templates=self.templates,
+            headings=self.headings,
+            redirect_aliases=self.redirect_aliases,
+        )
+
+        # Extract list of namespaces
+        self.namespaces = {
+            ns.name: ns.id for ns in dump.mwiterator.site_info.namespaces
+        }
+
+        page_count = 0
+        rev_count = 0
+
+        schema = build_schema(
+            table,
+            diff=self.diff,
+            persist=self.persist,
+            text=self.text,
+            regex_revision_pairs=self.regex_revision_pairs,
+            regex_comment_pairs=self.regex_comment_pairs,
+        )
+
+        # Add RevisionText to table for diff/persist computation (extraction only, not output)
+        if (self.diff or self.persist != PersistMethod.none) and not self.text:
+            table.columns.append(tables.RevisionText())
+
+        # Initialize writer
+        writer = None
+
+        if self.output_jsonl:
+            append_mode = self.resume_point is not None
+            if self.output_jsonl_dir:
+                # Create directory for JSONL output
+                Path(self.output_file).mkdir(parents=True, exist_ok=True)
+                # Derive JSONL filename from input filename
+                if self.input_filename:
+                    jsonl_basename = os.path.basename(get_output_filename(self.input_filename, 'jsonl'))
+                else:
+                    jsonl_basename = "data.jsonl"
+                jsonl_path = Path(self.output_file) / jsonl_basename
+                print(f"Writing to JSONL: {jsonl_path} (append={append_mode})", file=sys.stderr)
+                writer = JSONLWriter(str(jsonl_path), schema, append=append_mode)
+            else:
+                writer = JSONLWriter(self.output_file, schema, append=append_mode)
+        else:
+            writer = pacsv.CSVWriter(
+                self.output_file,
+                schema,
+                write_options=pacsv.WriteOptions(delimiter="\t"),
+            )
+
+        # Initialize diff machinery
+        differ = None
+        fast_differ = None
+        if self.diff:
+            pywikidiff2 = require_pywikidiff2("--diff")
+            differ = pywikidiff2.pywikidiff2(
+                num_context_lines=1000000,
+                max_word_level_diff_complexity=-1,
+                moved_paragraph_detection_cutoff=-1,
+                words_cache_capacity=10000,
+                diff_cache_capacity=10000,
+                stats_cache_capacity=10000,
+            )
+            fast_differ = pywikidiff2.pywikidiff2(
+                num_context_lines=1000000,
+                max_word_level_diff_complexity=40000000,
+                moved_paragraph_detection_cutoff=100,
+                words_cache_capacity=-1,
+                diff_cache_capacity=-1,
+                stats_cache_capacity=-1,
+            )
+
+        # Write buffer: accumulate rows before flushing
+        write_buffer = defaultdict(list)
+        buffer_count = 0
+
+        def flush_buffer():
+            nonlocal write_buffer, buffer_count
+            if buffer_count == 0:
+                return
+            row_buffer = dict(write_buffer)
+            if self.output_jsonl:
+                writer.write_batch(row_buffer)
+            else:
+                writer.write(pa.record_batch(row_buffer, schema=schema))
+            write_buffer = defaultdict(list)
+            buffer_count = 0
+
+        # Iterate through pages
+        for page in dump:
+            # Skip namespaces not in the filter
+            if self.namespace_filter is not None:
+                if page.mwpage.namespace not in self.namespace_filter:
+                    continue
+
+            # Resume logic: skip pages before the resume point
+            is_resume_page = False
+            page_resume_revid = None
+            if self.resume_point is not None and not found_resume_point:
+                page_id = page.mwpage.id
+                resume_pageid, resume_revid = self.resume_point[0], self.resume_point[1]
+                if page_id < resume_pageid:
+                    continue
+                elif page_id == resume_pageid:
+                    is_resume_page = True
+                    page_resume_revid = resume_revid
+                else:
+                    found_resume_point = True
+
+            # Reset revert detector for new page
+            if self.revert_radius > 0:
+                reverts_column.rev_detector = mwreverts.Detector(radius=self.revert_radius)
+            else:
+                reverts_column.rev_detector = None
+
+            # State for this page
+            prev_text = ""
+            persist_state = None
+            persist_window = None
+
+            # Adaptive parse-skip: once we observe a parse timeout on this
+            # page, record the smallest revision text size that timed out;
+            # subsequent revisions at or above that size short-circuit the
+            # parse instead of waiting for another 60s SIGALRM. Resets per
+            # page so this only suppresses pages that have already proven
+            # pathological (e.g. zhwiki bot-vandalism log pages).
+            page_min_timeout_size: Union[int, None] = None
+            page_skipped_parses = 0
+
+            if self.persist != PersistMethod.none:
+                persist_window = deque(maxlen=PERSISTENCE_RADIUS)
+                if self.persist == PersistMethod.sequence:
+                    persist_state = mwpersistence.DiffState(
+                        SequenceMatcher(tokenizer=wikitext_split),
+                        revert_radius=PERSISTENCE_RADIUS,
+                    )
+                elif self.persist == PersistMethod.segment:
+                    persist_state = mwpersistence.DiffState(
+                        SegmentMatcher(tokenizer=wikitext_split),
+                        revert_radius=PERSISTENCE_RADIUS,
+                    )
+                elif self.persist == PersistMethod.wikidiff2:
+                    require_pywikidiff2("-p wikidiff2")
+                    from wikiq.wiki_diff_matcher import WikiDiffMatcher
+                    wikidiff_matcher = WikiDiffMatcher(tokenizer=wikitext_split)
+                    persist_state = mwpersistence.DiffState(
+                        wikidiff_matcher, revert_radius=PERSISTENCE_RADIUS
+                    )
+                else:
+                    persistence = require_mw_persistence()
+                    persist_state = persistence.State()
+
+            # Pending persistence values waiting for window to fill
+            pending_persistence = []
+
+            # Use peekable to detect last revision in page
+            revs_iter = peekable(page)
+
+            for revs in revs_iter:
+                # revs is either a single revision or a group (collapse_user mode)
+                revs = list(revs)
+                revs = fix_hex_digests(revs)
+                rev = revs[-1]  # Last revision in the group
+                is_last_in_page = revs_iter.peek(None) is None
+
+                # Skip revisions before resume point
+                if is_resume_page:
+                    if rev.id <= page_resume_revid:
+                        # Update state for correctness when we resume output
+                        if self.diff or self.persist != PersistMethod.none:
+                            prev_text = rev.text or ""
+                        if persist_state is not None:
+                            text = rev.text or ""
+                            if self.persist != PersistMethod.legacy:
+                                persist_state.update(text, rev.id)
+                            else:
+                                persist_state.process(text, rev.id)
+                        # Update revert detector so it has history for post-resume revisions
+                        if reverts_column.rev_detector is not None and not rev.deleted.text:
+                            reverts_column.rev_detector.process(rev.sha1, rev.id)
+                        if rev.id == page_resume_revid:
+                            found_resume_point = True
+                            is_resume_page = False
+                            print(f"Resuming output after revid {rev.id}", file=sys.stderr, flush=True)
+                        continue
+
+                rev_count += 1
+
+                # If this page has already produced a parse timeout, force
+                # the cache to "timed out" for any same-or-larger revision so
+                # extract_row returns nulls without spending another 60s on
+                # mwparserfromhell. See page_min_timeout_size init above.
+                if (
+                    wikitext_parser is not None
+                    and page_min_timeout_size is not None
+                    and rev.text is not None
+                    and len(rev.text) >= page_min_timeout_size
+                ):
+                    wikitext_parser.force_timeout(rev.text)
+                    page_skipped_parses += 1
+
+                # Extract base row data
+                row = table.extract_row(page.mwpage, revs)
+
+                # Record the smallest size at which this page has timed out
+                # so far, so future revisions at >= that size are auto-skipped.
+                if (
+                    wikitext_parser is not None
+                    and wikitext_parser.last_parse_timed_out
+                    and rev.text is not None
+                ):
+                    rev_size = len(rev.text)
+                    if page_min_timeout_size is None or rev_size < page_min_timeout_size:
+                        page_min_timeout_size = rev_size
+                        print(
+                            f"  parse timeout on pageid={page.mwpage.id} "
+                            f"title={page.mwpage.title!r} revid={rev.id} "
+                            f"text_bytes={rev_size:,} — will skip future revs "
+                            f">= {rev_size:,} bytes on this page",
+                            file=sys.stderr, flush=True,
+                        )
+
+                # Compute revert flag
+                if self.revert_radius == 0 or row["deleted"]:
+                    row["revert"] = None
+                else:
+                    row["revert"] = row["reverteds"] is not None
+
+                # Regex matching
+                regex_dict = self.matchmake_revision(rev)
+                for k, v in regex_dict.items():
+                    row[k] = v
+
+                # Compute diff
+                text = row.get("text", "") or ""
+                if self.diff:
+                    diff_result, timed_out = diff_with_timeout(differ, prev_text, text)
+                    if timed_out:
+                        print(f"WARNING! wikidiff2 timeout for rev: {rev.id}. Falling back to default limits.", file=sys.stderr)
+                        diff_result = fast_differ.inline_json_diff(prev_text, text)
+                    row["diff"] = [entry for entry in json.loads(diff_result)["diff"] if entry["type"] != 0]
+                    row["diff_timeout"] = timed_out
+
+                # Compute persistence
+                if persist_state is not None:
+                    if self.persist != PersistMethod.legacy:
+                        _, tokens_added, tokens_removed = persist_state.update(text, rev.id)
+                    else:
+                        _, tokens_added, tokens_removed = persist_state.process(text, rev.id)
+
+                    persist_window.append((rev.id, tokens_added, tokens_removed))
+                    pending_persistence.append(row)
+
+                    # When window is full, emit persistence for oldest revision
+                    if len(persist_window) == PERSISTENCE_RADIUS:
+                        old_rev_id, old_tokens_added, old_tokens_removed = persist_window.popleft()
+                        oldest_row = pending_persistence.pop(0)
+                        num_token_revs, num_tokens = calculate_persistence(old_tokens_added)
+                        oldest_row["token_revs"] = num_token_revs
+                        oldest_row["tokens_added"] = num_tokens
+                        oldest_row["tokens_removed"] = len(old_tokens_removed)
+                        oldest_row["tokens_window"] = PERSISTENCE_RADIUS - 1
+
+                        # Remove text if not outputting it
+                        if not self.text and "text" in oldest_row:
+                            del oldest_row["text"]
+
+                        # Add to write buffer
+                        for k, v in oldest_row.items():
+                            write_buffer[k].append(v)
+                        buffer_count += 1
+
+                        if buffer_count >= self.batch_size:
+                            flush_buffer()
+
+                # Update prev_text for next iteration
+                if self.diff or self.persist != PersistMethod.none:
+                    prev_text = text
+
+                # If no persistence, write row directly
+                if persist_state is None:
+                    if not self.text and "text" in row:
+                        del row["text"]
+
+                    for k, v in row.items():
+                        write_buffer[k].append(v)
+                    buffer_count += 1
+
+                    if buffer_count >= self.batch_size:
+                        flush_buffer()
+
+                # Check for shutdown
+                if self.shutdown_requested:
+                    print("Shutdown requested, closing writers...", file=sys.stderr)
+                    break
+
+            # End of page: flush remaining persistence window
+            if persist_state is not None and not self.shutdown_requested:
+                for i, (pending_row, window_item) in enumerate(zip(pending_persistence, persist_window)):
+                    rev_id, tokens_added, tokens_removed = window_item
+                    num_token_revs, num_tokens = calculate_persistence(tokens_added)
+                    pending_row["token_revs"] = num_token_revs
+                    pending_row["tokens_added"] = num_tokens
+                    pending_row["tokens_removed"] = len(tokens_removed)
+                    pending_row["tokens_window"] = len(persist_window) - (i + 1)
+
+                    if not self.text and "text" in pending_row:
+                        del pending_row["text"]
+
+                    for k, v in pending_row.items():
+                        write_buffer[k].append(v)
+                    buffer_count += 1
+
+            if self.shutdown_requested:
+                break
+            page_count += 1
+            if page_skipped_parses > 0:
+                print(
+                    f"  pageid={page.mwpage.id} done, skipped {page_skipped_parses} "
+                    f"parse(s) after timeout at {page_min_timeout_size:,} bytes",
+                    file=sys.stderr, flush=True,
+                )
+
+        # Flush remaining buffer
+        flush_buffer()
+
+        # Cancel time limit timer
+        self._cancel_time_limit_timer(time_limit_timer)
+
+        print(
+            "Done: %s revisions and %s pages." % (rev_count, page_count),
+            file=sys.stderr,
+        )
+
+        # Close all writers
+        if writer is not None:
+            writer.close()
+
+def match_archive_suffix(input_filename):
+    if re.match(r".*\.7z$", input_filename):
+        cmd = ["7za", "x", "-so", input_filename]
+    elif re.match(r".*\.gz$", input_filename):
+        cmd = ["zcat", input_filename]
+    elif re.match(r".*\.bz2$", input_filename):
+        cmd = ["bzcat", "-dk", input_filename]
+    else:
+        raise ValueError("Unrecognized file type: %s" % input_filename)
+    return cmd
+
+
+def open_input_file(input_filename, fandom_2020=False):
+    cmd = match_archive_suffix(input_filename)
+    if fandom_2020:
+        cmd.append("*.xml")
+    try:
+        return Popen(cmd, stdout=PIPE).stdout
+    except NameError:
+        return open(input_filename, "r")
+
+
+def get_output_filename(input_filename, output_format='tsv') -> str:
+    """Generate output filename based on input filename and format.
+
+    Args:
+        input_filename: Input dump file path
+        output_format: 'tsv' or 'jsonl'
+    """
+    output_filename = re.sub(r"\.(7z|gz|bz2)?$", "", input_filename)
+    output_filename = re.sub(r"\.xml", "", output_filename)
+    if output_format == 'jsonl':
+        output_filename = output_filename + ".jsonl"
+    else:
+        output_filename = output_filename + ".tsv"
+    return output_filename
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Parse MediaWiki XML database dumps into tab delimited data."
+    )
+
+    # arguments for the input direction
+    parser.add_argument(
+        "dumpfiles",
+        metavar="DUMPFILE",
+        nargs="*",
+        type=str,
+        help="Filename of the compressed or uncompressed XML database dump. If absent, we'll look for content on stdin and output on stdout.",
+    )
+
+    parser.add_argument(
+        "-o",
+        "--output",
+        metavar="OUTPUT",
+        dest="output",
+        type=str,
+        nargs=1,
+        help="Output file or directory. Format is detected from extension: .jsonl for JSONL, otherwise TSV.",
+    )
+
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"wikiq {__version__}",
+    )
+
+    parser.add_argument(
+        "-s",
+        "--stdout",
+        dest="stdout",
+        action="store_true",
+        help="Write output to standard out (do not create dump file)",
+    )
+
+    parser.add_argument(
+        "--print-schema",
+        dest="print_schema",
+        action="store_true",
+        help="Print the Spark-compatible JSON schema for the output and exit. No dump file is processed.",
+    )
+
+    parser.add_argument(
+        "--collapse-user",
+        dest="collapse_user",
+        action="store_true",
+        help="Operate only on the final revision made by user a user within all sequences of consecutive edits made by a user. This can be useful for addressing issues with text persistence measures.",
+    )
+
+    parser.add_argument(
+        "-p",
+        "--persistence",
+        dest="persist",
+        default=None,
+        const="",
+        type=str,
+        choices=["", "wikidiff2", "segment", "sequence", "legacy"],
+        nargs="?",
+        help="Compute and report measures of content persistent: (1) persistent token revisions, (2) tokens added, and (3) number of revision used in computing the first measure. This may by slow. The default is no persistence. -p=sequence, which uses the same algorithm as in the past, but with improvements to wikitext parsing. Use -p=legacy for old behavior used in older research projects. -p=segment attempts advanced persistence calculation method that is robust to content moves, but prone to bugs, and slower. -p=wikidiff2 is like segment, but uses the wikidiff2 algorithm, which (should be) faster and more robust.",
+    )
+
+    parser.add_argument(
+        "-n",
+        "--namespace-include",
+        dest="namespace_filter",
+        type=int,
+        action="append",
+        help="Id number of namespace to include. Can be specified more than once.",
+    )
+
+    parser.add_argument(
+        "-rr",
+        "--revert-radius",
+        dest="revert_radius",
+        type=int,
+        action="store",
+        default=15,
+        help="Number of edits to check when looking for reverts (default: 15)",
+    )
+
+    parser.add_argument(
+        "-RP",
+        "--revision-pattern",
+        dest="regex_match_revision",
+        default=None,
+        type=str,
+        action="append",
+        help="The regular expression to search for in revision text. The regex must be surrounded by quotes.",
+    )
+
+    parser.add_argument(
+        "-RPl",
+        "--revision-pattern-label",
+        dest="regex_revision_label",
+        default=None,
+        type=str,
+        action="append",
+        help="The label for the outputted column based on matching the regex in revision text.",
+    )
+
+    parser.add_argument(
+        "-RPc",
+        "--revision-pattern-count",
+        dest="regex_revision_output_count",
+        action="store_true",
+        help="If present, this will cause the revision patterns to return counts of the number of matches instead of the text of the matches themselves. It will affect all revision patterns.",
+    )
+
+    parser.add_argument(
+        "-CP",
+        "--comment-pattern",
+        dest="regex_match_comment",
+        default=None,
+        type=str,
+        action="append",
+        help="The regular expression to search for in comments of revisions.",
+    )
+
+    parser.add_argument(
+        "-CPl",
+        "--comment-pattern-label",
+        dest="regex_comment_label",
+        default=None,
+        type=str,
+        action="append",
+        help="The label for the outputted column based on matching the regex in comments.",
+    )
+
+    parser.add_argument(
+        "-CPc",
+        "--comment-pattern-count",
+        dest="regex_comment_output_count",
+        action="store_true",
+        help="If present, this will cause the comment patterns to return counts of the number of matches instead of the text of the matches themselves. It will affect all comment patterns.",
+    )
+
+    parser.add_argument(
+        "-d",
+        "--diff",
+        dest="diff",
+        default=False,
+        action="store_true",
+        help="Output a diff structure for each revision with information about changed or moved lines.",
+    )
+
+    parser.add_argument(
+        "-t",
+        "--text",
+        dest="text",
+        default=False,
+        action="store_true",
+        help="Output the text of the revision.",
+    )
+
+    parser.add_argument(
+        "--external-links",
+        dest="external_links",
+        action="store_true",
+        default=False,
+        help="Extract external links from each revision using mwparserfromhell.",
+    )
+
+    parser.add_argument(
+        "--citations",
+        dest="citations",
+        action="store_true",
+        default=False,
+        help="Extract citations (ref tags and cite templates) from each revision.",
+    )
+
+    parser.add_argument(
+        "--wikilinks",
+        dest="wikilinks",
+        action="store_true",
+        default=False,
+        help="Extract internal wikilinks from each revision.",
+    )
+
+    parser.add_argument(
+        "--templates",
+        dest="templates",
+        action="store_true",
+        default=False,
+        help="Extract templates with their parameters from each revision.",
+    )
+
+    parser.add_argument(
+        "--headings",
+        dest="headings",
+        action="store_true",
+        default=False,
+        help="Extract section headings from each revision.",
+    )
+
+    parser.add_argument(
+        "--redirect-aliases",
+        dest="redirect_aliases",
+        default=None,
+        type=str,
+        action="append",
+        help="Additional localized redirect keyword recognized alongside #REDIRECT when detecting redirect revisions (e.g. OMDIRIGERING on Swedish wikis). May be given multiple times or as a comma-separated list.",
+    )
+
+    parser.add_argument(
+        "--fandom-2020",
+        dest="fandom_2020",
+        action="store_true",
+        help="Whether the archive is from the fandom 2020 dumps by Wikiteam. These dumps can have multiple .xml files in their archives.",
+    )
+
+    parser.add_argument(
+        "--batch-size",
+        dest="batch_size",
+        default=1500,
+        type=int,
+        help="How many revisions to process in each batch.",
+    )
+
+    parser.add_argument(
+        "--resume",
+        dest="resume",
+        action="store_true",
+        help="Resume processing from the last successfully written revision in the output file.",
+    )
+
+    parser.add_argument(
+        "--time-limit",
+        dest="time_limit",
+        type=float,
+        default=0,
+        help="Time limit in hours before graceful shutdown. Set to 0 to disable (default).",
+    )
+
+    args = parser.parse_args()
+
+    # set persistence method
+
+    if args.persist is None:
+        persist = PersistMethod.none
+    elif args.persist == "segment":
+        persist = PersistMethod.segment
+    elif args.persist == "legacy":
+        persist = PersistMethod.legacy
+    elif args.persist == "wikidiff2":
+        persist = PersistMethod.wikidiff2
+    else:
+        persist = PersistMethod.sequence
+
+    # Check for the optional dependencies up front. Both are used deep in the
+    # per-revision loop, and a run can stream for hours before reaching them.
+    if args.diff:
+        require_pywikidiff2("--diff")
+    if persist == PersistMethod.wikidiff2:
+        require_pywikidiff2("-p wikidiff2")
+    elif persist == PersistMethod.legacy:
+        require_mw_persistence()
+
+    if args.namespace_filter is not None:
+        namespaces = args.namespace_filter
+    else:
+        namespaces = None
+
+    # --redirect-aliases may be given multiple times or comma-separated
+    redirect_aliases = None
+    if args.redirect_aliases:
+        redirect_aliases = [
+            alias.strip()
+            for arg in args.redirect_aliases
+            for alias in arg.split(",")
+            if alias.strip()
+        ]
+
+    # Handle --print-schema: build and output schema, then exit
+    if args.print_schema:
+        regex_revision_pairs = make_regex_pairs(args.regex_match_revision, args.regex_revision_label, args.regex_revision_output_count)
+        regex_comment_pairs = make_regex_pairs(args.regex_match_comment, args.regex_comment_label, args.regex_comment_output_count)
+
+        table, _, _ = build_table(
+            text=args.text,
+            collapse_user=args.collapse_user,
+            external_links=args.external_links,
+            citations=args.citations,
+            wikilinks=args.wikilinks,
+            templates=args.templates,
+            headings=args.headings,
+            redirect_aliases=redirect_aliases,
+        )
+        schema = build_schema(
+            table,
+            diff=args.diff,
+            persist=persist,
+            text=args.text,
+            regex_revision_pairs=regex_revision_pairs,
+            regex_comment_pairs=regex_comment_pairs,
+        )
+
+        spark_schema = pyarrow_to_spark_schema(schema)
+        print(json.dumps(spark_schema, indent=2))
+        sys.exit(0)
+
+    print(f"wikiq {__version__}", file=sys.stderr)
+    print(args, file=sys.stderr)
+    if len(args.dumpfiles) > 0:
+        for filename in args.dumpfiles:
+            # Determine output file path before opening input (so resume errors are caught early)
+            if args.output:
+                output = args.output[0]
+            else:
+                output = "."
+
+            # Detect output format from extension
+            output_jsonl_dir = output.endswith(".jsonl.d")
+            output_jsonl = output.endswith(".jsonl") or output_jsonl_dir
+
+            if args.stdout:
+                output_file = sys.stdout.buffer
+            elif output_jsonl:
+                # Output is a JSONL file path - use it directly
+                output_file = output
+            elif os.path.isdir(output):
+                # Output is a directory - derive filename from input
+                output_filename = os.path.join(output, os.path.basename(filename))
+                output_file = get_output_filename(output_filename, output_format='tsv')
+            else:
+                output_file = output
+
+            # Handle resume functionality before opening input file
+            resume_point = None
+            if args.resume:
+                if output_jsonl and not args.stdout:
+                    # JSONL: get resume point from last line of file
+                    resume_point = get_resume_point(output_file, input_file=filename)
+                    if resume_point is not None:
+                        pageid, revid = resume_point[0], resume_point[1]
+                        print(f"Resuming from: pageid={pageid}, revid={revid}", file=sys.stderr)
+                else:
+                    sys.exit("Error: --resume only works with JSONL output (not stdout or TSV)")
+
+            # Now open the input file
+            print("Processing file: %s" % filename, file=sys.stderr)
+            input_file = open_input_file(filename, args.fandom_2020)
+
+            time_limit_seconds = args.time_limit * 3600 if args.time_limit > 0 else None
+
+            wikiq = WikiqParser(
+                input_file,
+                output_file,
+                collapse_user=args.collapse_user,
+                persist=persist,
+                namespaces=namespaces,
+                revert_radius=args.revert_radius,
+                regex_match_revision=args.regex_match_revision,
+                regex_revision_label=args.regex_revision_label,
+                regex_revision_output_count=args.regex_revision_output_count,
+                regex_match_comment=args.regex_match_comment,
+                regex_comment_label=args.regex_comment_label,
+                regex_comment_output_count=args.regex_comment_output_count,
+                text=args.text,
+                diff=args.diff,
+                output_jsonl=output_jsonl,
+                output_jsonl_dir=output_jsonl_dir,
+                batch_size=args.batch_size,
+                resume_point=resume_point,
+                external_links=args.external_links,
+                citations=args.citations,
+                wikilinks=args.wikilinks,
+                templates=args.templates,
+                headings=args.headings,
+                redirect_aliases=redirect_aliases,
+                time_limit_seconds=time_limit_seconds,
+                input_filename=filename,
+            )
+
+            # Register signal handlers for graceful shutdown (CLI only)
+            def handle_shutdown(signum, frame):
+                sig_name = signal.Signals(signum).name
+                print(f"\nReceived {sig_name}, requesting graceful shutdown...", file=sys.stderr)
+                wikiq.request_shutdown()
+
+            original_sigterm = signal.signal(signal.SIGTERM, handle_shutdown)
+            original_sigint = signal.signal(signal.SIGINT, handle_shutdown)
+            original_sigusr1 = signal.signal(signal.SIGUSR1, handle_shutdown)
+            original_sigusr2 = signal.signal(signal.SIGUSR2, handle_shutdown)
+
+            try:
+                wikiq.process()
+            finally:
+                # Restore original signal handlers
+                signal.signal(signal.SIGTERM, original_sigterm)
+                signal.signal(signal.SIGINT, original_sigint)
+                signal.signal(signal.SIGUSR1, original_sigusr1)
+                signal.signal(signal.SIGUSR2, original_sigusr2)
+
+            # close things
+            input_file.close()
+
+    else:
+        if args.resume:
+            print("Warning: --resume cannot be used with stdin/stdout", file=sys.stderr)
+
+        time_limit_seconds = args.time_limit * 3600 if args.time_limit > 0 else None
+
+        wikiq = WikiqParser(
+            sys.stdin,
+            sys.stdout,
+            collapse_user=args.collapse_user,
+            persist=persist,
+            # persist_legacy=args.persist_legacy,
+            namespaces=namespaces,
+            revert_radius=args.revert_radius,
+            regex_match_revision=args.regex_match_revision,
+            regex_revision_label=args.regex_revision_label,
+            regex_revision_output_count=args.regex_revision_output_count,
+            regex_match_comment=args.regex_match_comment,
+            regex_comment_label=args.regex_comment_label,
+            regex_comment_output_count=args.regex_comment_output_count,
+            diff=args.diff,
+            text=args.text,
+            batch_size=args.batch_size,
+            resume_point=None,
+            external_links=args.external_links,
+            citations=args.citations,
+            wikilinks=args.wikilinks,
+            templates=args.templates,
+            headings=args.headings,
+            redirect_aliases=redirect_aliases,
+            time_limit_seconds=time_limit_seconds,
+        )
+
+        # Register signal handlers for graceful shutdown (CLI only)
+        def handle_shutdown(signum, frame):
+            sig_name = signal.Signals(signum).name
+            print(f"\nReceived {sig_name}, requesting graceful shutdown...", file=sys.stderr)
+            wikiq.request_shutdown()
+
+        original_sigterm = signal.signal(signal.SIGTERM, handle_shutdown)
+        original_sigint = signal.signal(signal.SIGINT, handle_shutdown)
+        original_sigusr1 = signal.signal(signal.SIGUSR1, handle_shutdown)
+
+        try:
+            wikiq.process()
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGTERM, original_sigterm)
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGUSR1, original_sigusr1)
+
+    # stop_words = "a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your"
+    # stop_words = stop_words.split(",")
+
+
+if __name__ == "__main__":
+    main()