internet-archive-extractor 0.0.1__tar.gz

internet_archive_extractor-0.0.1/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: internet-archive-extractor
+Version: 0.0.1
+Summary: Tool for extracting archived web sites from the Internet Archive saving as WARC files.
+Author-email: Victor Harbo Johnston <vijo@cas.au.dk>
+License: MIT
+Project-URL: Homepage, https://github.com/WEB-CHILD/InternetArchiveExtractor
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: certifi
+Requires-Dist: charset-normalizer
+Requires-Dist: idna
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pysqlite3
+Requires-Dist: python-dateutil
+Requires-Dist: python-magic
+Requires-Dist: pytz
+Requires-Dist: pywaybackup
+Requires-Dist: requests
+Requires-Dist: six
+Requires-Dist: tqdm
+Requires-Dist: tzdata
+Requires-Dist: urllib3
+Requires-Dist: warcio
+
+# The project currently only supports converting output from Bitdruids WaybackUp program to be converted to WARC files.
+
+## WaybackupToWarc
+
+This project is designed to read a CSV file containing URLs, remove any instances of port 80 from those URLs, and generate WARC-GZ files based on the cleaned data.
+The CSV file can be constructed by using the following tool: [Python Wayback Machine Downloader](https://github.com/bitdruid/python-wayback-machine-downloader)
+
+## Project Structure
+
+```
+WaybackupToWarc
+├── src
+│   ├── main.py        # Main script for processing CSV and generating WARC files
+│   └── utils.py       # Utility functions for reading CSV and modifying URLs
+├── requirements.txt    # List of dependencies for the project
+└── README.md           # Documentation for the project
+```
+
+## Requirements
+
+To run this project, you need to install the following dependencies:
+
+- `warcio`: For creating WARC files.
+- `pandas`: For handling CSV data.
+
+You can install the required packages using pip:
+
+```
+pip install -r requirements.txt
+```
+
+## Usage
+
+1. Place your CSV file in the appropriate directory.
+2. Update the `src/main.py` file to specify the path to your CSV file.
+3. Run the main script:
+
+```
+python src/main.py
+```
+
+This will process the CSV file, remove port 80 from the URLs, and generate the corresponding WARC-GZ files.
+
+## Contributing
+
+Feel free to submit issues or pull requests if you have suggestions or improvements for the project.

internet_archive_extractor-0.0.1/README.md

@@ -0,0 +1,46 @@
+# The project currently only supports converting output from Bitdruids WaybackUp program to be converted to WARC files.
+
+## WaybackupToWarc
+
+This project is designed to read a CSV file containing URLs, remove any instances of port 80 from those URLs, and generate WARC-GZ files based on the cleaned data.
+The CSV file can be constructed by using the following tool: [Python Wayback Machine Downloader](https://github.com/bitdruid/python-wayback-machine-downloader)
+
+## Project Structure
+
+```
+WaybackupToWarc
+├── src
+│   ├── main.py        # Main script for processing CSV and generating WARC files
+│   └── utils.py       # Utility functions for reading CSV and modifying URLs
+├── requirements.txt    # List of dependencies for the project
+└── README.md           # Documentation for the project
+```
+
+## Requirements
+
+To run this project, you need to install the following dependencies:
+
+- `warcio`: For creating WARC files.
+- `pandas`: For handling CSV data.
+
+You can install the required packages using pip:
+
+```
+pip install -r requirements.txt
+```
+
+## Usage
+
+1. Place your CSV file in the appropriate directory.
+2. Update the `src/main.py` file to specify the path to your CSV file.
+3. Run the main script:
+
+```
+python src/main.py
+```
+
+This will process the CSV file, remove port 80 from the URLs, and generate the corresponding WARC-GZ files.
+
+## Contributing
+
+Feel free to submit issues or pull requests if you have suggestions or improvements for the project.

internet_archive_extractor-0.0.1/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "internet-archive-extractor"           
+version = "0.0.1"
+description = "Tool for extracting archived web sites from the Internet Archive saving as WARC files."
+readme = "README.md"
+license = { text = "MIT" }
+authors = [{ name = "Victor Harbo Johnston", email = "vijo@cas.au.dk" }]
+requires-python = ">=3.8"
+dependencies = [
+    "certifi",
+    "charset-normalizer",
+    "idna",
+    "numpy",
+    "pandas",
+    "pysqlite3",
+    "python-dateutil",
+    "python-magic",
+    "pytz",
+    "pywaybackup",
+    "requests",
+    "six",
+    "tqdm",
+    "tzdata",
+    "urllib3",
+    "warcio"
+]
+
+[project.urls]
+Homepage = "https://github.com/WEB-CHILD/InternetArchiveExtractor"

internet_archive_extractor-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

internet_archive_extractor-0.0.1/src/internet_archive_downloader.py

@@ -0,0 +1,106 @@
+from pywaybackup import PyWayBackup
+from wayback_date_object import WaybackDateObject
+import re
+
+from utils import import_urls_from_csv
+
+
+def get_wayback_date_and_archived_url(wayback_url: str):
+    """
+    Extracts the archive date and archived URL from a Wayback Machine URL.
+
+    Args:
+        wayback_url (str): The URL from the Wayback Machine in the format
+            'https://web.archive.org/web/<timestamp>/<archived_url>'.
+
+    Returns:
+        tuple: A tuple containing:
+            - date (WaybackDateObject): The extracted date as a WaybackDateObject.
+            - archived_url (str): The original URL archived by the Wayback Machine.
+
+    Raises:
+        AttributeError: If the input URL does not match the expected Wayback Machine format.
+    """
+    match = re.match(r"https://web\.archive\.org/web/(\d+)/(.*)", wayback_url)
+    if match:
+        date = WaybackDateObject(match.group(1))
+        archived_url = match.group(2)
+        return date, archived_url
+
+def download_urls_from_csv(csv_file_path: str, url_column_name: str):
+    """
+    Reads a CSV file containing Internet Archive URLs (eg. https://web.archive.org/web/20251002062751/https://cas.au.dk/erc-webchild),
+    retrieves their corresponding Wayback Machine archived URLs and dates, and downloads the archived content for each URL for a period of two weeks around the archived date.
+
+    Args:
+        csv_file_path (str): The file path to the CSV file containing the Internet Archive URLs.
+        url_column_name (str): The name of the column in the CSV file that contains the URLs.
+
+    Returns:
+        None
+
+    Side Effects:
+        - Downloads archived content for each URL from the Wayback Machine.
+        - Handles and prints TypeError exceptions that may occur during download.
+    """
+    internet_archive_urls = import_urls_from_csv(csv_file_path, url_column_name)
+
+    for url in internet_archive_urls:
+        wayback_date, archived_url = get_wayback_date_and_archived_url(url)
+
+        end_date = WaybackDateObject(wayback_date.wayback_format())
+        end_date.increment_week()
+
+        start_date = WaybackDateObject(wayback_date.wayback_format())
+        start_date.decrement_week()
+
+        try:
+            download_single_url(archived_url, start_date.wayback_format(), end_date.wayback_format())
+        except TypeError as e:
+            print()
+        
+    
+def download_single_url(url: str, start_date: str, end_date: str):
+    """
+    Downloads all available snapshots of a given URL from the Internet Archive's Wayback Machine within a specified date range.
+
+    Args:
+        url (str): The URL to download snapshots for.
+        start_date (str): The start date (inclusive) in 'YYYYMMDD' format.
+        end_date (str): The end date (inclusive) in 'YYYYMMDD' format.
+
+    Returns:
+        None
+
+    Side Effects:
+        - Prints progress and debug information to the console.
+        - Downloads and saves the snapshots to disk.
+        - Prints the relative paths of the downloaded snapshots.
+    """
+
+    print(f"Downloading {url} from {start_date} to {end_date}")
+    backup = PyWayBackup(
+    url=url,
+    all=True,
+    start=start_date,
+    end=end_date,
+    silent=False,
+    debug=True,
+    log=True,
+    keep=True,
+    workers=1,
+    reset=False,
+    explicit=True
+    )
+
+    backup.run()
+    backup_paths = backup.paths(rel=True)
+    print(backup_paths)
+
+
+def main():
+    # Currently only doesnt support other files than the one presented here. Just need convertng to useing arguments.
+    download_urls_from_csv("./resources/curated_urls.csv", "Internet_Archive_URL")
+
+if __name__ == "__main__":
+    main()

internet_archive_extractor-0.0.1/src/internet_archive_extractor.egg-info/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: internet-archive-extractor
+Version: 0.0.1
+Summary: Tool for extracting archived web sites from the Internet Archive saving as WARC files.
+Author-email: Victor Harbo Johnston <vijo@cas.au.dk>
+License: MIT
+Project-URL: Homepage, https://github.com/WEB-CHILD/InternetArchiveExtractor
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: certifi
+Requires-Dist: charset-normalizer
+Requires-Dist: idna
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pysqlite3
+Requires-Dist: python-dateutil
+Requires-Dist: python-magic
+Requires-Dist: pytz
+Requires-Dist: pywaybackup
+Requires-Dist: requests
+Requires-Dist: six
+Requires-Dist: tqdm
+Requires-Dist: tzdata
+Requires-Dist: urllib3
+Requires-Dist: warcio
+
+# The project currently only supports converting output from Bitdruids WaybackUp program to be converted to WARC files.
+
+## WaybackupToWarc
+
+This project is designed to read a CSV file containing URLs, remove any instances of port 80 from those URLs, and generate WARC-GZ files based on the cleaned data.
+The CSV file can be constructed by using the following tool: [Python Wayback Machine Downloader](https://github.com/bitdruid/python-wayback-machine-downloader)
+
+## Project Structure
+
+```
+WaybackupToWarc
+├── src
+│   ├── main.py        # Main script for processing CSV and generating WARC files
+│   └── utils.py       # Utility functions for reading CSV and modifying URLs
+├── requirements.txt    # List of dependencies for the project
+└── README.md           # Documentation for the project
+```
+
+## Requirements
+
+To run this project, you need to install the following dependencies:
+
+- `warcio`: For creating WARC files.
+- `pandas`: For handling CSV data.
+
+You can install the required packages using pip:
+
+```
+pip install -r requirements.txt
+```
+
+## Usage
+
+1. Place your CSV file in the appropriate directory.
+2. Update the `src/main.py` file to specify the path to your CSV file.
+3. Run the main script:
+
+```
+python src/main.py
+```
+
+This will process the CSV file, remove port 80 from the URLs, and generate the corresponding WARC-GZ files.
+
+## Contributing
+
+Feel free to submit issues or pull requests if you have suggestions or improvements for the project.

internet_archive_extractor-0.0.1/src/internet_archive_extractor.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+src/internet_archive_downloader.py
+src/main.py
+src/utils.py
+src/wayback_date_object.py
+src/waybackup_to_warc.py
+src/internet_archive_extractor.egg-info/PKG-INFO
+src/internet_archive_extractor.egg-info/SOURCES.txt
+src/internet_archive_extractor.egg-info/dependency_links.txt
+src/internet_archive_extractor.egg-info/requires.txt
+src/internet_archive_extractor.egg-info/top_level.txt

internet_archive_extractor-0.0.1/src/internet_archive_extractor.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

internet_archive_extractor-0.0.1/src/internet_archive_extractor.egg-info/requires.txt

@@ -0,0 +1,16 @@
+certifi
+charset-normalizer
+idna
+numpy
+pandas
+pysqlite3
+python-dateutil
+python-magic
+pytz
+pywaybackup
+requests
+six
+tqdm
+tzdata
+urllib3
+warcio

internet_archive_extractor-0.0.1/src/internet_archive_extractor.egg-info/top_level.txt

@@ -0,0 +1,5 @@
+internet_archive_downloader
+main
+utils
+wayback_date_object
+waybackup_to_warc

internet_archive_extractor-0.0.1/src/main.py

@@ -0,0 +1,56 @@
+import sys
+import argparse
+from enum import Enum
+from waybackup_to_warc import combine_csv_files, process_csv_file, COMBINED_CSV_PATH
+from internet_archive_downloader import download_urls_from_csv
+
+parser = argparse.ArgumentParser(description="Internet Archive Extractor")
+
+parser.add_argument("mode", help="The mode to run the script in: 'download', 'convert' or 'full'.")
+parser.add_argument("input", help="The input file or directory path.")
+parser.add_argument("--output", help="The output file name for the generated WARC file. Only applicable for modes: 'convert' or 'full'.")
+parser.add_argument("--column_name", default="Internet_Archive_URL", help="The column name in the CSV file that contains the URLs for download. Default is 'Internet_Archive_URL'.")
+
+class Mode(Enum):
+    """
+    Enum for the different modes of operation.
+    """
+    FULL = 1
+    DOWNLOAD = 2
+    CONVERT = 3
+
+args = parser.parse_args()
+
+try:
+    Mode(args.mode.upper())  
+except ValueError:
+    try:
+        Mode[args.mode.upper()]  
+    except KeyError:
+        print(f"Invalid mode: {args.mode}. Choose from 'download', 'convert' or 'full'.")
+        sys.exit(1)
+
+def choose_mode():
+    if args.mode.upper() == Mode.DOWNLOAD.name:
+        print("Download mode selected.")
+        download_urls_from_csv(args.input, args.column_name)
+    elif args.mode.upper() == Mode.CONVERT.name:
+        print("Convert mode selected.")
+        combine_csv_files(args.input, COMBINED_CSV_PATH)
+        process_csv_file(COMBINED_CSV_PATH, 'output', args.output)
+    elif args.mode.upper() == Mode.FULL.name:
+        print("Full mode selected.")
+
+        download_urls_from_csv(args.input, args.column_name)
+        combine_csv_files("waybackup_snapshots", COMBINED_CSV_PATH)
+        process_csv_file(COMBINED_CSV_PATH, 'output', args.output)
+    else:
+        print(f"Invalid mode: {args.mode}. Choose from 'download', 'convert' or 'full'.")
+        sys.exit(1)
+
+def main():
+    choose_mode()
+
+
+if __name__ == "__main__":
+    main()

internet_archive_extractor-0.0.1/src/utils.py

@@ -0,0 +1,45 @@
+def read_csv(file_path):
+    import pandas as pd
+    return pd.read_csv(file_path, sep=";")
+
+def remove_port_80(url):
+    if ':80' in url:
+        return url.replace(':80', '')
+    return url
+
+def clean_urls(dataframe):
+    dataframe['url_archive'] = dataframe['url_archive'].apply(lambda x: remove_port_80(x))
+    dataframe['url_origin'] = dataframe['url_origin'].apply(lambda x: remove_port_80(x))
+    return dataframe
+
+def create_warc_gz(file_path, dataframe):
+    from warcio.archiveiterator import ArchiveIterator
+    from warcio.warcwriter import WARCWriter
+    import gzip
+
+    with gzip.open(file_path, 'wb') as stream:
+        writer = WARCWriter(stream, gzip=True)
+        for index, row in dataframe.iterrows():
+            writer.write_webpage(row['url_origin'], row['timestamp'], content_type='text/html')
+            writer.write_webpage(row['url_archive'], row['timestamp'], content_type='text/html')
+
+def import_urls_from_csv(file_path, column_name):
+    """
+    Imports URLs from a specified column in a CSV file.
+
+    Args:
+        file_path (str): The path to the CSV file.
+        column_name (str): The name of the column containing URLs.
+
+    Returns:
+        list: A list of URLs extracted from the specified column.
+
+    Raises:
+        FileNotFoundError: If the specified file does not exist.
+        KeyError: If the specified column does not exist in the CSV file.
+        pd.errors.EmptyDataError: If the CSV file is empty.
+    """
+
+    df = read_csv(file_path)
+    url_list = df[column_name].tolist()
+    return url_list

internet_archive_extractor-0.0.1/src/wayback_date_object.py

@@ -0,0 +1,89 @@
+from datetime import datetime, timedelta
+
+
+class WaybackDateObject:
+    def __init__(self, waybackdate_str):
+        self.year = waybackdate_str[0:4]
+        self.month = waybackdate_str[4:6]
+        self.day = waybackdate_str[6:8]
+        self.hour = waybackdate_str[8:10]
+        self.minute = waybackdate_str[10:12]
+        self.second = waybackdate_str[12:14]
+
+    @classmethod
+    def from_values(self, year, month, day, hour, minute, second):
+        self.year = year
+        self.month = month
+        self.day = day
+        self.hour = hour
+        self.minute = minute
+        self.second = second
+
+    def pretty_print(self):
+        """
+        Returns a human-readable string representation of the date and time.
+        Example: '2003-04-09 19:30:11'
+        """
+        return f"{self.year}-{self.month}-{self.day} {self.hour}:{self.minute}:{self.second}"
+    
+    def wayback_format(self):
+        """
+        Returns the date and time in the original Wayback date format.
+        Example: '20030409193011'
+        """
+        return f"{self.year}{self.month}{self.day}{self.hour}{self.minute}{self.second}"
+    
+    def increment_day(self):
+        """
+        Increments the day by one, adjusting month and year as necessary.
+        Note: This method does not account for leap years or varying month lengths.
+        """
+        day = int(self.day)
+        month = int(self.month)
+        year = int(self.year)
+
+        # Days in each month (not accounting for leap years)
+        days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+
+        day += 1
+        if day > days_in_month[month - 1]:
+            day = 1
+            month += 1
+            if month > 12:
+                month = 1
+                year += 1
+
+        self.day = f"{day:02d}"
+        self.month = f"{month:02d}"
+        self.year = str(year)
+
+    def to_datetime(self):
+        """Converts the WaybackDateObject to a Python datetime object."""
+        return datetime(
+            int(self.year), int(self.month), int(self.day),
+            int(self.hour), int(self.minute), int(self.second)
+        )
+
+    def from_datetime(self, dt):
+        """Updates the WaybackDateObject from a Python datetime object."""
+        self.year = f"{dt.year:04d}"
+        self.month = f"{dt.month:02d}"
+        self.day = f"{dt.day:02d}"
+        self.hour = f"{dt.hour:02d}"
+        self.minute = f"{dt.minute:02d}"
+        self.second = f"{dt.second:02d}"
+
+    def increment_week(self):
+        """
+        Increments the date by 7 days.
+        """
+        dt = self.to_datetime() + timedelta(days=7)
+        self.from_datetime(dt)
+
+    def decrement_week(self):
+        """
+        Decrements the date by 7 days.
+        """
+        dt = self.to_datetime() - timedelta(days=7)
+        self.from_datetime(dt)
+

internet_archive_extractor-0.0.1/src/waybackup_to_warc.py

@@ -0,0 +1,209 @@
+import glob
+import os
+import sys
+import pandas as pd
+from warcio.archiveiterator import ArchiveIterator
+from warcio.warcwriter import WARCWriter
+from warcio.recordloader import ArcWarcRecord
+from warcio.statusandheaders import StatusAndHeaders
+import csv
+from datetime import datetime
+
+
+COMBINED_CSV_PATH = "combined_output.csv"
+
+def remove_port_80(url):
+    if ":80" in url:
+        return url.replace(":80", "")
+    return url
+
+def process_csv(file_path):
+    data = pd.read_csv(file_path)
+    data['url_archive'] = data['url_archive'].apply(remove_port_80)
+    data['url_origin'] = data['url_origin'].apply(remove_port_80)
+    return data
+
+def write_404_warc_entry(writer, url, warc_date):
+    http_headers = StatusAndHeaders('404 Not Found', [('Content-Type', 'text/html')], protocol='HTTP/1.0')
+    warc_type = 'response'
+    record = writer.create_warc_record(
+        url,
+        warc_type,
+        payload = None,
+        http_headers = http_headers,
+        warc_headers_dict={'WARC-Date': warc_date} if warc_date else None
+    )
+    print(f"Writing 404 record for URL: {url}")
+    writer.write_record(record)
+
+def write_500_warc_entry(writer, url, warc_date):
+    http_headers = StatusAndHeaders('500 Internal Server Error', [('Content-Type', 'text/html')], protocol='HTTP/1.0')
+    warc_type = 'response'
+    record = writer.create_warc_record(
+        url,
+        warc_type,
+        payload = None,
+        http_headers = http_headers,
+        warc_headers_dict={'WARC-Date': warc_date} if warc_date else None
+    )
+    print(f"Writing 500 record for URL: {url}")
+    writer.write_record(record)
+
+def create_warc_gz(data, output_dir, output_filename):
+    """
+    Creates a compressed WARC (Web ARChive) file (.warc.gz) from a list of data entries.
+    Each entry in `data` should be a dictionary containing at least the following keys:
+        - 'url_origin': The original URL of the resource.
+        - 'file': The local file path to the resource content.
+        - 'timestamp': The timestamp of the capture in 'YYYYMMDDHHMMSS' format.
+        - 'response': The HTTP response code as a string or integer (e.g., '200', '404', '500').
+    The function processes each entry and writes a corresponding WARC record:
+        - For HTTP 404 and 500 responses, special WARC records are created using helper functions.
+        - For successful responses (HTTP 200), the content is read from the specified file and written as a WARC response record.
+        - The content type is inferred from the file extension.
+        - If the file does not exist, the entry is skipped and a warning is printed.
+    Parameters:
+        data (list of dict): List of dictionaries containing resource metadata and file paths.
+        output_dir (str): Directory where the output WARC file will be saved.
+        output_filename (str): Base name for the output WARC file (without extension).
+    Side Effects:
+        - Creates the output directory if it does not exist.
+        - Writes a compressed WARC file to disk.
+        - Prints progress and summary information to stdout.
+    Returns:
+        None
+    """
+
+    total_counter = 0
+    success_counter = 0
+    internal_service_error_counter = 0
+    not_found_counter = 0
+    os.makedirs(output_dir, exist_ok=True)
+    warc_path = os.path.join(output_dir, output_filename + '.warc.gz')
+    with open(warc_path, 'wb') as stream:
+        writer = WARCWriter(stream, gzip=True)
+        for row in data:
+            total_counter += 1
+            response_code = str(row['response']).strip()
+            url = row['url_origin']
+            file_path = row['file']
+
+            if total_counter % 5000 == 0:
+                print(f"Processing entry number: {total_counter}:")
+            # Convert timestamp to ISO 8601 format for WARC-Date
+            try:
+                dt = datetime.strptime(row['timestamp'].strip(), "%Y%m%d%H%M%S")
+                warc_date = dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+            except Exception as e:
+                print(f"Invalid timestamp {row['timestamp']}: {e}")
+                warc_date = None
+
+            # As 404 AND 500 
+
+            if response_code == '404':
+                write_404_warc_entry(writer, url, warc_date)
+                not_found_counter += 1
+                continue
+            elif response_code == '500':
+                write_500_warc_entry(writer, url, warc_date)
+                internal_service_error_counter += 1
+                continue
+
+        
+            if not os.path.isfile(file_path):
+                print(f"File not found: {file_path}")
+                print(f"Timestamp for URL is: {row['timestamp']}")
+                print(f"Response code is: {row['response']}")
+                # TODO: If response code is 404 a 404 record should be created
+                # TODO: If response code is 500 a 500 record should be created etc. 
+                continue
+
+            # Set content type based on file extension (simple default)
+            content_type = 'text/html'
+            ext = os.path.splitext(file_path)[1].lower()
+            if ext in ['.jpg', '.jpeg']:
+                content_type = 'image/jpeg'
+            elif ext == '.png':
+                content_type = 'image/png'
+            elif ext == '.gif':
+                content_type = 'image/gif'
+            elif ext == '.css':
+                content_type = 'text/css'
+            elif ext == '.js':
+                content_type = 'application/javascript'
+            elif ext == '.pdf':
+                content_type = 'application/pdf'
+            elif ext == '.txt':
+                content_type = 'text/plain'
+
+        
+            http_headers = StatusAndHeaders('200 OK', [('Content-Type', content_type)], protocol='HTTP/1.0')
+            warc_type = 'response'
+
+            with open(file_path, 'rb') as payload:
+                record = writer.create_warc_record(
+                    url,
+                    warc_type,
+                    payload=payload,
+                    http_headers=http_headers,
+                    warc_headers_dict={'WARC-Date': warc_date} if warc_date else None
+                )
+                writer.write_record(record)
+            success_counter += 1
+
+
+        print(
+            f"\nWARC creation summary:\n"
+            f"  Successful records:     {success_counter}\n"
+            f"  Not found (404):        {not_found_counter}\n"
+            f"  Internal errors (500):  {internal_service_error_counter}\n"
+            f"  Total processed:        {len(data)}"
+        )
+
+def read_csv(input_csv):
+    with open(input_csv, newline='', encoding='utf-8') as csvfile:
+        reader = csv.DictReader(csvfile)
+        return list(reader)
+
+def process_csv_file(csv_file_path, output_dir, output_filename):
+    data = read_csv(csv_file_path)
+    create_warc_gz(data, output_dir, output_filename)
+
+def combine_csv_files(input_directory, output_file):
+    """
+    Combines all CSV files in the specified directory into a single CSV file.
+    This is used to aggregate the multiple CSV files constructed when downloading content from the Internet Archive into one for further processing into WARC files.
+    """
+    # Find all CSV files in the directory
+    csv_files = glob.glob(os.path.join(input_directory, "*.csv"))
+    # Read and concatenate all CSV files
+    df_list = [pd.read_csv(f) for f in csv_files]
+    combined_df = pd.concat(df_list, ignore_index=True)
+    # Write the combined DataFrame to a new CSV file
+    combined_df.to_csv(output_file, index=False)
+    print(f"Combined {len(csv_files)} files into {output_file}")
+          
+
+def main():
+    """
+    Main entry point for the script. Expects two command-line arguments: the path to a directory with multiple CSV files and the desired filename for the output warc.gx file.
+    - Combines CSV files from the provided path into a single file with the name 'combined_output.csv'.
+    - Processes the combined CSV file and generates a warc.gz file using the specified output filename.
+    Usage:
+        python main.py <csv_file_path> <output_filename>
+    """
+
+    if len(sys.argv) < 3:
+        print("Usage: python main.py <csv_file_path> <output_filename>")
+        sys.exit(1)
+
+    csv_file_path = sys.argv[1]
+    output_filename = sys.argv[2]
+    
+    combine_csv_files(csv_file_path, COMBINED_CSV_PATH)
+    process_csv_file(COMBINED_CSV_PATH, 'output', output_filename)
+
+
+
+if __name__ == "__main__":
+    main()