zipline_polygon_bundle 0.1.6__py3-none-any.whl

zipline_polygon_bundle/concat_all_aggs.py

@@ -0,0 +1,246 @@
+from .config import PolygonConfig
+
+import shutil
+from typing import Iterator, Tuple
+
+import argparse
+import glob
+import os
+
+import pyarrow as pa
+from pyarrow import dataset as pa_ds
+from pyarrow import csv as pa_csv
+
+import pandas as pd
+
+
+PARTITION_COLUMN_NAME = "part"
+PARTITION_KEY_LENGTH = 2
+
+
+def to_partition_key(s: str) -> str:
+    """
+    Partition key is low cardinality and must be filesystem-safe.
+    The reason for partitioning is to keep the parquet files from getting too big.
+    10 years of minute aggs for US stocks is 83GB gzipped.  A single parquet would be 62GB on disk.
+    Currently the first two characters so files stay under 1GB.  Weird characters are replaced with "A".
+    """
+    k = (s + "A")[0:PARTITION_KEY_LENGTH].upper()
+    if k.isalpha():
+        return k
+    # Replace non-alpha characters with "A".
+    k = "".join([c if c.isalpha() else "A" for c in k])
+    return k
+
+
+def generate_tables_from_csv_files(
+    paths: list,
+    schema: pa.Schema,
+    start_timestamp: pd.Timestamp,
+    limit_timestamp: pd.Timestamp,
+) -> Iterator[pa.Table]:
+    empty_table = schema.empty_table()
+    # TODO: Find which column(s) need to be cast to int64 from the schema.
+    empty_table = empty_table.set_column(
+        empty_table.column_names.index("window_start"),
+        "window_start",
+        empty_table.column("window_start").cast(pa.int64()),
+    )
+    csv_schema = empty_table.schema
+
+    tables_read_count = 0
+    skipped_table_count = 0
+    for path in paths:
+        convert_options = pa_csv.ConvertOptions(
+            column_types=csv_schema,
+            strings_can_be_null=False,
+            quoted_strings_can_be_null=False,
+        )
+
+        table = pa.csv.read_csv(path, convert_options=convert_options)
+        tables_read_count += 1
+        table = table.set_column(
+            table.column_names.index("window_start"),
+            "window_start",
+            table.column("window_start").cast(schema.field("window_start").type),
+        )
+        if PARTITION_COLUMN_NAME in schema.names:
+            table = table.append_column(
+                PARTITION_COLUMN_NAME,
+                pa.array(
+                    [
+                        to_partition_key(ticker)
+                        for ticker in table.column("ticker").to_pylist()
+                    ]
+                ),
+            )
+        expr = (
+            pa.compute.field("window_start")
+            >= pa.scalar(start_timestamp, type=schema.field("window_start").type)
+        ) & (
+            pa.compute.field("window_start")
+            < pa.scalar(
+                limit_timestamp,
+                type=schema.field("window_start").type,
+            )
+        )
+        table = table.filter(expr)
+
+        # TODO: Also check that these rows are within range for this file's date (not just the whole session).
+        # And if we're doing that (figuring date for each file), we can just skip reading the file.
+        # Might able to do a single comparison using compute.days_between.
+        # https://arrow.apache.org/docs/python/generated/pyarrow.compute.days_between.html
+
+        if table.num_rows == 0:
+            skipped_table_count += 1
+            continue
+
+        yield table
+    print(f"{tables_read_count=} {skipped_table_count=}")
+
+
+def generate_csv_agg_tables(
+    config: PolygonConfig,
+) -> Tuple[list[str], pa.Schema, Iterator[pa.Table]]:
+    """zipline does bundle ingestion one ticker at a time."""
+    # We sort by path because they have the year and month in the dir names and the date in the filename.
+    paths = sorted(
+        list(
+            glob.glob(
+                os.path.join(config.aggs_dir, config.csv_paths_pattern),
+                recursive="**" in config.csv_paths_pattern,
+            )
+        )
+    )
+
+    print(f"{len(paths)=}")
+    if len(paths) > 0:
+        print(f"{paths[0]=}")
+        print(f"{paths[-1]=}")
+
+    # Polygon Aggregate flatfile timestamps are in nanoseconds (like trades), not milliseconds as the docs say.
+    # I make the timestamp timezone-aware because that's how Unix timestamps work and it may help avoid mistakes.
+    timestamp_type = pa.timestamp("ns", tz="UTC")
+
+    # But we can't use the timestamp type in the schema here because it's not supported by the CSV reader.
+    # So we'll use int64 and cast it after reading the CSV file.
+    # https://github.com/apache/arrow/issues/44030
+
+    # strptime(3) (used by CSV reader for timestamps in ConvertOptions.timestamp_parsers) supports Unix timestamps (%s) and milliseconds (%f) but not nanoseconds.
+    # https://www.geeksforgeeks.org/how-to-use-strptime-with-milliseconds-in-python/
+    # Actually that's the wrong strptime (it's Python's).  C++ strptime(3) doesn't even support %f.
+    # https://github.com/apache/arrow/issues/39839#issuecomment-1915981816
+    # Also I don't think you can use those in a format string without a separator.
+
+    # Polygon price scale is 4 decimal places (i.e. hundredths of a penny), but we'll use 10 because we have precision to spare.
+    # price_type = pa.decimal128(precision=38, scale=10)
+    # 64bit float a little overkill but avoids any plausible truncation error.
+    price_type = pa.float64()
+
+    polygon_aggs_schema = pa.schema(
+        [
+            pa.field("ticker", pa.string(), nullable=False),
+            pa.field("volume", pa.int64(), nullable=False),
+            pa.field("open", price_type, nullable=False),
+            pa.field("close", price_type, nullable=False),
+            pa.field("high", price_type, nullable=False),
+            pa.field("low", price_type, nullable=False),
+            pa.field("window_start", timestamp_type, nullable=False),
+            pa.field("transactions", pa.int64(), nullable=False),
+        ]
+    )
+    if config.agg_time == "minute":
+        polygon_aggs_schema = polygon_aggs_schema.append(
+            pa.field(PARTITION_COLUMN_NAME, pa.string(), nullable=False)
+        )
+
+    return (
+        paths,
+        polygon_aggs_schema,
+        generate_tables_from_csv_files(
+            paths=paths,
+            schema=polygon_aggs_schema,
+            start_timestamp=config.start_timestamp,
+            limit_timestamp=config.end_timestamp + pd.to_timedelta(1, unit="day"),
+        ),
+    )
+
+
+def generate_batches_from_tables(tables):
+    for table in tables:
+        for batch in table.to_batches():
+            yield batch
+
+
+def concat_all_aggs_from_csv(
+    config: PolygonConfig,
+    overwrite: bool = False,
+) -> str:
+    paths, schema, tables = generate_csv_agg_tables(config)
+
+    if len(paths) < 1:
+        raise ValueError(f"No Polygon CSV flat files found in {config.aggs_dir=}")
+    by_ticker_aggs_arrow_dir = config.by_ticker_aggs_arrow_dir(paths[0], paths[-1])
+    if os.path.exists(by_ticker_aggs_arrow_dir):
+        if overwrite:
+            print(f"Removing {by_ticker_aggs_arrow_dir=}")
+            shutil.rmtree(by_ticker_aggs_arrow_dir)
+        else:
+            print(f"Found existing {by_ticker_aggs_arrow_dir=}")
+            return by_ticker_aggs_arrow_dir
+
+    partitioning = None
+    if PARTITION_COLUMN_NAME in schema.names:
+        partitioning = pa_ds.partitioning(
+            pa.schema([(PARTITION_COLUMN_NAME, pa.string())]), flavor="hive"
+        )
+
+    # scanner = pa_ds.Scanner.from_batches(source=generate_batches_from_tables(tables), schema=schema)
+    pa_ds.write_dataset(
+        generate_batches_from_tables(tables),
+        schema=schema,
+        base_dir=by_ticker_aggs_arrow_dir,
+        partitioning=partitioning,
+        format="parquet",
+        existing_data_behavior="overwrite_or_ignore",
+    )
+    print(f"Concatenated aggregates to {by_ticker_aggs_arrow_dir=}")
+    return by_ticker_aggs_arrow_dir
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--calendar_name", default="XNYS")
+
+    parser.add_argument("--start_session", default="2014-06-16")
+    parser.add_argument("--end_session", default="2024-09-06")
+    # parser.add_argument("--start_session", default="2020-01-01")
+    # parser.add_argument("--end_session", default="2020-12-31")
+
+    parser.add_argument("--agg_time", default="day")
+
+    parser.add_argument("--overwrite", action="store_true")
+
+    # TODO: These defaults should be None but for dev convenience they are set for my local config.
+    parser.add_argument("--data_dir", default="/Volumes/Oahu/Mirror/files.polygon.io")
+    # parser.add_argument("--aggs_pattern", default="**/*.csv.gz")
+    # parser.add_argument("--aggs_pattern", default="2020/10/**/*.csv.gz")
+
+    args = parser.parse_args()
+
+    # Maybe the way to do this is to use the os.environ as the argparser defaults.
+    if args.data_dir:
+        os.environ["POLYGON_DATA_DIR"] = args.data_dir
+
+    config = PolygonConfig(
+        environ=os.environ,
+        calendar_name=args.calendar_name,
+        start_session=args.start_session,
+        end_session=args.end_session,
+        agg_time=args.agg_time,
+    )
+
+    concat_all_aggs_from_csv(
+        config=config,
+        overwrite=args.overwrite,
+    )

zipline_polygon_bundle/concat_all_aggs_partitioned.py

@@ -0,0 +1,173 @@
+from .config import PolygonConfig
+
+from typing import Iterator
+
+import argparse
+import glob
+import os
+
+import pyarrow as pa
+from pyarrow import dataset as pa_ds
+from pyarrow import csv as pa_csv
+
+PARTITION_COLUMN_NAME = "part"
+
+
+# To work across all reasonable filesystems, we need to escape the characters in partition keys that are treated weirdly in filenames.
+def partition_key_escape(c: str) -> str:
+    return ("^" + c.upper()) if c.islower() else ("%" + "%02X" % ord(c))
+
+
+def to_partition_key(s: str) -> str:
+    if s.isalnum() and s.isupper():
+        return s
+    return "".join(
+        [f"{c if (c.isupper() or c.isdigit()) else partition_key_escape(c)}" for c in s]
+    )
+
+
+def read_csv_table(path, timestamp_type: pa.TimestampType, convert_options):
+    table = pa.csv.read_csv(path, convert_options=convert_options)
+    table = table.set_column(
+        table.column_names.index("window_start"),
+        "window_start",
+        table.column("window_start").cast(timestamp_type),
+    )
+    return table
+
+
+def csv_agg_scanner(
+    paths: list, schema: pa.Schema, timestamp_type: pa.TimestampType
+) -> Iterator[pa.RecordBatch]:
+    for path in paths:
+        convert_options = pa_csv.ConvertOptions(
+            column_types=schema,
+            strings_can_be_null=False,
+            quoted_strings_can_be_null=False,
+        )
+
+        print(f"{path=}")
+        table = read_csv_table(
+            path=path, timestamp_type=timestamp_type, convert_options=convert_options
+        )
+
+        table = table.append_column(
+            PARTITION_COLUMN_NAME,
+            pa.array(
+                [to_partition_key(ticker) for ticker in table.column("ticker").to_pylist()]
+            ),
+        )
+
+        for batch in table.to_batches():
+            yield batch
+
+
+def concat_all_aggs_from_csv(
+    config: PolygonConfig,
+    aggs_pattern: str = "**/*.csv.gz",
+) -> list:
+    """zipline does bundle ingestion one ticker at a time."""
+
+    # We sort by path because they have the year and month in the dir names and the date in the filename.
+    paths = sorted(
+        list(
+            glob.glob(
+                os.path.join(config.aggs_dir, aggs_pattern),
+                recursive="**" in aggs_pattern,
+            )
+        )
+    )
+
+    # Polygon Aggregate flatfile timestamps are in nanoseconds (like trades), not milliseconds as the docs say.
+    # I make the timestamp timezone-aware because that's how Unix timestamps work and it may help avoid mistakes.
+    timestamp_type = pa.timestamp("ns", tz="UTC")
+
+    # But we can't use the timestamp type in the schema here because it's not supported by the CSV reader.
+    # So we'll use int64 and cast it after reading the CSV file.
+    # https://github.com/apache/arrow/issues/44030
+
+    # strptime(3) (used by CSV reader for timestamps in ConvertOptions.timestamp_parsers) supports Unix timestamps (%s) and milliseconds (%f) but not nanoseconds.
+    # https://www.geeksforgeeks.org/how-to-use-strptime-with-milliseconds-in-python/
+    # Actually that's the wrong strptime (it's Python's).  C++ strptime(3) doesn't even support %f.
+    # https://github.com/apache/arrow/issues/39839#issuecomment-1915981816
+    # Also I don't think you can use those in a format string without a separator.
+
+    # Polygon price scale is 4 decimal places (i.e. hundredths of a penny), but we'll use 10 because we have precision to spare.
+    price_type = pa.decimal128(precision=38, scale=10)
+
+    polygon_aggs_schema = pa.schema(
+        [
+            pa.field("ticker", pa.string(), nullable=False),
+            pa.field("volume", pa.int64(), nullable=False),
+            pa.field("open", price_type, nullable=False),
+            pa.field("close", price_type, nullable=False),
+            pa.field("high", price_type, nullable=False),
+            pa.field("low", price_type, nullable=False),
+            pa.field("window_start", pa.int64(), nullable=False),
+            pa.field("transactions", pa.int64(), nullable=False),
+        ]
+    )
+
+    partitioned_schema = polygon_aggs_schema.append(
+        pa.field(PARTITION_COLUMN_NAME, pa.string(), nullable=False)
+    )
+    agg_scanner = pa_ds.Scanner.from_batches(
+        csv_agg_scanner(paths=paths, schema=polygon_aggs_schema, timestamp_type=timestamp_type),
+        schema=partitioned_schema
+    )
+
+    by_ticker_base_dir = os.path.join(
+        config.by_ticker_dir,
+        f"{config.agg_time}_{config.start_timestamp.date().isoformat()}_{config.end_timestamp.date().isoformat()}.hive",
+    )
+    partition_by_ticker = pa.dataset.partitioning(
+        pa.schema([(PARTITION_COLUMN_NAME, pa.string())]), flavor="hive"
+    )
+    pa_ds.write_dataset(
+        agg_scanner,
+        base_dir=by_ticker_base_dir,
+        format="parquet",
+        partitioning=partition_by_ticker,
+        existing_data_behavior="overwrite_or_ignore",
+        max_partitions=config.max_partitions,
+        max_open_files=config.max_open_files,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--calendar_name", default="XNYS")
+
+    parser.add_argument("--start_session", default="2014-06-16")
+    parser.add_argument("--end_session", default="2024-09-06")
+    # parser.add_argument("--start_session", default="2020-10-07")
+    # parser.add_argument("--end_session", default="2020-10-15")
+    # parser.add_argument("--aggs_pattern", default="2020/10/**/*.csv.gz")
+    parser.add_argument("--aggs_pattern", default="**/*.csv.gz")
+
+    parser.add_argument("--overwrite", action="store_true")
+
+    # TODO: These defaults should be None but for dev convenience they are set for my local config.
+    parser.add_argument("--agg_time", default="day")
+    parser.add_argument("--data_dir", default="/Volumes/Oahu/Mirror/files.polygon.io")
+
+    args = parser.parse_args()
+
+    # Maybe the way to do this is to use the os.environ as the argparser defaults.
+    environ = dict(os.environ.items())
+    if args.data_dir:
+        environ["POLYGON_DATA_DIR"] = args.data_dir
+    if args.agg_time:
+        environ["POLYGON_AGG_TIME"] = args.agg_time
+
+    config = PolygonConfig(
+        environ=os.environ,
+        calendar_name=args.calendar_name,
+        start_session=args.start_session,
+        end_session=args.end_session,
+    )
+
+    concat_all_aggs_from_csv(
+        config=config,
+        aggs_pattern=args.aggs_pattern,
+    )

zipline_polygon_bundle/config.py

@@ -0,0 +1,113 @@
+from exchange_calendars.calendar_helpers import Date, parse_date, parse_timestamp
+from zipline.utils.calendar_utils import get_calendar
+
+import os
+import pandas as pd
+
+
+class PolygonConfig:
+    def __init__(
+        self,
+        environ: dict,
+        calendar_name: str,
+        start_session: Date,
+        end_session: Date,
+        agg_time: str = "day",
+    ):
+        if agg_time not in ["minute", "day"]:
+            raise ValueError(f"agg_time must be 'minute' or 'day', got '{agg_time}'")
+        self.calendar_name = calendar_name
+        self.start_timestamp = (
+            parse_date(start_session, calendar=self.calendar)
+            if start_session
+            else self.calendar.first_session
+        )
+        self.end_timestamp = (
+            parse_date(end_session, calendar=self.calendar)
+            if end_session
+            else self.calendar.last_session
+        )
+        self.max_workers = None
+        if environ.get("POLYGON_MAX_WORKERS", "").strip() != "":
+            self.max_workers = int(environ.get("POLYGON_MAX_WORKERS"))
+        self.api_key = environ.get("POLYGON_API_KEY")
+        self.data_dir = environ.get("POLYGON_DATA_DIR", "data/files.polygon.io")
+        self.cik_cusip_mapping_csv_path = environ.get(
+            "CIK_CUSIP_MAPS_CSV", os.path.join(self.data_dir, "cik-cusip-maps.csv")
+        )
+        self.asset_subdir = environ.get("POLYGON_ASSET_SUBDIR", "us_stocks_sip")
+        self.market = environ.get("POLYGON_MARKET", "stocks")
+        self.tickers_dir = environ.get(
+            "POLYGON_TICKERS_DIR",
+            os.path.join(os.path.join(self.data_dir, "tickers"), self.asset_subdir),
+        )
+        self.tickers_csv_path = environ.get(
+            "POLYGON_TICKERS_CSV",
+            os.path.join(
+                self.tickers_dir,
+                f"tickers_{self.start_timestamp.date().isoformat()}_{self.end_timestamp.date().isoformat()}.csv",
+            ),
+        )
+        self.flat_files_dir = environ.get(
+            "POLYGON_FLAT_FILES_DIR", os.path.join(self.data_dir, "flatfiles")
+        )
+        self.csv_paths_pattern = environ.get("POLYGON_FLAT_FILES_CSV_PATTERN", "**/*.csv.gz")
+        self.agg_time = agg_time
+        self.asset_files_dir = os.path.join(self.flat_files_dir, self.asset_subdir)
+        self.minute_aggs_dir = os.path.join(self.asset_files_dir, "minute_aggs_v1")
+        self.day_aggs_dir = os.path.join(self.asset_files_dir, "day_aggs_v1")
+        self.aggs_dir = (
+            self.minute_aggs_dir if self.agg_time == "minute" else self.day_aggs_dir
+        )
+        # TODO: The "by ticker" files are temporary/intermediate and should/could be in the zipline data dir.
+        self.minute_by_ticker_dir = os.path.join(
+            self.asset_files_dir, "minute_by_ticker_v1"
+        )
+        self.day_by_ticker_dir = os.path.join(self.asset_files_dir, "day_by_ticker_v1")
+        self.by_ticker_dir = (
+            self.minute_by_ticker_dir
+            if self.agg_time == "minute"
+            else self.day_by_ticker_dir
+        )
+        self.arrow_format = environ.get("POLYGON_ARROW_FORMAT", "parquet" if self.agg_time == "day" else "hive")
+        # self.by_ticker_hive_dir = os.path.join(
+        #     self.by_ticker_dir,
+        #     f"{self.agg_time}_{self.start_timestamp.date().isoformat()}_{self.end_timestamp.date().isoformat()}.hive",
+        # )
+        self.cache_dir = os.path.join(self.asset_files_dir, "api_cache")
+
+    @property
+    def calendar(self):
+        return get_calendar(self.calendar_name)
+
+    def ticker_file_path(self, date: pd.Timestamp):
+        ticker_year_dir = os.path.join(
+            self.tickers_dir, f"tickers_{date.strftime('%Y')}"
+        )
+        os.makedirs(ticker_year_dir, exist_ok=True)
+        return os.path.join(
+            ticker_year_dir, f"tickers_{date.date().isoformat()}.parquet"
+        )
+    
+    def file_path_to_name(self, path: str):
+        return os.path.basename(path).removesuffix(".gz").removesuffix(".csv")
+
+    def by_ticker_aggs_arrow_dir(self, first_path: str, last_path: str):
+        return os.path.join(
+            self.by_ticker_dir,
+            f"{self.file_path_to_name(first_path)}_{self.file_path_to_name(last_path)}.arrow",
+        )
+
+    def api_cache_path(
+        self, start_date: Date, end_date: Date, filename: str, extension=".parquet"
+    ):
+        start_str = parse_date(start_date, calendar=self.calendar).date().isoformat()
+        end_str = parse_date(end_date, calendar=self.calendar).date().isoformat()
+        return os.path.join(
+            self.cache_dir, f"{start_str}_{end_str}/{filename}{extension}"
+        )
+
+
+if __name__ == "__main__":
+    config = PolygonConfig(os.environ, "XNYS", "2003-10-01", "2023-01-01")
+    print(config.__dict__)

zipline_polygon_bundle/polygon_file_reader.py

@@ -0,0 +1,104 @@
+from .config import PolygonConfig
+
+import os
+import glob
+from concurrent.futures import ProcessPoolExecutor
+import pandas as pd
+
+
+def convert_timestamp(x):
+    """
+    Polygon timestamps are in nanoseconds, milliseconds, or seconds.
+    We can decide automatically based on the size of the number because the only overlaps
+    are during the first few months of 1970.  And zero is always the same in any case.
+    """
+    try:
+        unix_time = int(x)
+        return pd.to_datetime(
+            unix_time,
+            unit=(
+                "ns"
+                if unix_time > 100_000_000_000_000
+                else "ms" if unix_time > 10_000_000_000 else "s"
+            ),
+        )
+    except Exception as e:
+        print(f"ERROR: Failed to convert '{x}': {e}")
+        return pd.NaT
+
+
+def convert_csv_to_parquet(path, extension, compression):
+    print(path)
+    try:
+        bars_df = pd.read_csv(
+            path,
+            compression=compression,
+            converters={"window_start": convert_timestamp},
+            dtype={"ticker": "str"},
+        )
+        # bars_df["ticker"] = bars_df["ticker"].astype(str)
+        # bars_df.info()
+        # print(f"{bars_df["ticker"].str.len().max()=}")
+        if len(bars_df) == 0:
+            print(f"WARNING: Empty {path}")
+            return
+        # if len(bars_df) < 100000:
+        #     print(f"WARNING: Short {path}")
+        # Don't change the data.  We're just converting to Parquet to save time.
+        # bars_df.set_index(["window_start", "ticker"], inplace=True)
+        # bars_df.sort_index(inplace=True)
+        parquet_path = path.removesuffix(extension) + ".parquet"
+        bars_df.to_parquet(parquet_path)
+        # fp.write(parquet_path, bars_df, has_nulls=False, write_index=False, fixed_text={"ticker": bars_df["ticker"].str.len().max()})
+        if not os.path.exists(parquet_path):
+            print(f"ERROR: Failed to write {parquet_path}")
+    except Exception as e:
+        print(f"Failed for {path}: {e}")
+
+
+def process_all_csv_to_parquet(
+    aggs_dir,
+    recursive=True,
+    extension=".csv.gz",
+    compression="infer",
+    force=False,
+    max_workers=None,
+):
+    """Big CSV files are very slow to read.  So we only read them once and convert them to Parquet."""
+    csv_pattern = f"**/*{extension}" if recursive else f"*{extension}"
+    paths = list(glob.glob(os.path.join(aggs_dir, csv_pattern), recursive=recursive))
+    if force:
+        print(f"Removing Parquet files that may exist for {len(paths)} CSV files.")
+        for path in paths:
+            parquet_path = path.removesuffix(extension) + ".parquet"
+            if os.path.exists(parquet_path):
+                print(f"Removing {parquet_path}")
+                os.remove(parquet_path)
+    else:
+        csv_file_count = len(paths)
+        paths = [
+            path
+            for path in paths
+            if not os.path.exists(path.removesuffix(extension) + ".parquet")
+        ]
+        if len(paths) < csv_file_count:
+            print(f"Skipping {csv_file_count - len(paths)} already converted files.")
+    if max_workers == 0:
+        for path in paths:
+            convert_csv_to_parquet(path, extension=extension, compression=compression)
+    else:
+        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+            executor.map(
+                convert_csv_to_parquet,
+                paths,
+                [extension] * len(paths),
+                [compression] * len(paths),
+            )
+
+
+if __name__ == "__main__":
+    # os.environ["POLYGON_DATA_DIR"] = "/Volumes/Oahu/Mirror/files.polygon.io"
+    config = PolygonConfig(
+        environ=os.environ, calendar_name="XNYS", start_session=None, end_session=None
+    )
+    process_all_csv_to_parquet(config.aggs_dir)