zipline_polygon_bundle 0.2.3__py3-none-any.whl → 0.2.4__py3-none-any.whl

zipline_polygon_bundle/__init__.py

@@ -8,7 +8,7 @@ from .config import PolygonConfig
 from .nyse_all_hours_calendar import NYSE_ALL_HOURS, register_nyse_all_hours_calendar
 from .concat_all_aggs import concat_all_aggs_from_csv, generate_csv_agg_tables
 from .adjustments import load_splits, load_dividends, load_conditions
-from .trades import trades_schema, trades_dataset, cast_trades
+from .trades import trades_schema, trades_dataset, cast_trades, ordinary_trades_mask
 from .trades import custom_aggs_partitioning, custom_aggs_schema, trades_to_custom_aggs, convert_trades_to_custom_aggs
 from .trades import get_aggs_dates, generate_csv_trades_tables
 # from .compute_signals import compute_signals_for_all_custom_aggs
@@ -32,6 +32,7 @@ __all__ = [
     "trades_schema",
     "trades_dataset",
     "cast_trades",
+    "ordinary_trades_mask",
     "get_aggs_dates",
     "generate_csv_trades_tables",
     "custom_aggs_partitioning",

zipline_polygon_bundle/bundle.py

@@ -203,6 +203,16 @@ def process_minute_table(
                 metadata.loc[sid, "auto_close_date"] = auto_close_date
             else:
                 # Add a row to the metadata DataFrame. Don't forget to add an exchange field.
+                # metadata = pd.DataFrame(
+                #     columns=(
+                #         "start_date",
+                #         "end_date",
+                #         "auto_close_date",
+                #         "symbol",
+                #         "exchange",
+                #         "asset_name",
+                #     )
+                # )
                 metadata.loc[sid] = (
                     start_date,
                     end_date,

zipline_polygon_bundle/trades.py

@@ -17,6 +17,29 @@ import numpy as np
 import pandas as pd
 
 
+# Polygon Trade Conditions codes that don't reflect a current market-priced trade.
+# https://polygon.io/docs/rest/stocks/market-operations/condition-codes
+# Odd lots are excluded because although their volume counts the prices don't.
+EXCLUDED_CONDITION_CODES = {
+    # 2,   # Average Price
+    # 7,   # Cash Sale
+    10,  # Derivatively Priced
+    # 12,  # Form T / Extended Hours
+    13,  # Extended Hours (Sold Out Of Sequence)
+    # 15,  # Official Close
+    # 16,  # Official Open
+    20,  # Next Day
+    21,  # Price Variation
+    # 22,  # Prior Reference
+    29,  # Seller
+    32,  # Sold (Out of Sequence)
+    # 33,  # Sold + Stopped
+    41,  # Trade Thru Exempt
+    52,  # Contingent Trade
+    53   # Qualified Contingent Trade
+}
+
+
 def trades_schema(raw: bool = False) -> pa.Schema:
     # There is some problem reading the timestamps as timestamps so we have to read as integer then change the schema.
     # Polygon Aggregate flatfile timestamps are in nanoseconds (like trades), not milliseconds as the docs say.
@@ -89,9 +112,15 @@ def cast_strings_to_list(
     split_array = pa_compute.split_pattern(filled_column, pattern=separator)
 
     # Cast each element in the resulting lists to integers
-    int_list_array = pa_compute.cast(split_array, pa.list_(value_type))
+    return pa_compute.cast(split_array, pa.list_(value_type))
 
-    return int_list_array
+
+def ordinary_trades_mask(table: pa.Table) -> pa.BooleanArray:
+    conditions_dict = table["conditions"].combine_chunks().dictionary_encode()
+    list_of_codes = cast_strings_to_list(conditions_dict.dictionary).to_pylist()
+    code_dictionary = pa.array(set(codes).isdisjoint(EXCLUDED_CONDITION_CODES) for codes in list_of_codes)
+    include_mask = pa.DictionaryArray.from_arrays(conditions_dict.indices, code_dictionary).dictionary_decode()
+    return pa_compute.and_(include_mask, pa_compute.equal(table["correction"], "0"))
 
 
 def cast_trades(trades) -> pa.Table:
@@ -117,6 +146,8 @@ def custom_aggs_schema(raw: bool = False) -> pa.Schema:
             pa.field("window_start", timestamp_type, nullable=False),
             pa.field("transactions", pa.int64(), nullable=False),
             pa.field("vwap", price_type, nullable=False),
+            pa.field("traded_value", price_type, nullable=False),
+            pa.field("cumulative_traded_value", price_type, nullable=False),
             pa.field("date", pa.date32(), nullable=False),
             pa.field("year", pa.uint16(), nullable=False),
             pa.field("month", pa.uint8(), nullable=False),
@@ -190,17 +221,10 @@ def trades_to_custom_aggs(
     config: PolygonConfig,
     date: datetime.date,
     table: pa.Table,
-    include_trf: bool = False,
 ) -> pa.Table:
     print(f"{date=} {pa.default_memory_pool()=}")
-    # print(f"{datetime.datetime.now()=} {date=} {pa.default_memory_pool()=}")
-    # print(f"{resource.getrusage(resource.RUSAGE_SELF).ru_maxrss=}")
-    table = table.filter(pa_compute.greater(table["size"], 0))
-    table = table.filter(pa_compute.equal(table["correction"], "0"))
-    if not include_trf:
-        table = table.filter(pa_compute.not_equal(table["exchange"], 4))
     table = table.append_column(
-        "price_total", pa_compute.multiply(table["price"], table["size"])
+        "traded_value", pa_compute.multiply(table["price"], table["size"])
     )
     table = table.append_column(
         "window_start",
@@ -208,13 +232,14 @@ def trades_to_custom_aggs(
             table["sip_timestamp"], multiple=config.agg_timedelta.seconds, unit="second"
         ),
     )
+    table = table.sort_by([("ticker", "ascending"), ("sip_timestamp", "ascending")])
     table = table.group_by(["ticker", "window_start"], use_threads=False).aggregate(
         [
             ("price", "first"),
             ("price", "max"),
             ("price", "min"),
             ("price", "last"),
-            ("price_total", "sum"),
+            ("traded_value", "sum"),
             ("size", "sum"),
             ([], "count_all"),
         ]
@@ -226,13 +251,21 @@ def trades_to_custom_aggs(
             "price_min": "low",
             "price_last": "close",
             "size_sum": "volume",
-            "price_total_sum": "total",
+            "traded_value_sum": "traded_value",
             "count_all": "transactions",
         }
     )
+    table = table.sort_by([("ticker", "ascending"), ("window_start", "ascending")])
     table = table.append_column(
-        "vwap", pa_compute.divide(table["total"], table["volume"])
+        "vwap", pa_compute.divide(table["traded_value"], table["volume"])
     )
+    # Calculate cumulative traded value by ticker
+    traded_values_by_ticker = table.group_by("ticker").aggregate([("traded_value", "list")])
+    cumulative_sum_arrays = [
+        pa_compute.cumulative_sum(pa.array(values_list)) for values_list in traded_values_by_ticker["traded_value_list"].combine_chunks()
+    ]
+    table = table.append_column("cumulative_traded_value", pa.concat_arrays(cumulative_sum_arrays))
+    
     # table.append_column('date', pa.array([date] * len(table), type=pa.date32()))
     # table.append_column('year', pa.array([date.year] * len(table), type=pa.uint16()))
     # table.append_column('month', pa.array([date.month] * len(table), type=pa.uint8()))
@@ -249,7 +282,6 @@ def trades_to_custom_aggs(
             [to_partition_key(ticker) for ticker in table.column("ticker").to_pylist()]
         ),
     )
-    table = table.sort_by([("window_start", "ascending"), ("ticker", "ascending")])
     # print(f"aggs {date=} {table.to_pandas().head()=}")
     return table
 
@@ -279,22 +311,11 @@ def convert_trades_to_custom_aggs(
 
     # pa.set_memory_pool()
 
-    # pa_ds.write_dataset(
-    #     generate_custom_agg_batches_from_tables(config),
-    #     schema=custom_aggs_schema(),
-    #     filesystem=config.filesystem,
-    #     base_dir=config.aggs_dir,
-    #     partitioning=custom_aggs_partitioning(),
-    #     format="parquet",
-    #     existing_data_behavior="overwrite_or_ignore",
-    #     # max_open_files = MAX_FILES_OPEN,
-    #     # min_rows_per_group = MIN_ROWS_PER_GROUP,
-    # )
-
     for date, trades_table in generate_csv_trades_tables(config):
-        aggs_table = trades_to_custom_aggs(config, date, trades_table)
         pa_ds.write_dataset(
-            aggs_table,
+            trades_to_custom_aggs(config,
+                                  date,
+                                  trades_table.filter(ordinary_trades_mask(trades_table))),
             filesystem=config.filesystem,
             base_dir=config.aggs_dir,
             partitioning=custom_aggs_partitioning(),
@@ -304,7 +325,17 @@ def convert_trades_to_custom_aggs(
             # max_open_files=10,
             # min_rows_per_group=MIN_ROWS_PER_GROUP,
         )
-        del aggs_table
+        # pa_ds.write_dataset(
+        #     trades_to_custom_events(config,
+        #                             date,
+        #                             trades_table.filter(pa_compute.invert(ordinary_trades_mask(trades_table)))),
+        #     filesystem=config.filesystem,
+        #     base_dir=config.events_dir,
+        #     partitioning=custom_events_partitioning(),
+        #     format="parquet",
+        #     existing_data_behavior="overwrite_or_ignore",
+        #     file_visitor=file_visitor,
+        # )
         del trades_table
 
     # with ProcessPoolExecutor(max_workers=1) as executor:

{zipline_polygon_bundle-0.2.3.dist-info → zipline_polygon_bundle-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: zipline_polygon_bundle
-Version: 0.2.3
+Version: 0.2.4
 Summary: A zipline-reloaded data provider bundle for Polygon.io
 License: GNU AFFERO GENERAL PUBLIC LICENSE
                                 Version 3, 19 November 2007
@@ -769,7 +769,7 @@ register_polygon_equities_bundle(
 
 ```bash
 sudo apt-get update
-sudo apt-get install python3-dev python3-poetry
+sudo apt-get install python3-dev python3-poetry gcc-multilib
 
 CFLAGS=$(python3-config --includes) pip install git+https://github.com/fovi-llc/zipline-arrow.git
 ```

{zipline_polygon_bundle-0.2.3.dist-info → zipline_polygon_bundle-0.2.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
-zipline_polygon_bundle/__init__.py,sha256=KGN5kBi021Eiz_GDtxVRTUdXgYWe6loG_C8XcrVNHrY,1765
+zipline_polygon_bundle/__init__.py,sha256=QlH8njI5zj1WM5EFDqR3vXy80rXF4qeG3hnPPkaMqE0,1815
 zipline_polygon_bundle/adjustments.py,sha256=4garYK7RUrYyCIhCm0ZqHsk3y2bCt9vHUkWoHvVniTA,8233
-zipline_polygon_bundle/bundle.py,sha256=7f_rpVBhR1XyOJ1e7Lulq1Uh4DWJmHxFQKZNfz9OSgQ,19805
+zipline_polygon_bundle/bundle.py,sha256=EPv3uTgLZFkv2Fi3o5d3Yun0NQ-iHXGVmajVTxUjln4,20161
 zipline_polygon_bundle/compute_signals.py,sha256=FxcMuwMmxuvyy45y1avdL_uFEn0B4_2ekcv_B4AyPo0,10115
 zipline_polygon_bundle/concat_all_aggs.py,sha256=Nuj0pytQAVoK8OK7qx5m3jWCV8uJIPsa0XHnmicgSmg,12066
 zipline_polygon_bundle/concat_all_aggs_partitioned.py,sha256=AQq4ai5u5GyclWzQq2C8zIvHl_zjvLiDtxarNejwCQ4,6325
@@ -11,8 +11,8 @@ zipline_polygon_bundle/process_all_aggs.py,sha256=MVhb8xn9-DngSNSrRIpMG4XAgHjMXk
 zipline_polygon_bundle/quotes.py,sha256=yFjlPiQXPp0t6w2Bo96VLtYSqITP7WCLwMp5CH3zx1E,4260
 zipline_polygon_bundle/split_aggs_by_ticker.py,sha256=HI_3nuN6E_VCq7LfOj4Dib_qm8wYME-jdXXX4rt-9YI,2150
 zipline_polygon_bundle/tickers_and_names.py,sha256=BjYquIlSBQGd1yDW3m3cGuXKVvUfh_waYwdMR7eAhuM,15402
-zipline_polygon_bundle/trades.py,sha256=XK2ed06ekByAVCimCDtJUIQ3HZaQbfKc0BXC9orHoJg,20192
-zipline_polygon_bundle-0.2.3.dist-info/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-zipline_polygon_bundle-0.2.3.dist-info/METADATA,sha256=a7tw9uwGWQ-cmRk3xzc5r8WYZ03276_3NYajuuxcQR4,51908
-zipline_polygon_bundle-0.2.3.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-zipline_polygon_bundle-0.2.3.dist-info/RECORD,,
+zipline_polygon_bundle/trades.py,sha256=OXRILPa7Hyx-cyEe0u1BVhoncpotOzG_dhh_TPHLCBI,21818
+zipline_polygon_bundle-0.2.4.dist-info/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+zipline_polygon_bundle-0.2.4.dist-info/METADATA,sha256=bh8uyn9tcvitKwjf7lKo1uGeXclD3V62PiRfrr4NBh8,51921
+zipline_polygon_bundle-0.2.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+zipline_polygon_bundle-0.2.4.dist-info/RECORD,,

{zipline_polygon_bundle-0.2.3.dist-info → zipline_polygon_bundle-0.2.4.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.2
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any