gnomepy 2.26.0__tar.gz → 2.26.2__tar.gz

{gnomepy-2.26.0 → gnomepy-2.26.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gnomepy
-Version: 2.26.0
+Version: 2.26.2
 Summary: 
 Author: mprey
 Author-email: masonprey7@gmail.com

{gnomepy-2.26.0 → gnomepy-2.26.2}/gnomepy/_fs.py

@@ -55,7 +55,7 @@ def fs_read_json(fs: pafs.FileSystem, path: str) -> dict:
 
 def fs_write_json(fs: pafs.FileSystem, path: str, data: dict) -> None:
     content = (json.dumps(data, indent=2, default=str) + "\n").encode()
-    with fs.open_output_stream(path) as f:
+    with fs.open_output_stream(path, compression=None) as f:
         f.write(content)
 
 
@@ -110,5 +110,7 @@ def fs_read_bytes(fs: pafs.FileSystem, path: str) -> bytes:
 
 
 def fs_write_bytes(fs: pafs.FileSystem, path: str, data: bytes) -> None:
-    with fs.open_output_stream(path) as f:
+    # pyarrow would otherwise compress by suffix, so bytes already gzipped for a .gz path got gzipped
+    # again; open_input_file never decompresses, so the write must store them exactly as given.
+    with fs.open_output_stream(path, compression=None) as f:
         f.write(data)

{gnomepy-2.26.0 → gnomepy-2.26.2}/gnomepy/data/datasets.py

@@ -36,6 +36,13 @@ MANAGED_COLUMNS = tuple(f.name for f in MANAGED_FIELDS)
 KEY_PROPERTY = "gnome.key"
 COMMIT_ATTEMPTS = 4
 TRANSFORMS: dict[str, Transform] = {"identity": IdentityTransform(), "day": DayTransform(), "month": MonthTransform()}
+# Arrow hands back an integer or boolean column as float64 or object when the batch has a null in it,
+# and as int64 or bool when it doesn't, so a row's hash would depend on the rows published with it.
+NULLABLE_DTYPES = {
+    pa.int8(): pd.Int8Dtype(), pa.int16(): pd.Int16Dtype(), pa.int32(): pd.Int32Dtype(), pa.int64(): pd.Int64Dtype(),
+    pa.uint8(): pd.UInt8Dtype(), pa.uint16(): pd.UInt16Dtype(), pa.uint32(): pd.UInt32Dtype(),
+    pa.uint64(): pd.UInt64Dtype(), pa.bool_(): pd.BooleanDtype(),
+}
 
 
 @dataclass(frozen=True)
@@ -226,7 +233,7 @@ def _as_stored(spec: TableSpec, df: pd.DataFrame) -> pd.DataFrame:
     # Hash the values as Iceberg will store them: the same data sent as float, int or a
     # nullable extension dtype, or read back by load(), must hash the same or it re-appends.
     rows = pa.Table.from_pandas(_to_microseconds(df[list(spec.schema.names)]), schema=spec.schema, preserve_index=False)
-    return rows.to_pandas()
+    return rows.to_pandas(types_mapper=NULLABLE_DTYPES.get)
 
 
 def _to_microseconds(df: pd.DataFrame) -> pd.DataFrame:

{gnomepy-2.26.0 → gnomepy-2.26.2}/gnomepy/reporting/report.py

@@ -417,7 +417,7 @@ class BacktestReport:
         if path_str.startswith("s3://"):
             fs, normalized = resolve_fs(path_str)
             content = html.encode("utf-8")
-            with fs.open_output_stream(normalized) as f:
+            with fs.open_output_stream(normalized, compression=None) as f:
                 f.write(content)
         else:
             Path(path_str).write_text(html)
@@ -429,7 +429,7 @@ class BacktestReport:
         if path_str.startswith("s3://"):
             fs, normalized = resolve_fs(path_str)
             content = (json.dumps(data, indent=2, default=str) + "\n").encode("utf-8")
-            with fs.open_output_stream(normalized) as f:
+            with fs.open_output_stream(normalized, compression=None) as f:
                 f.write(content)
         else:
             Path(path_str).write_text(json.dumps(data, indent=2, default=str) + "\n")

{gnomepy-2.26.0 → gnomepy-2.26.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "gnomepy"
-version = "2.26.0"
+version = "2.26.2"
 description = ""
 authors = [
     {name = "mprey",email = "masonprey7@gmail.com"}