containerio 2.1.1__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {containerio-2.1.1 → containerio-2.2.0}/PKG-INFO +9 -3
- {containerio-2.1.1 → containerio-2.2.0}/README.md +8 -2
- {containerio-2.1.1 → containerio-2.2.0}/pyproject.toml +1 -1
- {containerio-2.1.1 → containerio-2.2.0}/src/containerio/handler.py +43 -0
- {containerio-2.1.1 → containerio-2.2.0}/src/containerio/__init__.py +0 -0
- {containerio-2.1.1 → containerio-2.2.0}/src/containerio/azure_config.py +0 -0
- {containerio-2.1.1 → containerio-2.2.0}/src/containerio/session.py +0 -0
- {containerio-2.1.1 → containerio-2.2.0}/src/containerio/tools/cli.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: containerio
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: Azure Blob Storage handler with unified local/cloud interface
|
|
5
5
|
Author: Alliance SwissPass
|
|
6
6
|
Requires-Dist: polars>=1.30.0
|
|
@@ -406,6 +406,7 @@ StorageHandler(
|
|
|
406
406
|
| `read_parquet_blob(blob_name)` | Read parquet file, returns `(DataFrame, metadata)` |
|
|
407
407
|
| `read_csv_blob(blob_name, **kwargs)` | Read CSV file into DataFrame |
|
|
408
408
|
| `read_excel_blob(blob_name, **kwargs)` | Read Excel file into DataFrame |
|
|
409
|
+
| `read_json_blob(blob_name, **kwargs)` | Read JSON file into DataFrame |
|
|
409
410
|
| `scan_parquet_blob(blob_name)` | Lazy scan parquet, returns `(LazyFrame, metadata)` |
|
|
410
411
|
| `scan_csv_blob(blob_name, **kwargs)` | Lazy scan CSV file |
|
|
411
412
|
|
|
@@ -416,6 +417,7 @@ StorageHandler(
|
|
|
416
417
|
| `write_parquet_blob(blob_name, df)` | Write DataFrame to parquet |
|
|
417
418
|
| `write_csv_blob(blob_name, df, **kwargs)` | Write DataFrame to CSV |
|
|
418
419
|
| `write_excel_blob(blob_name, df, **kwargs)` | Write DataFrame to Excel |
|
|
420
|
+
| `write_json_blob(blob_name, df, **kwargs)` | Write DataFrame to JSON |
|
|
419
421
|
| `sink_parquet_blob(blob_name, lf)` | Stream LazyFrame to parquet |
|
|
420
422
|
|
|
421
423
|
#### Utility Methods
|
|
@@ -620,8 +622,12 @@ The CI pipeline will automatically:
|
|
|
620
622
|
|
|
621
623
|
The repository includes a Crow CI pipeline that automatically builds, tests, and publishes the package when a git tag is pushed:
|
|
622
624
|
|
|
623
|
-
|
|
624
|
-
|
|
625
|
+
1. Merge all branches you want to have part of a new release into main
|
|
626
|
+
1. Check tag for upcoming semantic release version with `git sv rn`
|
|
627
|
+
1. If not already added, add your private key (whose public key is associated with your Codefloe Git user) to the SSH agent.
|
|
628
|
+
`ssh-add ~/.ssh/<your-private-key>`
|
|
629
|
+
1. Generate tag with version based on git commit messages with `git sv tag`
|
|
630
|
+
1. Create and push a git tag (e.g., `git push --tags`)
|
|
625
631
|
|
|
626
632
|
The pipeline will:
|
|
627
633
|
- Run linting and tests
|
|
@@ -392,6 +392,7 @@ StorageHandler(
|
|
|
392
392
|
| `read_parquet_blob(blob_name)` | Read parquet file, returns `(DataFrame, metadata)` |
|
|
393
393
|
| `read_csv_blob(blob_name, **kwargs)` | Read CSV file into DataFrame |
|
|
394
394
|
| `read_excel_blob(blob_name, **kwargs)` | Read Excel file into DataFrame |
|
|
395
|
+
| `read_json_blob(blob_name, **kwargs)` | Read JSON file into DataFrame |
|
|
395
396
|
| `scan_parquet_blob(blob_name)` | Lazy scan parquet, returns `(LazyFrame, metadata)` |
|
|
396
397
|
| `scan_csv_blob(blob_name, **kwargs)` | Lazy scan CSV file |
|
|
397
398
|
|
|
@@ -402,6 +403,7 @@ StorageHandler(
|
|
|
402
403
|
| `write_parquet_blob(blob_name, df)` | Write DataFrame to parquet |
|
|
403
404
|
| `write_csv_blob(blob_name, df, **kwargs)` | Write DataFrame to CSV |
|
|
404
405
|
| `write_excel_blob(blob_name, df, **kwargs)` | Write DataFrame to Excel |
|
|
406
|
+
| `write_json_blob(blob_name, df, **kwargs)` | Write DataFrame to JSON |
|
|
405
407
|
| `sink_parquet_blob(blob_name, lf)` | Stream LazyFrame to parquet |
|
|
406
408
|
|
|
407
409
|
#### Utility Methods
|
|
@@ -606,8 +608,12 @@ The CI pipeline will automatically:
|
|
|
606
608
|
|
|
607
609
|
The repository includes a Crow CI pipeline that automatically builds, tests, and publishes the package when a git tag is pushed:
|
|
608
610
|
|
|
609
|
-
|
|
610
|
-
|
|
611
|
+
1. Merge all branches you want to have part of a new release into main
|
|
612
|
+
1. Check tag for upcoming semantic release version with `git sv rn`
|
|
613
|
+
1. If not already added, add your private key (whose public key is associated with your Codefloe Git user) to the SSH agent.
|
|
614
|
+
`ssh-add ~/.ssh/<your-private-key>`
|
|
615
|
+
1. Generate tag with version based on git commit messages with `git sv tag`
|
|
616
|
+
1. Create and push a git tag (e.g., `git push --tags`)
|
|
611
617
|
|
|
612
618
|
The pipeline will:
|
|
613
619
|
- Run linting and tests
|
|
@@ -655,6 +655,25 @@ class StorageHandler:
|
|
|
655
655
|
console = Console()
|
|
656
656
|
console.print(tree)
|
|
657
657
|
|
|
658
|
+
def read_json_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
659
|
+
"""Reads a JSON file from storage into a Polars DataFrame.
|
|
660
|
+
|
|
661
|
+
Args:
|
|
662
|
+
blob_name (str): The path/name of the file to read.
|
|
663
|
+
**kwargs: Additional keyword arguments passed to
|
|
664
|
+
`polars.read_json`.
|
|
665
|
+
|
|
666
|
+
Returns:
|
|
667
|
+
pl.DataFrame: The content of the JSON file as a Polars
|
|
668
|
+
DataFrame.
|
|
669
|
+
"""
|
|
670
|
+
if self.local:
|
|
671
|
+
file_path = self.base_path / blob_name
|
|
672
|
+
return pl.read_json(source=file_path, **kwargs)
|
|
673
|
+
else:
|
|
674
|
+
data = self._read_blob_as_bytes(blob_name)
|
|
675
|
+
return pl.read_json(source=BytesIO(data), **kwargs)
|
|
676
|
+
|
|
658
677
|
def read_csv_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
659
678
|
"""Reads a CSV file from storage into a Polars DataFrame.
|
|
660
679
|
|
|
@@ -838,6 +857,30 @@ class StorageHandler:
|
|
|
838
857
|
except Exception as e:
|
|
839
858
|
raise RuntimeError(f"Failed to write blob '{blob_name}': {e}") from e
|
|
840
859
|
|
|
860
|
+
def write_json_blob(self, blob_name: str, df: pl.DataFrame, **kwargs: Dict) -> None:
|
|
861
|
+
"""Writes a Polars DataFrame to a JSON file in storage.
|
|
862
|
+
|
|
863
|
+
Args:
|
|
864
|
+
blob_name (str): The path/name of the file to write to.
|
|
865
|
+
df (pl.DataFrame): The DataFrame to write.
|
|
866
|
+
**kwargs: Additional keyword arguments passed to
|
|
867
|
+
`DataFrame.write_json`.
|
|
868
|
+
|
|
869
|
+
Returns:
|
|
870
|
+
None
|
|
871
|
+
"""
|
|
872
|
+
if self.local:
|
|
873
|
+
file_path = self.base_path / blob_name
|
|
874
|
+
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
875
|
+
df.write_json(file=file_path, **kwargs)
|
|
876
|
+
else:
|
|
877
|
+
# Write the DataFrame to an in-memory stream
|
|
878
|
+
buffer = BytesIO()
|
|
879
|
+
df.write_json(file=buffer, **kwargs)
|
|
880
|
+
buffer.seek(0)
|
|
881
|
+
# Upload the bytes to Azure Blob Storage
|
|
882
|
+
self._write_blob_as_bytes(blob_name, buffer.getvalue())
|
|
883
|
+
|
|
841
884
|
def write_csv_blob(self, blob_name: str, df: pl.DataFrame, **kwargs: Dict) -> None:
|
|
842
885
|
"""Writes a Polars DataFrame to a CSV file in storage.
|
|
843
886
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|