permafrost-framework 0.6.0__tar.gz

permafrost_framework-0.6.0/LICENSE

@@ -0,0 +1,121 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship made available under
+      the License, as indicated by a copyright notice that is included in
+      or attached to the work (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other transformations
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean, as submitted to the Licensor for inclusion
+      in the Work by the copyright owner or by an individual or Legal Entity
+      authorized to submit on behalf of the copyright owner.
+
+      "Contributor" shall mean Licensor and any Legal Entity on behalf of
+      whom a Contribution has been received by the Licensor and included
+      within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work; and
+
+      (d) If the Work includes a "NOTICE" text file, you must include a
+          readable copy of the attribution notices contained within such
+          NOTICE file.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution submitted for inclusion in the Work shall be under
+      the terms and conditions of this License, without any additional terms.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or agreed
+      to in writing, Licensor provides the Work on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.
+
+   8. Limitation of Liability. In no event and under no legal theory shall
+      any Contributor be liable for damages arising from this License.
+
+   9. Accepting Warranty or Additional Liability. While redistributing the
+      Work, You may offer acceptance of support, warranty, indemnity, or
+      other liability obligations. However, in accepting such obligations,
+      You may charge a fee, and You must make it clear that any warranties
+      are Your own, and not on behalf of the other Contributors.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2026 Permafrost Contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

permafrost_framework-0.6.0/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.4
+Name: permafrost-framework
+Version: 0.6.0
+Summary: Distributed intelligent compression for long-term data archival
+Author: Permafrost Contributors
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/caua-ferreira/permafrost-framework
+Project-URL: Repository, https://github.com/caua-ferreira/permafrost-framework
+Keywords: compression,archival,lzma,cold-storage,permafrost,duckdb,distributed
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Archiving :: Compression
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=12.0.0
+Requires-Dist: zstandard>=0.21.0
+Requires-Dist: duckdb>=0.9.0
+Requires-Dist: typer>=0.9.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: fastapi>=0.100.0
+Requires-Dist: uvicorn>=0.23.0
+Requires-Dist: httpx>=0.24.0
+Provides-Extra: s3
+Requires-Dist: boto3>=1.26.0; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: google-cloud-storage>=2.10.0; extra == "gcs"
+Provides-Extra: azure
+Requires-Dist: azure-storage-blob>=12.17.0; extra == "azure"
+Provides-Extra: all-cloud
+Requires-Dist: boto3>=1.26.0; extra == "all-cloud"
+Requires-Dist: google-cloud-storage>=2.10.0; extra == "all-cloud"
+Requires-Dist: azure-storage-blob>=12.17.0; extra == "all-cloud"
+Provides-Extra: spark
+Requires-Dist: pyspark>=4.0; extra == "spark"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: pyspark>=4.0; extra == "dev"
+Dynamic: license-file
+
+# ❄️ Permafrost Data Framework
+
+<div align="center">
+
+[![PyPI version](https://badge.fury.io/py/permafrost-framework.svg)](https://pypi.org/project/permafrost-framework/)
+[![Tests](https://github.com/caua-ferreira/permafrost-framework/actions/workflows/tests.yml/badge.svg)](https://github.com/caua-ferreira/permafrost-framework/actions)
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.10%2B-yellow)](https://pypi.org/project/permafrost-framework/)
+[![Docs](https://img.shields.io/badge/docs-mkdocs-00D4FF)](https://caua-ferreira.github.io/permafrost-framework)
+
+**Plataforma distribuída de compressão inteligente para arquivamento digital de longo prazo.**
+
+</div>
+
+## Instalação
+
+```bash
+pip install permafrost-framework
+```
+
+## Quick Start
+
+```python
+import permafrost as pf
+
+metrics = pf.freeze(df, "vendas.permafrost", codec=pf.CODEC_LZMA2, partition_by="ano")
+print(f"Ratio: {metrics['ratio']:.2f}x")
+
+df_back = pf.thaw("vendas.permafrost", verify=True)
+df_2023 = pf.thaw("vendas.permafrost", filter={"ano": 2023})  # sparse index
+```
+
+## Benchmarks (medidos)
+
+| Dado | Original | .permafrost | Ratio |
+|------|----------|-------------|-------|
+| CSV corporativo 80k linhas | 5.85 MB | **0.678 MB** | **8.37x** |
+| JSONL social media 5k posts | 1.44 MB | **0.043 MB** | **33x** |
+| 1 TB no Glacier Deep Archive | $0.99/mês | **$0.12/mês** | **-88%** |
+
+91/91 testes passando — ver [EVIDENCE.md](EVIDENCE.md).
+
+## Docs
+
+[caua-ferreira.github.io/permafrost-framework](https://caua-ferreira.github.io/permafrost-framework)
+
+## Licença
+
+Apache License 2.0

permafrost_framework-0.6.0/README.md

@@ -0,0 +1,49 @@
+# ❄️ Permafrost Data Framework
+
+<div align="center">
+
+[![PyPI version](https://badge.fury.io/py/permafrost-framework.svg)](https://pypi.org/project/permafrost-framework/)
+[![Tests](https://github.com/caua-ferreira/permafrost-framework/actions/workflows/tests.yml/badge.svg)](https://github.com/caua-ferreira/permafrost-framework/actions)
+[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.10%2B-yellow)](https://pypi.org/project/permafrost-framework/)
+[![Docs](https://img.shields.io/badge/docs-mkdocs-00D4FF)](https://caua-ferreira.github.io/permafrost-framework)
+
+**Plataforma distribuída de compressão inteligente para arquivamento digital de longo prazo.**
+
+</div>
+
+## Instalação
+
+```bash
+pip install permafrost-framework
+```
+
+## Quick Start
+
+```python
+import permafrost as pf
+
+metrics = pf.freeze(df, "vendas.permafrost", codec=pf.CODEC_LZMA2, partition_by="ano")
+print(f"Ratio: {metrics['ratio']:.2f}x")
+
+df_back = pf.thaw("vendas.permafrost", verify=True)
+df_2023 = pf.thaw("vendas.permafrost", filter={"ano": 2023})  # sparse index
+```
+
+## Benchmarks (medidos)
+
+| Dado | Original | .permafrost | Ratio |
+|------|----------|-------------|-------|
+| CSV corporativo 80k linhas | 5.85 MB | **0.678 MB** | **8.37x** |
+| JSONL social media 5k posts | 1.44 MB | **0.043 MB** | **33x** |
+| 1 TB no Glacier Deep Archive | $0.99/mês | **$0.12/mês** | **-88%** |
+
+91/91 testes passando — ver [EVIDENCE.md](EVIDENCE.md).
+
+## Docs
+
+[caua-ferreira.github.io/permafrost-framework](https://caua-ferreira.github.io/permafrost-framework)
+
+## Licença
+
+Apache License 2.0

permafrost_framework-0.6.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "permafrost-framework"
+version = "0.6.0"
+description = "Distributed intelligent compression for long-term data archival"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.10"
+authors = [{ name = "Permafrost Contributors" }]
+keywords = ["compression","archival","lzma","cold-storage","permafrost","duckdb","distributed"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: System :: Archiving :: Compression",
+]
+dependencies = [
+    "numpy>=1.24.0", "pandas>=2.0.0", "pyarrow>=12.0.0",
+    "zstandard>=0.21.0", "duckdb>=0.9.0",
+    "typer>=0.9.0", "rich>=13.0.0",
+    "fastapi>=0.100.0", "uvicorn>=0.23.0", "httpx>=0.24.0",
+]
+
+[project.optional-dependencies]
+s3        = ["boto3>=1.26.0"]
+gcs       = ["google-cloud-storage>=2.10.0"]
+azure     = ["azure-storage-blob>=12.17.0"]
+all-cloud = ["boto3>=1.26.0","google-cloud-storage>=2.10.0","azure-storage-blob>=12.17.0"]
+spark     = ["pyspark>=4.0"]
+dev       = ["pytest>=7.0","pytest-cov","build","twine","pyspark>=4.0"]
+
+[project.urls]
+Homepage  = "https://github.com/caua-ferreira/permafrost-framework"
+Repository = "https://github.com/caua-ferreira/permafrost-framework"
+
+[project.scripts]
+permafrost = "permafrost.cli:app"
+
+[tool.setuptools.packages.find]
+where = ["src"]

permafrost_framework-0.6.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

permafrost_framework-0.6.0/src/permafrost/__init__.py

@@ -0,0 +1,114 @@
+"""
+Permafrost Data Framework
+=========================
+Plataforma distribuída de compressão inteligente para arquivamento digital de longo prazo.
+
+Uso rápido:
+    from permafrost import freeze, thaw, audit
+    from permafrost import PermafrostCatalog, SchemaDetector
+    from permafrost import freeze_to, thaw_from          # cloud
+    from permafrost import PermafrostMaster, PermafrostWorker, PermafrostClient  # cluster
+
+Formatos suportados:
+    freeze(df, "arquivo.permafrost")                     # DataFrame tabular
+    freeze(detector.detect("dados.jsonl")[0], ...)       # JSONL / NoSQL
+    freeze_file("dados.csv", "saida.permafrost")         # streaming, sem carregar tudo na RAM
+    freeze_to(df, "s3://bucket/dados.permafrost")        # direto para cloud
+
+Exemplos:
+    >>> import permafrost as pf
+    >>> metrics = pf.freeze(df, "vendas.permafrost", codec=pf.CODEC_LZMA2)
+    >>> print(f"Ratio: {metrics['ratio']:.2f}x")
+    >>> df_back = pf.thaw("vendas.permafrost")
+    >>> info = pf.audit("vendas.permafrost")   # sem descomprimir
+
+Links:
+    GitHub: https://github.com/caua-ferreira/permafrost-framework
+    Docs:   https://github.com/caua-ferreira/permafrost-framework/tree/main/docs
+"""
+
+__version__  = "0.6.0"
+__author__   = "Permafrost Contributors"
+__license__  = "Apache-2.0"
+
+# ── Core codec ────────────────────────────────────────────────────────────────
+from permafrost.codec import (
+    freeze,
+    thaw,
+    audit,
+    # Codec IDs
+    CODEC_ZSTD,
+    CODEC_LZMA2,
+    CODEC_ZPAQ,
+    # Quantization levels
+    QUANT_NONE,
+    QUANT_HIGH,
+    QUANT_MEDIUM,
+    QUANT_LOW,
+    # Format constants
+    MAGIC,
+    EOF_MAGIC,
+)
+
+# ── Schema detection (SQL + NoSQL + JSONL) ────────────────────────────────────
+from permafrost.schema_detector import (
+    SchemaDetector,
+    DataType,
+    FieldKind,
+)
+
+# ── Chunk mode (streaming — datasets > RAM) ───────────────────────────────────
+from permafrost.chunk_mode import (
+    freeze_stream,
+    freeze_file,
+    thaw_iter,
+)
+
+# ── Catalog (DuckDB index) ────────────────────────────────────────────────────
+from permafrost.catalog import PermafrostCatalog
+
+# ── Cloud storage adapters ────────────────────────────────────────────────────
+from permafrost.storage import (
+    LocalAdapter,
+    S3Adapter,
+    GCSAdapter,
+    AzureAdapter,
+    storage_from_uri,
+    parse_uri,
+    freeze_to,
+    thaw_from,
+    audit_remote,
+)
+
+# ── Cluster (distributed processing) ─────────────────────────────────────────
+from permafrost.cluster import (
+    PermafrostMaster,
+    PermafrostWorker,
+    PermafrostClient,
+)
+
+__all__ = [
+    # Core
+    "freeze", "thaw", "audit",
+    # Codecs
+    "CODEC_ZSTD", "CODEC_LZMA2", "CODEC_ZPAQ",
+    # Quant levels
+    "QUANT_NONE", "QUANT_HIGH", "QUANT_MEDIUM", "QUANT_LOW",
+    # Schema
+    "SchemaDetector", "DataType", "FieldKind",
+    # Chunk mode
+    "freeze_stream", "freeze_file", "thaw_iter",
+    # Catalog
+    "PermafrostCatalog",
+    # Storage
+    "LocalAdapter", "S3Adapter", "GCSAdapter", "AzureAdapter",
+    "storage_from_uri", "parse_uri", "freeze_to", "thaw_from", "audit_remote",
+    # Cluster
+    "PermafrostMaster", "PermafrostWorker", "PermafrostClient",
+]
+# ── Spark DataSource API v2 ───────────────────────────────────────────────────
+try:
+    from permafrost.spark import PermafrostDataSource, register as spark_register
+    __all__ += ["PermafrostDataSource", "spark_register"]
+except ImportError:
+    pass   # PySpark não instalado — ok

permafrost_framework-0.6.0/src/permafrost/__main__.py

@@ -0,0 +1,64 @@
+"""
+Entrypoint para execução via `python -m permafrost`.
+
+Uso:
+  python -m permafrost master [--host HOST] [--port PORT]
+  python -m permafrost worker --master URL [--host HOST] [--port PORT] [--id ID]
+  python -m permafrost freeze arquivo.csv
+  python -m permafrost thaw arquivo.permafrost
+"""
+import sys
+
+def main():
+    if len(sys.argv) < 2:
+        print("Uso: python -m permafrost <comando>")
+        print("Comandos: master | worker | freeze | thaw | audit | catalog")
+        sys.exit(1)
+
+    cmd = sys.argv[1]
+
+    if cmd == "master":
+        import argparse, uvicorn
+        from permafrost.cluster import PermafrostMaster
+        p = argparse.ArgumentParser(description="Permafrost Master node")
+        p.add_argument("--host", default="0.0.0.0")
+        p.add_argument("--port", type=int, default=8700)
+        p.add_argument("--max-retries", type=int, default=3)
+        args = p.parse_args(sys.argv[2:])
+        master = PermafrostMaster(host=args.host, port=args.port)
+        master.MAX_RETRIES = args.max_retries
+        print(f"❄  Permafrost Master iniciando em {args.host}:{args.port}")
+        uvicorn.run(master.app, host=args.host, port=args.port, log_level="info")
+
+    elif cmd == "worker":
+        import argparse, uvicorn
+        from permafrost.cluster import PermafrostWorker
+        p = argparse.ArgumentParser(description="Permafrost Worker node")
+        p.add_argument("--master", required=True, help="URL do master (ex: http://master:8700)")
+        p.add_argument("--host", default="0.0.0.0")
+        p.add_argument("--port", type=int, default=8801)
+        p.add_argument("--id",   default=None, help="ID único do worker")
+        args = p.parse_args(sys.argv[2:])
+        worker = PermafrostWorker(
+            master_url=args.master,
+            host=args.host,
+            port=args.port,
+            worker_id=args.id,
+        )
+        print(f"❄  Permafrost Worker {worker.worker_id} → {args.master}")
+        worker.run(auto_register=True)
+
+    elif cmd in ("freeze", "thaw", "audit", "verify", "catalog"):
+        # Delegar para a CLI typer
+        from permafrost.cli import app
+        sys.argv = ["permafrost"] + sys.argv[1:]
+        app()
+
+    else:
+        print(f"Comando desconhecido: {cmd}")
+        print("Comandos disponíveis: master | worker | freeze | thaw | audit | catalog")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()