local-data-platform 0.1.0__tar.gz

local_data_platform-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.1
+Name: local-data-platform
+Version: 0.1.0
+Summary: 
+Author: Tushar Choudhary
+Author-email: 151359025+tusharchou@users.noreply.github.com
+Requires-Python: >=3.12,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: pyarrow (>=17.0.0,<18.0.0)
+Requires-Dist: pyiceberg (>=0.7.1,<0.8.0)
+Description-Content-Type: text/markdown
+
+# Poetry Documentation
+
+`cd local-data-platform`
+
+`poetry init`
+
+`poetry install`
+
+`poetry run python hello_world`

local_data_platform-0.1.0/README.md

@@ -0,0 +1,9 @@
+# Poetry Documentation
+
+`cd local-data-platform`
+
+`poetry init`
+
+`poetry install`
+
+`poetry run python hello_world`

local_data_platform-0.1.0/local_data_platform/catalog/sql.py

@@ -0,0 +1,10 @@
+from pyiceberg.catalog.sql import SqlCatalog
+
+class LocalCatalog(SqlCatalog):
+
+    def __init__(self,  name: str, **properties: str):
+        # Set up the SQL catalog using SQLite and the defined warehouse path
+        super().__init__(name, **properties)
+
+
+

local_data_platform-0.1.0/local_data_platform/catalog.py

@@ -0,0 +1,27 @@
+import os
+from pyiceberg.catalog.sql import SqlCatalog
+
+class LocalCatalog(SqlCatalog):
+
+    def __init__(self,  name: str, **properties: str):
+        super().__init__(name, **properties)
+
+
+
+# Define the warehouse path
+warehouse_path = "./tmp/warehouse"
+
+# Ensure the directory exists
+os.makedirs(warehouse_path, exist_ok=True)
+
+# Set up the SQL catalog using SQLite and the defined warehouse path
+catalog = SqlCatalog(
+    "pyiceberg_catalog_db",
+    **{
+        "uri": f"sqlite:///{warehouse_path}/pyiceberg_catalog.db",  # Ensure .db file extension
+        "warehouse": f"file://{warehouse_path}",
+    },
+)
+
+# Verify if the catalog is set up correctly
+print("Catalog set up successfully:", catalog)

local_data_platform-0.1.0/local_data_platform/download.py

@@ -0,0 +1,22 @@
+import requests
+import pyarrow.parquet as pq
+
+#
+#
+# response = requests.get(
+#     'https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2023-01.parquet',
+# )
+#
+#
+# if response.status_code == 200:
+#     with open("/tmp/yellow_tripdata_2023-01.parquet", "w") as f:
+#         f.write(response.content)
+
+df = pq.read_table("/tmp/yellow_tripdata_2023-01.parquet")
+
+catalog.create_namespace("nyc_taxi")
+
+table = catalog.create_table(
+    "nyc_taxi.taxi_dataset",
+    schema=df.schema,
+)

local_data_platform-0.1.0/local_data_platform/hello_world.py

@@ -0,0 +1,2 @@
+def hello_world():
+    print("Hello, world")

local_data_platform-0.1.0/local_data_platform/pipelines/__init__.py

@@ -0,0 +1,17 @@
+from abc import ABC
+
+class Pipelines(ABC):
+    source: str
+    target: str
+
+    def __init__(self):
+        pass
+
+    def extract(self):
+        pass
+
+    def transform(self):
+        pass
+
+    def load(self):
+        pass

local_data_platform-0.1.0/local_data_platform/pipelines/python/base_pipeline.py

@@ -0,0 +1,10 @@
+class BasePipeline():
+    source
+    target
+    def extract(self):
+        pass
+
+    def transform(self):
+        pass
+    def load(self):
+        pass

local_data_platform-0.1.0/local_data_platform/pipelines/python/ingestion/create_table.py

@@ -0,0 +1,3 @@
+
+
+class LakehouseTable

local_data_platform-0.1.0/local_data_platform/source/base_source.py

@@ -0,0 +1,18 @@
+from abc import ABC
+
+class SourceFile(ABC):
+    """
+    A base class for SourceFile implementation
+
+    Args:
+        location (str): A URI or a path to a local file
+
+    Attributes:
+        location (str): The path to SourceFile instance
+        exists (boo): w
+
+    """
+    _name: str
+
+    def __init__(self, location: str):
+        self

local_data_platform-0.1.0/local_data_platform/source/parquet/read.py

@@ -0,0 +1,21 @@
+import os
+import  pyarrow.parquet as pq
+
+SAMPLE_PARQUET = os.path.abspath('/Users/tushar/PycharmProjects/local-data-platform/local-data-platform/sample_data/nyc_yellow_taxi/parquet/2023-01/yellow_tripdata_2023-01.parquet')
+
+# cwd_path = os.getcwd()
+# print(type(cwd_path))
+# print(cwd_path)
+# print(SAMPLE_PARQUET)
+# # print(path)
+# print(__file__)
+# print(os.path.dirname(__file__))
+# print(os.path.realpath(os.path.dirname(__file__)))
+# print(os.path.realpath(path))
+# real_path = os.path.realpath(path)
+# print("File exists:", os.path.exists(SAMPLE_PARQUET))
+# print("File exists:", os.path.exists(cwd_path))
+# print(cwd_path)
+df = pq.read_table(SAMPLE_PARQUET)
+#
+print(df.get_total_buffer_size())

local_data_platform-0.1.0/pyproject.toml

@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "local-data-platform"
+version = "0.1.0"
+description = ""
+authors = ["Tushar Choudhary <151359025+tusharchou@users.noreply.github.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.12"
+pyiceberg = "^0.7.1"
+pyarrow = "^17.0.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"