laktory 0.0.1__tar.gz

laktory-0.0.1/.github/workflows/release.yml

@@ -0,0 +1,30 @@
+name: release
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  release:
+    if: ${{ github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release') }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install flit
+        run: pip install flit
+
+      - name: Build and publish laktory to pypi
+        run: make publish
+        env:
+          FLIT_INDEX_URL: ${{ vars.FLIT_INDEX_URL }}
+          FLIT_USERNAME: ${{ vars.FLIT_USERNAME }}
+          FLIT_PASSWORD: ${{ secrets.FLIT_PASSWORD }}

laktory-0.0.1/.github/workflows/test.yml

@@ -0,0 +1,40 @@
+name: test
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+  push:
+    branches:
+      - main
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        pyVersion:
+          - '3.9'
+          - '3.10'
+          - '3.11'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+#      - name: Unshallow
+#        run: git fetch --prune --unshallow
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.pyVersion }}
+
+      - name: Install flit
+        run: pip install flit
+
+      - name: Run tests
+        run: make dev install test
+
+#      - name: Publish test coverage
+#        uses: codecov/codecov-action@v1

laktory-0.0.1/.gitignore

@@ -0,0 +1,45 @@
+# --------------------------------------------------------------------------- #
+# Compiled files                                                              #
+# --------------------------------------------------------------------------- #
+
+# Compiled files
+**pytest_cache**
+**/**.pyc
+
+# --------------------------------------------------------------------------- #
+# Environment and configurations                                              #
+# --------------------------------------------------------------------------- #
+
+# PyCharm configuration
+**/.idea/**
+
+# Local settings and environment variables
+#settings_local/*.*
+
+# --------------------------------------------------------------------------- #
+# Installation                                                                #
+# --------------------------------------------------------------------------- #
+
+#**/*.egg-info/**
+dist
+#**/dist/**
+
+# --------------------------------------------------------------------------- #
+# Testing and coverage                                                        #
+# --------------------------------------------------------------------------- #
+
+# Coverage
+.coverage
+coverage.xml
+htmlcov
+
+# Test results
+junit
+
+# Tox files - Global
+#.tox_pip_cache_sdist
+#.tox_pip_cache_whl
+
+# Tox files - Package-specific
+#**/.tox/**
+

laktory-0.0.1/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Release History
+
+## [0.0.1] - 2023-07-13
+**Added**
+- Initial pypi release.

laktory-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 opencubes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

laktory-0.0.1/Makefile

@@ -0,0 +1,17 @@
+install:
+	flit install
+
+dev:
+	flit install -s
+
+test:
+	pytest --junitxml=junit/test-results.xml --cov=laktory --cov-report=xml --cov-report=html tests
+
+coverage:
+	open htmlcov/index.html
+
+build:
+	flit build
+
+publish:
+	flit publish

laktory-0.0.1/PKG-INFO

@@ -0,0 +1,26 @@
+Metadata-Version: 2.1
+Name: laktory
+Version: 0.0.1
+Summary: A DataOps framework for building a lakehouse
+Keywords: one,two
+Author-email: Olivier Soucy <osoucy.transactions@gmail.com>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Dist: pydantic>=2
+Requires-Dist: pyyaml
+Requires-Dist: pyspark
+Requires-Dist: black ; extra == "dev"
+Requires-Dist: flit ; extra == "dev"
+Requires-Dist: pytest ; extra == "test"
+Requires-Dist: pytest-cov ; extra == "test"
+Project-URL: Bug Tracker, https://github.com/opencubes-ai/laktory/issues
+Project-URL: Homepage, https://github.com/opencubes-ai/laktory
+Provides-Extra: dev
+Provides-Extra: test
+
+# laktory
+A DataOps framework for building a lakehouse.
+

laktory-0.0.1/README.md

@@ -0,0 +1,2 @@
+# laktory
+A DataOps framework for building a lakehouse.

laktory-0.0.1/laktory/__init__.py

@@ -0,0 +1,3 @@
+from ._version import VERSION
+
+__version__ = VERSION

laktory-0.0.1/laktory/_version.py

@@ -0,0 +1 @@
+VERSION = "0.0.1"

laktory-0.0.1/laktory/contants.py

@@ -0,0 +1,17 @@
+import inspect
+from pyspark.sql import types
+
+
+SUPPORTED_TYPES = {}
+for k in vars(types):
+    o = getattr(types, k)
+    if "Type" in k and inspect.isclass(o):
+        try:
+            typ = o()
+        except TypeError:
+            continue
+
+        if not hasattr(typ, "simpleString"):
+            continue
+
+        SUPPORTED_TYPES[typ.simpleString()] = typ

laktory-0.0.1/laktory/models/__init__.py

@@ -0,0 +1,4 @@
+from .catalog import Catalog
+from .column import Column
+from .database import Database
+from .table import Table

laktory-0.0.1/laktory/models/base.py

@@ -0,0 +1,17 @@
+import yaml
+import json
+from pydantic import BaseModel as _BaseModel
+
+
+class BaseModel(_BaseModel):
+    pass
+
+    @classmethod
+    def model_validate_yaml(cls, fp):
+        data = yaml.safe_load(fp)
+        return cls.model_validate(data)
+
+    @classmethod
+    def model_validate_json_file(cls, fp):
+        data = json.load(fp)
+        return cls.model_validate(data)

laktory-0.0.1/laktory/models/catalog.py

@@ -0,0 +1,11 @@
+from pydantic import computed_field
+
+from laktory.models.base import BaseModel
+from laktory.models.database import Database
+
+
+class Catalog(BaseModel):
+    name: str
+    comment: str = None
+    databases: list[Database] = []
+    is_unity: bool = True

laktory-0.0.1/laktory/models/column.py

@@ -0,0 +1,54 @@
+from typing import Literal
+from pydantic import computed_field
+
+from laktory.contants import SUPPORTED_TYPES
+from laktory.models.base import BaseModel
+
+
+class Column(BaseModel):
+    name: str
+    type: Literal[tuple(SUPPORTED_TYPES.keys())] = "string"
+    comment: str = None
+    unit: str = None
+    pii: bool = None
+    func_name: str = None
+    input_cols: list[str] = []
+    func_kwargs: dict = {}
+    parent_id: str = None
+
+    @computed_field
+    @property
+    def table_name(self) -> str:
+        if self.parent_id is None:
+            return None
+        return self.parent_id.split(".")[-1]
+
+    @computed_field
+    @property
+    def schema_name(self) -> str:
+        if self.parent_id is None or len(self.parent_id.split(".")) < 2:
+            return None
+        return self.parent_id.split(".")[-2]
+
+    @computed_field
+    @property
+    def database_name(self) -> str:
+        return self.schema_name
+
+    @computed_field
+    @property
+    def catalog_name(self) -> str:
+        if self.parent_id is None or len(self.parent_id.split(".")) < 3:
+            return None
+        return self.parent_id.split(".")[-3]
+
+
+if __name__ == "__main__":
+    speed = Column(
+        name="airspeed",
+        type="double",
+        unit="kt",
+        parent_id="lakehouse.flights.f012",
+    )
+
+    print(speed)

laktory-0.0.1/laktory/models/database.py

@@ -0,0 +1,18 @@
+from pydantic import computed_field
+
+from laktory.models.base import BaseModel
+from laktory.models.table import Table
+
+
+class Database(BaseModel):
+    name: str
+    comment: str = None
+    tables: list[Table] = []
+    parent_id: str = None
+
+    @computed_field
+    @property
+    def catalog_name(self) -> str:
+        if self.parent_id is None or len(self.parent_id.split(".")) < 1:
+            return None
+        return self.parent_id.split(".")[-1]

laktory-0.0.1/laktory/models/dummy.py

@@ -0,0 +1,30 @@
+from typing import Literal
+
+from pydantic import Field
+from pydantic import ConfigDict
+
+from laktory.contants import SUPPORTED_TYPES
+from laktory.models.base import BaseModel
+
+
+class Column(BaseModel):
+    name: str
+    type: Literal[tuple(SUPPORTED_TYPES.keys())] = "string"
+    comment: str = None
+    unit: str = None
+    pii: bool = None
+    udf_name: str = None
+    input_cols: list[str] = []
+    udf_kwargs: dict = {}
+    test: int = Field(...)  # Set as required field, but put at end of list of fields
+
+    model_config = ConfigDict(extra="forbid")
+
+
+if __name__ == "__main__":
+    speed = Column(
+        name="airspeed",
+        type="double",
+        unit="kt",
+        test=2,
+    )

laktory-0.0.1/laktory/models/table.py

@@ -0,0 +1,58 @@
+from typing import Literal
+
+from pydantic import computed_field
+
+from laktory.models.base import BaseModel
+from laktory.models.column import Column
+
+
+class Table(BaseModel):
+    name: str
+    columns: list[Column] = []
+    primary_key: str = None
+    comment: str = None
+    parent_id: str = None
+
+    # Lakehouse
+    # event_name: str = None
+    # pipeline_name: str = None
+    zone: Literal["BRONZE", "SILVER", "SILVER_STAR", "GOLD"] = None
+    # joins
+    # expectations
+
+    @computed_field
+    @property
+    def database_name(self) -> str:
+        if self.parent_id is None or len(self.parent_id.split(".")) < 1:
+            return None
+        return self.parent_id.split(".")[-1]
+
+    @computed_field
+    @property
+    def schema_name(self) -> str:
+        return self.database_name
+
+    @computed_field
+    @property
+    def catalog_name(self) -> str:
+        if self.parent_id is None or len(self.parent_id.split(".")) < 2:
+            return None
+        return self.parent_id.split(".")[-2]
+
+
+if __name__ == "__main__":
+    table = Table(
+        name="f1549",
+        columns=[
+            {
+                "name": "airspeed",
+                "type": "double",
+            },
+            {
+                "name": "altitude",
+                "type": "double",
+            },
+        ],
+    )
+
+    print(table)

laktory-0.0.1/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "laktory"
+authors = [
+    {name = "Olivier Soucy", email = "osoucy.transactions@gmail.com"},
+]
+description = "A DataOps framework for building a lakehouse"
+readme = "README.md"
+requires-python = ">=3.9"
+keywords = ["one", "two"]
+license = {text = "MIT"}
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dynamic = ["version"]
+dependencies = [
+    "pydantic>=2",
+    "pyyaml",
+    "pyspark",
+]
+[project.optional-dependencies]
+dev = [
+    "black",
+    "flit",
+]
+test = [
+    "pytest",
+    "pytest-cov",
+#    "tox"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/opencubes-ai/laktory"
+"Bug Tracker" = "https://github.com/opencubes-ai/laktory/issues"

laktory-0.0.1/tests/airspeed.json

@@ -0,0 +1,6 @@
+{
+  "name": "airspeed",
+  "type": "double",
+  "unit": "kt",
+  "parent_id": "lakehouse.flights.f1549"
+}

laktory-0.0.1/tests/airspeed.yaml

@@ -0,0 +1,4 @@
+name: "airspeed"
+type: "double"
+unit: "kt"
+parent_id: "lakehouse.flights.f1549"

laktory-0.0.1/tests/test_catalog.py

@@ -0,0 +1,13 @@
+from laktory.models import Catalog
+
+
+def test_model():
+    cat = Catalog(
+        name="lakehouse",
+    )
+
+    assert cat.name == "lakehouse"
+
+
+if __name__ == "__main__":
+    test_model()

laktory-0.0.1/tests/test_column.py

@@ -0,0 +1,42 @@
+import os
+
+from laktory.models import Column
+
+AIRSPEED = {
+    "name": "airspeed",
+    "type": "double",
+    "unit": "kt",
+    "parent_id": "lakehouse.flights.f1549",
+}
+
+root_dir = os.path.dirname(__file__)
+
+
+def test_model():
+    c0 = Column(**AIRSPEED)
+    c1 = Column.model_validate(AIRSPEED)
+    assert c1.type == "double"
+    assert c1.catalog_name == "lakehouse"
+    assert c1.schema_name == "flights"
+    assert c1.table_name == "f1549"
+    assert "func_name" in c1.model_fields
+    assert "table_name" in c1.model_computed_fields
+    assert c0 == c1
+
+
+def test_read():
+    c0 = Column(**AIRSPEED)
+
+    with open(f"{root_dir}/airspeed.yaml", "r") as fp:
+        c1 = Column.model_validate_yaml(fp)
+
+    with open(f"{root_dir}/airspeed.json", "r") as fp:
+        c2 = Column.model_validate_json_file(fp)
+
+    assert c1 == c0
+    assert c2 == c0
+
+
+if __name__ == "__main__":
+    test_model()
+    test_read()

laktory-0.0.1/tests/test_constants.py

@@ -0,0 +1,14 @@
+from pyspark.sql.types import DoubleType
+from pyspark.sql.types import StringType
+from laktory.contants import SUPPORTED_TYPES
+
+
+def test_constants():
+
+    assert SUPPORTED_TYPES["double"] == DoubleType()
+    assert "str" not in SUPPORTED_TYPES
+    assert SUPPORTED_TYPES["string"] == StringType()
+
+
+if __name__ == "__main__":
+    test_constants()

laktory-0.0.1/tests/test_database.py

@@ -0,0 +1,50 @@
+from laktory.models import Table
+from laktory.models import Database
+from laktory.models import Column
+
+
+def test_model():
+    db = Database(
+        name="flights",
+        tables=[
+            Table(
+                name="f1549",
+                columns=[
+                    {
+                        "name": "airspeed",
+                        "type": "double",
+                    },
+                    {
+                        "name": "altitude",
+                        "type": "double",
+                    },
+                ],
+                zone="SILVER",
+                parent_id="lakehouse.flights",
+            ),
+            Table(
+                name="f0002",
+                columns=[
+                    {
+                        "name": "airspeed",
+                        "type": "double",
+                    },
+                    {
+                        "name": "altitude",
+                        "type": "double",
+                    },
+                ],
+                zone="SILVER",
+                parent_id="lakehouse.flights",
+            ),
+        ],
+    )
+
+    print(db.tables[0].columns[0].name)
+
+    assert db.tables[0].columns[0].name == "airspeed"
+    assert type(db.tables[0].columns[0]) == Column
+
+
+if __name__ == "__main__":
+    test_model()

laktory-0.0.1/tests/test_table.py

@@ -0,0 +1,39 @@
+import pytest
+from pydantic import ValidationError
+
+from laktory.models import Column
+from laktory.models import Table
+
+
+def test_model():
+    table = Table(
+        name="f1549",
+        columns=[
+            {
+                "name": "airspeed",
+                "type": "double",
+            },
+            {
+                "name": "altitude",
+                "type": "double",
+            },
+        ],
+        zone="SILVER",
+        parent_id="lakehouse.flights",
+    )
+
+    assert table.columns == [
+        Column(name="airspeed", type="double"),
+        Column(name="altitude", type="double"),
+    ]
+    assert table.catalog_name == "lakehouse"
+    assert table.schema_name == "flights"
+    assert table.zone == "SILVER"
+
+    # Invalid zone
+    with pytest.raises(ValidationError):
+        Table(name="f0001", zone="ROUGE")
+
+
+if __name__ == "__main__":
+    test_model()