pyfabric-dev 0.5.2__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyfabric_dev-0.5.2/pyfabric_dev.egg-info → pyfabric_dev-0.6.0}/PKG-INFO +4 -4
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/README.md +1 -1
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/__init__.py +1 -1
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/spark.py +13 -19
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0/pyfabric_dev.egg-info}/PKG-INFO +4 -4
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/SOURCES.txt +2 -1
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/requires.txt +2 -2
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyproject.toml +3 -3
- pyfabric_dev-0.6.0/tests/test_spark_session.py +27 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/LICENSE +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/NOTICE +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/__init__.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/generate_notebooks.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/run_notebook.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/run_pipeline.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/run_tests_parallel.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/defs.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/fabric.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/functions.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/local_config.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/local_env.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/mock_notebookutils.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/__init__.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/hooks.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/notebook.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/pipeline.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/dependency_links.txt +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/entry_points.txt +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/top_level.txt +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/setup.cfg +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/tests/test_functions_merge.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/tests/test_generate_notebooks.py +0 -0
- {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/tests/test_smoke.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyfabric-dev
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Local development framework for Microsoft Fabric notebooks: source-of-truth Python, generated .Notebook artifacts, mocked Fabric APIs, parallel local testing.
|
|
5
5
|
Author-email: CashHero <engineering@cashhero.ai>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -21,8 +21,8 @@ Requires-Python: >=3.13
|
|
|
21
21
|
Description-Content-Type: text/markdown
|
|
22
22
|
License-File: LICENSE
|
|
23
23
|
License-File: NOTICE
|
|
24
|
-
Requires-Dist: pyspark>=
|
|
25
|
-
Requires-Dist: delta-spark>=
|
|
24
|
+
Requires-Dist: pyspark>=4.1.1
|
|
25
|
+
Requires-Dist: delta-spark>=4.2.0
|
|
26
26
|
Requires-Dist: azure-identity>=1.15.0
|
|
27
27
|
Requires-Dist: azure-keyvault-secrets>=4.7.0
|
|
28
28
|
Requires-Dist: python-dotenv>=1.0.0
|
|
@@ -63,7 +63,7 @@ Fabric notebooks have two pain points:
|
|
|
63
63
|
|
|
64
64
|
## Install
|
|
65
65
|
|
|
66
|
-
Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark).
|
|
66
|
+
Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark; JDK 21 matches Fabric Runtime 2.0).
|
|
67
67
|
|
|
68
68
|
```bash
|
|
69
69
|
pip install pyfabric-dev
|
|
@@ -30,7 +30,7 @@ Fabric notebooks have two pain points:
|
|
|
30
30
|
|
|
31
31
|
## Install
|
|
32
32
|
|
|
33
|
-
Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark).
|
|
33
|
+
Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark; JDK 21 matches Fabric Runtime 2.0).
|
|
34
34
|
|
|
35
35
|
```bash
|
|
36
36
|
pip install pyfabric-dev
|
|
@@ -15,7 +15,6 @@ import logging
|
|
|
15
15
|
import os
|
|
16
16
|
import sys
|
|
17
17
|
from logging import Logger
|
|
18
|
-
from pathlib import Path
|
|
19
18
|
from typing import Literal
|
|
20
19
|
|
|
21
20
|
from pyspark.sql import SparkSession
|
|
@@ -157,7 +156,7 @@ def _create_local_spark_session() -> SparkSession:
|
|
|
157
156
|
from pyfabric_dev.local_config import SPARK_MASTER, SPARK_WAREHOUSE_DIR, SPARK_METASTORE_DIR
|
|
158
157
|
|
|
159
158
|
try:
|
|
160
|
-
import delta
|
|
159
|
+
import delta # noqa: F401 — detects whether delta-spark is installed
|
|
161
160
|
import importlib.metadata
|
|
162
161
|
from pyspark import __version__ as spark_version
|
|
163
162
|
|
|
@@ -202,8 +201,15 @@ def _create_local_spark_session() -> SparkSession:
|
|
|
202
201
|
.config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
|
|
203
202
|
.config("spark.sql.sources.default", "delta")
|
|
204
203
|
|
|
204
|
+
# Fabric Runtime 2.0 defaults that differ from upstream Spark 4.1
|
|
205
|
+
# (https://learn.microsoft.com/fabric/data-engineering/runtime-2-0-migration-guide).
|
|
206
|
+
# Fabric keeps ANSI mode off, so a bad cast yields null instead of failing
|
|
207
|
+
# the job; upstream Spark 4 turns it on. New Delta tables get deletion
|
|
208
|
+
# vectors. Runtime 2.0 no longer sets delta.optimizeWrite.enabled.
|
|
209
|
+
.config("spark.sql.ansi.enabled", "false")
|
|
210
|
+
.config("spark.databricks.delta.properties.defaults.enableDeletionVectors", "true")
|
|
211
|
+
|
|
205
212
|
# Delta Lake optimizations
|
|
206
|
-
.config("spark.databricks.delta.optimizeWrite.enabled", "true")
|
|
207
213
|
.config("spark.sql.sources.partitionOverwriteMode", "dynamic")
|
|
208
214
|
|
|
209
215
|
# Compression optimizations
|
|
@@ -220,23 +226,11 @@ def _create_local_spark_session() -> SparkSession:
|
|
|
220
226
|
.config("spark.sql.legacy.createHiveTableByDefault", "false")
|
|
221
227
|
)
|
|
222
228
|
|
|
223
|
-
#
|
|
229
|
+
# Delta Lake JARs: delta-spark's helper resolves the Maven artifact built for
|
|
230
|
+
# the installed Spark (e.g. io.delta:delta-spark_4.1_2.13:4.2.0).
|
|
224
231
|
if delta_version:
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
delta_jars = list(delta_package_path.glob("**/delta-*.jar"))
|
|
228
|
-
|
|
229
|
-
if delta_jars:
|
|
230
|
-
jar_paths = ",".join([str(jar.absolute()) for jar in delta_jars])
|
|
231
|
-
builder = builder.config("spark.jars", jar_paths)
|
|
232
|
-
else:
|
|
233
|
-
# Use Maven coordinates
|
|
234
|
-
spark_major_minor = ".".join(spark_version.split(".")[:2])
|
|
235
|
-
scala_version = "2.12" if spark_major_minor.startswith("3.") else "2.13"
|
|
236
|
-
delta_packages = f"io.delta:delta-spark_{scala_version}:{delta_version}"
|
|
237
|
-
builder = builder.config("spark.jars.packages", delta_packages)
|
|
238
|
-
except Exception:
|
|
239
|
-
pass
|
|
232
|
+
from delta import configure_spark_with_delta_pip
|
|
233
|
+
builder = configure_spark_with_delta_pip(builder)
|
|
240
234
|
|
|
241
235
|
return builder.getOrCreate()
|
|
242
236
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyfabric-dev
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Local development framework for Microsoft Fabric notebooks: source-of-truth Python, generated .Notebook artifacts, mocked Fabric APIs, parallel local testing.
|
|
5
5
|
Author-email: CashHero <engineering@cashhero.ai>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -21,8 +21,8 @@ Requires-Python: >=3.13
|
|
|
21
21
|
Description-Content-Type: text/markdown
|
|
22
22
|
License-File: LICENSE
|
|
23
23
|
License-File: NOTICE
|
|
24
|
-
Requires-Dist: pyspark>=
|
|
25
|
-
Requires-Dist: delta-spark>=
|
|
24
|
+
Requires-Dist: pyspark>=4.1.1
|
|
25
|
+
Requires-Dist: delta-spark>=4.2.0
|
|
26
26
|
Requires-Dist: azure-identity>=1.15.0
|
|
27
27
|
Requires-Dist: azure-keyvault-secrets>=4.7.0
|
|
28
28
|
Requires-Dist: python-dotenv>=1.0.0
|
|
@@ -63,7 +63,7 @@ Fabric notebooks have two pain points:
|
|
|
63
63
|
|
|
64
64
|
## Install
|
|
65
65
|
|
|
66
|
-
Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark).
|
|
66
|
+
Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark; JDK 21 matches Fabric Runtime 2.0).
|
|
67
67
|
|
|
68
68
|
```bash
|
|
69
69
|
pip install pyfabric-dev
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pyfabric-dev"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Local development framework for Microsoft Fabric notebooks: source-of-truth Python, generated .Notebook artifacts, mocked Fabric APIs, parallel local testing."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "Apache-2.0" }
|
|
@@ -25,8 +25,8 @@ classifiers = [
|
|
|
25
25
|
]
|
|
26
26
|
|
|
27
27
|
dependencies = [
|
|
28
|
-
"pyspark>=
|
|
29
|
-
"delta-spark>=
|
|
28
|
+
"pyspark>=4.1.1",
|
|
29
|
+
"delta-spark>=4.2.0",
|
|
30
30
|
"azure-identity>=1.15.0",
|
|
31
31
|
"azure-keyvault-secrets>=4.7.0",
|
|
32
32
|
"python-dotenv>=1.0.0",
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""The local session mirrors Fabric Runtime 2.0's defaults, not upstream Spark 4.1's."""
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
import pyfabric_dev.local_env # noqa: F401 (bootstrap mocks + local dirs)
|
|
5
|
+
from pyfabric_dev.spark import create_spark_session
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.fixture(scope="module")
|
|
9
|
+
def spark():
|
|
10
|
+
return create_spark_session()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_ansi_mode_off_like_fabric(spark):
|
|
14
|
+
assert spark.conf.get("spark.sql.ansi.enabled") == "false"
|
|
15
|
+
# Under ANSI (the upstream Spark 4 default) this raises instead of returning null.
|
|
16
|
+
assert spark.sql("SELECT CAST('x' AS INT) AS v").first()["v"] is None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_new_delta_tables_get_deletion_vectors(spark):
|
|
20
|
+
table = "test_session_deletion_vectors_tbl"
|
|
21
|
+
spark.sql(f"DROP TABLE IF EXISTS {table}")
|
|
22
|
+
spark.range(1).write.format("delta").saveAsTable(table)
|
|
23
|
+
try:
|
|
24
|
+
props = spark.sql(f"DESCRIBE DETAIL {table}").first()["properties"]
|
|
25
|
+
assert props.get("delta.enableDeletionVectors") == "true"
|
|
26
|
+
finally:
|
|
27
|
+
spark.sql(f"DROP TABLE IF EXISTS {table}")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|