pyfabric-dev 0.5.2__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. {pyfabric_dev-0.5.2/pyfabric_dev.egg-info → pyfabric_dev-0.6.0}/PKG-INFO +4 -4
  2. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/README.md +1 -1
  3. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/__init__.py +1 -1
  4. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/spark.py +13 -19
  5. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0/pyfabric_dev.egg-info}/PKG-INFO +4 -4
  6. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/SOURCES.txt +2 -1
  7. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/requires.txt +2 -2
  8. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyproject.toml +3 -3
  9. pyfabric_dev-0.6.0/tests/test_spark_session.py +27 -0
  10. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/LICENSE +0 -0
  11. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/NOTICE +0 -0
  12. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/__init__.py +0 -0
  13. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/generate_notebooks.py +0 -0
  14. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/run_notebook.py +0 -0
  15. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/run_pipeline.py +0 -0
  16. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/cli/run_tests_parallel.py +0 -0
  17. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/defs.py +0 -0
  18. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/fabric.py +0 -0
  19. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/functions.py +0 -0
  20. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/local_config.py +0 -0
  21. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/local_env.py +0 -0
  22. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/mock_notebookutils.py +0 -0
  23. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/__init__.py +0 -0
  24. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/hooks.py +0 -0
  25. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/notebook.py +0 -0
  26. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev/runners/pipeline.py +0 -0
  27. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/dependency_links.txt +0 -0
  28. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/entry_points.txt +0 -0
  29. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/pyfabric_dev.egg-info/top_level.txt +0 -0
  30. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/setup.cfg +0 -0
  31. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/tests/test_functions_merge.py +0 -0
  32. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/tests/test_generate_notebooks.py +0 -0
  33. {pyfabric_dev-0.5.2 → pyfabric_dev-0.6.0}/tests/test_smoke.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyfabric-dev
3
- Version: 0.5.2
3
+ Version: 0.6.0
4
4
  Summary: Local development framework for Microsoft Fabric notebooks: source-of-truth Python, generated .Notebook artifacts, mocked Fabric APIs, parallel local testing.
5
5
  Author-email: CashHero <engineering@cashhero.ai>
6
6
  License: Apache-2.0
@@ -21,8 +21,8 @@ Requires-Python: >=3.13
21
21
  Description-Content-Type: text/markdown
22
22
  License-File: LICENSE
23
23
  License-File: NOTICE
24
- Requires-Dist: pyspark>=3.5.0
25
- Requires-Dist: delta-spark>=3.2.0
24
+ Requires-Dist: pyspark>=4.1.1
25
+ Requires-Dist: delta-spark>=4.2.0
26
26
  Requires-Dist: azure-identity>=1.15.0
27
27
  Requires-Dist: azure-keyvault-secrets>=4.7.0
28
28
  Requires-Dist: python-dotenv>=1.0.0
@@ -63,7 +63,7 @@ Fabric notebooks have two pain points:
63
63
 
64
64
  ## Install
65
65
 
66
- Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark).
66
+ Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark; JDK 21 matches Fabric Runtime 2.0).
67
67
 
68
68
  ```bash
69
69
  pip install pyfabric-dev
@@ -30,7 +30,7 @@ Fabric notebooks have two pain points:
30
30
 
31
31
  ## Install
32
32
 
33
- Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark).
33
+ Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark; JDK 21 matches Fabric Runtime 2.0).
34
34
 
35
35
  ```bash
36
36
  pip install pyfabric-dev
@@ -1,3 +1,3 @@
1
1
  """pyfabric-dev: local development framework for Microsoft Fabric notebooks."""
2
2
 
3
- __version__ = "0.5.2"
3
+ __version__ = "0.6.0"
@@ -15,7 +15,6 @@ import logging
15
15
  import os
16
16
  import sys
17
17
  from logging import Logger
18
- from pathlib import Path
19
18
  from typing import Literal
20
19
 
21
20
  from pyspark.sql import SparkSession
@@ -157,7 +156,7 @@ def _create_local_spark_session() -> SparkSession:
157
156
  from pyfabric_dev.local_config import SPARK_MASTER, SPARK_WAREHOUSE_DIR, SPARK_METASTORE_DIR
158
157
 
159
158
  try:
160
- import delta
159
+ import delta # noqa: F401 — detects whether delta-spark is installed
161
160
  import importlib.metadata
162
161
  from pyspark import __version__ as spark_version
163
162
 
@@ -202,8 +201,15 @@ def _create_local_spark_session() -> SparkSession:
202
201
  .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
203
202
  .config("spark.sql.sources.default", "delta")
204
203
 
204
+ # Fabric Runtime 2.0 defaults that differ from upstream Spark 4.1
205
+ # (https://learn.microsoft.com/fabric/data-engineering/runtime-2-0-migration-guide).
206
+ # Fabric keeps ANSI mode off, so a bad cast yields null instead of failing
207
+ # the job; upstream Spark 4 turns it on. New Delta tables get deletion
208
+ # vectors. Runtime 2.0 no longer sets delta.optimizeWrite.enabled.
209
+ .config("spark.sql.ansi.enabled", "false")
210
+ .config("spark.databricks.delta.properties.defaults.enableDeletionVectors", "true")
211
+
205
212
  # Delta Lake optimizations
206
- .config("spark.databricks.delta.optimizeWrite.enabled", "true")
207
213
  .config("spark.sql.sources.partitionOverwriteMode", "dynamic")
208
214
 
209
215
  # Compression optimizations
@@ -220,23 +226,11 @@ def _create_local_spark_session() -> SparkSession:
220
226
  .config("spark.sql.legacy.createHiveTableByDefault", "false")
221
227
  )
222
228
 
223
- # Configure Delta Lake JARs
229
+ # Delta Lake JARs: delta-spark's helper resolves the Maven artifact built for
230
+ # the installed Spark (e.g. io.delta:delta-spark_4.1_2.13:4.2.0).
224
231
  if delta_version:
225
- try:
226
- delta_package_path = Path(delta.__file__).parent
227
- delta_jars = list(delta_package_path.glob("**/delta-*.jar"))
228
-
229
- if delta_jars:
230
- jar_paths = ",".join([str(jar.absolute()) for jar in delta_jars])
231
- builder = builder.config("spark.jars", jar_paths)
232
- else:
233
- # Use Maven coordinates
234
- spark_major_minor = ".".join(spark_version.split(".")[:2])
235
- scala_version = "2.12" if spark_major_minor.startswith("3.") else "2.13"
236
- delta_packages = f"io.delta:delta-spark_{scala_version}:{delta_version}"
237
- builder = builder.config("spark.jars.packages", delta_packages)
238
- except Exception:
239
- pass
232
+ from delta import configure_spark_with_delta_pip
233
+ builder = configure_spark_with_delta_pip(builder)
240
234
 
241
235
  return builder.getOrCreate()
242
236
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyfabric-dev
3
- Version: 0.5.2
3
+ Version: 0.6.0
4
4
  Summary: Local development framework for Microsoft Fabric notebooks: source-of-truth Python, generated .Notebook artifacts, mocked Fabric APIs, parallel local testing.
5
5
  Author-email: CashHero <engineering@cashhero.ai>
6
6
  License: Apache-2.0
@@ -21,8 +21,8 @@ Requires-Python: >=3.13
21
21
  Description-Content-Type: text/markdown
22
22
  License-File: LICENSE
23
23
  License-File: NOTICE
24
- Requires-Dist: pyspark>=3.5.0
25
- Requires-Dist: delta-spark>=3.2.0
24
+ Requires-Dist: pyspark>=4.1.1
25
+ Requires-Dist: delta-spark>=4.2.0
26
26
  Requires-Dist: azure-identity>=1.15.0
27
27
  Requires-Dist: azure-keyvault-secrets>=4.7.0
28
28
  Requires-Dist: python-dotenv>=1.0.0
@@ -63,7 +63,7 @@ Fabric notebooks have two pain points:
63
63
 
64
64
  ## Install
65
65
 
66
- Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark).
66
+ Requires **Python 3.13 or 3.14** and a JDK 17+ on `$PATH` (for PySpark; JDK 21 matches Fabric Runtime 2.0).
67
67
 
68
68
  ```bash
69
69
  pip install pyfabric-dev
@@ -27,4 +27,5 @@ pyfabric_dev/runners/notebook.py
27
27
  pyfabric_dev/runners/pipeline.py
28
28
  tests/test_functions_merge.py
29
29
  tests/test_generate_notebooks.py
30
- tests/test_smoke.py
30
+ tests/test_smoke.py
31
+ tests/test_spark_session.py
@@ -1,5 +1,5 @@
1
- pyspark>=3.5.0
2
- delta-spark>=3.2.0
1
+ pyspark>=4.1.1
2
+ delta-spark>=4.2.0
3
3
  azure-identity>=1.15.0
4
4
  azure-keyvault-secrets>=4.7.0
5
5
  python-dotenv>=1.0.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "pyfabric-dev"
7
- version = "0.5.2"
7
+ version = "0.6.0"
8
8
  description = "Local development framework for Microsoft Fabric notebooks: source-of-truth Python, generated .Notebook artifacts, mocked Fabric APIs, parallel local testing."
9
9
  readme = "README.md"
10
10
  license = { text = "Apache-2.0" }
@@ -25,8 +25,8 @@ classifiers = [
25
25
  ]
26
26
 
27
27
  dependencies = [
28
- "pyspark>=3.5.0",
29
- "delta-spark>=3.2.0",
28
+ "pyspark>=4.1.1",
29
+ "delta-spark>=4.2.0",
30
30
  "azure-identity>=1.15.0",
31
31
  "azure-keyvault-secrets>=4.7.0",
32
32
  "python-dotenv>=1.0.0",
@@ -0,0 +1,27 @@
1
+ """The local session mirrors Fabric Runtime 2.0's defaults, not upstream Spark 4.1's."""
2
+ import pytest
3
+
4
+ import pyfabric_dev.local_env # noqa: F401 (bootstrap mocks + local dirs)
5
+ from pyfabric_dev.spark import create_spark_session
6
+
7
+
8
+ @pytest.fixture(scope="module")
9
+ def spark():
10
+ return create_spark_session()
11
+
12
+
13
+ def test_ansi_mode_off_like_fabric(spark):
14
+ assert spark.conf.get("spark.sql.ansi.enabled") == "false"
15
+ # Under ANSI (the upstream Spark 4 default) this raises instead of returning null.
16
+ assert spark.sql("SELECT CAST('x' AS INT) AS v").first()["v"] is None
17
+
18
+
19
+ def test_new_delta_tables_get_deletion_vectors(spark):
20
+ table = "test_session_deletion_vectors_tbl"
21
+ spark.sql(f"DROP TABLE IF EXISTS {table}")
22
+ spark.range(1).write.format("delta").saveAsTable(table)
23
+ try:
24
+ props = spark.sql(f"DESCRIBE DETAIL {table}").first()["properties"]
25
+ assert props.get("delta.enableDeletionVectors") == "true"
26
+ finally:
27
+ spark.sql(f"DROP TABLE IF EXISTS {table}")
File without changes
File without changes
File without changes