sqlmesh 0.184.3.dev10__py3-none-any.whl → 0.184.3.dev12__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.184.3.dev10'
-__version_tuple__ = version_tuple = (0, 184, 3, 'dev10')
+__version__ = version = '0.184.3.dev12'
+__version_tuple__ = version_tuple = (0, 184, 3, 'dev12')

sqlmesh/core/engine_adapter/athena.py

@@ -8,7 +8,7 @@ from sqlmesh.utils.aws import validate_s3_uri, parse_s3_uri
 from sqlmesh.core.engine_adapter.mixins import PandasNativeFetchDFSupportMixin, RowDiffMixin
 from sqlmesh.core.engine_adapter.trino import TrinoEngineAdapter
 from sqlmesh.core.node import IntervalUnit
-import os
+import posixpath
 from sqlmesh.utils.errors import SQLMeshError
 from sqlmesh.core.engine_adapter.shared import (
     CatalogSupport,
@@ -403,11 +403,13 @@ class AthenaEngineAdapter(PandasNativeFetchDFSupportMixin, RowDiffMixin):
 
         elif self.s3_warehouse_location:
             # If the user has set `s3_warehouse_location` in the connection config, the base URI is <s3_warehouse_location>/<catalog>/<schema>/
-            base_uri = os.path.join(self.s3_warehouse_location, table.catalog or "", table.db or "")
+            base_uri = posixpath.join(
+                self.s3_warehouse_location, table.catalog or "", table.db or ""
+            )
         else:
             return None
 
-        full_uri = validate_s3_uri(os.path.join(base_uri, table.text("this") or ""), base=True)
+        full_uri = validate_s3_uri(posixpath.join(base_uri, table.text("this") or ""), base=True)
         return exp.LocationProperty(this=exp.Literal.string(full_uri))
 
     def _find_matching_columns(

sqlmesh/core/model/cache.py

@@ -15,6 +15,7 @@ from sqlmesh.core import constants as c
 from sqlmesh.core.model.definition import ExternalModel, Model, SqlModel, _Model
 from sqlmesh.utils.cache import FileCache
 from sqlmesh.utils.hashing import crc32
+from sqlmesh.utils.windows import IS_WINDOWS
 
 from dataclasses import dataclass
 
@@ -149,8 +150,10 @@ class OptimizedQueryCache:
 
 
 def optimized_query_cache_pool(optimized_query_cache: OptimizedQueryCache) -> ProcessPoolExecutor:
+    # fork doesnt work on Windows. ref: https://docs.python.org/3/library/multiprocessing.html#multiprocessing-start-methods
+    context_type = "spawn" if IS_WINDOWS else "fork"
     return ProcessPoolExecutor(
-        mp_context=mp.get_context("fork"),
+        mp_context=mp.get_context(context_type),
         initializer=_init_optimized_query_cache,
         initargs=(optimized_query_cache,),
         max_workers=c.MAX_FORK_WORKERS,

sqlmesh/utils/cache.py

@@ -12,6 +12,7 @@ from sqlglot import __version__ as SQLGLOT_VERSION
 from sqlmesh.utils import sanitize_name
 from sqlmesh.utils.date import to_datetime
 from sqlmesh.utils.errors import SQLMeshError
+from sqlmesh.utils.windows import IS_WINDOWS, fix_windows_path
 
 logger = logging.getLogger(__name__)
 
@@ -132,4 +133,8 @@ class FileCache(t.Generic[T]):
 
     def _cache_entry_path(self, name: str, entry_id: str = "") -> Path:
         entry_file_name = "__".join(p for p in (self._cache_version, name, entry_id) if p)
-        return self._path / sanitize_name(entry_file_name)
+        full_path = self._path / sanitize_name(entry_file_name)
+        if IS_WINDOWS:
+            # handle paths longer than 260 chars
+            full_path = fix_windows_path(full_path)
+        return full_path

sqlmesh/utils/windows.py

@@ -0,0 +1,14 @@
+import platform
+from pathlib import Path
+
+IS_WINDOWS = platform.system() == "Windows"
+
+
+def fix_windows_path(path: Path) -> Path:
+    """
+    Windows paths are limited to 260 characters: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
+    Users can change this by updating a registry entry but we cant rely on that.
+    We can quite commonly generate a cache file path that exceeds 260 characters which causes a FileNotFound error.
+    If we prefix the path with "\\?\" then we can have paths up to 32,767 characters
+    """
+    return Path("\\\\?\\" + str(path.absolute()))

{sqlmesh-0.184.3.dev10.dist-info → sqlmesh-0.184.3.dev12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.184.3.dev10
+Version: 0.184.3.dev12
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.184.3.dev10.dist-info → sqlmesh-0.184.3.dev12.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=DRovDXI-9D9xeYDP7t8Fq7mI31S2btlN-LHPOK3y7PA,7240
-sqlmesh/_version.py,sha256=jFzBB3tos-4Z7mIfsDO0P_7HdqxK3J2sRk8mGivtBkI,530
+sqlmesh/_version.py,sha256=zRNKjsN4O0fNwBz4jqBqT0B-wSHyRN-M0SWZHxxCSZ8,530
 sqlmesh/magics.py,sha256=aeOyTdeNnyGzGtJXECDAbqJ8OAplrV4UAYl0I2hhtig,39670
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -58,7 +58,7 @@ sqlmesh/core/config/scheduler.py,sha256=essKKkJ6jD5gRT_l-WiKwGKVO1hpDlnf_UdrCvyN
 sqlmesh/core/config/ui.py,sha256=jsO-S6_d9NkLZGG5pT4mgKgxMF34KzkDociZAMvCX3U,278
 sqlmesh/core/engine_adapter/__init__.py,sha256=two_WTs4UnSNp-fscJh8vOunJq0Xi37EuDuMjKsKuNE,2281
 sqlmesh/core/engine_adapter/_typing.py,sha256=BAM_U7Vsi0k37XHu_nNnu3whXxuTVDey_uZe3aelKYY,1033
-sqlmesh/core/engine_adapter/athena.py,sha256=kVkGhDhjVd_8zsvq7B94kr_CHD0bct6XXFkhzrOjQOY,26158
+sqlmesh/core/engine_adapter/athena.py,sha256=WBVAB_YzBJXdiqq51LTd2jmYoMtGrLmtsA8IrIFVnQ0,26199
 sqlmesh/core/engine_adapter/base.py,sha256=XQeJi4Rr5VllnQKFZBYXNs1G142T16ZOVU_l9UYvzoE,100539
 sqlmesh/core/engine_adapter/base_postgres.py,sha256=DOsOdCJyQAVfA9S0GznuH0J1bonMqYVP_yJLDYcxFYM,6772
 sqlmesh/core/engine_adapter/bigquery.py,sha256=gnd2cFc7v95tT_E9XMHHgccS-QwSF2sJHsL8xfcSguk,49249
@@ -84,7 +84,7 @@ sqlmesh/core/metric/__init__.py,sha256=H1HmoD5IwN4YWe9iJXyueLYNmTQFZwok5nSWNJcZI
 sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw9x9M,7201
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
 sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu4,1644
-sqlmesh/core/model/cache.py,sha256=hzpGPXxp2LFMEp6gm-8DULJKCBDGf-1kTQcfNt0xrRM,7028
+sqlmesh/core/model/cache.py,sha256=s-aEFuUmUKITaWHvYbOyRarsjsgoA6vfASK-f-MwJ9k,7257
 sqlmesh/core/model/common.py,sha256=J3ukpCEvZyfiYlG38aMhqxlR-Tyq2Sm2dzg5SOnldXc,16279
 sqlmesh/core/model/decorator.py,sha256=v_sVudNROGrtj5cZ3vgcOFnwPdzwoUGY6REAkQpIMQc,8927
 sqlmesh/core/model/definition.py,sha256=vR9Y48WTZTHbjlvuQltueXNxBtVoWpxCw4D9cikRqw4,108422
@@ -249,7 +249,7 @@ sqlmesh/migrations/v0083_use_sql_for_scd_time_data_type_data_hash.py,sha256=0r-S
 sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/utils/__init__.py,sha256=4vN2s-t0p0c9l4erdOFONl_j8StaW6Eljt_SBffMW80,10898
 sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
-sqlmesh/utils/cache.py,sha256=y58l95ka8RgSX-r6Opav4RBKWLnNYLUYqLfDOPI8HzI,4526
+sqlmesh/utils/cache.py,sha256=GITQ3sldwG4vlI0wKIDI7yOr1CND1EI3necAmeCEW8g,4743
 sqlmesh/utils/concurrency.py,sha256=HvK_AwVfy3Npzmsthmd73Z5tLhY-Anrb472IkaoRjOw,9041
 sqlmesh/utils/config.py,sha256=-SjIn7Z0HHnMtjPavQIvFcSaeg86XLsGd3sGIwOKrTE,1941
 sqlmesh/utils/connection_pool.py,sha256=To1zr9d5MQOZ5im09jLQKO4DFmfzH6s_q75cEa1y0PE,11390
@@ -266,8 +266,9 @@ sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,83
 sqlmesh/utils/pandas.py,sha256=uKI-ygEtMcGDRzXdBu0RkGmMBkYxwu5O2Wabho841Ls,2196
 sqlmesh/utils/pydantic.py,sha256=-u69PDY-4FLR7ODNKBoN7PsPNmZ0wdhsJcCAG2P6vyY,11061
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
+sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.184.3.dev10.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.184.3.dev12.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=h-jIuvBTQ-QurWEds39umXiJjP_TWPQmkHTiizCAeOI,5472
 web/client/dist/assets/Audits-DIPZFtrx.js,sha256=Yn24xPBT3znH0fQJJaBLb1p1wjJz1nWjrZcm18e63W0,1041
@@ -367,26 +368,26 @@ web/server/app.py,sha256=b6qEJ4btjaSm2kWwQUGV--S3VE8QW0Um7f4rBdaoB_M,59
 web/server/console.py,sha256=K2fJpiZbibWaQa2OvN7YbeI9sxkXtkXwA8lJKsXfq-k,14529
 web/server/exceptions.py,sha256=I2PPpChwhDWawQENr5kHg6DjyRGeiJ6OWZsCzEZhVzs,1533
 web/server/main.py,sha256=9PaDa8sigo1z0AxtI3sQ3fHmDqs6uXwP-_IvoMwkznw,3199
-web/server/models.py,sha256=SUztvOYmlHDMVOt7C36-C3tKBwm7UNRkkYeid9bc5n8,15527
+web/server/models.py,sha256=NUI2xqDcav20uuC2BZ9xDL2x0v16-gpVT1Sb-WHfAcg,15518
 web/server/openapi.py,sha256=NOg__YQg7arVdajQv5CvOA5KLTQCfGZFvwnjARuM6iE,586
 web/server/settings.py,sha256=PuWrOmn5LpJMBTymCMQreqMsL93tClTJ6wlcAZXjgCo,3755
-web/server/utils.py,sha256=h_CpJe_2PvWhAZ_S2q8SWRf5olvdjAj3lnhusiVXexY,3286
+web/server/utils.py,sha256=WodQI75kQTXVCO_g1g9Sf0_b-9yoYFieb_xcKRpZ470,3986
 web/server/watcher.py,sha256=5biJ8gbHmGNUrooMFWRmw00XDVg0bMx6Hub571cs6TI,5246
 web/server/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/server/api/endpoints/__init__.py,sha256=FcXl56-kPqbKgCZKOx0kjE50hn37q1m-nwOCxI9tRcc,932
-web/server/api/endpoints/commands.py,sha256=GuYgrstMxpKYAHaH7EGYTLZdY8J21nlNQX2f-EK66ZY,8243
+web/server/api/endpoints/commands.py,sha256=MVlxC-QG6VMEcboHYeuTdtsd7WgPpqJcQDa1OLxZtLI,8367
 web/server/api/endpoints/directories.py,sha256=Aa8U4pTs3Saj5DnJ9TDnX9mZois-EXgpcBm5GpNNM_s,1822
 web/server/api/endpoints/environments.py,sha256=g3YC6j4LLlG_wjJqn-Hrz78BWmgS7vhUhzaycsiez00,2124
 web/server/api/endpoints/events.py,sha256=tjhOhGe9jt7L1mOLLapl5qTvicDRnP5lPZHSq9OnSVI,932
-web/server/api/endpoints/files.py,sha256=UnZPScF-HANNNZhNFhxfAPihcU3p85mB7P2rqFc2wG4,6698
+web/server/api/endpoints/files.py,sha256=HogdDH35tHBPLu_4vwj2aXrAqikA2UdkCOLv5QXf0Ng,6720
 web/server/api/endpoints/lineage.py,sha256=re2mNADlUQsBZVeA7jKw857mQvMxwijynP5iVB7YAnY,5769
 web/server/api/endpoints/meta.py,sha256=QKWWrHewGh5JS8mCSWXTKCIQwXw-TQfaT9tWt_SsNY4,1032
-web/server/api/endpoints/models.py,sha256=SzS5bshzdXyM6hb948sZ_3k-xhpGCHkxkb7pHzPjM_o,4675
+web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodrafB7Ndk,4697
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.184.3.dev10.dist-info/METADATA,sha256=Vsg4xUkm0XISAr4Mr31UoAvyqBR3PxXLiyT9p-TNg-I,25499
-sqlmesh-0.184.3.dev10.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-sqlmesh-0.184.3.dev10.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.184.3.dev10.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.184.3.dev10.dist-info/RECORD,,
+sqlmesh-0.184.3.dev12.dist-info/METADATA,sha256=0DY4ysd-BS4Bqm9s3CORlButuv1lwL7tY2wSFzagOf8,25499
+sqlmesh-0.184.3.dev12.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+sqlmesh-0.184.3.dev12.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.184.3.dev12.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.184.3.dev12.dist-info/RECORD,,

web/server/api/endpoints/commands.py

@@ -22,6 +22,7 @@ from web.server.utils import (
     df_to_pyarrow_bytes,
     run_in_executor,
 )
+from pathlib import Path
 
 router = APIRouter()
 
@@ -143,7 +144,7 @@ async def test(
     test_output = io.StringIO()
     try:
         result = context.test(
-            tests=[str(context.path / test)] if test else None,
+            tests=[str(context.path / Path(test))] if test else None,
             verbosity=verbosity,
             stream=test_output,
         )
@@ -160,11 +161,17 @@ async def test(
         test_output.getvalue(),
         context._test_connection_config._engine_adapter.DIALECT,
     )
+
+    def _test_path(test: ModelTest) -> t.Optional[str]:
+        if path := test.path_relative_to(context.path):
+            return path.as_posix()
+        return None
+
     return models.TestResult(
         errors=[
             models.TestErrorOrFailure(
                 name=test.test_name,
-                path=test.path_relative_to(context.path),
+                path=_test_path(test),
                 tb=tb,
             )
             for test, tb in ((t.cast(ModelTest, test), tb) for test, tb in result.errors)
@@ -172,7 +179,7 @@ async def test(
         failures=[
             models.TestErrorOrFailure(
                 name=test.test_name,
-                path=test.path_relative_to(context.path),
+                path=_test_path(test),
                 tb=tb,
             )
             for test, tb in ((t.cast(ModelTest, test), tb) for test, tb in result.failures)
@@ -180,7 +187,7 @@ async def test(
         skipped=[
             models.TestSkipped(
                 name=test.test_name,
-                path=test.path_relative_to(context.path),
+                path=_test_path(test),
                 reason=reason,
             )
             for test, reason in (
@@ -190,7 +197,7 @@ async def test(
         successes=[
             models.TestCase(
                 name=test.test_name,
-                path=test.path_relative_to(context.path),
+                path=_test_path(test),
             )
             for test in (t.cast(ModelTest, test) for test in result.successes)
         ],

web/server/api/endpoints/files.py

@@ -145,13 +145,13 @@ def _get_directory(path: str | Path, settings: Settings) -> models.Directory:
                     directories.append(
                         models.Directory(
                             name=entry.name,
-                            path=str(relative_path),
+                            path=str(relative_path.as_posix()),
                             directories=_directories,
                             files=_files,
                         )
                     )
                 elif entry.is_file(follow_symlinks=False):
-                    files.append(models.File(name=entry.name, path=str(relative_path)))
+                    files.append(models.File(name=entry.name, path=str(relative_path.as_posix())))
         return sorted(directories, key=lambda x: x.name), sorted(files, key=lambda x: x.name)
 
     directories, files = walk_path(path)

web/server/api/endpoints/models.py

@@ -124,8 +124,8 @@ def serialize_model(context: Context, model: Model, render_query: bool = False)
     return models.Model(
         name=model.name,
         fqn=model.fqn,
-        path=str(model._path.absolute().relative_to(context.path)),
-        full_path=str(model._path.absolute()),
+        path=str(model._path.absolute().relative_to(context.path).as_posix()),
+        full_path=str(model._path.absolute().as_posix()),
         dialect=dialect,
         columns=columns,
         details=details,

web/server/models.py

@@ -420,7 +420,7 @@ class TableDiff(PydanticModel):
 
 class TestCase(PydanticModel):
     name: str
-    path: pathlib.Path
+    path: str
 
 
 class TestErrorOrFailure(TestCase):

web/server/utils.py

@@ -16,6 +16,7 @@ from sqlmesh.core import constants as c
 from web.server.console import api_console
 from web.server.exceptions import ApiException
 from web.server.settings import Settings, get_context, get_settings
+from sqlmesh.utils.windows import IS_WINDOWS
 
 R = t.TypeVar("R")
 
@@ -75,6 +76,16 @@ def replace_file(src: Path, dst: Path) -> None:
     """Move a file or directory at src to dst."""
     if src != dst:
         try:
+            if IS_WINDOWS:
+                # os.replace() behaves differently on Windows so we have to do some extra checks
+                if dst.exists():
+                    if src.exists() and src.is_dir() and dst.is_file():
+                        raise OSError("Cant rename directory to existing file")
+                    elif src.exists() and src.is_file() and dst.is_dir():
+                        raise OSError("Cant rename file to existing directory")
+                    elif dst.is_dir() and not any(dst.iterdir()):
+                        # target dir exists but is empty, delete it so replace() succeeds
+                        dst.rmdir()
             src.replace(dst)
         except FileNotFoundError:
             raise HTTPException(status_code=HTTP_404_NOT_FOUND)