castor-extractor 0.24.52__tar.gz → 0.24.55__tar.gz

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.24.55 - 2025-09-19
+
+* Fix encoding in LocalStorage - force to utf-8
+
+## 0.24.54 - 2025-09-18
+
+* SqlServer: fix typo in the extraction query of schemas
+
+## 0.24.53 - 2025-09-18
+
+* Fix CSV field size to support running on Windows
+
 ## 0.24.52 - 2025-09-18
 
 * SqlServer : improve extraction of users and technical owners

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.52
+Version: 0.24.55
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -36,7 +36,7 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=2,<3)
+Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
@@ -215,6 +215,18 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.55 - 2025-09-19
+
+* Fix encoding in LocalStorage - force to utf-8
+
+## 0.24.54 - 2025-09-18
+
+* SqlServer: fix typo in the extraction query of schemas
+
+## 0.24.53 - 2025-09-18
+
+* Fix CSV field size to support running on Windows
+
 ## 0.24.52 - 2025-09-18
 
 * SqlServer : improve extraction of users and technical owners

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/castor_extractor/utils/__init__.py

@@ -26,7 +26,7 @@ from .collection import (
     group_by,
     mapping_from_rows,
 )
-from .constants import OUTPUT_DIR
+from .constants import ENCODING_UTF8, OUTPUT_DIR
 from .deprecate import deprecate_python
 from .env import from_env
 from .files import explode, search_files

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/castor_extractor/utils/constants.py

@@ -1 +1,2 @@
 OUTPUT_DIR = "CASTOR_OUTPUT_DIRECTORY"
+ENCODING_UTF8 = "utf-8"

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/castor_extractor/utils/formatter.py

@@ -1,6 +1,7 @@
 """convert files to csv"""
 
 import csv
+import ctypes
 import json
 import logging
 import re
@@ -21,6 +22,8 @@ CSV_OPTIONS: CsvOptions = {
     "quotechar": '"',
 }
 
+CSV_FIELD_SIZE_MB = 100
+
 ScalarValue = Union[int, float, None, str]
 
 logger = logging.getLogger(__name__)
@@ -132,6 +135,31 @@ class Formatter(ABC):
         pass
 
 
+def _set_csv_field_size_limit(target_limit_mb: int) -> None:
+    """
+    Safely set the maximum CSV field size limit across platforms.
+
+    This function wraps `csv.field_size_limit()` to avoid `OverflowError` on
+    Windows, where the maximum C long is only 32 bits (2^31 - 1). On Linux and
+    macOS, the C long is typically 64 bits, allowing much larger values.
+
+    The requested limit is specified in megabytes and converted to bytes.
+
+    It is then clamped to the maximum value supported by:
+      - the current platform's C long
+      - Python's `sys.maxsize`
+      - the requested target limit
+    """
+    target_limit_bytes = target_limit_mb * 1024**2
+
+    # max value of C long for the current platform
+    platform_c_long = (1 << (8 * ctypes.sizeof(ctypes.c_long) - 1)) - 1
+
+    limit_bytes = min(target_limit_bytes, sys.maxsize, platform_c_long)
+
+    csv.field_size_limit(limit_bytes)
+
+
 class CsvFormatter(Formatter):
     """
     Serialize/Deserialize CSV
@@ -141,9 +169,7 @@ class CsvFormatter(Formatter):
         return "csv"
 
     # increase the size limit (some fields are very large)
-    # Limit to 100 MB. Value must be smaller than the C long maximum value.
-    size_limit = min(sys.maxsize, 100 * 1024**3)
-    csv.field_size_limit(size_limit)
+    _set_csv_field_size_limit(target_limit_mb=CSV_FIELD_SIZE_MB)
 
     @staticmethod
     def serialize(buffer: IO[str], data: Iterable[dict]) -> bool:

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/castor_extractor/utils/store.py

@@ -4,6 +4,7 @@ from collections.abc import Iterable, Iterator
 from io import StringIO
 from typing import Optional
 
+from .constants import ENCODING_UTF8
 from .formatter import CsvFormatter, Formatter
 from .time import current_timestamp
 from .write import timestamped_filename
@@ -58,11 +59,11 @@ class LocalStorage(AbstractStorage):
 
     def put(self, name: str, data: Iterable[dict]) -> str:
         path = self.path(name)
-        with open(path, "w") as file:
+        with open(path, "w", encoding=ENCODING_UTF8) as file:
             self._formatter.serialize(file, data)
         return path
 
     def get(self, name: str) -> Iterator[dict]:
         path = self.path(name)
-        with open(path, "r") as file:
+        with open(path, "r", encoding=ENCODING_UTF8) as file:
             return self._formatter.deserialize(StringIO(file.read()))

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/castor_extractor/utils/write.py

@@ -3,11 +3,11 @@ import logging
 import os
 import sys
 from datetime import datetime
+from importlib.metadata import version
 from typing import Any
 
-import pkg_resources
+from ..utils import ENCODING_UTF8
 
-ENCODING = "utf8"
 SUMMARY_FILENAME = "summary.json"
 
 logger = logging.getLogger(__name__)
@@ -33,15 +33,14 @@ def write_json(filename: str, data: Any):
     """
     write the data to a json file at path filename
     """
-    with open(filename, "w", encoding=ENCODING) as f:
+    with open(filename, "w", encoding=ENCODING_UTF8) as f:
         json.dump(data, f)
         logger.info(f"Wrote output file: {filename} ({f.tell()} bytes)")
 
 
 def _current_version() -> str:
     """fetch the current version of castor extractor running"""
-    packages = pkg_resources.require("castor-extractor")
-    return str(packages[0].version)
+    return version("castor-extractor")
 
 
 def write_summary(output_directory: str, ts: int, **kwargs):

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/castor_extractor/warehouse/sqlserver/queries/schema.sql

@@ -17,7 +17,7 @@ SELECT
     schema_name = s.name,
     schema_id = CAST(d.database_id AS VARCHAR(10)) + '_' + CAST(s.schema_id AS VARCHAR(10)),
     schema_owner = u.name,
-    schema_owner_id = u.uid
+    schema_owner_id = u.name
 FROM [{database}].sys.schemas AS s
 INNER JOIN ids AS i
     ON s.name = i.table_schema

{castor_extractor-0.24.52 → castor_extractor-0.24.55}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.52"
+version = "0.24.55"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"
@@ -32,7 +32,7 @@ google-api-core = ">=2.1.1, <3.0.0"
 google-api-python-client = {version = "^2.121.0", optional = true}
 google-auth = "^2"
 google-cloud-core = ">=2.1.0, <3.0.0"
-google-cloud-storage = "^2"
+google-cloud-storage = "^3.1.0"
 google-resumable-media = "^2.0.3"
 googleapis-common-protos = "^1.53.0"
 looker-sdk = { version = "^25.0.0", optional = true }
@@ -197,15 +197,16 @@ pytest-freezegun = "freezegun"
 DEP004 = ["pytest", "requests_mock", "freezegun"]
 # SQL Alchemy needs some dependencies to be installed
 DEP002 = [
+  "numpy",                      # See comment on numpy in dependencies
+  "pandas",                     # See comment on pandas in dependencies
   "psycopg2-binary",
+  "pymssql",
+  "pymysql",
+  "setuptools",
   "snowflake-connector-python",
   "snowflake-sqlalchemy",
   "sqlalchemy-bigquery",
   "sqlalchemy-redshift",
-  "pymysql",
-  "pymssql",
-  "numpy",                      # See comment on numpy in dependencies
-  "pandas",                     # See comment on pandas in dependencies
 ]