castor-extractor 0.24.52__py3-none-any.whl → 0.24.55__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.24.55 - 2025-09-19
+
+* Fix encoding in LocalStorage - force to utf-8
+
+## 0.24.54 - 2025-09-18
+
+* SqlServer: fix typo in the extraction query of schemas
+
+## 0.24.53 - 2025-09-18
+
+* Fix CSV field size to support running on Windows
+
 ## 0.24.52 - 2025-09-18
 
 * SqlServer : improve extraction of users and technical owners

castor_extractor/utils/__init__.py

@@ -26,7 +26,7 @@ from .collection import (
     group_by,
     mapping_from_rows,
 )
-from .constants import OUTPUT_DIR
+from .constants import ENCODING_UTF8, OUTPUT_DIR
 from .deprecate import deprecate_python
 from .env import from_env
 from .files import explode, search_files

castor_extractor/utils/constants.py

@@ -1 +1,2 @@
 OUTPUT_DIR = "CASTOR_OUTPUT_DIRECTORY"
+ENCODING_UTF8 = "utf-8"

castor_extractor/utils/formatter.py

@@ -1,6 +1,7 @@
 """convert files to csv"""
 
 import csv
+import ctypes
 import json
 import logging
 import re
@@ -21,6 +22,8 @@ CSV_OPTIONS: CsvOptions = {
     "quotechar": '"',
 }
 
+CSV_FIELD_SIZE_MB = 100
+
 ScalarValue = Union[int, float, None, str]
 
 logger = logging.getLogger(__name__)
@@ -132,6 +135,31 @@ class Formatter(ABC):
         pass
 
 
+def _set_csv_field_size_limit(target_limit_mb: int) -> None:
+    """
+    Safely set the maximum CSV field size limit across platforms.
+
+    This function wraps `csv.field_size_limit()` to avoid `OverflowError` on
+    Windows, where the maximum C long is only 32 bits (2^31 - 1). On Linux and
+    macOS, the C long is typically 64 bits, allowing much larger values.
+
+    The requested limit is specified in megabytes and converted to bytes.
+
+    It is then clamped to the maximum value supported by:
+      - the current platform's C long
+      - Python's `sys.maxsize`
+      - the requested target limit
+    """
+    target_limit_bytes = target_limit_mb * 1024**2
+
+    # max value of C long for the current platform
+    platform_c_long = (1 << (8 * ctypes.sizeof(ctypes.c_long) - 1)) - 1
+
+    limit_bytes = min(target_limit_bytes, sys.maxsize, platform_c_long)
+
+    csv.field_size_limit(limit_bytes)
+
+
 class CsvFormatter(Formatter):
     """
     Serialize/Deserialize CSV
@@ -141,9 +169,7 @@ class CsvFormatter(Formatter):
         return "csv"
 
     # increase the size limit (some fields are very large)
-    # Limit to 100 MB. Value must be smaller than the C long maximum value.
-    size_limit = min(sys.maxsize, 100 * 1024**3)
-    csv.field_size_limit(size_limit)
+    _set_csv_field_size_limit(target_limit_mb=CSV_FIELD_SIZE_MB)
 
     @staticmethod
     def serialize(buffer: IO[str], data: Iterable[dict]) -> bool:

castor_extractor/utils/store.py

@@ -4,6 +4,7 @@ from collections.abc import Iterable, Iterator
 from io import StringIO
 from typing import Optional
 
+from .constants import ENCODING_UTF8
 from .formatter import CsvFormatter, Formatter
 from .time import current_timestamp
 from .write import timestamped_filename
@@ -58,11 +59,11 @@ class LocalStorage(AbstractStorage):
 
     def put(self, name: str, data: Iterable[dict]) -> str:
         path = self.path(name)
-        with open(path, "w") as file:
+        with open(path, "w", encoding=ENCODING_UTF8) as file:
             self._formatter.serialize(file, data)
         return path
 
     def get(self, name: str) -> Iterator[dict]:
         path = self.path(name)
-        with open(path, "r") as file:
+        with open(path, "r", encoding=ENCODING_UTF8) as file:
             return self._formatter.deserialize(StringIO(file.read()))

castor_extractor/utils/write.py

@@ -3,11 +3,11 @@ import logging
 import os
 import sys
 from datetime import datetime
+from importlib.metadata import version
 from typing import Any
 
-import pkg_resources
+from ..utils import ENCODING_UTF8
 
-ENCODING = "utf8"
 SUMMARY_FILENAME = "summary.json"
 
 logger = logging.getLogger(__name__)
@@ -33,15 +33,14 @@ def write_json(filename: str, data: Any):
     """
     write the data to a json file at path filename
     """
-    with open(filename, "w", encoding=ENCODING) as f:
+    with open(filename, "w", encoding=ENCODING_UTF8) as f:
         json.dump(data, f)
         logger.info(f"Wrote output file: {filename} ({f.tell()} bytes)")
 
 
 def _current_version() -> str:
     """fetch the current version of castor extractor running"""
-    packages = pkg_resources.require("castor-extractor")
-    return str(packages[0].version)
+    return version("castor-extractor")
 
 
 def write_summary(output_directory: str, ts: int, **kwargs):

castor_extractor/warehouse/sqlserver/queries/schema.sql

@@ -17,7 +17,7 @@ SELECT
     schema_name = s.name,
     schema_id = CAST(d.database_id AS VARCHAR(10)) + '_' + CAST(s.schema_id AS VARCHAR(10)),
     schema_owner = u.name,
-    schema_owner_id = u.uid
+    schema_owner_id = u.name
 FROM [{database}].sys.schemas AS s
 INNER JOIN ids AS i
     ON s.name = i.table_schema

{castor_extractor-0.24.52.dist-info → castor_extractor-0.24.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.52
+Version: 0.24.55
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -36,7 +36,7 @@ Requires-Dist: google-api-core (>=2.1.1,<3.0.0)
 Requires-Dist: google-api-python-client (>=2.121.0,<3.0.0) ; extra == "lookerstudio" or extra == "all"
 Requires-Dist: google-auth (>=2,<3)
 Requires-Dist: google-cloud-core (>=2.1.0,<3.0.0)
-Requires-Dist: google-cloud-storage (>=2,<3)
+Requires-Dist: google-cloud-storage (>=3.1.0,<4.0.0)
 Requires-Dist: google-resumable-media (>=2.0.3,<3.0.0)
 Requires-Dist: googleapis-common-protos (>=1.53.0,<2.0.0)
 Requires-Dist: looker-sdk (>=25.0.0,<26.0.0) ; extra == "looker" or extra == "all"
@@ -215,6 +215,18 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.55 - 2025-09-19
+
+* Fix encoding in LocalStorage - force to utf-8
+
+## 0.24.54 - 2025-09-18
+
+* SqlServer: fix typo in the extraction query of schemas
+
+## 0.24.53 - 2025-09-18
+
+* Fix CSV field size to support running on Windows
+
 ## 0.24.52 - 2025-09-18
 
 * SqlServer : improve extraction of users and technical owners

{castor_extractor-0.24.52.dist-info → castor_extractor-0.24.55.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=ruS47cNmG5EMJYGYtkGGyhh7A5NgNz4TxzS8h0lP_Co,20477
+CHANGELOG.md,sha256=y8BAidkUDrMoQLEfu3LJLiqxoEUzI5hJZs4CUN_e1H0,20711
 Dockerfile,sha256=xQ05-CFfGShT3oUqaiumaldwA288dj9Yb_pxofQpufg,301
 DockerfileUsage.md,sha256=2hkJQF-5JuuzfPZ7IOxgM6QgIQW7l-9oRMFVwyXC4gE,998
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
@@ -95,7 +95,7 @@ castor_extractor/uploader/settings.py,sha256=sUZpg9eHemM99DMrBW8bnlMuoTmCmLCKq-D
 castor_extractor/uploader/upload.py,sha256=b2g9vWWjXWbt8Ms7brTc7OK_I7Z-1VSibNbppGoB2oQ,4764
 castor_extractor/uploader/upload_test.py,sha256=UgN7TnT9Chn6KVzRcAX0Tuvp7-tps3ugxGitlgb9TSY,462
 castor_extractor/uploader/utils.py,sha256=otAaySj5aeem6f0CTd0Te6ioJ6uP2J1p348j-SdIwDI,802
-castor_extractor/utils/__init__.py,sha256=z_BdKTUyuug3I5AzCuSGrAVskfLax4_olfORIjhZw_M,1691
+castor_extractor/utils/__init__.py,sha256=LhcSQe50m5iv2zlEPfJUa3VY4pDcoMctqLByAWNV7As,1706
 castor_extractor/utils/argument_parser.py,sha256=S4EcIh3wNDjs3fOrQnttCcPsAmG8m_Txl7xvEh0Q37s,283
 castor_extractor/utils/argument_parser_test.py,sha256=wnyLFJ74iEiPxxLSbwFtckR7FIHxsFOVU38ljs9gqRA,633
 castor_extractor/utils/batch.py,sha256=SFlLmJgVjV2nVhIrjVIEp8wJ9du4dKKHq8YVYubnwQQ,448
@@ -119,12 +119,12 @@ castor_extractor/utils/client/uri.py,sha256=jmP9hY-6PRqdc3-vAOdtll_U6q9VCqSqmBAN
 castor_extractor/utils/client/uri_test.py,sha256=1XKF6qSseCeD4G4ckaNO07JXfGbt7XUVinOZdpEYrDQ,259
 castor_extractor/utils/collection.py,sha256=g2HmB0ievvYHWaZ8iEzkcPPkrBFsh6R6b_liBqcsMjc,3044
 castor_extractor/utils/collection_test.py,sha256=mlw33u4VidazQwWxJMvaFeYX3VB5CAj6rqRG-cRsLrw,2884
-castor_extractor/utils/constants.py,sha256=qBQprS9U66mS-RIBXiLujdTSV3WvGv40Bc0khP4Abdk,39
+castor_extractor/utils/constants.py,sha256=xEUk-B__cqHPKz5_Ta9kHIsiR-a9qTXzpsTY-SzPRHo,63
 castor_extractor/utils/deprecate.py,sha256=aBIN2QqZUx5CBNZMFfOUhi8QqtPqRcJtmrN6xqfm-y8,805
 castor_extractor/utils/env.py,sha256=TqdtB50U8LE0993WhhEhpy89TJrHbjtIKjvg6KQ-5q0,596
 castor_extractor/utils/files.py,sha256=qKbfu5FRjsQdKnRmaJNd5EdX_F6gf5C5tV8LdoYKxs0,1527
 castor_extractor/utils/files_test.py,sha256=omRT3XSjaSAywYUoLh1SGWqYzl4UwBYKSYA9_7mXd_E,1542
-castor_extractor/utils/formatter.py,sha256=KFUIiT1mSMS464trrpneFC2aIC0MjocHS0BLzX0WkIk,4969
+castor_extractor/utils/formatter.py,sha256=wys8LD5sB39g37aqIpJ3X-YXVspcsUmcuUFvZA4ODHg,5811
 castor_extractor/utils/formatter_test.csv,sha256=UCNqPs8-xrY1AdMSpuctVFXInQe3Z_EABP4rF-Jw5ks,3802
 castor_extractor/utils/formatter_test.json,sha256=yPP_z1ZEavaUskC-Hx33uGlwKoInHYOFKqsJ9NgwIFo,12527
 castor_extractor/utils/formatter_test.py,sha256=VPlRTPQOaAeCySNs1wU1jd3bMppqxkVpD1dyCLt6p94,1856
@@ -148,7 +148,7 @@ castor_extractor/utils/salesforce/constants.py,sha256=7yPmUeyn4IHQiHLDutXE0L_OBd
 castor_extractor/utils/salesforce/credentials.py,sha256=m_11LIaBrYVgH2bLo-QnxaIY5KhEdtfVXz9r2lb_fd0,1123
 castor_extractor/utils/salesforce/credentials_test.py,sha256=FQRyNk2Jsh6KtYiW20oL43CVnGjXLcAjdFATkE7jK0s,586
 castor_extractor/utils/salesforce/pagination.py,sha256=wJq0rKLdacFRggyHwB6Fh3K6iXPvL4QWhsDvZdjQjM8,849
-castor_extractor/utils/store.py,sha256=hnyrFwCsL48e9QrsBns-n8FospujZrkUy1P2YHAh_C0,2067
+castor_extractor/utils/store.py,sha256=KAg5TzLd8jak1Gh5NK-iPu2buQIYNofx6lpXye7MRDU,2152
 castor_extractor/utils/string.py,sha256=IQqNum7CJwuSvDGPbTAmz46YwtYDYgJKeXY7iixdjI4,2370
 castor_extractor/utils/string_test.py,sha256=u3P2tAPhyfCLvD19rH_JcpHhPuWTHUdg0z_N_-Kxwno,2501
 castor_extractor/utils/time.py,sha256=jmP1QWg4lv21Jp_Oy71lfJ47hjNOSgHiBOFf964RMPU,1732
@@ -158,7 +158,7 @@ castor_extractor/utils/url.py,sha256=0YaKAz3EC5PgTb5A2TNOlxf1DANK40yw6hs7ArEtJaU
 castor_extractor/utils/url_test.py,sha256=LWzNdOZqjrDeLmvhPBYmP35mzhm7jGAXi021thiro1Y,1425
 castor_extractor/utils/validation.py,sha256=dRvC9SoFVecVZuLQNN3URq37yX2sBSW3-NxIxkcol5o,1894
 castor_extractor/utils/validation_test.py,sha256=A7P6VmI0kYX2aGIeEN12y7LsY7Kpm8pE4bdVFhbBAMw,1184
-castor_extractor/utils/write.py,sha256=Z_RYm47XeHiUPPUMYMuAjQrVZ18CAkL3daQHQG1XPlM,2148
+castor_extractor/utils/write.py,sha256=KQVWF29N766avzmSb129IUWrId5c_8BtnYhVLmU6YIs,2133
 castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/visualization/domo/__init__.py,sha256=1axOCPm4RpdIyUt9LQEvlMvbOPllW8rk63h6EjVgJ0Y,111
 castor_extractor/visualization/domo/assets.py,sha256=bK1urFR2tnlWkVkkhR32mAKMoKbESNlop-CNGx-65PY,206
@@ -428,14 +428,14 @@ castor_extractor/warehouse/sqlserver/queries/.sqlfluff,sha256=yy0KQdz8I_67vnXyX8
 castor_extractor/warehouse/sqlserver/queries/column.sql,sha256=ojiUQQnHXdWMbgaYOcxKBiwfi7rtu_tyamK6r4t4IBM,2929
 castor_extractor/warehouse/sqlserver/queries/database.sql,sha256=4dPeBCn85MEOXr1f-DPXxiI3RvvoE_1n8lsbTs26E0I,150
 castor_extractor/warehouse/sqlserver/queries/query.sql,sha256=bkENw7QovlG4MyYe5q3XNPs3ajUr_3bNzpbm0Y2upYo,821
-castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=jUnZ10kIZk44nKQ5KiyjZ0YFdypYQj__OlPDRq71EAw,909
+castor_extractor/warehouse/sqlserver/queries/schema.sql,sha256=Fq_8-tCnArayON3fjd2oMWM9nuYaXX3aZMr5jOOfMuw,910
 castor_extractor/warehouse/sqlserver/queries/table.sql,sha256=ggzatJOlOfGkMG1NS-hD-n1-3WLbV9Yh8IsQrEFO5X4,2831
 castor_extractor/warehouse/sqlserver/queries/user.sql,sha256=MAlnTis43E3Amu1e1Oz_qhaX8Bz-iN0Lrbf9RiohX7Y,99
 castor_extractor/warehouse/sqlserver/queries/view_ddl.sql,sha256=9rynvx6MWg3iZzrWPB7haZfVKEPkxulzryE2g19x804,315
 castor_extractor/warehouse/sqlserver/query.py,sha256=c8f7_SEMR17DhbtzuYphWqWDQ0sCRy-nR442RRBZVYw,1773
 castor_extractor/warehouse/synapse/queries/column.sql,sha256=lNcFoIW3Y0PFOqoOzJEXmPvZvfAsY0AP63Mu2LuPzPo,1351
-castor_extractor-0.24.52.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
-castor_extractor-0.24.52.dist-info/METADATA,sha256=0wd_HdsZnM75f8hXza9FNIvjipmHGUDHOe5yjIYX1Ig,27930
-castor_extractor-0.24.52.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-castor_extractor-0.24.52.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
-castor_extractor-0.24.52.dist-info/RECORD,,
+castor_extractor-0.24.55.dist-info/LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
+castor_extractor-0.24.55.dist-info/METADATA,sha256=MhFCdByqa4_T7A4-Mb96-ISq07W6BP7M-RHgjSfI8iY,28172
+castor_extractor-0.24.55.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+castor_extractor-0.24.55.dist-info/entry_points.txt,sha256=_F-qeZCybjoMkNb9ErEhnyqXuG6afHIFQhakdBHZsr4,1803
+castor_extractor-0.24.55.dist-info/RECORD,,