cognite-extractor-utils 7.5.9__py3-none-any.whl → 7.5.11__py3-none-any.whl

cognite/extractorutils/__init__.py

@@ -16,7 +16,7 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.5.9"
+__version__ = "7.5.11"
 from .base import Extractor
 
 __all__ = ["Extractor"]

cognite/extractorutils/unstable/configuration/models.py

@@ -1,3 +1,4 @@
+import os
 import re
 from datetime import timedelta
 from enum import Enum
@@ -215,6 +216,36 @@ class ConnectionConfig(ConfigModel):
 
         return CogniteClient(client_config)
 
+    @classmethod
+    def from_environment(cls) -> "ConnectionConfig":
+        auth: AuthenticationConfig
+        if "COGNITE_CLIENT_SECRET" in os.environ:
+            auth = _ClientCredentialsConfig(
+                type="client-credentials",
+                client_id=os.environ["COGNITE_CLIENT_ID"],
+                client_secret=os.environ["COGNITE_CLIENT_SECRET"],
+                token_url=os.environ["COGNITE_TOKEN_URL"],
+                scopes=os.environ["COGNITE_TOKEN_SCOPES"].split(","),
+            )
+        elif "COGNITE_CLIENT_CERTIFICATE_PATH" in os.environ:
+            auth = _ClientCertificateConfig(
+                type="client-certificate",
+                client_id=os.environ["COGNITE_CLIENT_ID"],
+                path=Path(os.environ["COGNITE_CLIENT_CERTIFICATE_PATH"]),
+                password=os.environ.get("COGNITE_CLIENT_CERTIFICATE_PATH"),
+                authority_url=os.environ["COGNITE_AUTHORITY_URL"],
+                scopes=os.environ["COGNITE_TOKEN_SCOPES"].split(","),
+            )
+        else:
+            raise KeyError("Missing auth, either COGNITE_CLIENT_SECRET or COGNITE_CLIENT_CERTIFICATE_PATH must be set")
+
+        return ConnectionConfig(
+            project=os.environ["COGNITE_PROJECT"],
+            base_url=os.environ["COGNITE_BASE_URL"],
+            integration=os.environ["COGNITE_INTEGRATION"],
+            authentication=auth,
+        )
+
 
 class CronConfig(ConfigModel):
     type: Literal["cron"]

cognite/extractorutils/uploader/files.py

@@ -25,6 +25,7 @@ from typing import (
 )
 from urllib.parse import ParseResult, urlparse
 
+from h11._util import LocalProtocolError
 from httpx import URL, Client, Headers, Request, StreamConsumed, SyncByteStream
 from requests.utils import super_len
 
@@ -481,6 +482,19 @@ class IOFileUploadQueue(AbstractUploadQueue):
             try:
                 upload_file(read_file, file_meta)
 
+            except LocalProtocolError as e:
+                error_message = str(e)
+                if (
+                    "Too much data for declared Content" in error_message
+                    or "Too little data for declared Content-Length" in error_message
+                ):
+                    self.logger.error(
+                        f"Content Length Mismatch while uploading file: {file_meta} error message: {error_message}"
+                    )
+
+                self.add_entry_failure_logger(file_name=str(file_meta.name), error=e)
+                self.errors.append(e)
+
             except Exception as e:
                 self.logger.exception(
                     f"Unexpected error while uploading file: {file_meta.external_id} {file_meta.name}"

{cognite_extractor_utils-7.5.9.dist-info → cognite_extractor_utils-7.5.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cognite-extractor-utils
-Version: 7.5.9
+Version: 7.5.11
 Summary: Utilities for easier development of extractors for CDF
 Project-URL: repository, https://github.com/cognitedata/python-extractor-utils
 Author-email: Mathias Lohne <mathias.lohne@cognite.com>
@@ -14,7 +14,7 @@ Requires-Dist: azure-identity>=1.14.0
 Requires-Dist: azure-keyvault-secrets>=4.7.0
 Requires-Dist: cognite-sdk>=7.59.0
 Requires-Dist: croniter>=6.0.0
-Requires-Dist: dacite>=1.6.0
+Requires-Dist: dacite<1.9.0,>=1.6.0
 Requires-Dist: decorator>=5.1.1
 Requires-Dist: httpx<1,>=0.27.0
 Requires-Dist: jsonlines>=4.0.0

{cognite_extractor_utils-7.5.9.dist-info → cognite_extractor_utils-7.5.11.dist-info}/RECORD

@@ -1,4 +1,4 @@
-cognite/extractorutils/__init__.py,sha256=9ImaA1fzZeO4aXeQjkMOpz_E7Nky0b4IP9htbfQLNjI,764
+cognite/extractorutils/__init__.py,sha256=QMrJrW8mM_pXGYl31dhRBsnmCVY7HtLNYMUG_X-BvLw,765
 cognite/extractorutils/_inner_util.py,sha256=ZMZIBwFpSD39RzLZq_OJlrFtVt8NFOtU4ObzAG8vTB4,1779
 cognite/extractorutils/base.py,sha256=-jkGgUMYKdhFReua8ZoGUQwT1qbOvg6nTrt0lyJOGVc,16354
 cognite/extractorutils/exceptions.py,sha256=4qreRiTwZH9lyLIKR67TP02MUxN9oYhCd2vFTb5bRME,1125
@@ -21,7 +21,7 @@ cognite/extractorutils/unstable/__init__.py,sha256=L6nqJHjylpk67CE-PbXJyb_TBI4yj
 cognite/extractorutils/unstable/configuration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/extractorutils/unstable/configuration/exceptions.py,sha256=2-0jUp9IFwzk2QTQzWLoGgW1KOApk9UmPmLwtwRUBaE,591
 cognite/extractorutils/unstable/configuration/loaders.py,sha256=guBSVz7nd3dfsNEfPEHUUmU5WoYdPQmBBaJk2FuKXVw,3640
-cognite/extractorutils/unstable/configuration/models.py,sha256=vrwe53CZroujNVVpgQbmZEOywtR8PMJ0exi9gBjc5Gw,8160
+cognite/extractorutils/unstable/configuration/models.py,sha256=YYGXxSwct04J9A6NDlW0kpKHvlvtPxuLeMHgq5KwH-g,9567
 cognite/extractorutils/unstable/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cognite/extractorutils/unstable/core/_dto.py,sha256=tvvy39cvf-QT28GWz5FpqxQ5vAVk0t69JoPPhpWlweY,1293
 cognite/extractorutils/unstable/core/_messaging.py,sha256=D9rOW8fijryXffbm90d8VTf2vy5FmwVGU-H0O-cn-EI,68
@@ -40,11 +40,11 @@ cognite/extractorutils/uploader/_metrics.py,sha256=J2LJXb19L_SLSJ_voNIQHYLp0pjxU
 cognite/extractorutils/uploader/assets.py,sha256=a6LKbGjWh29Lj0PnL1bdhP88e27FKh8xfTFTsOBUOew,5713
 cognite/extractorutils/uploader/data_modeling.py,sha256=h4uRnppU2dVBquJz0Uy2SKNy5A1Gq-g7ZAlERu5vjIU,3614
 cognite/extractorutils/uploader/events.py,sha256=soU8TslPFIA7Ck4ATR_a7MHGitztRcn0cEt7LIbMPsE,5643
-cognite/extractorutils/uploader/files.py,sha256=U3wBU8iJNbOIP-JV_K3tRVHkB1gsb4lCjNlWbr-wg-0,26553
+cognite/extractorutils/uploader/files.py,sha256=ag3TUrImJdAUiOSXugagD03-9WGlFpIyd31z3WMJU20,27180
 cognite/extractorutils/uploader/raw.py,sha256=8duMk9uVJPK3O6C0nzwnvrl0UzBtJs_T_L1la0Vo89k,6713
 cognite/extractorutils/uploader/time_series.py,sha256=rs-jEChEZymxc98TYePpSRstPjQnDJd97vjOEYc-0-A,26568
 cognite/extractorutils/uploader/upload_failure_handler.py,sha256=wbUSXUP26rbmK2NGdQMzcWQwNOh2sXYOnlj0A_sk1n8,2003
-cognite_extractor_utils-7.5.9.dist-info/METADATA,sha256=jInc1LEbqmrE65cSI1kdktojnU0Y8AU5muy98PjmT7I,5272
-cognite_extractor_utils-7.5.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cognite_extractor_utils-7.5.9.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-cognite_extractor_utils-7.5.9.dist-info/RECORD,,
+cognite_extractor_utils-7.5.11.dist-info/METADATA,sha256=Ywvqxw4d9cEHRmAzbjjiNwdAmgb2FB2mxpo9XhLmGBE,5280
+cognite_extractor_utils-7.5.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cognite_extractor_utils-7.5.11.dist-info/licenses/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+cognite_extractor_utils-7.5.11.dist-info/RECORD,,