hydroserverpy 1.1.1__tar.gz → 1.2.0__tar.gz

{hydroserverpy-1.1.1/src/hydroserverpy.egg-info → hydroserverpy-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.1.1
+Version: 1.2.0
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = hydroserverpy
-version = 1.1.1
+version = 1.2.0
 
 [options]
 package_dir = 

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0}/src/hydroserverpy/api/models/etl/data_source.py

@@ -129,7 +129,7 @@ class DataSource(HydroServerModel, DataSourceFields, OrchestrationConfigurationF
             return
 
         if self.settings["extractor"]["type"] == "local":
-            with open(self.settings["extractor"]["path"]) as data_file:
+            with open(self.settings["extractor"]["sourceUri"]) as data_file:
                 loader = HydroServerETLCSV(
                     self._connection, data_file=data_file, data_source=self
                 )
@@ -137,7 +137,7 @@ class DataSource(HydroServerModel, DataSourceFields, OrchestrationConfigurationF
         elif self.settings["extractor"]["type"] == "HTTP":
             with tempfile.NamedTemporaryFile(mode="w+") as temp_file:
                 response = requests.get(
-                    self.settings["extractor"]["urlTemplate"],
+                    self.settings["extractor"]["sourceUri"],
                     stream=True,
                     timeout=60,
                 )

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0}/src/hydroserverpy/api/services/iam/workspace.py

@@ -178,7 +178,7 @@ class WorkspaceService(EndpointService):
                     None,
                     ...,
                 )
-                else None
+                else expires_at
             )
         }
         headers = {"Content-type": "application/json"}

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0}/src/hydroserverpy/api/services/sta/datastream.py

@@ -195,7 +195,7 @@ class DatastreamService(SensorThingsService):
                     None,
                     ...,
                 )
-                else None
+                else phenomenon_begin_time
             ),
             "phenomenonEndTime": (
                 phenomenon_end_time.isoformat()
@@ -204,7 +204,7 @@ class DatastreamService(SensorThingsService):
                     None,
                     ...,
                 )
-                else None
+                else phenomenon_end_time
             ),
             "resultBeginTime": (
                 result_begin_time.isoformat()
@@ -213,7 +213,7 @@ class DatastreamService(SensorThingsService):
                     None,
                     ...,
                 )
-                else None
+                else result_begin_time
             ),
             "resultEndTime": (
                 result_end_time.isoformat()
@@ -222,7 +222,7 @@ class DatastreamService(SensorThingsService):
                     None,
                     ...,
                 )
-                else None
+                else result_end_time
             ),
             "isPrivate": is_private,
             "isVisible": is_visible,

hydroserverpy-1.2.0/src/hydroserverpy/etl/extractors/base.py

@@ -0,0 +1,57 @@
+from abc import abstractmethod
+import logging
+import pandas as pd
+from datetime import datetime
+
+from src.hydroserverpy.etl.timestamp_parser import TimestampParser
+
+
+class Extractor:
+    def __init__(self, settings: dict):
+        self.settings = settings
+        self.source_uri = settings["sourceUri"]
+
+    def resolve_placeholder_variables(self, payload, loader):
+        logging.info(f"Creating runtime variables...")
+        filled = {}
+        for var in self.settings.get("placeholderVariables", []):
+            name = var["name"]
+            var_type = var.get("type", None)
+
+            if var_type == "runTime":
+                logging.info(f"Resolving runtime var: {name}")
+                if var.get("runTimeValue", None) == "latestObservationTimestamp":
+                    value = loader.earliest_begin_date(payload)
+                elif var.get("runTimeValue", None) == "jobExecutionTime":
+                    value = pd.Timestamp.now(tz="UTC")
+            elif var_type == "perPayload":
+                logging.info(f"Resolving payload var: {name}")
+                payload_vars = payload.get("extractorVariables", {})
+                if name not in payload_vars:
+                    raise KeyError(f"Missing per-payload variable '{name}'")
+                value = payload_vars[name]
+            else:
+                continue
+
+            if isinstance(value, (datetime, pd.Timestamp)):
+                fmt = var.get("timestampFormat", "ISO8601")
+                offset = var.get("timestampOffset", "+0000")
+                parser = TimestampParser(fmt, offset)
+                value = parser.format(value)
+
+            filled[name] = value
+        if not filled:
+            return self.source_uri
+        return self.format_uri(filled)
+
+    def format_uri(self, placeholder_variables):
+        try:
+            uri = self.source_uri.format(**placeholder_variables)
+        except KeyError as e:
+            missing_key = e.args[0]
+            raise KeyError(f"Missing placeholder variable: {missing_key}")
+        return uri
+
+    @abstractmethod
+    def extract(self):
+        pass

hydroserverpy-1.2.0/src/hydroserverpy/etl/extractors/http_extractor.py

@@ -0,0 +1,29 @@
+import logging
+import requests
+from io import BytesIO
+from .base import Extractor
+
+
+class HTTPExtractor(Extractor):
+    def __init__(self, settings: dict):
+        super().__init__(settings)
+
+    def extract(self, payload, loader=None):
+        """
+        Downloads the file from the HTTP/HTTPS server and returns a file-like object.
+        """
+        url = self.resolve_placeholder_variables(payload, loader)
+        logging.info(f"Requesting data from → {url}")
+
+        try:
+            response = requests.get(url)
+        except Exception as e:
+            logging.error(f"Failed to fetch {url}: {e}")
+            raise
+
+        data = BytesIO()
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                data.write(chunk)
+        data.seek(0)
+        return data

hydroserverpy-1.2.0/src/hydroserverpy/etl/extractors/local_file_extractor.py

@@ -0,0 +1,19 @@
+import logging
+from .base import Extractor
+
+
+class LocalFileExtractor(Extractor):
+    def __init__(self, settings: object):
+        super().__init__(settings)
+
+    def extract(self):
+        """
+        Opens the file and returns a file-like object.
+        """
+        try:
+            file_handle = open(self.source_uri, "r")
+            logging.info(f"Successfully opened file '{self.source_uri}'.")
+            return file_handle
+        except Exception as e:
+            logging.error(f"Error opening file '{self.source_uri}': {e}")
+            return None

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0}/src/hydroserverpy/etl/loaders/base.py

@@ -9,5 +9,5 @@ class Loader(ABC):
         pass
 
     @abstractmethod
-    def get_data_requirements(self, df: pd.DataFrame) -> Dict[str, pd.Timestamp]:
+    def earliest_begin_date(self, payload_mappings) -> str:
         pass

hydroserverpy-1.2.0/src/hydroserverpy/etl/loaders/hydroserver_loader.py

@@ -0,0 +1,70 @@
+from hydroserverpy import HydroServer
+from typing import Optional
+
+from .base import Loader
+import logging
+import pandas as pd
+
+
+class HydroServerLoader(HydroServer, Loader):
+    """
+    A class that extends the HydroServer client with ETL-specific functionalities.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        email: Optional[str] = None,
+        password: Optional[str] = None,
+        apikey: Optional[str] = None,
+    ):
+        super().__init__(
+            host=host,
+            email=email,
+            password=password,
+            apikey=apikey,
+        )
+        self._begin_cache: dict[str, str] = {}
+
+    def load(self, data: pd.DataFrame, payload) -> None:
+        """
+        Load observations from a DataFrame to the HydroServer.
+        :param data: A Pandas DataFrame where each column corresponds to a datastream.
+        """
+        begin_date = self.earliest_begin_date(payload)
+        new_data = data[data["timestamp"] > begin_date]
+        for col in new_data.columns.difference(["timestamp"]):
+            df = (
+                new_data[["timestamp", col]]
+                .rename(columns={col: "value"})
+                .dropna(subset=["value"])
+            )
+            if df.empty:
+                logging.warning(f"No new data for {col}, skipping.")
+                continue
+            logging.info(f"loading dataframe {df}")
+            logging.info(f"dtypes: {df.dtypes}")
+
+            df["value"] = pd.to_numeric(df["value"], errors="raise")
+            self.datastreams.load_observations(uid=col, observations=df)
+
+    def _fetch_earliest_begin(self, mappings: list[dict]) -> pd.Timestamp:
+        timestamps = []
+        for m in mappings:
+            ds = self.datastreams.get(uid=m["targetIdentifier"])
+            if not ds:
+                raise RuntimeError(f"Datastream {m['targetIdentifier']} not found.")
+            raw = ds.phenomenon_end_time or "1970-01-01"
+            ts = pd.to_datetime(raw, utc=True)
+            logging.info(f"timestamp {ts}")
+            timestamps.append(ts)
+        return min(timestamps)
+
+    def earliest_begin_date(self, payload: dict) -> pd.Timestamp:
+        """
+        Return earliest begin date for a payload, or compute+cache it on first call.
+        """
+        key = payload["name"]
+        if key not in self._begin_cache:
+            self._begin_cache[key] = self._fetch_earliest_begin(payload["mappings"])
+        return self._begin_cache[key]

hydroserverpy-1.2.0/src/hydroserverpy/etl/timestamp_parser.py

@@ -0,0 +1,75 @@
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Union
+import pandas as pd
+
+
+class TimestampParser:
+    def __init__(
+        self, timestamp_format: str = "ISO8601", timestamp_offset: str = "+0000"
+    ):
+        VALID_KEYS = {"utc", "iso8601", "constant"}
+        self.timestamp_offset = timestamp_offset
+        self.timestamp_format = timestamp_format
+
+        if (
+            self.timestamp_format.lower() not in VALID_KEYS
+            and "%" not in self.timestamp_format
+        ):
+            raise ValueError(
+                f"timestamp_format must be one of {', '.join(VALID_KEYS)} "
+                "or a valid strftime pattern."
+            )
+
+    def parse_series(self, raw_series: pd.Series) -> pd.Series:
+        s = raw_series.str.strip()
+        if self.timestamp_format.lower() == "utc":
+            parsed = pd.to_datetime(s, utc=True, errors="coerce")
+
+        elif self.timestamp_format.lower() == "iso8601":
+            parsed = pd.to_datetime(s, errors="coerce").dt.tz_convert("UTC")
+
+        elif self.timestamp_format.lower() == "constant":
+            off = self.timestamp_offset.strip()
+            if not (len(off) == 5 and off[0] in "+-"):
+                raise ValueError(f"Invalid timestamp_offset: {off}")
+            sign = 1 if off[0] == "+" else -1
+            hrs, mins = int(off[1:3]), int(off[3:5])
+            tz = timezone(timedelta(minutes=sign * (hrs * 60 + mins)))
+            naive = pd.to_datetime(s, errors="coerce")
+            parsed = naive.dt.tz_localize(tz).dt.tz_convert("UTC")
+
+        else:
+            parsed = pd.to_datetime(
+                s, format=self.timestamp_format, errors="coerce"
+            ).dt.tz_localize("UTC")
+
+        if parsed.isna().any():
+            bad_rows = s[parsed.isna()].head(5).tolist()
+            logging.warning(
+                f"{parsed.isna().sum()} timestamps failed to parse. "
+                f"Sample bad values: {bad_rows}"
+            )
+
+        return parsed
+
+    def format(self, dt: Union[datetime, pd.Timestamp]) -> str:
+        if isinstance(dt, pd.Timestamp):
+            dt = dt.to_pydatetime()
+
+        fmt = self.timestamp_format.lower()
+        if fmt == "utc":
+            return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
+
+        if fmt == "iso8601":
+            return dt.astimezone(timezone.utc).isoformat()
+
+        if fmt == "constant":
+            off = self.timestamp_offset.strip()
+            sign = 1 if off[0] == "+" else -1
+            hrs, mins = int(off[1:3]), int(off[3:5])
+            tz = timezone(timedelta(minutes=sign * (hrs * 60 + mins)))
+            return dt.astimezone(tz).strftime("%Y-%m-%dT%H:%M:%S")
+
+        # custom strftime
+        return dt.strftime(self.timestamp_format)

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0}/src/hydroserverpy/etl/transformers/base.py

@@ -1,8 +1,7 @@
 from abc import ABC, abstractmethod
-from datetime import timedelta, timezone
 import logging
 from typing import Union
-import pandas as pd
+from src.hydroserverpy.etl.timestamp_parser import TimestampParser
 
 
 class Transformer(ABC):
@@ -17,6 +16,10 @@ class Transformer(ABC):
             # Users will always interact in 1-based, so if the key is a column index, convert to 0-based
             self.timestamp_key = self.timestamp_key - 1
 
+        self.timestamp_parser = TimestampParser(
+            self.timestamp_format, self.timestamp_offset
+        )
+
     @abstractmethod
     def transform(self, *args, **kwargs) -> None:
         pass
@@ -55,7 +58,7 @@ class Transformer(ABC):
         to_keep = ["timestamp", *expected]
         df.drop(columns=df.columns.difference(to_keep), inplace=True)
 
-        df["timestamp"] = self._parse_timestamps(df["timestamp"])
+        df["timestamp"] = self.timestamp_parser.parse_series(df["timestamp"])
 
         df.drop_duplicates(subset=["timestamp"], keep="last")
         logging.info(f"standardized dataframe created: {df.shape}")
@@ -63,55 +66,3 @@ class Transformer(ABC):
         logging.info(f"{df.head()}")
 
         return df
-
-    def _parse_timestamps(self, raw_series: pd.Series) -> pd.Series:
-        """Return a Series of pandas UTC datetimes for the four supported modes."""
-        logging.info(f"parsing timestamps. Format: {self.timestamp_format}")
-
-        fmt = self.timestamp_format.lower()
-
-        VALID_KEYS = {"utc", "iso8601", "constant"}
-        if fmt not in VALID_KEYS and "%" not in self.timestamp_format:
-            raise ValueError(
-                f"timestamp_format must be one of {', '.join(VALID_KEYS)} "
-                "or a valid strftime pattern."
-            )
-
-        series = raw_series.str.strip()
-
-        if fmt == "utc":
-            # Accept Z-suffix, no offset, fractional seconds, etc.
-            parsed = pd.to_datetime(series, utc=True, errors="coerce")
-
-        elif fmt == "iso8601":
-            # pandas reads the embedded offset, then we shift to UTC
-            parsed = pd.to_datetime(series, errors="coerce").dt.tz_convert("UTC")
-
-        elif fmt == "constant":
-            offset = str(self.timestamp_offset).strip()
-            if not (len(offset) == 5 and offset[0] in "+-"):
-                raise ValueError(f"Invalid timestampOffset: {self.timestamp_offset}")
-
-            sign_multiplier = 1 if offset[0] == "+" else -1
-            hours = int(offset[1:3])
-            minutes = int(offset[3:5])
-            total_minutes = sign_multiplier * (hours * 60 + minutes)
-            local_timezone = timezone(timedelta(minutes=total_minutes))
-
-            naive_times = pd.to_datetime(series, errors="coerce")
-            localized_times = naive_times.dt.tz_localize(local_timezone)
-            parsed = localized_times.dt.tz_convert("UTC")
-
-        else:
-            logging.info(f"timestamp format is custom {self.timestamp_format}")
-            parsed = pd.to_datetime(
-                series, format=self.timestamp_format, errors="coerce"
-            ).dt.tz_localize("UTC")
-
-        if parsed.isna().any():
-            bad_rows = series[parsed.isna()].head(5).tolist()
-            logging.warning(
-                f"{parsed.isna().sum()} timestamps failed to parse. Sample bad values: {bad_rows}"
-            )
-
-        return parsed

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0/src/hydroserverpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydroserverpy
-Version: 1.1.1
+Version: 1.2.0
 Requires-Python: <4,>=3.9
 License-File: LICENSE
 Requires-Dist: requests>=2

{hydroserverpy-1.1.1 → hydroserverpy-1.2.0}/src/hydroserverpy.egg-info/SOURCES.txt

@@ -52,6 +52,7 @@ src/hydroserverpy/api/services/sta/thing.py
 src/hydroserverpy/api/services/sta/unit.py
 src/hydroserverpy/etl/__init__.py
 src/hydroserverpy/etl/hydroserver_etl.py
+src/hydroserverpy/etl/timestamp_parser.py
 src/hydroserverpy/etl/types.py
 src/hydroserverpy/etl/extractors/__init__.py
 src/hydroserverpy/etl/extractors/base.py

hydroserverpy-1.1.1/src/hydroserverpy/etl/extractors/base.py

@@ -1,13 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Dict
-from ..types import TimeRange
-
-
-class Extractor(ABC):
-    @abstractmethod
-    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
-        pass
-
-    @abstractmethod
-    def extract(self):
-        pass

hydroserverpy-1.1.1/src/hydroserverpy/etl/extractors/http_extractor.py

@@ -1,99 +0,0 @@
-import logging
-from hydroserverpy.etl.types import TimeRange
-import requests
-from io import BytesIO
-from typing import Dict
-from .base import Extractor
-
-
-class HTTPExtractor(Extractor):
-    def __init__(self, settings: object):
-        self.url = settings["urlTemplate"]
-        # self.url = self.format_url(url, url_variables or {})
-        # self.params = settings.get('params', )
-        # self.headers = headers
-        # self.auth = auth
-
-    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
-        pass
-        # TODO: Uncomment this once url templates work on in the Data Management App
-        # start_times = [
-        #     req["start_time"] for req in data_requirements.values() if req["start_time"]
-        # ]
-
-        # if start_times:
-        #     oldest_start_time = min(start_times)
-        #     start_time_key = self.params.pop("start_time_key", None)
-        #     if start_time_key:
-        #         self.params[start_time_key] = oldest_start_time
-        #         logging.info(
-        #             f"Set start_time to {oldest_start_time} and removed 'start_time_key'"
-        #         )
-        #     else:
-        #         logging.warning("'start_time_key' not found in params.")
-
-        # end_times = [
-        #     req["end_time"] for req in data_requirements.values() if req["end_time"]
-        # ]
-
-        # if end_times:
-        #     newest_end_time = max(end_times)
-        #     end_time_key = self.params.pop("end_time_key", None)
-        #     if end_time_key:
-        #         self.params[end_time_key] = newest_end_time
-        #         logging.info(
-        #             f"Set end_time to {newest_end_time} and removed 'end_time_key'"
-        #         )
-        #     else:
-        #         logging.warning("'end_time_key' not found in params.")
-
-    def extract(self):
-        """
-        Downloads the file from the HTTP/HTTPS server and returns a file-like object.
-        """
-
-        logging.info(f"Requesting data from → {self.url}")
-
-        # endpoints = [
-        #     "https://httpbin.org/get",
-        #     "https://jsonplaceholder.typicode.com/posts/1",
-        #     "https://api.github.com",
-        #     "https://api.ipify.org?format=json",
-        #     "https://www.python.org/",
-        #     "https://waterservices.usgs.gov/nwis/iv/?&format=json&sites=01646500&parameterCd=00060",
-        #     "https://datahub.io/core/country-list/r/data.csv",
-        #     "https://raw.githubusercontent.com/cs109/2014_data/master/countries.csv",
-        #     # "https://rain-flow.slco.org/export/file/?delimiter=comma&site_id=68&data_start=2025-04-09&data_end=2025-05-09&device_id=2",
-        #     # "https://rain-flow.slco.org/export/file/?mime=txt&delimiter=comma&site_id=68&data_start=2025-05-09%2000:00:00&data_end=2025-05-09%2023:59:59&device_id=2"
-        # ]
-        # for url in endpoints:
-        #     try:
-        #         r = requests.get(url, timeout=10)
-        #         print(f"{url:50} → {r.status_code}")
-        #     except Exception as e:
-        #         print(f"{url:50} → ERROR: {e}")
-
-        try:
-            response = requests.get(self.url)
-        except Exception as e:
-            logging.error(f"Failed to fetch {repr(self.url)}: {e}")
-            raise
-
-        logging.info(f"Received response")
-
-        data = BytesIO()
-        for chunk in response.iter_content(chunk_size=8192):
-            if chunk:
-                data.write(chunk)
-        data.seek(0)
-        return data
-
-    @staticmethod
-    def format_url(url_template, url_variables):
-        try:
-            url = url_template.format(**url_variables)
-        except KeyError as e:
-            missing_key = e.args[0]
-            raise KeyError(f"Missing configuration url_variable: {missing_key}")
-
-        return url

hydroserverpy-1.1.1/src/hydroserverpy/etl/extractors/local_file_extractor.py

@@ -1,29 +0,0 @@
-import logging
-from typing import Dict
-
-from .base import Extractor
-from ..types import TimeRange
-
-
-class LocalFileExtractor(Extractor):
-    def __init__(self, settings: object):
-        if "path" not in settings:
-            message = "Missing required setting 'path' in LocalFileExtractor settings."
-            logging.error(message)
-            raise ValueError(message)
-        self.path = settings["path"]
-
-    def prepare_params(self, data_requirements: Dict[str, TimeRange]):
-        pass
-
-    def extract(self):
-        """
-        Opens the file and returns a file-like object.
-        """
-        try:
-            file_handle = open(self.path, "r")
-            logging.info(f"Successfully opened file '{self.path}'.")
-            return file_handle
-        except Exception as e:
-            logging.error(f"Error opening file '{self.path}': {e}")
-            return None

hydroserverpy-1.1.1/src/hydroserverpy/etl/loaders/hydroserver_loader.py

@@ -1,91 +0,0 @@
-import datetime
-from hydroserverpy import HydroServer
-from typing import Dict, Optional
-
-from hydroserverpy.etl.types import TimeRange
-from .base import Loader
-import logging
-import pandas as pd
-
-
-class HydroServerLoader(HydroServer, Loader):
-    """
-    A class that extends the HydroServer client with ETL-specific functionalities.
-    """
-
-    def __init__(
-        self,
-        host: str,
-        email: Optional[str] = None,
-        password: Optional[str] = None,
-        apikey: Optional[str] = None,
-    ):
-        super().__init__(
-            host=host,
-            email=email,
-            password=password,
-            apikey=apikey,
-        )
-
-    def load(self, data: pd.DataFrame, payload_settings) -> None:
-        """
-        Load observations from a DataFrame to the HydroServer.
-
-        :param data: A Pandas DataFrame where each column corresponds to a datastream.
-        """
-        mappings = payload_settings["mappings"]
-        time_ranges = self.get_data_requirements(mappings)
-        for ds_id in data.columns:
-            if ds_id == "timestamp":
-                continue
-
-            df = data[["timestamp", ds_id]].copy()
-            df.rename(columns={ds_id: "value"}, inplace=True)
-            df.dropna(subset=["value"], inplace=True)
-
-            # ensure the timestamp column is UTC‑aware
-            timestamp_column = df["timestamp"]
-            if timestamp_column.dt.tz is None:
-                df["timestamp"] = timestamp_column.dt.tz_localize("UTC")
-
-            time_range = time_ranges[ds_id]
-            start_ts = pd.to_datetime(time_range["start_time"], utc=True)
-
-            if start_ts:
-                df = df[df["timestamp"] > start_ts]
-            logging.info(f"start cutoff for data loading {start_ts}")
-            if df.empty:
-                logging.warning(
-                    f"No new data to upload for datastream {ds_id}. Skipping."
-                )
-                continue
-            self.datastreams.load_observations(uid=ds_id, observations=df)
-
-    def get_data_requirements(self, source_target_map) -> Dict[str, TimeRange]:
-        """
-        Each target system needs to be able to answer the question: 'What data do you need?'
-        and return a time range for each target time series. Usually the answer will be
-        'anything newer than my most recent observation'.
-        """
-        data_requirements = {}
-        target_ids = [mapping["targetIdentifier"] for mapping in source_target_map]
-        for id in target_ids:
-            datastream = self.datastreams.get(uid=id)
-            if not datastream:
-                message = "Couldn't fetch target datastream. ETL process aborted."
-                logging.error(message)
-                raise message
-
-            start_ts = pd.Timestamp(
-                datastream.phenomenon_end_time or "1970-01-01T00:00:00Z"
-            )
-            if start_ts.tzinfo is None:
-                start_ts = start_ts.tz_localize("UTC")
-
-            end_ts = pd.Timestamp.now(tz="UTC")
-
-            data_requirements[id] = {
-                "start_time": start_ts.isoformat(),
-                "end_time": end_ts.isoformat(),
-            }
-        return data_requirements