pipeline-eds 0.2.4__py3-none-any.whl

pipeline/time_manager.py

@@ -0,0 +1,211 @@
+from datetime import datetime, timezone
+from typing import Union
+import click
+try:
+    from zoneinfo import ZoneInfo
+except ImportError:
+    from backports.zoneinfo import ZoneInfo
+
+
+class TimeManager:
+    """
+    TimeManager is a flexible utility for handling various datetime representations.
+
+    Supports initialization from:
+    - ISO 8601 string (e.g., "2025-07-19T15:00:00Z")
+    - Formatted datetime string (e.g., "2025-07-19 15:00:00")
+    - Unix timestamp as int or float
+    - datetime.datetime object
+
+    Example usage:
+        tm1 = TimeManager("2025-07-19T15:00:00Z")        # ISO 8601
+        tm2 = TimeManager("2025-07-19 15:00:00")         # formatted string
+        tm3 = TimeManager(1752946800)                    # unix int
+        tm4 = TimeManager(1752946800.0)                  # unix float
+        tm5 = TimeManager(datetime(2025, 7, 19, 15, 0))  # datetime object
+
+        print(tm1.as_formatted_date_time())  # → '2025-07-19 15:00:00'
+        print(tm1.as_formatted_time())  # → '15:00:00'
+        print(tm1.as_isoz())                  # → '2025-07-19T15:00:00Z'
+        print(tm1.as_unix())                 # → 1752946800
+        print(tm1.as_datetime())             # → datetime.datetime(2025, 7, 19, 15, 0)
+
+        rounded_tm = tm1.round_down_to_nearest_five()
+        print(rounded_tm.as_formatted_date_time())
+
+        now_tm = TimeManager.now()
+        now_rounded_tm = TimeManager.now_rounded_to_five()
+    """
+    
+    HOW_TO_UTCZ_DOC =  """
+    # HOW TO CONVERT TIME BEFORE USING TIMEMANAGER
+    ## 1. Create a datetime in Central Time
+    central_time = datetime(2025, 7, 19, 10, 0, tzinfo=ZoneInfo("America/Chicago"))
+
+    ## 2. Convert to UTC
+    utc_time = central_time.astimezone(ZoneInfo("UTC"))
+
+    ## 3. Use the TimeManager class to ensure ISO format (with Z). 
+    utc_time_z = TimeManager(utc_time).as_isoz()
+
+    print("Central:", central_time)
+    print("UTC:    ", utc_time)
+
+    # ALTERNATIVE METHODS 
+    ## - Prepare single timestamp (top of the hour UTC)
+    ``` 
+    import datetime
+    timestamp = datetime.datetime.now(datetime.timezone.utc).replace(minute=0, second=0, microsecond=0)
+    timestamp_str = timestamp.strftime('%Y-%m-%d %H:%M:%S')
+    ```
+
+    """
+
+    def __init__(self, timestamp: Union[str, int, float, datetime]):
+        if isinstance(timestamp, datetime):
+            self._dt = timestamp.replace(tzinfo=timezone.utc)
+        elif isinstance(timestamp, (int, float)):
+            self._dt = datetime.fromtimestamp(timestamp, tz=timezone.utc)
+            '''
+        elif isinstance(timestamp, str):
+            try:
+                # Use fromisoformat (Python 3.7+)
+                self._dt = datetime.fromisoformat(timestamp).replace(tzinfo=timezone.utc)
+            except ValueError:
+                self._dt = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
+            '''
+        elif isinstance(timestamp, str):
+            try:
+                if timestamp.endswith("Z"):
+                    # Strip 'Z' and parse as UTC
+                    self._dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+                else:
+                    # Try ISO 8601 string with offset (e.g., +00:00)
+                    self._dt = datetime.fromisoformat(timestamp).astimezone(timezone.utc)
+            except ValueError:
+                # Fallback to "YYYY-MM-DD HH:MM:SS"
+                self._dt = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
+
+            '''
+        elif isinstance(timestamp, str):
+            try:
+                # Try ISO 8601 with 'Z'
+                self._dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+            except ValueError:
+                # Try formatted string without timezone
+                self._dt = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
+            '''
+        else:
+            raise TypeError(f"Unsupported timestamp type: {type(timestamp)}")
+
+    def as_datetime(self) -> datetime:
+        """Return the internal datetime object (UTC)."""
+        return self._dt
+    
+    def as_safe_isoformat_for_filename(self) -> str:
+        """
+        Returns an ISO 8601 formatted UTC time string safe for use in filenames.
+        Example: '2025-07-19T23-35-00Z'
+        """
+        return self.as_datetime().isoformat().replace(":", "-") + "Z"
+
+    def as_unix(self):# -> int:
+        """Return the Unix timestamp as an integer."""
+        return int(self._dt.timestamp())
+
+    def as_isoz(self):# -> str:
+        """Return ISO 8601 string (UTC) with 'Z' suffix."""
+        return self._dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+    
+    def as_iso(self):# -> str:
+        """Return ISO 8601, like datetime.fromtimestamp(ts).isoformat()."""
+        return self._dt.isoformat()
+
+    def as_formatted_date_time(self):# -> str:
+        """Return formatted string 'YYYY-MM-DD HH:MM:SS'."""
+        return self._dt.strftime("%Y-%m-%d %H:%M:%S")
+
+    def as_formatted_time(self):# -> str:
+        """Return formatted string 'HH:MM:SS'."""
+        return self._dt.strftime("%H:%M:%S")
+    
+    def as_excel(self):# -> float:
+        """Returns Excel serial number for Windows (based on 1899-12-30 epoch)."""
+        unix_ts = self.as_unix()
+        return unix_ts / 86400 + 25569  # 86400 seconds in a day
+
+    def round_down_to_nearest_five(self):# -> "TimeManager":
+        """Return new TimeManager rounded down to nearest 5-minute mark."""
+        minute = (self._dt.minute // 5) * 5
+        rounded_dt = self._dt.replace(minute=minute, second=0, microsecond=0)
+        return TimeManager(rounded_dt).as_unix()
+
+    @staticmethod
+    def now():# -> "TimeManager":
+        """Return current UTC time as a TimeManager."""
+        return TimeManager(datetime.now(timezone.utc)).as_unix()
+    
+
+    @staticmethod
+    #def from_local(dt: datetime, zone_name: str) -> "TimeManager":
+    def from_local(dt, zone_name):
+        """
+        Convert a local datetime in the given time zone to UTC and return a TimeManager instance.
+
+        Args:
+            dt (datetime): The local datetime (can be naive or aware).
+            zone_name (str): A valid IANA time zone string, e.g. 'America/Chicago'.
+
+        Returns:
+            TimeManager: A new instance based on the UTC version of the datetime.
+        """
+        if dt.tzinfo is None:
+            local_dt = dt.replace(tzinfo=ZoneInfo(zone_name))
+        else:
+            local_dt = dt.astimezone(ZoneInfo(zone_name))
+        utc_dt = local_dt.astimezone(timezone.utc)
+        return TimeManager(utc_dt)
+
+        
+    @staticmethod
+    def now_rounded_to_five():# -> "TimeManager":
+        """Return current UTC time rounded down to nearest 5 minutes."""
+        now = datetime.now(timezone.utc)
+        minute = (now.minute // 5) * 5
+        rounded = now.replace(minute=minute, second=0, microsecond=0)
+        return TimeManager(rounded).as_unix()
+
+    def __repr__(self):
+        return f"TimeManager({self.as_isoz()})"
+
+    def __str__(self):
+        return self.as_formatted_date_time()
+   
+@click.command()
+def main():
+    click.echo("WELCOME TO THE `TimeManager` CLASS")
+    click.echo("pipx install mulch")
+    click.echo("from mulch.time_manager import TimeManager")
+    click.echo("")
+    click.echo("test>> click.MultiCommand(True)")
+    click.MultiCommand(True)
+    click.echo("test>> click.MultiCommand()")
+    click.MultiCommand()
+
+    msg0 = ''' python time_manager.py             # runs 'main' by default if you set it as default command (else error)
+python time_manager.py main        # run main
+python time_manager.py easteregg   # run easteregg
+python time_manager.py howto-utcz  # show how-to UTCZ doc
+python time_manager.py license     # show license
+'''
+    click.echo(msg0)
+def easteregg():
+    click.echo("from mulch.philosophy import Philosophy")
+    click.echo("PS C:Users/user/dev/pipeline >> poetry run python -m mulch.philosphy --help # click ")
+
+def howto_utcz():
+    click.echo(TimeManager.HOW_TO_UTCZ_DOC)
+def license():
+    pass
+if __name__ == "__main__":
+    main()

pipeline/workspace_manager.py

@@ -0,0 +1,253 @@
+import os
+import toml
+import logging
+from pathlib import Path
+
+'''
+Goal:
+Implement default-workspace.toml variable: use-most-recently-edited-workspace-directory 
+'''
+
+# Configure logging (adjust level as needed)
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+class WorkspaceManager:
+    # It has been chosen to not make the WorkspaceManager a singleton if there is to be batch processing.
+
+    WORKSPACES_DIR_NAME = 'workspaces'
+    QUERIES_DIR_NAME = 'queries'
+    IMPORTS_DIR_NAME = 'imports'
+    EXPORTS_DIR_NAME = 'exports'
+    SCRIPTS_DIR_NAME = 'scripts'
+    CONFIGURATIONS_DIR_NAME = 'configurations'
+    SECRETS_DIR_NAME ='secrets'
+    LOGS_DIR_NAME = 'logs'
+    CONFIGURATION_FILE_NAME = 'configuration.toml'
+    SECRETS_YAML_FILE_NAME ='secrets.yaml'
+    SECRETS_EXAMPLE_YAML_FILE_NAME ='secrets-example.yaml'
+    DEFAULT_WORKSPACE_TOML_FILE_NAME = 'default-workspace.toml'
+    TIMESTAMPS_JSON_FILE_NAME = 'timestamps_success.json'
+    ROOT_DIR = Path(__file__).resolve().parents[2]  # root directory
+    
+    
+    # This climbs out of /src/pipeline/ to find the root.
+    # parents[0] → The directory that contains the (this) Python file.
+    # parents[1] → The parent of that directory.
+    # parents[2] → The grandparent directory (which should be the root), if root_pipeline\src\pipeline\
+    # This organization anticipates PyPi packaging.
+
+    
+    def __init__(self, workspace_name):
+        self.workspace_name = workspace_name
+        self.workspaces_dir = self.get_workspaces_dir()
+        self.workspace_dir = self.get_workspace_dir()
+        self.configurations_dir = self.get_configurations_dir()
+        self.exports_dir = self.get_exports_dir()
+        self.imports_dir = self.get_imports_dir()
+        self.queries_dir = self.get_queries_dir()
+        self.secrets_dir = self.get_secrets_dir()
+        self.scripts_dir = self.get_scripts_dir()
+        self.logs_dir = self.get_logs_dir()
+        self.aggregate_dir = self.get_aggregate_dir()
+
+        
+        self.check_and_create_dirs(list_dirs = 
+                                    [self.workspace_dir, 
+                                    self.exports_dir, 
+                                    self.imports_dir, 
+                                    self.secrets_dir, 
+                                    self.scripts_dir, 
+                                    self.logs_dir,
+                                    self.aggregate_dir])
+
+    def get_workspaces_dir(self):
+        return self.ROOT_DIR / self.WORKSPACES_DIR_NAME
+
+    def get_workspace_dir(self):
+        return self.get_workspaces_dir() / self.workspace_name
+
+    def get_exports_dir(self):
+        return self.workspace_dir / self.EXPORTS_DIR_NAME
+    
+    def get_exports_file_path(self, filename):
+        # Return the full path to the export file
+        return self.exports_dir / filename
+
+    def get_aggregate_dir(self):
+        # This is for five-minute aggregation data to be stored between hourly bulk passes
+        # This should become defunct once the tabular trend data request is functional 
+        return self.exports_dir / 'aggregate'
+    
+    def get_configurations_dir(self):
+        return self.workspace_dir / self.CONFIGURATIONS_DIR_NAME
+    
+    def get_configuration_file_path(self):
+        # Return the full path to the config file or create it from the fallback copy if it exists
+        file_path = self.get_configurations_dir() / self.CONFIGURATION_FILE_NAME
+        return file_path
+    
+    def get_logs_dir(self):
+        return self.workspace_dir / self.LOGS_DIR_NAME
+
+    def get_imports_dir(self):
+        return self.workspace_dir / self.IMPORTS_DIR_NAME
+
+    def get_imports_file_path(self, filename):
+        # Return the full path to the export file
+        return self.imports_dir / filename
+        
+    def get_secrets_dir(self):
+        return self.workspace_dir / self.SECRETS_DIR_NAME
+
+    def get_secrets_file_path(self):
+        # Return the full path to the config file
+        file_path = self.secrets_dir / self.SECRETS_YAML_FILE_NAME
+        if not file_path.exists():
+            logging.warning(f"Secrets sonfiguration file {self.SECRETS_YAML_FILE_NAME} not found in:\n{self.secrets_dir}.\nHint: Copy and edit the {self.SECRETS_YAML_FILE_NAME}.")
+            print("\n")
+            choice = str(input(f"Auto-copy {self.SECRETS_EXAMPLE_YAML_FILE_NAME} [Y] or sys.exit() [n] ? "))
+            if choice.lower().startswith("y"):
+                file_path = self.get_secrets_file_path_or_copy()
+            else:
+                # edge case, expected once per machine, or less, if the user knows to set up a secrets.yaml file.
+                import sys 
+                sys.exit()
+        return file_path
+    
+    def get_secrets_file_path_or_copy(self):
+        # Return the full path to the config file or create it from the fallback copy if it exists
+        file_path = self.secrets_dir / self.SECRETS_YAML_FILE_NAME
+        fallback_file_path = self.secrets_dir / self.SECRETS_EXAMPLE_YAML_FILE_NAME
+        if not file_path.exists() and fallback_file_path.exists():
+            import shutil
+            shutil.copy(fallback_file_path, file_path)
+            print(f"{self.SECRETS_YAML_FILE_NAME} not found, copied from {self.SECRETS_YAML_FILE_NAME}")
+        elif not file_path.exists() and not fallback_file_path.exists():
+            raise FileNotFoundError(f"Configuration file {self.SECRETS_YAML_FILE_NAME} nor {self.SECRETS_EXAMPLE_YAML_FILE_NAME} not found in directory '{self.secrets_dir}'.")
+        return file_path
+
+    def get_scripts_dir(self):
+        return self.workspace_dir / self.SCRIPTS_DIR_NAME
+
+    def get_scripts_file_path(self, filename):
+        # Return the full path to the config file
+        return self.get_scripts_dir() / filename
+    
+    def get_queries_dir(self):
+        return self.workspace_dir / self.QUERIES_DIR_NAME
+    
+    def get_queries_file_path(self,filename): #
+        # Return the full path to the config file
+        filepath = self.get_queries_dir() / filename
+        if not filepath.exists():
+            raise FileNotFoundError(f"Query filepath={filepath} not found. \nPossible reason: You are in the wrong project directory.")
+        return filepath    
+    
+    def get_timestamp_success_file_path(self):
+        # Return the full path to the timestamp file
+        filepath = self.get_queries_dir() / self.TIMESTAMPS_JSON_FILE_NAME
+        logging.info(f"WorkspaceManager.get_timestamp_success_file_path() = {filepath}")
+        return filepath
+
+    def check_and_create_dirs(self, list_dirs):
+        for dir_path in list_dirs:
+            if not dir_path.exists():
+                dir_path.mkdir(parents=True, exist_ok=True)
+
+    @classmethod
+    def get_cwd(cls) -> Path:
+        """Return current workspace directory, not the source code root, as a Path instance."""
+        # Quick and dirty, not representative of the complex truth or opportunity.
+        #cls.ROOT_DIR / 'workspaces' / cls.identify_default_workspace_name()
+        
+        # Pre-Exisiting function, generated some time before July 24, 2025. May as well use that instead. It is good to use your own library. Benefit from having built it.
+        return cls.identify_default_workspace_name()
+
+    @classmethod
+    def get_all_workspaces_names(cls):
+        """
+        Return a list of all workspace names found in the workspaces directory.
+        """
+        workspaces_dir = cls.ROOT_DIR / cls.WORKSPACES_DIR_NAME
+        if not workspaces_dir.exists():
+            raise FileNotFoundError(f"Workspaces directory not found at: {workspaces_dir}")
+        
+        workspace_dirs = [
+            p.name for p in workspaces_dir.iterdir()
+            if p.is_dir() and not p.name.startswith('.')  # skip hidden/system folders
+        ]
+        return workspace_dirs
+
+    @classmethod
+    def identify_default_workspace_path(cls):
+        """
+        Class method that reads default-workspace.toml to identify the default-workspace path.
+        """
+        workspace_name = cls.identify_default_workspace_name()
+        workspaces_dir = cls.ROOT_DIR / cls.WORKSPACES_DIR_NAME
+        default_workspace_path = workspaces_dir / workspace_name
+        if not default_workspace_path.exists():
+            raise FileNotFoundError(f"Default workspace directory not found: {default_workspace_path}")
+        return default_workspace_path
+    @classmethod
+    def identify_default_workspace_name(cls):
+        """
+        Class method that reads default-workspace.toml to identify the default-workspace.
+        """
+         
+        workspaces_dir = cls.ROOT_DIR / cls.WORKSPACES_DIR_NAME
+        logging.info(f"workspaces_dir = {workspaces_dir}\n")
+        default_toml_path = workspaces_dir / cls.DEFAULT_WORKSPACE_TOML_FILE_NAME
+
+        if not default_toml_path.exists():
+            raise FileNotFoundError(f"Missing {cls.DEFAULT_WORKSPACE_TOML_FILE_NAME} in {workspaces_dir}")
+
+        with open(default_toml_path, 'r') as f:
+            data = toml.load(f)
+            logging.debug(f"data = {data}") 
+        try:
+            return data['default-workspace']['workspace'] # This dictates the proper formatting of the TOML file.
+        except KeyError as e:
+            raise KeyError(f"Missing key in {cls.DEFAULT_WORKSPACE_TOML_FILE_NAME}: {e}")
+        
+    def get_default_query_file_paths_list(self):
+        
+        default_query_path = self.get_queries_dir()/ 'default-queries.toml'
+        
+        with open(default_query_path, 'r') as f:
+            query_config = toml.load(f)
+        filenames = query_config['default-query']['files']
+        if not isinstance(filenames, list):
+            raise ValueError("Expected a list under ['default-query']['files'] in default-queries.toml")
+        paths = [self.get_queries_file_path(fname) for fname in filenames]
+
+        for path in paths:
+            if not os.path.exists(path):
+                raise FileNotFoundError(f"Query file not found: {path}")
+        return paths
+
+    @property
+    def name(self):
+        return self.workspace_name
+    
+def establish_default_workspace():
+    workspace_name = WorkspaceManager.identify_default_workspace_name()
+    logging.info(f"workspace_name = {workspace_name}")
+    workspace_manager = WorkspaceManager(workspace_name)
+    logging.info(f"WorkspaceManager.get_workspace_dir() = {WorkspaceManager.get_workspace_dir()}")
+    return 
+
+def demo_establish_default_workspace():
+    establish_default_workspace()
+
+if __name__ == "__main__":
+    import sys
+    cmd = sys.argv[1] if len(sys.argv) > 1 else "default"
+
+    if cmd == "demo-default":
+        demo_establish_default_workspace()
+    else:
+        print("Usage options: \n" 
+        "poetry run python -m pipeline.api.eds demo-default \n")  
+
+    

pipeline_eds-0.2.4.dist-info/LICENSE

@@ -0,0 +1,14 @@
+BSD-3 License
+
+Copyright (c) 2025 George Clayton Bennett
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+