pymscada 0.1.11b10__py3-none-any.whl → 0.2.0__py3-none-any.whl

pymscada/iodrivers/witsapi_POC.py

@@ -0,0 +1,246 @@
+import aiohttp
+import asyncio
+import datetime
+
+class WitsAPIClient:
+    def __init__(self, url, client_id, client_secret):
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.base_url = url
+        self.session = None
+        
+    async def __aenter__(self):
+        """Create session and get token on entry"""
+        self.session = aiohttp.ClientSession()
+        return self
+        
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Close session on exit"""
+        if self.session:
+            await self.session.close()
+    
+    async def get_token(self):
+        """Get a new OAuth token"""
+        auth_url = f"{self.base_url}/login/oauth2/token"
+        data = {
+            "grant_type": "client_credentials",
+            "client_id": self.client_id,
+            "client_secret": self.client_secret
+        }
+        
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded"
+        }
+        
+        try:
+            async with self.session.post(auth_url, data=data, headers=headers) as response:
+                if response.status == 200:
+                    result = await response.json()
+                    self.session.headers.update({
+                        "Authorization": f"Bearer {result['access_token']}"
+                    })
+                    return result["access_token"]
+                else:
+                    error_text = await response.text()
+                    return None
+        except Exception as e:
+            return None
+
+    async def get(self, endpoint):
+        """Make a GET request to the WITS API"""
+        url = f"{self.base_url}/{endpoint}"
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 200:
+                    result = await response.json()
+                    return result
+                else:
+                    error_text = await response.text()
+                    return None
+        except Exception as e:
+            return None
+
+    async def get_schedules(self):
+        """Get list of schedules for which pricing data is available"""
+        endpoint = "api/market-prices/v1/schedules"
+        return await self.get(endpoint)
+
+    async def get_nodes(self):
+        """Get list of GXP/GIP nodes supported by this API"""
+        endpoint = "api/market-prices/v1/nodes"
+        return await self.get(endpoint)
+
+    async def get_schedule_prices(self, schedule='RTD', market_type='E', nodes=None, 
+                                back=None, forward=None, from_date=None, to_date=None, 
+                                island=None, offset=0):
+        """Get prices for a single schedule
+        Args:
+            schedule: Schedule type (e.g. 'RTD' for Real Time Dispatch)
+            market_type: 'E' for energy prices, 'R' for reserve prices
+            nodes: List of node IDs to filter by
+            back: Number of trading periods to look back (1-48)
+            forward: Number of trading periods to look ahead (1-48)
+            from_date: Start datetime (RFC3339 format)
+            to_date: End datetime (RFC3339 format)
+            island: Filter by island ('NI' or 'SI')
+            offset: Pagination offset
+        """
+        endpoint = f"api/market-prices/v1/schedules/{schedule}/prices"
+        params = {
+            'marketType': market_type,
+            'offset': offset
+        }
+        if nodes:
+            params['nodes'] = ','.join(nodes) if isinstance(nodes, list) else nodes
+        if back:
+            params['back'] = min(back, 48)
+        if forward:
+            params['forward'] = min(forward, 48)
+        if from_date:
+            params['from'] = from_date
+        if to_date:
+            params['to'] = to_date
+        if island:
+            params['island'] = island
+        query = '&'.join(f"{k}={v}" for k, v in params.items())
+        return await self.get(f"{endpoint}?{query}")
+
+    async def get_multi_schedule_prices(self, schedules, market_type='E', nodes=None,
+                                      back=None, forward=None, from_date=None, 
+                                      to_date=None, island=None, offset=0):
+        """Get prices across multiple schedules
+        Args:
+            schedules: List of schedule types
+            market_type: 'E' for energy prices, 'R' for reserve prices
+            nodes: List of node IDs to filter by
+            back: Number of trading periods to look back (1-48)
+            forward: Number of trading periods to look ahead (1-48)
+            from_date: Start datetime (RFC3339 format)
+            to_date: End datetime (RFC3339 format)
+            island: Filter by island ('NI' or 'SI')
+            offset: Pagination offset
+        """
+        endpoint = "api/market-prices/v1/prices"
+        params = {
+            'schedules': ','.join(schedules) if isinstance(schedules, list) else schedules,
+            'marketType': market_type,
+            'offset': offset
+        }
+        if nodes:
+            params['nodes'] = ','.join(nodes) if isinstance(nodes, list) else nodes
+        if back:
+            params['back'] = min(back, 48)
+        if forward:
+            params['forward'] = min(forward, 48)
+        if from_date:
+            params['from'] = from_date
+        if to_date:
+            params['to'] = to_date
+        if island:
+            params['island'] = island
+        query = '&'.join(f"{k}={v}" for k, v in params.items())
+        return await self.get(f"{endpoint}?{query}")
+
+    def parse_prices(self, response):
+        """Parse API response into structured price dictionary
+        Returns dict in format:
+            {node: {trading_time_utc: {schedule: [price, last_run_utc]}}}
+        """
+        if not response:
+            return {}
+        prices = {}
+        for schedule_data in response:
+            schedule = schedule_data['schedule']
+            if 'prices' not in schedule_data:
+                continue
+            for price in schedule_data['prices']:
+                node = price['node']
+                trading_time = int(datetime.datetime.fromisoformat(
+                    price['tradingDateTime'].replace('Z', '+00:00')
+                ).timestamp())
+                last_run = int(datetime.datetime.fromisoformat(
+                    price['lastRunTime'].replace('Z', '+00:00')
+                ).timestamp())
+                
+                if node not in prices:
+                    prices[node] = {}
+                if trading_time not in prices[node]:
+                    prices[node][trading_time] = {}
+                prices[node][trading_time][schedule] = [price['price'], last_run]
+        
+        # Create RTD_forecast schedule
+        for node in prices:
+            for trading_time in prices[node]:
+                if 'RTD' in prices[node][trading_time]:
+                    prices[node][trading_time]['RTD_forecast'] = prices[node][trading_time]['RTD']
+                else:
+                    # Find most recent schedule by last run time
+                    latest_schedule = None
+                    latest_last_run = 0
+                    for schedule in prices[node][trading_time]:
+                        if prices[node][trading_time][schedule][1] > latest_last_run:
+                            latest_last_run = prices[node][trading_time][schedule][1]
+                            latest_schedule = schedule
+                    if latest_schedule:
+                        prices[node][trading_time]['RTD_forecast'] = \
+                            prices[node][trading_time][latest_schedule]
+        
+        return prices
+    
+    def print_prices(self, prices):
+        """Print prices in structured format with time information"""
+        now = datetime.datetime.now(datetime.timezone.utc)
+        now_ts = now.timestamp()
+        for node in sorted(prices.keys()):
+            print(f" - {node}:")
+            for trading_time in sorted(prices[node].keys()):
+                time_diff = trading_time - now_ts
+                # For future times on 30 minute boundaries, show half-hour intervals
+                if time_diff > 0 and trading_time % 1800 == 0:
+                    half_hours = int(time_diff / 1800)
+                    time_str = f"(+{half_hours})"
+                else:
+                    # For past times or non-30min intervals, show actual time
+                    dt = datetime.datetime.fromtimestamp(trading_time, 
+                                                       datetime.timezone.utc)
+                    time_str = f"({dt.strftime('%Y-%m-%d %H:%M:%S')})"
+                
+                print(f"   - Trading Time UTC: {trading_time} {time_str}")
+                for schedule in sorted(prices[node][trading_time].keys()):
+                    if schedule in ['RTD', 'PRSS', 'PRSL', 'RTD_forecast']:
+                        price, last_run = prices[node][trading_time][schedule]
+                        last_run_dt = datetime.datetime.fromtimestamp(
+                            last_run, datetime.timezone.utc)
+                        print(f"        {schedule:12} Price: {price:8.2f}, "
+                              f"Last Run: {last_run_dt.strftime('%H:%M:%S')}")
+
+
+async def main(config):
+    async with WitsAPIClient(url=config['url'],client_id=config['client_id'],
+                             client_secret=config['client_secret']) as client:
+        token = await client.get_token()
+        if token:
+            multi_prices = await client.get_multi_schedule_prices(
+                schedules=config['schedules'],
+                nodes=config['gxp_list'],
+                back=config['back'],
+                forward=config['forward']
+            )
+            if multi_prices:
+                prices_dict = client.parse_prices(multi_prices)
+                client.print_prices(prices_dict)
+        await asyncio.sleep(1)
+
+
+CONFIG = {
+    'url': 'https://api.electricityinfo.co.nz',
+    'gxp_list': ['MAT1101', 'CYD2201', 'BEN2201'],
+    'schedules': ['RTD', 'PRSS', 'PRSL'],
+    'back': 2,
+    'forward': 72,
+    'client_id': 'xx',
+    'client_secret': 'xx'
+}
+
+if __name__ == "__main__":
+    asyncio.run(main(CONFIG))

pymscada/main.py

@@ -39,7 +39,7 @@ async def run():
     if not options.verbose:
         root_logger.setLevel(logging.WARNING)
     factory = ModuleFactory()
-    module = factory.create_module(options.module_name, options)
+    module = factory.create_module(options)
     if module is not None:
         if hasattr(module, 'start'):
             await module.start()

pymscada/module_config.py

@@ -5,19 +5,18 @@ from textwrap import dedent
 from importlib.metadata import version
 import logging
 from pymscada.config import Config
-from pymscada.console import Console
 
 class ModuleArgument:
     def __init__(self, args: tuple[str, ...], kwargs: dict[str, Any]):
         self.args = args
-        self.kwargs = kwargs
+        self.kwargs = kwargs 
 
 class ModuleDefinition:
     """Defines a module's configuration and behavior."""
-    def __init__(self, name: str, help: str, module_class: Type[Any], *,
+    def __init__(self, name: str, help: str, module_class: Any, *,
                  config: bool = True, tags: bool = True,
                  epilog: Optional[str] = None,
-                 extra_args: list[ModuleArgument] = None,
+                 extra_args: Optional[list[ModuleArgument]] = None,
                  await_future: bool = True):
         self.name = name
         self.help = help
@@ -59,6 +58,17 @@ def create_module_registry():
             module_class='pymscada.opnotes:OpNotes',
             tags=False
         ),
+        ModuleDefinition(
+            name='alarms',
+            help='alarms',
+            module_class='pymscada.alarms:Alarms'
+        ),
+        ModuleDefinition(
+            name='callout',
+            help='alarm callout notifications',
+            module_class='pymscada.callout:Callout',
+            tags=False
+        ),
         ModuleDefinition(
             name='validate',
             help='validate config files',
@@ -75,9 +85,10 @@ def create_module_registry():
         ModuleDefinition(
             name='checkout',
             help='create example config files',
-            module_class='pymscada.checkout:checkout',
+            module_class='pymscada.checkout:Checkout',
             config=False,
             tags=False,
+            await_future=False,
             epilog=dedent("""
                 To add to systemd:
                   su -
@@ -145,6 +156,18 @@ def create_module_registry():
             module_class='pymscada.iodrivers.snmp_client:SnmpClient',
             tags=False
         ),
+        ModuleDefinition(
+            name='witsapi',
+            help='poll WITS GXP pricing real time dispatch and forecast',
+            module_class='pymscada.iodrivers.witsapi:WitsAPIClient',
+            tags=False,
+            epilog=dedent("""
+                WITS_CLIENT_ID and WITS_CLIENT_SECRET can be set in the wits.yaml
+                or as environment variables:
+                  vi ~/.bashrc
+                  export WITS_CLIENT_ID='your_client_id'
+                  export WITS_CLIENT_SECRET='your_client_secret'""")
+        ),
         ModuleDefinition(
             name='console',
             help='interactive bus console',
@@ -156,12 +179,12 @@ def create_module_registry():
                 like to see correctly typed values and set values."""),
             extra_args=[
                 ModuleArgument(
-                    ('-p', '--port'),
+                    ('-p', '--bus-port'),
                     {'action': 'store', 'type': int, 'default': 1324,
                      'help': 'connect to port (default: 1324)'}
                 ),
                 ModuleArgument(
-                    ('-i', '--ip'),
+                    ('-i', '--bus-ip'),
                     {'action': 'store', 'default': 'localhost',
                      'help': 'connect to ip address (default: localhost)'}
                 )
@@ -208,9 +231,11 @@ class ModuleFactory:
                 parser.add_argument(*arg.args, **arg.kwargs)
         return parser
 
-    def create_module(self, module_name: str, options: argparse.Namespace):
+    def create_module(self, options: argparse.Namespace):
         """Create a module instance based on configuration and options."""
-        module_def = self.modules[module_name]
+        if options.module_name not in self.modules:
+            raise ValueError(f'{options.module_name} does not exist')
+        module_def = self.modules[options.module_name]
         logging.info(f'Python Mobile SCADA {version("pymscada")} '
                      f'starting {module_def.name}')
         # Import the module class only when needed
@@ -220,13 +245,14 @@ class ModuleFactory:
             actual_class = getattr(module, class_name)
         else:
             actual_class = module_def.module_class
-            
         kwargs = {}
         if module_def.config:
-            kwargs.update(Config(options.config))  
+            kwargs.update(Config(options.config))
         if module_def.tags:
             kwargs['tag_info'] = dict(Config(options.tags))
-        if module_name == 'console':
-            return Console(options.ip, options.port,
-                         kwargs.get('tag_info',{}))
+        if module_def.extra_args:
+            for arg in module_def.extra_args:
+                arg_name = arg.args[-1].lstrip('-').replace('-', '_')
+                if hasattr(options, arg_name):
+                    kwargs[arg_name] = getattr(options, arg_name)
         return actual_class(**kwargs)

pymscada/opnotes.py

@@ -1,6 +1,7 @@
 """Operator Notes."""
 import logging
 import sqlite3  # note that sqlite3 has blocking calls
+import socket
 from pymscada.bus_client import BusClient
 from pymscada.tag import Tag
 
@@ -8,36 +9,95 @@ from pymscada.tag import Tag
 class OpNotes:
     """Connect to bus_ip:bus_port, store and provide Operator Notes."""
 
-    def __init__(self, bus_ip: str = '127.0.0.1', bus_port: int = 1324,
-                 db: str = None, rta_tag: str = '__opnotes__',
-                 table: str = 'opnotes') -> None:
+    def __init__(
+        self,
+        bus_ip: str = '127.0.0.1',
+        bus_port: int = 1324,
+        db: str | None = None,
+        table: str = 'opnotes',
+        rta_tag: str = '__opnotes__'
+    ) -> None:
         """
         Connect to bus_ip:bus_port, serve and update operator notes database.
 
-        TODO
-        Write something.
+        Open an Operator notes table, creating if necessary. Provide additions,
+        updates, deletions and history requests via the rta_tag.
 
         Event loop must be running.
+
+        For testing only: bus_ip can be None to skip connection.
         """
         if db is None:
             raise SystemExit('OpNotes db must be defined')
+        if bus_ip is None:
+            logging.warning('OpNotes has bus_ip=None, only use for testing')
+        else:
+            try:
+                socket.gethostbyname(bus_ip)
+            except socket.gaierror as e:
+                raise ValueError(f'Cannot resolve IP/hostname: {e}')
+            if not isinstance(bus_port, int):
+                raise TypeError('bus_port must be an integer')
+            if not 1024 <= bus_port <= 65535:
+                raise ValueError('bus_port must be between 1024 and 65535')
+        if not isinstance(rta_tag, str) or not rta_tag:
+            raise ValueError('rta_tag must be a non-empty string')
+        if not isinstance(table, str) or not table:
+            raise ValueError('table must be a non-empty string')
+
         logging.warning(f'OpNotes {bus_ip} {bus_port} {db} {rta_tag}')
         self.connection = sqlite3.connect(db)
         self.table = table
         self.cursor = self.connection.cursor()
+        self._init_table()
+        self.busclient = BusClient(bus_ip, bus_port, module='OpNotes')
+        self.rta = Tag(rta_tag, dict)
+        self.rta.value = {}
+        self.busclient.add_callback_rta(rta_tag, self.rta_cb)
+
+    def _init_table(self):
+        """Initialize or upgrade the database table schema."""
         query = (
             'CREATE TABLE IF NOT EXISTS ' + self.table +
             '(id INTEGER PRIMARY KEY ASC, '
             'date_ms INTEGER, '
             'site TEXT, '
             'by TEXT, '
-            'note TEXT)'
+            'note TEXT, '
+            'abnormal INTEGER)'
         )
         self.cursor.execute(query)
-        self.busclient = BusClient(bus_ip, bus_port, module='OpNotes')
-        self.rta = Tag(rta_tag, dict)
-        self.rta.value = {}
-        self.busclient.add_callback_rta(rta_tag, self.rta_cb)
+        self.cursor.execute(f"PRAGMA table_info({self.table})")
+        columns = {col[1]: col[2] for col in self.cursor.fetchall()}
+        if 'abnormal' not in columns:
+            # Add abnormal as INTEGER from original schema
+            logging.warning(f'Upgrading {self.table} schema to include '
+                          'abnormal INTEGER, CANNOT revert automatically!')
+            self.cursor.execute(
+                f'ALTER TABLE {self.table} ADD COLUMN abnormal INTEGER')
+        elif columns['abnormal'].upper() == 'BOOLEAN':
+            # Change abnormal from BOOLEAN to INTEGER
+            logging.warning(f'Upgrading {self.table} abnormal from BOOLEAN to '
+                          'INTEGER, CANNOT revert automatically!')
+            self.cursor.execute(
+                f'CREATE TABLE {self.table}_new '
+                '(id INTEGER PRIMARY KEY ASC, '
+                'date_ms INTEGER, '
+                'site TEXT, '
+                'by TEXT, '
+                'note TEXT, '
+                'abnormal INTEGER)'
+            )
+            self.cursor.execute(
+                f'INSERT INTO {self.table}_new '
+                f'SELECT id, date_ms, site, by, note, '
+                f'CASE WHEN abnormal THEN 1 ELSE 0 END '
+                f'FROM {self.table}'
+            )
+            self.cursor.execute(f'DROP TABLE {self.table}')
+            self.cursor.execute(
+                f'ALTER TABLE {self.table}_new RENAME TO {self.table}'
+            )
 
     def rta_cb(self, request):
         """Respond to Request to Author and publish on rta_tag as needed."""
@@ -48,8 +108,10 @@ class OpNotes:
                 logging.info(f'add {request}')
                 with self.connection:
                     self.cursor.execute(
-                        f'INSERT INTO {self.table} (date_ms, site, by, note) '
-                        'VALUES(:date_ms, :site, :by, :note) RETURNING *;',
+                        f'INSERT INTO {self.table} '
+                        '(date_ms, site, by, note, abnormal) '
+                        'VALUES(:date_ms, :site, :by, :note, :abnormal) '
+                        'RETURNING *;',
                         request)
                     res = self.cursor.fetchone()
                     self.rta.value = {
@@ -57,7 +119,8 @@ class OpNotes:
                         'date_ms': res[1],
                         'site': res[2],
                         'by': res[3],
-                        'note': res[4]
+                        'note': res[4],
+                        'abnormal': res[5]
                     }
             except sqlite3.IntegrityError as error:
                 logging.warning(f'OpNotes rta_cb {error}')
@@ -67,14 +130,15 @@ class OpNotes:
                 with self.connection:
                     self.cursor.execute(
                         f'REPLACE INTO {self.table} VALUES(:id, :date_ms, '
-                        ':site, :by, :note) RETURNING *;', request)
+                        ':site, :by, :note, :abnormal) RETURNING *;', request)
                     res = self.cursor.fetchone()
                     self.rta.value = {
                         'id': res[0],
                         'date_ms': res[1],
                         'site': res[2],
                         'by': res[3],
-                        'note': res[4]
+                        'note': res[4],
+                        'abnormal': res[5]
                     }
             except sqlite3.IntegrityError as error:
                 logging.warning(f'OpNotes rta_cb {error}')
@@ -101,7 +165,8 @@ class OpNotes:
                             'date_ms': res[1],
                             'site': res[2],
                             'by': res[3],
-                            'note': res[4]
+                            'note': res[4],
+                            'abnormal': res[5]
                         }
             except sqlite3.IntegrityError as error:
                 logging.warning(f'OpNotes rta_cb {error}')

pymscada/protocol_constants.py

@@ -1,44 +1,62 @@
 """
 Protocol description and protocol constants.
 
-Bus holds a tag forever, assigns a tag id forever, holds a tag value with len
-< 1000 forever, otherwise wipes periodically. So assign tagnames once, at the
-start of your program; use RTA as an update messenger, share whole structures
-rarely.
+Bus holds a tag forever, assigning a new ID for each new tagname.
 
-- version 16-bit unsigned int == 0x01
-- command 16-bit unsigned int
-- tag_id 32-bit unsigned int     0 is not a valid tag_id
-- size 32-bit unsigned int
-  - if size == 0xff  continuation mandatory
-  - size 0x00 completes an empty continuation
-- time_us 128-bit unsigned int, UTC microseconds
-- data size of 8-bit char
+Bus protocol message format:
+- version: 8-bit unsigned int == 0x01
+- command: 8-bit unsigned int
+- tag_id: 16-bit unsigned int (0 is valid for ID requests)
+- size: 16-bit unsigned int
+- time_us: 64-bit unsigned int, UTC microseconds
+- data: variable length bytes of size
 
-command
-- CMD.ID data is tagname
-  - reply: CMD_ID with tag_id and data as tagname
-- CMD.SET id, data is typed or json packed
-  - no reply
-- CMD.UNSUB id
-  - no reply
-- CMD.GET id
-- CMD.RTA id, data is request to author
-- CMD.SUB id
-  - reply: SET id and value, value may be None
-- CMD.LIST
-  - size == 0x00
-    - tags with values newer than time_us
-  - size > 0x00
-    - ^text matches start of tagname
-    - text$ matches start of tagname
-    - text matches anywhere in tagname
-  - reply: LIST data as space separated tagnames
-- CMD.LOG data to logging.warning
+Commands:
+- CMD.ID: Query/inform tag ID
+  - Request: data is tagname as bytes
+  - Reply: CMD.ID with assigned tag_id and tagname as data
+  - Error: CMD.ERR if tagname undefined
+  
+- CMD.SET: Set a tag value
+  - Data format: TYPE byte + typed value
+  - No reply
+  - Error: CMD.ERR if tag_id invalid
+  
+- CMD.GET: Get a tag value
+  - Request: empty data
+  - Reply: CMD.SET with current value
+  - Error: CMD.ERR if tag_id invalid
+  
+- CMD.RTA: Request to author
+  - Request: JSON encoded request
+  - Reply: Comes from target client, not server
+  - Error: CMD.ERR if tag_id invalid or target gone
+  
+- CMD.SUB: Subscribe to tag updates
+  - Request: empty data
+  - Reply: CMD.SET with current value
+  - Error: CMD.ERR if tag_id invalid
+  
+- CMD.UNSUB: Unsubscribe from tag
+  - No reply
+  - Error: CMD.ERR if tag_id invalid
+  
+- CMD.LIST: List tags
+  - Empty data: tags newer than time_us
+  - ^text: tags starting with text
+  - text$: tags ending with text 
+  - text: tags containing text
+  - Reply: CMD.LIST with space-separated tagnames
+  
+- CMD.LOG: Log message
+  - Data: Message to log (max 300 bytes)
+  - Updates __bus__ tag with client address and message
+
+Large messages are split into MAX_LEN chunks. Final chunk size < MAX_LEN.
 """
 
 # Tuning constants
-MAX_LEN = 65535 - 14  # TODO fix server(?) when 3
+MAX_LEN = 65535 - 14  # Maximum data size per message
 
 from enum import IntEnum
 

pymscada/tag.py

@@ -18,28 +18,6 @@ TYPES = {
 }
 
 
-def tag_for_web(tagname: str, tag: dict):
-    """Correct tag dictionary in place to be suitable for web client."""
-    tag['name'] = tagname
-    tag['id'] = None
-    if 'desc' not in tag:
-        tag['desc'] = tag.name
-    if 'multi' in tag:
-        tag['type'] = 'int'
-    else:
-        if 'type' not in tag:
-            tag['type'] = 'float'
-        else:
-            if tag['type'] not in TYPES:
-                tag['type'] = 'str'
-    if tag['type'] == 'int':
-        tag['dp'] = 0
-    elif tag['type'] == 'float' and 'dp' not in tag:
-        tag['dp'] = 2
-    elif tag['type'] == 'str' and 'dp' in tag:
-        del tag['dp']
-
-
 class UniqueTag(type):
     """Super Tag class only create unique tags for unique tag names."""