pymscada 0.1.11b10__py3-none-any.whl → 0.2.0rc2__py3-none-any.whl

pymscada/bus_client.py

@@ -12,8 +12,9 @@ class BusClient:
     """
     Connects to a Bus Server.
 
-    await client.connect() to make the connection. If bus server connection
-    fails, die.
+    The client is created without a connection. client.start() creates the
+    connection and checks the tags at that time. When the connection fails
+    the client dies. A connection is mandatory for the client to run.
     """
 
     def __init__(self, ip: str = '127.0.0.1', port: int = 1324, tag_info=None,
@@ -125,7 +126,6 @@ class BusClient:
 
     async def read(self):
         """Read forever."""
-        await self.open_connection()
         while True:
             try:
                 head = await self.reader.readexactly(14)
@@ -223,4 +223,5 @@ class BusClient:
 
     async def start(self):
         """Start async."""
+        await self.open_connection()
         self.read_task = asyncio.create_task(self.read())

pymscada/bus_server.py

@@ -3,6 +3,7 @@ import asyncio
 from struct import pack, unpack
 import time
 import logging
+import socket
 import pymscada.protocol_constants as pc
 
 
@@ -143,8 +144,12 @@ class BusServer:
 
     __slots__ = ('ip', 'port', 'server', 'connections', 'bus_tag')
 
-    def __init__(self, ip: str = '127.0.0.1', port: int = 1324,
-                 bus_tag: str = '__bus__'):
+    def __init__(
+        self,
+        ip: str = '127.0.0.1',
+        port: int = 1324,
+        bus_tag: str = '__bus__'
+    ) -> None:
         """
         Serve Tags on ip:port, echoing changes to any subscribers.
 
@@ -154,6 +159,13 @@ class BusServer:
 
         Event loop must be running.
         """
+        try:
+            socket.gethostbyname(ip)
+        except socket.gaierror as e:
+            raise ValueError(f'Cannot resolve IP/hostname: {e}')
+        if not isinstance(bus_tag, str):
+            raise ValueError('bus_tag must be a string')
+
         self.ip = ip
         self.port = port
         self.server = None

pymscada/checkout.py

@@ -6,100 +6,98 @@ import sys
 from pymscada.config import get_demo_files, get_pdf_files
 
 
-PATH = {
-    '__PYTHON__': Path(f'{sys.exec_prefix}/bin/python').absolute(),
-    '__PYMSCADA__': Path(sys.argv[0]).absolute(),
-    '__DIR__': Path('.').absolute(),
-    '__HOME__': Path.home().absolute(),
-    '__USER__': getpass.getuser()
-}
-if sys.platform == "win32":
-    PATH = {
-        '__PYTHON__': Path(f'{sys.exec_prefix}/python.exe').absolute(),
-        '__PYMSCADA__': Path(sys.argv[0]).absolute(),
-        '__DIR__': Path('.').absolute(),
-        '__HOME__': Path.home(),
-        '__USER__': getpass.getuser()
-    }
+class Checkout:
+    """Create and manage configuration files."""
+    
+    def __init__(self, **kwargs):
+        """Initialize paths and settings."""
+        self.path = {
+            '__PYTHON__': Path(f'{sys.exec_prefix}/bin/python').absolute(),
+            '__PYMSCADA__': Path(sys.argv[0]).absolute(),
+            '__DIR__': Path('.').absolute(),
+            '__HOME__': Path.home().absolute(),
+            '__USER__': getpass.getuser()
+        }
+        if sys.platform == "win32":
+            self.path['__PYTHON__'] = Path(f'{sys.exec_prefix}/python.exe').absolute()
+        
+        self.overwrite = kwargs.get('overwrite', False)
+        self.diff = kwargs.get('diff', False)
 
-def make_history():
-    """Make the history folder if missing."""
-    history_dir = PATH['__DIR__'].joinpath('history')
-    if not history_dir.exists():
-        print("making 'history' folder")
-        history_dir.mkdir()
+    def make_history(self):
+        """Make the history folder if missing."""
+        history_dir = self.path['__DIR__'].joinpath('history')
+        if not history_dir.exists():
+            print("making 'history' folder")
+            history_dir.mkdir()
 
+    def make_pdf(self):
+        """Make the pdf folder if missing."""
+        pdf_dir = self.path['__DIR__'].joinpath('pdf')
+        if not pdf_dir.exists():
+            print('making pdf dir')
+            pdf_dir.mkdir()
+        for pdf_file in get_pdf_files():
+            target = pdf_dir.joinpath(pdf_file.name)
+            target.write_bytes(pdf_file.read_bytes())
 
-def make_pdf():
-    """Make the pdf folder if missing."""
-    pdf_dir = PATH['__DIR__'].joinpath('pdf')
-    if not pdf_dir.exists():
-        print('making pdf dir')
-        pdf_dir.mkdir()
-    for pdf_file in get_pdf_files():
-        target = pdf_dir.joinpath(pdf_file.name)
-        target.write_bytes(pdf_file.read_bytes())
+    def make_config(self):
+        """Make the config folder, if missing, and copy files in."""
+        config_dir = self.path['__DIR__'].joinpath('config')
+        if not config_dir.exists():
+            print('making config dir')
+            config_dir.mkdir()
+        for config_file in get_demo_files():
+            target = config_dir.joinpath(config_file.name)
+            rt = 'Creating '
+            if target.exists():
+                if self.overwrite:
+                    rt = 'Replacing '
+                    target.unlink()
+                else:
+                    continue
+            print(f'{rt} {target}')
+            rd_bytes = config_file.read_bytes()
+            if target.name.lower() != 'readme.md':
+                for k, v in self.path.items():
+                    rd_bytes = rd_bytes.replace(k.encode(), str(v).encode())
+            target.write_bytes(rd_bytes)
 
+    def read_with_subst(self, file: Path):
+        """Read the file and replace DIR markers."""
+        rd = file.read_bytes().decode()
+        for k, v in self.path.items():
+            rd = rd.replace(k, str(v))
+        lines = rd.splitlines()
+        return lines
 
-def make_config(overwrite: bool):
-    """Make the config folder, if missing, and copy files in."""
-    config_dir = PATH['__DIR__'].joinpath('config')
-    if not config_dir.exists():
-        print('making config dir')
-        config_dir.mkdir()
-    for config_file in get_demo_files():
-        target = config_dir.joinpath(config_file.name)
-        rt = 'Creating '
-        if target.exists():
-            if overwrite:
-                rt = 'Replacing '
-                target.unlink()
+    def compare_config(self):
+        """Compare old and new config."""
+        config_dir = self.path['__DIR__'].joinpath('config')
+        if not config_dir.exists():
+            print('No config dir, are you in the right directory')
+            return
+        for config_file in get_demo_files():
+            target = config_dir.joinpath(config_file.name)
+            if target.exists():
+                new_lines = self.read_with_subst(config_file)
+                old_lines = self.read_with_subst(target)
+                diff = list(difflib.unified_diff(old_lines, new_lines,
+                            fromfile=str(target), tofile=str(config_file)))
+                if len(diff):
+                    print('\n'.join(diff), '\n')
             else:
-                continue
-        print(f'{rt} {target}')
-        rd_bytes = config_file.read_bytes()
-        if target.name.lower() != 'readme.md':
-            for k, v in PATH.items():
-                rd_bytes = rd_bytes.replace(k.encode(), str(v).encode())
-        target.write_bytes(rd_bytes)
+                print(f'\n--- MISSING FILE\n\n+++ {config_file}')
 
-
-def read_with_subst(file: Path):
-    """Read the file and replace DIR markers."""
-    rd = file.read_bytes().decode()
-    for k, v in PATH.items():
-        rd = rd.replace(k, str(v))
-    lines = rd.splitlines()
-    return lines
-
-
-def compare_config():
-    """Compare old and new config."""
-    config_dir = PATH['__DIR__'].joinpath('config')
-    if not config_dir.exists():
-        print('No config dir, are you in the right directory')
-        return
-    for config_file in get_demo_files():
-        target = config_dir.joinpath(config_file.name)
-        if target.exists():
-            new_lines = read_with_subst(config_file)
-            old_lines = read_with_subst(target)
-            diff = list(difflib.unified_diff(old_lines, new_lines,
-                        fromfile=str(target), tofile=str(config_file)))
-            if len(diff):
-                print('\n'.join(diff), '\n')
+    async def start(self):
+        """Execute checkout process."""
+        for name in ['__PYTHON__', '__PYMSCADA__', '__DIR__', '__HOME__']:
+            if not self.path[name].exists():
+                raise SystemExit(f'{self.path[name]} is missing')
+        
+        if self.diff:
+            self.compare_config()
         else:
-            print(f'\n--- MISSING FILE\n\n+++ {config_file}')
-
-
-def checkout(overwrite=False, diff=False):
-    """Do it."""
-    for name in ['__PYTHON__', '__PYMSCADA__', '__DIR__', '__HOME__']:
-        if not PATH[name].exists():
-            raise SystemExit(f'{PATH[name]} is missing')
-    if diff:
-        compare_config()
-    else:
-        make_history()
-        make_pdf()
-        make_config(overwrite)
+            self.make_history()
+            self.make_pdf()
+            self.make_config()

pymscada/console.py

@@ -3,7 +3,7 @@ import asyncio
 import logging
 import sys
 from pymscada.bus_client import BusClient
-from pymscada.tag import Tag, tag_for_web
+from pymscada.tag import Tag, tag_for_web, TagInfo
 try:
     import termios
     import tty
@@ -153,7 +153,7 @@ class Console:
     """Provide a text console to interact with a Bus."""
 
     def __init__(self, bus_ip: str = '127.0.0.1', bus_port: int = 1324,
-                 tag_info: dict = {}):
+                 tag_info: dict[str, TagInfo] = {}) -> None:
         """
         Connect to bus_ip:bus_port and provide console interaction with a Bus.
 

pymscada/demo/openweather.yaml

@@ -2,7 +2,7 @@ bus_ip: 127.0.0.1
 bus_port: 1324
 proxy:
 api:
-  api_key: ${OPENWEATHER_API_KEY}
+  api_key: ${OPENWEATHERMAP_API_KEY}
   units: metric
   locations:
     Murupara:

pymscada/demo/{pymscada-io-accuweather.service → pymscada-io-openweather.service}

@@ -1,11 +1,11 @@
 [Unit]
-    Description=pymscada - AccuWeather client
+    Description=pymscada - Open Weather client
     BindsTo=pymscada-bus.service
     After=pymscada-bus.service
 
 [Service]
     WorkingDirectory=__DIR__
-    ExecStart=__PYMSCADA__ accuweatherclient --config __DIR__/config/accuweather.yaml
+    ExecStart=__PYMSCADA__ openweatherclient --config __DIR__/config/openweather.yaml
     Restart=always
     RestartSec=5
     User=__USER__

pymscada/files.py

@@ -10,13 +10,13 @@ class Files():
     """Connect to bus_ip:bus_port, store and provide a value history."""
 
     def __init__(self, bus_ip: str = '127.0.0.1', bus_port: int = 1324,
-                 path: str = '', files: list = None,
+                 path: str = '', files: list[dict] = [],
                  rta_tag: str = '__files__') -> None:
         """
         Connect to bus_ip:bus_port, serve and update files.
 
-        TODO
-        Write something.
+        Scans paths in files at the root defined by path. These should
+        be readable by wwwserver so that download links work.
 
         Event loop must be running.
         """

pymscada/history.py

@@ -1,11 +1,34 @@
-"""Store and provide history."""
+"""Store and provide history.
+
+History File Structure
+---------------------
+History files are binary files stored as <tagname>_<time_us>.dat where time_us
+is the microsecond timestamp of the first entry in that file.
+
+Each file contains a series of fixed-size records (16 bytes each):
+- For integer tags: 8 bytes timestamp (uint64) + 8 bytes value (int64)
+- For float tags: 8 bytes timestamp (uint64) + 8 bytes value (double)
+
+Files are organized in chunks:
+- Each chunk is 1024 records (16KB)
+- Each file contains up to 64 chunks (1MB)
+- New files are created when:
+  1. Current file reaches max size (64 chunks)
+  2. Manual flush() is called
+  3. Application shutdown
+
+Timestamps are stored as microseconds since epoch in network byte order (big-endian).
+Values are also stored in network byte order.
+"""
 import atexit
 import logging
 from pathlib import Path
 from struct import pack, pack_into, unpack_from, error
 import time
+import socket
+from typing import TypedDict, Optional
 from pymscada.bus_client import BusClient
-from pymscada.tag import Tag, TYPES
+from pymscada.tag import Tag, TagInfo, TYPES
 
 
 ITEM_SIZE = 16  # Q + q, Q or d
@@ -14,6 +37,16 @@ CHUNK_SIZE = ITEM_COUNT * ITEM_SIZE
 FILE_CHUNKS = 64
 
 
+class Request(TypedDict, total=False):
+    """Type definition for request dictionary."""
+    tagname: str 
+    start_ms: Optional[int]  # Allow web client to use native ms
+    start_us: Optional[int]  # Native for pymscada server
+    end_ms: Optional[int]
+    end_us: Optional[int]
+    __rta_id__: Optional[int]  # Empty for a change that must be broadcast
+
+
 def tag_for_history(tagname: str, tag: dict):
     """Correct tag dictionary in place to be suitable for web client."""
     tag['name'] = tagname
@@ -188,9 +221,14 @@ class TagHistory():
 class History():
     """Connect to bus_ip:bus_port, store and provide a value history."""
 
-    def __init__(self, bus_ip: str = '127.0.0.1', bus_port: int = 1324,
-                 path: str = 'history', tag_info: dict = {},
-                 rta_tag: str = '__history__') -> None:
+    def __init__(
+        self,
+        bus_ip: str = '127.0.0.1',
+        bus_port: int = 1324,
+        path: str = 'history',
+        tag_info: TagInfo = {},
+        rta_tag: str | None = '__history__'
+    ) -> None:
         """
         Connect to bus_ip:bus_port, store and provide a value history.
 
@@ -200,6 +238,23 @@ class History():
 
         Event loop must be running.
         """
+        if not isinstance(bus_ip, str):
+            raise ValueError("bus_ip must be a string")
+        try:
+            socket.gethostbyname(bus_ip)
+        except socket.gaierror:
+            raise ValueError(f"Invalid bus_ip: {bus_ip}")
+        if not isinstance(bus_port, int):
+            raise ValueError("bus_port must be an integer")
+        if not 1024 <= bus_port <= 65535:
+            raise ValueError(f"bus_port must be between 1024 and 65535")
+        if not isinstance(path, str):
+            raise ValueError("path must be a string")
+        if not isinstance(tag_info, dict):
+            raise ValueError("tag_info must be a dictionary")
+        if not isinstance(rta_tag, (str, type(None))):
+            raise ValueError("rta_tag must be a string or None")
+
         self.busclient = BusClient(bus_ip, bus_port, module='History')
         self.path = path
         self.tags: dict[str, Tag] = {}
@@ -217,7 +272,7 @@ class History():
         self.rta.value = b'\x00\x00\x00\x00\x00\x00'
         self.busclient.add_callback_rta(rta_tag, self.rta_cb)
 
-    def rta_cb(self, request):
+    def rta_cb(self, request: Request):
         """Respond to bus requests for data to publish on rta."""
         if 'start_ms' in request:
             request['start_us'] = request['start_ms'] * 1000

pymscada/iodrivers/openweather.py

@@ -2,6 +2,7 @@
 import asyncio
 import aiohttp
 import logging
+import socket
 from time import time
 from pymscada.bus_client import BusClient
 from pymscada.periodic import Periodic
@@ -10,8 +11,14 @@ from pymscada.tag import Tag
 class OpenWeatherClient:
     """Get weather data from OpenWeather Current and Forecast APIs."""
 
-    def __init__(self, bus_ip: str = '127.0.0.1', bus_port: int = 1324,
-                 proxy: str = None, api: dict = {}, tags: dict = {}) -> None:
+    def __init__(
+        self,
+        bus_ip: str | None = '127.0.0.1',
+        bus_port: int = 1324,
+        proxy: str | None = None,
+        api: dict = {},
+        tags: list = []
+    ) -> None:
         """
         Connect to bus on bus_ip:bus_port.
 
@@ -21,6 +28,18 @@ class OpenWeatherClient:
         - times: list of hours ahead to fetch forecast data for
         - units: optional units (standard, metric, imperial)
         """
+        if bus_ip is not None:
+            try:
+                socket.gethostbyname(bus_ip)
+            except socket.gaierror:
+                raise ValueError(f"Invalid bus_ip: {bus_ip}")
+        if not isinstance(proxy, str) and proxy is not None:
+            raise ValueError("proxy must be a string or None")
+        if not isinstance(api, dict):
+            raise ValueError("api must be a dictionary")
+        if not isinstance(tags, list):
+            raise ValueError("tags must be a list")
+
         self.busclient = None
         if bus_ip is not None:
             self.busclient = BusClient(bus_ip, bus_port, module='OpenWeather')
@@ -41,82 +60,144 @@ class OpenWeatherClient:
 
     def update_tags(self, location, data, suffix):
         """Update tags for forecast weather."""
+        if 'dt' not in data:
+            logging.error(f'No timestamp in data for {location}, skipping update')
+            return
         for parameter in self.parameters:
             tagname = f"{location}_{parameter}{suffix}"
-            if parameter == 'Temp':
-                value = data['main']['temp']
-            elif parameter == 'WindSpeed':
-                value = data['wind']['speed']
-            elif parameter == 'WindDir':
-                value = data['wind'].get('deg', 0)
-            elif parameter == 'Rain':
-                value = data.get('rain', {}).get('1h', 0)
             try:
-                self.tags[tagname].value = value
-            except KeyError:
-                logging.warning(f'{tagname} not found setting weather value')
+                if parameter == 'Temp':
+                    main_data = data.get('main', {})
+                    value = main_data.get('temp', 0)
+                elif parameter == 'WindSpeed':
+                    wind_data = data.get('wind', {})
+                    value = wind_data.get('speed', 0)
+                elif parameter == 'WindDir':
+                    wind_data = data.get('wind', {})
+                    value = wind_data.get('deg', 0)
+                elif parameter == 'Rain':
+                    rain_data = data.get('rain', {})
+                    value = rain_data.get('1h', 0)
+                else:
+                    logging.warning(f'Unknown parameter {parameter} for {tagname}')
+                    continue
+                time_us = int(data['dt'] * 1_000_000)
+                self.tags[tagname].value = value, time_us, self.map_bus
+                logging.debug(f'Updated {tagname} = {value} at timestamp {data["dt"]}')
+            except Exception as e:
+                logging.error(
+                    f'Error updating {tagname}: {type(e).__name__} - {str(e)}'
+                )
 
     async def handle_response(self):
         """Handle responses from the API."""
         while True:
-            location, data = await self.queue.get()
-            now = int(time())
-            if 'dt' in data:
-                self.update_tags(location, data, '')
-            elif 'list' in data:
-                for forecast in data['list']:
-                    hours_ahead = int((forecast['dt'] - now) / 3600)
-                    if hours_ahead not in self.times:
-                        continue
-                    suffix = f'_{hours_ahead:02d}'
-                    self.update_tags(location, forecast, suffix)
+            try:
+                location, data = await self.queue.get()
+                logging.debug(f'Processing data for {location}')
+                
+                if 'dt' in data:  # Current weather data
+                    self.update_tags(location, data, '')
+                elif 'list' in data:  # Forecast data
+                    now = int(time())
+                    for forecast in data['list']:
+                        hours_ahead = int((forecast['dt'] - now) / 3600)
+                        if hours_ahead in self.times:
+                            suffix = f'_{hours_ahead:02d}'
+                            self.update_tags(location, forecast, suffix)
+                
+                self.queue.task_done()
+            except Exception as e:
+                logging.error(f'Error handling response: {type(e).__name__} - {str(e)}')
 
     async def fetch_current_data(self):
         """Fetch current weather data for all locations."""
-        logging.info('fetching current')
-        if self.session is None:
-            self.session = aiohttp.ClientSession()
-        for location, coords in self.locations.items():
-            base_params = {'lat': coords['lat'], 'lon': coords['lon'],
-                'appid': self.api_key, 'units': self.units }
-            try:
-                async with self.session.get(self.current_url,
-                        params=base_params, proxy=self.proxy) as resp:
-                    if resp.status == 200:
-                        self.queue.put_nowait((location, await resp.json()))
-                    else:
-                        logging.warning('OpenWeather current API error for '
-                            f'{location}: {resp.status}')
-            except Exception as e:
-                logging.warning('OpenWeather current API error for '
-                    f'{location}: {e}')
+        try:
+            if self.session is None:
+                self.session = aiohttp.ClientSession()
+            
+            for location, coords in self.locations.items():
+                base_params = {
+                    'lat': coords.get('lat'),
+                    'lon': coords.get('lon'),
+                    'appid': self.api_key,
+                    'units': self.units
+                }
+                
+                # Validate required parameters
+                if not all(base_params.values()):
+                    logging.error(
+                        f'Missing required parameters for {location}: '
+                        f'{[k for k, v in base_params.items() if not v]}'
+                    )
+                    continue
+                    
+                try:
+                    async with self.session.get(
+                        self.current_url,
+                        params=base_params,
+                        proxy=self.proxy,
+                        timeout=30  # Add timeout
+                    ) as resp:
+                        if resp.status == 200:
+                            data = await resp.json()
+                            logging.debug(
+                                f'Received current weather data for {location}'
+                            )
+                            await self.queue.put((location, data))
+                        else:
+                            error_text = await resp.text()
+                            logging.error(
+                                f'OpenWeather API error for {location}: '
+                                f'Status: {resp.status}, Response: {error_text[:200]}'
+                            )
+                            
+                except asyncio.TimeoutError:
+                    logging.error(f'Timeout fetching data for {location}')
+                except aiohttp.ClientError as e:
+                    logging.error(
+                        f'Network error for {location}: {type(e).__name__} - {str(e)}'
+                    )
+                except Exception as e:
+                    logging.error(
+                        f'Unexpected error for {location}: {type(e).__name__} - {str(e)}'
+                    )
+                    
+        except Exception as e:
+            logging.error(f'Fatal error in fetch_current_data: {type(e).__name__} - {str(e)}')
 
     async def fetch_forecast_data(self):
         """Fetch forecast weather data for all locations."""
-        logging.info('fetching forecast')
         if self.session is None:
             self.session = aiohttp.ClientSession()
         for location, coords in self.locations.items():
-            base_params = {'lat': coords['lat'], 'lon': coords['lon'],
-                'appid': self.api_key, 'units': self.units }
+            base_params = {
+                'lat': coords['lat'], 
+                'lon': coords['lon'],
+                'appid': self.api_key, 
+                'units': self.units 
+            }
             try:
                 async with self.session.get(self.forecast_url,
                         params=base_params, proxy=self.proxy) as resp:
                     if resp.status == 200:
-                        self.queue.put_nowait((location, await resp.json()))
+                        data = await resp.json()
+                        logging.info(f'Queue forecast {location} {data}')
+                        await self.queue.put((location, data))
                     else:
-                        logging.warning('OpenWeather forecast API error '
-                            f'for {location}: {resp.status}')
+                        logging.error(f'OpenWeather forecast API error for '
+                            f'{location}: Status:{resp.status}, '
+                            f'Response:{await resp.text()}')
             except Exception as e:
-                logging.warning('OpenWeather forecast API error for '
-                    f'{location}: {e}')
+                logging.error(f'OpenWeather forecast API error for {location}: '
+                    f'Exception:{type(e).__name__}, Message:{str(e)}')
 
     async def poll(self):
         """Poll OpenWeather APIs every 10 minutes."""
         now = int(time())
         if now % 600 == 0:  # Every 10 minutes
             asyncio.create_task(self.fetch_current_data())
-        if now % 10800 == 60:  # Every 3 hours, offset by 1 minute
+        if now % 3600 == 60:  # Every 3 hours, offset by 1 minute
             asyncio.create_task(self.fetch_forecast_data())
 
     async def start(self):