pymscada 0.1.11b9__tar.gz → 0.2.0rc1__tar.gz

pymscada-0.2.0rc1/MANIFEST.in

@@ -0,0 +1,2 @@
+graft src/pymscada/demo
+graft src/pymscada/pdf

{pymscada-0.1.11b9 → pymscada-0.2.0rc1}/PKG-INFO

@@ -1,25 +1,26 @@
 Metadata-Version: 2.1
 Name: pymscada
-Version: 0.1.11b9
+Version: 0.2.0rc1
 Summary: Shared tag value SCADA with python backup and Angular UI
-Author-Email: Jamie Walton <jamie@walton.net.nz>
+Author-email: Jamie Walton <jamie@walton.net.nz>
 License: GPL-3.0-or-later
+Project-URL: Homepage, https://github.com/jamie0walton/pymscada
+Project-URL: Bug Tracker, https://github.com/jamie0walton/pymscada/issues
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: JavaScript
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Environment :: Console
 Classifier: Development Status :: 1 - Planning
-Project-URL: Homepage, https://github.com/jamie0walton/pymscada
-Project-URL: Bug Tracker, https://github.com/jamie0walton/pymscada/issues
 Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: PyYAML>=6.0.1
 Requires-Dist: aiohttp>=3.8.5
-Requires-Dist: pymscada-html>=0.1.10b5
+Requires-Dist: pymscada-html==0.2.0rc1
 Requires-Dist: cerberus>=1.3.5
 Requires-Dist: pycomm3>=1.2.14
 Requires-Dist: pysnmplib>=5.0.24
-Description-Content-Type: text/markdown
 
 # pymscada
 #### [Docs](https://github.com/jamie0walton/pymscada/blob/main/docs/README.md)

{pymscada-0.1.11b9 → pymscada-0.2.0rc1}/pyproject.toml

@@ -1,20 +1,21 @@
 [project]
 name = "pymscada"
-version = "0.1.11b9"
+version = "0.2.0rc1"
 description = "Shared tag value SCADA with python backup and Angular UI"
 authors = [
-    { name = "Jamie Walton", email = "jamie@walton.net.nz" },
+    {name = "Jamie Walton", email = "jamie@walton.net.nz"},
 ]
 dependencies = [
-    "PyYAML>=6.0.1",
-    "aiohttp>=3.8.5",
-    "pymscada-html>=0.1.10b5",
-    "cerberus>=1.3.5",
-    "pycomm3>=1.2.14",
-    "pysnmplib>=5.0.24",
+    "PyYAML>=6.0.1",          # all
+    "aiohttp>=3.8.5",         # www_server
+    "pymscada-html==0.2.0rc1",   # www_server
+    "cerberus>=1.3.5",        # validate
+    "pycomm3>=1.2.14",        # logix_client
+    "pysnmplib>=5.0.24",      # DON'T use pysnmp, dead
 ]
 requires-python = ">=3.9"
 readme = "README.md"
+license = {text = "GPL-3.0-or-later"}
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: JavaScript",
@@ -24,21 +25,9 @@ classifiers = [
     "Development Status :: 1 - Planning",
 ]
 
-[project.license]
-text = "GPL-3.0-or-later"
-
-[project.scripts]
-pymscada = "pymscada.__main__:cmd_line"
-
-[project.urls]
-Homepage = "https://github.com/jamie0walton/pymscada"
-"Bug Tracker" = "https://github.com/jamie0walton/pymscada/issues"
-
 [build-system]
-requires = [
-    "pdm-backend",
-]
-build-backend = "pdm.backend"
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
 
 [tool.pdm.dev-dependencies]
 test = [
@@ -46,14 +35,21 @@ test = [
     "flake8>=6.1.0",
     "flake8-docstrings>=1.7.0",
     "pytest-asyncio>=0.21.1",
-    "pytest-cov>=4.1.0",
+    "pytest-cov>=4.1.0"
 ]
 pdm = []
 
 [tool.coverage.run]
 omit = [
-    "tests/*",
+    'tests/*',
 ]
 
+[project.scripts]
+pymscada = "pymscada.__main__:cmd_line"
+
+[project.urls]
+"Homepage" = "https://github.com/jamie0walton/pymscada"
+"Bug Tracker" = "https://github.com/jamie0walton/pymscada/issues"
+
 [tool.pytest.ini_options]
 addopts = "-v -s"

pymscada-0.2.0rc1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pymscada-0.2.0rc1/src/pymscada/checkout.py

@@ -0,0 +1,103 @@
+"""Create base config folder and check out demo files."""
+import difflib
+import getpass
+from pathlib import Path
+import sys
+from pymscada.config import get_demo_files, get_pdf_files
+
+
+class Checkout:
+    """Create and manage configuration files."""
+    
+    def __init__(self, **kwargs):
+        """Initialize paths and settings."""
+        self.path = {
+            '__PYTHON__': Path(f'{sys.exec_prefix}/bin/python').absolute(),
+            '__PYMSCADA__': Path(sys.argv[0]).absolute(),
+            '__DIR__': Path('.').absolute(),
+            '__HOME__': Path.home().absolute(),
+            '__USER__': getpass.getuser()
+        }
+        if sys.platform == "win32":
+            self.path['__PYTHON__'] = Path(f'{sys.exec_prefix}/python.exe').absolute()
+        
+        self.overwrite = kwargs.get('overwrite', False)
+        self.diff = kwargs.get('diff', False)
+
+    def make_history(self):
+        """Make the history folder if missing."""
+        history_dir = self.path['__DIR__'].joinpath('history')
+        if not history_dir.exists():
+            print("making 'history' folder")
+            history_dir.mkdir()
+
+    def make_pdf(self):
+        """Make the pdf folder if missing."""
+        pdf_dir = self.path['__DIR__'].joinpath('pdf')
+        if not pdf_dir.exists():
+            print('making pdf dir')
+            pdf_dir.mkdir()
+        for pdf_file in get_pdf_files():
+            target = pdf_dir.joinpath(pdf_file.name)
+            target.write_bytes(pdf_file.read_bytes())
+
+    def make_config(self):
+        """Make the config folder, if missing, and copy files in."""
+        config_dir = self.path['__DIR__'].joinpath('config')
+        if not config_dir.exists():
+            print('making config dir')
+            config_dir.mkdir()
+        for config_file in get_demo_files():
+            target = config_dir.joinpath(config_file.name)
+            rt = 'Creating '
+            if target.exists():
+                if self.overwrite:
+                    rt = 'Replacing '
+                    target.unlink()
+                else:
+                    continue
+            print(f'{rt} {target}')
+            rd_bytes = config_file.read_bytes()
+            if target.name.lower() != 'readme.md':
+                for k, v in self.path.items():
+                    rd_bytes = rd_bytes.replace(k.encode(), str(v).encode())
+            target.write_bytes(rd_bytes)
+
+    def read_with_subst(self, file: Path):
+        """Read the file and replace DIR markers."""
+        rd = file.read_bytes().decode()
+        for k, v in self.path.items():
+            rd = rd.replace(k, str(v))
+        lines = rd.splitlines()
+        return lines
+
+    def compare_config(self):
+        """Compare old and new config."""
+        config_dir = self.path['__DIR__'].joinpath('config')
+        if not config_dir.exists():
+            print('No config dir, are you in the right directory')
+            return
+        for config_file in get_demo_files():
+            target = config_dir.joinpath(config_file.name)
+            if target.exists():
+                new_lines = self.read_with_subst(config_file)
+                old_lines = self.read_with_subst(target)
+                diff = list(difflib.unified_diff(old_lines, new_lines,
+                            fromfile=str(target), tofile=str(config_file)))
+                if len(diff):
+                    print('\n'.join(diff), '\n')
+            else:
+                print(f'\n--- MISSING FILE\n\n+++ {config_file}')
+
+    async def start(self):
+        """Execute checkout process."""
+        for name in ['__PYTHON__', '__PYMSCADA__', '__DIR__', '__HOME__']:
+            if not self.path[name].exists():
+                raise SystemExit(f'{self.path[name]} is missing')
+        
+        if self.diff:
+            self.compare_config()
+        else:
+            self.make_history()
+            self.make_pdf()
+            self.make_config()

{pymscada-0.1.11b9 → pymscada-0.2.0rc1}/src/pymscada/demo/openweather.yaml

@@ -2,7 +2,7 @@ bus_ip: 127.0.0.1
 bus_port: 1324
 proxy:
 api:
-  api_key: ${OPENWEATHER_API_KEY}
+  api_key: ${OPENWEATHERMAP_API_KEY}
   units: metric
   locations:
     Murupara:

pymscada-0.1.11b9/src/pymscada/demo/pymscada-io-accuweather.service → pymscada-0.2.0rc1/src/pymscada/demo/pymscada-io-openweather.service

@@ -1,11 +1,11 @@
 [Unit]
-    Description=pymscada - AccuWeather client
+    Description=pymscada - Open Weather client
     BindsTo=pymscada-bus.service
     After=pymscada-bus.service
 
 [Service]
     WorkingDirectory=__DIR__
-    ExecStart=__PYMSCADA__ accuweatherclient --config __DIR__/config/accuweather.yaml
+    ExecStart=__PYMSCADA__ openweatherclient --config __DIR__/config/openweather.yaml
     Restart=always
     RestartSec=5
     User=__USER__

{pymscada-0.1.11b9 → pymscada-0.2.0rc1}/src/pymscada/history.py

@@ -1,11 +1,33 @@
-"""Store and provide history."""
+"""Store and provide history.
+
+History File Structure
+---------------------
+History files are binary files stored as <tagname>_<time_us>.dat where time_us
+is the microsecond timestamp of the first entry in that file.
+
+Each file contains a series of fixed-size records (16 bytes each):
+- For integer tags: 8 bytes timestamp (uint64) + 8 bytes value (int64)
+- For float tags: 8 bytes timestamp (uint64) + 8 bytes value (double)
+
+Files are organized in chunks:
+- Each chunk is 1024 records (16KB)
+- Each file contains up to 64 chunks (1MB)
+- New files are created when:
+  1. Current file reaches max size (64 chunks)
+  2. Manual flush() is called
+  3. Application shutdown
+
+Timestamps are stored as microseconds since epoch in network byte order (big-endian).
+Values are also stored in network byte order.
+"""
 import atexit
 import logging
 from pathlib import Path
 from struct import pack, pack_into, unpack_from, error
 import time
+from typing import TypedDict, Optional
 from pymscada.bus_client import BusClient
-from pymscada.tag import Tag, TYPES
+from pymscada.tag import Tag, TagInfo, TYPES
 
 
 ITEM_SIZE = 16  # Q + q, Q or d
@@ -14,6 +36,16 @@ CHUNK_SIZE = ITEM_COUNT * ITEM_SIZE
 FILE_CHUNKS = 64
 
 
+class Request(TypedDict, total=False):
+    """Type definition for request dictionary."""
+    tagname: str 
+    start_ms: Optional[int]  # Allow web client to use native ms
+    start_us: Optional[int]  # Native for pymscada server
+    end_ms: Optional[int]
+    end_us: Optional[int]
+    __rta_id__: Optional[int]  # Empty for a change that must be broadcast
+
+
 def tag_for_history(tagname: str, tag: dict):
     """Correct tag dictionary in place to be suitable for web client."""
     tag['name'] = tagname
@@ -189,7 +221,7 @@ class History():
     """Connect to bus_ip:bus_port, store and provide a value history."""
 
     def __init__(self, bus_ip: str = '127.0.0.1', bus_port: int = 1324,
-                 path: str = 'history', tag_info: dict = {},
+                 path: str = 'history', tag_info: TagInfo = {},
                  rta_tag: str = '__history__') -> None:
         """
         Connect to bus_ip:bus_port, store and provide a value history.
@@ -217,7 +249,7 @@ class History():
         self.rta.value = b'\x00\x00\x00\x00\x00\x00'
         self.busclient.add_callback_rta(rta_tag, self.rta_cb)
 
-    def rta_cb(self, request):
+    def rta_cb(self, request: Request):
         """Respond to bus requests for data to publish on rta."""
         if 'start_ms' in request:
             request['start_us'] = request['start_ms'] * 1000

pymscada-0.2.0rc1/src/pymscada/iodrivers/openweather.py

@@ -0,0 +1,190 @@
+"""Poll OpenWeather current and forecast APIs."""
+import asyncio
+import aiohttp
+import logging
+from time import time
+from pymscada.bus_client import BusClient
+from pymscada.periodic import Periodic
+from pymscada.tag import Tag
+
+class OpenWeatherClient:
+    """Get weather data from OpenWeather Current and Forecast APIs."""
+
+    def __init__(self, bus_ip: str = '127.0.0.1', bus_port: int = 1324,
+                 proxy: str = None, api: dict = {}, tags: dict = {}) -> None:
+        """
+        Connect to bus on bus_ip:bus_port.
+
+        api dict should contain:
+        - api_key: OpenWeatherMap API key
+        - locations: dict of location names and coordinates
+        - times: list of hours ahead to fetch forecast data for
+        - units: optional units (standard, metric, imperial)
+        """
+        self.busclient = None
+        if bus_ip is not None:
+            self.busclient = BusClient(bus_ip, bus_port, module='OpenWeather')
+        self.proxy = proxy
+        self.map_bus = id(self)
+        self.tags = {tagname: Tag(tagname, float) for tagname in tags}
+        self.api_key = api['api_key']
+        self.units = api.get('units', 'standard')
+        self.locations = api.get('locations', {})
+        self.parameters = api.get('parameters', {})
+        self.times = api.get('times', [3, 6, 12, 24, 48])
+        self.current_url = "https://api.openweathermap.org/data/2.5/weather"
+        self.forecast_url = "https://api.openweathermap.org/data/2.5/forecast"
+        self.queue = asyncio.Queue()
+        self.session = None
+        self.handle = None
+        self.periodic = None
+
+    def update_tags(self, location, data, suffix):
+        """Update tags for forecast weather."""
+        if 'dt' not in data:
+            logging.error(f'No timestamp in data for {location}, skipping update')
+            return
+        for parameter in self.parameters:
+            tagname = f"{location}_{parameter}{suffix}"
+            try:
+                if parameter == 'Temp':
+                    main_data = data.get('main', {})
+                    value = main_data.get('temp', 0)
+                elif parameter == 'WindSpeed':
+                    wind_data = data.get('wind', {})
+                    value = wind_data.get('speed', 0)
+                elif parameter == 'WindDir':
+                    wind_data = data.get('wind', {})
+                    value = wind_data.get('deg', 0)
+                elif parameter == 'Rain':
+                    rain_data = data.get('rain', {})
+                    value = rain_data.get('1h', 0)
+                else:
+                    logging.warning(f'Unknown parameter {parameter} for {tagname}')
+                    continue
+                time_us = int(data['dt'] * 1_000_000)
+                self.tags[tagname].value = value, time_us, self.map_bus
+                logging.debug(f'Updated {tagname} = {value} at timestamp {data["dt"]}')
+            except Exception as e:
+                logging.error(
+                    f'Error updating {tagname}: {type(e).__name__} - {str(e)}'
+                )
+
+    async def handle_response(self):
+        """Handle responses from the API."""
+        while True:
+            try:
+                location, data = await self.queue.get()
+                logging.debug(f'Processing data for {location}')
+                
+                if 'dt' in data:  # Current weather data
+                    self.update_tags(location, data, '')
+                elif 'list' in data:  # Forecast data
+                    now = int(time())
+                    for forecast in data['list']:
+                        hours_ahead = int((forecast['dt'] - now) / 3600)
+                        if hours_ahead in self.times:
+                            suffix = f'_{hours_ahead:02d}'
+                            self.update_tags(location, forecast, suffix)
+                
+                self.queue.task_done()
+            except Exception as e:
+                logging.error(f'Error handling response: {type(e).__name__} - {str(e)}')
+
+    async def fetch_current_data(self):
+        """Fetch current weather data for all locations."""
+        try:
+            if self.session is None:
+                self.session = aiohttp.ClientSession()
+            
+            for location, coords in self.locations.items():
+                base_params = {
+                    'lat': coords.get('lat'),
+                    'lon': coords.get('lon'),
+                    'appid': self.api_key,
+                    'units': self.units
+                }
+                
+                # Validate required parameters
+                if not all(base_params.values()):
+                    logging.error(
+                        f'Missing required parameters for {location}: '
+                        f'{[k for k, v in base_params.items() if not v]}'
+                    )
+                    continue
+                    
+                try:
+                    async with self.session.get(
+                        self.current_url,
+                        params=base_params,
+                        proxy=self.proxy,
+                        timeout=30  # Add timeout
+                    ) as resp:
+                        if resp.status == 200:
+                            data = await resp.json()
+                            logging.debug(
+                                f'Received current weather data for {location}'
+                            )
+                            await self.queue.put((location, data))
+                        else:
+                            error_text = await resp.text()
+                            logging.error(
+                                f'OpenWeather API error for {location}: '
+                                f'Status: {resp.status}, Response: {error_text[:200]}'
+                            )
+                            
+                except asyncio.TimeoutError:
+                    logging.error(f'Timeout fetching data for {location}')
+                except aiohttp.ClientError as e:
+                    logging.error(
+                        f'Network error for {location}: {type(e).__name__} - {str(e)}'
+                    )
+                except Exception as e:
+                    logging.error(
+                        f'Unexpected error for {location}: {type(e).__name__} - {str(e)}'
+                    )
+                    
+        except Exception as e:
+            logging.error(f'Fatal error in fetch_current_data: {type(e).__name__} - {str(e)}')
+
+    async def fetch_forecast_data(self):
+        """Fetch forecast weather data for all locations."""
+        if self.session is None:
+            self.session = aiohttp.ClientSession()
+        for location, coords in self.locations.items():
+            base_params = {
+                'lat': coords['lat'], 
+                'lon': coords['lon'],
+                'appid': self.api_key, 
+                'units': self.units 
+            }
+            try:
+                async with self.session.get(self.forecast_url,
+                        params=base_params, proxy=self.proxy) as resp:
+                    if resp.status == 200:
+                        data = await resp.json()
+                        logging.info(f'Queue forecast {location} {data}')
+                        await self.queue.put((location, data))
+                    else:
+                        logging.error(f'OpenWeather forecast API error for '
+                            f'{location}: Status:{resp.status}, '
+                            f'Response:{await resp.text()}')
+            except Exception as e:
+                logging.error(f'OpenWeather forecast API error for {location}: '
+                    f'Exception:{type(e).__name__}, Message:{str(e)}')
+
+    async def poll(self):
+        """Poll OpenWeather APIs every 10 minutes."""
+        now = int(time())
+        if now % 600 == 0:  # Every 10 minutes
+            asyncio.create_task(self.fetch_current_data())
+        if now % 3600 == 60:  # Every 3 hours, offset by 1 minute
+            asyncio.create_task(self.fetch_forecast_data())
+
+    async def start(self):
+        """Start bus connection and API polling."""
+        if self.busclient is not None:
+            await self.busclient.start()
+        self.handle = asyncio.create_task(self.handle_response())
+        self.periodic = Periodic(self.poll, 1.0)
+        await self.periodic.start()

{pymscada-0.1.11b9 → pymscada-0.2.0rc1}/src/pymscada/main.py

@@ -39,7 +39,7 @@ async def run():
     if not options.verbose:
         root_logger.setLevel(logging.WARNING)
     factory = ModuleFactory()
-    module = factory.create_module(options.module_name, options)
+    module = factory.create_module(options)
     if module is not None:
         if hasattr(module, 'start'):
             await module.start()

{pymscada-0.1.11b9 → pymscada-0.2.0rc1}/src/pymscada/module_config.py

@@ -75,9 +75,10 @@ def create_module_registry():
         ModuleDefinition(
             name='checkout',
             help='create example config files',
-            module_class='pymscada.checkout:checkout',
+            module_class='pymscada.checkout:Checkout',
             config=False,
             tags=False,
+            await_future=False,
             epilog=dedent("""
                 To add to systemd:
                   su -
@@ -156,12 +157,12 @@ def create_module_registry():
                 like to see correctly typed values and set values."""),
             extra_args=[
                 ModuleArgument(
-                    ('-p', '--port'),
+                    ('-p', '--bus-port'),
                     {'action': 'store', 'type': int, 'default': 1324,
                      'help': 'connect to port (default: 1324)'}
                 ),
                 ModuleArgument(
-                    ('-i', '--ip'),
+                    ('-i', '--bus-ip'),
                     {'action': 'store', 'default': 'localhost',
                      'help': 'connect to ip address (default: localhost)'}
                 )
@@ -208,9 +209,11 @@ class ModuleFactory:
                 parser.add_argument(*arg.args, **arg.kwargs)
         return parser
 
-    def create_module(self, module_name: str, options: argparse.Namespace):
+    def create_module(self, options: argparse.Namespace):
         """Create a module instance based on configuration and options."""
-        module_def = self.modules[module_name]
+        if options.module_name not in self.modules:
+            raise ValueError(f'{options.module_name} does not exist')
+        module_def = self.modules[options.module_name]
         logging.info(f'Python Mobile SCADA {version("pymscada")} '
                      f'starting {module_def.name}')
         # Import the module class only when needed
@@ -220,13 +223,14 @@ class ModuleFactory:
             actual_class = getattr(module, class_name)
         else:
             actual_class = module_def.module_class
-            
         kwargs = {}
         if module_def.config:
-            kwargs.update(Config(options.config))  
+            kwargs.update(Config(options.config))
         if module_def.tags:
             kwargs['tag_info'] = dict(Config(options.tags))
-        if module_name == 'console':
-            return Console(options.ip, options.port,
-                         kwargs.get('tag_info',{}))
+        if module_def.extra_args:
+            for arg in module_def.extra_args:
+                arg_name = arg.args[-1].lstrip('-').replace('-', '_')
+                if hasattr(options, arg_name):
+                    kwargs[arg_name] = getattr(options, arg_name)
         return actual_class(**kwargs)