chellow 1698931736.0.0__tar.gz → 1765896461.0.0__tar.gz

chellow-1765896461.0.0/.github/workflows/chellow.yml

@@ -0,0 +1,95 @@
+name: Chellow
+
+on:
+  push:
+    branches: [main, test]
+  pull_request:
+    branches: [main, test]
+
+jobs:
+  # Label of the container job
+  test:
+    runs-on: ubuntu-latest
+    container: python:3.11
+
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run Tox
+        run: |
+          tox
+        env:
+          PGHOST: postgres
+          PGPORT: 5432
+
+  build:
+    runs-on: ubuntu-latest
+    container: python:3.11
+    needs: test
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build a binary wheel and a source tarball
+      run: |
+        echo "__version__ = \"$(date +%s).0.0\"" > VERSION
+        python -m build 
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  deploy_main:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      id-token: write
+    if: github.ref == 'refs/heads/main'
+    environment:
+      name: pypi
+      url: https://pypi.org/p/chellow
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  deploy_test:
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      id-token: write
+    if: github.ref == 'refs/heads/test'
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/chellow
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/

{chellow-1698931736.0.0 → chellow-1765896461.0.0}/PKG-INFO

@@ -1,30 +1,30 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: chellow
-Version: 1698931736.0.0
+Version: 1765896461.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Requires-Python: >=3.9
-Requires-Dist: flask-restx==1.2.0
-Requires-Dist: flask==2.3.3
-Requires-Dist: odio==0.0.22
-Requires-Dist: openpyxl==3.1.2
-Requires-Dist: pep3143daemon==0.0.6
-Requires-Dist: pg8000==1.30.1
+Requires-Python: >=3.11
+Requires-Dist: flask-restx==1.3.2
+Requires-Dist: flask==3.1.2
+Requires-Dist: markdown-it-py==4.0.0
+Requires-Dist: odio==0.0.23
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: paramiko==4.0.0
+Requires-Dist: pg8000==1.31.5
 Requires-Dist: pip>=9.0.1
-Requires-Dist: psutil==5.9.5
-Requires-Dist: pympler==1.0.1
-Requires-Dist: pypdf==3.17.0
-Requires-Dist: pysftp==0.2.9
+Requires-Dist: psutil==7.1.1
+Requires-Dist: pympler==1.1
+Requires-Dist: pypdf==6.4.0
 Requires-Dist: python-dateutil==2.8.2
-Requires-Dist: pytz==2022.6
-Requires-Dist: requests==2.31.0
-Requires-Dist: sqlalchemy==2.0.20
-Requires-Dist: waitress==2.1.2
-Requires-Dist: xlrd==2.0.1
-Requires-Dist: zish==0.1.11
+Requires-Dist: pytz==2025.2
+Requires-Dist: requests==2.32.5
+Requires-Dist: sqlalchemy==2.0.44
+Requires-Dist: waitress==3.0.2
+Requires-Dist: xlrd==2.0.2
+Requires-Dist: zish==0.1.12
 Description-Content-Type: text/markdown
 
 # Chellow
@@ -46,7 +46,7 @@ Chellow is a web application for checking UK electricity and gas bills. It's des
 for organizations with high electricity consumption. The software is hosted at
 https://github.com/WessexWater/chellow.
 
-[![Build Status](https://github.com/WessexWater/chellow/workflows/chellow/badge.svg)](https://github.com/WessexWater/chellow/workflows/chellow)
+[![Build Status](https://github.com/WessexWater/chellow/actions/workflows/chellow.yml/badge.svg)](https://github.com/WessexWater/chellow/actions/workflows/chellow.yml)
 
 
 ## Installation
@@ -54,7 +54,7 @@ https://github.com/WessexWater/chellow.
 Chellow is a Python web application that uses the PostgreSQL database. To install
 Chellow, follow these steps:
 
-* Install [PostgreSQL](http://www.postgresql.org/) 12
+* Install [PostgreSQL](http://www.postgresql.org/) 15
 * Create a PostgreSQL database: `createdb --encoding=UTF8 chellow`
 * Set up the following environment variables to configure Chellow:
 
@@ -74,7 +74,7 @@ in Windows an environment variable can be set by doing:
 
 `set PGUSER=postgres`
 
-* Install Python 3.9 (tested on the [CPython 3.9.8](http://www.python.org/) interpreter)
+* Install Python 3.11
 * Create a virtual environment: `python3 -m venv venv`
 * Activate the virtual environment: `source venv/bin/activate`
 * Install Chellow: `pip install chellow`
@@ -84,6 +84,19 @@ in Windows an environment variable can be set by doing:
   `admin@example.com` and the password `admin`, and then the home page should appear.
   Change the admin password from the `users` page.
 
+### Azure
+
+SSH into the machine and you'll be in ``/home/site/wwwroot``.
+
+* Create a virtual environment: ``python -m venv antenv``
+* Activate the virtual environment with: ``source antenv/bin/activate``
+*
+In ``/home/site/wwwroot`` create a file called ``app.py`` containing the following:
+
+    import chellow
+    
+    app = chellow.create_app()
+
 
 ### Manually Upgrading Chellow
 

{chellow-1698931736.0.0 → chellow-1765896461.0.0}/README.md

@@ -17,7 +17,7 @@ Chellow is a web application for checking UK electricity and gas bills. It's des
 for organizations with high electricity consumption. The software is hosted at
 https://github.com/WessexWater/chellow.
 
-[![Build Status](https://github.com/WessexWater/chellow/workflows/chellow/badge.svg)](https://github.com/WessexWater/chellow/workflows/chellow)
+[![Build Status](https://github.com/WessexWater/chellow/actions/workflows/chellow.yml/badge.svg)](https://github.com/WessexWater/chellow/actions/workflows/chellow.yml)
 
 
 ## Installation
@@ -25,7 +25,7 @@ https://github.com/WessexWater/chellow.
 Chellow is a Python web application that uses the PostgreSQL database. To install
 Chellow, follow these steps:
 
-* Install [PostgreSQL](http://www.postgresql.org/) 12
+* Install [PostgreSQL](http://www.postgresql.org/) 15
 * Create a PostgreSQL database: `createdb --encoding=UTF8 chellow`
 * Set up the following environment variables to configure Chellow:
 
@@ -45,7 +45,7 @@ in Windows an environment variable can be set by doing:
 
 `set PGUSER=postgres`
 
-* Install Python 3.9 (tested on the [CPython 3.9.8](http://www.python.org/) interpreter)
+* Install Python 3.11
 * Create a virtual environment: `python3 -m venv venv`
 * Activate the virtual environment: `source venv/bin/activate`
 * Install Chellow: `pip install chellow`
@@ -55,6 +55,19 @@ in Windows an environment variable can be set by doing:
   `admin@example.com` and the password `admin`, and then the home page should appear.
   Change the admin password from the `users` page.
 
+### Azure
+
+SSH into the machine and you'll be in ``/home/site/wwwroot``.
+
+* Create a virtual environment: ``python -m venv antenv``
+* Activate the virtual environment with: ``source antenv/bin/activate``
+*
+In ``/home/site/wwwroot`` create a file called ``app.py`` containing the following:
+
+    import chellow
+    
+    app = chellow.create_app()
+
 
 ### Manually Upgrading Chellow
 

chellow-1765896461.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1765896461.0.0"

chellow-1765896461.0.0/bin/chellow_upgrader.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+cd /var/www/chellow
+source venv/bin/activate
+
+if [[ $* == *--test* ]] ; then
+	if pip list --outdated --extra-index-url https://test.pypi.org/simple/ | grep chellow ; then
+		echo "Found a new Chellow version."
+
+		echo "Stopping Chellow"
+		/usr/bin/systemctl stop chellow
+
+		echo "Doing a pip upgrade of Chellow."
+		pip install chellow --upgrade --extra-index-url https://test.pypi.org/simple/
+
+		echo "Starting Chellow"
+		/usr/bin/systemctl start chellow
+	else
+		echo "No new version found."
+	fi
+else
+	if pip list --outdated | grep chellow ; then
+		echo "Found a new Chellow version."
+
+		echo "Stopping Chellow"
+		/usr/bin/systemctl stop chellow
+
+		echo "Doing a pip upgrade of Chellow."
+		pip install chellow --upgrade
+
+		echo "Starting Chellow"
+		/usr/bin/systemctl start chellow
+	else
+		echo "No new version found."
+	fi
+fi

{chellow-1698931736.0.0 → chellow-1765896461.0.0}/chellow/__init__.py

@@ -1,9 +1,15 @@
+import json
 import os
+from base64 import b64decode
 from datetime import datetime as Datetime
 from importlib.metadata import version
+from pathlib import Path
 
 from flask import Flask, Response, g, make_response, render_template, request
 
+from markdown_it import MarkdownIt
+
+from markupsafe import Markup
 
 from sqlalchemy import select
 
@@ -16,14 +22,16 @@ import chellow.bank_holidays
 import chellow.dloads
 import chellow.e.bmarketidx
 import chellow.e.bsuos
+import chellow.e.elexon
 import chellow.e.hh_importer
 import chellow.e.rcrc
 import chellow.e.system_price
-import chellow.e.tlms
 import chellow.e.views
+import chellow.fake_batch_updater
 import chellow.gas.cv
 import chellow.gas.views
 import chellow.national_grid
+import chellow.rrun
 import chellow.testing
 from chellow.models import (
     Contract,
@@ -50,22 +58,22 @@ TEMPLATE_FORMATS = {
 
 def get_importer_modules():
     return (
-        chellow.e.rcrc,
+        chellow.e.elexon,
         chellow.e.bsuos,
-        chellow.e.system_price,
-        chellow.e.hh_importer,
+        chellow.e.lcc,
         chellow.testing,
-        chellow.e.tlms,
         chellow.bank_holidays,
         chellow.gas.cv,
         chellow.e.bmarketidx,
         chellow.national_grid,
         chellow.rate_server,
+        chellow.rrun,
+        chellow.fake_batch_updater,
     )
 
 
-def create_app(testing=False):
-    app = Flask("chellow", instance_relative_config=True)
+def create_app(testing=False, instance_path=None):
+    app = Flask("chellow", instance_relative_config=True, instance_path=instance_path)
     app.wsgi_app = MsProxy(app.wsgi_app)
     app.secret_key = os.urandom(24)
     start_sqlalchemy()
@@ -73,14 +81,13 @@ def create_app(testing=False):
     app.register_blueprint(chellow.views.home)
     app.register_blueprint(chellow.e.views.e)
     app.register_blueprint(chellow.gas.views.gas)
-    chellow.utils.root_path = app.root_path
 
     api = chellow.api.api
     api.init_app(app, endpoint="/api/v1")
 
     if not testing:
         db_upgrade(app.root_path)
-        chellow.dloads.startup(app.instance_path)
+        chellow.e.hh_importer.startup()
 
         with Session() as sess:
             configuration = sess.execute(
@@ -90,6 +97,8 @@ def create_app(testing=False):
             api_props = props.get("api", {})
             api.description = api_props.get("description", "Access Chellow data")
 
+    chellow.dloads.startup(Path(app.instance_path), run_deleter=(not testing))
+
     for module in get_importer_modules():
         if not testing:
             module.startup()
@@ -113,20 +122,6 @@ def create_app(testing=False):
         )
         print(msg)
 
-        try:
-            scheme = request.headers["X-Forwarded-Proto"]
-        except KeyError:
-            config_contract = Contract.get_non_core_by_name(g.sess, "configuration")
-            props = config_contract.make_properties()
-            scheme = props.get("redirect_scheme", "http")
-
-        try:
-            host = request.headers["X-Forwarded-Host"]
-        except KeyError:
-            host = request.host
-
-        chellow.utils.url_root = scheme + "://" + host
-
     @app.before_request
     def check_permissions(*args, **kwargs):
         g.user = None
@@ -137,17 +132,45 @@ def create_app(testing=False):
             g.config = {}
         ad_props = g.config.get("ad_authentication", {})
         ad_auth_on = ad_props.get("on", False)
+        easy_auth_props = g.config.get("easy_auth", {})
+        easy_auth_on = easy_auth_props.get("on", False)
+        path = request.path
+        if path in ("/health", "/robots933456.txt"):
+            return
         if ad_auth_on:
             username = request.headers["X-Isrw-Proxy-Logon-User"].upper()
-            user = g.sess.query(User).filter(User.email_address == username).first()
+            user = g.sess.scalars(
+                select(User).where(User.email_address == username)
+            ).first()
             if user is None:
                 try:
                     username = ad_props["default_user"]
-                    user = (
-                        g.sess.query(User)
-                        .filter(User.email_address == username)
-                        .first()
-                    )
+                    user = g.sess.scalars(
+                        select(User).where(User.email_address == username)
+                    ).first()
+                except KeyError:
+                    user = None
+            if user is not None:
+                g.user = user
+        elif easy_auth_on:
+            jwt_enc = request.headers["X-MS-CLIENT-PRINCIPAL"]
+            jwt = json.loads(b64decode(jwt_enc))
+            for claim in jwt["claims"]:
+                if claim["typ"] == (
+                    "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/"
+                    "emailaddress"
+                ):
+                    username = claim["val"]
+                    user = g.sess.scalars(
+                        select(User).where(User.email_address == username)
+                    ).first()
+                    break
+            if user is None:
+                try:
+                    username = easy_auth_props["default_user"]
+                    user = g.sess.scalars(
+                        select(User).where(User.email_address == username)
+                    ).first()
                 except KeyError:
                     user = None
             if user is not None:
@@ -166,25 +189,20 @@ def create_app(testing=False):
                         key = None
 
                     email = ips[key]
-                    g.user = (
-                        g.sess.query(User).filter(User.email_address == email).first()
-                    )
+                    g.user = g.sess.scalars(
+                        select(User).where(User.email_address == email)
+                    ).first()
                 except KeyError:
                     pass
             else:
-                user = (
-                    g.sess.query(User)
-                    .filter(User.email_address == auth.username)
-                    .first()
-                )
+                user = g.sess.scalars(
+                    select(User).where(User.email_address == auth.username)
+                ).first()
                 if user is not None and user.password_matches(auth.password):
                     g.user = user
 
         # Got our user
-        path = request.path
         method = request.method
-        if path in ("/health",):
-            return
 
         if g.user is not None:
             if "X-Isrw-Proxy-Logon-User" in request.headers:
@@ -237,7 +255,9 @@ def create_app(testing=False):
             g.sess.commit()
             return
 
-        if g.user is None or (not ad_auth_on and auth is None):
+        if (not easy_auth_on) and (
+            (g.user is None) or (not ad_auth_on and auth is None)
+        ):
             return Response(
                 "Could not verify your access level for that URL.\n"
                 "You have to login with proper credentials",
@@ -263,14 +283,8 @@ def create_app(testing=False):
                 except NotFound:
                     pass
 
-        for importer in (
-            chellow.e.bsuos.bsuos_importer,
-            chellow.gas.cv.cv_importer,
-            chellow.e.bmarketidx.bmarketidx_importer,
-            chellow.national_grid.importer,
-            chellow.rate_server.importer,
-            chellow.testing.tester,
-        ):
+        for mod in get_importer_modules():
+            importer = mod.get_importer()
             if importer is not None and importer.global_alert is not None:
                 global_alerts.append(importer.global_alert)
 
@@ -317,6 +331,11 @@ def create_app(testing=False):
     def dumps_filter(d):
         return dumps(d)
 
+    @app.template_filter("markdown")
+    def markdown_filter(txt):
+        md = MarkdownIt()
+        return Markup(md.render(txt))
+
     return app
 
 

{chellow-1698931736.0.0 → chellow-1765896461.0.0}/chellow/bank_holidays.py

@@ -12,26 +12,37 @@ from zish import loads
 from chellow.models import Contract, RateScript, Session
 from chellow.utils import (
     HH,
+    ct_datetime,
     hh_format,
+    to_utc,
     utc_datetime,
     utc_datetime_now,
     utc_datetime_parse,
 )
 
 
-ELEXON_PORTAL_SCRIPTING_KEY_KEY = "elexonportal_scripting_key"
-
-
 bh_importer = None
 
 
 def _run(log_f, sess):
     log_f("Starting to check bank holidays")
-    contract = Contract.get_non_core_by_name(sess, "bank_holidays")
+    contract_name = "bank_holidays"
+    contract = Contract.find_non_core_by_name(sess, contract_name)
+    if contract is None:
+        contract = Contract.insert_non_core(
+            sess,
+            contract_name,
+            "",
+            {"enabled": True},
+            to_utc(ct_datetime(1996, 4, 1)),
+            None,
+            {},
+        )
+        sess.commit()
     contract_props = contract.make_properties()
 
-    if contract_props.get("enabled", False):
-        url_str = contract_props["url"]
+    if contract_props.get("enabled", True):
+        url_str = "https://www.gov.uk/bank-holidays/england-and-wales.ics"
 
         log_f(f"Downloading from {url_str}.")
         res = requests.get(url_str)
@@ -94,6 +105,7 @@ class BankHolidayImporter(threading.Thread):
         self.going = threading.Event()
         self.PROXY_HOST_KEY = "proxy.host"
         self.PROXY_PORT_KEY = "proxy.port"
+        self.global_alert = None
 
     def stop(self):
         self.stopped.set()
@@ -120,9 +132,13 @@ class BankHolidayImporter(threading.Thread):
             if self.lock.acquire(False):
                 try:
                     with Session() as sess:
+                        self.global_alert = None
                         _run(self.log, sess)
                 except BaseException:
                     self.log(f"Outer problem {traceback.format_exc()}")
+                    self.global_alert = (
+                        "There's a problem with the Bank Holiday importer"
+                    )
                 finally:
                     self.lock.release()
                     self.log("Finished checking bank holidays.")