chellow 1712839751.0.0__tar.gz → 1712929217.0.0__tar.gz

{chellow-1712839751.0.0 → chellow-1712929217.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1712839751.0.0
+Version: 1712929217.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -11,13 +11,13 @@ Requires-Dist: flask-restx==1.2.0
 Requires-Dist: flask==2.3.3
 Requires-Dist: odio==0.0.22
 Requires-Dist: openpyxl==3.1.2
+Requires-Dist: paramiko==3.4.0
 Requires-Dist: pep3143daemon==0.0.6
 Requires-Dist: pg8000==1.31.1
 Requires-Dist: pip>=9.0.1
 Requires-Dist: psutil==5.9.5
 Requires-Dist: pympler==1.0.1
 Requires-Dist: pypdf==3.17.0
-Requires-Dist: pysftp==0.2.9
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2022.6
 Requires-Dist: requests==2.31.0

chellow-1712929217.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1712929217.0.0"

{chellow-1712839751.0.0 → chellow-1712929217.0.0}/chellow/e/hh_importer.py

@@ -8,13 +8,13 @@ import traceback
 from collections import defaultdict, deque
 from datetime import datetime as Datetime, timedelta as Timedelta
 from decimal import Decimal
-from io import TextIOWrapper
+from io import StringIO, TextIOWrapper
+from pathlib import Path
 
 import jinja2
 
-import paramiko
+from paramiko import AutoAddPolicy, RSAKey, SFTPError, SSHClient
 
-import pysftp
 
 import requests
 
@@ -174,7 +174,7 @@ class HhImportTask(threading.Thread):
             self.log(f"Problem {e}")
             sess.rollback()
             self.is_error = True
-        except Exception:
+        except BaseException:
             self.log(f"Unknown Exception {traceback.format_exc()}")
             sess.rollback()
             self.is_error = True
@@ -231,19 +231,19 @@ class HhImportTask(threading.Thread):
         fl = None
 
         for directory in directories:
-            self.log("Checking the directory '" + directory + "'.")
+            self.log(f"Checking the directory '{directory}'.")
             try:
                 last_import_key = last_import_keys[directory]
             except KeyError:
                 last_import_key = last_import_keys[directory] = ""
 
             dir_path = home_path + "/" + directory
-            ftp.cwd(dir_path)
+            ftp.chdir(dir_path)
             files = []
-            for fname in ftp.nlst():
+            for fname in ftp.listdir():
                 fpath = dir_path + "/" + fname
                 try:
-                    ftp.cwd(fpath)
+                    ftp.chdir(fpath)
                     continue  # directory
                 except ftplib.error_perm:
                     pass
@@ -266,7 +266,7 @@ class HhImportTask(threading.Thread):
             key, fpath = fl
             self.log(f"Attempting to download {fpath} with key {key}.")
             f = tempfile.TemporaryFile()
-            ftp.retrbinary("RETR " + fpath, f.write)
+            ftp.retrbinary(f"RETR {fpath}", f.write)
             self.log("File downloaded successfully.")
             ftp.quit()
             self.log("Logged out.")
@@ -290,25 +290,30 @@ class HhImportTask(threading.Thread):
             contract = Contract.get_dc_by_id(sess, self.contract_id)
             contract.update_state(state)
             sess.commit()
-            self.log("Finished loading '" + fpath)
+            self.log(f"Finished loading '{fpath}'")
             return True
 
     def sftp_handler(self, sess, properties, contract):
-        host_name = properties["hostname"]
-        user_name = properties["username"]
-        password = properties["password"]
-        try:
-            port = properties["port"]
-        except KeyError:
-            port = None
+        hostname = properties["hostname"]
+        username = properties["username"]
+        password = properties.get("password")
+        private_key = properties.get("private_key")
+        pkey = None
+        if private_key is not None:
+            pkf = StringIO.StringIO(private_key)
+            pkf.seek(0)
+            pkey = RSAKey.from_private_key(pkf)
+
+        port = properties.get("port", 22)
         file_type = properties["file_type"]
         directories = properties["directories"]
+        known_hosts = properties["known_hosts"].strip()
         state = contract.make_state()
 
         try:
             last_import_keys = state["last_import_keys"]
         except KeyError:
-            last_import_keys = state["last_import_keys"] = []
+            last_import_keys = state["last_import_keys"] = {}
 
         ct_now = ct_datetime_now()
         try:
@@ -323,36 +328,60 @@ class HhImportTask(threading.Thread):
             latest_imports = state["latest_imports"] = []
 
         sess.rollback()
-        self.log(f"Connecting to sftp server at {host_name}:{port}.")
-        cnopts = pysftp.CnOpts()
-        cnopts.hostkeys = None
-        ftp = pysftp.Connection(
-            host_name, username=user_name, password=password, cnopts=cnopts
+        self.log(f"Connecting to sftp server at {hostname}:{port}.")
+
+        client = SSHClient()
+        client.set_missing_host_key_policy(AutoAddPolicy)
+
+        client.connect(
+            hostname,
+            port=port,
+            username=username,
+            password=password,
+            pkey=pkey,
+            key_filename=None,
+            timeout=120,
+            banner_timeout=120,
+            auth_timeout=120,
+            channel_timeout=120,
+            look_for_keys=False,
         )
-        ftp.timeout = 120
-        home_path = ftp.pwd
+        host_keys = client.get_host_keys()
+        with tempfile.TemporaryDirectory() as tempdir:
+            td = Path(tempdir)
+            hfname = td / "hosts"
+            host_keys.save(hfname)
+            with hfname.open() as hf:
+                host_keys_str = hf.read().strip()
+
+        if host_keys_str != known_hosts:
+            raise BadRequest(
+                f"The returned host keys {host_keys_str} don't match the known "
+                f"hosts {known_hosts} property"
+            )
+
+        ftp = client.open_sftp()
 
         f = None
 
         for directory in directories:
-            self.log("Checking the directory '" + directory + "'.")
+            self.log(f"Checking the directory '{directory}'.")
             try:
                 last_import_key = last_import_keys[directory]
             except KeyError:
                 last_import_key = last_import_keys[directory] = ""
 
-            dir_path = home_path + "/" + directory
-            ftp.cwd(dir_path)
+            ftp.chdir(directory)
             files = []
             for attr in ftp.listdir_attr():
-                fpath = dir_path + "/" + attr.filename
+                fpath = f"{directory}/{attr.filename}"
                 try:
-                    ftp.cwd(fpath)
+                    ftp.chdir(fpath)
                     continue  # directory
-                except paramiko.SFTPError:
+                except SFTPError:
                     pass
 
-                key = str(attr.st_mtime) + "_" + attr.filename
+                key = f"{attr.st_mtime}_{attr.filename}"
                 if key > last_import_key:
                     files.append((key, fpath))
 
@@ -364,6 +393,7 @@ class HhImportTask(threading.Thread):
         if f is None:
             self.log("No new files found.")
             ftp.close()
+            client.close()
             self.log("Logged out.")
             return False
         else:

{chellow-1712839751.0.0 → chellow-1712929217.0.0}/chellow/e/views.py

@@ -105,7 +105,11 @@ from chellow.utils import (
     utc_datetime,
     utc_datetime_now,
 )
-from chellow.views import chellow_redirect as credirect, hx_redirect as chx_redirect
+from chellow.views import (
+    chellow_redirect as credirect,
+    hx_redirect as chx_redirect,
+    requires_editor,
+)
 
 
 def chellow_redirect(path, code=None):
@@ -1075,6 +1079,44 @@ def dc_contract_get(dc_contract_id):
             )
 
 
+@e.route("/dc_contracts/<int:dc_contract_id>/properties")
+@requires_editor
+def dc_contract_properties_get(dc_contract_id):
+    contract = Contract.get_dc_by_id(g.sess, dc_contract_id)
+    return render_template("dc_contract_properties.html", dc_contract=contract)
+
+
+@e.route("/dc_contracts/<int:dc_contract_id>/properties/edit")
+@requires_editor
+def dc_contract_properties_edit_get(dc_contract_id):
+    dc_contract = Contract.get_dc_by_id(g.sess, dc_contract_id)
+    return render_template("dc_contract_properties_edit.html", dc_contract=dc_contract)
+
+
+@e.route("/dc_contracts/<int:dc_contract_id>/properties/edit", methods=["POST"])
+def dc_contract_properties_edit_post(dc_contract_id):
+    dc_contract = None
+    try:
+        dc_contract = Contract.get_dc_by_id(g.sess, dc_contract_id)
+        properties = req_zish("properties")
+        dc_contract.update(
+            dc_contract.name, dc_contract.party, dc_contract.charge_script, properties
+        )
+        g.sess.commit()
+        return chellow_redirect(f"/dc_contracts/{dc_contract.id}/properties", 303)
+    except BadRequest as e:
+        flash(e.description)
+        if dc_contract is None:
+            raise e
+        else:
+            return make_response(
+                render_template(
+                    "dc_contract_properties_edit.html", dc_contract=dc_contract
+                ),
+                400,
+            )
+
+
 @e.route("/dc_contracts/<int:dc_contract_id>/edit")
 def dc_contract_edit_get(dc_contract_id):
     parties = (
@@ -1127,9 +1169,8 @@ def dc_contract_edit_post(contract_id):
             party_id = req_int("party_id")
             name = req_str("name")
             charge_script = req_str("charge_script")
-            properties = req_zish("properties")
             party = Party.get_by_id(g.sess, party_id)
-            contract.update(name, party, charge_script, properties)
+            contract.update(name, party, charge_script, contract.properties)
             g.sess.commit()
             return chellow_redirect(f"/dc_contracts/{contract.id}", 303)
     except BadRequest as e:

{chellow-1712839751.0.0 → chellow-1712929217.0.0}/chellow/templates/e/dc_contract.html

@@ -66,6 +66,9 @@
 	</table>
 	<br>
 	<ul>
+		<li>
+			<a href="/e/dc_contracts/{{dc_contract.id}}/properties">Properties</a>
+		</li>
 		<li>
 			<a href="/e/dc_batches?dc_contract_id={{dc_contract.id}}">Batches</a>
 		</li>
@@ -99,10 +102,6 @@
 
 	<pre>{{dc_contract.charge_script}}</pre>
 
-	<h3>Properties</h3>
-
-	<pre>{{dc_contract.properties }}</pre>
-
 	<h3>State</h3>
 
 	<pre>{{dc_contract.state }}</pre>

{chellow-1712839751.0.0 → chellow-1712929217.0.0}/chellow/templates/e/dc_contract_edit.html

@@ -42,58 +42,10 @@
 				{{input_textarea(
 					'charge_script', dc_contract.charge_script, 40, 80)}}
 
-				<label>Properties</label>
-				{{ input_textarea(
-					'properties', dc_contract.properties, 40, 80) }}
-
 				<input type="submit" value="Update">
 			</fieldset>
 		</form>
 
-		<h4>Example</h4>
-
-		<p>For the SFTP protocol:</p>
-		<code>
-			<pre>
-{
-  "enabled": true,
-	"protocol": "sftp",
-	"file_type": ".df2",
-	"hostname": "example.com",
-	"username": "username",
-	"password": "password",
-	"directories": ["downloads1", "downloads2"]
-	"mpan_map": {  \\ Optional
-	  "99 0993 2821 985": null,  \\ Ignore MPAN
-	},
-}
-			</pre>
-		</code>
-		<p>For the HTTPS protocol:</p>
-		<code>
-			<pre>
-				{% raw %}
-{
-  "enabled": true,
-  "protocol": "https",
-  "download_days": 8,
-  "parser": "meniscus",
-  "url_template": "https://data.example.com/?from={{chunk_start.strftime('%d/%m/%Y')}}&amp;to={{chunk_finish.strftime('%d/%m/%Y')}}",
-  "url_values": {
-    "99 4298 4729 917": {
-      "name1": val1,
-      "name2": val2,
-    }
-  }
-	"mpan_map": {  \\ Optional
-	  "99 0993 2821 985": null,  \\ Ignore MPAN
-	},
-}
-
-				{% endraw %}
-			</pre>
-		</code>
-
 		<form action="/e/dc_contracts/{{dc_contract.id}}/edit" method="post">
 			<fieldset>
 				<legend>Update State</legend>

chellow-1712929217.0.0/chellow/templates/e/dc_contract_properties.html

@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+
+{% block title %}
+	» DC Contracts » {{dc_contract.name}} » Properties
+{% endblock %}
+
+{% block nav %}
+	<a href="/e/dc_contracts">DC Contracts</a> &raquo;
+	<a href="/e/dc_contracts/{{dc_contract.id}}">{{dc_contract.name}}</a> &raquo;
+	Properties [<a href="/e/dc_contracts/{{dc_contract.id}}/properties/edit">edit</a>]
+{% endblock %}
+
+{% block content %}
+
+	<pre>{{dc_contract.properties }}</pre>
+
+{% endblock %}

chellow-1712929217.0.0/chellow/templates/e/dc_contract_properties_edit.html

@@ -0,0 +1,71 @@
+{% extends "base.html" %}
+
+{% block title %}
+	» DC Contracts » {{dc_contract.name}} » Properties » Edit
+{% endblock %}
+
+{% block nav %}
+	<a href="/e/dc_contracts">DC Contracts</a> &raquo;
+	<a href="/e/dc_contracts/{{dc_contract.id}}">{{dc_contract.name}}</a> &raquo;
+	<a href="/e/dc_contracts/{{dc_contract.id}}/properties">Properties</a> &raquo;
+ 	Edit
+{% endblock %}
+
+{% block content %}
+
+	<form action="/e/dc_contracts/{{dc_contract.id}}/properties/edit" method="post">
+		<fieldset>
+			<legend>Update Properties</legend>
+
+			<label>Properties</label>
+			{{ input_textarea('properties', dc_contract.properties, 40, 80, show_pos=True) }}
+
+			<input type="submit" value="Update">
+		</fieldset>
+	</form>
+
+	<h4>Example</h4>
+
+	<p>For the SFTP protocol:</p>
+	<code>
+		<pre>
+{
+  "enabled": true,
+  "protocol": "sftp",
+  "file_type": ".df2",
+  "hostname": "example.com",
+  "username": "username",
+  "password": "password", /* Remove if using private key */
+  "private_key": "private_key", /* Remove if using password */
+  "directories": ["downloads1", "downloads2"],
+  "mpan_map": {  /* Optional */
+    "99 0993 2821 985": null,  /* Ignore MPAN */
+  },
+}
+		</pre>
+	</code>
+	<p>For the HTTPS protocol:</p>
+	<code>
+		<pre>
+			{% raw %}
+{
+  "enabled": true,
+  "protocol": "https",
+  "download_days": 8,
+  "parser": "meniscus",
+  "url_template": "https://data.example.com/?from={{chunk_start.strftime('%d/%m/%Y')}}&amp;to={{chunk_finish.strftime('%d/%m/%Y')}}",
+  "url_values": {
+    "99 4298 4729 917": {
+      "name1": val1,
+      "name2": val2,
+    }
+  }
+  "mpan_map": {  \\ Optional
+    "99 0993 2821 985": null,  \\ Ignore MPAN
+  },
+}
+
+			{% endraw %}
+		</pre>
+	</code>
+{% endblock %}

{chellow-1712839751.0.0 → chellow-1712929217.0.0}/chellow/views.py

@@ -11,6 +11,7 @@ import types
 from collections import OrderedDict
 from datetime import datetime as Datetime
 from decimal import Decimal
+from functools import wraps
 from importlib import import_module
 from io import DEFAULT_BUFFER_SIZE, StringIO
 from itertools import chain, islice
@@ -52,7 +53,7 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.attributes import flag_modified
 
-from werkzeug.exceptions import BadRequest
+from werkzeug.exceptions import BadRequest, Forbidden
 
 import chellow.bank_holidays
 import chellow.dloads
@@ -147,6 +148,17 @@ def hx_redirect(path, status=None):
     return res
 
 
+def requires_editor(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        if g.user.user_role.code == "editor":
+            return f(*args, **kwargs)
+        else:
+            raise Forbidden("You must be an editor to do this.")
+
+    return decorated_function
+
+
 @home.route("/configuration", methods=["GET"])
 def configuration():
     config = Contract.get_non_core_by_name(g.sess, "configuration")

{chellow-1712839751.0.0 → chellow-1712929217.0.0}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 	"waitress==2.1.2",
 	"pep3143daemon==0.0.6",
 	"pip>=9.0.1",
-	"pysftp==0.2.9",
+	"paramiko==3.4.0",
 	"pympler==1.0.1",
 	"psutil==5.9.5",
 	"xlrd==2.0.1",

chellow-1712839751.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1712839751.0.0"