chellow 1712909617.0.0__tar.gz → 1712929217.0.0__tar.gz

{chellow-1712909617.0.0 → chellow-1712929217.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1712909617.0.0
+Version: 1712929217.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -11,13 +11,13 @@ Requires-Dist: flask-restx==1.2.0
 Requires-Dist: flask==2.3.3
 Requires-Dist: odio==0.0.22
 Requires-Dist: openpyxl==3.1.2
+Requires-Dist: paramiko==3.4.0
 Requires-Dist: pep3143daemon==0.0.6
 Requires-Dist: pg8000==1.31.1
 Requires-Dist: pip>=9.0.1
 Requires-Dist: psutil==5.9.5
 Requires-Dist: pympler==1.0.1
 Requires-Dist: pypdf==3.17.0
-Requires-Dist: pysftp==0.2.9
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2022.6
 Requires-Dist: requests==2.31.0

chellow-1712929217.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1712929217.0.0"

{chellow-1712909617.0.0 → chellow-1712929217.0.0}/chellow/e/hh_importer.py

@@ -8,13 +8,13 @@ import traceback
 from collections import defaultdict, deque
 from datetime import datetime as Datetime, timedelta as Timedelta
 from decimal import Decimal
-from io import TextIOWrapper
+from io import StringIO, TextIOWrapper
+from pathlib import Path
 
 import jinja2
 
-import paramiko
+from paramiko import AutoAddPolicy, RSAKey, SFTPError, SSHClient
 
-import pysftp
 
 import requests
 
@@ -174,7 +174,7 @@ class HhImportTask(threading.Thread):
             self.log(f"Problem {e}")
             sess.rollback()
             self.is_error = True
-        except Exception:
+        except BaseException:
             self.log(f"Unknown Exception {traceback.format_exc()}")
             sess.rollback()
             self.is_error = True
@@ -231,19 +231,19 @@ class HhImportTask(threading.Thread):
         fl = None
 
         for directory in directories:
-            self.log("Checking the directory '" + directory + "'.")
+            self.log(f"Checking the directory '{directory}'.")
             try:
                 last_import_key = last_import_keys[directory]
             except KeyError:
                 last_import_key = last_import_keys[directory] = ""
 
             dir_path = home_path + "/" + directory
-            ftp.cwd(dir_path)
+            ftp.chdir(dir_path)
             files = []
-            for fname in ftp.nlst():
+            for fname in ftp.listdir():
                 fpath = dir_path + "/" + fname
                 try:
-                    ftp.cwd(fpath)
+                    ftp.chdir(fpath)
                     continue  # directory
                 except ftplib.error_perm:
                     pass
@@ -266,7 +266,7 @@ class HhImportTask(threading.Thread):
             key, fpath = fl
             self.log(f"Attempting to download {fpath} with key {key}.")
             f = tempfile.TemporaryFile()
-            ftp.retrbinary("RETR " + fpath, f.write)
+            ftp.retrbinary(f"RETR {fpath}", f.write)
             self.log("File downloaded successfully.")
             ftp.quit()
             self.log("Logged out.")
@@ -290,25 +290,30 @@ class HhImportTask(threading.Thread):
             contract = Contract.get_dc_by_id(sess, self.contract_id)
             contract.update_state(state)
             sess.commit()
-            self.log("Finished loading '" + fpath)
+            self.log(f"Finished loading '{fpath}'")
             return True
 
     def sftp_handler(self, sess, properties, contract):
-        host_name = properties["hostname"]
-        user_name = properties["username"]
-        password = properties["password"]
-        try:
-            port = properties["port"]
-        except KeyError:
-            port = None
+        hostname = properties["hostname"]
+        username = properties["username"]
+        password = properties.get("password")
+        private_key = properties.get("private_key")
+        pkey = None
+        if private_key is not None:
+            pkf = StringIO.StringIO(private_key)
+            pkf.seek(0)
+            pkey = RSAKey.from_private_key(pkf)
+
+        port = properties.get("port", 22)
         file_type = properties["file_type"]
         directories = properties["directories"]
+        known_hosts = properties["known_hosts"].strip()
         state = contract.make_state()
 
         try:
             last_import_keys = state["last_import_keys"]
         except KeyError:
-            last_import_keys = state["last_import_keys"] = []
+            last_import_keys = state["last_import_keys"] = {}
 
         ct_now = ct_datetime_now()
         try:
@@ -323,36 +328,60 @@ class HhImportTask(threading.Thread):
             latest_imports = state["latest_imports"] = []
 
         sess.rollback()
-        self.log(f"Connecting to sftp server at {host_name}:{port}.")
-        cnopts = pysftp.CnOpts()
-        cnopts.hostkeys = None
-        ftp = pysftp.Connection(
-            host_name, username=user_name, password=password, cnopts=cnopts
+        self.log(f"Connecting to sftp server at {hostname}:{port}.")
+
+        client = SSHClient()
+        client.set_missing_host_key_policy(AutoAddPolicy)
+
+        client.connect(
+            hostname,
+            port=port,
+            username=username,
+            password=password,
+            pkey=pkey,
+            key_filename=None,
+            timeout=120,
+            banner_timeout=120,
+            auth_timeout=120,
+            channel_timeout=120,
+            look_for_keys=False,
         )
-        ftp.timeout = 120
-        home_path = ftp.pwd
+        host_keys = client.get_host_keys()
+        with tempfile.TemporaryDirectory() as tempdir:
+            td = Path(tempdir)
+            hfname = td / "hosts"
+            host_keys.save(hfname)
+            with hfname.open() as hf:
+                host_keys_str = hf.read().strip()
+
+        if host_keys_str != known_hosts:
+            raise BadRequest(
+                f"The returned host keys {host_keys_str} don't match the known "
+                f"hosts {known_hosts} property"
+            )
+
+        ftp = client.open_sftp()
 
         f = None
 
         for directory in directories:
-            self.log("Checking the directory '" + directory + "'.")
+            self.log(f"Checking the directory '{directory}'.")
             try:
                 last_import_key = last_import_keys[directory]
             except KeyError:
                 last_import_key = last_import_keys[directory] = ""
 
-            dir_path = home_path + "/" + directory
-            ftp.cwd(dir_path)
+            ftp.chdir(directory)
             files = []
             for attr in ftp.listdir_attr():
-                fpath = dir_path + "/" + attr.filename
+                fpath = f"{directory}/{attr.filename}"
                 try:
-                    ftp.cwd(fpath)
+                    ftp.chdir(fpath)
                     continue  # directory
-                except paramiko.SFTPError:
+                except SFTPError:
                     pass
 
-                key = str(attr.st_mtime) + "_" + attr.filename
+                key = f"{attr.st_mtime}_{attr.filename}"
                 if key > last_import_key:
                     files.append((key, fpath))
 
@@ -364,6 +393,7 @@ class HhImportTask(threading.Thread):
         if f is None:
             self.log("No new files found.")
             ftp.close()
+            client.close()
             self.log("Logged out.")
             return False
         else:

{chellow-1712909617.0.0 → chellow-1712929217.0.0}/chellow/e/views.py

@@ -1093,13 +1093,28 @@ def dc_contract_properties_edit_get(dc_contract_id):
     return render_template("dc_contract_properties_edit.html", dc_contract=dc_contract)
 
 
-@e.route("/dc_contracts/<int:contract_id>/properties/edit", methods=["POST"])
-def dc_contract_properties_edit_post(contract_id):
-    contract = Contract.get_dc_by_id(g.sess, contract_id)
-    properties = req_zish("properties")
-    contract.update(contract.name, contract.party, contract.charge_script, properties)
-    g.sess.commit()
-    return chellow_redirect(f"/dc_contracts/{contract.id}/properties", 303)
+@e.route("/dc_contracts/<int:dc_contract_id>/properties/edit", methods=["POST"])
+def dc_contract_properties_edit_post(dc_contract_id):
+    dc_contract = None
+    try:
+        dc_contract = Contract.get_dc_by_id(g.sess, dc_contract_id)
+        properties = req_zish("properties")
+        dc_contract.update(
+            dc_contract.name, dc_contract.party, dc_contract.charge_script, properties
+        )
+        g.sess.commit()
+        return chellow_redirect(f"/dc_contracts/{dc_contract.id}/properties", 303)
+    except BadRequest as e:
+        flash(e.description)
+        if dc_contract is None:
+            raise e
+        else:
+            return make_response(
+                render_template(
+                    "dc_contract_properties_edit.html", dc_contract=dc_contract
+                ),
+                400,
+            )
 
 
 @e.route("/dc_contracts/<int:dc_contract_id>/edit")

{chellow-1712909617.0.0 → chellow-1712929217.0.0}/chellow/templates/e/dc_contract_properties.html

@@ -6,7 +6,7 @@
 
 {% block nav %}
 	<a href="/e/dc_contracts">DC Contracts</a> &raquo;
-	<a href="/e/dc_contracts/{{dc_contract.id}}">dc_contract.name</a> &raquo;
+	<a href="/e/dc_contracts/{{dc_contract.id}}">{{dc_contract.name}}</a> &raquo;
 	Properties [<a href="/e/dc_contracts/{{dc_contract.id}}/properties/edit">edit</a>]
 {% endblock %}
 

{chellow-1712909617.0.0 → chellow-1712929217.0.0}/chellow/templates/e/dc_contract_properties_edit.html

@@ -18,7 +18,7 @@
 			<legend>Update Properties</legend>
 
 			<label>Properties</label>
-			{{ input_textarea('properties', dc_contract.properties, 40, 80) }}
+			{{ input_textarea('properties', dc_contract.properties, 40, 80, show_pos=True) }}
 
 			<input type="submit" value="Update">
 		</fieldset>
@@ -35,10 +35,11 @@
   "file_type": ".df2",
   "hostname": "example.com",
   "username": "username",
-  "password": "password",
-  "directories": ["downloads1", "downloads2"]
-  "mpan_map": {  \\ Optional
-    "99 0993 2821 985": null,  \\ Ignore MPAN
+  "password": "password", /* Remove if using private key */
+  "private_key": "private_key", /* Remove if using password */
+  "directories": ["downloads1", "downloads2"],
+  "mpan_map": {  /* Optional */
+    "99 0993 2821 985": null,  /* Ignore MPAN */
   },
 }
 		</pre>

{chellow-1712909617.0.0 → chellow-1712929217.0.0}/pyproject.toml

@@ -27,7 +27,7 @@ dependencies = [
 	"waitress==2.1.2",
 	"pep3143daemon==0.0.6",
 	"pip>=9.0.1",
-	"pysftp==0.2.9",
+	"paramiko==3.4.0",
 	"pympler==1.0.1",
 	"psutil==5.9.5",
 	"xlrd==2.0.1",

chellow-1712909617.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1712909617.0.0"