chellow 1712909617.0.0__py3-none-any.whl → 1712929217.0.0__py3-none-any.whl

chellow/e/hh_importer.py

@@ -8,13 +8,13 @@ import traceback
 from collections import defaultdict, deque
 from datetime import datetime as Datetime, timedelta as Timedelta
 from decimal import Decimal
-from io import TextIOWrapper
+from io import StringIO, TextIOWrapper
+from pathlib import Path
 
 import jinja2
 
-import paramiko
+from paramiko import AutoAddPolicy, RSAKey, SFTPError, SSHClient
 
-import pysftp
 
 import requests
 
@@ -174,7 +174,7 @@ class HhImportTask(threading.Thread):
             self.log(f"Problem {e}")
             sess.rollback()
             self.is_error = True
-        except Exception:
+        except BaseException:
             self.log(f"Unknown Exception {traceback.format_exc()}")
             sess.rollback()
             self.is_error = True
@@ -231,19 +231,19 @@ class HhImportTask(threading.Thread):
         fl = None
 
         for directory in directories:
-            self.log("Checking the directory '" + directory + "'.")
+            self.log(f"Checking the directory '{directory}'.")
             try:
                 last_import_key = last_import_keys[directory]
             except KeyError:
                 last_import_key = last_import_keys[directory] = ""
 
             dir_path = home_path + "/" + directory
-            ftp.cwd(dir_path)
+            ftp.chdir(dir_path)
             files = []
-            for fname in ftp.nlst():
+            for fname in ftp.listdir():
                 fpath = dir_path + "/" + fname
                 try:
-                    ftp.cwd(fpath)
+                    ftp.chdir(fpath)
                     continue  # directory
                 except ftplib.error_perm:
                     pass
@@ -266,7 +266,7 @@ class HhImportTask(threading.Thread):
             key, fpath = fl
             self.log(f"Attempting to download {fpath} with key {key}.")
             f = tempfile.TemporaryFile()
-            ftp.retrbinary("RETR " + fpath, f.write)
+            ftp.retrbinary(f"RETR {fpath}", f.write)
             self.log("File downloaded successfully.")
             ftp.quit()
             self.log("Logged out.")
@@ -290,25 +290,30 @@ class HhImportTask(threading.Thread):
             contract = Contract.get_dc_by_id(sess, self.contract_id)
             contract.update_state(state)
             sess.commit()
-            self.log("Finished loading '" + fpath)
+            self.log(f"Finished loading '{fpath}'")
             return True
 
     def sftp_handler(self, sess, properties, contract):
-        host_name = properties["hostname"]
-        user_name = properties["username"]
-        password = properties["password"]
-        try:
-            port = properties["port"]
-        except KeyError:
-            port = None
+        hostname = properties["hostname"]
+        username = properties["username"]
+        password = properties.get("password")
+        private_key = properties.get("private_key")
+        pkey = None
+        if private_key is not None:
+            pkf = StringIO.StringIO(private_key)
+            pkf.seek(0)
+            pkey = RSAKey.from_private_key(pkf)
+
+        port = properties.get("port", 22)
         file_type = properties["file_type"]
         directories = properties["directories"]
+        known_hosts = properties["known_hosts"].strip()
         state = contract.make_state()
 
         try:
             last_import_keys = state["last_import_keys"]
         except KeyError:
-            last_import_keys = state["last_import_keys"] = []
+            last_import_keys = state["last_import_keys"] = {}
 
         ct_now = ct_datetime_now()
         try:
@@ -323,36 +328,60 @@ class HhImportTask(threading.Thread):
             latest_imports = state["latest_imports"] = []
 
         sess.rollback()
-        self.log(f"Connecting to sftp server at {host_name}:{port}.")
-        cnopts = pysftp.CnOpts()
-        cnopts.hostkeys = None
-        ftp = pysftp.Connection(
-            host_name, username=user_name, password=password, cnopts=cnopts
+        self.log(f"Connecting to sftp server at {hostname}:{port}.")
+
+        client = SSHClient()
+        client.set_missing_host_key_policy(AutoAddPolicy)
+
+        client.connect(
+            hostname,
+            port=port,
+            username=username,
+            password=password,
+            pkey=pkey,
+            key_filename=None,
+            timeout=120,
+            banner_timeout=120,
+            auth_timeout=120,
+            channel_timeout=120,
+            look_for_keys=False,
         )
-        ftp.timeout = 120
-        home_path = ftp.pwd
+        host_keys = client.get_host_keys()
+        with tempfile.TemporaryDirectory() as tempdir:
+            td = Path(tempdir)
+            hfname = td / "hosts"
+            host_keys.save(hfname)
+            with hfname.open() as hf:
+                host_keys_str = hf.read().strip()
+
+        if host_keys_str != known_hosts:
+            raise BadRequest(
+                f"The returned host keys {host_keys_str} don't match the known "
+                f"hosts {known_hosts} property"
+            )
+
+        ftp = client.open_sftp()
 
         f = None
 
         for directory in directories:
-            self.log("Checking the directory '" + directory + "'.")
+            self.log(f"Checking the directory '{directory}'.")
             try:
                 last_import_key = last_import_keys[directory]
             except KeyError:
                 last_import_key = last_import_keys[directory] = ""
 
-            dir_path = home_path + "/" + directory
-            ftp.cwd(dir_path)
+            ftp.chdir(directory)
             files = []
             for attr in ftp.listdir_attr():
-                fpath = dir_path + "/" + attr.filename
+                fpath = f"{directory}/{attr.filename}"
                 try:
-                    ftp.cwd(fpath)
+                    ftp.chdir(fpath)
                     continue  # directory
-                except paramiko.SFTPError:
+                except SFTPError:
                     pass
 
-                key = str(attr.st_mtime) + "_" + attr.filename
+                key = f"{attr.st_mtime}_{attr.filename}"
                 if key > last_import_key:
                     files.append((key, fpath))
 
@@ -364,6 +393,7 @@ class HhImportTask(threading.Thread):
         if f is None:
             self.log("No new files found.")
             ftp.close()
+            client.close()
             self.log("Logged out.")
             return False
         else:

chellow/e/views.py

@@ -1093,13 +1093,28 @@ def dc_contract_properties_edit_get(dc_contract_id):
     return render_template("dc_contract_properties_edit.html", dc_contract=dc_contract)
 
 
-@e.route("/dc_contracts/<int:contract_id>/properties/edit", methods=["POST"])
-def dc_contract_properties_edit_post(contract_id):
-    contract = Contract.get_dc_by_id(g.sess, contract_id)
-    properties = req_zish("properties")
-    contract.update(contract.name, contract.party, contract.charge_script, properties)
-    g.sess.commit()
-    return chellow_redirect(f"/dc_contracts/{contract.id}/properties", 303)
+@e.route("/dc_contracts/<int:dc_contract_id>/properties/edit", methods=["POST"])
+def dc_contract_properties_edit_post(dc_contract_id):
+    dc_contract = None
+    try:
+        dc_contract = Contract.get_dc_by_id(g.sess, dc_contract_id)
+        properties = req_zish("properties")
+        dc_contract.update(
+            dc_contract.name, dc_contract.party, dc_contract.charge_script, properties
+        )
+        g.sess.commit()
+        return chellow_redirect(f"/dc_contracts/{dc_contract.id}/properties", 303)
+    except BadRequest as e:
+        flash(e.description)
+        if dc_contract is None:
+            raise e
+        else:
+            return make_response(
+                render_template(
+                    "dc_contract_properties_edit.html", dc_contract=dc_contract
+                ),
+                400,
+            )
 
 
 @e.route("/dc_contracts/<int:dc_contract_id>/edit")

chellow/templates/e/dc_contract_properties.html

@@ -6,7 +6,7 @@
 
 {% block nav %}
 	<a href="/e/dc_contracts">DC Contracts</a> &raquo;
-	<a href="/e/dc_contracts/{{dc_contract.id}}">dc_contract.name</a> &raquo;
+	<a href="/e/dc_contracts/{{dc_contract.id}}">{{dc_contract.name}}</a> &raquo;
 	Properties [<a href="/e/dc_contracts/{{dc_contract.id}}/properties/edit">edit</a>]
 {% endblock %}
 

chellow/templates/e/dc_contract_properties_edit.html

@@ -18,7 +18,7 @@
 			<legend>Update Properties</legend>
 
 			<label>Properties</label>
-			{{ input_textarea('properties', dc_contract.properties, 40, 80) }}
+			{{ input_textarea('properties', dc_contract.properties, 40, 80, show_pos=True) }}
 
 			<input type="submit" value="Update">
 		</fieldset>
@@ -35,10 +35,11 @@
   "file_type": ".df2",
   "hostname": "example.com",
   "username": "username",
-  "password": "password",
-  "directories": ["downloads1", "downloads2"]
-  "mpan_map": {  \\ Optional
-    "99 0993 2821 985": null,  \\ Ignore MPAN
+  "password": "password", /* Remove if using private key */
+  "private_key": "private_key", /* Remove if using password */
+  "directories": ["downloads1", "downloads2"],
+  "mpan_map": {  /* Optional */
+    "99 0993 2821 985": null,  /* Ignore MPAN */
   },
 }
 		</pre>

{chellow-1712909617.0.0.dist-info → chellow-1712929217.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1712909617.0.0
+Version: 1712929217.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -11,13 +11,13 @@ Requires-Dist: flask-restx==1.2.0
 Requires-Dist: flask==2.3.3
 Requires-Dist: odio==0.0.22
 Requires-Dist: openpyxl==3.1.2
+Requires-Dist: paramiko==3.4.0
 Requires-Dist: pep3143daemon==0.0.6
 Requires-Dist: pg8000==1.31.1
 Requires-Dist: pip>=9.0.1
 Requires-Dist: psutil==5.9.5
 Requires-Dist: pympler==1.0.1
 Requires-Dist: pypdf==3.17.0
-Requires-Dist: pysftp==0.2.9
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2022.6
 Requires-Dist: requests==2.31.0

{chellow-1712909617.0.0.dist-info → chellow-1712929217.0.0.dist-info}/RECORD

@@ -24,7 +24,7 @@ chellow/e/dno_rate_parser.py,sha256=5MYEbcYngh7n6Szn2h9zLTwQKXH-WHKy6c80PYwNLLQ,
 chellow/e/duos.py,sha256=ISTcNqe9KNjVNM2Qs8IBoQxnmSXOt5W_G7tZxp4T78M,28870
 chellow/e/elexon.py,sha256=ALhXS9Es7PV0z9ukPbIramn3cf3iLyFi-PMWPSm5iOs,5487
 chellow/e/energy_management.py,sha256=aXC2qlGt3FAODlNl_frWzVYAQrJLP8FFOiNX3m-QE_Y,12388
-chellow/e/hh_importer.py,sha256=nhmlWDNBm8U4salBud8ivdaDGF1lkY82NqQCrGuPYRI,20483
+chellow/e/hh_importer.py,sha256=QBKdnT3Z4A6fsGqODE9bBGVdWde6Hu8OJfkXVhShQz0,21485
 chellow/e/hh_parser_bg_csv.py,sha256=W5SU2MSpa8BGA0VJw1JXF-IwbCNLFy8fe35yxLZ7gEw,2453
 chellow/e/hh_parser_df2.py,sha256=ynNNRMpLm9qs23tJjfm0vGSaAduXfPMw_7-WfQHXliQ,4209
 chellow/e/hh_parser_simple_csv.py,sha256=RN4QOLvTQeoPrpvXvQ9hkOBZnR5piybLfjCiSJdjpjs,2112
@@ -38,7 +38,7 @@ chellow/e/system_price.py,sha256=3cPWohHmmBI9v7fENLBzXQkpAeC3r0s5xV29Nmr6k_E,583
 chellow/e/tlms.py,sha256=kHAy2A2d1Mma7x4GdNDyrzscJd5DpJtDBYiZEg241Dw,8538
 chellow/e/tnuos.py,sha256=KoMPJDIXfE4zwhSDuywGF1ooxjTYLVjtF7BkSFm6X24,4158
 chellow/e/triad.py,sha256=S6LEMHvUKhAZe0-yfLIRciYDZ8IKMn1jh1TmmsbQD3s,13588
-chellow/e/views.py,sha256=EdNlMI-cutMhWiog_PUBOW1jWu-u4MLOCHxv1uyIGVQ,213550
+chellow/e/views.py,sha256=8Rmm6XuNGXACFSSX_LpomyTC4oS2-nljRlisjEyUVJ4,213975
 chellow/e/bill_parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/bill_parsers/activity_mop_stark_xlsx.py,sha256=UgWXDPzQkQghyj_lfgBqoSJpHB-t-qOdSaB8qY6GLog,4071
 chellow/e/bill_parsers/annual_mop_stark_xlsx.py,sha256=-HMoIfa_utXYKA44RuC0Xqv3vd2HLeQU_4P0iBUd3WA,4219
@@ -187,8 +187,8 @@ chellow/templates/e/dc_contract.html,sha256=b6DLDrGdzN9Rri56pFvfpE7QZKYXGHib2veY
 chellow/templates/e/dc_contract_edit.html,sha256=YraU0_MTBjmzBCMdsuvtIYm7K58hnDly5wUZ-6n8XzY,2148
 chellow/templates/e/dc_contract_hh_import.html,sha256=UODiBFNohb60MjH1w-9JW1JE0O9GR2uKPGw-lD7Da5g,848
 chellow/templates/e/dc_contract_hh_imports.html,sha256=eXFDGyzSgag4JRism81_p5yTzQOjCIXaVkQ8tl3dDcM,8172
-chellow/templates/e/dc_contract_properties.html,sha256=fOHCBeAI7k9yi9vBCtaYDpYRB9zk1WuMaO1pnNCMxes,451
-chellow/templates/e/dc_contract_properties_edit.html,sha256=852aWHaDsIa_VZdn9sLQzjP7ynueXa0CKL4ZHaOBmmM,1585
+chellow/templates/e/dc_contract_properties.html,sha256=2EmA91gYWZcTOHQ3PxXQrpTOb8dm0OGjwRWsW6qNI9s,455
+chellow/templates/e/dc_contract_properties_edit.html,sha256=TGYcHTlWbk5WmjZPkdbHhRVZcOpEfhowCp2penZcGiE,1704
 chellow/templates/e/dc_contracts.html,sha256=v8czpEcMzndngfWNXIKXib6Xrz9OwJjQAmNQNrQ6Y08,1705
 chellow/templates/e/dc_contracts_add.html,sha256=2lrGrNqAoKp16OiMoNDNlJxBMW3QdSPfgEFheSg826s,710
 chellow/templates/e/dc_rate_script.html,sha256=gfKUHV2IMsTQT-cYfjl4aiEBqGEWSj3uW22UZpoyGq4,1219
@@ -363,6 +363,6 @@ chellow/templates/g/supply_note_edit.html,sha256=6UQf_qbhFDys3cVsTp-c7ABWZpggW9R
 chellow/templates/g/supply_notes.html,sha256=WR3YwGh_qqTklSJ7JqWX6BKBc9rk_jMff4RiWZiF2CM,936
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1712909617.0.0.dist-info/METADATA,sha256=qV7hEsZpgKlO6cp3iu5A8YJhHVmh28JRt1sVz42GH64,12203
-chellow-1712909617.0.0.dist-info/WHEEL,sha256=as-1oFTWSeWBgyzh0O_qF439xqBe6AbBgt4MfYe5zwY,87
-chellow-1712909617.0.0.dist-info/RECORD,,
+chellow-1712929217.0.0.dist-info/METADATA,sha256=y7dYnKInV3Z2jmy6Z1aKrqsJ-hwDU4ncat88dfZKzmo,12205
+chellow-1712929217.0.0.dist-info/WHEEL,sha256=as-1oFTWSeWBgyzh0O_qF439xqBe6AbBgt4MfYe5zwY,87
+chellow-1712929217.0.0.dist-info/RECORD,,