otter-service-stdalone 0.1.16__tar.gz → 0.1.18__tar.gz

{otter_service_stdalone-0.1.16 → otter_service_stdalone-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: otter_service_stdalone
-Version: 0.1.16
+Version: 0.1.18
 Summary: Grading Service for Instructors using Otter Grader
 Home-page: https://github.com/sean-morris/otter-service-stdalone
 Author: Sean Morris

{otter_service_stdalone-0.1.16 → otter_service_stdalone-0.1.18}/setup.cfg

@@ -23,7 +23,7 @@ python_requires = >=3.8
 where = src
 
 [options.package_data]
-otter_service_stdalone = index.html
+otter_service_stdalone = index.html, secrets/*.yaml, static_templates/*.html
 
 [options.entry_points]
 console_scripts = 

otter_service_stdalone-0.1.18/src/otter_service_stdalone/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.18"

{otter_service_stdalone-0.1.16 → otter_service_stdalone-0.1.18}/src/otter_service_stdalone/access_sops_keys.py

@@ -14,10 +14,10 @@ def get(course_key, key_name, sops_path=None, secrets_file=None) -> str:
     :return: the secret
     """
     try:
-        secret = get_via_env(course_key, key_name)
-        if secret is None:
-            secret = get_via_sops(course_key, key_name, sops_path=sops_path, secrets_file=secrets_file)
-        return secret
+        sec = get_via_env(course_key, key_name)
+        if sec is None:
+            sec = get_via_sops(course_key, key_name, sops_path=sops_path, secrets_file=secrets_file)
+        return sec
     except Exception as ex:
         raise Exception(f"Key not decrypted: {key_name}; please configure: Error: {ex}") from ex
 
@@ -38,8 +38,8 @@ def get_via_sops(course_key, key, sops_path=None, secrets_file=None):
 
         secrets_file = secrets_file
 
-        sops_output = subprocess.check_output([sops_path, "-d", secrets_file], stderr=subprocess.STDOUT)
-        dct = yaml.safe_load(sops_output)
+        sops_ot = subprocess.check_output([sops_path, "-d", secrets_file], stderr=subprocess.STDOUT)
+        dct = yaml.safe_load(sops_ot)
         if course_key is None:
             return dct[key]
         return dct[course_key][key]

otter_service_stdalone-0.1.18/src/otter_service_stdalone/app.py

@@ -0,0 +1,235 @@
+import tornado
+import tornado.ioloop
+import tornado.web
+import tornado.auth
+import os
+import uuid
+from otter_service_stdalone import fs_logging as log
+from otter_service_stdalone import user_auth as u_auth
+from otter_service_stdalone import grade_notebooks
+from zipfile import ZipFile, ZIP_DEFLATED
+
+
+__UPLOADS__ = "/tmp/uploads"
+log_debug = f'{os.environ.get("ENVIRONMENT")}-debug'
+log_error = f'{os.environ.get("ENVIRONMENT")}-logs'
+
+state = str(uuid.uuid4())  # used to protect against cross-site request forgery attacks.
+
+
+class LoginHandler(tornado.web.RequestHandler):
+    """Initiaties login auth by authorizing access to github auth api
+
+    Args:
+        tornado (tornado.web.RequestHandler): The request handler
+    """
+    async def get(self):
+        await u_auth.handle_authorization(self, state)
+
+
+class BaseHandler(tornado.web.RequestHandler):
+    """This is the super class for the handlers. get_current_user is called by
+    any handler that decorated with @tornado.web.authenticated
+
+    Args:
+        tornado (tornado.web.RequestHandler): The request handler
+    """
+    def get_current_user(self):
+        return self.get_secure_cookie("user")
+
+    def write_error(self, status_code, **kwargs):
+        log.write_logs("Http Error", f"{status_code} Error", "", "info", log_error)
+        if status_code == 403:
+            self.set_status(403)
+            self.render("static_templates/403.html")
+        else:
+            self.set_status(500)
+            self.render("static_templates/500.html")
+
+
+class GitHubOAuthHandler(BaseHandler):
+    """Handles GitHubOAuth
+
+    Args:
+        tornado (tornado.web.RequestHandler): The request handler
+    """
+    async def get(self):
+        code = self.get_argument('code', False)
+        arg_state = self.get_argument('state', False)
+        access_token = await u_auth.get_acess_token(arg_state, state, code)
+        if access_token:
+            user = await u_auth.get_github_username(access_token)
+            is_org_member = await u_auth.handle_is_org_member(access_token, user)
+            if is_org_member:
+                self.set_secure_cookie("user", user, expires_days=7)
+                self.redirect("/")
+            else:
+                raise tornado.web.HTTPError(403)
+        else:
+            raise tornado.web.HTTPError(500)
+
+
+class MainHandler(BaseHandler):
+    """This is the initial landing page for application
+
+    Args:
+        BaseHandler (BaseHandler): super class
+    """
+    @tornado.web.authenticated
+    async def get(self):
+        self.render("index.html", message=None)
+
+
+class Download(BaseHandler):
+    """The class handling a request to download results
+
+    Args:
+        tornado (tornado.web.RequestHandler): The download request handler
+    """
+    @tornado.web.authenticated
+    def get(self):
+        # this just redirects to login and displays main page
+        self.render("index.html", message=None)
+
+    @tornado.web.authenticated
+    async def post(self):
+        """the post method that accepts the code used to locate the results
+        the user wants to download
+        """
+        download_code = self.get_argument('download')
+        directory = f"{__UPLOADS__}/{download_code}"
+        if download_code == "":
+            m = "Download: Code Not Given!"
+            log.write_logs(download_code, m, f"{download_code}", "debug", log_debug)
+            msg = "Please enter the download code to see your result."
+            self.render("index.html",  download_message=msg)
+        elif not os.path.exists(f"{directory}"):
+            m = "Download: Directory for Code Not existing"
+            log.write_logs(download_code, m, f"{download_code}", "debug", log_debug)
+            msg = "The download code appears to not be correct or expired "
+            msg += f"- results are deleted regularly: {download_code}."
+            msg += "Please check the code or upload your notebooks "
+            msg += "and autograder.zip for grading again."
+            self.render("index.html",  download_message=msg)
+        elif not os.path.exists(f"{directory}/grading-logs.txt"):
+            m = "Download: Results Not Ready"
+            log.write_logs(download_code, m, f"{download_code}", "debug", log_debug)
+            msg = "The results of your download are not ready yet. "
+            msg += "Please check back."
+            self.render("index.html",  download_message=msg, dcode=download_code)
+        else:
+            if not os.path.isfile(f"{directory}/final_grades.csv"):
+                m = "Download: final_grades.csv does not exist"
+                t = "Problem grading notebooks see stack trace"
+                log.write_logs(download_code, m, t, "debug", log_debug)
+                with open(f"{directory}/final_grades.csv", "a") as f:
+                    m = "There was a problem grading your notebooks. Please see grading-logs.txt"
+                    f.write(m)
+                    f.close()
+            m = "Download Success: Creating results.zip"
+            log.write_logs(download_code, m, "", "debug", log_debug)
+            with ZipFile(f"{directory}/results.zip", 'w') as zipF:
+                for file in ["final_grades.csv", "grading-logs.txt"]:
+                    if os.path.isfile(f"{directory}/{file}"):
+                        zipF.write(f"{directory}/{file}", file, compress_type=ZIP_DEFLATED)
+
+            self.set_header('Content-Type', 'application/octet-stream')
+            self.set_header("Content-Description", "File Transfer")
+            m = f"attachment; filename=results-{download_code}.zip"
+            self.set_header('Content-Disposition', m)
+            with open(f"{directory}/results.zip", 'rb') as f:
+                try:
+                    while True:
+                        data = f.read(4096)
+                        if not data:
+                            break
+                        self.write(data)
+                    self.finish()
+                except Exception as exc:
+                    self.write(exc)
+
+
+class Upload(BaseHandler):
+    """This is the upload handler for users to upload autograder.zip and notebooks
+
+    Args:
+        tornado (tornado.web.RequestHandler): The upload request handler
+    """
+    @tornado.web.authenticated
+    def get(self):
+        # this just redirects to login and displays main page
+        self.render("index.html", message=None)
+
+    @tornado.web.authenticated
+    async def post(self):
+        """this handles the post request and asynchronously launches the grader
+        """
+        g = grade_notebooks.GradeNotebooks()
+        files = self.request.files
+        results_path = str(uuid.uuid4())
+        autograder = self.request.files['autograder'][0] if "autograder" in files else None
+        notebooks = self.request.files['notebooks'][0] if "notebooks" in files else None
+        log.write_logs(results_path, "Step 1: Upload accepted", "", "debug", log_debug)
+        if autograder is not None and notebooks is not None:
+            notebooks_fname = notebooks['filename']
+            notebooks_extn = os.path.splitext(notebooks_fname)[1]
+            notebooks_name = results_path + notebooks_extn
+            autograder_fname = autograder['filename']
+            autograder_extn = os.path.splitext(autograder_fname)[1]
+            autograder_name = str(uuid.uuid4()) + autograder_extn
+            if not os.path.exists(__UPLOADS__):
+                os.mkdir(__UPLOADS__)
+            auto_p = f"{__UPLOADS__}/{autograder_name}"
+            notebooks_path = f"{__UPLOADS__}/{notebooks_name}"
+            m = "Step 2a: Uploaded File Names Determined"
+            log.write_logs(results_path, m, f"notebooks path: {notebooks_path}", "debug", log_debug)
+            fh = open(auto_p, 'wb')
+            fh.write(autograder['body'])
+
+            fh = open(notebooks_path, 'wb')
+            fh.write(notebooks['body'])
+            m = "Step 3: Uploaded Files Written to Disk"
+            log.write_logs(results_path, m, f"Results Code: {results_path}", "debug", log_debug)
+            m = "Please save this code. You can retrieve your files by submitting this code "
+            m += f"in the \"Results\" section to the right: {results_path}"
+            self.render("index.html", message=m)
+            try:
+                await g.grade(auto_p, notebooks_path, results_path)
+            except Exception as e:
+                log.write_logs(results_path, "Grading Problem", str(e), "error", log_error)
+        else:
+            m = "Step 2b: Uploaded Files not given"
+            log.write_logs(results_path, m, "", "debug", log_debug)
+            m = "It looks like you did not set the notebooks or autograder.zip or both!"
+            self.render("index.html", message=m)
+
+
+settings = {
+    "cookie_secret": str(uuid.uuid4()),
+    "xsrf_cookies": True,
+    "login_url": "/login"
+}
+
+application = tornado.web.Application([
+        (r"/", MainHandler),
+        (r"/login", LoginHandler),
+        (r"/upload", Upload),
+        (r"/download", Download),
+        (r"/oauth_callback", GitHubOAuthHandler),
+        ], **settings, debug=False)
+
+
+def main():
+    """the web servers entry point
+    """
+    try:
+        application.listen(80)
+        log.write_logs("Server Start", "Starting Server", "", "info", log_debug)
+        tornado.ioloop.IOLoop.instance().start()
+    except Exception as e:
+        m = "Server Starting error"
+        log.write_logs("Server Start Error", m, str(e), "error", log_debug)
+
+
+if __name__ == "__main__":
+    main()

{otter_service_stdalone-0.1.16 → otter_service_stdalone-0.1.18}/src/otter_service_stdalone/fs_logging.py

@@ -50,7 +50,7 @@ def write_logs(id, msg, trace, type, collection):
         try:
             db = firestore.client()
             # this redirects FireStore to local emulator when local testing!
-            if os.getenv("ENVIRONMENT") == "otter-stdalone-docker-local-test":
+            if "local" in os.getenv("ENVIRONMENT"):
                 channel = grpc.insecure_channel("host.docker.internal:8080")
                 transport = firestore_grpc_transport.FirestoreGrpcTransport(channel=channel)
                 db._firestore_api_internal = firestore_client.FirestoreClient(transport=transport)

otter_service_stdalone-0.1.18/src/otter_service_stdalone/grade_notebooks.py

@@ -0,0 +1,103 @@
+import asyncio
+import async_timeout
+from otter_service_stdalone import fs_logging as log, upload_handle as uh
+import os
+
+log_debug = f'{os.environ.get("ENVIRONMENT")}-debug'
+log_count = f'{os.environ.get("ENVIRONMENT")}-count'
+log_error = f'{os.environ.get("ENVIRONMENT")}-logs'
+
+
+class GradeNotebooks():
+    """The class contains the async grade method for executing
+        otter grader as well as a function for logging the number of 
+        notebooks to be graded
+    """
+
+    def count_ipynb_files(self, directory, extension):
+        """this count the files for logging purposes"""
+        count = 0
+        for filename in os.listdir(directory):
+            if filename.endswith(extension):
+                count += 1
+        return count
+
+    async def grade(self, p, notebooks_path, results_id):
+        """Calls otter grade asynchronously and writes the various log files
+        and results of grading generating by otter-grader
+
+        Args:
+            p (str): the path to autograder.zip -- the solutions
+            notebooks_path (str): the path to the folder of notebooks to be graded
+            results_id (str): used for identifying logs
+
+        Raises:
+            Exception: Timeout Exception is raised if async takes longer than 20 min
+
+        Returns:
+            boolean: True is the process completes; otherwise an Exception is thrown
+        """
+        try:
+            notebook_folder = uh.handle_upload(notebooks_path, results_id)
+            notebook_count = self.count_ipynb_files(notebook_folder, ".ipynb")
+            log.write_logs(results_id, f"{notebook_count}",
+                           "",
+                           "info",
+                           f'{os.environ.get("ENVIRONMENT")}-count')
+            log.write_logs(results_id, "Step 5: Notebook Folder configured for grader",
+                           f"Notebook Folder: {notebook_folder}",
+                           "debug",
+                           log_debug)
+            command = [
+                'otter', 'grade',
+                '-a', p,
+                '-p', notebook_folder,
+                "--ext", "ipynb",
+                "--containers", "10",
+                "-o", notebook_folder,
+                "-v"
+            ]
+            log.write_logs(results_id, f"Step 6: Grading Start: {notebook_folder}",
+                           " ".join(command),
+                           "debug",
+                           log_debug)
+            process = await asyncio.create_subprocess_exec(
+                *command,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE
+            )
+
+            # this is waiting for communication back from the process
+            # some images are quite big and take some time to build the first
+            # time through - like 20 min for otter-grader
+            async with async_timeout.timeout(2000):
+                stdout, stderr = await process.communicate()
+
+                with open(f"{notebook_folder}/grading-output.txt", "w") as f:
+                    for line in stdout.decode().splitlines():
+                        f.write(line + "\n")
+                log.write_logs(results_id, "Step 7: Grading: Finished: Write: grading-output.txt",
+                               f"{notebook_folder}/grading-output.txt",
+                               "debug",
+                               log_debug)
+                with open(f"{notebook_folder}/grading-logs.txt", "w") as f:
+                    for line in stderr.decode().splitlines():
+                        f.write(line + "\n")
+                log.write_logs(results_id, "Step 8: Grading: Finished: Write grading-logs.txt",
+                               f"{notebook_folder}/grading-logs.txt",
+                               "debug",
+                               log_debug)
+                log.write_logs(results_id, f"Step 9: Grading: Finished: {notebook_folder}",
+                               " ".join(command),
+                               "debug",
+                               log_debug)
+                log.write_logs(results_id, f"Grading: Finished: {notebook_folder}",
+                               " ".join(command),
+                               "info",
+                               log_error)
+                return True
+        except asyncio.TimeoutError:
+            raise Exception(f'Grading timed out for {notebook_folder}')
+        except Exception as e:
+            raise e

otter_service_stdalone-0.1.18/src/otter_service_stdalone/secrets/gh_key.dev.yaml

@@ -0,0 +1,17 @@
+github_access_id: ENC[AES256_GCM,data:L97KjBroMsmVUYBOwIhGirB2nns=,iv:ZcBj74anvs43HziVBeQQRwl7hoEtX36+2wEQbFPNS0c=,tag:ptO+Tsx37xk1pWAGjctuhQ==,type:str]
+github_access_secret: ENC[AES256_GCM,data:5DuzOLRI8HwW5KhskZQW+HzwPiIHBlYcBCMpfAt34dZg9lOB+ivrcw==,iv:Y4H6MrJT1dnO1HsOy6XYDQfUERXCssu+7/FP5SEEnCw=,tag:UYp6KrHWG4XrxY8LroSO6Q==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops
+          created_at: "2024-04-03T18:29:45Z"
+          enc: CiUA67O9AArlsf5j4XCvPmf4qs3txe7+SxbVuJggNgW3GLh2DdACEkkAYHATHigxI73bJa6VfxZDxj5KLXsnz0ql72q7r7dzgSCwwjtFRrqizyloq2JvIoJskpjBcZBIe4VVVXHXsg8TUX/iMEB2G0Wm
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-04-03T18:29:46Z"
+    mac: ENC[AES256_GCM,data:QWZetePfWsqQQtCKc6O++f1falvJujzYMhxqRQFPmoUmQrFHBZ06v4XPPUciZGcrSgkhuyQwgHYXXSap1rDba2/D2i8jy8n8jC+hop60HLsveY53+ahYu3Fs91iQh3UdYg5ecTE0I/6prCdqVNL8nshYSBILYVNVG4KJAI7+G4s=,iv:4Beqcu1nYExqiv58Cs/YlCFGS1UV/MkAC5SWdmHr9Jw=,tag:h5PXnYsDtTFBFqkfGpPaJw==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1
+

otter_service_stdalone-0.1.18/src/otter_service_stdalone/secrets/gh_key.local.yaml

@@ -0,0 +1,16 @@
+github_access_id: ENC[AES256_GCM,data:MULsaNCCrl+bF/HvLO00HKfCw54=,iv:Wi4txCUYoM+Zi3CHTSjqSuGAwUToZdggDOVuaZndHLA=,tag:CtKnanwlrdXLnPs74ZsqOQ==,type:str]
+github_access_secret: ENC[AES256_GCM,data:h/cKH4nRjylg8ELxXsTvRuEuuB0P87R6SLqrbEBkQrjjgT+Bv/SVhQ==,iv:WmFvG6nijf9u0NE3ykJDaVxLdhbnDeiXbnexlYJDClE=,tag:HSO2CzLpCiqJJ9UdLn3+aw==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops
+          created_at: "2024-03-26T22:42:25Z"
+          enc: CiUA67O9AORb6b4pqLYE7IAfXT1a6ZHA9TycUT/TmUecInMxiq8aEkkAYHATHnnEuC/W1e4eckSfEz+yQe5Eu7CrX7X8f8DTBIFLLyu6tFgSKjr3VxitXNmBhOY5rQ53e+M/QbgjeJ2LexKjuWCFck+A
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-03-26T22:42:26Z"
+    mac: ENC[AES256_GCM,data:n0QV+da9B/zogfbFCjY94Sr+uaTZBUjyjIvqOh7nIf/OCTjLwN6dsJlF6QCLlWIChTS+hfPn+8gEmJpn7gTrJIYfzO77ByP2hIxu+CAjLGHJ4V3/QrY3odPPzbiRv6KXpVpCt/JWXGcoimKMeFStwtaUDDWqh/xDbb7UwciXFxE=,iv:GAML0oOeBpI2SUW3NkNNJbWlhGBP4AyjUlZVG+qnds0=,tag:+1xj3mMmL33pBSWT6NyAjw==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1

otter_service_stdalone-0.1.18/src/otter_service_stdalone/secrets/gh_key.prod.yaml

@@ -0,0 +1,16 @@
+github_access_id: ENC[AES256_GCM,data:c8f28uHdSyPTkYqGeYQLTVilBfM=,iv:xF06JH1aACwzifY6dLpGyNC/Gp4UfUrRHhHEKRq86RQ=,tag:crN2FEg5rcam9XlWO26Bag==,type:str]
+github_access_secret: ENC[AES256_GCM,data:paO5S9N9cn4/fiyG6P16550hL/r7dz8EHRMJLTaPdZgBtrHnaMq6HQ==,iv:qPAEDBcRJYyq0tPj7vPDPd+x1zhEhaTZm7TTOW3JNyU=,tag:9lKfrBt1Fnaq9T5J8jy4Hw==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops
+          created_at: "2024-04-03T18:24:30Z"
+          enc: CiUA67O9AKNwFJ3SWLlDu7J9GHFv6BLZeu15atOKMwR5Z1pXtaPEEkkAYHATHmUfzxyp7nC6ly2ICTtW6rd7b9qG4w2N2n4rlC+0/eT0MjTQIDB4WBa65YGfp+8dYQXS6S6sBTK+ozbX6nXtcinddCcr
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-04-03T18:24:31Z"
+    mac: ENC[AES256_GCM,data:kCdn15/6xjxE3YYJMNJJaLmP7gswC2gEPop+ptkyq+TPu1ufG6kGaCaDbXScr6w7Qe/RXnzdeOdTp/nF7nSRL9YRH6acR2RXcWuV6ngWFUM+YJXgiM6uDB1Ps+yn2OdzEa1v8LZo0gGRxraHdScqTxJ0bL7koFQBKiJf7CWdhD8=,iv:VoFzGQLkufxBFZVtGMxK9x2jF2jIpVPvjdWB8Z3Edo4=,tag:KCix8V1a7V9vCbt4A7kxtA==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1

otter_service_stdalone-0.1.18/src/otter_service_stdalone/secrets/gh_key.staging.yaml

@@ -0,0 +1,16 @@
+github_access_id: ENC[AES256_GCM,data:3R9AMcMccaaF+1z3zuu2aehmJHI=,iv:mnhhWTTXsbAJN0FdoFzpc8WrImYLSM1chk3lqwxFSkw=,tag:RSBoplL8Ve7MjTW6Zz7WMQ==,type:str]
+github_access_secret: ENC[AES256_GCM,data:+ebdQ0Jxy/bRe5plcuZZIrDoULu02jE+vj2HxoERYDG462YRONtiww==,iv:fHLqi4D1ixCdoOK5NASkkVLdX7kExnh8nOIOPaksnN0=,tag:9Jl2lJajSzO4UzJ/T+A8sw==,type:str]
+sops:
+    kms: []
+    gcp_kms:
+        - resource_id: projects/ucb-datahub-2018/locations/global/keyRings/datahub/cryptoKeys/sops
+          created_at: "2024-04-03T18:27:16Z"
+          enc: CiUA67O9AKF7k4zHdbhRYNzT1cKTzkS1by8G/1516HbJQBlOsnf2EkkAYHATHos+I2G4tn7GIRJhjrsXLeN++ZUUyICCn3VB1Wzcm2udHOfce4GGGc0XcQc18Nvn1Zr/ToroSv9qI8DarocbEHZX+H/l
+    azure_kv: []
+    hc_vault: []
+    age: []
+    lastmodified: "2024-04-03T18:27:17Z"
+    mac: ENC[AES256_GCM,data:Q/A3QkCxo1GdcSB/gtAyB3DFaXKrcAtpzXpe6GJYD8jD1v5N7rAW/DFo459r5jfAM+9W4Ks3cwTheBJgJnqxfHRuWp6FcP6GefNuScGsiRequp/FQwJcTMnAgLOGCsOGbSfzzsNVEm7j0VpkTkgavr/rFrqAlVC1MFT++bJsXTw=,iv:P86phogwEJL0ZMkBWH+qJWS8MyHbI1/0iM399cyuLxQ=,tag:MdO0PHXoe893Mk0GTfms+g==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.7.1

otter_service_stdalone-0.1.18/src/otter_service_stdalone/static_templates/403.html

@@ -0,0 +1,24 @@
+<html>
+<head> 
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> 
+    <title>403 Forbidden</title>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
+    <script defer src="https://use.fontawesome.com/releases/v5.3.1/js/all.js"></script>
+</head>
+<body>
+    <section class="container">
+        <div class="hero is-info welcome is-small">
+            <div class="hero-body">
+                <div class="container">
+                    <h1 class="title">403 Forbidden</h1>
+                    <p class="subtitle colored is-3">
+                        You do not have permission to access this resource or your authentication needs to be renewed. 
+                        Please <a style='color:#005b96' href="/login">login</a> to try again or 
+                        email sean.smorris@berkeley.edu for help.
+                    </p>
+                </div>
+            </div>
+        </div>
+    </section>
+</body>
+</html>

otter_service_stdalone-0.1.18/src/otter_service_stdalone/static_templates/500.html

@@ -0,0 +1,24 @@
+<html>
+<head> 
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> 
+    <title>Access Problem</title>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.4/css/bulma.min.css">
+    <script defer src="https://use.fontawesome.com/releases/v5.3.1/js/all.js"></script>
+</head>
+<body>
+    <section class="container">
+        <div class="hero is-info welcome is-small">
+            <div class="hero-body">
+                <div class="container">
+                    <h1 class="title">500 Forbidden - Access Problem</h1>
+                    <p class="subtitle colored is-3">
+                        We are unable to establish access for you.
+                        Please email: sean.smorris@berkeley.edu or try 
+                        <a style='color:#005b96' href="/login">logging</a> in again.
+                    </p>
+                </div>
+            </div>
+        </div>
+    </section>
+</body>
+</html>

otter_service_stdalone-0.1.18/src/otter_service_stdalone/user_auth.py

@@ -0,0 +1,120 @@
+import json
+import os
+from otter_service_stdalone import fs_logging as log, access_sops_keys
+import requests
+import tornado
+import urllib.parse
+
+
+log_coll = f'{os.environ.get("ENVIRONMENT")}-debug'
+environment_name = os.environ.get("ENVIRONMENT").split("-")[-1]
+gh_key_path = os.path.join(os.path.dirname(__file__), f"secrets/gh_key.{environment_name}.yaml")
+github_id = access_sops_keys.get(None, "github_access_id", secrets_file=gh_key_path)
+github_secret = access_sops_keys.get(None, "github_access_secret", secrets_file=gh_key_path)
+
+
+async def handle_authorization(form, state):
+    """authorizes via oauth app token access github auth api
+
+    Parameters:
+        - form (tornado.web.RequestHandler): The request handler used to re-direct for
+          authorization. The redirect url is configured at the github endpoint for the app
+        - state (int): random uuid4 number generated to ensure communication between endpoints is
+          not compromised
+    """
+    log.write_logs("Auth Workflow", "UserAuth: Get: Authorizing", "", "info", log_coll)
+    q_params = f"client_id={github_id}&state={state}&scope=read:org"
+    form.redirect(f'https://github.com/login/oauth/authorize?{q_params}')
+
+
+async def get_acess_token(arg_state, state, code):
+    """requests and returns the access token or None
+
+    Parameters:
+        - arg_state (int): the state argument being passed on the url string; this will be compared
+          to the state value generated in this file to ensure they are the same!
+        - state (int): random uuid4 number generated to ensure communication between endpoints is
+          not compromised
+        - code (str): the code that is returned from the authorization request
+
+    Returns:
+        - str: the access token used for subsequent calls to the api; or None
+    """
+    http_client = tornado.httpclient.AsyncHTTPClient()
+    params = {
+        'client_id': github_id,
+        'client_secret': github_secret,
+        'code': code,
+        'redirect_uri': f"{os.environ.get('GRADER_DNS')}/oauth_callback"
+    }
+    m = "UserAuth: GitHubOAuthHandler: Getting Access Token"
+    log.write_logs("Auth Workflow", m, "", "info", log_coll)
+    response = await http_client.fetch(
+        'https://github.com/login/oauth/access_token',
+        method='POST',
+        headers={'Accept': 'application/json'},
+        body=urllib.parse.urlencode(params)
+    )
+    resp = json.loads(response.body.decode())
+    access_token = resp['access_token']
+    if arg_state != state:
+        access_token = None
+        m = "UserAuth: GitHubOAuthHandler: Cross-Site Forgery possible - aborting"
+        log.write_logs("Auth Workflow", m, "", "info", log_coll)
+    else:
+        m = "UserAuth: GitHubOAuthHandler: Access Token Granted"
+        log.write_logs("Auth Workflow", m, "", "info", log_coll)
+    return access_token
+
+
+async def handle_is_org_member(access_token, user):
+    """the final authorization is to make sure the member has access to the application by being
+    a part of the correct organizaton
+
+    Parameters:
+        - access_token (str): The OAuth access token.
+        - user (str): the authenticated GitHub user name
+
+    Returns:
+        - boolean: True user is in the GH org, False otherwise
+    """
+    log.write_logs("Auth Workflow", "UserAuth: Get: Check Membership", "", "info", log_coll)
+    if user:
+        org_name = os.environ.get("AUTH_ORG")
+        url = f'https://api.github.com/orgs/{org_name}/members/{user}'
+        headers = {
+            'Authorization': f'token {access_token}',
+            'Accept': 'application/vnd.github.v3+json',
+        }
+        response = requests.get(url, headers=headers)
+        return response.status_code == 204
+    else:
+        return False
+
+
+async def get_github_username(access_token):
+    """
+    Retrieve the GitHub username of the authenticated user.
+
+    Parameters:
+    - access_token (str): The OAuth access token.
+
+    Returns:
+    - str: The username of the authenticated user, or None if not found.
+    """
+    url = 'https://api.github.com/user'
+    headers = {
+        'Authorization': f'token {access_token}',
+        'Accept': 'application/vnd.github.v3+json',
+    }
+
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        m = "UserAuth: Get: UserName - Success"
+        log.write_logs("Auth Workflow", m, "", "info", log_coll)
+        user_info = response.json()
+        return user_info.get('login')
+    else:
+        m = f"UserAuth: Get: UserName - Fail:{response.status_code}"
+        log.write_logs("Auth Workflow", m, "", "info", log_coll)
+        return None

{otter_service_stdalone-0.1.16 → otter_service_stdalone-0.1.18}/src/otter_service_stdalone.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: otter_service_stdalone
-Version: 0.1.16
+Version: 0.1.18
 Summary: Grading Service for Instructors using Otter Grader
 Home-page: https://github.com/sean-morris/otter-service-stdalone
 Author: Sean Morris

{otter_service_stdalone-0.1.16 → otter_service_stdalone-0.1.18}/src/otter_service_stdalone.egg-info/SOURCES.txt

@@ -5,11 +5,19 @@ src/otter_service_stdalone/__init__.py
 src/otter_service_stdalone/access_sops_keys.py
 src/otter_service_stdalone/app.py
 src/otter_service_stdalone/fs_logging.py
+src/otter_service_stdalone/grade_notebooks.py
 src/otter_service_stdalone/index.html
 src/otter_service_stdalone/upload_handle.py
+src/otter_service_stdalone/user_auth.py
 src/otter_service_stdalone.egg-info/PKG-INFO
 src/otter_service_stdalone.egg-info/SOURCES.txt
 src/otter_service_stdalone.egg-info/dependency_links.txt
 src/otter_service_stdalone.egg-info/entry_points.txt
 src/otter_service_stdalone.egg-info/top_level.txt
+src/otter_service_stdalone/secrets/gh_key.dev.yaml
+src/otter_service_stdalone/secrets/gh_key.local.yaml
+src/otter_service_stdalone/secrets/gh_key.prod.yaml
+src/otter_service_stdalone/secrets/gh_key.staging.yaml
+src/otter_service_stdalone/static_templates/403.html
+src/otter_service_stdalone/static_templates/500.html
 tests/test_upload_handle.py

otter_service_stdalone-0.1.16/src/otter_service_stdalone/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.16"

otter_service_stdalone-0.1.16/src/otter_service_stdalone/app.py

@@ -1,296 +0,0 @@
-import tornado
-import tornado.ioloop
-import tornado.web
-import tornado.auth
-import os
-import uuid
-from otter_service_stdalone import fs_logging as log, upload_handle as uh
-from zipfile import ZipFile, ZIP_DEFLATED
-import asyncio
-import async_timeout
-import urllib.parse
-from otter_service_stdalone import access_sops_keys
-import json
-
-__UPLOADS__ = "/tmp/uploads"
-
-
-class GradeNotebooks():
-    """The class contains the async grade method for executing
-        otter grader
-    """
-    async def grade(self, p, notebooks_path, results_id):
-        """Calls otter grade asynchronously and writes the various log files
-        and results of grading generating by otter-grader
-
-        Args:
-            p (str): the path to autograder.zip -- the solutions
-            notebooks_path (str): the path to the folder of notebooks to be graded
-            results_id (str): used for identifying logs
-
-        Raises:
-            Exception: Timeout Exception is raised if async takes longer than 20 min
-
-        Returns:
-            boolean: True is the process completes; otherwise an Exception is thrown
-        """
-        try:
-            notebook_folder = uh.handle_upload(notebooks_path, results_id)
-            log.write_logs(results_id, "Step 5: Notebook Folder configured for grader",
-                           f"Notebook Folder: {notebook_folder}",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            command = [
-                'otter', 'grade',
-                '-a', p,
-                '-p', notebook_folder,
-                "--ext", "ipynb",
-                "--containers", "10",
-                "-o", notebook_folder,
-                "-v"
-            ]
-            log.write_logs(results_id, f"Step 6: Grading Start: {notebook_folder}",
-                           " ".join(command),
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            process = await asyncio.create_subprocess_exec(
-                *command,
-                stdin=asyncio.subprocess.PIPE,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE
-            )
-
-            # this is waiting for communication back from the process
-            # some images are quite big and take some time to build the first
-            # time through - like 20 min for otter-grader
-            async with async_timeout.timeout(2000):
-                stdout, stderr = await process.communicate()
-
-                with open(f"{notebook_folder}/grading-output.txt", "w") as f:
-                    for line in stdout.decode().splitlines():
-                        f.write(line + "\n")
-                log.write_logs(results_id, "Step 7: Grading: Finished: Write: grading-output.txt",
-                               f"{notebook_folder}/grading-output.txt",
-                               "debug",
-                               f'{os.environ.get("ENVIRONMENT")}-debug')
-                with open(f"{notebook_folder}/grading-logs.txt", "w") as f:
-                    for line in stderr.decode().splitlines():
-                        f.write(line + "\n")
-                log.write_logs(results_id, "Step 8: Grading: Finished: Write grading-logs.txt",
-                               f"{notebook_folder}/grading-logs.txt",
-                               "debug",
-                               f'{os.environ.get("ENVIRONMENT")}-debug')
-                log.write_logs(results_id, f"Step 9: Grading: Finished: {notebook_folder}",
-                               " ".join(command),
-                               "debug",
-                               f'{os.environ.get("ENVIRONMENT")}-debug')
-                log.write_logs(results_id, f"Grading: Finished: {notebook_folder}",
-                               " ".join(command),
-                               "info",
-                               f'{os.environ.get("ENVIRONMENT")}-logs')
-                return True
-        except asyncio.TimeoutError:
-            raise Exception(f'Grading timed out for {notebook_folder}')
-        except Exception as e:
-            raise e
-
-
-class Userform(tornado.web.RequestHandler):
-    """This is the initial landing page for application
-
-    Args:
-        tornado (tornado.web.RequestHandler): The request handler
-    """
-    async def get(self):
-        # User will be redirected here by GitHub after authorization
-        code = self.get_argument('code', False)
-        if code:
-            access_token = await self.get_authenticated_user(code)
-            self.write(f'Your GitHub access token is: {access_token}')
-        else:
-            self.render("index.html", message=None)
-  
-    async def get_authenticated_user(self, code):
-        secrets_file = os.path.join(os.path.dirname(__file__), "secrets/gh_key.yaml")
-        github_id = access_sops_keys.get(None, "github_access_id", secrets_file=secrets_file)
-        github_secret = access_sops_keys.get(None, "github_access_secret", secrets_file=secrets_file)
-
-        http_client = tornado.httpclient.AsyncHTTPClient()
-        params = {
-            'client_id': github_id,
-            'client_secret': github_secret,
-            'code': code,
-            'redirect_uri': "https://grader.datahub.berkeley.edu/oauth_callback"
-        }
-        response = await http_client.fetch(
-            'https://github.com/login/oauth/access_token',
-            method='POST',
-            headers={'Accept': 'application/json'},
-            body=urllib.parse.urlencode(params)
-        )
-        return json.loads(response.body.decode())['access_token']
-
-
-class Download(tornado.web.RequestHandler):
-    """The class handling a request to download results
-
-    Args:
-        tornado (tornado.web.RequestHandler): The download request handler
-    """
-    async def post(self):
-        """the post method that accepts the code used to locate the results
-        the user wants to download
-        """
-        code = self.get_argument('download')
-        directory = f"{__UPLOADS__}/{code}"
-        if code == "":
-            log.write_logs(code, "Download: Code Not Given!",
-                           f"{code}",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            msg = "Please enter the download code to see your result."
-            self.render("index.html",  download_message=msg)
-        elif not os.path.exists(f"{directory}"):
-            log.write_logs(code, "Download: Directory for Code Not existing",
-                           f"{code}",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            msg = "The download code appears to not be correct or expired "
-            msg += f"- results are deleted regularly: {code}."
-            msg += "Please check the code or upload your notebooks "
-            msg += "and autograder.zip for grading again."
-            self.render("index.html",  download_message=msg)
-        elif not os.path.exists(f"{directory}/grading-logs.txt"):
-            log.write_logs(code, "Download: Results Not Ready",
-                           f"{code}",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            msg = "The results of your download are not ready yet. "
-            msg += "Please check back."
-            self.render("index.html",  download_message=msg, dcode=code)
-        else:
-            if not os.path.isfile(f"{directory}/final_grades.csv"):
-                log.write_logs(code, "Download: final_grades.csv does not exist",
-                               "Problem grading notebooks see stack trace",
-                               "debug",
-                               f'{os.environ.get("ENVIRONMENT")}-debug')
-                with open(f"{directory}/final_grades.csv", "a") as f:
-                    m = "There was a problem grading your notebooks. Please see grading-logs.txt"
-                    f.write(m)
-                    f.close()
-
-            log.write_logs(code, "Download Success: Creating results.zip",
-                           "",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            with ZipFile(f"{directory}/results.zip", 'w') as zipF:
-                for file in ["final_grades.csv", "grading-logs.txt"]:
-                    if os.path.isfile(f"{directory}/{file}"):
-                        zipF.write(f"{directory}/{file}", file, compress_type=ZIP_DEFLATED)
-
-            self.set_header('Content-Type', 'application/octet-stream')
-            self.set_header("Content-Description", "File Transfer")
-            self.set_header('Content-Disposition', f"attachment; filename=results-{code}.zip")
-            with open(f"{directory}/results.zip", 'rb') as f:
-                try:
-                    while True:
-                        data = f.read(4096)
-                        if not data:
-                            break
-                        self.write(data)
-                    self.finish()
-                except Exception as exc:
-                    self.write(exc)
-
-
-class Upload(tornado.web.RequestHandler):
-    """This is the upload handler for users to upload autograder.zip and notebooks
-
-    Args:
-        tornado (tornado.web.RequestHandler): The upload request handler
-    """
-    async def post(self):
-        """this handles the post request and asynchronously launches the grader
-        """
-        g = GradeNotebooks()
-        files = self.request.files
-        results_path = str(uuid.uuid4())
-        autograder = self.request.files['autograder'][0] if "autograder" in files else None
-        notebooks = self.request.files['notebooks'][0] if "notebooks" in files else None
-        log.write_logs(results_path, "Step 1: Upload accepted",
-                       "",
-                       "debug",
-                       f'{os.environ.get("ENVIRONMENT")}-debug')
-        if autograder is not None and notebooks is not None:
-            notebooks_fname = notebooks['filename']
-            notebooks_extn = os.path.splitext(notebooks_fname)[1]
-            notebooks_name = results_path + notebooks_extn
-            autograder_fname = autograder['filename']
-            autograder_extn = os.path.splitext(autograder_fname)[1]
-            autograder_name = str(uuid.uuid4()) + autograder_extn
-            if not os.path.exists(__UPLOADS__):
-                os.mkdir(__UPLOADS__)
-            auto_p = f"{__UPLOADS__}/{autograder_name}"
-            notebooks_path = f"{__UPLOADS__}/{notebooks_name}"
-            log.write_logs(results_path, "Step 2a: Uploaded File Names Determined",
-                           f"notebooks path: {notebooks_path}",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            fh = open(auto_p, 'wb')
-            fh.write(autograder['body'])
-
-            fh = open(notebooks_path, 'wb')
-            fh.write(notebooks['body'])
-            log.write_logs(results_path, "Step 3: Uploaded Files Written to Disk",
-                           f"Results Code: {results_path}",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            m = "Please save this code. You can retrieve your files by submitting this code "
-            m += f"in the \"Results\" section to the right: {results_path}"
-            self.render("index.html", message=m)
-            try:
-                await g.grade(auto_p, notebooks_path, results_path)
-            except Exception as e:
-                log.write_logs(results_path, "Grading Problem",
-                               str(e),
-                               "error",
-                               f'{os.environ.get("ENVIRONMENT")}-logs')
-        else:
-            log.write_logs(results_path, "Step 2b: Uploaded Files not given",
-                           "",
-                           "debug",
-                           f'{os.environ.get("ENVIRONMENT")}-debug')
-            m = "It looks like you did not set the notebooks or autograder.zip or both!"
-            self.render("index.html", message=m)
-
-
-settings = {
-    "cookie_secret": str(uuid.uuid4()),
-    "xsrf_cookies": True
-}
-
-application = tornado.web.Application([
-        (r"/", Userform),
-        (r"/upload", Upload),
-        (r"/download", Download),
-        (r"/auth/github", Userform),
-        ], **settings, debug=True)
-
-
-def main():
-    """the web servers entry point
-    """
-    try:
-        application.listen(80)
-        msg = f'{os.environ.get("ENVIRONMENT")}-debug'
-        log.write_logs("Server Start", "Starting Server", "", "info", msg)
-        tornado.ioloop.IOLoop.instance().start()
-    except Exception as e:
-        log.write_logs("Server Start Error", "Server Starting error",
-                       str(e),
-                       "error",
-                       f'{os.environ.get("ENVIRONMENT")}-debug')
-
-
-if __name__ == "__main__":
-    main()