numerapi 2.15.0__tar.gz → 2.16.0__tar.gz

{numerapi-2.15.0 → numerapi-2.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: numerapi
-Version: 2.15.0
+Version: 2.16.0
 Summary: Automatically download and upload data for the Numerai machine learning competition
 Home-page: https://github.com/uuazed/numerapi
 Maintainer: uuazed

{numerapi-2.15.0 → numerapi-2.16.0}/numerapi/base_api.py

@@ -705,6 +705,134 @@ class Api:
         utils.replace(results, "updatedAt", utils.parse_datetime_string)
         return results
 
+    def round_model_performances_v2(self, model_id: str):
+        """Fetch round model performance of a user.
+
+        Args:
+            model_id (str)
+
+        Returns:
+            list of dicts: list of round model performance entries
+
+            For each entry in the list, there is a dict with the following
+            content:
+
+                * atRisk (`float`)
+                * corrMultiplier (`float` or None)
+                * tcMultiplier (`float` or None)
+                * roundNumber (`int`)
+                * roundOpenTime (`datetime`)
+                * roundResolveTime (`datetime`)
+                * roundResolved (`bool`)
+                * roundTarget (`str`)
+                * submissionScores (`dict`)
+                    * date (`datetime`)
+                    * day (`int`)
+                    * displayName (`str`): name of the metric
+                    * payoutPending (`float`)
+                    * payoutSettled (`float`)
+                    * percentile (`float`)
+                    * value (`float`): value of the metric
+        """
+
+        query = """
+          query($modelId: String!
+                $tournament: Int!) {
+            v2RoundModelPerformances(modelId: $modelId
+                                     tournament: $tournament) {
+                atRisk
+                corrMultiplier
+                tcMultiplier
+                roundNumber,
+                roundOpenTime,
+                roundResolveTime,
+                roundResolved,
+                roundTarget,
+                submissionScores {
+                    date,
+                    day,
+                    displayName,
+                    payoutPending,
+                    payoutSettled,
+                    percentile,
+                    value
+                }
+            }
+          }
+        """
+        arguments = {'modelId': model_id, 'tournament': self.tournament_id}
+        data = self.raw_query(query, arguments)['data']
+        performances = data['v2RoundModelPerformances']
+        for perf in performances:
+            utils.replace(perf, "roundOpenTime", utils.parse_datetime_string)
+            utils.replace(perf, "roundResolveTime", utils.parse_datetime_string)
+            utils.replace(perf, "atRisk", utils.parse_float_string)
+            if perf["submissionScores"]:
+                for submission in perf["submissionScores"]:
+                    utils.replace(
+                        submission, "date", utils.parse_datetime_string)
+                    utils.replace(
+                        submission, "payoutPending", utils.parse_float_string)
+                    utils.replace(
+                        submission, "payoutSettled",utils.parse_float_string)
+        return performances
+
+    def intra_round_scores(self, model_id: str):
+        """Fetch intra-round scores for your model.
+
+        While only the final scores are relevant for payouts, it might be
+        interesting to look how your scores evolve throughout a round.
+
+        Args:
+            model_id (str)
+
+        Returns:
+            list of dicts: list of intra-round model performance entries
+
+            For each entry in the list, there is a dict with the following
+            content:
+
+                * roundNumber (`int`)
+                * intraRoundSubmissionScores (`dict`)
+                    * date (`datetime`)
+                    * day (`int`)
+                    * displayName (`str`): name of the metric
+                    * payoutPending (`float`)
+                    * payoutSettled (`float`)
+                    * percentile (`float`)
+                    * value (`float`): value of the metric
+        """
+
+        query = """
+          query($modelId: String!
+                $tournament: Int!) {
+            v2RoundModelPerformances(modelId: $modelId
+                                     tournament: $tournament) {
+                roundNumber,
+                intraRoundSubmissionScores {
+                    date,
+                    day,
+                    displayName,
+                    payoutPending,
+                    payoutSettled,
+                    percentile,
+                    value
+                }
+            }
+          }
+        """
+        arguments = {'modelId': model_id, 'tournament': self.tournament_id}
+        data = self.raw_query(query, arguments)['data']
+        performances = data['v2RoundModelPerformances']
+        for perf in performances:
+            if perf["intraRoundSubmissionScores"]:
+                for score in perf["intraRoundSubmissionScores"]:
+                    utils.replace(score, "date", utils.parse_datetime_string)
+                    fun = utils.parse_float_string
+                    utils.replace(score, "payoutPending", fun)
+                    utils.replace(score, "payoutSettled", fun)
+        return performances
+
     def round_model_performances(self, username: str) -> List[Dict]:
         """Fetch round model performance of a user.
 
@@ -790,7 +918,8 @@ class Api:
         else:
             raise ValueError("round_model_performances is not available for ",
                              f"tournament {self.tournament_id}")
-
+        self.logger.warning(
+            "Deprecated soon. Checkout round_model_performances_v2.")
         query = f"""
           query($username: String!) {{
             {endpoint}(modelName: $username) {{
@@ -1120,15 +1249,62 @@ class Api:
         res = self.raw_query(query, arguments, authorization=True)
         return res['data']['model']["name"]
 
+    def pipeline_status(self, date: str = None) -> Dict:
+        """Get status of Numerai's scoring pipeline
+
+        Args:
+            date (str, optional): date in YYYY-MM-DD format. Defaults to today.
+
+        Returns:
+            dict: pipeline status information including the following fields:
+                * dataReadyAt (`str`)
+                * isScoringDay (`bool`)
+                * resolvedAt (`datetime`)
+                * scoredAt (`datetime`)
+                * startedAt (`datetime`)
+                * tournament (`str`)
+
+        Example:
+            >>> napi = NumerAPI()
+            >>> napi.pipeline_status()
+        """
+        if date is None:
+            date = datetime.date.today().isoformat()
+        tournament = "classic" if self.tournament_id == 8 else "signals"
+        query = """
+            query($tournament: String! $date: String) {
+                pipelineStatus(date: $date, tournament: $tournament) {
+                    dataReadyAt
+                    isScoringDay
+                    resolvedAt
+                    scoredAt
+                    startedAt
+                    tournament
+                }
+            }
+        """
+        arguments = {'tournament': tournament, "date": date}
+        res = self.raw_query(query, arguments)["data"]["pipelineStatus"]
+        for field in res.keys():
+            if field.endswith("At"):
+                utils.replace(res, field, utils.parse_datetime_string)
+        return res
+
     def model_upload(self, file_path: str,
                  tournament: int = None,
-                 model_id: str = None) -> str:
+                 model_id: str = None,
+                 data_version: str = None,
+                 docker_image: str = None) -> str:
         """Upload pickled model to numerai.
 
         Args:
             file_path (str): pickle file, needs to endwith .pkl
             tournament (int): ID of the tournament (optional)
             model_id (str): Target model UUID
+            data_version (str, optional): which data version to use. ID or name.
+                    Check available options with 'model_upload_data_versions'
+            docker_image (str, optional): which docker image to use. ID or name.
+                    Check available options with 'model_upload_docker_images'
 
         Returns:
             str: model_upload_id
@@ -1139,6 +1315,23 @@ class Api:
             >>> api.model_upload("example.pkl", model_id=model_id)
             '93c46857-fed9-4594-981e-82db2b358daf'
         """
+        if data_version is not None:
+            if not utils.is_valid_uuid(data_version):
+                data_versions = self.model_upload_data_versions()
+                if data_version not in data_versions:
+                    msg = "'data_version' needs to be one of"
+                    msg += f"{list(data_versions.keys())}"
+                    raise ValueError(msg)
+                data_version = data_versions[data_version]
+        if docker_image is not None:
+            if not utils.is_valid_uuid(docker_image):
+                docker_images = self.model_upload_docker_images()
+                if docker_image not in docker_images:
+                    msg = "'docker_image' needs to be one of"
+                    msg += f"{list(docker_images.keys())}"
+                    raise ValueError(msg)
+                docker_image = docker_images[docker_image]
+
         auth_query = '''
             query($filename: String! $modelId: String) {
                 computePickleUploadAuth(filename: $filename
@@ -1159,16 +1352,157 @@ class Api:
         create_query = '''
             mutation($filename: String!
                      $tournament: Int!
-                     $modelId: String) {
+                     $modelId: String
+                     $dataVersionId: String
+                     $dockerImageId: String) {
                 createComputePickleUpload(filename: $filename
                                           tournament: $tournament
-                                          modelId: $modelId) {
+                                          modelId: $modelId
+                                          dataVersionId: $dataVersionId
+                                          dockerImageId: $dockerImageId) {
                     id
                 }
             }'''
+
         tournament = self.tournament_id if tournament is None else tournament
         arguments = {'filename': upload_auth['filename'],
                      'tournament': tournament,
-                     'modelId': model_id}
+                     'modelId': model_id,
+                     'dataVersionId': data_version,
+                     'dockerImageId': docker_image}
         create = self.raw_query(create_query, arguments, authorization=True)
         return create['data']['createComputePickleUpload']['id']
+
+    def model_upload_data_versions(self) -> Dict:
+        """ Get available data version for model uploads
+
+        Returns:
+            dict[str, str]: name to ID mapping
+
+        Example:
+            >>> api = NumerAPI(secret_key="..", public_id="..")
+            >>> api.model_upload_data_versions()
+            {'v4.1': 'a76bafa1-b25a-4f22-9add-65b528a0f3d0'}
+
+        """
+        query = '''
+            query {
+                computePickleDataVersions {
+                    name
+                    id
+                }
+            }
+        '''
+        data = self.raw_query(query, authorization=True)['data']
+        res = {
+            item["name"]: item["id"]
+            for item in data["computePickleDataVersions"]}
+        return res
+
+    def model_upload_docker_images(self) -> Dict:
+        """ Get available docker images for model uploads
+
+        Returns:
+            dict[str, str]: name to ID mapping
+
+        Example:
+            >>> api = NumerAPI(secret_key="..", public_id="..")
+            >>> api.model_upload_docker_images()
+            {'Python 3.10': 'c72ae05e-2831-4c50-b20f-c2fe01c206ef',
+             'Python 3.9': '5a32b827-cd9a-40a9-a99d-e58401120a0b',
+               ...
+            }
+        """
+        query = '''
+            query {
+                computePickleDockerImages {
+                    name
+                    id
+                }
+            }
+        '''
+        data = self.raw_query(query, authorization=True)['data']
+        res = {
+            item["name"]: item["id"]
+            for item in data["computePickleDockerImages"]}
+        return res
+
+    def submission_ids(self, model_id: str):
+        """ Get all submission ids from a model
+
+        Args:
+            model_id (str)
+
+        Returns:
+            list of dicts: list of submissions
+
+            For each entry in the list, there is a dict with the following
+            content:
+
+                * insertedAt (`datetime`)
+                * filename (`str`)
+                * id (`str`)
+
+         Example:
+            >>> api = NumerAPI(secret_key="..", public_id="..")
+            >>> model_id = napi.get_models()["uuazed"]
+            >>> api.submission_ids(model_id)
+        """
+        query = """
+            query($modelId: String) {
+                submissions(modelId: $modelId) {
+                    id
+                    filename
+                    insertedAt
+                }
+            }
+        """
+        raw = self.raw_query(query, {"modelId": model_id}, authorization=True)
+        data = raw["data"]["submissions"]
+        utils.replace(data, "insertedAt", utils.parse_datetime_string)
+        return data
+
+    def download_submission(self, submission_id: str = None,
+                            model_id: str = None, dest_path: str = None) -> str:
+        """ Download previous submissions from numerai
+
+        Args:
+            submission_id (str, optional): the submission to be downloaded
+            model_id (str, optional): if provided, the latest submission of that
+                                      model gets downloaded
+            dest_path (str, optional): where to save the downloaded file
+
+        Returns:
+            str: path to downloaded file
+
+        Example:
+            # fetch latest submission
+            >>> api = NumerAPI(secret_key="..", public_id="..")
+            >>> model_id = napi.get_models()["uuazed"]
+            >>> api.download_submission(model_id=model_id)
+            # fetch older submssion
+            >>> ids = submission_ids(model_id)
+            >>> import random; submission_id = random.choice(ids)["id"]
+            >>> api.download_submission(submission_id=submission_id)
+        """
+        msg = "You need to provide one of `model_id` and `submission_id"
+        assert model_id or submission_id, msg
+        auth_query = '''
+            query($id: String) {
+                submissionDownloadAuth(id: $id) {
+                    filename
+                    url
+                }
+            }
+        '''
+        if not submission_id:
+            ids = self.submission_ids(model_id)
+            submission_id = max(ids, key=lambda x: x['insertedAt'])["id"]
+
+        data = self.raw_query(
+            auth_query, {'id': submission_id},
+            authorization=True)['data']["submissionDownloadAuth"]
+        if dest_path is None:
+            dest_path = data["filename"]
+        path = utils.download_file(data["url"], dest_path)
+        return path

{numerapi-2.15.0 → numerapi-2.16.0}/numerapi/numerapi.py

@@ -71,13 +71,13 @@ class NumerAPI(base_api.Api):
         args = {'round': round_num}
         return self.raw_query(query, args)['data']['listDatasets']
 
-    def download_dataset(self, filename: str = "numerai_live_data.csv",
+    def download_dataset(self, filename: str = None,
                          dest_path: str = None,
                          round_num: int = None) -> None:
         """ Download specified file for the given round.
 
         Args:
-            filename (str): file to be downloaded, defaults to live data
+            filename (str, optional): file to be downloaded
             dest_path (str, optional): complete path where the file should be
                 stored, defaults to the same name as the source file
             round_num (int, optional): tournament round you are interested in.

{numerapi-2.15.0 → numerapi-2.16.0}/numerapi/utils.py

@@ -5,6 +5,7 @@ import decimal
 import logging
 import time
 import datetime
+import uuid
 import json
 from typing import Optional, Dict
 
@@ -103,6 +104,8 @@ def post_with_err_handling(url: str, body: str, headers: Dict,
             resp = requests.post(url, json=body,
                                  headers=headers, timeout=timeout)
         resp.raise_for_status()
+        return resp.json()
+
     except requests.exceptions.HTTPError as err:
         logger.error(f"Http Error: {err}")
     except requests.exceptions.ConnectionError as err:
@@ -111,13 +114,16 @@ def post_with_err_handling(url: str, body: str, headers: Dict,
         logger.error(f"Timeout Error: {err}")
     except requests.exceptions.RequestException as err:
         logger.error(f"Oops, something went wrong: {err}")
-
-    try:
-        return resp.json()
-    except UnboundLocalError:
-        # `r` isn't available, probably because the try/except above failed
-        pass
     except json.decoder.JSONDecodeError as err:
         logger.error(f"Did not receive a valid JSON: {err}")
 
     return {}
+
+
+def is_valid_uuid(val: str) -> bool:
+    """ check if the given string is a valid UUID """
+    try:
+        uuid.UUID(str(val))
+        return True
+    except ValueError:
+        return False

{numerapi-2.15.0 → numerapi-2.16.0}/numerapi.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: numerapi
-Version: 2.15.0
+Version: 2.16.0
 Summary: Automatically download and upload data for the Numerai machine learning competition
 Home-page: https://github.com/uuazed/numerapi
 Maintainer: uuazed

{numerapi-2.15.0 → numerapi-2.16.0}/setup.py

@@ -6,7 +6,7 @@ def load(path):
     return open(path, 'r').read()
 
 
-numerapi_version = '2.15.0'
+numerapi_version = '2.16.0'
 
 
 classifiers = [