kaggle 1.7.3b1__py3-none-any.whl

kagglesdk/kaggle_http_client.py

@@ -0,0 +1,316 @@
+import binascii
+import codecs
+import json
+import os
+import urllib.parse
+from io import BytesIO
+
+import requests
+from urllib3.fields import RequestField
+
+from kagglesdk.kaggle_env import get_endpoint, get_env, KaggleEnv
+from kagglesdk.kaggle_object import KaggleObject
+from typing import Type
+
+# TODO (http://b/354237483) Generate the client from the existing one.
+# This was created from kaggle_api_client.py, prior to recent changes to
+# auth handling. The new client requires KAGGLE_API_TOKEN, so it is not
+# currently usable by the CLI.
+
+# TODO: Extend kapigen to add a boolean to these requests indicating that they use forms.
+REQUESTS_REQUIRING_FORMS = ['ApiUploadDatasetFileRequest', 'ApiCreateSubmissionRequest', 'ApiStartSubmissionUploadRequest', 'ApiUploadModelFileRequest']
+
+def _headers_to_str(headers):
+  return '\n'.join(f'{k}: {v}' for k, v in headers.items())
+
+
+def _get_apikey_creds():
+  apikey_filename = os.path.expanduser('~/.kaggle/kaggle.json')
+  if not os.path.exists(apikey_filename):
+    return None
+
+  kaggle_json = None
+  with open(apikey_filename) as apikey_file:
+    kaggle_json = apikey_file.read()
+
+  if not kaggle_json or not kaggle_json.strip():
+    return None
+
+  api_key_data = json.loads(kaggle_json)
+  username = api_key_data['username']
+  api_key = api_key_data['key']
+  return username, api_key
+
+
+def clean_data(data):
+  if isinstance(data, dict):
+    return {to_lower_camel_case(k): clean_data(v) for k, v in data.items() if v is not None}
+  if isinstance(data, list):
+    return [clean_data(v) for v in data if v is not None]
+  if data is True:
+    return 'true'
+  if data is False:
+    return 'false'
+  return data
+
+def find_words(source, left='{', right='}'):
+  words = []
+  split_str = source.split(left)
+
+  for s in split_str[1:]:
+    split_s = s.split(right)
+    if len(split_s) > 1:
+      words.append(split_s[0])
+
+  return words
+
+def to_camel_case(snake_str):
+  return "".join(x.capitalize() for x in snake_str.lower().split("_"))
+
+def to_lower_camel_case(snake_str):
+  # https://stackoverflow.com/questions/19053707/converting-snake-case-to-lower-camel-case-lowercamelcase
+  # We capitalize the first letter of each component except the first one
+  # with the 'capitalize' method and join them together.
+  camel_string = to_camel_case(snake_str)
+  return snake_str[0].lower() + camel_string[1:]
+
+class KaggleHttpClient(object):
+  _xsrf_cookie_name = 'XSRF-TOKEN'
+  _csrf_cookie_name = "CSRF-TOKEN"
+  _xsrf_cookies = (_xsrf_cookie_name, _csrf_cookie_name)
+  _xsrf_header_name = 'X-XSRF-TOKEN'
+
+  def __init__(self,
+               env: KaggleEnv = None,
+               verbose: bool = False,
+               renew_iap_token=None,
+               username=None,
+               password=None):
+    self._env = env or get_env()
+    self._signed_in = None
+    self._endpoint = get_endpoint(self._env)
+    self._verbose = verbose
+    self._session = None
+    self._username = username
+    self._password = password
+
+  def call(self, service_name: str, request_name: str, request: KaggleObject,
+           response_type: Type[KaggleObject]):
+    self._init_session()
+    http_request = self._prepare_request(service_name, request_name, request)
+
+    http_response = self._session.send(http_request)
+
+    response = self._prepare_response(response_type, http_response)
+    return response
+
+  def _prepare_request(self, service_name: str, request_name: str,
+                       request: KaggleObject):
+    request_url = self._get_request_url(request)
+    method = request.method()
+    data= ''
+    if method == 'GET':
+      data = request.__class__.to_dict(request, ignore_defaults=False)
+      if request.endpoint_path():
+        words = find_words(request.endpoint_path())
+        list(map(data.pop, [to_lower_camel_case(w) for w in words]))
+        if len(data) == 0:
+          data = None
+      if data:
+        request_url = f'{request_url}?{urllib.parse.urlencode(clean_data(data))}'
+      data = ''
+      self._session.headers.update({
+        'Accept': 'application/json',
+        'Content-Type': 'text/plain',
+        })
+    elif method == 'POST':
+      data = request.to_field_map(request, ignore_defaults=True)
+      if isinstance(data, dict):
+        fields = request.body_fields()
+        if fields is not None:
+          if fields != '*':
+            data = data[fields]
+        data = clean_data(data)
+        if self.requires_form(request):
+          data, content_type = self.make_form(data)
+        else:
+          content_type = 'application/json'
+          data = json.dumps(data)
+        self._session.headers.update({
+          'Accept': 'application/json',
+          'Content-Type': content_type,
+          })
+    http_request = requests.Request(
+      method=method,
+      url=request_url,
+      data=data,
+      headers=self._session.headers,
+      # cookies=self._get_xsrf_cookies(),
+      auth=self._session.auth)
+    prepared_request = http_request.prepare()
+    self._print_request(prepared_request)
+    return prepared_request
+
+  def _get_xsrf_cookies(self):
+    cookies = requests.cookies.RequestsCookieJar()
+    for cookie in self._session.cookies:
+      if cookie.name in KaggleHttpClient._xsrf_cookies:
+        cookies[cookie.name] = cookie.value
+    return cookies
+
+  def _prepare_response(self, response_type, http_response):
+    self._print_response(http_response)
+    http_response.raise_for_status()
+    if 'application/json' in http_response.headers['Content-Type']:
+      resp = http_response.json()
+      if 'code' in resp and resp['code'] >= 400:
+        raise requests.exceptions.HTTPError(
+          resp['message'], response=http_response)
+    if response_type is None:  # Method doesn't have a return type
+      return None
+    return response_type.prepare_from(http_response)
+
+  def _print_request(self, request):
+    if not self._verbose:
+      return
+    self._print('---------------------Request----------------------')
+    self._print(
+      f'{request.method} {request.url}\n{_headers_to_str(request.headers)}\n\n{request.body}'
+      )
+    self._print('--------------------------------------------------')
+
+  def _print_response(self, response, body=True):
+    if not self._verbose:
+      return
+    self._print('---------------------Response---------------------')
+    self._print(f'{response.status_code}\n{_headers_to_str(response.headers)}')
+    if body:
+      self._print(f'\n{response.text}')
+    self._print('--------------------------------------------------')
+
+  def _print(self, message: str):
+    if self._verbose:
+      print(message)
+
+  def __enter__(self):
+    self._init_session()
+    return self
+
+  def __exit__(self, exc_type, exc_value, tb):
+    if self._session is not None:
+      self._session.close()
+
+  def _init_session(self):
+    if self._session is not None:
+      return self._session
+
+    self._session = requests.Session()
+    self._session.headers.update({
+      'User-Agent': 'kaggle-api/v1.7.0',  # Was: V2
+      'Content-Type': 'application/x-www-form-urlencoded',  # Was: /json
+      })
+
+    iap_token = self._get_iap_token_if_required()
+    if iap_token is not None:
+      self._session.headers.update({
+        # https://cloud.google.com/iap/docs/authentication-howto#authenticating_from_proxy-authorization_header
+        'Proxy-Authorization': f'Bearer {iap_token}',
+        })
+
+    self._try_fill_auth()
+    # self._fill_xsrf_token(iap_token)  # TODO Make this align with original handler.
+
+  def _get_iap_token_if_required(self):
+    if self._env not in (KaggleEnv.STAGING, KaggleEnv.ADMIN):
+      return None
+    iap_token = os.getenv('KAGGLE_IAP_TOKEN')
+    if iap_token is None:
+      raise Exception(f'Must set KAGGLE_IAP_TOKEN to access "{self._endpoint}"')
+    return iap_token
+
+  def _fill_xsrf_token(self, iap_token):
+    initial_get_request = requests.Request(
+      method='GET',
+      url=self._endpoint,
+      headers=self._session.headers,
+      auth=self._session.auth)
+    prepared_request = initial_get_request.prepare()
+    self._print_request(prepared_request)
+
+    http_response = self._session.send(prepared_request)
+
+    self._print_response(http_response, body=False)
+    if iap_token is not None and http_response.status_code in (401, 403):
+      raise requests.exceptions.HTTPError('IAP token invalid or expired')
+    http_response.raise_for_status()
+
+    self._session.headers.update({
+      KaggleHttpClient._xsrf_header_name:
+        self._session.cookies[KaggleHttpClient._xsrf_cookie_name],
+      })
+
+  class BearerAuth(requests.auth.AuthBase):
+
+    def __init__(self, token):
+      self.token = token
+
+    def __call__(self, r):
+      r.headers["Authorization"] = f"Bearer {self.token}"
+      return r
+
+  def _try_fill_auth(self):
+    if self._signed_in is not None:
+      return
+
+    api_token = os.getenv('KAGGLE_API_TOKEN')
+    if api_token is not None:
+      self._session.auth = KaggleHttpClient.BearerAuth(api_token)
+      self._signed_in = True
+      return
+
+    if self._username and self._password:
+      apikey_creds = self._username, self._password
+    else:
+      apikey_creds = _get_apikey_creds()
+    if apikey_creds is not None:
+      self._session.auth = apikey_creds
+      self._signed_in = True
+      return
+
+    self._signed_in = False
+
+  def _get_request_url(self, request):
+    return f'{self._endpoint}{request.endpoint()}'
+
+  @staticmethod
+  def make_form(fields):
+    body = BytesIO()
+    boundary = binascii.hexlify(os.urandom(16)).decode()
+    writer = codecs.lookup("utf-8")[3]
+
+    for field in fields.items():
+      field = RequestField.from_tuples(*field)
+      body.write(f"--{boundary}\r\n".encode("latin-1"))
+
+      writer(body).write(field.render_headers())
+      data = field.data
+
+      if isinstance(data, int):
+        data = str(data)
+
+      if isinstance(data, str):
+        writer(body).write(data)
+      else:
+        body.write(data)
+
+      body.write(b"\r\n")
+
+    body.write(f"--{boundary}--\r\n".encode("latin-1"))
+
+    content_type = f"multipart/form-data; boundary={boundary}"
+
+    return body.getvalue(), content_type
+
+  @staticmethod
+  def requires_form(request):
+    return type(request).__name__ in REQUESTS_REQUIRING_FORMS

kagglesdk/kaggle_object.py

@@ -0,0 +1,293 @@
+import json
+import re
+from datetime import datetime, timedelta
+from google.protobuf.field_mask_pb2 import FieldMask
+
+
+class ObjectSerializer(object):
+  def __init__(self, to_dict_value, from_dict_value):
+    self.to_dict_value = to_dict_value
+    self.from_dict_value = from_dict_value
+
+
+class PredefinedSerializer(ObjectSerializer):
+  def __init__(self):
+    """Predefined objects such as int, float etc are serialized/deserialized directly."""
+    ObjectSerializer.__init__(self, lambda cls, v, _: v, lambda cls, v: v)
+
+
+# Adapted from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case
+_pascal_to_upper_snake_case_regex = re.compile('((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))')
+
+
+def _pascal_case_to_upper_snake_case(string):
+  return _pascal_to_upper_snake_case_regex.sub(r'_\1', string).upper()
+
+
+def _convert (camel_input):
+  words = re.findall(r'[A-Z]?[a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|\d+', camel_input)
+  return '_'.join(map(str.lower, words))
+
+
+class EnumSerializer(ObjectSerializer):
+  def __init__(self):
+    """
+    Enum objects are serialized using their ".name" field and deserialized by indexing the string in the Enum type.
+    Example:
+    class Foo(Enum):
+       TEST = 1
+    foo = Foo.TEST
+    foo.name # => returns "TEST"
+    Foo["TEST"] # => returns Foo.TEST enum value.
+    """
+    ObjectSerializer.__init__(self,
+                              lambda cls, v, _: EnumSerializer._to_str(cls, v),
+                              lambda cls, v: EnumSerializer._from_str(cls, v))
+
+  @staticmethod
+  def _to_str(cls, v):
+    # "v" corresponds to an enum instance: Example foo or Foo.Test above.
+    # "cls" corresponds to the enum type Foo above.
+    #enum_prefix = f'{_pascal_case_to_upper_snake_case(cls.__name__)}_'
+    #if v.name.startswith(enum_prefix):
+    #  return v.name
+    #return f'{enum_prefix}{v.name}'
+    enum_prefix = f'{_pascal_case_to_upper_snake_case(cls.__name__)}_'
+    if v.name.find(enum_prefix) == 0:
+      return v.name[len(enum_prefix):].lower()
+    return v.name
+
+  @staticmethod
+  def _from_str(cls, v):
+    # "v" corresponds to enum string: Example "TEST" above.
+    # "cls" corresponds to the enum type Foo above.
+    # enum_items = {item.name: item for item in cls}
+    # if v in enum_items:
+    #   return enum_items[v]
+    # 
+    # # Try with enum prefix. Example: EnvironmentType.JSON -> "ENVIRONMENT_TYPE_JSON"
+    # enum_prefix = _pascal_case_to_upper_snake_case(cls.__name__)
+    # if v.startswith(enum_prefix):
+    #   ix_start = len(enum_prefix) + 1
+    #   return enum_items[v[ix_start:]]
+    # 
+    # return enum_items[f'{enum_prefix}_{v}']
+    try:
+      return cls[v]
+    except KeyError:
+      dct = vars(cls)
+      n = v.lower()
+      nn = _convert(v).lower()
+      enum_prefix = _pascal_case_to_upper_snake_case(cls.__name__).lower()
+      for key in dct.keys():
+        k = key.lower()
+        if k == n:
+          return dct[key]
+        if k.startswith(enum_prefix) and k.endswith(n) or k.endswith(nn):
+          return dct[key]
+      raise
+
+
+class ListSerializer(ObjectSerializer):
+  def __init__(self, item_serializer: ObjectSerializer):
+    """
+    Lists are serialized based on the type they contain. Since objects are generated from proto files, a list always
+    contains objects of the same type, which is serialized/deserialized using "item_serializer".
+    """
+    ObjectSerializer.__init__(self,
+                              lambda cls, l, ignore_defaults: [item_serializer.to_dict_value(cls, v, ignore_defaults) for v in l],
+                              lambda cls, l: [item_serializer.from_dict_value(cls, v) for v in l])
+
+
+class MapSerializer(ObjectSerializer):
+  def __init__(self, item_serializer: ObjectSerializer):
+    """
+    Maps are serialized based on type of their values. Since maps keys are always predefined types, we don't need a
+    serializer for them.
+    """
+    ObjectSerializer.__init__(self,
+                              lambda cls, d, ignore_defaults: {k: item_serializer.to_dict_value(cls, v, ignore_defaults) for k, v in d.items()},
+                              lambda cls, d: {k: item_serializer.from_dict_value(cls, v) for k, v in d.items()})
+
+
+class DateTimeSerializer(ObjectSerializer):
+  def __init__(self):
+    """Date times are serialized/deserialized as a string in iso format"""
+    ObjectSerializer.__init__(self,
+                              lambda cls, dt, _: DateTimeSerializer._to_str(dt),
+                              lambda _, v: DateTimeSerializer._from_str(v))
+
+  @staticmethod
+  def _to_str(dt):
+    return dt.isoformat(timespec='milliseconds') + 'Z'
+
+  @staticmethod
+  def _from_str(v):
+    v = v.rstrip('Z')
+    fields = v.rsplit('.', maxsplit=1)
+    if len(fields) == 1:
+      return datetime.fromisoformat(v)
+    (dt, nanos) = fields
+    millis = nanos[:3]
+    try:
+      return datetime.fromisoformat(f'{dt}.{millis}')
+    except ValueError:
+      return datetime.fromisoformat(dt) # Python 3.9, 3.10
+
+
+class TimeDeltaSerializer(ObjectSerializer):
+  def __init__(self):
+    """Time deltas are serialized/deserialized as a string in "mm:ss" format"""
+    ObjectSerializer.__init__(self,
+                              lambda cls, t, _: TimeDeltaSerializer._to_dict_value(t),
+                              lambda cls, v: TimeDeltaSerializer._from_dict_value(v))
+
+  @staticmethod
+  def _to_dict_value(delta):
+    seconds = int(delta.total_seconds())
+    minutes = seconds // 60
+    seconds -= minutes * 60
+    return '{}:{:02}'.format(int(minutes), int(seconds))
+
+  @staticmethod
+  def _from_dict_value(value):
+    (minutes, seconds) = value.split(':')
+    return timedelta(minutes=int(minutes), seconds=int(seconds))
+
+
+class FieldMaskSerializer(ObjectSerializer):
+  def __init__(self):
+    """Field masks are serialized/deserialized as a string that contains a list of paths with a comma delimiter"""
+    ObjectSerializer.__init__(self,
+                              lambda cls, m, _: m.ToJsonString(),
+                              lambda cls, v: FieldMaskSerializer._from_joined_paths(v))
+
+  @staticmethod
+  def _from_joined_paths(joined_paths):
+    mask = FieldMask()
+    mask.FromJsonString(joined_paths)
+    return mask
+
+
+class KaggleObjectSerializer(ObjectSerializer):
+  def __init__(self):
+    """
+    Kaggle objects (i.e., proto-generated types that inherit from KaggleObject) have custom "to_dict" and "from_dict"
+    methods that serialize/deserialize them to/from dictionaries.
+    """
+    ObjectSerializer.__init__(self,
+                              # "v" is an instance of a KaggleObject. For example: "req = ListCompetitionsRequest()".
+                              # So "req.to_dict()" returns a dictionary with keys as json field names. Example:
+                              # '{"pageSize": 10, "page": 2}'
+                              lambda cls, v, ignore_defaults: cls.to_dict(v, ignore_defaults),
+                              # "cls" is the type of a KaggleObject. For example: ListCompetitionsRequest. All
+                              # generated Kaggle objects have "from_dict" class method that takes a dict to create a
+                              # new instance of the object. See "KaggleObject" class definition below.
+                              lambda cls, v: cls.from_dict(v))
+
+
+class FieldMetadata(object):
+  def __init__(self, json_name, field_name, private_field_name, field_type, default_value, serializer, optional=False):
+    self.json_name = json_name
+    self.field_name = field_name
+    self.private_field_name = private_field_name
+    self.field_type = field_type
+    self.default_value = default_value
+    self.serializer = serializer
+    self.optional = optional
+
+  def get_as_dict_item(self, instance, ignore_defaults=True):
+    value = getattr(instance, self.private_field_name)
+    if ignore_defaults and value == self.default_value:
+      return None
+    if value is None:
+      return None
+    return self.serializer.to_dict_value(self.field_type, value, ignore_defaults)
+
+  def set_from_dict(self, instance, json_dict):
+    if self.json_name not in json_dict:
+      return  # Ignore unknown fields
+    value = json_dict[self.json_name]
+    if value == self.default_value:
+      return  # Ignore default values
+    try:
+      setattr(instance, self.private_field_name, self.serializer.from_dict_value(self.field_type, value))
+    except Exception as e:
+      raise
+
+
+class KaggleObject(object):
+  def endpoint(self):
+    raise 'Error: endpoint must be defined by the request object'
+
+  @staticmethod
+  def endpoint_path():
+    return None
+
+  @staticmethod
+  def body_fields():
+    return None
+
+  @classmethod
+  def prepare_from(cls, http_response):
+    return cls.from_json(http_response.text)
+
+  @staticmethod
+  def method():
+    return "GET"
+
+  def _freeze(self):
+    self._is_frozen = True
+
+  def __setattr__(self, key, value):
+    if hasattr(self, '_is_frozen') and not hasattr(self, key):
+      raise AttributeError(f'Unknown field for {self.__class__.__name__}: {key}')
+    object.__setattr__(self, key, value)
+
+  def to_dict(self, ignore_defaults=True):
+    kv_pairs = [(field.json_name, field.get_as_dict_item(self, ignore_defaults)) for field in self._fields]
+    return {k: v for (k, v) in kv_pairs if not ignore_defaults or v is not None}
+
+  @staticmethod
+  def to_field_map(self, ignore_defaults=True):
+    kv_pairs = [(field.field_name, field.get_as_dict_item(self, ignore_defaults)) for field in self._fields]
+    return {k: v for (k, v) in kv_pairs if not ignore_defaults or v is not None}
+
+  @classmethod
+  def from_dict(cls, json_dict):
+    instance = cls()
+    for field in cls._fields:
+      field.set_from_dict(instance, json_dict)
+    return instance
+
+  @classmethod
+  def from_json(cls, json_str):
+    return cls.from_dict(json.loads(json_str))
+
+  @staticmethod
+  def to_json(self, ignore_defaults=True):
+    return json.dumps(KaggleObject.to_dict(self, ignore_defaults))
+
+  def __str__(self):
+    return KaggleObject.to_json(self, ignore_defaults=False)
+
+  def __repr__(self):
+    return KaggleObject.to_json(self, ignore_defaults=False)
+
+  def __contains__(self, field_name):
+    field = self._get_field(field_name)
+    value = getattr(self, field.private_field_name)
+    if field.optional:
+      return value is not None
+    else:
+      return value != field.default_value
+
+  def __delattr__(self, field_name):
+    field = self._get_field(field_name)
+    setattr(self, field.private_field_name, field.default_value)
+
+  def _get_field(self, field_name):
+    field = next((f for f in self._fields if f.field_name == field_name), None)
+    if field is None:
+      raise ValueError(f'Protocol message {self.__class__.__name__} has no "{field_name}" field.')
+    return field