singer-python 6.2.0__tar.gz → 6.2.2__tar.gz

{singer_python-6.2.0/singer_python.egg-info → singer_python-6.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: singer-python
-Version: 6.2.0
+Version: 6.2.2
 Summary: Singer.io utility library
 Home-page: http://singer.io
 Author: Stitch

{singer_python-6.2.0 → singer_python-6.2.2}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 import subprocess
 
 setup(name="singer-python",
-      version='6.2.0',
+      version='6.2.2',
       description="Singer.io utility library",
       author="Stitch",
       classifiers=['Programming Language :: Python :: 3 :: Only'],

{singer_python-6.2.0 → singer_python-6.2.2}/singer/schema_generation.py

@@ -1,6 +1,3 @@
-import dateutil.parser
-
-
 def add_observation(acc, path):
 
     node = acc
@@ -18,19 +15,26 @@ def add_observations(acc, path, data):
         for key in data:
             add_observations(acc, path + ["object", key], data[key])
     elif isinstance(data, list):
+        if len(data) == 0:
+            add_observations(acc, path + ["array"], None)
         for item in data:
             add_observations(acc, path + ["array"], item)
     elif isinstance(data, str):
-        # If the string parses as a date, add an observation that its a date
         try:
-            data = dateutil.parser.parse(data)
-        except (dateutil.parser.ParserError, OverflowError):
-            data = None
-        if data:
-            add_observation(acc, path + ["date"])
-        else:
-            add_observation(acc, path + ["string"])
-
+            # If the string parses as a int, add an observation that it's a integer
+            int(data)
+            add_observation(acc, path + ["integer"])
+            return acc
+        except (ValueError, TypeError):
+            pass
+        try:
+            # If the string parses as a float, add an observation that it's a number
+            float(data)
+            add_observation(acc, path + ["number"])
+            return acc
+        except (ValueError, TypeError):
+            pass
+        add_observation(acc, path + ["string"])
     elif isinstance(data, bool):
         add_observation(acc, path + ["boolean"])
     elif isinstance(data, int):
@@ -45,9 +49,13 @@ def add_observations(acc, path, data):
     return acc
 
 def to_json_schema(obs):
-    result = {'type': ['null']}
+    types = []
+    # add schema types in a specific order to anyOf list
+    for key in ['array', 'object', 'number', 'integer', 'boolean', 'string', 'null']:
+        if key not in obs:
+            continue
 
-    for key in obs:
+        result = {'type': ['null']}
 
         if key == 'object':
             result['type'] += ['object']
@@ -60,9 +68,6 @@ def to_json_schema(obs):
             result['type'] += ['array']
             result['items'] = to_json_schema(obs['array'])
 
-        elif key == 'date':
-            result['type'] += ['string']
-            result['format'] = 'date-time'
         elif key == 'string':
             result['type'] += ['string']
 
@@ -83,7 +88,15 @@ def to_json_schema(obs):
         else:
             raise Exception("Unexpected data type " + key)
 
-    return result
+        types.append(result)
+
+    if len(types) == 0:
+        return {'type': ['null', 'string']}
+
+    if len(types) == 1:
+        return types[0]
+
+    return {'anyOf': types}
 
 def generate_schema(records):
     obs = {}

{singer_python-6.2.0 → singer_python-6.2.2}/singer/transform.py

@@ -185,6 +185,8 @@ class Transformer:
             success, transformed_data = self.transform_recur(data, subschema, path)
             if success:
                 return success, transformed_data
+            else:
+                self.errors.pop()
         else: # pylint: disable=useless-else-on-loop
             # exhaused all schemas and didn't return, so we failed :-(
             self.errors.append(Error(path, data, schema, logging_level=LOGGER.level))
@@ -266,13 +268,13 @@ class Transformer:
             else:
                 return False, None
 
-        elif schema.get("format") == "date-time":
+        elif typ == "string" and schema.get("format") == "date-time":
             data = self._transform_datetime(data)
             if data is None:
                 return False, None
 
             return True, data
-        elif schema.get("format") == "singer.decimal":
+        elif typ == "string" and schema.get("format") == "singer.decimal":
             if data is None:
                 return False, None
 

{singer_python-6.2.0 → singer_python-6.2.2/singer_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: singer-python
-Version: 6.2.0
+Version: 6.2.2
 Summary: Singer.io utility library
 Home-page: http://singer.io
 Author: Stitch

{singer_python-6.2.0 → singer_python-6.2.2}/tests/test_schema_generation.py

@@ -10,7 +10,7 @@ class TestSchemaGeneration(unittest.TestCase):
                 'a': {'type': ['null', 'integer']},
                 'b': {'type': ['null', 'string']},
                 'c': {'type': ['null', 'boolean']},
-                'dt': {'type': ['null', 'string'], 'format': 'date-time'}
+                'dt': {'type': ['null', 'string']}
             }
         }
         self.assertEqual(expected_schema, generate_schema(records))
@@ -23,24 +23,20 @@ class TestSchemaGeneration(unittest.TestCase):
         ]
         expected_schema = {
             'type': ['null', 'object'],
-            'properties': {
-                'a': {'type': {'null', 'integer', 'string', 'boolean'}},
-                'b': {'type': ['null', 'string']},
-                'c': {'type': {'null', 'integer', 'string'}, 'format': 'singer.decimal'},
-                'd': {
-                    'type': {'null', 'array', 'object'},
-                    'items': {'type': {'null', 'integer', 'string'}},
-                    'properties': {'one': {'type': ['null', 'integer']},
-                                   'two': {'type': ['null', 'string']}}
-
-                }
-            }
+            'properties': {'a': {'anyOf': [{'type': ['null', 'integer']},
+                                           {'type': ['null', 'boolean']},
+                                           {'type': ['null', 'string']}]},
+                           'b': {'type': ['null', 'string']},
+                           'c': {'anyOf': [{'type': ['null', 'string'], 'format': 'singer.decimal'},
+                                           {'type': ['null', 'integer']}]},
+                'd': {'anyOf': [{'type': ['null', 'array'],
+                                 'items': {'anyOf': [{'type': ['null', 'integer']},
+                                                     {'type': ['null', 'string']}]}},
+                                {'type': ['null', 'object'],
+                                 'properties': {'one': {'type': ['null', 'integer']},
+                                                'two': {'type': ['null', 'string']}}}]}}
         }
         actual_schema = generate_schema(records)
-        actual_schema['properties']['a']['type'] = set(actual_schema['properties']['a']['type'])
-        actual_schema['properties']['c']['type'] = set(actual_schema['properties']['c']['type'])
-        actual_schema['properties']['d']['type'] = set(actual_schema['properties']['d']['type'])
-        actual_schema['properties']['d']['items']['type'] = set(actual_schema['properties']['d']['items']['type'])
         self.assertEqual(expected_schema, actual_schema)
 
     def test_nested_structue_schema(self):

{singer_python-6.2.0 → singer_python-6.2.2}/tests/test_transform.py

@@ -25,7 +25,7 @@ class TestTransform(unittest.TestCase):
 
     def test_multi_type_object_transform(self):
         schema =  {"type": ["null", "object", "string"],
-                   "properties": {"whatever": {"type": "date-time",
+                   "properties": {"whatever": {"type": "string",
                                                "format": "date-time"}}}
         data = {"whatever": "2017-01-01"}
         expected = {"whatever": "2017-01-01T00:00:00.000000Z"}
@@ -36,7 +36,7 @@ class TestTransform(unittest.TestCase):
 
     def test_multi_type_array_transform(self):
         schema =  {"type": ["null", "array", "integer"],
-                   "items": {"type": "date-time", "format": "date-time"}}
+                   "items": {"type": "string", "format": "date-time"}}
         data = ["2017-01-01"]
         expected = ["2017-01-01T00:00:00.000000Z"]
         self.assertEqual(expected, transform(data, schema))