singer-python 5.13.2__tar.gz → 5.14.0__tar.gz

{singer_python-5.13.2/singer_python.egg-info → singer_python-5.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: singer-python
-Version: 5.13.2
+Version: 5.14.0
 Summary: Singer.io utility library
 Home-page: http://singer.io
 Author: Stitch

{singer_python-5.13.2 → singer_python-5.14.0}/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 import subprocess
 
 setup(name="singer-python",
-      version='5.13.2',
+      version='5.14.0',
       description="Singer.io utility library",
       author="Stitch",
       classifiers=['Programming Language :: Python :: 3 :: Only'],

singer_python-5.14.0/singer/schema_generation.py

@@ -0,0 +1,92 @@
+import dateutil.parser
+
+
+def add_observation(acc, path):
+
+    node = acc
+    for i in range(0, len(path) - 1):
+        k = path[i]
+        if k not in node:
+            node[k] = {}
+        node = node[k]
+
+    node[path[-1]] = True
+
+# pylint: disable=too-many-branches
+def add_observations(acc, path, data):
+    if isinstance(data, dict):
+        for key in data:
+            add_observations(acc, path + ["object", key], data[key])
+    elif isinstance(data, list):
+        for item in data:
+            add_observations(acc, path + ["array"], item)
+    elif isinstance(data, str):
+        # If the string parses as a date, add an observation that its a date
+        try:
+            data = dateutil.parser.parse(data)
+        except (dateutil.parser.ParserError, OverflowError):
+            data = None
+        if data:
+            add_observation(acc, path + ["date"])
+        else:
+            add_observation(acc, path + ["string"])
+
+    elif isinstance(data, bool):
+        add_observation(acc, path + ["boolean"])
+    elif isinstance(data, int):
+        add_observation(acc, path + ["integer"])
+    elif isinstance(data, float):
+        add_observation(acc, path + ["number"])
+    elif data is None:
+        add_observation(acc, path + ["null"])
+    else:
+        raise Exception("Unexpected value " + repr(data) + " at path " + repr(path))
+
+    return acc
+
+def to_json_schema(obs):
+    result = {'type': ['null']}
+
+    for key in obs:
+
+        if key == 'object':
+            result['type'] += ['object']
+            if 'properties' not in result:
+                result['properties'] = {}
+                for obj_key in obs['object']:
+                    result['properties'][obj_key] = to_json_schema(obs['object'][obj_key])
+
+        elif key == 'array':
+            result['type'] += ['array']
+            result['items'] = to_json_schema(obs['array'])
+
+        elif key == 'date':
+            result['type'] += ['string']
+            result['format'] = 'date-time'
+        elif key == 'string':
+            result['type'] += ['string']
+
+        elif key == 'boolean':
+            result['type'] += ['boolean']
+
+        elif key == 'integer':
+            result['type'] += ['integer']
+
+        elif key == 'number':
+            # Use type=string, format=singer.decimal
+            result['type'] += ['string']
+            result['format'] = 'singer.decimal'
+
+        elif key == 'null':
+            pass
+
+        else:
+            raise Exception("Unexpected data type " + key)
+
+    return result
+
+def generate_schema(records):
+    obs = {}
+    for record in records:
+        obs = add_observations(obs, [], record)
+    return to_json_schema(obs)

{singer_python-5.13.2 → singer_python-5.14.0/singer_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: singer-python
-Version: 5.13.2
+Version: 5.14.0
 Summary: Singer.io utility library
 Home-page: http://singer.io
 Author: Stitch

{singer_python-5.13.2 → singer_python-5.14.0}/singer_python.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ singer/metadata.py
 singer/metrics.py
 singer/requests.py
 singer/schema.py
+singer/schema_generation.py
 singer/statediff.py
 singer/transform.py
 singer/utils.py
@@ -28,6 +29,7 @@ tests/test_exceptions.py
 tests/test_metadata.py
 tests/test_metrics.py
 tests/test_schema.py
+tests/test_schema_generation.py
 tests/test_singer.py
 tests/test_statediff.py
 tests/test_transform.py

{singer_python-5.13.2 → singer_python-5.14.0}/tests/test_catalog.py

@@ -25,7 +25,7 @@ class TestGetSelectedStreams(unittest.TestCase):
              CatalogEntry(tap_stream_id='c',schema=Schema(),metadata=[])])
         state = {}
         selected_streams = catalog.get_selected_streams(state)
-        self.assertEquals([e for e in selected_streams],[selected_entry])
+        self.assertEqual([e for e in selected_streams],[selected_entry])
 
     def test_resumes_currently_syncing_stream(self):
         selected_entry_a = CatalogEntry(tap_stream_id='a',
@@ -44,7 +44,7 @@ class TestGetSelectedStreams(unittest.TestCase):
              selected_entry_c])
         state = {'currently_syncing': 'c'}
         selected_streams = catalog.get_selected_streams(state)
-        self.assertEquals([e for e in selected_streams][0],selected_entry_c)
+        self.assertEqual([e for e in selected_streams][0],selected_entry_c)
 
 class TestToDictAndFromDict(unittest.TestCase):
 
@@ -141,4 +141,4 @@ class TestGetStream(unittest.TestCase):
              CatalogEntry(tap_stream_id='b'),
              CatalogEntry(tap_stream_id='c')])
         entry = catalog.get_stream('b')
-        self.assertEquals('b', entry.tap_stream_id)
+        self.assertEqual('b', entry.tap_stream_id)

{singer_python-5.13.2 → singer_python-5.14.0}/tests/test_exceptions.py

@@ -14,8 +14,8 @@ class TestSingerErrors(unittest.TestCase):
             raise SingerError(error_text)
 
         expected_text = "SingerError\n" + error_text
-        self.assertEquals(expected_text,
-                          str(test_run.exception))
+        self.assertEqual(expected_text,
+                         str(test_run.exception))
 
     def test_SingerConfigurationError_prints_correctly(self):
         error_text = "An error occured"
@@ -24,8 +24,8 @@ class TestSingerErrors(unittest.TestCase):
             raise SingerConfigurationError(error_text)
 
         expected_text = "SingerConfigurationError\n" + error_text
-        self.assertEquals(expected_text,
-                          str(test_run.exception))
+        self.assertEqual(expected_text,
+                         str(test_run.exception))
 
     def test_SingerDiscoveryError_prints_correctly(self):
         error_text = "An error occured"
@@ -34,8 +34,8 @@ class TestSingerErrors(unittest.TestCase):
             raise SingerDiscoveryError(error_text)
 
         expected_text = "SingerDiscoveryError\n" + error_text
-        self.assertEquals(expected_text,
-                          str(test_run.exception))
+        self.assertEqual(expected_text,
+                         str(test_run.exception))
 
     def test_SingerSyncError_prints_correctly(self):
         error_text = "An error occured"
@@ -44,8 +44,8 @@ class TestSingerErrors(unittest.TestCase):
             raise SingerSyncError(error_text)
 
         expected_text = "SingerSyncError\n" + error_text
-        self.assertEquals(expected_text,
-                          str(test_run.exception))
+        self.assertEqual(expected_text,
+                         str(test_run.exception))
 
     def test_SingerRetryableRequestError_prints_correctly(self):
         error_text = "An error occured"
@@ -54,8 +54,8 @@ class TestSingerErrors(unittest.TestCase):
             raise SingerRetryableRequestError(error_text)
 
         expected_text = "SingerRetryableRequestError\n" + error_text
-        self.assertEquals(expected_text,
-                          str(test_run.exception))
+        self.assertEqual(expected_text,
+                         str(test_run.exception))
 
     def test_SingerError_prints_multiple_lines_correctly(self):
         error_text = "\n".join(["Line 1", "Line 2", "Line 3"])
@@ -64,5 +64,5 @@ class TestSingerErrors(unittest.TestCase):
             raise SingerError(error_text)
 
         expected_text = "SingerError\n" + error_text
-        self.assertEquals(expected_text,
-                          str(test_run.exception))
+        self.assertEqual(expected_text,
+                         str(test_run.exception))

{singer_python-5.13.2 → singer_python-5.14.0}/tests/test_schema.py

@@ -44,38 +44,38 @@ class TestSchema(unittest.TestCase):
                         additionalProperties=True)
 
     def test_string_to_dict(self):
-        self.assertEquals(self.string_dict, self.string_obj.to_dict())
+        self.assertEqual(self.string_dict, self.string_obj.to_dict())
 
     def test_integer_to_dict(self):
-        self.assertEquals(self.integer_dict, self.integer_obj.to_dict())
+        self.assertEqual(self.integer_dict, self.integer_obj.to_dict())
 
     def test_array_to_dict(self):
-        self.assertEquals(self.array_dict, self.array_obj.to_dict())
+        self.assertEqual(self.array_dict, self.array_obj.to_dict())
 
     def test_object_to_dict(self):
-        self.assertEquals(self.object_dict, self.object_obj.to_dict())
+        self.assertEqual(self.object_dict, self.object_obj.to_dict())
 
     def test_string_from_dict(self):
-        self.assertEquals(self.string_obj, Schema.from_dict(self.string_dict))
+        self.assertEqual(self.string_obj, Schema.from_dict(self.string_dict))
 
     def test_integer_from_dict(self):
-        self.assertEquals(self.integer_obj, Schema.from_dict(self.integer_dict))
+        self.assertEqual(self.integer_obj, Schema.from_dict(self.integer_dict))
 
     def test_array_from_dict(self):
-        self.assertEquals(self.array_obj, Schema.from_dict(self.array_dict))
+        self.assertEqual(self.array_obj, Schema.from_dict(self.array_dict))
 
     def test_object_from_dict(self):
-        self.assertEquals(self.object_obj, Schema.from_dict(self.object_dict))
+        self.assertEqual(self.object_obj, Schema.from_dict(self.object_dict))
 
     def test_repr_atomic(self):
-        self.assertEquals(self.string_obj, eval(repr(self.string_obj)))
+        self.assertEqual(self.string_obj, eval(repr(self.string_obj)))
 
     def test_repr_recursive(self):
-        self.assertEquals(self.object_obj, eval(repr(self.object_obj)))
+        self.assertEqual(self.object_obj, eval(repr(self.object_obj)))
 
     def test_object_from_dict_with_defaults(self):
         schema = Schema.from_dict(self.object_dict, inclusion='automatic')
-        self.assertEquals('whatever', schema.inclusion,
-                          msg='The schema value should override the default')
-        self.assertEquals('automatic', schema.properties['a_string'].inclusion)
-        self.assertEquals('automatic', schema.properties['an_array'].items.inclusion)
+        self.assertEqual('whatever', schema.inclusion,
+                         msg='The schema value should override the default')
+        self.assertEqual('automatic', schema.properties['a_string'].inclusion)
+        self.assertEqual('automatic', schema.properties['an_array'].items.inclusion)

singer_python-5.14.0/tests/test_schema_generation.py

@@ -0,0 +1,76 @@
+import unittest
+from singer.schema_generation import generate_schema
+
+class TestSchemaGeneration(unittest.TestCase):
+    def test_simple_schema(self):
+        records = [{'a': 1, 'b': 'two', 'c': True, 'dt': '2000-01-01T00:11:22Z'}]
+        expected_schema = {
+            'type': ['null', 'object'],
+            'properties': {
+                'a': {'type': ['null', 'integer']},
+                'b': {'type': ['null', 'string']},
+                'c': {'type': ['null', 'boolean']},
+                'dt': {'type': ['null', 'string'], 'format': 'date-time'}
+            }
+        }
+        self.assertEqual(expected_schema, generate_schema(records))
+
+    def test_mix_n_match_records_schema(self):
+        records = [
+            {'a': 1, 'b': 'b'},
+            {'a': 'two', 'c': 7, 'd': [1, 'two']},
+            {'a': True, 'c': 7.7, 'd': {'one': 1, 'two': 'two'}}
+        ]
+        expected_schema = {
+            'type': ['null', 'object'],
+            'properties': {
+                'a': {'type': {'null', 'integer', 'string', 'boolean'}},
+                'b': {'type': ['null', 'string']},
+                'c': {'type': {'null', 'integer', 'string'}, 'format': 'singer.decimal'},
+                'd': {
+                    'type': {'null', 'array', 'object'},
+                    'items': {'type': {'null', 'integer', 'string'}},
+                    'properties': {'one': {'type': ['null', 'integer']},
+                                   'two': {'type': ['null', 'string']}}
+
+                }
+            }
+        }
+        actual_schema = generate_schema(records)
+        actual_schema['properties']['a']['type'] = set(actual_schema['properties']['a']['type'])
+        actual_schema['properties']['c']['type'] = set(actual_schema['properties']['c']['type'])
+        actual_schema['properties']['d']['type'] = set(actual_schema['properties']['d']['type'])
+        actual_schema['properties']['d']['items']['type'] = set(actual_schema['properties']['d']['items']['type'])
+        self.assertEqual(expected_schema, actual_schema)
+
+    def test_nested_structue_schema(self):
+        records = [{'a': {'b': {'c': [{'d': 7}]}, 'e': [[1, 2, 3]]}}]
+        expected_schema = {
+            'type': ['null', 'object'],
+            'properties': {
+                'a': {
+                    'type': ['null', 'object'],
+                    'properties': {
+                        'b': {
+                            'type': ['null', 'object'],
+                            'properties': {
+                                'c': {
+                                    'type': ['null', 'array'],
+                                    'items': {
+                                        'type': ['null', 'object'],
+                                        'properties': {'d': {'type': ['null', 'integer']}}
+                                    }
+                                }
+                            }
+                        },
+                        'e': {
+                            'type': ['null', 'array'],
+                            'items': {
+                                'type': ['null', 'array'],
+                                'items': {'type': ['null', 'integer']}}
+                        }
+                    }
+                }
+            }
+        }
+        self.assertEqual(expected_schema, generate_schema(records))

{singer_python-5.13.2 → singer_python-5.14.0}/tests/test_transform.py

@@ -264,11 +264,11 @@ class TestTransform(unittest.TestCase):
         nan = {'percentage': decimal.Decimal('NaN')}
         snan = {'percentage': decimal.Decimal('sNaN')}
 
-        self.assertEquals(inf, transform(inf, schema))
-        self.assertEquals(negative_inf, transform(negative_inf, schema))
-        self.assertEquals({'percentage': '1.4142135623730951'}, transform(root2, schema))
-        self.assertEquals({'percentage': 'NaN'}, transform(nan, schema))
-        self.assertEquals({'percentage': 'NaN'}, transform(snan, schema))
+        self.assertEqual(inf, transform(inf, schema))
+        self.assertEqual(negative_inf, transform(negative_inf, schema))
+        self.assertEqual({'percentage': '1.4142135623730951'}, transform(root2, schema))
+        self.assertEqual({'percentage': 'NaN'}, transform(nan, schema))
+        self.assertEqual({'percentage': 'NaN'}, transform(snan, schema))
 
 
         str1 = {'percentage':'0.1'}
@@ -276,11 +276,11 @@ class TestTransform(unittest.TestCase):
         str3 = {'percentage': '1E+13'}
         str4 = {'percentage': '100'}
         str5 = {'percentage': '-100'}
-        self.assertEquals(str1, transform(str1, schema))
-        self.assertEquals({'percentage': '1E-13'}, transform(str2, schema))
-        self.assertEquals({'percentage': '1E+13'}, transform(str3, schema))
-        self.assertEquals({'percentage': '100'}, transform(str4, schema))
-        self.assertEquals({'percentage': '-100'}, transform(str5, schema))
+        self.assertEqual(str1, transform(str1, schema))
+        self.assertEqual({'percentage': '1E-13'}, transform(str2, schema))
+        self.assertEqual({'percentage': '1E+13'}, transform(str3, schema))
+        self.assertEqual({'percentage': '100'}, transform(str4, schema))
+        self.assertEqual({'percentage': '-100'}, transform(str5, schema))
 
         float1 = {'percentage': 12.0000000000000000000000000001234556}
         float2 = {'percentage': 0.0123}
@@ -288,28 +288,28 @@ class TestTransform(unittest.TestCase):
         float4 = {'percentage': -100.0123}
         float5 = {'percentage': 0.000001}
         float6 = {'percentage': 0.0000001}
-        self.assertEquals({'percentage':'12.0'}, transform(float1, schema))
-        self.assertEquals({'percentage':'0.0123'}, transform(float2, schema))
-        self.assertEquals({'percentage':'100.0123'}, transform(float3, schema))
-        self.assertEquals({'percentage':'-100.0123'}, transform(float4, schema))
-        self.assertEquals({'percentage':'0.000001'}, transform(float5, schema))
-        self.assertEquals({'percentage':'1E-7'}, transform(float6, schema))
+        self.assertEqual({'percentage':'12.0'}, transform(float1, schema))
+        self.assertEqual({'percentage':'0.0123'}, transform(float2, schema))
+        self.assertEqual({'percentage':'100.0123'}, transform(float3, schema))
+        self.assertEqual({'percentage':'-100.0123'}, transform(float4, schema))
+        self.assertEqual({'percentage':'0.000001'}, transform(float5, schema))
+        self.assertEqual({'percentage':'1E-7'}, transform(float6, schema))
 
         int1 = {'percentage': 123}
         int2 = {'percentage': 0}
         int3 = {'percentage': -1000}
-        self.assertEquals({'percentage':'123'}, transform(int1, schema))
-        self.assertEquals({'percentage':'0'}, transform(int2, schema))
-        self.assertEquals({'percentage':'-1000'}, transform(int3, schema))
+        self.assertEqual({'percentage':'123'}, transform(int1, schema))
+        self.assertEqual({'percentage':'0'}, transform(int2, schema))
+        self.assertEqual({'percentage':'-1000'}, transform(int3, schema))
 
         dec1 = {'percentage': decimal.Decimal('1.1010101')}
         dec2 = {'percentage': decimal.Decimal('.111111111111111111111111')}
         dec3 = {'percentage': decimal.Decimal('-.111111111111111111111111')}
         dec4 = {'percentage': decimal.Decimal('100')}
-        self.assertEquals({'percentage':'1.1010101'}, transform(dec1, schema))
-        self.assertEquals({'percentage':'0.111111111111111111111111'}, transform(dec2, schema))
-        self.assertEquals({'percentage':'-0.111111111111111111111111'}, transform(dec3, schema))
-        self.assertEquals({'percentage':'100'}, transform(dec4, schema))
+        self.assertEqual({'percentage':'1.1010101'}, transform(dec1, schema))
+        self.assertEqual({'percentage':'0.111111111111111111111111'}, transform(dec2, schema))
+        self.assertEqual({'percentage':'-0.111111111111111111111111'}, transform(dec3, schema))
+        self.assertEqual({'percentage':'100'}, transform(dec4, schema))
 
         bad1 = {'percentage': 'fsdkjl'}
         with self.assertRaises(SchemaMismatch):
@@ -317,7 +317,7 @@ class TestTransform(unittest.TestCase):
 
         badnull = {'percentage': None}
         with self.assertRaises(SchemaMismatch):
-            self.assertEquals({'percentage':None}, transform(badnull, schema))
+            self.assertEqual({'percentage':None}, transform(badnull, schema))
 
 class TestTransformsWithMetadata(unittest.TestCase):