pysus 0.4.0__py3.6-linux-x86_64.egg

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. EGG-INFO/PKG-INFO +30 -0
  2. EGG-INFO/SOURCES.txt +75 -0
  3. EGG-INFO/dependency_links.txt +1 -0
  4. EGG-INFO/native_libs.txt +2 -0
  5. EGG-INFO/not-zip-safe +1 -0
  6. EGG-INFO/requires.txt +7 -0
  7. EGG-INFO/top_level.txt +2 -0
  8. __pycache__/_readdbc.cpython-36.pyc +0 -0
  9. _readdbc.abi3.so +0 -0
  10. _readdbc.py +7 -0
  11. pysus/__init__.py +2 -0
  12. pysus/__pycache__/__init__.cpython-36.pyc +0 -0
  13. pysus/online_data/CIHA.py +62 -0
  14. pysus/online_data/CNES.py +24 -0
  15. pysus/online_data/SIA.py +92 -0
  16. pysus/online_data/SIH.py +61 -0
  17. pysus/online_data/SIM.py +155 -0
  18. pysus/online_data/__init__.py +22 -0
  19. pysus/online_data/__pycache__/CIHA.cpython-36.pyc +0 -0
  20. pysus/online_data/__pycache__/CNES.cpython-36.pyc +0 -0
  21. pysus/online_data/__pycache__/SIA.cpython-36.pyc +0 -0
  22. pysus/online_data/__pycache__/SIH.cpython-36.pyc +0 -0
  23. pysus/online_data/__pycache__/SIM.cpython-36.pyc +0 -0
  24. pysus/online_data/__pycache__/__init__.cpython-36.pyc +0 -0
  25. pysus/online_data/__pycache__/sinasc.cpython-36.pyc +0 -0
  26. pysus/online_data/sinasc.py +44 -0
  27. pysus/preprocessing/__init__.py +8 -0
  28. pysus/preprocessing/__pycache__/__init__.cpython-36.pyc +0 -0
  29. pysus/preprocessing/__pycache__/decoders.cpython-36.pyc +0 -0
  30. pysus/preprocessing/__pycache__/sinan.cpython-36.pyc +0 -0
  31. pysus/preprocessing/decoders.py +96 -0
  32. pysus/preprocessing/sinan.py +103 -0
  33. pysus/tests/__init__.py +8 -0
  34. pysus/tests/__pycache__/__init__.cpython-36.pyc +0 -0
  35. pysus/tests/__pycache__/test_decoders.cpython-36.pyc +0 -0
  36. pysus/tests/__pycache__/test_sinan.cpython-36.pyc +0 -0
  37. pysus/tests/__pycache__/test_utilities.cpython-36.pyc +0 -0
  38. pysus/tests/test_data/__init__.py +6 -0
  39. pysus/tests/test_data/__pycache__/__init__.cpython-36.pyc +0 -0
  40. pysus/tests/test_data/__pycache__/test_ciha.cpython-36.pyc +0 -0
  41. pysus/tests/test_data/__pycache__/test_sia.cpython-36.pyc +0 -0
  42. pysus/tests/test_data/__pycache__/test_sih.cpython-36.pyc +0 -0
  43. pysus/tests/test_data/__pycache__/test_sim.cpython-36.pyc +0 -0
  44. pysus/tests/test_data/__pycache__/test_sinasc.cpython-36.pyc +0 -0
  45. pysus/tests/test_data/test_ciha.py +23 -0
  46. pysus/tests/test_data/test_sia.py +25 -0
  47. pysus/tests/test_data/test_sih.py +18 -0
  48. pysus/tests/test_data/test_sim.py +41 -0
  49. pysus/tests/test_data/test_sinasc.py +20 -0
  50. pysus/tests/test_decoders.py +38 -0
  51. pysus/tests/test_sinan.py +35 -0
  52. pysus/tests/test_utilities.py +18 -0
  53. pysus/utilities/__init__.py +6 -0
  54. pysus/utilities/__pycache__/__init__.cpython-36.pyc +0 -0
  55. pysus/utilities/__pycache__/_build_readdbc.cpython-36.pyc +0 -0
  56. pysus/utilities/__pycache__/readdbc.cpython-36.pyc +0 -0
  57. pysus/utilities/_build_readdbc.py +32 -0
  58. pysus/utilities/_readdbc.c +897 -0
  59. pysus/utilities/_readdbc.cpython-36m-x86_64-linux-gnu.so +0 -0
  60. pysus/utilities/_readdbc.o +0 -0
  61. pysus/utilities/blast.c +982 -0
  62. pysus/utilities/blast.h +75 -0
  63. pysus/utilities/blast.o +0 -0
  64. pysus/utilities/readdbc.py +52 -0
@@ -0,0 +1,96 @@
1
+ #-*- coding:utf-8 -*-
2
+ u"""
3
+ This module contains a set of functions to decode
4
+ commonly encoded variables
5
+ Created on 19/07/16
6
+ by fccoelho
7
+ license: GPL V3 or Later
8
+ """
9
+
10
+ __docformat__ = 'restructuredtext en'
11
+ import numpy as np
12
+ from datetime import timedelta
13
+
14
+ @np.vectorize
15
+ def decodifica_idade_SINAN(idade, unidade='Y'):
16
+ """
17
+ Em tabelas do SINAN frequentemente a idade é representada como um inteiro que precisa ser parseado
18
+ para retornar a idade em uma unidade cronológica padrão.
19
+ :param unidade: unidade da idade: 'Y': anos, 'M' meses, 'D': dias, 'H': horas
20
+ :param idade: inteiro ou sequencia de inteiros codificados.
21
+ :return:
22
+ """
23
+ fator = {'Y': 1., 'M': 12., 'D': 365., 'H': 365*24.}
24
+ if idade >= 4000: #idade em anos
25
+ idade_anos = idade - 4000
26
+ elif idade >= 3000 and idade < 4000: #idade em meses
27
+ idade_anos = (idade-3000)/12.
28
+ elif idade >= 2000 and idade < 3000: #idade em dias
29
+ idade_anos = (idade-2000)/365.
30
+ elif idade >= 1000 and idade < 2000: # idade em horas
31
+ idade_anos = (idade-1000)/(365*24.)
32
+ else:
33
+ #print(idade)
34
+ idade_anos = np.nan
35
+ #raise ValueError("Idade inválida")
36
+ idade_dec = idade_anos*fator[unidade]
37
+ return idade_dec
38
+
39
+ @np.vectorize
40
+ def decodifica_idade_SIM(idade, unidade="D"):
41
+ """
42
+ Em tabelas do SIM a idade encontra-se codificada
43
+ :param idade: valor original da tabela do SIM
44
+ :param unidade: Unidade de saida desejada: 'Y': anos, 'M' meses, 'D': dias, 'H': horas. Valor default: 'D'
45
+ :return:
46
+ """
47
+ fator = {'Y': 365., 'M': 30., 'D': 1., 'H': 1/24.}
48
+ if idade.startswith('1'):
49
+ idade = timedelta(hours=int(idade[1:])).days
50
+ elif idade.startswith('2'):
51
+ idade = timedelta(days=int(idade[1:])).days
52
+ elif idade.startswith('3'):
53
+ idade = timedelta(days=int(idade[1:]) * 30).days
54
+ elif idade.startswith('4'):
55
+ idade = timedelta(days=int(idade[1:]) * 365).days
56
+ else:
57
+ idade = np.nan
58
+ return idade/fator.get(unidade, 1)
59
+
60
+
61
+ def is_valid_geocode(geocodigo):
62
+ """
63
+ Returns True if the geocode is valid
64
+ :param geocodigo:
65
+ :return:
66
+ """
67
+ if len(str(geocodigo)) != 7:
68
+ raise ValueError('Geocode must have 7 digtis')
69
+ dig = int(str(geocodigo)[-1])
70
+ if dig == calculate_digit(geocodigo):
71
+ return True
72
+ else:
73
+ return False
74
+
75
+
76
+ def calculate_digit(geocode):
77
+ """
78
+ Calcula o digito verificador do geocódigo de município com 6 dígitos
79
+ :param geocode: geocódigo com 6 dígitos
80
+ :return: dígito verificador
81
+ """
82
+ peso = [1, 2, 1, 2, 1, 2, 0]
83
+ soma = 0
84
+ geocode = str(geocode)
85
+ for i in range(6):
86
+ valor = int(geocode[i]) * peso[i]
87
+ soma += sum([int(d) for d in str(valor)]) if valor > 9 else valor
88
+ dv = 0 if soma % 10 == 0 else (10 - (soma % 10))
89
+ return dv
90
+
91
+
92
+ def add_dv(geocodigo):
93
+ if len(str(geocodigo)) == 7:
94
+ return geocodigo
95
+ else:
96
+ return int(str(geocodigo) + str(calculate_digit(geocodigo)))
@@ -0,0 +1,103 @@
1
+ from dbfread import DBF
2
+ import pandas as pd
3
+ import requests
4
+ import geocoder
5
+ from functools import lru_cache
6
+ import os
7
+
8
+
9
+ def read_sinan_dbf(fname, encoding) -> pd.DataFrame:
10
+ """
11
+ Read SINAN dbf file returning a Pandas Dataframe with
12
+ :param fname: dbf file name
13
+ :param encoding: Encoding of the dbf
14
+ :return: pandas dataframe
15
+ """
16
+ db = DBF(fname, encoding=encoding)
17
+ df = pd.DataFrame(list(db))
18
+
19
+ def convert_week(x):
20
+ try:
21
+ w = int(x) % 100
22
+ except ValueError:
23
+ w = pd.np.nan
24
+ return w
25
+ for cname in df.columns:
26
+ df[cname].replace('', pd.np.nan, inplace=True)
27
+ if cname.startswith(('NU', 'ID')):
28
+ try:
29
+ df[cname] = pd.to_numeric(df[cname])
30
+ except ValueError as e:
31
+ # certain IDs can be alphanumerical
32
+ pass
33
+ elif cname.startswith('SEM'):
34
+ df[cname] = df[cname].map(convert_week)
35
+
36
+ return df
37
+
38
+
39
+ @lru_cache(maxsize=None)
40
+ def get_geocodes(geoc):
41
+ """
42
+ Return city name and state two letter code from geocode
43
+ :param geoc:
44
+ :return:
45
+ """
46
+ url = "http://cidades.ibge.gov.br/services/jSonpMuns.php?busca=330&featureClass=P&style=full&maxRows=5&name_startsWith={}".format(
47
+ geoc)
48
+ resp = requests.get(url)
49
+ for d in resp.json()['municipios']:
50
+ if int(geoc) == int(d['c']):
51
+ return [d['n'].encode('latin-1').decode('utf-8'), d['s']]
52
+
53
+ else:
54
+ raise KeyError('could not find geocode {} in '.format(geoc))
55
+
56
+
57
+ def _address_generator(df, default=''):
58
+ for l in df.iterrows():
59
+ l = dict(l[1])
60
+ try:
61
+ l['cidade'] = ','.join(get_geocodes(l['ID_MN_RESI']))
62
+ except:
63
+ print("Could not find geocode {} using default")
64
+ l['cidade'] = default
65
+ yield l['NU_NOTIFIC'], "{NM_LOGRADO}, {NU_NUMERO}, {NM_BAIRRO}, {cidade}, Brasil".format(**l)
66
+
67
+
68
+ def geocode(sinan_df, outfile, default_city):
69
+ """
70
+ Geocode cases based on addresses included.
71
+ :param default_city: default city to use in case of bad Geocode found in file. It can be "city, state"
72
+ :param sinan_df: Dataframe generated from sinan DBF
73
+ :param outfile: File on Which
74
+ """
75
+ addrs = _address_generator(sinan_df, default_city)
76
+ if os.path.exists(outfile):
77
+ mode = 'a'
78
+ coords = pd.read_csv(outfile)
79
+ geocoded = coords.NU_NOTIFIC.tolist()
80
+ else:
81
+ mode = 'w'
82
+ geocoded = []
83
+ with open(outfile, mode) as of:
84
+ if mode == 'w':
85
+ of.write("NU_NOTIFIC,latitude,longitude\n")
86
+ for nu, ad in addrs:
87
+ # ad = ad.encode('latin-1').decode('utf-8')
88
+ if nu in geocoded:
89
+ continue
90
+ location = geocoder.google(ad)
91
+ if location is None:
92
+ raise NameError("Google could not find {}".format(ad))
93
+ if location.latlng == []:
94
+ print("Search for {} returned {} as coordinates, trying reduced address:".format(ad, location.latlng))
95
+ ad = ','.join(ad.split(',')[2:])
96
+ print(ad)
97
+ location = geocoder.google(ad)
98
+ try:
99
+ of.write("{},{},{}\n".format(nu, location.latlng[0], location.latlng[1]))
100
+ print("Successfully geolocated {}".format(ad))
101
+ except IndexError:
102
+ print("Search for {} returned {} as coordinates, skipping".format(ad, location.latlng))
103
+ of.write("{},nan,nan\n".format(nu))
@@ -0,0 +1,8 @@
1
+ #-*- coding:utf-8 -*-
2
+ u"""
3
+ Created on 19/07/16
4
+ by fccoelho
5
+ license: GPL V3 or Later
6
+ """
7
+
8
+ __docformat__ = 'restructuredtext en'
@@ -0,0 +1,6 @@
1
+ u"""
2
+ Created on 12/12/18
3
+ by fccoelho
4
+ license: GPL V3 or Later
5
+ """
6
+
@@ -0,0 +1,23 @@
1
+ __author__ = 'fccoelho'
2
+
3
+ import unittest
4
+ from pysus.online_data.CIHA import download
5
+ import pandas as pd
6
+
7
+ unittest.skip("too slow to run om travis")
8
+ class SIHTestCase(unittest.TestCase):
9
+ def test_download_CIH(self):
10
+ df = download('mg', 2009, 7)
11
+
12
+ self.assertGreater(len(df), 0)
13
+ self.assertIn('DIAG_PRINC', df.columns)
14
+ self.assertIsInstance(df, pd.DataFrame)
15
+
16
+ def test_download_CIHA(self):
17
+ df = download('MG', 2013, 10)
18
+ self.assertGreater(len(df), 0)
19
+ self.assertIn('DIAG_PRINC', df.columns)
20
+ self.assertIsInstance(df, pd.DataFrame)
21
+
22
+ if __name__ == '__main__':
23
+ unittest.main()
@@ -0,0 +1,25 @@
1
+ __author__ = 'fccoelho'
2
+
3
+ import unittest
4
+ from pysus.online_data.SIA import download
5
+ import pandas as pd
6
+
7
+ unittest.skip("too slow to run om travis")
8
+ class SIATestCase(unittest.TestCase):
9
+ def test_download_after_2008(self):
10
+ df, df2 = download('to', 2009, 12)
11
+ self.assertGreater(len(df), 0)
12
+ self.assertIn('PA_CODUNI', df.columns)
13
+ self.assertIn('CODUNI', df2.columns)
14
+ self.assertIsInstance(df, pd.DataFrame)
15
+ self.assertIsInstance(df2, pd.DataFrame)
16
+
17
+ def test_download_before_2008(self):
18
+ df, _ = download('mg', 2006, 10)
19
+ self.assertGreater(len(df), 0)
20
+ self.assertIn('PA_CODUNI', df.columns)
21
+ self.assertIsInstance(df, pd.DataFrame)
22
+
23
+
24
+ if __name__ == '__main__':
25
+ unittest.main()
@@ -0,0 +1,18 @@
1
+ __author__ = 'fccoelho'
2
+
3
+ import unittest
4
+ from pysus.online_data.SIH import download
5
+ import pandas as pd
6
+
7
+ unittest.skip("too slow to run om travis")
8
+ class SIHTestCase(unittest.TestCase):
9
+ def test_download(self):
10
+ df = download('to', 2009, 12)
11
+ df2 = download('AC', 2013, 10)
12
+ self.assertGreater(len(df), 0)
13
+ self.assertGreater(len(df2), 0)
14
+ self.assertIsInstance(df, pd.DataFrame)
15
+
16
+
17
+ if __name__ == '__main__':
18
+ unittest.main()
@@ -0,0 +1,41 @@
1
+ __author__ = 'fccoelho'
2
+
3
+ import unittest
4
+ from unittest import skip
5
+ from pysus.online_data.SIM import download, get_CID10_table, get_CID9_table, get_municipios, get_ocupations
6
+
7
+
8
+ class TestDownload(unittest.TestCase):
9
+ def test_download_CID10(self):
10
+ df = download('ba', 2007)
11
+ self.assertIn('IDADEMAE', df.columns)
12
+ self.assertGreater(len(df), 0)
13
+
14
+ def test_download_CID9(self):
15
+ df = download('mg', 1987)
16
+ self.assertIn('NECROPSIA', df.columns)
17
+ self.assertGreater(len(df), 0)
18
+
19
+ def test_get_cid10(self):
20
+ df = get_CID10_table()
21
+ self.assertIn('CID10', df.columns)
22
+ self.assertGreater(len(df), 0)
23
+
24
+ def test_get_cid9(self):
25
+ df = get_CID9_table()
26
+ self.assertIn('DESCRICAO', df.columns)
27
+ self.assertGreater(len(df), 0)
28
+
29
+ def test_get_mun(self):
30
+ df = get_municipios()
31
+ self.assertIn('MUNCOD', df.columns)
32
+ self.assertGreater(len(df), 0)
33
+
34
+ def test_get_ocup(self):
35
+ df = get_ocupations()
36
+ self.assertIn('CODIGO', df.columns)
37
+ self.assertGreater(len(df), 0)
38
+
39
+
40
+ if __name__ == '__main__':
41
+ unittest.main()
@@ -0,0 +1,20 @@
1
+ __author__ = 'fccoelho'
2
+
3
+ import unittest
4
+ from unittest import skip
5
+ from pysus.online_data.sinasc import download
6
+
7
+
8
+ class TestDownload(unittest.TestCase):
9
+ def test_download_new(self):
10
+ df = download('SE', 2015)
11
+ self.assertIn('IDADEMAE', df.columns)
12
+ self.assertGreater(len(df), 0)
13
+ def test_download_old(self):
14
+ df = download('AL', 1994)
15
+ self.assertIn('IDADE_MAE', df.columns)
16
+ self.assertGreater(len(df), 0)
17
+
18
+
19
+ if __name__ == '__main__':
20
+ unittest.main()
@@ -0,0 +1,38 @@
1
+ # -*- coding:utf-8 -*-
2
+ u"""
3
+ Created on 19/07/16
4
+ by fccoelho
5
+ license: GPL V3 or Later
6
+ """
7
+
8
+ import unittest
9
+
10
+ import numpy as np
11
+ from numpy.testing import *
12
+
13
+ from pysus.preprocessing import decoders
14
+
15
+
16
+ class TestDecoder(unittest.TestCase):
17
+ def test_decodifica_idade_retorna_em_anos(self):
18
+ res = decoders.decodifica_idade_SINAN(4010, unidade='Y')
19
+ self.assertEqual(res, 10)
20
+ res = decoders.decodifica_idade_SINAN(3120, unidade='Y')
21
+ self.assertEqual(res, 10)
22
+ res = decoders.decodifica_idade_SINAN(2365, unidade='Y')
23
+ self.assertAlmostEqual(res, 1, places=4)
24
+ res = decoders.decodifica_idade_SINAN(1480, unidade='Y')
25
+ self.assertAlmostEqual(res, 0.0547, places=3)
26
+
27
+ def test_decodifica_lista_idades_retorna_em_anos(self):
28
+ res = decoders.decodifica_idade_SINAN([4010] * 3, unidade='Y')
29
+ assert_array_equal(res, np.array([10] * 3))
30
+ res = decoders.decodifica_idade_SINAN([3120] * 4, unidade='Y')
31
+ assert_array_equal(res, np.array([10] * 4))
32
+ res = decoders.decodifica_idade_SINAN([2365] * 2, unidade='Y')
33
+ assert_array_equal(res, np.array([1, 1]))
34
+ res = decoders.decodifica_idade_SINAN([1480] * 5, unidade='Y')
35
+ assert_array_almost_equal(res, np.array([0.0547] * 5), decimal=3)
36
+
37
+ def test_verifica_geocodigo(self):
38
+ self.assertTrue(decoders.is_valid_geocode(3304557))
@@ -0,0 +1,35 @@
1
+ import unittest
2
+ from pysus.preprocessing.sinan import read_sinan_dbf, geocode
3
+ import pandas as pd
4
+ import datetime
5
+ import numpy as np
6
+
7
+
8
+
9
+
10
+ class TestSinanDF(unittest.TestCase):
11
+ def test_read_dbf(self):
12
+ df = read_sinan_dbf('test_data/EPR-2016-06-01-2016.dbf', encoding='latin-1')
13
+ self.assertIsInstance(df, pd.DataFrame)
14
+ for cname in df.columns:
15
+ if cname.startswith('DT_'):
16
+ self.assertIsInstance(df[cname][0], datetime.date)
17
+ elif cname.startswith('SEM'):
18
+ self.assertLessEqual(df[cname][0], 52)
19
+ self.assertIsInstance(df[cname][0], (int, np.int64))
20
+ elif cname.startswith(('NU', 'ID')):
21
+ if cname == 'ID_AGRAVO': continue
22
+ self.assertIsInstance(df[cname][0], (int, float, np.int64),
23
+ msg="Failed on column {}, type:{}".format(cname, type(df[cname][0])))
24
+
25
+ def test_type_convertion(self):
26
+ df = read_sinan_dbf('test_data/EPR-2016-06-01-2016.dbf', encoding='latin-1')
27
+ assert not all(df.dtypes == 'object')
28
+
29
+ def test_geocode(self):
30
+ df = pd.read_pickle('test_data/chik.pickle')
31
+ # geocode(sinan_df=df, outfile='chik_2016.csv', default_city='Rio de Janeiro')
32
+
33
+
34
+ if __name__ == '__main__':
35
+ unittest.main()
@@ -0,0 +1,18 @@
1
+ u"""
2
+ Created on 17/08/16
3
+ by fccoelho
4
+ license: GPL V3 or Later
5
+ """
6
+
7
+ import unittest
8
+
9
+ import pandas as pd
10
+
11
+ from pysus.utilities.readdbc import read_dbc
12
+
13
+
14
+ class TestReadDbc(unittest.TestCase):
15
+ def test_read_dbc(self):
16
+ df = read_dbc(b'test_data/sids.dbc')
17
+ self.assertIsInstance(df, pd.DataFrame)
18
+ self.assertGreater(df.size, 0)
@@ -0,0 +1,6 @@
1
+ u"""
2
+ Created on 16/08/16
3
+ by fccoelho
4
+ license: GPL V3 or Later
5
+ """
6
+
@@ -0,0 +1,32 @@
1
+ u"""
2
+ Created on 16/08/16
3
+ by fccoelho
4
+ license: GPL V3 or Later
5
+ """
6
+ import os
7
+ from cffi import FFI
8
+
9
+ ffibuilder = FFI()
10
+
11
+ PATH = os.path.dirname(__file__)
12
+
13
+ with open(os.path.join(PATH, 'c-src/dbc2dbf.c'),'r') as f:
14
+ ffibuilder.set_source("_readdbc",
15
+ f.read(),
16
+ libraries=["c"],
17
+ sources=[os.path.join(PATH, "c-src/blast.c")],
18
+ include_dirs=[os.path.join(PATH, "c-src/")]
19
+ )
20
+ ffibuilder.cdef(
21
+ """
22
+ static unsigned inf(void *how, unsigned char **buf);
23
+ static int outf(void *how, unsigned char *buf, unsigned len);
24
+ void dbc2dbf(char** input_file, char** output_file);
25
+ """
26
+ )
27
+
28
+ with open(os.path.join(PATH, "c-src/blast.h")) as f:
29
+ ffibuilder.cdef(f.read(), override=True)
30
+
31
+ if __name__ == "__main__":
32
+ ffibuilder.compile(verbose=True)