pysus 0.4.0__py3.7-linux-x86_64.egg
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- EGG-INFO/PKG-INFO +30 -0
- EGG-INFO/SOURCES.txt +75 -0
- EGG-INFO/dependency_links.txt +1 -0
- EGG-INFO/native_libs.txt +3 -0
- EGG-INFO/not-zip-safe +1 -0
- EGG-INFO/requires.txt +7 -0
- EGG-INFO/top_level.txt +2 -0
- __pycache__/_readdbc.cpython-37.pyc +0 -0
- _readdbc.abi3.so +0 -0
- _readdbc.py +7 -0
- pysus/__init__.py +2 -0
- pysus/__pycache__/__init__.cpython-37.pyc +0 -0
- pysus/online_data/CIHA.py +62 -0
- pysus/online_data/CNES.py +24 -0
- pysus/online_data/SIA.py +92 -0
- pysus/online_data/SIH.py +61 -0
- pysus/online_data/SIM.py +155 -0
- pysus/online_data/SINAN.py +77 -0
- pysus/online_data/__init__.py +22 -0
- pysus/online_data/__pycache__/CIHA.cpython-37.pyc +0 -0
- pysus/online_data/__pycache__/CNES.cpython-37.pyc +0 -0
- pysus/online_data/__pycache__/SIA.cpython-37.pyc +0 -0
- pysus/online_data/__pycache__/SIH.cpython-37.pyc +0 -0
- pysus/online_data/__pycache__/SIM.cpython-37.pyc +0 -0
- pysus/online_data/__pycache__/SINAN.cpython-37.pyc +0 -0
- pysus/online_data/__pycache__/__init__.cpython-37.pyc +0 -0
- pysus/online_data/__pycache__/sinasc.cpython-37.pyc +0 -0
- pysus/online_data/sinasc.py +44 -0
- pysus/preprocessing/__init__.py +8 -0
- pysus/preprocessing/__pycache__/__init__.cpython-37.pyc +0 -0
- pysus/preprocessing/__pycache__/decoders.cpython-37.pyc +0 -0
- pysus/preprocessing/__pycache__/sinan.cpython-37.pyc +0 -0
- pysus/preprocessing/decoders.py +96 -0
- pysus/preprocessing/sinan.py +103 -0
- pysus/tests/__init__.py +8 -0
- pysus/tests/__pycache__/__init__.cpython-37.pyc +0 -0
- pysus/tests/__pycache__/test_decoders.cpython-37.pyc +0 -0
- pysus/tests/__pycache__/test_sinan.cpython-37.pyc +0 -0
- pysus/tests/__pycache__/test_utilities.cpython-37.pyc +0 -0
- pysus/tests/test_data/__init__.py +6 -0
- pysus/tests/test_data/__pycache__/__init__.cpython-37.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_ciha.cpython-37.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sia.cpython-37.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sih.cpython-37.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sim.cpython-37.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sinasc.cpython-37.pyc +0 -0
- pysus/tests/test_data/test_ciha.py +23 -0
- pysus/tests/test_data/test_sia.py +25 -0
- pysus/tests/test_data/test_sih.py +18 -0
- pysus/tests/test_data/test_sim.py +41 -0
- pysus/tests/test_data/test_sinasc.py +20 -0
- pysus/tests/test_decoders.py +38 -0
- pysus/tests/test_sinan.py +44 -0
- pysus/tests/test_utilities.py +18 -0
- pysus/utilities/__init__.py +6 -0
- pysus/utilities/__pycache__/__init__.cpython-37.pyc +0 -0
- pysus/utilities/__pycache__/_build_readdbc.cpython-37.pyc +0 -0
- pysus/utilities/__pycache__/readdbc.cpython-37.pyc +0 -0
- pysus/utilities/_build_readdbc.py +32 -0
- pysus/utilities/_readdbc.c +966 -0
- pysus/utilities/_readdbc.cpython-36m-x86_64-linux-gnu.so +0 -0
- pysus/utilities/_readdbc.cpython-37m-x86_64-linux-gnu.so +0 -0
- pysus/utilities/_readdbc.o +0 -0
- pysus/utilities/blast.c +982 -0
- pysus/utilities/blast.h +75 -0
- pysus/utilities/blast.o +0 -0
- pysus/utilities/readdbc.py +52 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Download SINASC data from DATASUS FTP server
|
|
3
|
+
Created on 01/11/17
|
|
4
|
+
by fccoelho
|
|
5
|
+
license: GPL V3 or Later
|
|
6
|
+
"""
|
|
7
|
+
import os
|
|
8
|
+
from ftplib import FTP
|
|
9
|
+
from pysus.utilities.readdbc import read_dbc
|
|
10
|
+
from pysus.online_data import CACHEPATH
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def download(state, year, cache=True):
|
|
15
|
+
"""
|
|
16
|
+
Downloads data directly from Datasus ftp server
|
|
17
|
+
:param state: two-letter state identifier: MG == Minas Gerais
|
|
18
|
+
:param year: 4 digit integer
|
|
19
|
+
:return: pandas dataframe
|
|
20
|
+
"""
|
|
21
|
+
state = state.upper()
|
|
22
|
+
if year < 1994:
|
|
23
|
+
raise ValueError("SINASC does not contain data before 1994")
|
|
24
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
25
|
+
ftp.login()
|
|
26
|
+
if year >= 1996:
|
|
27
|
+
ftp.cwd('/dissemin/publicos/SINASC/NOV/DNRES')
|
|
28
|
+
fname = 'DN{}{}.DBC'.format(state, year)
|
|
29
|
+
else:
|
|
30
|
+
ftp.cwd('/dissemin/publicos/SINASC/ANT/DNRES')
|
|
31
|
+
fname = 'DNR{}{}.DBC'.format(state, str(year)[-2:])
|
|
32
|
+
cachefile = os.path.join(CACHEPATH, 'SINASC_'+fname.split('.')[0] + '_.parquet')
|
|
33
|
+
if os.path.exists(cachefile):
|
|
34
|
+
df = pd.read_parquet(cachefile)
|
|
35
|
+
return df
|
|
36
|
+
|
|
37
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
38
|
+
df = read_dbc(fname, encoding='iso-8859-1')
|
|
39
|
+
if cache:
|
|
40
|
+
df.to_parquet(cachefile)
|
|
41
|
+
os.unlink(fname)
|
|
42
|
+
return df
|
|
43
|
+
|
|
44
|
+
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#-*- coding:utf-8 -*-
|
|
2
|
+
u"""
|
|
3
|
+
This module contains a set of functions to decode
|
|
4
|
+
commonly encoded variables
|
|
5
|
+
Created on 19/07/16
|
|
6
|
+
by fccoelho
|
|
7
|
+
license: GPL V3 or Later
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__docformat__ = 'restructuredtext en'
|
|
11
|
+
import numpy as np
|
|
12
|
+
from datetime import timedelta
|
|
13
|
+
|
|
14
|
+
@np.vectorize
|
|
15
|
+
def decodifica_idade_SINAN(idade, unidade='Y'):
|
|
16
|
+
"""
|
|
17
|
+
Em tabelas do SINAN frequentemente a idade é representada como um inteiro que precisa ser parseado
|
|
18
|
+
para retornar a idade em uma unidade cronológica padrão.
|
|
19
|
+
:param unidade: unidade da idade: 'Y': anos, 'M' meses, 'D': dias, 'H': horas
|
|
20
|
+
:param idade: inteiro ou sequencia de inteiros codificados.
|
|
21
|
+
:return:
|
|
22
|
+
"""
|
|
23
|
+
fator = {'Y': 1., 'M': 12., 'D': 365., 'H': 365*24.}
|
|
24
|
+
if idade >= 4000: #idade em anos
|
|
25
|
+
idade_anos = idade - 4000
|
|
26
|
+
elif idade >= 3000 and idade < 4000: #idade em meses
|
|
27
|
+
idade_anos = (idade-3000)/12.
|
|
28
|
+
elif idade >= 2000 and idade < 3000: #idade em dias
|
|
29
|
+
idade_anos = (idade-2000)/365.
|
|
30
|
+
elif idade >= 1000 and idade < 2000: # idade em horas
|
|
31
|
+
idade_anos = (idade-1000)/(365*24.)
|
|
32
|
+
else:
|
|
33
|
+
#print(idade)
|
|
34
|
+
idade_anos = np.nan
|
|
35
|
+
#raise ValueError("Idade inválida")
|
|
36
|
+
idade_dec = idade_anos*fator[unidade]
|
|
37
|
+
return idade_dec
|
|
38
|
+
|
|
39
|
+
@np.vectorize
|
|
40
|
+
def decodifica_idade_SIM(idade, unidade="D"):
|
|
41
|
+
"""
|
|
42
|
+
Em tabelas do SIM a idade encontra-se codificada
|
|
43
|
+
:param idade: valor original da tabela do SIM
|
|
44
|
+
:param unidade: Unidade de saida desejada: 'Y': anos, 'M' meses, 'D': dias, 'H': horas. Valor default: 'D'
|
|
45
|
+
:return:
|
|
46
|
+
"""
|
|
47
|
+
fator = {'Y': 365., 'M': 30., 'D': 1., 'H': 1/24.}
|
|
48
|
+
if idade.startswith('1'):
|
|
49
|
+
idade = timedelta(hours=int(idade[1:])).days
|
|
50
|
+
elif idade.startswith('2'):
|
|
51
|
+
idade = timedelta(days=int(idade[1:])).days
|
|
52
|
+
elif idade.startswith('3'):
|
|
53
|
+
idade = timedelta(days=int(idade[1:]) * 30).days
|
|
54
|
+
elif idade.startswith('4'):
|
|
55
|
+
idade = timedelta(days=int(idade[1:]) * 365).days
|
|
56
|
+
else:
|
|
57
|
+
idade = np.nan
|
|
58
|
+
return idade/fator.get(unidade, 1)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def is_valid_geocode(geocodigo):
|
|
62
|
+
"""
|
|
63
|
+
Returns True if the geocode is valid
|
|
64
|
+
:param geocodigo:
|
|
65
|
+
:return:
|
|
66
|
+
"""
|
|
67
|
+
if len(str(geocodigo)) != 7:
|
|
68
|
+
raise ValueError('Geocode must have 7 digtis')
|
|
69
|
+
dig = int(str(geocodigo)[-1])
|
|
70
|
+
if dig == calculate_digit(geocodigo):
|
|
71
|
+
return True
|
|
72
|
+
else:
|
|
73
|
+
return False
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def calculate_digit(geocode):
|
|
77
|
+
"""
|
|
78
|
+
Calcula o digito verificador do geocódigo de município com 6 dígitos
|
|
79
|
+
:param geocode: geocódigo com 6 dígitos
|
|
80
|
+
:return: dígito verificador
|
|
81
|
+
"""
|
|
82
|
+
peso = [1, 2, 1, 2, 1, 2, 0]
|
|
83
|
+
soma = 0
|
|
84
|
+
geocode = str(geocode)
|
|
85
|
+
for i in range(6):
|
|
86
|
+
valor = int(geocode[i]) * peso[i]
|
|
87
|
+
soma += sum([int(d) for d in str(valor)]) if valor > 9 else valor
|
|
88
|
+
dv = 0 if soma % 10 == 0 else (10 - (soma % 10))
|
|
89
|
+
return dv
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def add_dv(geocodigo):
|
|
93
|
+
if len(str(geocodigo)) == 7:
|
|
94
|
+
return geocodigo
|
|
95
|
+
else:
|
|
96
|
+
return int(str(geocodigo) + str(calculate_digit(geocodigo)))
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from dbfread import DBF
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import requests
|
|
4
|
+
import geocoder
|
|
5
|
+
from functools import lru_cache
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def read_sinan_dbf(fname, encoding) -> pd.DataFrame:
|
|
10
|
+
"""
|
|
11
|
+
Read SINAN dbf file returning a Pandas Dataframe with
|
|
12
|
+
:param fname: dbf file name
|
|
13
|
+
:param encoding: Encoding of the dbf
|
|
14
|
+
:return: pandas dataframe
|
|
15
|
+
"""
|
|
16
|
+
db = DBF(fname, encoding=encoding)
|
|
17
|
+
df = pd.DataFrame(list(db))
|
|
18
|
+
|
|
19
|
+
def convert_week(x):
|
|
20
|
+
try:
|
|
21
|
+
w = int(x) % 100
|
|
22
|
+
except ValueError:
|
|
23
|
+
w = pd.np.nan
|
|
24
|
+
return w
|
|
25
|
+
for cname in df.columns:
|
|
26
|
+
df[cname].replace('', pd.np.nan, inplace=True)
|
|
27
|
+
if cname.startswith(('NU', 'ID')):
|
|
28
|
+
try:
|
|
29
|
+
df[cname] = pd.to_numeric(df[cname])
|
|
30
|
+
except ValueError as e:
|
|
31
|
+
# certain IDs can be alphanumerical
|
|
32
|
+
pass
|
|
33
|
+
elif cname.startswith('SEM'):
|
|
34
|
+
df[cname] = df[cname].map(convert_week)
|
|
35
|
+
|
|
36
|
+
return df
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@lru_cache(maxsize=None)
|
|
40
|
+
def get_geocodes(geoc):
|
|
41
|
+
"""
|
|
42
|
+
Return city name and state two letter code from geocode
|
|
43
|
+
:param geoc:
|
|
44
|
+
:return:
|
|
45
|
+
"""
|
|
46
|
+
url = "http://cidades.ibge.gov.br/services/jSonpMuns.php?busca=330&featureClass=P&style=full&maxRows=5&name_startsWith={}".format(
|
|
47
|
+
geoc)
|
|
48
|
+
resp = requests.get(url)
|
|
49
|
+
for d in resp.json()['municipios']:
|
|
50
|
+
if int(geoc) == int(d['c']):
|
|
51
|
+
return [d['n'].encode('latin-1').decode('utf-8'), d['s']]
|
|
52
|
+
|
|
53
|
+
else:
|
|
54
|
+
raise KeyError('could not find geocode {} in '.format(geoc))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _address_generator(df, default=''):
|
|
58
|
+
for l in df.iterrows():
|
|
59
|
+
l = dict(l[1])
|
|
60
|
+
try:
|
|
61
|
+
l['cidade'] = ','.join(get_geocodes(l['ID_MN_RESI']))
|
|
62
|
+
except:
|
|
63
|
+
print("Could not find geocode {} using default")
|
|
64
|
+
l['cidade'] = default
|
|
65
|
+
yield l['NU_NOTIFIC'], "{NM_LOGRADO}, {NU_NUMERO}, {NM_BAIRRO}, {cidade}, Brasil".format(**l)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def geocode(sinan_df, outfile, default_city):
|
|
69
|
+
"""
|
|
70
|
+
Geocode cases based on addresses included.
|
|
71
|
+
:param default_city: default city to use in case of bad Geocode found in file. It can be "city, state"
|
|
72
|
+
:param sinan_df: Dataframe generated from sinan DBF
|
|
73
|
+
:param outfile: File on Which
|
|
74
|
+
"""
|
|
75
|
+
addrs = _address_generator(sinan_df, default_city)
|
|
76
|
+
if os.path.exists(outfile):
|
|
77
|
+
mode = 'a'
|
|
78
|
+
coords = pd.read_csv(outfile)
|
|
79
|
+
geocoded = coords.NU_NOTIFIC.tolist()
|
|
80
|
+
else:
|
|
81
|
+
mode = 'w'
|
|
82
|
+
geocoded = []
|
|
83
|
+
with open(outfile, mode) as of:
|
|
84
|
+
if mode == 'w':
|
|
85
|
+
of.write("NU_NOTIFIC,latitude,longitude\n")
|
|
86
|
+
for nu, ad in addrs:
|
|
87
|
+
# ad = ad.encode('latin-1').decode('utf-8')
|
|
88
|
+
if nu in geocoded:
|
|
89
|
+
continue
|
|
90
|
+
location = geocoder.google(ad)
|
|
91
|
+
if location is None:
|
|
92
|
+
raise NameError("Google could not find {}".format(ad))
|
|
93
|
+
if location.latlng == []:
|
|
94
|
+
print("Search for {} returned {} as coordinates, trying reduced address:".format(ad, location.latlng))
|
|
95
|
+
ad = ','.join(ad.split(',')[2:])
|
|
96
|
+
print(ad)
|
|
97
|
+
location = geocoder.google(ad)
|
|
98
|
+
try:
|
|
99
|
+
of.write("{},{},{}\n".format(nu, location.latlng[0], location.latlng[1]))
|
|
100
|
+
print("Successfully geolocated {}".format(ad))
|
|
101
|
+
except IndexError:
|
|
102
|
+
print("Search for {} returned {} as coordinates, skipping".format(ad, location.latlng))
|
|
103
|
+
of.write("{},nan,nan\n".format(nu))
|
pysus/tests/__init__.py
ADDED
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
__author__ = 'fccoelho'
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
from pysus.online_data.CIHA import download
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
unittest.skip("too slow to run om travis")
|
|
8
|
+
class SIHTestCase(unittest.TestCase):
|
|
9
|
+
def test_download_CIH(self):
|
|
10
|
+
df = download('mg', 2009, 7)
|
|
11
|
+
|
|
12
|
+
self.assertGreater(len(df), 0)
|
|
13
|
+
self.assertIn('DIAG_PRINC', df.columns)
|
|
14
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
15
|
+
|
|
16
|
+
def test_download_CIHA(self):
|
|
17
|
+
df = download('MG', 2013, 10)
|
|
18
|
+
self.assertGreater(len(df), 0)
|
|
19
|
+
self.assertIn('DIAG_PRINC', df.columns)
|
|
20
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
21
|
+
|
|
22
|
+
if __name__ == '__main__':
|
|
23
|
+
unittest.main()
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
__author__ = 'fccoelho'
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
from pysus.online_data.SIA import download
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
unittest.skip("too slow to run om travis")
|
|
8
|
+
class SIATestCase(unittest.TestCase):
|
|
9
|
+
def test_download_after_2008(self):
|
|
10
|
+
df, df2 = download('to', 2009, 12)
|
|
11
|
+
self.assertGreater(len(df), 0)
|
|
12
|
+
self.assertIn('PA_CODUNI', df.columns)
|
|
13
|
+
self.assertIn('CODUNI', df2.columns)
|
|
14
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
15
|
+
self.assertIsInstance(df2, pd.DataFrame)
|
|
16
|
+
|
|
17
|
+
def test_download_before_2008(self):
|
|
18
|
+
df, _ = download('mg', 2006, 10)
|
|
19
|
+
self.assertGreater(len(df), 0)
|
|
20
|
+
self.assertIn('PA_CODUNI', df.columns)
|
|
21
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if __name__ == '__main__':
|
|
25
|
+
unittest.main()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
__author__ = 'fccoelho'
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
from pysus.online_data.SIH import download
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
unittest.skip("too slow to run om travis")
|
|
8
|
+
class SIHTestCase(unittest.TestCase):
|
|
9
|
+
def test_download(self):
|
|
10
|
+
df = download('to', 2009, 12)
|
|
11
|
+
df2 = download('AC', 2013, 10)
|
|
12
|
+
self.assertGreater(len(df), 0)
|
|
13
|
+
self.assertGreater(len(df2), 0)
|
|
14
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
if __name__ == '__main__':
|
|
18
|
+
unittest.main()
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
__author__ = 'fccoelho'
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
from unittest import skip
|
|
5
|
+
from pysus.online_data.SIM import download, get_CID10_table, get_CID9_table, get_municipios, get_ocupations
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TestDownload(unittest.TestCase):
|
|
9
|
+
def test_download_CID10(self):
|
|
10
|
+
df = download('ba', 2007)
|
|
11
|
+
self.assertIn('IDADEMAE', df.columns)
|
|
12
|
+
self.assertGreater(len(df), 0)
|
|
13
|
+
|
|
14
|
+
def test_download_CID9(self):
|
|
15
|
+
df = download('mg', 1987)
|
|
16
|
+
self.assertIn('NECROPSIA', df.columns)
|
|
17
|
+
self.assertGreater(len(df), 0)
|
|
18
|
+
|
|
19
|
+
def test_get_cid10(self):
|
|
20
|
+
df = get_CID10_table()
|
|
21
|
+
self.assertIn('CID10', df.columns)
|
|
22
|
+
self.assertGreater(len(df), 0)
|
|
23
|
+
|
|
24
|
+
def test_get_cid9(self):
|
|
25
|
+
df = get_CID9_table()
|
|
26
|
+
self.assertIn('DESCRICAO', df.columns)
|
|
27
|
+
self.assertGreater(len(df), 0)
|
|
28
|
+
|
|
29
|
+
def test_get_mun(self):
|
|
30
|
+
df = get_municipios()
|
|
31
|
+
self.assertIn('MUNCOD', df.columns)
|
|
32
|
+
self.assertGreater(len(df), 0)
|
|
33
|
+
|
|
34
|
+
def test_get_ocup(self):
|
|
35
|
+
df = get_ocupations()
|
|
36
|
+
self.assertIn('CODIGO', df.columns)
|
|
37
|
+
self.assertGreater(len(df), 0)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
if __name__ == '__main__':
|
|
41
|
+
unittest.main()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
__author__ = 'fccoelho'
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
from unittest import skip
|
|
5
|
+
from pysus.online_data.sinasc import download
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TestDownload(unittest.TestCase):
|
|
9
|
+
def test_download_new(self):
|
|
10
|
+
df = download('SE', 2015)
|
|
11
|
+
self.assertIn('IDADEMAE', df.columns)
|
|
12
|
+
self.assertGreater(len(df), 0)
|
|
13
|
+
def test_download_old(self):
|
|
14
|
+
df = download('AL', 1994)
|
|
15
|
+
self.assertIn('IDADE_MAE', df.columns)
|
|
16
|
+
self.assertGreater(len(df), 0)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
if __name__ == '__main__':
|
|
20
|
+
unittest.main()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# -*- coding:utf-8 -*-
|
|
2
|
+
u"""
|
|
3
|
+
Created on 19/07/16
|
|
4
|
+
by fccoelho
|
|
5
|
+
license: GPL V3 or Later
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import unittest
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
from numpy.testing import *
|
|
12
|
+
|
|
13
|
+
from pysus.preprocessing import decoders
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TestDecoder(unittest.TestCase):
|
|
17
|
+
def test_decodifica_idade_retorna_em_anos(self):
|
|
18
|
+
res = decoders.decodifica_idade_SINAN(4010, unidade='Y')
|
|
19
|
+
self.assertEqual(res, 10)
|
|
20
|
+
res = decoders.decodifica_idade_SINAN(3120, unidade='Y')
|
|
21
|
+
self.assertEqual(res, 10)
|
|
22
|
+
res = decoders.decodifica_idade_SINAN(2365, unidade='Y')
|
|
23
|
+
self.assertAlmostEqual(res, 1, places=4)
|
|
24
|
+
res = decoders.decodifica_idade_SINAN(1480, unidade='Y')
|
|
25
|
+
self.assertAlmostEqual(res, 0.0547, places=3)
|
|
26
|
+
|
|
27
|
+
def test_decodifica_lista_idades_retorna_em_anos(self):
|
|
28
|
+
res = decoders.decodifica_idade_SINAN([4010] * 3, unidade='Y')
|
|
29
|
+
assert_array_equal(res, np.array([10] * 3))
|
|
30
|
+
res = decoders.decodifica_idade_SINAN([3120] * 4, unidade='Y')
|
|
31
|
+
assert_array_equal(res, np.array([10] * 4))
|
|
32
|
+
res = decoders.decodifica_idade_SINAN([2365] * 2, unidade='Y')
|
|
33
|
+
assert_array_equal(res, np.array([1, 1]))
|
|
34
|
+
res = decoders.decodifica_idade_SINAN([1480] * 5, unidade='Y')
|
|
35
|
+
assert_array_almost_equal(res, np.array([0.0547] * 5), decimal=3)
|
|
36
|
+
|
|
37
|
+
def test_verifica_geocodigo(self):
|
|
38
|
+
self.assertTrue(decoders.is_valid_geocode(3304557))
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from pysus.preprocessing.sinan import read_sinan_dbf, geocode
|
|
3
|
+
from pysus.online_data.SINAN import download, list_diseases
|
|
4
|
+
import pandas as pd
|
|
5
|
+
import datetime
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TestSINANDownload(unittest.TestCase):
|
|
10
|
+
def test_download(self):
|
|
11
|
+
df = download('AC', 2007, 'Animais Peçonhentos')
|
|
12
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
13
|
+
|
|
14
|
+
def test_lista_agravos(self):
|
|
15
|
+
lista = list_diseases()
|
|
16
|
+
self.assertIsInstance(lista, list)
|
|
17
|
+
self.assertGreater(len(lista), 0)
|
|
18
|
+
|
|
19
|
+
class TestSinanDF(unittest.TestCase):
|
|
20
|
+
def test_read_dbf(self):
|
|
21
|
+
df = read_sinan_dbf('test_data/EPR-2016-06-01-2016.dbf', encoding='latin-1')
|
|
22
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
23
|
+
for cname in df.columns:
|
|
24
|
+
if cname.startswith('DT_'):
|
|
25
|
+
self.assertIsInstance(df[cname][0], datetime.date)
|
|
26
|
+
elif cname.startswith('SEM'):
|
|
27
|
+
self.assertLessEqual(df[cname][0], 52)
|
|
28
|
+
self.assertIsInstance(df[cname][0], (int, np.int64))
|
|
29
|
+
elif cname.startswith(('NU', 'ID')):
|
|
30
|
+
if cname == 'ID_AGRAVO': continue
|
|
31
|
+
self.assertIsInstance(df[cname][0], (int, float, np.int64),
|
|
32
|
+
msg="Failed on column {}, type:{}".format(cname, type(df[cname][0])))
|
|
33
|
+
|
|
34
|
+
def test_type_convertion(self):
|
|
35
|
+
df = read_sinan_dbf('test_data/EPR-2016-06-01-2016.dbf', encoding='latin-1')
|
|
36
|
+
assert not all(df.dtypes == 'object')
|
|
37
|
+
|
|
38
|
+
def test_geocode(self):
|
|
39
|
+
df = pd.read_pickle('test_data/chik.pickle')
|
|
40
|
+
# geocode(sinan_df=df, outfile='chik_2016.csv', default_city='Rio de Janeiro')
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == '__main__':
|
|
44
|
+
unittest.main()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Created on 17/08/16
|
|
3
|
+
by fccoelho
|
|
4
|
+
license: GPL V3 or Later
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import unittest
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from pysus.utilities.readdbc import read_dbc
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TestReadDbc(unittest.TestCase):
|
|
15
|
+
def test_read_dbc(self):
|
|
16
|
+
df = read_dbc(b'test_data/sids.dbc')
|
|
17
|
+
self.assertIsInstance(df, pd.DataFrame)
|
|
18
|
+
self.assertGreater(df.size, 0)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Created on 16/08/16
|
|
3
|
+
by fccoelho
|
|
4
|
+
license: GPL V3 or Later
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
from cffi import FFI
|
|
8
|
+
|
|
9
|
+
ffibuilder = FFI()
|
|
10
|
+
|
|
11
|
+
PATH = os.path.dirname(__file__)
|
|
12
|
+
|
|
13
|
+
with open(os.path.join(PATH, 'c-src/dbc2dbf.c'),'r') as f:
|
|
14
|
+
ffibuilder.set_source("_readdbc",
|
|
15
|
+
f.read(),
|
|
16
|
+
libraries=["c"],
|
|
17
|
+
sources=[os.path.join(PATH, "c-src/blast.c")],
|
|
18
|
+
include_dirs=[os.path.join(PATH, "c-src/")]
|
|
19
|
+
)
|
|
20
|
+
ffibuilder.cdef(
|
|
21
|
+
"""
|
|
22
|
+
static unsigned inf(void *how, unsigned char **buf);
|
|
23
|
+
static int outf(void *how, unsigned char *buf, unsigned len);
|
|
24
|
+
void dbc2dbf(char** input_file, char** output_file);
|
|
25
|
+
"""
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
with open(os.path.join(PATH, "c-src/blast.h")) as f:
|
|
29
|
+
ffibuilder.cdef(f.read(), override=True)
|
|
30
|
+
|
|
31
|
+
if __name__ == "__main__":
|
|
32
|
+
ffibuilder.compile(verbose=True)
|