pysus 0.4.0__py3.6-linux-x86_64.egg
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- EGG-INFO/PKG-INFO +30 -0
- EGG-INFO/SOURCES.txt +75 -0
- EGG-INFO/dependency_links.txt +1 -0
- EGG-INFO/native_libs.txt +2 -0
- EGG-INFO/not-zip-safe +1 -0
- EGG-INFO/requires.txt +7 -0
- EGG-INFO/top_level.txt +2 -0
- __pycache__/_readdbc.cpython-36.pyc +0 -0
- _readdbc.abi3.so +0 -0
- _readdbc.py +7 -0
- pysus/__init__.py +2 -0
- pysus/__pycache__/__init__.cpython-36.pyc +0 -0
- pysus/online_data/CIHA.py +62 -0
- pysus/online_data/CNES.py +24 -0
- pysus/online_data/SIA.py +92 -0
- pysus/online_data/SIH.py +61 -0
- pysus/online_data/SIM.py +155 -0
- pysus/online_data/__init__.py +22 -0
- pysus/online_data/__pycache__/CIHA.cpython-36.pyc +0 -0
- pysus/online_data/__pycache__/CNES.cpython-36.pyc +0 -0
- pysus/online_data/__pycache__/SIA.cpython-36.pyc +0 -0
- pysus/online_data/__pycache__/SIH.cpython-36.pyc +0 -0
- pysus/online_data/__pycache__/SIM.cpython-36.pyc +0 -0
- pysus/online_data/__pycache__/__init__.cpython-36.pyc +0 -0
- pysus/online_data/__pycache__/sinasc.cpython-36.pyc +0 -0
- pysus/online_data/sinasc.py +44 -0
- pysus/preprocessing/__init__.py +8 -0
- pysus/preprocessing/__pycache__/__init__.cpython-36.pyc +0 -0
- pysus/preprocessing/__pycache__/decoders.cpython-36.pyc +0 -0
- pysus/preprocessing/__pycache__/sinan.cpython-36.pyc +0 -0
- pysus/preprocessing/decoders.py +96 -0
- pysus/preprocessing/sinan.py +103 -0
- pysus/tests/__init__.py +8 -0
- pysus/tests/__pycache__/__init__.cpython-36.pyc +0 -0
- pysus/tests/__pycache__/test_decoders.cpython-36.pyc +0 -0
- pysus/tests/__pycache__/test_sinan.cpython-36.pyc +0 -0
- pysus/tests/__pycache__/test_utilities.cpython-36.pyc +0 -0
- pysus/tests/test_data/__init__.py +6 -0
- pysus/tests/test_data/__pycache__/__init__.cpython-36.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_ciha.cpython-36.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sia.cpython-36.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sih.cpython-36.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sim.cpython-36.pyc +0 -0
- pysus/tests/test_data/__pycache__/test_sinasc.cpython-36.pyc +0 -0
- pysus/tests/test_data/test_ciha.py +23 -0
- pysus/tests/test_data/test_sia.py +25 -0
- pysus/tests/test_data/test_sih.py +18 -0
- pysus/tests/test_data/test_sim.py +41 -0
- pysus/tests/test_data/test_sinasc.py +20 -0
- pysus/tests/test_decoders.py +38 -0
- pysus/tests/test_sinan.py +35 -0
- pysus/tests/test_utilities.py +18 -0
- pysus/utilities/__init__.py +6 -0
- pysus/utilities/__pycache__/__init__.cpython-36.pyc +0 -0
- pysus/utilities/__pycache__/_build_readdbc.cpython-36.pyc +0 -0
- pysus/utilities/__pycache__/readdbc.cpython-36.pyc +0 -0
- pysus/utilities/_build_readdbc.py +32 -0
- pysus/utilities/_readdbc.c +897 -0
- pysus/utilities/_readdbc.cpython-36m-x86_64-linux-gnu.so +0 -0
- pysus/utilities/_readdbc.o +0 -0
- pysus/utilities/blast.c +982 -0
- pysus/utilities/blast.h +75 -0
- pysus/utilities/blast.o +0 -0
- pysus/utilities/readdbc.py +52 -0
EGG-INFO/PKG-INFO
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 1.0
|
|
2
|
+
Name: PySUS
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Tools for dealing with Brazil's Public health data
|
|
5
|
+
Home-page: https://github.com/fccoelho/PySUS
|
|
6
|
+
Author: Flavio Codeco Coelho
|
|
7
|
+
Author-email: fccoelho@gmail.com
|
|
8
|
+
License: gpl-v3
|
|
9
|
+
Description:
|
|
10
|
+
PySUS
|
|
11
|
+
=====
|
|
12
|
+
|
|
13
|
+
This package collects a set of utilities for handling with public databases published by Brazil's DATASUS
|
|
14
|
+
The documentation of how to use PySUS can be found [here](http://pysus.readthedocs.io/en/latest/)
|
|
15
|
+
|
|
16
|
+
Features
|
|
17
|
+
--------
|
|
18
|
+
|
|
19
|
+
- Decode encoded patient age to any time unit (years, months, etc)
|
|
20
|
+
- Convert `.dbc` files to DBF databases or read them into pandas dataframes. DBC files are basically DBFs compressed by a proprietary algorithm.
|
|
21
|
+
- Loads SINAN files into Pandas Dataframes
|
|
22
|
+
- Geocodes SINAN notified cases in batch. You can use your Google API KEY to avoid Google's free limits.
|
|
23
|
+
|
|
24
|
+
Instalation
|
|
25
|
+
-----------
|
|
26
|
+
|
|
27
|
+
`$ sudo pip install PySUS`
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
Platform: UNKNOWN
|
EGG-INFO/SOURCES.txt
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
.gitignore
|
|
2
|
+
.travis.yml
|
|
3
|
+
MANIFEST.in
|
|
4
|
+
README.md
|
|
5
|
+
requirements.txt
|
|
6
|
+
setup.cfg
|
|
7
|
+
setup.py
|
|
8
|
+
.circleci/config.yml
|
|
9
|
+
.idea/PySUS.iml
|
|
10
|
+
.idea/misc.xml
|
|
11
|
+
.idea/modules.xml
|
|
12
|
+
.idea/inspectionProfiles/Project_Default.xml
|
|
13
|
+
PySUS.egg-info/PKG-INFO
|
|
14
|
+
PySUS.egg-info/SOURCES.txt
|
|
15
|
+
PySUS.egg-info/dependency_links.txt
|
|
16
|
+
PySUS.egg-info/not-zip-safe
|
|
17
|
+
PySUS.egg-info/requires.txt
|
|
18
|
+
PySUS.egg-info/top_level.txt
|
|
19
|
+
docs/Makefile
|
|
20
|
+
docs/make.bat
|
|
21
|
+
docs/source/Chikungunya.ipynb
|
|
22
|
+
docs/source/Dengue.ipynb
|
|
23
|
+
docs/source/IT_SIHSUS_1603.pdf
|
|
24
|
+
docs/source/Zika.ipynb
|
|
25
|
+
docs/source/conf.py
|
|
26
|
+
docs/source/data-sources.rst
|
|
27
|
+
docs/source/index.rst
|
|
28
|
+
docs/source/rio.html
|
|
29
|
+
docs/source/tutorials.rst
|
|
30
|
+
docs/source/.ipynb_checkpoints/Chikungunya-checkpoint.ipynb
|
|
31
|
+
docs/source/.ipynb_checkpoints/Dengue-checkpoint.ipynb
|
|
32
|
+
docs/source/.ipynb_checkpoints/Zika-checkpoint.ipynb
|
|
33
|
+
pysus/__init__.py
|
|
34
|
+
pysus/Notebooks/Analizing SINASC Data.ipynb
|
|
35
|
+
pysus/Notebooks/Analyzing SIH.ipynb
|
|
36
|
+
pysus/Notebooks/.ipynb_checkpoints/Dengue-checkpoint.ipynb
|
|
37
|
+
pysus/online_data/CIHA.py
|
|
38
|
+
pysus/online_data/CNES.py
|
|
39
|
+
pysus/online_data/SIA.py
|
|
40
|
+
pysus/online_data/SIH.py
|
|
41
|
+
pysus/online_data/SIM.py
|
|
42
|
+
pysus/online_data/__init__.py
|
|
43
|
+
pysus/online_data/sinasc.py
|
|
44
|
+
pysus/preprocessing/__init__.py
|
|
45
|
+
pysus/preprocessing/decoders.py
|
|
46
|
+
pysus/preprocessing/sinan.py
|
|
47
|
+
pysus/tests/__init__.py
|
|
48
|
+
pysus/tests/test_decoders.py
|
|
49
|
+
pysus/tests/test_sinan.py
|
|
50
|
+
pysus/tests/test_utilities.py
|
|
51
|
+
pysus/tests/test_data/EPR-2016-06-01-2016.dbf
|
|
52
|
+
pysus/tests/test_data/__init__.py
|
|
53
|
+
pysus/tests/test_data/chik.pickle
|
|
54
|
+
pysus/tests/test_data/sids.dbc
|
|
55
|
+
pysus/tests/test_data/test_ciha.py
|
|
56
|
+
pysus/tests/test_data/test_sia.py
|
|
57
|
+
pysus/tests/test_data/test_sih.py
|
|
58
|
+
pysus/tests/test_data/test_sim.py
|
|
59
|
+
pysus/tests/test_data/test_sinasc.py
|
|
60
|
+
pysus/utilities/.directory
|
|
61
|
+
pysus/utilities/__init__.py
|
|
62
|
+
pysus/utilities/_build_readdbc.py
|
|
63
|
+
pysus/utilities/_readdbc.c
|
|
64
|
+
pysus/utilities/_readdbc.cpython-36m-x86_64-linux-gnu.so
|
|
65
|
+
pysus/utilities/_readdbc.o
|
|
66
|
+
pysus/utilities/blast.c
|
|
67
|
+
pysus/utilities/blast.h
|
|
68
|
+
pysus/utilities/blast.o
|
|
69
|
+
pysus/utilities/readdbc.py
|
|
70
|
+
pysus/utilities/__pycache__/__init__.cpython-35.pyc
|
|
71
|
+
pysus/utilities/__pycache__/_build_readdbc.cpython-35.pyc
|
|
72
|
+
pysus/utilities/build/_readdbc.c
|
|
73
|
+
pysus/utilities/c-src/blast.c
|
|
74
|
+
pysus/utilities/c-src/blast.h
|
|
75
|
+
pysus/utilities/c-src/dbc2dbf.c
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
EGG-INFO/native_libs.txt
ADDED
EGG-INFO/not-zip-safe
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
EGG-INFO/requires.txt
ADDED
EGG-INFO/top_level.txt
ADDED
|
Binary file
|
_readdbc.abi3.so
ADDED
|
Binary file
|
_readdbc.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
def __bootstrap__():
|
|
2
|
+
global __bootstrap__, __loader__, __file__
|
|
3
|
+
import sys, pkg_resources, imp
|
|
4
|
+
__file__ = pkg_resources.resource_filename(__name__, '_readdbc.abi3.so')
|
|
5
|
+
__loader__ = None; del __bootstrap__, __loader__
|
|
6
|
+
imp.load_dynamic(__name__,__file__)
|
|
7
|
+
__bootstrap__()
|
pysus/__init__.py
ADDED
|
Binary file
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Download data from CIHA and CIH (Old)
|
|
3
|
+
Hospital and Ambulatorial information system
|
|
4
|
+
http://ciha.datasus.gov.br/CIHA/index.php?area=03
|
|
5
|
+
|
|
6
|
+
Created on 12/12/18
|
|
7
|
+
by fccoelho
|
|
8
|
+
license: GPL V3 or Later
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
from ftplib import FTP
|
|
13
|
+
from pysus.utilities.readdbc import read_dbc
|
|
14
|
+
from pysus.online_data import CACHEPATH
|
|
15
|
+
from dbfread import DBF
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def download(state: str, year: int, month: int, cache: bool=True) -> object:
|
|
20
|
+
"""
|
|
21
|
+
Download CIHA records for state, year and month and returns dataframe
|
|
22
|
+
:param month: 1 to 12
|
|
23
|
+
:param state: 2 letter state code
|
|
24
|
+
:param year: 4 digit integer
|
|
25
|
+
"""
|
|
26
|
+
state = state.upper()
|
|
27
|
+
year2 = str(year)[-2:]
|
|
28
|
+
month = str(month).zfill(2)
|
|
29
|
+
if year < 2008:
|
|
30
|
+
raise ValueError("CIHA does not contain data before 2008")
|
|
31
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
32
|
+
ftp.login()
|
|
33
|
+
if year > 2008 and year < 2011:
|
|
34
|
+
ftype = 'DBC'
|
|
35
|
+
ftp.cwd('/dissemin/publicos/CIH/200801_201012/Dados')
|
|
36
|
+
fname = 'CR{}{}{}.dbc'.format(state, year2, month)
|
|
37
|
+
if year >= 2011:
|
|
38
|
+
ftype = 'DBC'
|
|
39
|
+
ftp.cwd('/dissemin/publicos/CIHA/201101_/Dados'.format(year))
|
|
40
|
+
fname = 'CIHA{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
|
|
41
|
+
cachefile = os.path.join(CACHEPATH, 'CIHA_' + fname.split('.')[0] + '_.parquet')
|
|
42
|
+
if os.path.exists(cachefile):
|
|
43
|
+
df = pd.read_parquet(cachefile)
|
|
44
|
+
return df
|
|
45
|
+
df = _fetch_file(fname, ftp, ftype)
|
|
46
|
+
if cache:
|
|
47
|
+
df.to_parquet(cachefile)
|
|
48
|
+
return df
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _fetch_file(fname, ftp, ftype):
|
|
52
|
+
try:
|
|
53
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
54
|
+
except:
|
|
55
|
+
raise Exception("File {} not available".format(fname))
|
|
56
|
+
if ftype == 'DBC':
|
|
57
|
+
df = read_dbc(fname, encoding='iso-8859-1')
|
|
58
|
+
elif ftype == 'DBF':
|
|
59
|
+
dbf = DBF(fname, encoding='iso-8859-1')
|
|
60
|
+
df = pd.DataFrame(list(dbf))
|
|
61
|
+
os.unlink(fname)
|
|
62
|
+
return df
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Created on 12/12/18
|
|
3
|
+
by fccoelho
|
|
4
|
+
license: GPL V3 or Later
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
from ftplib import FTP
|
|
8
|
+
from pysus.utilities.readdbc import read_dbc
|
|
9
|
+
from dbfread import DBF
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def profissionais(state: str, year: int, month: int = 0):
|
|
14
|
+
"""
|
|
15
|
+
Fetch registry of health professionals
|
|
16
|
+
:param state:
|
|
17
|
+
:param year:
|
|
18
|
+
:param month:
|
|
19
|
+
:return:
|
|
20
|
+
"""
|
|
21
|
+
state = state.upper()
|
|
22
|
+
month = str(month).zfill(2)
|
|
23
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
24
|
+
ftp.login()
|
pysus/online_data/SIA.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Downloads SIA data from Datasus FTP server
|
|
3
|
+
Created on 21/09/18
|
|
4
|
+
by fccoelho
|
|
5
|
+
license: GPL V3 or Later
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from ftplib import FTP
|
|
10
|
+
from pysus.utilities.readdbc import read_dbc
|
|
11
|
+
from dbfread import DBF
|
|
12
|
+
import pandas as pd
|
|
13
|
+
from pysus.online_data import CACHEPATH
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def download(state: str, year: int, month: int, cache: bool =True) -> object:
|
|
17
|
+
"""
|
|
18
|
+
Download SIH records for state year and month and returns dataframe
|
|
19
|
+
:param month: 1 to 12
|
|
20
|
+
:param state: 2 letter state code
|
|
21
|
+
:param year: 4 digit integer
|
|
22
|
+
:param cache: whether to cache files locally. default is True
|
|
23
|
+
:return: Two dataframes PA and BI
|
|
24
|
+
"""
|
|
25
|
+
state = state.upper()
|
|
26
|
+
year2 = str(year)[-2:]
|
|
27
|
+
month = str(month).zfill(2)
|
|
28
|
+
if year < 1992:
|
|
29
|
+
raise ValueError("SIH does not contain data before 1994")
|
|
30
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
31
|
+
ftp.login()
|
|
32
|
+
if year < 2008 and year > 1994:
|
|
33
|
+
ftype = 'DBC'
|
|
34
|
+
ftp.cwd('/dissemin/publicos/SIASUS/199407_200712/Dados')
|
|
35
|
+
fname = 'PA{}{}{}.dbc'.format(state, year2, month)
|
|
36
|
+
fname2 = None
|
|
37
|
+
if year >= 2008:
|
|
38
|
+
ftype = 'DBC'
|
|
39
|
+
ftp.cwd('/dissemin/publicos/SIASUS/200801_/Dados'.format(year))
|
|
40
|
+
fname = 'PA{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
|
|
41
|
+
fname2 = 'BI{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
|
|
42
|
+
# Check in Cache
|
|
43
|
+
cachefile = os.path.join(CACHEPATH, 'SIA_' + fname.split('.')[0] + '_.parquet')
|
|
44
|
+
if os.path.exists(cachefile):
|
|
45
|
+
df = pd.read_parquet(cachefile)
|
|
46
|
+
else:
|
|
47
|
+
df = _fetch_file(fname, ftp, ftype)
|
|
48
|
+
if cache:
|
|
49
|
+
df.to_parquet(cachefile)
|
|
50
|
+
if fname2 is not None:
|
|
51
|
+
cachefile2 = os.path.join(CACHEPATH, 'SIA_' + fname2.split('.')[0] + '_.parquet')
|
|
52
|
+
if os.path.exists(cachefile2): # reads from cache
|
|
53
|
+
df2 = pd.read_parquet(cachefile2)
|
|
54
|
+
else: # fetches from DataSUS
|
|
55
|
+
try:
|
|
56
|
+
df2 = _fetch_file(fname2, ftp, ftype)
|
|
57
|
+
if cache: #saves to cache
|
|
58
|
+
df2.to_parquet(cachefile2)
|
|
59
|
+
except Exception as e:
|
|
60
|
+
df2 = None
|
|
61
|
+
print(e)
|
|
62
|
+
else:
|
|
63
|
+
df2 = None
|
|
64
|
+
|
|
65
|
+
return df, df2
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _fetch_file(fname, ftp, ftype):
|
|
69
|
+
"""
|
|
70
|
+
Does the FTP fetching.
|
|
71
|
+
:param fname: file name
|
|
72
|
+
:param ftp: ftp connection object
|
|
73
|
+
:param ftype: file type: DBF|DBC
|
|
74
|
+
:return: pandas dataframe
|
|
75
|
+
"""
|
|
76
|
+
print("Downloading {}...".format(fname))
|
|
77
|
+
try:
|
|
78
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
79
|
+
except:
|
|
80
|
+
try:
|
|
81
|
+
ftp.retrbinary('RETR {}'.format(fname.lower()), open(fname, 'wb').write)
|
|
82
|
+
except:
|
|
83
|
+
raise Exception("File {} not available".format(fname))
|
|
84
|
+
if ftype == 'DBC':
|
|
85
|
+
df = read_dbc(fname, encoding='iso-8859-1')
|
|
86
|
+
elif ftype == 'DBF':
|
|
87
|
+
dbf = DBF(fname, encoding='iso-8859-1')
|
|
88
|
+
df = pd.DataFrame(list(dbf))
|
|
89
|
+
os.unlink(fname)
|
|
90
|
+
return df
|
|
91
|
+
|
|
92
|
+
|
pysus/online_data/SIH.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Downloads SIH data from Datasus FTP server
|
|
3
|
+
Created on 21/09/18
|
|
4
|
+
by fccoelho
|
|
5
|
+
license: GPL V3 or Later
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from ftplib import FTP
|
|
10
|
+
from pysus.utilities.readdbc import read_dbc
|
|
11
|
+
from pysus.online_data import CACHEPATH
|
|
12
|
+
from dbfread import DBF
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def download(state: str, year: int, month: int, cache: bool=True) -> object:
|
|
17
|
+
"""
|
|
18
|
+
Download SIH records for state year and month and returns dataframe
|
|
19
|
+
:param month: 1 to 12
|
|
20
|
+
:param state: 2 letter state code
|
|
21
|
+
:param year: 4 digit integer
|
|
22
|
+
"""
|
|
23
|
+
state = state.upper()
|
|
24
|
+
year2 = int(str(year)[-2:])
|
|
25
|
+
month = str(month).zfill(2)
|
|
26
|
+
if year < 1992:
|
|
27
|
+
raise ValueError("SIH does not contain data before 1994")
|
|
28
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
29
|
+
ftp.login()
|
|
30
|
+
if year < 2008:
|
|
31
|
+
ftype = 'DBC'
|
|
32
|
+
ftp.cwd('/dissemin/publicos/SIHSUS/199201_200712/Dados')
|
|
33
|
+
fname = 'RD{}{}{}.dbc'.format(state, year2, month)
|
|
34
|
+
if year >= 2008:
|
|
35
|
+
ftype = 'DBC'
|
|
36
|
+
ftp.cwd('/dissemin/publicos/SIHSUS/200801_/Dados'.format(year))
|
|
37
|
+
fname = 'RD{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
|
|
38
|
+
cachefile = os.path.join(CACHEPATH, 'SIH_' + fname.split('.')[0] + '_.parquet')
|
|
39
|
+
if os.path.exists(cachefile):
|
|
40
|
+
df = pd.read_parquet(cachefile)
|
|
41
|
+
return df
|
|
42
|
+
df = _fetch_file(fname, ftp, ftype)
|
|
43
|
+
if cache:
|
|
44
|
+
df.to_parquet(cachefile)
|
|
45
|
+
return df
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _fetch_file(fname, ftp, ftype):
|
|
49
|
+
try:
|
|
50
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
51
|
+
except:
|
|
52
|
+
raise Exception("File {} not available".format(fname))
|
|
53
|
+
if ftype == 'DBC':
|
|
54
|
+
df = read_dbc(fname, encoding='iso-8859-1')
|
|
55
|
+
elif ftype == 'DBF':
|
|
56
|
+
dbf = DBF(fname, encoding='iso-8859-1')
|
|
57
|
+
df = pd.DataFrame(list(dbf))
|
|
58
|
+
os.unlink(fname)
|
|
59
|
+
return df
|
|
60
|
+
|
|
61
|
+
|
pysus/online_data/SIM.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Download Mortality records from SIM Datasus
|
|
3
|
+
Created on 12/12/18
|
|
4
|
+
by fccoelho
|
|
5
|
+
license: GPL V3 or Later
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from ftplib import FTP
|
|
10
|
+
from pysus.utilities.readdbc import read_dbc
|
|
11
|
+
from pysus.online_data import CACHEPATH
|
|
12
|
+
from dbfread import DBF
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def download(state, year, cache=True):
|
|
17
|
+
"""
|
|
18
|
+
Downloads data directly from Datasus ftp server
|
|
19
|
+
:param state: two-letter state identifier: MG == Minas Gerais
|
|
20
|
+
:param year: 4 digit integer
|
|
21
|
+
:return: pandas dataframe
|
|
22
|
+
"""
|
|
23
|
+
year2 = str(year)[-2:].zfill(2)
|
|
24
|
+
state = state.upper()
|
|
25
|
+
if year < 1979:
|
|
26
|
+
raise ValueError("SIM does not contain data before 1979")
|
|
27
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
28
|
+
ftp.login()
|
|
29
|
+
if year >= 1996:
|
|
30
|
+
ftp.cwd('/dissemin/publicos/SIM/CID10/DORES')
|
|
31
|
+
fname = 'DO{}{}.DBC'.format(state, year)
|
|
32
|
+
else:
|
|
33
|
+
ftp.cwd('/dissemin/publicos/SIM/CID9/DORES')
|
|
34
|
+
fname = 'DOR{}{}.DBC'.format(state, year2)
|
|
35
|
+
cachefile = os.path.join(CACHEPATH, 'SIM_'+fname.split('.')[0] + '_.parquet')
|
|
36
|
+
if os.path.exists(cachefile):
|
|
37
|
+
df = pd.read_parquet(cachefile)
|
|
38
|
+
return df
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
42
|
+
except:
|
|
43
|
+
try:
|
|
44
|
+
ftp.retrbinary('RETR {}'.format(fname.upper()), open(fname, 'wb').write)
|
|
45
|
+
except:
|
|
46
|
+
raise Exception("File {} not available".format(fname))
|
|
47
|
+
|
|
48
|
+
df = read_dbc(fname, encoding='iso-8859-1')
|
|
49
|
+
if cache:
|
|
50
|
+
df.to_parquet(cachefile)
|
|
51
|
+
os.unlink(fname)
|
|
52
|
+
return df
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def get_CID10_table(cache=True):
|
|
56
|
+
"""
|
|
57
|
+
Fetch the CID10 table
|
|
58
|
+
:param cache:
|
|
59
|
+
:return:
|
|
60
|
+
"""
|
|
61
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
62
|
+
ftp.login()
|
|
63
|
+
ftp.cwd('/dissemin/publicos/SIM/CID10/TABELAS')
|
|
64
|
+
fname = 'CID10.DBF'
|
|
65
|
+
cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
|
|
66
|
+
if os.path.exists(cachefile):
|
|
67
|
+
df = pd.read_parquet(cachefile)
|
|
68
|
+
return df
|
|
69
|
+
try:
|
|
70
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
71
|
+
except:
|
|
72
|
+
raise Exception('Could not download {}'.format(fname))
|
|
73
|
+
dbf = DBF(fname, encoding='iso-8859-1')
|
|
74
|
+
df = pd.DataFrame(list(dbf))
|
|
75
|
+
if cache:
|
|
76
|
+
df.to_parquet(cachefile)
|
|
77
|
+
os.unlink(fname)
|
|
78
|
+
return df
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def get_CID9_table(cache=True):
|
|
82
|
+
"""
|
|
83
|
+
Fetch the CID9 table
|
|
84
|
+
:param cache:
|
|
85
|
+
:return:
|
|
86
|
+
"""
|
|
87
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
88
|
+
ftp.login()
|
|
89
|
+
ftp.cwd('/dissemin/publicos/SIM/CID9/TABELAS')
|
|
90
|
+
fname = 'CID9.DBF'
|
|
91
|
+
cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
|
|
92
|
+
if os.path.exists(cachefile):
|
|
93
|
+
df = pd.read_parquet(cachefile)
|
|
94
|
+
return df
|
|
95
|
+
try:
|
|
96
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
97
|
+
except:
|
|
98
|
+
raise Exception('Could not download {}'.format(fname))
|
|
99
|
+
dbf = DBF(fname, encoding='iso-8859-1')
|
|
100
|
+
df = pd.DataFrame(list(dbf))
|
|
101
|
+
if cache:
|
|
102
|
+
df.to_parquet(cachefile)
|
|
103
|
+
os.unlink(fname)
|
|
104
|
+
return df
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def get_municipios(cache=True):
|
|
108
|
+
"""
|
|
109
|
+
Get municipality metadata
|
|
110
|
+
:param cache:
|
|
111
|
+
:return:
|
|
112
|
+
"""
|
|
113
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
114
|
+
ftp.login()
|
|
115
|
+
ftp.cwd('/dissemin/publicos/SIM/CID10/TABELAS')
|
|
116
|
+
fname = 'CADMUN.DBF'
|
|
117
|
+
cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
|
|
118
|
+
if os.path.exists(cachefile):
|
|
119
|
+
df = pd.read_parquet(cachefile)
|
|
120
|
+
return df
|
|
121
|
+
try:
|
|
122
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
123
|
+
except:
|
|
124
|
+
raise Exception('Could not download {}'.format(fname))
|
|
125
|
+
dbf = DBF(fname, encoding='iso-8859-1')
|
|
126
|
+
df = pd.DataFrame(list(dbf))
|
|
127
|
+
if cache:
|
|
128
|
+
df.to_parquet(cachefile)
|
|
129
|
+
os.unlink(fname)
|
|
130
|
+
return df
|
|
131
|
+
|
|
132
|
+
def get_ocupations(cache=True):
|
|
133
|
+
"""
|
|
134
|
+
Fetch ocupations table
|
|
135
|
+
:param cache:
|
|
136
|
+
:return:
|
|
137
|
+
"""
|
|
138
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
139
|
+
ftp.login()
|
|
140
|
+
ftp.cwd('/dissemin/publicos/SIM/CID10/TABELAS')
|
|
141
|
+
fname = 'TABOCUP.DBF'
|
|
142
|
+
cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
|
|
143
|
+
if os.path.exists(cachefile):
|
|
144
|
+
df = pd.read_parquet(cachefile)
|
|
145
|
+
return df
|
|
146
|
+
try:
|
|
147
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
148
|
+
except:
|
|
149
|
+
raise Exception('Could not download {}'.format(fname))
|
|
150
|
+
dbf = DBF(fname, encoding='iso-8859-1')
|
|
151
|
+
df = pd.DataFrame(list(dbf))
|
|
152
|
+
if cache:
|
|
153
|
+
df.to_parquet(cachefile)
|
|
154
|
+
os.unlink(fname)
|
|
155
|
+
return df
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
u"""
|
|
2
|
+
Created on 21/09/18
|
|
3
|
+
by fccoelho
|
|
4
|
+
license: GPL V3 or Later
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
# create pysus cache directory
|
|
10
|
+
if not os.path.exists(os.path.join(Path.home(), 'pysus')):
|
|
11
|
+
os.mkdir(os.path.join(Path.home(), 'pysus'))
|
|
12
|
+
|
|
13
|
+
CACHEPATH = os.path.join(Path.home(), 'pysus')
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def cache_contents():
|
|
17
|
+
"""
|
|
18
|
+
List the files currently cached in ~/pysus
|
|
19
|
+
:return:
|
|
20
|
+
"""
|
|
21
|
+
cached_data = os.listdir(CACHEPATH)
|
|
22
|
+
return [os.path.join(CACHEPATH, f) for f in cached_data]
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Download SINASC data from DATASUS FTP server
|
|
3
|
+
Created on 01/11/17
|
|
4
|
+
by fccoelho
|
|
5
|
+
license: GPL V3 or Later
|
|
6
|
+
"""
|
|
7
|
+
import os
|
|
8
|
+
from ftplib import FTP
|
|
9
|
+
from pysus.utilities.readdbc import read_dbc
|
|
10
|
+
from pysus.online_data import CACHEPATH
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def download(state, year, cache=True):
|
|
15
|
+
"""
|
|
16
|
+
Downloads data directly from Datasus ftp server
|
|
17
|
+
:param state: two-letter state identifier: MG == Minas Gerais
|
|
18
|
+
:param year: 4 digit integer
|
|
19
|
+
:return: pandas dataframe
|
|
20
|
+
"""
|
|
21
|
+
state = state.upper()
|
|
22
|
+
if year < 1994:
|
|
23
|
+
raise ValueError("SINASC does not contain data before 1994")
|
|
24
|
+
ftp = FTP('ftp.datasus.gov.br')
|
|
25
|
+
ftp.login()
|
|
26
|
+
if year >= 1996:
|
|
27
|
+
ftp.cwd('/dissemin/publicos/SINASC/NOV/DNRES')
|
|
28
|
+
fname = 'DN{}{}.DBC'.format(state, year)
|
|
29
|
+
else:
|
|
30
|
+
ftp.cwd('/dissemin/publicos/SINASC/ANT/DNRES')
|
|
31
|
+
fname = 'DNR{}{}.DBC'.format(state, str(year)[-2:])
|
|
32
|
+
cachefile = os.path.join(CACHEPATH, 'SINASC_'+fname.split('.')[0] + '_.parquet')
|
|
33
|
+
if os.path.exists(cachefile):
|
|
34
|
+
df = pd.read_parquet(cachefile)
|
|
35
|
+
return df
|
|
36
|
+
|
|
37
|
+
ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
|
|
38
|
+
df = read_dbc(fname, encoding='iso-8859-1')
|
|
39
|
+
if cache:
|
|
40
|
+
df.to_parquet(cachefile)
|
|
41
|
+
os.unlink(fname)
|
|
42
|
+
return df
|
|
43
|
+
|
|
44
|
+
|
|
Binary file
|
|
Binary file
|
|
Binary file
|