pysus 0.4.0__py3.6-linux-x86_64.egg

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. EGG-INFO/PKG-INFO +30 -0
  2. EGG-INFO/SOURCES.txt +75 -0
  3. EGG-INFO/dependency_links.txt +1 -0
  4. EGG-INFO/native_libs.txt +2 -0
  5. EGG-INFO/not-zip-safe +1 -0
  6. EGG-INFO/requires.txt +7 -0
  7. EGG-INFO/top_level.txt +2 -0
  8. __pycache__/_readdbc.cpython-36.pyc +0 -0
  9. _readdbc.abi3.so +0 -0
  10. _readdbc.py +7 -0
  11. pysus/__init__.py +2 -0
  12. pysus/__pycache__/__init__.cpython-36.pyc +0 -0
  13. pysus/online_data/CIHA.py +62 -0
  14. pysus/online_data/CNES.py +24 -0
  15. pysus/online_data/SIA.py +92 -0
  16. pysus/online_data/SIH.py +61 -0
  17. pysus/online_data/SIM.py +155 -0
  18. pysus/online_data/__init__.py +22 -0
  19. pysus/online_data/__pycache__/CIHA.cpython-36.pyc +0 -0
  20. pysus/online_data/__pycache__/CNES.cpython-36.pyc +0 -0
  21. pysus/online_data/__pycache__/SIA.cpython-36.pyc +0 -0
  22. pysus/online_data/__pycache__/SIH.cpython-36.pyc +0 -0
  23. pysus/online_data/__pycache__/SIM.cpython-36.pyc +0 -0
  24. pysus/online_data/__pycache__/__init__.cpython-36.pyc +0 -0
  25. pysus/online_data/__pycache__/sinasc.cpython-36.pyc +0 -0
  26. pysus/online_data/sinasc.py +44 -0
  27. pysus/preprocessing/__init__.py +8 -0
  28. pysus/preprocessing/__pycache__/__init__.cpython-36.pyc +0 -0
  29. pysus/preprocessing/__pycache__/decoders.cpython-36.pyc +0 -0
  30. pysus/preprocessing/__pycache__/sinan.cpython-36.pyc +0 -0
  31. pysus/preprocessing/decoders.py +96 -0
  32. pysus/preprocessing/sinan.py +103 -0
  33. pysus/tests/__init__.py +8 -0
  34. pysus/tests/__pycache__/__init__.cpython-36.pyc +0 -0
  35. pysus/tests/__pycache__/test_decoders.cpython-36.pyc +0 -0
  36. pysus/tests/__pycache__/test_sinan.cpython-36.pyc +0 -0
  37. pysus/tests/__pycache__/test_utilities.cpython-36.pyc +0 -0
  38. pysus/tests/test_data/__init__.py +6 -0
  39. pysus/tests/test_data/__pycache__/__init__.cpython-36.pyc +0 -0
  40. pysus/tests/test_data/__pycache__/test_ciha.cpython-36.pyc +0 -0
  41. pysus/tests/test_data/__pycache__/test_sia.cpython-36.pyc +0 -0
  42. pysus/tests/test_data/__pycache__/test_sih.cpython-36.pyc +0 -0
  43. pysus/tests/test_data/__pycache__/test_sim.cpython-36.pyc +0 -0
  44. pysus/tests/test_data/__pycache__/test_sinasc.cpython-36.pyc +0 -0
  45. pysus/tests/test_data/test_ciha.py +23 -0
  46. pysus/tests/test_data/test_sia.py +25 -0
  47. pysus/tests/test_data/test_sih.py +18 -0
  48. pysus/tests/test_data/test_sim.py +41 -0
  49. pysus/tests/test_data/test_sinasc.py +20 -0
  50. pysus/tests/test_decoders.py +38 -0
  51. pysus/tests/test_sinan.py +35 -0
  52. pysus/tests/test_utilities.py +18 -0
  53. pysus/utilities/__init__.py +6 -0
  54. pysus/utilities/__pycache__/__init__.cpython-36.pyc +0 -0
  55. pysus/utilities/__pycache__/_build_readdbc.cpython-36.pyc +0 -0
  56. pysus/utilities/__pycache__/readdbc.cpython-36.pyc +0 -0
  57. pysus/utilities/_build_readdbc.py +32 -0
  58. pysus/utilities/_readdbc.c +897 -0
  59. pysus/utilities/_readdbc.cpython-36m-x86_64-linux-gnu.so +0 -0
  60. pysus/utilities/_readdbc.o +0 -0
  61. pysus/utilities/blast.c +982 -0
  62. pysus/utilities/blast.h +75 -0
  63. pysus/utilities/blast.o +0 -0
  64. pysus/utilities/readdbc.py +52 -0
EGG-INFO/PKG-INFO ADDED
@@ -0,0 +1,30 @@
1
+ Metadata-Version: 1.0
2
+ Name: PySUS
3
+ Version: 0.4.0
4
+ Summary: Tools for dealing with Brazil's Public health data
5
+ Home-page: https://github.com/fccoelho/PySUS
6
+ Author: Flavio Codeco Coelho
7
+ Author-email: fccoelho@gmail.com
8
+ License: gpl-v3
9
+ Description:
10
+ PySUS
11
+ =====
12
+
13
+ This package collects a set of utilities for handling with public databases published by Brazil's DATASUS
14
+ The documentation of how to use PySUS can be found [here](http://pysus.readthedocs.io/en/latest/)
15
+
16
+ Features
17
+ --------
18
+
19
+ - Decode encoded patient age to any time unit (years, months, etc)
20
+ - Convert `.dbc` files to DBF databases or read them into pandas dataframes. DBC files are basically DBFs compressed by a proprietary algorithm.
21
+ - Loads SINAN files into Pandas Dataframes
22
+ - Geocodes SINAN notified cases in batch. You can use your Google API KEY to avoid Google's free limits.
23
+
24
+ Instalation
25
+ -----------
26
+
27
+ `$ sudo pip install PySUS`
28
+
29
+
30
+ Platform: UNKNOWN
EGG-INFO/SOURCES.txt ADDED
@@ -0,0 +1,75 @@
1
+ .gitignore
2
+ .travis.yml
3
+ MANIFEST.in
4
+ README.md
5
+ requirements.txt
6
+ setup.cfg
7
+ setup.py
8
+ .circleci/config.yml
9
+ .idea/PySUS.iml
10
+ .idea/misc.xml
11
+ .idea/modules.xml
12
+ .idea/inspectionProfiles/Project_Default.xml
13
+ PySUS.egg-info/PKG-INFO
14
+ PySUS.egg-info/SOURCES.txt
15
+ PySUS.egg-info/dependency_links.txt
16
+ PySUS.egg-info/not-zip-safe
17
+ PySUS.egg-info/requires.txt
18
+ PySUS.egg-info/top_level.txt
19
+ docs/Makefile
20
+ docs/make.bat
21
+ docs/source/Chikungunya.ipynb
22
+ docs/source/Dengue.ipynb
23
+ docs/source/IT_SIHSUS_1603.pdf
24
+ docs/source/Zika.ipynb
25
+ docs/source/conf.py
26
+ docs/source/data-sources.rst
27
+ docs/source/index.rst
28
+ docs/source/rio.html
29
+ docs/source/tutorials.rst
30
+ docs/source/.ipynb_checkpoints/Chikungunya-checkpoint.ipynb
31
+ docs/source/.ipynb_checkpoints/Dengue-checkpoint.ipynb
32
+ docs/source/.ipynb_checkpoints/Zika-checkpoint.ipynb
33
+ pysus/__init__.py
34
+ pysus/Notebooks/Analizing SINASC Data.ipynb
35
+ pysus/Notebooks/Analyzing SIH.ipynb
36
+ pysus/Notebooks/.ipynb_checkpoints/Dengue-checkpoint.ipynb
37
+ pysus/online_data/CIHA.py
38
+ pysus/online_data/CNES.py
39
+ pysus/online_data/SIA.py
40
+ pysus/online_data/SIH.py
41
+ pysus/online_data/SIM.py
42
+ pysus/online_data/__init__.py
43
+ pysus/online_data/sinasc.py
44
+ pysus/preprocessing/__init__.py
45
+ pysus/preprocessing/decoders.py
46
+ pysus/preprocessing/sinan.py
47
+ pysus/tests/__init__.py
48
+ pysus/tests/test_decoders.py
49
+ pysus/tests/test_sinan.py
50
+ pysus/tests/test_utilities.py
51
+ pysus/tests/test_data/EPR-2016-06-01-2016.dbf
52
+ pysus/tests/test_data/__init__.py
53
+ pysus/tests/test_data/chik.pickle
54
+ pysus/tests/test_data/sids.dbc
55
+ pysus/tests/test_data/test_ciha.py
56
+ pysus/tests/test_data/test_sia.py
57
+ pysus/tests/test_data/test_sih.py
58
+ pysus/tests/test_data/test_sim.py
59
+ pysus/tests/test_data/test_sinasc.py
60
+ pysus/utilities/.directory
61
+ pysus/utilities/__init__.py
62
+ pysus/utilities/_build_readdbc.py
63
+ pysus/utilities/_readdbc.c
64
+ pysus/utilities/_readdbc.cpython-36m-x86_64-linux-gnu.so
65
+ pysus/utilities/_readdbc.o
66
+ pysus/utilities/blast.c
67
+ pysus/utilities/blast.h
68
+ pysus/utilities/blast.o
69
+ pysus/utilities/readdbc.py
70
+ pysus/utilities/__pycache__/__init__.cpython-35.pyc
71
+ pysus/utilities/__pycache__/_build_readdbc.cpython-35.pyc
72
+ pysus/utilities/build/_readdbc.c
73
+ pysus/utilities/c-src/blast.c
74
+ pysus/utilities/c-src/blast.h
75
+ pysus/utilities/c-src/dbc2dbf.c
@@ -0,0 +1 @@
1
+
@@ -0,0 +1,2 @@
1
+ _readdbc.abi3.so
2
+ pysus/utilities/_readdbc.cpython-36m-x86_64-linux-gnu.so
EGG-INFO/not-zip-safe ADDED
@@ -0,0 +1 @@
1
+
EGG-INFO/requires.txt ADDED
@@ -0,0 +1,7 @@
1
+ pandas
2
+ dbfread
3
+ cffi>=1.0.0
4
+ geocoder
5
+ requests
6
+ pyarrow
7
+ fastparquet
EGG-INFO/top_level.txt ADDED
@@ -0,0 +1,2 @@
1
+ _readdbc
2
+ pysus
Binary file
_readdbc.abi3.so ADDED
Binary file
_readdbc.py ADDED
@@ -0,0 +1,7 @@
1
+ def __bootstrap__():
2
+ global __bootstrap__, __loader__, __file__
3
+ import sys, pkg_resources, imp
4
+ __file__ = pkg_resources.resource_filename(__name__, '_readdbc.abi3.so')
5
+ __loader__ = None; del __bootstrap__, __loader__
6
+ imp.load_dynamic(__name__,__file__)
7
+ __bootstrap__()
pysus/__init__.py ADDED
@@ -0,0 +1,2 @@
1
+ from . import preprocessing
2
+ from . import utilities
@@ -0,0 +1,62 @@
1
+ u"""
2
+ Download data from CIHA and CIH (Old)
3
+ Hospital and Ambulatorial information system
4
+ http://ciha.datasus.gov.br/CIHA/index.php?area=03
5
+
6
+ Created on 12/12/18
7
+ by fccoelho
8
+ license: GPL V3 or Later
9
+ """
10
+
11
+ import os
12
+ from ftplib import FTP
13
+ from pysus.utilities.readdbc import read_dbc
14
+ from pysus.online_data import CACHEPATH
15
+ from dbfread import DBF
16
+ import pandas as pd
17
+
18
+
19
+ def download(state: str, year: int, month: int, cache: bool=True) -> object:
20
+ """
21
+ Download CIHA records for state, year and month and returns dataframe
22
+ :param month: 1 to 12
23
+ :param state: 2 letter state code
24
+ :param year: 4 digit integer
25
+ """
26
+ state = state.upper()
27
+ year2 = str(year)[-2:]
28
+ month = str(month).zfill(2)
29
+ if year < 2008:
30
+ raise ValueError("CIHA does not contain data before 2008")
31
+ ftp = FTP('ftp.datasus.gov.br')
32
+ ftp.login()
33
+ if year > 2008 and year < 2011:
34
+ ftype = 'DBC'
35
+ ftp.cwd('/dissemin/publicos/CIH/200801_201012/Dados')
36
+ fname = 'CR{}{}{}.dbc'.format(state, year2, month)
37
+ if year >= 2011:
38
+ ftype = 'DBC'
39
+ ftp.cwd('/dissemin/publicos/CIHA/201101_/Dados'.format(year))
40
+ fname = 'CIHA{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
41
+ cachefile = os.path.join(CACHEPATH, 'CIHA_' + fname.split('.')[0] + '_.parquet')
42
+ if os.path.exists(cachefile):
43
+ df = pd.read_parquet(cachefile)
44
+ return df
45
+ df = _fetch_file(fname, ftp, ftype)
46
+ if cache:
47
+ df.to_parquet(cachefile)
48
+ return df
49
+
50
+
51
+ def _fetch_file(fname, ftp, ftype):
52
+ try:
53
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
54
+ except:
55
+ raise Exception("File {} not available".format(fname))
56
+ if ftype == 'DBC':
57
+ df = read_dbc(fname, encoding='iso-8859-1')
58
+ elif ftype == 'DBF':
59
+ dbf = DBF(fname, encoding='iso-8859-1')
60
+ df = pd.DataFrame(list(dbf))
61
+ os.unlink(fname)
62
+ return df
@@ -0,0 +1,24 @@
1
+ u"""
2
+ Created on 12/12/18
3
+ by fccoelho
4
+ license: GPL V3 or Later
5
+ """
6
+ import os
7
+ from ftplib import FTP
8
+ from pysus.utilities.readdbc import read_dbc
9
+ from dbfread import DBF
10
+ import pandas as pd
11
+
12
+
13
+ def profissionais(state: str, year: int, month: int = 0):
14
+ """
15
+ Fetch registry of health professionals
16
+ :param state:
17
+ :param year:
18
+ :param month:
19
+ :return:
20
+ """
21
+ state = state.upper()
22
+ month = str(month).zfill(2)
23
+ ftp = FTP('ftp.datasus.gov.br')
24
+ ftp.login()
@@ -0,0 +1,92 @@
1
+ u"""
2
+ Downloads SIA data from Datasus FTP server
3
+ Created on 21/09/18
4
+ by fccoelho
5
+ license: GPL V3 or Later
6
+ """
7
+
8
+ import os
9
+ from ftplib import FTP
10
+ from pysus.utilities.readdbc import read_dbc
11
+ from dbfread import DBF
12
+ import pandas as pd
13
+ from pysus.online_data import CACHEPATH
14
+
15
+
16
+ def download(state: str, year: int, month: int, cache: bool =True) -> object:
17
+ """
18
+ Download SIH records for state year and month and returns dataframe
19
+ :param month: 1 to 12
20
+ :param state: 2 letter state code
21
+ :param year: 4 digit integer
22
+ :param cache: whether to cache files locally. default is True
23
+ :return: Two dataframes PA and BI
24
+ """
25
+ state = state.upper()
26
+ year2 = str(year)[-2:]
27
+ month = str(month).zfill(2)
28
+ if year < 1992:
29
+ raise ValueError("SIH does not contain data before 1994")
30
+ ftp = FTP('ftp.datasus.gov.br')
31
+ ftp.login()
32
+ if year < 2008 and year > 1994:
33
+ ftype = 'DBC'
34
+ ftp.cwd('/dissemin/publicos/SIASUS/199407_200712/Dados')
35
+ fname = 'PA{}{}{}.dbc'.format(state, year2, month)
36
+ fname2 = None
37
+ if year >= 2008:
38
+ ftype = 'DBC'
39
+ ftp.cwd('/dissemin/publicos/SIASUS/200801_/Dados'.format(year))
40
+ fname = 'PA{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
41
+ fname2 = 'BI{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
42
+ # Check in Cache
43
+ cachefile = os.path.join(CACHEPATH, 'SIA_' + fname.split('.')[0] + '_.parquet')
44
+ if os.path.exists(cachefile):
45
+ df = pd.read_parquet(cachefile)
46
+ else:
47
+ df = _fetch_file(fname, ftp, ftype)
48
+ if cache:
49
+ df.to_parquet(cachefile)
50
+ if fname2 is not None:
51
+ cachefile2 = os.path.join(CACHEPATH, 'SIA_' + fname2.split('.')[0] + '_.parquet')
52
+ if os.path.exists(cachefile2): # reads from cache
53
+ df2 = pd.read_parquet(cachefile2)
54
+ else: # fetches from DataSUS
55
+ try:
56
+ df2 = _fetch_file(fname2, ftp, ftype)
57
+ if cache: #saves to cache
58
+ df2.to_parquet(cachefile2)
59
+ except Exception as e:
60
+ df2 = None
61
+ print(e)
62
+ else:
63
+ df2 = None
64
+
65
+ return df, df2
66
+
67
+
68
+ def _fetch_file(fname, ftp, ftype):
69
+ """
70
+ Does the FTP fetching.
71
+ :param fname: file name
72
+ :param ftp: ftp connection object
73
+ :param ftype: file type: DBF|DBC
74
+ :return: pandas dataframe
75
+ """
76
+ print("Downloading {}...".format(fname))
77
+ try:
78
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
79
+ except:
80
+ try:
81
+ ftp.retrbinary('RETR {}'.format(fname.lower()), open(fname, 'wb').write)
82
+ except:
83
+ raise Exception("File {} not available".format(fname))
84
+ if ftype == 'DBC':
85
+ df = read_dbc(fname, encoding='iso-8859-1')
86
+ elif ftype == 'DBF':
87
+ dbf = DBF(fname, encoding='iso-8859-1')
88
+ df = pd.DataFrame(list(dbf))
89
+ os.unlink(fname)
90
+ return df
91
+
92
+
@@ -0,0 +1,61 @@
1
+ u"""
2
+ Downloads SIH data from Datasus FTP server
3
+ Created on 21/09/18
4
+ by fccoelho
5
+ license: GPL V3 or Later
6
+ """
7
+
8
+ import os
9
+ from ftplib import FTP
10
+ from pysus.utilities.readdbc import read_dbc
11
+ from pysus.online_data import CACHEPATH
12
+ from dbfread import DBF
13
+ import pandas as pd
14
+
15
+
16
+ def download(state: str, year: int, month: int, cache: bool=True) -> object:
17
+ """
18
+ Download SIH records for state year and month and returns dataframe
19
+ :param month: 1 to 12
20
+ :param state: 2 letter state code
21
+ :param year: 4 digit integer
22
+ """
23
+ state = state.upper()
24
+ year2 = int(str(year)[-2:])
25
+ month = str(month).zfill(2)
26
+ if year < 1992:
27
+ raise ValueError("SIH does not contain data before 1994")
28
+ ftp = FTP('ftp.datasus.gov.br')
29
+ ftp.login()
30
+ if year < 2008:
31
+ ftype = 'DBC'
32
+ ftp.cwd('/dissemin/publicos/SIHSUS/199201_200712/Dados')
33
+ fname = 'RD{}{}{}.dbc'.format(state, year2, month)
34
+ if year >= 2008:
35
+ ftype = 'DBC'
36
+ ftp.cwd('/dissemin/publicos/SIHSUS/200801_/Dados'.format(year))
37
+ fname = 'RD{}{}{}.dbc'.format(state, str(year2).zfill(2), month)
38
+ cachefile = os.path.join(CACHEPATH, 'SIH_' + fname.split('.')[0] + '_.parquet')
39
+ if os.path.exists(cachefile):
40
+ df = pd.read_parquet(cachefile)
41
+ return df
42
+ df = _fetch_file(fname, ftp, ftype)
43
+ if cache:
44
+ df.to_parquet(cachefile)
45
+ return df
46
+
47
+
48
+ def _fetch_file(fname, ftp, ftype):
49
+ try:
50
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
51
+ except:
52
+ raise Exception("File {} not available".format(fname))
53
+ if ftype == 'DBC':
54
+ df = read_dbc(fname, encoding='iso-8859-1')
55
+ elif ftype == 'DBF':
56
+ dbf = DBF(fname, encoding='iso-8859-1')
57
+ df = pd.DataFrame(list(dbf))
58
+ os.unlink(fname)
59
+ return df
60
+
61
+
@@ -0,0 +1,155 @@
1
+ u"""
2
+ Download Mortality records from SIM Datasus
3
+ Created on 12/12/18
4
+ by fccoelho
5
+ license: GPL V3 or Later
6
+ """
7
+
8
+ import os
9
+ from ftplib import FTP
10
+ from pysus.utilities.readdbc import read_dbc
11
+ from pysus.online_data import CACHEPATH
12
+ from dbfread import DBF
13
+ import pandas as pd
14
+
15
+
16
+ def download(state, year, cache=True):
17
+ """
18
+ Downloads data directly from Datasus ftp server
19
+ :param state: two-letter state identifier: MG == Minas Gerais
20
+ :param year: 4 digit integer
21
+ :return: pandas dataframe
22
+ """
23
+ year2 = str(year)[-2:].zfill(2)
24
+ state = state.upper()
25
+ if year < 1979:
26
+ raise ValueError("SIM does not contain data before 1979")
27
+ ftp = FTP('ftp.datasus.gov.br')
28
+ ftp.login()
29
+ if year >= 1996:
30
+ ftp.cwd('/dissemin/publicos/SIM/CID10/DORES')
31
+ fname = 'DO{}{}.DBC'.format(state, year)
32
+ else:
33
+ ftp.cwd('/dissemin/publicos/SIM/CID9/DORES')
34
+ fname = 'DOR{}{}.DBC'.format(state, year2)
35
+ cachefile = os.path.join(CACHEPATH, 'SIM_'+fname.split('.')[0] + '_.parquet')
36
+ if os.path.exists(cachefile):
37
+ df = pd.read_parquet(cachefile)
38
+ return df
39
+
40
+ try:
41
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
42
+ except:
43
+ try:
44
+ ftp.retrbinary('RETR {}'.format(fname.upper()), open(fname, 'wb').write)
45
+ except:
46
+ raise Exception("File {} not available".format(fname))
47
+
48
+ df = read_dbc(fname, encoding='iso-8859-1')
49
+ if cache:
50
+ df.to_parquet(cachefile)
51
+ os.unlink(fname)
52
+ return df
53
+
54
+
55
+ def get_CID10_table(cache=True):
56
+ """
57
+ Fetch the CID10 table
58
+ :param cache:
59
+ :return:
60
+ """
61
+ ftp = FTP('ftp.datasus.gov.br')
62
+ ftp.login()
63
+ ftp.cwd('/dissemin/publicos/SIM/CID10/TABELAS')
64
+ fname = 'CID10.DBF'
65
+ cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
66
+ if os.path.exists(cachefile):
67
+ df = pd.read_parquet(cachefile)
68
+ return df
69
+ try:
70
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
71
+ except:
72
+ raise Exception('Could not download {}'.format(fname))
73
+ dbf = DBF(fname, encoding='iso-8859-1')
74
+ df = pd.DataFrame(list(dbf))
75
+ if cache:
76
+ df.to_parquet(cachefile)
77
+ os.unlink(fname)
78
+ return df
79
+
80
+
81
+ def get_CID9_table(cache=True):
82
+ """
83
+ Fetch the CID9 table
84
+ :param cache:
85
+ :return:
86
+ """
87
+ ftp = FTP('ftp.datasus.gov.br')
88
+ ftp.login()
89
+ ftp.cwd('/dissemin/publicos/SIM/CID9/TABELAS')
90
+ fname = 'CID9.DBF'
91
+ cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
92
+ if os.path.exists(cachefile):
93
+ df = pd.read_parquet(cachefile)
94
+ return df
95
+ try:
96
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
97
+ except:
98
+ raise Exception('Could not download {}'.format(fname))
99
+ dbf = DBF(fname, encoding='iso-8859-1')
100
+ df = pd.DataFrame(list(dbf))
101
+ if cache:
102
+ df.to_parquet(cachefile)
103
+ os.unlink(fname)
104
+ return df
105
+
106
+
107
+ def get_municipios(cache=True):
108
+ """
109
+ Get municipality metadata
110
+ :param cache:
111
+ :return:
112
+ """
113
+ ftp = FTP('ftp.datasus.gov.br')
114
+ ftp.login()
115
+ ftp.cwd('/dissemin/publicos/SIM/CID10/TABELAS')
116
+ fname = 'CADMUN.DBF'
117
+ cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
118
+ if os.path.exists(cachefile):
119
+ df = pd.read_parquet(cachefile)
120
+ return df
121
+ try:
122
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
123
+ except:
124
+ raise Exception('Could not download {}'.format(fname))
125
+ dbf = DBF(fname, encoding='iso-8859-1')
126
+ df = pd.DataFrame(list(dbf))
127
+ if cache:
128
+ df.to_parquet(cachefile)
129
+ os.unlink(fname)
130
+ return df
131
+
132
+ def get_ocupations(cache=True):
133
+ """
134
+ Fetch ocupations table
135
+ :param cache:
136
+ :return:
137
+ """
138
+ ftp = FTP('ftp.datasus.gov.br')
139
+ ftp.login()
140
+ ftp.cwd('/dissemin/publicos/SIM/CID10/TABELAS')
141
+ fname = 'TABOCUP.DBF'
142
+ cachefile = os.path.join(CACHEPATH, 'SIM_' + fname.split('.')[0] + '_.parquet')
143
+ if os.path.exists(cachefile):
144
+ df = pd.read_parquet(cachefile)
145
+ return df
146
+ try:
147
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
148
+ except:
149
+ raise Exception('Could not download {}'.format(fname))
150
+ dbf = DBF(fname, encoding='iso-8859-1')
151
+ df = pd.DataFrame(list(dbf))
152
+ if cache:
153
+ df.to_parquet(cachefile)
154
+ os.unlink(fname)
155
+ return df
@@ -0,0 +1,22 @@
1
+ u"""
2
+ Created on 21/09/18
3
+ by fccoelho
4
+ license: GPL V3 or Later
5
+ """
6
+ import os
7
+ from pathlib import Path
8
+
9
+ # create pysus cache directory
10
+ if not os.path.exists(os.path.join(Path.home(), 'pysus')):
11
+ os.mkdir(os.path.join(Path.home(), 'pysus'))
12
+
13
+ CACHEPATH = os.path.join(Path.home(), 'pysus')
14
+
15
+
16
+ def cache_contents():
17
+ """
18
+ List the files currently cached in ~/pysus
19
+ :return:
20
+ """
21
+ cached_data = os.listdir(CACHEPATH)
22
+ return [os.path.join(CACHEPATH, f) for f in cached_data]
@@ -0,0 +1,44 @@
1
+ """
2
+ Download SINASC data from DATASUS FTP server
3
+ Created on 01/11/17
4
+ by fccoelho
5
+ license: GPL V3 or Later
6
+ """
7
+ import os
8
+ from ftplib import FTP
9
+ from pysus.utilities.readdbc import read_dbc
10
+ from pysus.online_data import CACHEPATH
11
+ import pandas as pd
12
+
13
+
14
+ def download(state, year, cache=True):
15
+ """
16
+ Downloads data directly from Datasus ftp server
17
+ :param state: two-letter state identifier: MG == Minas Gerais
18
+ :param year: 4 digit integer
19
+ :return: pandas dataframe
20
+ """
21
+ state = state.upper()
22
+ if year < 1994:
23
+ raise ValueError("SINASC does not contain data before 1994")
24
+ ftp = FTP('ftp.datasus.gov.br')
25
+ ftp.login()
26
+ if year >= 1996:
27
+ ftp.cwd('/dissemin/publicos/SINASC/NOV/DNRES')
28
+ fname = 'DN{}{}.DBC'.format(state, year)
29
+ else:
30
+ ftp.cwd('/dissemin/publicos/SINASC/ANT/DNRES')
31
+ fname = 'DNR{}{}.DBC'.format(state, str(year)[-2:])
32
+ cachefile = os.path.join(CACHEPATH, 'SINASC_'+fname.split('.')[0] + '_.parquet')
33
+ if os.path.exists(cachefile):
34
+ df = pd.read_parquet(cachefile)
35
+ return df
36
+
37
+ ftp.retrbinary('RETR {}'.format(fname), open(fname, 'wb').write)
38
+ df = read_dbc(fname, encoding='iso-8859-1')
39
+ if cache:
40
+ df.to_parquet(cachefile)
41
+ os.unlink(fname)
42
+ return df
43
+
44
+
@@ -0,0 +1,8 @@
1
+ #-*- coding:utf-8 -*-
2
+ u"""
3
+ Created on 19/07/16
4
+ by fccoelho
5
+ license: GPL V3 or Later
6
+ """
7
+
8
+ __docformat__ = 'restructuredtext en'