ingestr 0.14.2__py3-none-any.whl → 0.14.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/src/buildinfo.py +1 -1
- ingestr/src/destinations.py +1 -24
- ingestr/src/elasticsearch/helpers.py +35 -9
- ingestr/src/factory.py +2 -0
- ingestr/src/fluxx/__init__.py +9 -0
- ingestr/src/hubspot/__init__.py +6 -12
- ingestr/src/intercom/settings.py +3 -1
- ingestr/src/jira_source/__init__.py +314 -0
- ingestr/src/jira_source/helpers.py +452 -0
- ingestr/src/jira_source/settings.py +170 -0
- ingestr/src/mongodb/helpers.py +34 -6
- ingestr/src/sources.py +47 -0
- {ingestr-0.14.2.dist-info → ingestr-0.14.3.dist-info}/METADATA +1 -1
- {ingestr-0.14.2.dist-info → ingestr-0.14.3.dist-info}/RECORD +17 -14
- {ingestr-0.14.2.dist-info → ingestr-0.14.3.dist-info}/WHEEL +0 -0
- {ingestr-0.14.2.dist-info → ingestr-0.14.3.dist-info}/entry_points.txt +0 -0
- {ingestr-0.14.2.dist-info → ingestr-0.14.3.dist-info}/licenses/LICENSE.md +0 -0
ingestr/src/sources.py
CHANGED
|
@@ -1818,6 +1818,53 @@ class AsanaSource:
|
|
|
1818
1818
|
return src.with_resources(table)
|
|
1819
1819
|
|
|
1820
1820
|
|
|
1821
|
+
class JiraSource:
|
|
1822
|
+
resources = [
|
|
1823
|
+
"projects",
|
|
1824
|
+
"issues",
|
|
1825
|
+
"users",
|
|
1826
|
+
"issue_types",
|
|
1827
|
+
"statuses",
|
|
1828
|
+
"priorities",
|
|
1829
|
+
"resolutions",
|
|
1830
|
+
"project_versions",
|
|
1831
|
+
"project_components",
|
|
1832
|
+
]
|
|
1833
|
+
|
|
1834
|
+
def handles_incrementality(self) -> bool:
|
|
1835
|
+
return True
|
|
1836
|
+
|
|
1837
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1838
|
+
parsed_uri = urlparse(uri)
|
|
1839
|
+
params = parse_qs(parsed_uri.query)
|
|
1840
|
+
|
|
1841
|
+
base_url = f"https://{parsed_uri.netloc}"
|
|
1842
|
+
email = params.get("email")
|
|
1843
|
+
api_token = params.get("api_token")
|
|
1844
|
+
|
|
1845
|
+
if not email:
|
|
1846
|
+
raise ValueError("email must be specified in the URI query parameters")
|
|
1847
|
+
|
|
1848
|
+
if not api_token:
|
|
1849
|
+
raise ValueError("api_token is required for connecting to Jira")
|
|
1850
|
+
|
|
1851
|
+
if table not in self.resources:
|
|
1852
|
+
raise ValueError(
|
|
1853
|
+
f"Resource '{table}' is not supported for Jira source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
|
|
1854
|
+
)
|
|
1855
|
+
|
|
1856
|
+
import dlt
|
|
1857
|
+
|
|
1858
|
+
from ingestr.src.jira_source import jira_source
|
|
1859
|
+
|
|
1860
|
+
dlt.secrets["sources.jira_source.base_url"] = base_url
|
|
1861
|
+
dlt.secrets["sources.jira_source.email"] = email[0]
|
|
1862
|
+
dlt.secrets["sources.jira_source.api_token"] = api_token[0]
|
|
1863
|
+
|
|
1864
|
+
src = jira_source()
|
|
1865
|
+
return src.with_resources(table)
|
|
1866
|
+
|
|
1867
|
+
|
|
1821
1868
|
class DynamoDBSource:
|
|
1822
1869
|
AWS_ENDPOINT_PATTERN = re.compile(".*\.(.+)\.amazonaws\.com")
|
|
1823
1870
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.14.
|
|
3
|
+
Version: 0.14.3
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -2,17 +2,17 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
|
|
|
2
2
|
ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
|
|
3
3
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
4
4
|
ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
|
|
5
|
-
ingestr/src/buildinfo.py,sha256=
|
|
6
|
-
ingestr/src/destinations.py,sha256=
|
|
5
|
+
ingestr/src/buildinfo.py,sha256=lj41sOUyeqEVFBwxqKuI9w8Lpm2R3Hk2Mo4eesobpsw,20
|
|
6
|
+
ingestr/src/destinations.py,sha256=QtjE0AGs0WkPHaI2snWPHJ8HHi4lwXUBYLJPklz8Mvk,27772
|
|
7
7
|
ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
|
|
8
|
-
ingestr/src/factory.py,sha256=
|
|
8
|
+
ingestr/src/factory.py,sha256=7hOPrQSQowCvrVYBLTXE7BJl2pI7Eo5jdLkdpKSbkaw,7476
|
|
9
9
|
ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
|
|
10
10
|
ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
|
|
11
11
|
ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
|
|
12
12
|
ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
|
|
13
13
|
ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
|
|
14
14
|
ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
|
|
15
|
-
ingestr/src/sources.py,sha256=
|
|
15
|
+
ingestr/src/sources.py,sha256=eVnOiV9W2-9wobLC3Pux3gKgtcv90gwTSxQH7a8THoY,134045
|
|
16
16
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
17
17
|
ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
|
|
18
18
|
ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
|
|
@@ -47,7 +47,7 @@ ingestr/src/docebo/client.py,sha256=nki0kNQhN8VDz5cdqlQQPhr1JMPlcNEYKnWK3umAyOc,
|
|
|
47
47
|
ingestr/src/docebo/helpers.py,sha256=SaEjta6k3Lj-S5fvrheA5_xj7zfASMdOc_ihsqno5ko,3238
|
|
48
48
|
ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
|
|
49
49
|
ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
|
|
50
|
-
ingestr/src/elasticsearch/helpers.py,sha256=
|
|
50
|
+
ingestr/src/elasticsearch/helpers.py,sha256=twlNMHcJ0cPvRRLWgH8HX9LZLIQewn6uZwFGRKIcB8w,4470
|
|
51
51
|
ingestr/src/facebook_ads/__init__.py,sha256=d0pTXmtNp6Qh65aY-qF2DPz3jgHkZm3pkQn1_a3G5v0,9892
|
|
52
52
|
ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
|
|
53
53
|
ingestr/src/facebook_ads/helpers.py,sha256=c-WG008yU_zIdhFwljtqE2jfjVYuaVoNKldxcnJN3U4,9761
|
|
@@ -56,7 +56,7 @@ ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5v
|
|
|
56
56
|
ingestr/src/filesystem/__init__.py,sha256=42YAOHQxZ7TkTXC1eeaLUJpjqJ3l7DH7C8j927pV4pc,4353
|
|
57
57
|
ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
|
|
58
58
|
ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
|
|
59
|
-
ingestr/src/fluxx/__init__.py,sha256=
|
|
59
|
+
ingestr/src/fluxx/__init__.py,sha256=ILpfk-5U5NPqVTuMuEOfzmkbojttHDLBL-NYmItRlAM,566443
|
|
60
60
|
ingestr/src/fluxx/helpers.py,sha256=zJmlQWwiv9snnLqTygiWVZy7-0rGi_K427hRUuZeHEM,6352
|
|
61
61
|
ingestr/src/frankfurter/__init__.py,sha256=gOdL8ZqgHHYZByjtfE3WX3BTRHdYqyn9FpQwzDHSAx0,5089
|
|
62
62
|
ingestr/src/frankfurter/helpers.py,sha256=SpRr992OcSf7IDI5y-ToUdO6m6sGpqFz59LTY0ojchI,1502
|
|
@@ -83,15 +83,18 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
|
|
|
83
83
|
ingestr/src/google_sheets/helpers/data_processing.py,sha256=RNt2MYfdJhk4bRahnQVezpNg2x9z0vx60YFq2ukZ8vI,11004
|
|
84
84
|
ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
|
|
85
85
|
ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
|
|
86
|
-
ingestr/src/hubspot/__init__.py,sha256=
|
|
86
|
+
ingestr/src/hubspot/__init__.py,sha256=FCqjLeOjijdc9JC_NoDwtRqy3FDyY-szDi6UV7CdDN0,11548
|
|
87
87
|
ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b4,7883
|
|
88
88
|
ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
|
|
89
89
|
ingestr/src/influxdb/__init__.py,sha256=cYsGnDPNHRTe9pp14ogDQgPTCI9TOdyJm1MaNuQLHdk,1290
|
|
90
90
|
ingestr/src/influxdb/client.py,sha256=hCxSNREAWWEvvAV3RQbKaWp2-e_7EE8xmVRjTwLFEFo,1230
|
|
91
91
|
ingestr/src/intercom/__init__.py,sha256=rqorWFwcfcTYrCrpSsPPM2sGOc7qq5XbYZRCDVJXjyI,4451
|
|
92
92
|
ingestr/src/intercom/helpers.py,sha256=IljM0x4K70nuahidZaP7mtIlsHkPIcZq56j9mmuSck4,21074
|
|
93
|
-
ingestr/src/intercom/settings.py,sha256=
|
|
93
|
+
ingestr/src/intercom/settings.py,sha256=u_4P2tZiYsnlGjMTN4Ttr4bLHh1b272Pu5Q9YFq3ZCE,7053
|
|
94
94
|
ingestr/src/isoc_pulse/__init__.py,sha256=9b4eN4faatpiwTuRNPuYcEt1hEFDEjua9XhfakUigBk,4648
|
|
95
|
+
ingestr/src/jira_source/__init__.py,sha256=fy0lYAFVA9Skmdbx_gAI2iF8CXpgBEKSBsuNOQLaNo8,9178
|
|
96
|
+
ingestr/src/jira_source/helpers.py,sha256=obkFnVH45rDFTzen0L9y8UVxo3gdm2LFs21m_oB5ow0,15209
|
|
97
|
+
ingestr/src/jira_source/settings.py,sha256=Ufb-sGS-x_STtWJ-y6k69hP1CVtat_J5MtFPn51TUGE,2861
|
|
95
98
|
ingestr/src/kafka/__init__.py,sha256=QUHsGmdv5_E-3z0GDHXvbk39puwuGDBsyYSDhvbA89E,3595
|
|
96
99
|
ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
|
|
97
100
|
ingestr/src/kinesis/__init__.py,sha256=YretSz4F28tbkcPhd55mBp2Xk7XE9unyWx0nmvl8iEc,6235
|
|
@@ -107,7 +110,7 @@ ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnb
|
|
|
107
110
|
ingestr/src/mixpanel/__init__.py,sha256=s1QtqMP0BTGW6YtdCabJFWj7lEn7KujzELwGpBOQgfs,1796
|
|
108
111
|
ingestr/src/mixpanel/client.py,sha256=c_reouegOVYBOwHLfgYFwpmkba0Sxro1Zkml07NCYf0,3602
|
|
109
112
|
ingestr/src/mongodb/__init__.py,sha256=6-DvvaKL7XOPPRwItI7lSpoMQLEPzYubV6dKhpzbuME,7494
|
|
110
|
-
ingestr/src/mongodb/helpers.py,sha256=
|
|
113
|
+
ingestr/src/mongodb/helpers.py,sha256=BKb0F-AUWjFJikE9OPP9z5wFuMmJsf8YsyWhvQ9dC1k,38076
|
|
111
114
|
ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
|
|
112
115
|
ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
|
|
113
116
|
ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -169,8 +172,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
|
|
|
169
172
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
170
173
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
171
174
|
ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
|
|
172
|
-
ingestr-0.14.
|
|
173
|
-
ingestr-0.14.
|
|
174
|
-
ingestr-0.14.
|
|
175
|
-
ingestr-0.14.
|
|
176
|
-
ingestr-0.14.
|
|
175
|
+
ingestr-0.14.3.dist-info/METADATA,sha256=rZ95VgGDRpyp8xiaDCqjAlAaXrxy8B1eWFY_lQY15sg,15265
|
|
176
|
+
ingestr-0.14.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
177
|
+
ingestr-0.14.3.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
178
|
+
ingestr-0.14.3.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
179
|
+
ingestr-0.14.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|