airbyte-source-public-apis 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte-source-public-apis-0.2.0/PKG-INFO +70 -0
- airbyte-source-public-apis-0.2.0/README.md +67 -0
- airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/PKG-INFO +70 -0
- airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/SOURCES.txt +24 -0
- airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/dependency_links.txt +1 -0
- airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/entry_points.txt +2 -0
- airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/requires.txt +6 -0
- airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/top_level.txt +2 -0
- airbyte-source-public-apis-0.2.0/integration_tests/__init__.py +3 -0
- airbyte-source-public-apis-0.2.0/integration_tests/abnormal_state.json +5 -0
- airbyte-source-public-apis-0.2.0/integration_tests/acceptance.py +16 -0
- airbyte-source-public-apis-0.2.0/integration_tests/configured_catalog.json +22 -0
- airbyte-source-public-apis-0.2.0/integration_tests/invalid_config.json +3 -0
- airbyte-source-public-apis-0.2.0/integration_tests/sample_config.json +1 -0
- airbyte-source-public-apis-0.2.0/integration_tests/sample_state.json +5 -0
- airbyte-source-public-apis-0.2.0/setup.cfg +68 -0
- airbyte-source-public-apis-0.2.0/setup.py +39 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/__init__.py +8 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/components.py +14 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/manifest.yaml +60 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/run.py +14 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/schemas/categories.json +10 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/schemas/services.json +28 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/source.py +18 -0
- airbyte-source-public-apis-0.2.0/source_public_apis/spec.yaml +7 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: airbyte-source-public-apis
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Source implementation for Public Apis.
|
|
5
|
+
Author: Airbyte
|
|
6
|
+
Author-email: contact@airbyte.io
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: airbyte-cdk
|
|
9
|
+
Provides-Extra: tests
|
|
10
|
+
Requires-Dist: requests-mock~=1.9.3; extra == "tests"
|
|
11
|
+
Requires-Dist: pytest~=6.2; extra == "tests"
|
|
12
|
+
Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
|
|
13
|
+
|
|
14
|
+
# Public Apis Source
|
|
15
|
+
|
|
16
|
+
This is the repository for the Public Apis configuration based source connector.
|
|
17
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
|
|
21
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
|
|
22
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
23
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
24
|
+
|
|
25
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
|
|
26
|
+
and place them into `secrets/config.json`.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
31
|
+
```bash
|
|
32
|
+
airbyte-ci connectors --name=source-public-apis build
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
An image will be built with the tag `airbyte/source-public-apis:dev`.
|
|
36
|
+
|
|
37
|
+
**Via `docker build`:**
|
|
38
|
+
```bash
|
|
39
|
+
docker build -t airbyte/source-public-apis:dev .
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Then run any of the connector commands as follows:
|
|
43
|
+
```
|
|
44
|
+
docker run --rm airbyte/source-public-apis:dev spec
|
|
45
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
|
|
46
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
|
|
47
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
51
|
+
```bash
|
|
52
|
+
airbyte-ci connectors --name=source-public-apis test
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
56
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
57
|
+
|
|
58
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
59
|
+
We split dependencies between two groups, dependencies that are:
|
|
60
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
61
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
62
|
+
|
|
63
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
64
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
|
|
65
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
66
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
67
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
|
|
68
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
69
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
70
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Public Apis Source
|
|
2
|
+
|
|
3
|
+
This is the repository for the Public Apis configuration based source connector.
|
|
4
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
|
|
5
|
+
|
|
6
|
+
## Local development
|
|
7
|
+
|
|
8
|
+
#### Create credentials
|
|
9
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
|
|
10
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
|
|
11
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
12
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
13
|
+
|
|
14
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
|
|
15
|
+
and place them into `secrets/config.json`.
|
|
16
|
+
|
|
17
|
+
### Locally running the connector docker image
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
#### Build
|
|
21
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
22
|
+
```bash
|
|
23
|
+
airbyte-ci connectors --name=source-public-apis build
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
An image will be built with the tag `airbyte/source-public-apis:dev`.
|
|
27
|
+
|
|
28
|
+
**Via `docker build`:**
|
|
29
|
+
```bash
|
|
30
|
+
docker build -t airbyte/source-public-apis:dev .
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
#### Run
|
|
34
|
+
Then run any of the connector commands as follows:
|
|
35
|
+
```
|
|
36
|
+
docker run --rm airbyte/source-public-apis:dev spec
|
|
37
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
|
|
38
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
|
|
39
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Testing
|
|
43
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
44
|
+
```bash
|
|
45
|
+
airbyte-ci connectors --name=source-public-apis test
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### Customizing acceptance Tests
|
|
49
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
50
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
51
|
+
|
|
52
|
+
## Dependency Management
|
|
53
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
54
|
+
We split dependencies between two groups, dependencies that are:
|
|
55
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
56
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
57
|
+
|
|
58
|
+
### Publishing a new version of the connector
|
|
59
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
60
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
|
|
61
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
62
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
63
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
|
|
64
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
65
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
66
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
67
|
+
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: airbyte-source-public-apis
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Source implementation for Public Apis.
|
|
5
|
+
Author: Airbyte
|
|
6
|
+
Author-email: contact@airbyte.io
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: airbyte-cdk
|
|
9
|
+
Provides-Extra: tests
|
|
10
|
+
Requires-Dist: requests-mock~=1.9.3; extra == "tests"
|
|
11
|
+
Requires-Dist: pytest~=6.2; extra == "tests"
|
|
12
|
+
Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
|
|
13
|
+
|
|
14
|
+
# Public Apis Source
|
|
15
|
+
|
|
16
|
+
This is the repository for the Public Apis configuration based source connector.
|
|
17
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
|
|
21
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
|
|
22
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
23
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
24
|
+
|
|
25
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
|
|
26
|
+
and place them into `secrets/config.json`.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
31
|
+
```bash
|
|
32
|
+
airbyte-ci connectors --name=source-public-apis build
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
An image will be built with the tag `airbyte/source-public-apis:dev`.
|
|
36
|
+
|
|
37
|
+
**Via `docker build`:**
|
|
38
|
+
```bash
|
|
39
|
+
docker build -t airbyte/source-public-apis:dev .
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Then run any of the connector commands as follows:
|
|
43
|
+
```
|
|
44
|
+
docker run --rm airbyte/source-public-apis:dev spec
|
|
45
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
|
|
46
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
|
|
47
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
51
|
+
```bash
|
|
52
|
+
airbyte-ci connectors --name=source-public-apis test
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
56
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
57
|
+
|
|
58
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
59
|
+
We split dependencies between two groups, dependencies that are:
|
|
60
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
61
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
62
|
+
|
|
63
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
64
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
|
|
65
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
66
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
67
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
|
|
68
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
69
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
70
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
setup.cfg
|
|
3
|
+
setup.py
|
|
4
|
+
airbyte_source_public_apis.egg-info/PKG-INFO
|
|
5
|
+
airbyte_source_public_apis.egg-info/SOURCES.txt
|
|
6
|
+
airbyte_source_public_apis.egg-info/dependency_links.txt
|
|
7
|
+
airbyte_source_public_apis.egg-info/entry_points.txt
|
|
8
|
+
airbyte_source_public_apis.egg-info/requires.txt
|
|
9
|
+
airbyte_source_public_apis.egg-info/top_level.txt
|
|
10
|
+
integration_tests/__init__.py
|
|
11
|
+
integration_tests/abnormal_state.json
|
|
12
|
+
integration_tests/acceptance.py
|
|
13
|
+
integration_tests/configured_catalog.json
|
|
14
|
+
integration_tests/invalid_config.json
|
|
15
|
+
integration_tests/sample_config.json
|
|
16
|
+
integration_tests/sample_state.json
|
|
17
|
+
source_public_apis/__init__.py
|
|
18
|
+
source_public_apis/components.py
|
|
19
|
+
source_public_apis/manifest.yaml
|
|
20
|
+
source_public_apis/run.py
|
|
21
|
+
source_public_apis/source.py
|
|
22
|
+
source_public_apis/spec.yaml
|
|
23
|
+
source_public_apis/schemas/categories.json
|
|
24
|
+
source_public_apis/schemas/services.json
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
pytest_plugins = ("connector_acceptance_test.plugin",)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture(scope="session", autouse=True)
|
|
12
|
+
def connector_setup():
|
|
13
|
+
"""This fixture is a placeholder for external resources that acceptance test might require."""
|
|
14
|
+
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
|
|
15
|
+
yield
|
|
16
|
+
# TODO: clean up test dependencies
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"streams": [
|
|
3
|
+
{
|
|
4
|
+
"stream": {
|
|
5
|
+
"name": "services",
|
|
6
|
+
"json_schema": {},
|
|
7
|
+
"supported_sync_modes": ["full_refresh"]
|
|
8
|
+
},
|
|
9
|
+
"sync_mode": "full_refresh",
|
|
10
|
+
"destination_sync_mode": "overwrite"
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"stream": {
|
|
14
|
+
"name": "categories",
|
|
15
|
+
"json_schema": {},
|
|
16
|
+
"supported_sync_modes": ["full_refresh"]
|
|
17
|
+
},
|
|
18
|
+
"sync_mode": "full_refresh",
|
|
19
|
+
"destination_sync_mode": "overwrite"
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[metadata]
|
|
2
|
+
name = airbyte-source-public-apis
|
|
3
|
+
version = 0.2.0
|
|
4
|
+
author = Airbyte
|
|
5
|
+
author_email = contact@airbyte.io
|
|
6
|
+
long_description = # Public Apis Source
|
|
7
|
+
|
|
8
|
+
This is the repository for the Public Apis configuration based source connector.
|
|
9
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
|
|
13
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
|
|
14
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
15
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
16
|
+
|
|
17
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
|
|
18
|
+
and place them into `secrets/config.json`.
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
23
|
+
```bash
|
|
24
|
+
airbyte-ci connectors --name=source-public-apis build
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
An image will be built with the tag `airbyte/source-public-apis:dev`.
|
|
28
|
+
|
|
29
|
+
**Via `docker build`:**
|
|
30
|
+
```bash
|
|
31
|
+
docker build -t airbyte/source-public-apis:dev .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Then run any of the connector commands as follows:
|
|
35
|
+
```
|
|
36
|
+
docker run --rm airbyte/source-public-apis:dev spec
|
|
37
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
|
|
38
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
|
|
39
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
43
|
+
```bash
|
|
44
|
+
airbyte-ci connectors --name=source-public-apis test
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
48
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
49
|
+
|
|
50
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
51
|
+
We split dependencies between two groups, dependencies that are:
|
|
52
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
53
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
54
|
+
|
|
55
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
56
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
|
|
57
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
58
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
59
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
|
|
60
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
61
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
62
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
63
|
+
long_description_content_type = text/markdown
|
|
64
|
+
|
|
65
|
+
[egg_info]
|
|
66
|
+
tag_build =
|
|
67
|
+
tag_date = 0
|
|
68
|
+
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
from setuptools import find_packages, setup
|
|
7
|
+
|
|
8
|
+
MAIN_REQUIREMENTS = [
|
|
9
|
+
"airbyte-cdk",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"]
|
|
13
|
+
|
|
14
|
+
setup(
|
|
15
|
+
entry_points={
|
|
16
|
+
"console_scripts": [
|
|
17
|
+
"source-public-apis=source_public_apis.run:run",
|
|
18
|
+
],
|
|
19
|
+
},
|
|
20
|
+
description="Source implementation for Public Apis.",
|
|
21
|
+
packages=find_packages(),
|
|
22
|
+
install_requires=MAIN_REQUIREMENTS,
|
|
23
|
+
package_data={
|
|
24
|
+
"": [
|
|
25
|
+
# Include yaml files in the package (if any)
|
|
26
|
+
"*.yml",
|
|
27
|
+
"*.yaml",
|
|
28
|
+
# Include all json files in the package, up to 4 levels deep
|
|
29
|
+
"*.json",
|
|
30
|
+
"*/*.json",
|
|
31
|
+
"*/*/*.json",
|
|
32
|
+
"*/*/*/*.json",
|
|
33
|
+
"*/*/*/*/*.json",
|
|
34
|
+
]
|
|
35
|
+
},
|
|
36
|
+
extras_require={
|
|
37
|
+
"tests": TEST_REQUIREMENTS,
|
|
38
|
+
},
|
|
39
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from typing import Any, List, Mapping
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class CustomExtractor(RecordExtractor):
|
|
12
|
+
def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]:
|
|
13
|
+
|
|
14
|
+
return [{"name": cat} for cat in response.json()["categories"]]
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
version: "0.29.0"
|
|
2
|
+
|
|
3
|
+
definitions:
|
|
4
|
+
selector:
|
|
5
|
+
type: RecordSelector
|
|
6
|
+
extractor:
|
|
7
|
+
type: DpathExtractor
|
|
8
|
+
field_path: ["entries"]
|
|
9
|
+
|
|
10
|
+
selector_categories:
|
|
11
|
+
type: RecordSelector
|
|
12
|
+
extractor:
|
|
13
|
+
type: CustomRecordExtractor
|
|
14
|
+
class_name: source_public_apis.components.CustomExtractor
|
|
15
|
+
|
|
16
|
+
requester:
|
|
17
|
+
type: HttpRequester
|
|
18
|
+
url_base: "https://api.publicapis.org/"
|
|
19
|
+
http_method: "GET"
|
|
20
|
+
authenticator:
|
|
21
|
+
type: NoAuth
|
|
22
|
+
|
|
23
|
+
retriever:
|
|
24
|
+
type: SimpleRetriever
|
|
25
|
+
record_selector:
|
|
26
|
+
$ref: "#/definitions/selector"
|
|
27
|
+
paginator:
|
|
28
|
+
type: NoPagination
|
|
29
|
+
requester:
|
|
30
|
+
$ref: "#/definitions/requester"
|
|
31
|
+
|
|
32
|
+
base_stream:
|
|
33
|
+
type: DeclarativeStream
|
|
34
|
+
retriever:
|
|
35
|
+
$ref: "#/definitions/retriever"
|
|
36
|
+
|
|
37
|
+
categories_stream:
|
|
38
|
+
$ref: "#/definitions/base_stream"
|
|
39
|
+
retriever:
|
|
40
|
+
$ref: "#/definitions/retriever"
|
|
41
|
+
record_selector:
|
|
42
|
+
$ref: "#/definitions/selector_categories"
|
|
43
|
+
$parameters:
|
|
44
|
+
name: "categories"
|
|
45
|
+
path: "/categories"
|
|
46
|
+
|
|
47
|
+
services_stream:
|
|
48
|
+
$ref: "#/definitions/base_stream"
|
|
49
|
+
$parameters:
|
|
50
|
+
name: "services"
|
|
51
|
+
path: "/entries"
|
|
52
|
+
|
|
53
|
+
streams:
|
|
54
|
+
- "#/definitions/categories_stream"
|
|
55
|
+
- "#/definitions/services_stream"
|
|
56
|
+
|
|
57
|
+
check:
|
|
58
|
+
type: CheckStream
|
|
59
|
+
stream_names:
|
|
60
|
+
- "categories"
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from airbyte_cdk.entrypoint import launch
|
|
9
|
+
from source_public_apis import SourcePublicApis
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def run():
|
|
13
|
+
source = SourcePublicApis()
|
|
14
|
+
launch(source, sys.argv[1:])
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"type": "object",
|
|
4
|
+
"additionalProperties": true,
|
|
5
|
+
"properties": {
|
|
6
|
+
"API": {
|
|
7
|
+
"type": ["null", "string"]
|
|
8
|
+
},
|
|
9
|
+
"Description": {
|
|
10
|
+
"type": ["null", "string"]
|
|
11
|
+
},
|
|
12
|
+
"Auth": {
|
|
13
|
+
"type": ["null", "string"]
|
|
14
|
+
},
|
|
15
|
+
"HTTPS": {
|
|
16
|
+
"type": ["null", "boolean"]
|
|
17
|
+
},
|
|
18
|
+
"Cors": {
|
|
19
|
+
"type": ["null", "string"]
|
|
20
|
+
},
|
|
21
|
+
"Link": {
|
|
22
|
+
"type": ["null", "string"]
|
|
23
|
+
},
|
|
24
|
+
"Category": {
|
|
25
|
+
"type": ["null", "string"]
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
|
|
9
|
+
source connector.
|
|
10
|
+
|
|
11
|
+
WARNING: Do not modify this file.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Declarative Source
|
|
16
|
+
class SourcePublicApis(YamlDeclarativeSource):
|
|
17
|
+
def __init__(self):
|
|
18
|
+
super().__init__(**{"path_to_yaml": "manifest.yaml"})
|