airbyte-source-public-apis 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. airbyte-source-public-apis-0.2.0/PKG-INFO +70 -0
  2. airbyte-source-public-apis-0.2.0/README.md +67 -0
  3. airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/PKG-INFO +70 -0
  4. airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/SOURCES.txt +24 -0
  5. airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/dependency_links.txt +1 -0
  6. airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/entry_points.txt +2 -0
  7. airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/requires.txt +6 -0
  8. airbyte-source-public-apis-0.2.0/airbyte_source_public_apis.egg-info/top_level.txt +2 -0
  9. airbyte-source-public-apis-0.2.0/integration_tests/__init__.py +3 -0
  10. airbyte-source-public-apis-0.2.0/integration_tests/abnormal_state.json +5 -0
  11. airbyte-source-public-apis-0.2.0/integration_tests/acceptance.py +16 -0
  12. airbyte-source-public-apis-0.2.0/integration_tests/configured_catalog.json +22 -0
  13. airbyte-source-public-apis-0.2.0/integration_tests/invalid_config.json +3 -0
  14. airbyte-source-public-apis-0.2.0/integration_tests/sample_config.json +1 -0
  15. airbyte-source-public-apis-0.2.0/integration_tests/sample_state.json +5 -0
  16. airbyte-source-public-apis-0.2.0/setup.cfg +68 -0
  17. airbyte-source-public-apis-0.2.0/setup.py +39 -0
  18. airbyte-source-public-apis-0.2.0/source_public_apis/__init__.py +8 -0
  19. airbyte-source-public-apis-0.2.0/source_public_apis/components.py +14 -0
  20. airbyte-source-public-apis-0.2.0/source_public_apis/manifest.yaml +60 -0
  21. airbyte-source-public-apis-0.2.0/source_public_apis/run.py +14 -0
  22. airbyte-source-public-apis-0.2.0/source_public_apis/schemas/categories.json +10 -0
  23. airbyte-source-public-apis-0.2.0/source_public_apis/schemas/services.json +28 -0
  24. airbyte-source-public-apis-0.2.0/source_public_apis/source.py +18 -0
  25. airbyte-source-public-apis-0.2.0/source_public_apis/spec.yaml +7 -0
@@ -0,0 +1,70 @@
1
+ Metadata-Version: 2.1
2
+ Name: airbyte-source-public-apis
3
+ Version: 0.2.0
4
+ Summary: Source implementation for Public Apis.
5
+ Author: Airbyte
6
+ Author-email: contact@airbyte.io
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: airbyte-cdk
9
+ Provides-Extra: tests
10
+ Requires-Dist: requests-mock~=1.9.3; extra == "tests"
11
+ Requires-Dist: pytest~=6.2; extra == "tests"
12
+ Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
13
+
14
+ # Public Apis Source
15
+
16
+ This is the repository for the Public Apis configuration based source connector.
17
+ For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
18
+
19
+
20
+ **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
21
+ to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
22
+ Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
23
+ See `integration_tests/sample_config.json` for a sample config file.
24
+
25
+ **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
26
+ and place them into `secrets/config.json`.
27
+
28
+
29
+
30
+ **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
31
+ ```bash
32
+ airbyte-ci connectors --name=source-public-apis build
33
+ ```
34
+
35
+ An image will be built with the tag `airbyte/source-public-apis:dev`.
36
+
37
+ **Via `docker build`:**
38
+ ```bash
39
+ docker build -t airbyte/source-public-apis:dev .
40
+ ```
41
+
42
+ Then run any of the connector commands as follows:
43
+ ```
44
+ docker run --rm airbyte/source-public-apis:dev spec
45
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
46
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
47
+ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
48
+ ```
49
+
50
+ You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
51
+ ```bash
52
+ airbyte-ci connectors --name=source-public-apis test
53
+ ```
54
+
55
+ Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
56
+ If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
57
+
58
+ All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
59
+ We split dependencies between two groups, dependencies that are:
60
+ * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
61
+ * required for the testing need to go to `TEST_REQUIREMENTS` list
62
+
63
+ You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
64
+ 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
65
+ 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
66
+ 3. Make sure the `metadata.yaml` content is up to date.
67
+ 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
68
+ 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
69
+ 6. Pat yourself on the back for being an awesome contributor.
70
+ 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
@@ -0,0 +1,67 @@
1
+ # Public Apis Source
2
+
3
+ This is the repository for the Public Apis configuration based source connector.
4
+ For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
5
+
6
+ ## Local development
7
+
8
+ #### Create credentials
9
+ **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
10
+ to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
11
+ Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
12
+ See `integration_tests/sample_config.json` for a sample config file.
13
+
14
+ **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
15
+ and place them into `secrets/config.json`.
16
+
17
+ ### Locally running the connector docker image
18
+
19
+
20
+ #### Build
21
+ **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
22
+ ```bash
23
+ airbyte-ci connectors --name=source-public-apis build
24
+ ```
25
+
26
+ An image will be built with the tag `airbyte/source-public-apis:dev`.
27
+
28
+ **Via `docker build`:**
29
+ ```bash
30
+ docker build -t airbyte/source-public-apis:dev .
31
+ ```
32
+
33
+ #### Run
34
+ Then run any of the connector commands as follows:
35
+ ```
36
+ docker run --rm airbyte/source-public-apis:dev spec
37
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
38
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
39
+ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
40
+ ```
41
+
42
+ ## Testing
43
+ You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
44
+ ```bash
45
+ airbyte-ci connectors --name=source-public-apis test
46
+ ```
47
+
48
+ ### Customizing acceptance Tests
49
+ Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
50
+ If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
51
+
52
+ ## Dependency Management
53
+ All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
54
+ We split dependencies between two groups, dependencies that are:
55
+ * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
56
+ * required for the testing need to go to `TEST_REQUIREMENTS` list
57
+
58
+ ### Publishing a new version of the connector
59
+ You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
60
+ 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
61
+ 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
62
+ 3. Make sure the `metadata.yaml` content is up to date.
63
+ 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
64
+ 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
65
+ 6. Pat yourself on the back for being an awesome contributor.
66
+ 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
67
+
@@ -0,0 +1,70 @@
1
+ Metadata-Version: 2.1
2
+ Name: airbyte-source-public-apis
3
+ Version: 0.2.0
4
+ Summary: Source implementation for Public Apis.
5
+ Author: Airbyte
6
+ Author-email: contact@airbyte.io
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: airbyte-cdk
9
+ Provides-Extra: tests
10
+ Requires-Dist: requests-mock~=1.9.3; extra == "tests"
11
+ Requires-Dist: pytest~=6.2; extra == "tests"
12
+ Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
13
+
14
+ # Public Apis Source
15
+
16
+ This is the repository for the Public Apis configuration based source connector.
17
+ For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
18
+
19
+
20
+ **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
21
+ to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
22
+ Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
23
+ See `integration_tests/sample_config.json` for a sample config file.
24
+
25
+ **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
26
+ and place them into `secrets/config.json`.
27
+
28
+
29
+
30
+ **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
31
+ ```bash
32
+ airbyte-ci connectors --name=source-public-apis build
33
+ ```
34
+
35
+ An image will be built with the tag `airbyte/source-public-apis:dev`.
36
+
37
+ **Via `docker build`:**
38
+ ```bash
39
+ docker build -t airbyte/source-public-apis:dev .
40
+ ```
41
+
42
+ Then run any of the connector commands as follows:
43
+ ```
44
+ docker run --rm airbyte/source-public-apis:dev spec
45
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
46
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
47
+ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
48
+ ```
49
+
50
+ You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
51
+ ```bash
52
+ airbyte-ci connectors --name=source-public-apis test
53
+ ```
54
+
55
+ Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
56
+ If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
57
+
58
+ All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
59
+ We split dependencies between two groups, dependencies that are:
60
+ * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
61
+ * required for the testing need to go to `TEST_REQUIREMENTS` list
62
+
63
+ You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
64
+ 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
65
+ 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
66
+ 3. Make sure the `metadata.yaml` content is up to date.
67
+ 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
68
+ 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
69
+ 6. Pat yourself on the back for being an awesome contributor.
70
+ 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
@@ -0,0 +1,24 @@
1
+ README.md
2
+ setup.cfg
3
+ setup.py
4
+ airbyte_source_public_apis.egg-info/PKG-INFO
5
+ airbyte_source_public_apis.egg-info/SOURCES.txt
6
+ airbyte_source_public_apis.egg-info/dependency_links.txt
7
+ airbyte_source_public_apis.egg-info/entry_points.txt
8
+ airbyte_source_public_apis.egg-info/requires.txt
9
+ airbyte_source_public_apis.egg-info/top_level.txt
10
+ integration_tests/__init__.py
11
+ integration_tests/abnormal_state.json
12
+ integration_tests/acceptance.py
13
+ integration_tests/configured_catalog.json
14
+ integration_tests/invalid_config.json
15
+ integration_tests/sample_config.json
16
+ integration_tests/sample_state.json
17
+ source_public_apis/__init__.py
18
+ source_public_apis/components.py
19
+ source_public_apis/manifest.yaml
20
+ source_public_apis/run.py
21
+ source_public_apis/source.py
22
+ source_public_apis/spec.yaml
23
+ source_public_apis/schemas/categories.json
24
+ source_public_apis/schemas/services.json
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ source-public-apis = source_public_apis.run:run
@@ -0,0 +1,6 @@
1
+ airbyte-cdk
2
+
3
+ [tests]
4
+ requests-mock~=1.9.3
5
+ pytest~=6.2
6
+ pytest-mock~=3.6.1
@@ -0,0 +1,2 @@
1
+ integration_tests
2
+ source_public_apis
@@ -0,0 +1,3 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
@@ -0,0 +1,5 @@
1
+ {
2
+ "todo-stream-name": {
3
+ "todo-field-name": "todo-abnormal-value"
4
+ }
5
+ }
@@ -0,0 +1,16 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+
6
+ import pytest
7
+
8
+ pytest_plugins = ("connector_acceptance_test.plugin",)
9
+
10
+
11
+ @pytest.fixture(scope="session", autouse=True)
12
+ def connector_setup():
13
+ """This fixture is a placeholder for external resources that acceptance test might require."""
14
+ # TODO: setup test dependencies if needed. otherwise remove the TODO comments
15
+ yield
16
+ # TODO: clean up test dependencies
@@ -0,0 +1,22 @@
1
+ {
2
+ "streams": [
3
+ {
4
+ "stream": {
5
+ "name": "services",
6
+ "json_schema": {},
7
+ "supported_sync_modes": ["full_refresh"]
8
+ },
9
+ "sync_mode": "full_refresh",
10
+ "destination_sync_mode": "overwrite"
11
+ },
12
+ {
13
+ "stream": {
14
+ "name": "categories",
15
+ "json_schema": {},
16
+ "supported_sync_modes": ["full_refresh"]
17
+ },
18
+ "sync_mode": "full_refresh",
19
+ "destination_sync_mode": "overwrite"
20
+ }
21
+ ]
22
+ }
@@ -0,0 +1,3 @@
1
+ {
2
+ "invalid": "config_key"
3
+ }
@@ -0,0 +1,5 @@
1
+ {
2
+ "todo-stream-name": {
3
+ "todo-field-name": "value"
4
+ }
5
+ }
@@ -0,0 +1,68 @@
1
+ [metadata]
2
+ name = airbyte-source-public-apis
3
+ version = 0.2.0
4
+ author = Airbyte
5
+ author_email = contact@airbyte.io
6
+ long_description = # Public Apis Source
7
+
8
+ This is the repository for the Public Apis configuration based source connector.
9
+ For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis).
10
+
11
+
12
+ **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis)
13
+ to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file.
14
+ Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
15
+ See `integration_tests/sample_config.json` for a sample config file.
16
+
17
+ **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds`
18
+ and place them into `secrets/config.json`.
19
+
20
+
21
+
22
+ **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
23
+ ```bash
24
+ airbyte-ci connectors --name=source-public-apis build
25
+ ```
26
+
27
+ An image will be built with the tag `airbyte/source-public-apis:dev`.
28
+
29
+ **Via `docker build`:**
30
+ ```bash
31
+ docker build -t airbyte/source-public-apis:dev .
32
+ ```
33
+
34
+ Then run any of the connector commands as follows:
35
+ ```
36
+ docker run --rm airbyte/source-public-apis:dev spec
37
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check --config /secrets/config.json
38
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json
39
+ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
40
+ ```
41
+
42
+ You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
43
+ ```bash
44
+ airbyte-ci connectors --name=source-public-apis test
45
+ ```
46
+
47
+ Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
48
+ If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
49
+
50
+ All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
51
+ We split dependencies between two groups, dependencies that are:
52
+ * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
53
+ * required for the testing need to go to `TEST_REQUIREMENTS` list
54
+
55
+ You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
56
+ 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test`
57
+ 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
58
+ 3. Make sure the `metadata.yaml` content is up to date.
59
+ 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`).
60
+ 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
61
+ 6. Pat yourself on the back for being an awesome contributor.
62
+ 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
63
+ long_description_content_type = text/markdown
64
+
65
+ [egg_info]
66
+ tag_build =
67
+ tag_date = 0
68
+
@@ -0,0 +1,39 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+
6
+ from setuptools import find_packages, setup
7
+
8
+ MAIN_REQUIREMENTS = [
9
+ "airbyte-cdk",
10
+ ]
11
+
12
+ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"]
13
+
14
+ setup(
15
+ entry_points={
16
+ "console_scripts": [
17
+ "source-public-apis=source_public_apis.run:run",
18
+ ],
19
+ },
20
+ description="Source implementation for Public Apis.",
21
+ packages=find_packages(),
22
+ install_requires=MAIN_REQUIREMENTS,
23
+ package_data={
24
+ "": [
25
+ # Include yaml files in the package (if any)
26
+ "*.yml",
27
+ "*.yaml",
28
+ # Include all json files in the package, up to 4 levels deep
29
+ "*.json",
30
+ "*/*.json",
31
+ "*/*/*.json",
32
+ "*/*/*/*.json",
33
+ "*/*/*/*/*.json",
34
+ ]
35
+ },
36
+ extras_require={
37
+ "tests": TEST_REQUIREMENTS,
38
+ },
39
+ )
@@ -0,0 +1,8 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+
6
+ from .source import SourcePublicApis
7
+
8
+ __all__ = ["SourcePublicApis"]
@@ -0,0 +1,14 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from typing import Any, List, Mapping
6
+
7
+ import requests
8
+ from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
9
+
10
+
11
+ class CustomExtractor(RecordExtractor):
12
+ def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]:
13
+
14
+ return [{"name": cat} for cat in response.json()["categories"]]
@@ -0,0 +1,60 @@
1
+ version: "0.29.0"
2
+
3
+ definitions:
4
+ selector:
5
+ type: RecordSelector
6
+ extractor:
7
+ type: DpathExtractor
8
+ field_path: ["entries"]
9
+
10
+ selector_categories:
11
+ type: RecordSelector
12
+ extractor:
13
+ type: CustomRecordExtractor
14
+ class_name: source_public_apis.components.CustomExtractor
15
+
16
+ requester:
17
+ type: HttpRequester
18
+ url_base: "https://api.publicapis.org/"
19
+ http_method: "GET"
20
+ authenticator:
21
+ type: NoAuth
22
+
23
+ retriever:
24
+ type: SimpleRetriever
25
+ record_selector:
26
+ $ref: "#/definitions/selector"
27
+ paginator:
28
+ type: NoPagination
29
+ requester:
30
+ $ref: "#/definitions/requester"
31
+
32
+ base_stream:
33
+ type: DeclarativeStream
34
+ retriever:
35
+ $ref: "#/definitions/retriever"
36
+
37
+ categories_stream:
38
+ $ref: "#/definitions/base_stream"
39
+ retriever:
40
+ $ref: "#/definitions/retriever"
41
+ record_selector:
42
+ $ref: "#/definitions/selector_categories"
43
+ $parameters:
44
+ name: "categories"
45
+ path: "/categories"
46
+
47
+ services_stream:
48
+ $ref: "#/definitions/base_stream"
49
+ $parameters:
50
+ name: "services"
51
+ path: "/entries"
52
+
53
+ streams:
54
+ - "#/definitions/categories_stream"
55
+ - "#/definitions/services_stream"
56
+
57
+ check:
58
+ type: CheckStream
59
+ stream_names:
60
+ - "categories"
@@ -0,0 +1,14 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+
6
+ import sys
7
+
8
+ from airbyte_cdk.entrypoint import launch
9
+ from source_public_apis import SourcePublicApis
10
+
11
+
12
+ def run():
13
+ source = SourcePublicApis()
14
+ launch(source, sys.argv[1:])
@@ -0,0 +1,10 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "type": "object",
4
+ "additionalProperties": true,
5
+ "properties": {
6
+ "name": {
7
+ "type": ["null", "string"]
8
+ }
9
+ }
10
+ }
@@ -0,0 +1,28 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "type": "object",
4
+ "additionalProperties": true,
5
+ "properties": {
6
+ "API": {
7
+ "type": ["null", "string"]
8
+ },
9
+ "Description": {
10
+ "type": ["null", "string"]
11
+ },
12
+ "Auth": {
13
+ "type": ["null", "string"]
14
+ },
15
+ "HTTPS": {
16
+ "type": ["null", "boolean"]
17
+ },
18
+ "Cors": {
19
+ "type": ["null", "string"]
20
+ },
21
+ "Link": {
22
+ "type": ["null", "string"]
23
+ },
24
+ "Category": {
25
+ "type": ["null", "string"]
26
+ }
27
+ }
28
+ }
@@ -0,0 +1,18 @@
1
+ #
2
+ # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
+ #
4
+
5
+ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
6
+
7
+ """
8
+ This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
9
+ source connector.
10
+
11
+ WARNING: Do not modify this file.
12
+ """
13
+
14
+
15
+ # Declarative Source
16
+ class SourcePublicApis(YamlDeclarativeSource):
17
+ def __init__(self):
18
+ super().__init__(**{"path_to_yaml": "manifest.yaml"})
@@ -0,0 +1,7 @@
1
+ documentationUrl: https://docs.airbyte.com/integrations/sources/public-apis
2
+ connectionSpecification:
3
+ $schema: http://json-schema.org/draft-07/schema#
4
+ title: Public Apis Spec
5
+ type: object
6
+ additionalProperties: true
7
+ properties: {}