airbyte-source-commcare 0.1.0__tar.gz → 0.1.1__tar.gz

Sign up to get free protection for your applications and to get access to all the features.
Files changed (28) hide show
  1. airbyte_source_commcare-0.1.1/PKG-INFO +113 -0
  2. airbyte_source_commcare-0.1.1/README.md +91 -0
  3. airbyte_source_commcare-0.1.1/pyproject.toml +36 -0
  4. airbyte_source_commcare-0.1.1/source_commcare/schemas/TODO.md +30 -0
  5. airbyte-source-commcare-0.1.0/PKG-INFO +0 -102
  6. airbyte-source-commcare-0.1.0/README.md +0 -100
  7. airbyte-source-commcare-0.1.0/airbyte_source_commcare.egg-info/PKG-INFO +0 -102
  8. airbyte-source-commcare-0.1.0/airbyte_source_commcare.egg-info/SOURCES.txt +0 -23
  9. airbyte-source-commcare-0.1.0/airbyte_source_commcare.egg-info/dependency_links.txt +0 -1
  10. airbyte-source-commcare-0.1.0/airbyte_source_commcare.egg-info/entry_points.txt +0 -2
  11. airbyte-source-commcare-0.1.0/airbyte_source_commcare.egg-info/requires.txt +0 -9
  12. airbyte-source-commcare-0.1.0/airbyte_source_commcare.egg-info/top_level.txt +0 -3
  13. airbyte-source-commcare-0.1.0/integration_tests/__init__.py +0 -3
  14. airbyte-source-commcare-0.1.0/integration_tests/abnormal_state.json +0 -5
  15. airbyte-source-commcare-0.1.0/integration_tests/acceptance.py +0 -16
  16. airbyte-source-commcare-0.1.0/integration_tests/catalog.json +0 -1
  17. airbyte-source-commcare-0.1.0/integration_tests/configured_catalog.json +0 -16
  18. airbyte-source-commcare-0.1.0/integration_tests/invalid_config.json +0 -6
  19. airbyte-source-commcare-0.1.0/integration_tests/sample_config.json +0 -6
  20. airbyte-source-commcare-0.1.0/integration_tests/sample_state.json +0 -5
  21. airbyte-source-commcare-0.1.0/setup.cfg +0 -97
  22. airbyte-source-commcare-0.1.0/setup.py +0 -46
  23. airbyte-source-commcare-0.1.0/unit_tests/__init__.py +0 -3
  24. airbyte-source-commcare-0.1.0/unit_tests/test_source.py +0 -25
  25. {airbyte-source-commcare-0.1.0 → airbyte_source_commcare-0.1.1}/source_commcare/__init__.py +0 -0
  26. {airbyte-source-commcare-0.1.0 → airbyte_source_commcare-0.1.1}/source_commcare/run.py +0 -0
  27. {airbyte-source-commcare-0.1.0 → airbyte_source_commcare-0.1.1}/source_commcare/source.py +0 -0
  28. {airbyte-source-commcare-0.1.0 → airbyte_source_commcare-0.1.1}/source_commcare/spec.yaml +0 -0
@@ -0,0 +1,113 @@
1
+ Metadata-Version: 2.1
2
+ Name: airbyte-source-commcare
3
+ Version: 0.1.1
4
+ Summary: Source implementation for Commcare.
5
+ Home-page: https://airbyte.com
6
+ License: MIT
7
+ Author: Airbyte
8
+ Author-email: contact@airbyte.io
9
+ Requires-Python: >=3.9,<3.12
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Requires-Dist: airbyte-cdk (==0.80.0)
16
+ Requires-Dist: bigquery-schema-generator (==1.5.1)
17
+ Requires-Dist: flatten-json (==0.1.13)
18
+ Requires-Dist: gbqschema-converter (==1.2.1)
19
+ Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/commcare
20
+ Project-URL: Repository, https://github.com/airbytehq/airbyte
21
+ Description-Content-Type: text/markdown
22
+
23
+ # Commcare source connector
24
+
25
+
26
+ This is the repository for the Commcare source connector, written in Python.
27
+ For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/commcare).
28
+
29
+ ## Local development
30
+
31
+ ### Prerequisites
32
+ * Python (~=3.9)
33
+ * Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation)
34
+
35
+
36
+ ### Installing the connector
37
+ From this connector directory, run:
38
+ ```bash
39
+ poetry install --with dev
40
+ ```
41
+
42
+
43
+ ### Create credentials
44
+ **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/commcare)
45
+ to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commcare/spec.yaml` file.
46
+ Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
47
+ See `sample_files/sample_config.json` for a sample config file.
48
+
49
+
50
+ ### Locally running the connector
51
+ ```
52
+ poetry run source-commcare spec
53
+ poetry run source-commcare check --config secrets/config.json
54
+ poetry run source-commcare discover --config secrets/config.json
55
+ poetry run source-commcare read --config secrets/config.json --catalog sample_files/configured_catalog.json
56
+ ```
57
+
58
+ ### Running unit tests
59
+ To run unit tests locally, from the connector directory run:
60
+ ```
61
+ poetry run pytest unit_tests
62
+ ```
63
+
64
+ ### Building the docker image
65
+ 1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)
66
+ 2. Run the following command to build the docker image:
67
+ ```bash
68
+ airbyte-ci connectors --name=source-commcare build
69
+ ```
70
+
71
+ An image will be available on your host with the tag `airbyte/source-commcare:dev`.
72
+
73
+
74
+ ### Running as a docker container
75
+ Then run any of the connector commands as follows:
76
+ ```
77
+ docker run --rm airbyte/source-commcare:dev spec
78
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev check --config /secrets/config.json
79
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev discover --config /secrets/config.json
80
+ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commcare:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
81
+ ```
82
+
83
+ ### Running our CI test suite
84
+ You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
85
+ ```bash
86
+ airbyte-ci connectors --name=source-commcare test
87
+ ```
88
+
89
+ ### Customizing acceptance Tests
90
+ Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
91
+ If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
92
+
93
+ ### Dependency Management
94
+ All of your dependencies should be managed via Poetry.
95
+ To add a new dependency, run:
96
+ ```bash
97
+ poetry add <package-name>
98
+ ```
99
+
100
+ Please commit the changes to `pyproject.toml` and `poetry.lock` files.
101
+
102
+ ## Publishing a new version of the connector
103
+ You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
104
+ 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commcare test`
105
+ 2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)):
106
+ - bump the `dockerImageTag` value in in `metadata.yaml`
107
+ - bump the `version` value in `pyproject.toml`
108
+ 3. Make sure the `metadata.yaml` content is up to date.
109
+ 4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/commcare.md`).
110
+ 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
111
+ 6. Pat yourself on the back for being an awesome contributor.
112
+ 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
113
+ 8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry.
@@ -0,0 +1,91 @@
1
+ # Commcare source connector
2
+
3
+
4
+ This is the repository for the Commcare source connector, written in Python.
5
+ For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/commcare).
6
+
7
+ ## Local development
8
+
9
+ ### Prerequisites
10
+ * Python (~=3.9)
11
+ * Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation)
12
+
13
+
14
+ ### Installing the connector
15
+ From this connector directory, run:
16
+ ```bash
17
+ poetry install --with dev
18
+ ```
19
+
20
+
21
+ ### Create credentials
22
+ **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/commcare)
23
+ to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commcare/spec.yaml` file.
24
+ Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
25
+ See `sample_files/sample_config.json` for a sample config file.
26
+
27
+
28
+ ### Locally running the connector
29
+ ```
30
+ poetry run source-commcare spec
31
+ poetry run source-commcare check --config secrets/config.json
32
+ poetry run source-commcare discover --config secrets/config.json
33
+ poetry run source-commcare read --config secrets/config.json --catalog sample_files/configured_catalog.json
34
+ ```
35
+
36
+ ### Running unit tests
37
+ To run unit tests locally, from the connector directory run:
38
+ ```
39
+ poetry run pytest unit_tests
40
+ ```
41
+
42
+ ### Building the docker image
43
+ 1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)
44
+ 2. Run the following command to build the docker image:
45
+ ```bash
46
+ airbyte-ci connectors --name=source-commcare build
47
+ ```
48
+
49
+ An image will be available on your host with the tag `airbyte/source-commcare:dev`.
50
+
51
+
52
+ ### Running as a docker container
53
+ Then run any of the connector commands as follows:
54
+ ```
55
+ docker run --rm airbyte/source-commcare:dev spec
56
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev check --config /secrets/config.json
57
+ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev discover --config /secrets/config.json
58
+ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commcare:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
59
+ ```
60
+
61
+ ### Running our CI test suite
62
+ You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
63
+ ```bash
64
+ airbyte-ci connectors --name=source-commcare test
65
+ ```
66
+
67
+ ### Customizing acceptance Tests
68
+ Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
69
+ If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
70
+
71
+ ### Dependency Management
72
+ All of your dependencies should be managed via Poetry.
73
+ To add a new dependency, run:
74
+ ```bash
75
+ poetry add <package-name>
76
+ ```
77
+
78
+ Please commit the changes to `pyproject.toml` and `poetry.lock` files.
79
+
80
+ ## Publishing a new version of the connector
81
+ You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
82
+ 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commcare test`
83
+ 2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)):
84
+ - bump the `dockerImageTag` value in in `metadata.yaml`
85
+ - bump the `version` value in `pyproject.toml`
86
+ 3. Make sure the `metadata.yaml` content is up to date.
87
+ 4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/commcare.md`).
88
+ 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
89
+ 6. Pat yourself on the back for being an awesome contributor.
90
+ 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
91
+ 8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry.
@@ -0,0 +1,36 @@
1
+ [build-system]
2
+ requires = [
3
+ "poetry-core>=1.0.0",
4
+ ]
5
+ build-backend = "poetry.core.masonry.api"
6
+
7
+ [tool.poetry]
8
+ version = "0.1.1"
9
+ name = "airbyte-source-commcare"
10
+ description = "Source implementation for Commcare."
11
+ authors = [
12
+ "Airbyte <contact@airbyte.io>",
13
+ ]
14
+ license = "MIT"
15
+ readme = "README.md"
16
+ documentation = "https://docs.airbyte.com/integrations/sources/commcare"
17
+ homepage = "https://airbyte.com"
18
+ repository = "https://github.com/airbytehq/airbyte"
19
+ packages = [
20
+ { include = "source_commcare" },
21
+ ]
22
+
23
+ [tool.poetry.dependencies]
24
+ python = "^3.9,<3.12"
25
+ flatten-json = "==0.1.13"
26
+ bigquery-schema-generator = "==1.5.1"
27
+ gbqschema-converter = "==1.2.1"
28
+ airbyte-cdk = "0.80.0"
29
+
30
+ [tool.poetry.scripts]
31
+ source-commcare = "source_commcare.run:run"
32
+
33
+ [tool.poetry.group.dev.dependencies]
34
+ pytest-mock = "^3.6.1"
35
+ requests-mock = "^1.9.3"
36
+ pytest = "^6.1"
@@ -0,0 +1,30 @@
1
+ # TODO: Define your stream schemas
2
+
3
+ Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org).
4
+
5
+ The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it.
6
+
7
+ The schema of a stream is the return value of `Stream.get_json_schema`.
8
+
9
+ ## Static schemas
10
+
11
+ By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need.
12
+
13
+ Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files.
14
+
15
+ ## Dynamic schemas
16
+
17
+ If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org).
18
+
19
+ ## Dynamically modifying static schemas
20
+
21
+ Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value:
22
+
23
+ ```
24
+ def get_json_schema(self):
25
+ schema = super().get_json_schema()
26
+ schema['dynamically_determined_property'] = "property"
27
+ return schema
28
+ ```
29
+
30
+ Delete this file once you're done. Or don't. Up to you :)
@@ -1,102 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: airbyte-source-commcare
3
- Version: 0.1.0
4
- Summary: Source implementation for Commcare.
5
- Author: Airbyte
6
- Author-email: contact@airbyte.io
7
- Description-Content-Type: text/markdown
8
- Requires-Dist: airbyte-cdk
9
- Requires-Dist: bigquery_schema_generator~=1.5
10
- Requires-Dist: gbqschema_converter~=1.2.0
11
- Requires-Dist: flatten_json~=0.1.13
12
- Provides-Extra: tests
13
- Requires-Dist: requests-mock~=1.9.3; extra == "tests"
14
- Requires-Dist: pytest~=6.1; extra == "tests"
15
- Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
16
-
17
- # Commcare Source
18
-
19
- This is the repository for the Commcare source connector, written in Python.
20
- For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/commcare).
21
-
22
-
23
- **To iterate on this connector, make sure to complete this prerequisites section.**
24
-
25
-
26
- From this connector directory, create a virtual environment:
27
- ```
28
- python -m venv .venv
29
- ```
30
-
31
- This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your
32
- development environment of choice. To activate it from the terminal, run:
33
- ```
34
- source .venv/bin/activate
35
- pip install -r requirements.txt
36
- pip install '.[tests]'
37
- ```
38
- If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
39
-
40
- Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is
41
- used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`.
42
- If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything
43
- should work as you expect.
44
-
45
- **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/commcare)
46
- to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commcare/spec.yaml` file.
47
- Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
48
- See `integration_tests/sample_config.json` for a sample config file.
49
-
50
- **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source commcare test creds`
51
- and place them into `secrets/config.json`.
52
-
53
- ```
54
- python main.py spec
55
- python main.py check --config secrets/config.json
56
- python main.py discover --config secrets/config.json
57
- python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json
58
- ```
59
-
60
-
61
-
62
- **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
63
- ```bash
64
- airbyte-ci connectors --name=source-commcare build
65
- ```
66
-
67
- An image will be built with the tag `airbyte/source-commcare:dev`.
68
-
69
- **Via `docker build`:**
70
- ```bash
71
- docker build -t airbyte/source-commcare:dev .
72
- ```
73
-
74
- Then run any of the connector commands as follows:
75
- ```
76
- docker run --rm airbyte/source-commcare:dev spec
77
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev check --config /secrets/config.json
78
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev discover --config /secrets/config.json
79
- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commcare:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
80
- ```
81
-
82
- You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
83
- ```bash
84
- airbyte-ci connectors --name=source-commcare test
85
- ```
86
-
87
- Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
88
- If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
89
-
90
- All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
91
- We split dependencies between two groups, dependencies that are:
92
- * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
93
- * required for the testing need to go to `TEST_REQUIREMENTS` list
94
-
95
- You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
96
- 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commcare test`
97
- 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
98
- 3. Make sure the `metadata.yaml` content is up to date.
99
- 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/commcare.md`).
100
- 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
101
- 6. Pat yourself on the back for being an awesome contributor.
102
- 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
@@ -1,100 +0,0 @@
1
- # Commcare Source
2
-
3
- This is the repository for the Commcare source connector, written in Python.
4
- For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/commcare).
5
-
6
- ## Local development
7
-
8
- ### Prerequisites
9
- **To iterate on this connector, make sure to complete this prerequisites section.**
10
-
11
- #### Minimum Python version required `= 3.9.0`
12
-
13
- #### Build & Activate Virtual Environment and install dependencies
14
- From this connector directory, create a virtual environment:
15
- ```
16
- python -m venv .venv
17
- ```
18
-
19
- This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your
20
- development environment of choice. To activate it from the terminal, run:
21
- ```
22
- source .venv/bin/activate
23
- pip install -r requirements.txt
24
- pip install '.[tests]'
25
- ```
26
- If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
27
-
28
- Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is
29
- used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`.
30
- If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything
31
- should work as you expect.
32
-
33
- #### Create credentials
34
- **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/commcare)
35
- to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commcare/spec.yaml` file.
36
- Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
37
- See `integration_tests/sample_config.json` for a sample config file.
38
-
39
- **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source commcare test creds`
40
- and place them into `secrets/config.json`.
41
-
42
- ### Locally running the connector
43
- ```
44
- python main.py spec
45
- python main.py check --config secrets/config.json
46
- python main.py discover --config secrets/config.json
47
- python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json
48
- ```
49
-
50
- ### Locally running the connector docker image
51
-
52
-
53
- #### Build
54
- **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
55
- ```bash
56
- airbyte-ci connectors --name=source-commcare build
57
- ```
58
-
59
- An image will be built with the tag `airbyte/source-commcare:dev`.
60
-
61
- **Via `docker build`:**
62
- ```bash
63
- docker build -t airbyte/source-commcare:dev .
64
- ```
65
-
66
- #### Run
67
- Then run any of the connector commands as follows:
68
- ```
69
- docker run --rm airbyte/source-commcare:dev spec
70
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev check --config /secrets/config.json
71
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev discover --config /secrets/config.json
72
- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commcare:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
73
- ```
74
-
75
- ## Testing
76
- You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
77
- ```bash
78
- airbyte-ci connectors --name=source-commcare test
79
- ```
80
-
81
- ### Customizing acceptance Tests
82
- Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
83
- If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
84
-
85
- ## Dependency Management
86
- All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
87
- We split dependencies between two groups, dependencies that are:
88
- * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
89
- * required for the testing need to go to `TEST_REQUIREMENTS` list
90
-
91
- ### Publishing a new version of the connector
92
- You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
93
- 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commcare test`
94
- 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
95
- 3. Make sure the `metadata.yaml` content is up to date.
96
- 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/commcare.md`).
97
- 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
98
- 6. Pat yourself on the back for being an awesome contributor.
99
- 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
100
-
@@ -1,102 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: airbyte-source-commcare
3
- Version: 0.1.0
4
- Summary: Source implementation for Commcare.
5
- Author: Airbyte
6
- Author-email: contact@airbyte.io
7
- Description-Content-Type: text/markdown
8
- Requires-Dist: airbyte-cdk
9
- Requires-Dist: bigquery_schema_generator~=1.5
10
- Requires-Dist: gbqschema_converter~=1.2.0
11
- Requires-Dist: flatten_json~=0.1.13
12
- Provides-Extra: tests
13
- Requires-Dist: requests-mock~=1.9.3; extra == "tests"
14
- Requires-Dist: pytest~=6.1; extra == "tests"
15
- Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
16
-
17
- # Commcare Source
18
-
19
- This is the repository for the Commcare source connector, written in Python.
20
- For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/commcare).
21
-
22
-
23
- **To iterate on this connector, make sure to complete this prerequisites section.**
24
-
25
-
26
- From this connector directory, create a virtual environment:
27
- ```
28
- python -m venv .venv
29
- ```
30
-
31
- This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your
32
- development environment of choice. To activate it from the terminal, run:
33
- ```
34
- source .venv/bin/activate
35
- pip install -r requirements.txt
36
- pip install '.[tests]'
37
- ```
38
- If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
39
-
40
- Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is
41
- used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`.
42
- If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything
43
- should work as you expect.
44
-
45
- **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/commcare)
46
- to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commcare/spec.yaml` file.
47
- Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
48
- See `integration_tests/sample_config.json` for a sample config file.
49
-
50
- **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source commcare test creds`
51
- and place them into `secrets/config.json`.
52
-
53
- ```
54
- python main.py spec
55
- python main.py check --config secrets/config.json
56
- python main.py discover --config secrets/config.json
57
- python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json
58
- ```
59
-
60
-
61
-
62
- **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
63
- ```bash
64
- airbyte-ci connectors --name=source-commcare build
65
- ```
66
-
67
- An image will be built with the tag `airbyte/source-commcare:dev`.
68
-
69
- **Via `docker build`:**
70
- ```bash
71
- docker build -t airbyte/source-commcare:dev .
72
- ```
73
-
74
- Then run any of the connector commands as follows:
75
- ```
76
- docker run --rm airbyte/source-commcare:dev spec
77
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev check --config /secrets/config.json
78
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev discover --config /secrets/config.json
79
- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commcare:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
80
- ```
81
-
82
- You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
83
- ```bash
84
- airbyte-ci connectors --name=source-commcare test
85
- ```
86
-
87
- Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
88
- If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
89
-
90
- All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
91
- We split dependencies between two groups, dependencies that are:
92
- * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
93
- * required for the testing need to go to `TEST_REQUIREMENTS` list
94
-
95
- You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
96
- 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commcare test`
97
- 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
98
- 3. Make sure the `metadata.yaml` content is up to date.
99
- 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/commcare.md`).
100
- 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
101
- 6. Pat yourself on the back for being an awesome contributor.
102
- 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
@@ -1,23 +0,0 @@
1
- README.md
2
- setup.cfg
3
- setup.py
4
- airbyte_source_commcare.egg-info/PKG-INFO
5
- airbyte_source_commcare.egg-info/SOURCES.txt
6
- airbyte_source_commcare.egg-info/dependency_links.txt
7
- airbyte_source_commcare.egg-info/entry_points.txt
8
- airbyte_source_commcare.egg-info/requires.txt
9
- airbyte_source_commcare.egg-info/top_level.txt
10
- integration_tests/__init__.py
11
- integration_tests/abnormal_state.json
12
- integration_tests/acceptance.py
13
- integration_tests/catalog.json
14
- integration_tests/configured_catalog.json
15
- integration_tests/invalid_config.json
16
- integration_tests/sample_config.json
17
- integration_tests/sample_state.json
18
- source_commcare/__init__.py
19
- source_commcare/run.py
20
- source_commcare/source.py
21
- source_commcare/spec.yaml
22
- unit_tests/__init__.py
23
- unit_tests/test_source.py
@@ -1,2 +0,0 @@
1
- [console_scripts]
2
- source-commcare = source_commcare.run:run
@@ -1,9 +0,0 @@
1
- airbyte-cdk
2
- bigquery_schema_generator~=1.5
3
- gbqschema_converter~=1.2.0
4
- flatten_json~=0.1.13
5
-
6
- [tests]
7
- requests-mock~=1.9.3
8
- pytest~=6.1
9
- pytest-mock~=3.6.1
@@ -1,3 +0,0 @@
1
- integration_tests
2
- source_commcare
3
- unit_tests
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,5 +0,0 @@
1
- {
2
- "Assess a referred patient": {
3
- "indexed_on": "2023-11-25T20:30:30.2423"
4
- }
5
- }
@@ -1,16 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
-
6
- import pytest
7
-
8
- pytest_plugins = ("connector_acceptance_test.plugin",)
9
-
10
-
11
- @pytest.fixture(scope="session", autouse=True)
12
- def connector_setup():
13
- """This fixture is a placeholder for external resources that acceptance test might require."""
14
- # TODO: setup test dependencies if needed. otherwise remove the TODO comments
15
- yield
16
- # TODO: clean up test dependencies
@@ -1,16 +0,0 @@
1
- {
2
- "streams": [
3
- {
4
- "stream": {
5
- "name": "Assess a referred patient",
6
- "json_schema": {},
7
- "supported_sync_modes": ["full_refresh", "incremental"],
8
- "source_defined_cursor": true,
9
- "default_cursor_field": ["indexed_on"]
10
- },
11
- "sync_mode": "incremental",
12
- "cursor_field": ["indexed_on"],
13
- "destination_sync_mode": "append"
14
- }
15
- ]
16
- }
@@ -1,6 +0,0 @@
1
- {
2
- "app_id": "wrong app_id",
3
- "api_key": "wrong api key",
4
- "start_date": "This has the wrong format",
5
- "project_space": "project_space"
6
- }
@@ -1,6 +0,0 @@
1
- {
2
- "app_id": "App ID",
3
- "api_key": "API KEY",
4
- "project_space": "project_space",
5
- "start_date": "Start Date"
6
- }
@@ -1,5 +0,0 @@
1
- {
2
- "todo-stream-name": {
3
- "todo-field-name": "value"
4
- }
5
- }
@@ -1,97 +0,0 @@
1
- [metadata]
2
- name = airbyte-source-commcare
3
- version = 0.1.0
4
- author = Airbyte
5
- author_email = contact@airbyte.io
6
- long_description = # Commcare Source
7
-
8
- This is the repository for the Commcare source connector, written in Python.
9
- For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/commcare).
10
-
11
-
12
- **To iterate on this connector, make sure to complete this prerequisites section.**
13
-
14
-
15
- From this connector directory, create a virtual environment:
16
- ```
17
- python -m venv .venv
18
- ```
19
-
20
- This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your
21
- development environment of choice. To activate it from the terminal, run:
22
- ```
23
- source .venv/bin/activate
24
- pip install -r requirements.txt
25
- pip install '.[tests]'
26
- ```
27
- If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
28
-
29
- Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is
30
- used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`.
31
- If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything
32
- should work as you expect.
33
-
34
- **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/commcare)
35
- to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commcare/spec.yaml` file.
36
- Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
37
- See `integration_tests/sample_config.json` for a sample config file.
38
-
39
- **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source commcare test creds`
40
- and place them into `secrets/config.json`.
41
-
42
- ```
43
- python main.py spec
44
- python main.py check --config secrets/config.json
45
- python main.py discover --config secrets/config.json
46
- python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json
47
- ```
48
-
49
-
50
-
51
- **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
52
- ```bash
53
- airbyte-ci connectors --name=source-commcare build
54
- ```
55
-
56
- An image will be built with the tag `airbyte/source-commcare:dev`.
57
-
58
- **Via `docker build`:**
59
- ```bash
60
- docker build -t airbyte/source-commcare:dev .
61
- ```
62
-
63
- Then run any of the connector commands as follows:
64
- ```
65
- docker run --rm airbyte/source-commcare:dev spec
66
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev check --config /secrets/config.json
67
- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev discover --config /secrets/config.json
68
- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commcare:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
69
- ```
70
-
71
- You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
72
- ```bash
73
- airbyte-ci connectors --name=source-commcare test
74
- ```
75
-
76
- Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
77
- If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
78
-
79
- All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
80
- We split dependencies between two groups, dependencies that are:
81
- * required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
82
- * required for the testing need to go to `TEST_REQUIREMENTS` list
83
-
84
- You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
85
- 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commcare test`
86
- 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
87
- 3. Make sure the `metadata.yaml` content is up to date.
88
- 4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/commcare.md`).
89
- 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
90
- 6. Pat yourself on the back for being an awesome contributor.
91
- 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
92
- long_description_content_type = text/markdown
93
-
94
- [egg_info]
95
- tag_build =
96
- tag_date = 0
97
-
@@ -1,46 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
-
6
- from setuptools import find_packages, setup
7
-
8
- MAIN_REQUIREMENTS = [
9
- "airbyte-cdk",
10
- "bigquery_schema_generator~=1.5",
11
- "gbqschema_converter~=1.2.0",
12
- "flatten_json~=0.1.13",
13
- ]
14
-
15
- TEST_REQUIREMENTS = [
16
- "requests-mock~=1.9.3",
17
- "pytest~=6.1",
18
- "pytest-mock~=3.6.1",
19
- ]
20
-
21
- setup(
22
- entry_points={
23
- "console_scripts": [
24
- "source-commcare=source_commcare.run:run",
25
- ],
26
- },
27
- description="Source implementation for Commcare.",
28
- packages=find_packages(),
29
- install_requires=MAIN_REQUIREMENTS,
30
- package_data={
31
- "": [
32
- # Include yaml files in the package (if any)
33
- "*.yml",
34
- "*.yaml",
35
- # Include all json files in the package, up to 4 levels deep
36
- "*.json",
37
- "*/*.json",
38
- "*/*/*.json",
39
- "*/*/*/*.json",
40
- "*/*/*/*/*.json",
41
- ]
42
- },
43
- extras_require={
44
- "tests": TEST_REQUIREMENTS,
45
- },
46
- )
@@ -1,3 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
@@ -1,25 +0,0 @@
1
- #
2
- # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
3
- #
4
-
5
- from unittest.mock import MagicMock, Mock
6
-
7
- import pytest
8
- from source_commcare.source import SourceCommcare
9
-
10
-
11
- @pytest.fixture(name="config")
12
- def config_fixture():
13
- return {"api_key": "apikey", "app_id": "appid", "start_date": "2022-01-01T00:00:00Z"}
14
-
15
-
16
- def test_check_connection_ok(mocker, config):
17
- source = SourceCommcare()
18
- logger_mock = Mock()
19
- assert source.check_connection(logger_mock, config=config) == (True, None)
20
-
21
-
22
- def test_check_connection_fail(mocker, config):
23
- source = SourceCommcare()
24
- logger_mock = MagicMock()
25
- assert source.check_connection(logger_mock, config={}) == (False, None)