airbyte-source-facebook-pages 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte-source-facebook-pages-0.3.0/PKG-INFO +70 -0
- airbyte-source-facebook-pages-0.3.0/README.md +67 -0
- airbyte-source-facebook-pages-0.3.0/airbyte_source_facebook_pages.egg-info/PKG-INFO +70 -0
- airbyte-source-facebook-pages-0.3.0/airbyte_source_facebook_pages.egg-info/SOURCES.txt +96 -0
- airbyte-source-facebook-pages-0.3.0/airbyte_source_facebook_pages.egg-info/dependency_links.txt +1 -0
- airbyte-source-facebook-pages-0.3.0/airbyte_source_facebook_pages.egg-info/entry_points.txt +2 -0
- airbyte-source-facebook-pages-0.3.0/airbyte_source_facebook_pages.egg-info/requires.txt +6 -0
- airbyte-source-facebook-pages-0.3.0/airbyte_source_facebook_pages.egg-info/top_level.txt +2 -0
- airbyte-source-facebook-pages-0.3.0/integration_tests/__init__.py +0 -0
- airbyte-source-facebook-pages-0.3.0/integration_tests/abnormal_state.json +5 -0
- airbyte-source-facebook-pages-0.3.0/integration_tests/acceptance.py +16 -0
- airbyte-source-facebook-pages-0.3.0/integration_tests/configured_catalog.json +40 -0
- airbyte-source-facebook-pages-0.3.0/integration_tests/invalid_config.json +4 -0
- airbyte-source-facebook-pages-0.3.0/integration_tests/sample_config.json +5 -0
- airbyte-source-facebook-pages-0.3.0/integration_tests/sample_state.json +5 -0
- airbyte-source-facebook-pages-0.3.0/setup.cfg +68 -0
- airbyte-source-facebook-pages-0.3.0/setup.py +39 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/__init__.py +8 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/components.py +103 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/manifest.yaml +435 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/run.py +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/page.json +2515 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/page_insights.json +36 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/post.json +447 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/post_insights.json +36 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/adset.json +150 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/advideo.json +126 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/album.json +69 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/application.json +324 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/assigneduser.json +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/business.json +59 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/canvas.json +62 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/canvasbodyelement.json +8 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/chatplugin.json +41 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/comment.json +63 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/commercemerchantsettings.json +86 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/commercemerchantsettingssetupstatus.json +23 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/commerceorder.json +44 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/commerceordertransactiondetail.json +26 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/commercepayout.json +20 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/coverphoto.json +20 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/customusersettings.json +4 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/engagement.json +26 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/event.json +100 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/group.json +58 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/iguser.json +38 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/imagecopyright.json +50 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/insightsresult.json +23 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/instagramuser.json +32 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/instantarticle.json +26 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/instantarticleinsightsqueryresult.json +18 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/leadgenform.json +66 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/liveencoder.json +34 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/livevideo.json +71 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/location.json +44 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/mailingaddress.json +26 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/mediafingerprint.json +26 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/messagingfeaturereview.json +11 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/messagingfeaturestatus.json +8 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/messengerdestinationpagewelcomemessage.json +25 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/messengerprofile.json +26 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/nativeoffer.json +78 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/page.json +396 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pageadminnote.json +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagecalltoaction.json +55 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagecategory.json +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagecommerceeligibility.json +11 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pageparking.json +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagepaymentoptions.json +20 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagepost.json +179 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagerestaurantservices.json +35 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagerestaurantspecialties.json +20 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagesettings.json +11 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagestartinfo.json +11 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pagethreadowner.json +8 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/pageusermessagethreadlabel.json +11 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/persona.json +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/photo.json +68 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/place.json +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/post.json +200 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/privacy.json +23 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/productcatalog.json +32 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/profile.json +35 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/profilepicturesource.json +32 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/recommendation.json +27 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/rtbdynamicpost.json +36 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/shop.json +14 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/tab.json +32 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/targeting.json +278 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/unifiedthread.json +51 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/url.json +20 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/user.json +115 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/videocopyrightrule.json +24 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/videolist.json +34 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/schemas/shared/voipinfo.json +26 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/source.py +18 -0
- airbyte-source-facebook-pages-0.3.0/source_facebook_pages/spec.yaml +39 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: airbyte-source-facebook-pages
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Source implementation for Facebook Pages.
|
|
5
|
+
Author: Airbyte
|
|
6
|
+
Author-email: contact@airbyte.io
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: airbyte-cdk
|
|
9
|
+
Provides-Extra: tests
|
|
10
|
+
Requires-Dist: pytest~=6.1; extra == "tests"
|
|
11
|
+
Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
|
|
12
|
+
Requires-Dist: requests-mock; extra == "tests"
|
|
13
|
+
|
|
14
|
+
# Facebook Pages Source
|
|
15
|
+
|
|
16
|
+
This is the repository for the Facebook Pages configuration based source connector.
|
|
17
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/facebook-pages).
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-pages)
|
|
21
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_pages/spec.yaml` file.
|
|
22
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
23
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
24
|
+
|
|
25
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-pages test creds`
|
|
26
|
+
and place them into `secrets/config.json`.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
31
|
+
```bash
|
|
32
|
+
airbyte-ci connectors --name=source-facebook-pages build
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
An image will be built with the tag `airbyte/source-facebook-pages:dev`.
|
|
36
|
+
|
|
37
|
+
**Via `docker build`:**
|
|
38
|
+
```bash
|
|
39
|
+
docker build -t airbyte/source-facebook-pages:dev .
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Then run any of the connector commands as follows:
|
|
43
|
+
```
|
|
44
|
+
docker run --rm airbyte/source-facebook-pages:dev spec
|
|
45
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev check --config /secrets/config.json
|
|
46
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev discover --config /secrets/config.json
|
|
47
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
51
|
+
```bash
|
|
52
|
+
airbyte-ci connectors --name=source-facebook-pages test
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
56
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
57
|
+
|
|
58
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
59
|
+
We split dependencies between two groups, dependencies that are:
|
|
60
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
61
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
62
|
+
|
|
63
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
64
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-facebook-pages test`
|
|
65
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
66
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
67
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-pages.md`).
|
|
68
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
69
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
70
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Facebook Pages Source
|
|
2
|
+
|
|
3
|
+
This is the repository for the Facebook Pages configuration based source connector.
|
|
4
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/facebook-pages).
|
|
5
|
+
|
|
6
|
+
## Local development
|
|
7
|
+
|
|
8
|
+
#### Create credentials
|
|
9
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-pages)
|
|
10
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_pages/spec.yaml` file.
|
|
11
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
12
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
13
|
+
|
|
14
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-pages test creds`
|
|
15
|
+
and place them into `secrets/config.json`.
|
|
16
|
+
|
|
17
|
+
### Locally running the connector docker image
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
#### Build
|
|
21
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
22
|
+
```bash
|
|
23
|
+
airbyte-ci connectors --name=source-facebook-pages build
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
An image will be built with the tag `airbyte/source-facebook-pages:dev`.
|
|
27
|
+
|
|
28
|
+
**Via `docker build`:**
|
|
29
|
+
```bash
|
|
30
|
+
docker build -t airbyte/source-facebook-pages:dev .
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
#### Run
|
|
34
|
+
Then run any of the connector commands as follows:
|
|
35
|
+
```
|
|
36
|
+
docker run --rm airbyte/source-facebook-pages:dev spec
|
|
37
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev check --config /secrets/config.json
|
|
38
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev discover --config /secrets/config.json
|
|
39
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Testing
|
|
43
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
44
|
+
```bash
|
|
45
|
+
airbyte-ci connectors --name=source-facebook-pages test
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### Customizing acceptance Tests
|
|
49
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
50
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
51
|
+
|
|
52
|
+
## Dependency Management
|
|
53
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
54
|
+
We split dependencies between two groups, dependencies that are:
|
|
55
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
56
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
57
|
+
|
|
58
|
+
### Publishing a new version of the connector
|
|
59
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
60
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-facebook-pages test`
|
|
61
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
62
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
63
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-pages.md`).
|
|
64
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
65
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
66
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
67
|
+
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: airbyte-source-facebook-pages
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Source implementation for Facebook Pages.
|
|
5
|
+
Author: Airbyte
|
|
6
|
+
Author-email: contact@airbyte.io
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: airbyte-cdk
|
|
9
|
+
Provides-Extra: tests
|
|
10
|
+
Requires-Dist: pytest~=6.1; extra == "tests"
|
|
11
|
+
Requires-Dist: pytest-mock~=3.6.1; extra == "tests"
|
|
12
|
+
Requires-Dist: requests-mock; extra == "tests"
|
|
13
|
+
|
|
14
|
+
# Facebook Pages Source
|
|
15
|
+
|
|
16
|
+
This is the repository for the Facebook Pages configuration based source connector.
|
|
17
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/facebook-pages).
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-pages)
|
|
21
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_pages/spec.yaml` file.
|
|
22
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
23
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
24
|
+
|
|
25
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-pages test creds`
|
|
26
|
+
and place them into `secrets/config.json`.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
31
|
+
```bash
|
|
32
|
+
airbyte-ci connectors --name=source-facebook-pages build
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
An image will be built with the tag `airbyte/source-facebook-pages:dev`.
|
|
36
|
+
|
|
37
|
+
**Via `docker build`:**
|
|
38
|
+
```bash
|
|
39
|
+
docker build -t airbyte/source-facebook-pages:dev .
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Then run any of the connector commands as follows:
|
|
43
|
+
```
|
|
44
|
+
docker run --rm airbyte/source-facebook-pages:dev spec
|
|
45
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev check --config /secrets/config.json
|
|
46
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev discover --config /secrets/config.json
|
|
47
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
51
|
+
```bash
|
|
52
|
+
airbyte-ci connectors --name=source-facebook-pages test
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
56
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
57
|
+
|
|
58
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
59
|
+
We split dependencies between two groups, dependencies that are:
|
|
60
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
61
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
62
|
+
|
|
63
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
64
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-facebook-pages test`
|
|
65
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
66
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
67
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-pages.md`).
|
|
68
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
69
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
70
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
setup.cfg
|
|
3
|
+
setup.py
|
|
4
|
+
airbyte_source_facebook_pages.egg-info/PKG-INFO
|
|
5
|
+
airbyte_source_facebook_pages.egg-info/SOURCES.txt
|
|
6
|
+
airbyte_source_facebook_pages.egg-info/dependency_links.txt
|
|
7
|
+
airbyte_source_facebook_pages.egg-info/entry_points.txt
|
|
8
|
+
airbyte_source_facebook_pages.egg-info/requires.txt
|
|
9
|
+
airbyte_source_facebook_pages.egg-info/top_level.txt
|
|
10
|
+
integration_tests/__init__.py
|
|
11
|
+
integration_tests/abnormal_state.json
|
|
12
|
+
integration_tests/acceptance.py
|
|
13
|
+
integration_tests/configured_catalog.json
|
|
14
|
+
integration_tests/invalid_config.json
|
|
15
|
+
integration_tests/sample_config.json
|
|
16
|
+
integration_tests/sample_state.json
|
|
17
|
+
source_facebook_pages/__init__.py
|
|
18
|
+
source_facebook_pages/components.py
|
|
19
|
+
source_facebook_pages/manifest.yaml
|
|
20
|
+
source_facebook_pages/run.py
|
|
21
|
+
source_facebook_pages/source.py
|
|
22
|
+
source_facebook_pages/spec.yaml
|
|
23
|
+
source_facebook_pages/schemas/page.json
|
|
24
|
+
source_facebook_pages/schemas/page_insights.json
|
|
25
|
+
source_facebook_pages/schemas/post.json
|
|
26
|
+
source_facebook_pages/schemas/post_insights.json
|
|
27
|
+
source_facebook_pages/schemas/shared/adset.json
|
|
28
|
+
source_facebook_pages/schemas/shared/advideo.json
|
|
29
|
+
source_facebook_pages/schemas/shared/album.json
|
|
30
|
+
source_facebook_pages/schemas/shared/application.json
|
|
31
|
+
source_facebook_pages/schemas/shared/assigneduser.json
|
|
32
|
+
source_facebook_pages/schemas/shared/business.json
|
|
33
|
+
source_facebook_pages/schemas/shared/canvas.json
|
|
34
|
+
source_facebook_pages/schemas/shared/canvasbodyelement.json
|
|
35
|
+
source_facebook_pages/schemas/shared/chatplugin.json
|
|
36
|
+
source_facebook_pages/schemas/shared/comment.json
|
|
37
|
+
source_facebook_pages/schemas/shared/commercemerchantsettings.json
|
|
38
|
+
source_facebook_pages/schemas/shared/commercemerchantsettingssetupstatus.json
|
|
39
|
+
source_facebook_pages/schemas/shared/commerceorder.json
|
|
40
|
+
source_facebook_pages/schemas/shared/commerceordertransactiondetail.json
|
|
41
|
+
source_facebook_pages/schemas/shared/commercepayout.json
|
|
42
|
+
source_facebook_pages/schemas/shared/coverphoto.json
|
|
43
|
+
source_facebook_pages/schemas/shared/customusersettings.json
|
|
44
|
+
source_facebook_pages/schemas/shared/engagement.json
|
|
45
|
+
source_facebook_pages/schemas/shared/event.json
|
|
46
|
+
source_facebook_pages/schemas/shared/group.json
|
|
47
|
+
source_facebook_pages/schemas/shared/iguser.json
|
|
48
|
+
source_facebook_pages/schemas/shared/imagecopyright.json
|
|
49
|
+
source_facebook_pages/schemas/shared/insightsresult.json
|
|
50
|
+
source_facebook_pages/schemas/shared/instagramuser.json
|
|
51
|
+
source_facebook_pages/schemas/shared/instantarticle.json
|
|
52
|
+
source_facebook_pages/schemas/shared/instantarticleinsightsqueryresult.json
|
|
53
|
+
source_facebook_pages/schemas/shared/leadgenform.json
|
|
54
|
+
source_facebook_pages/schemas/shared/liveencoder.json
|
|
55
|
+
source_facebook_pages/schemas/shared/livevideo.json
|
|
56
|
+
source_facebook_pages/schemas/shared/location.json
|
|
57
|
+
source_facebook_pages/schemas/shared/mailingaddress.json
|
|
58
|
+
source_facebook_pages/schemas/shared/mediafingerprint.json
|
|
59
|
+
source_facebook_pages/schemas/shared/messagingfeaturereview.json
|
|
60
|
+
source_facebook_pages/schemas/shared/messagingfeaturestatus.json
|
|
61
|
+
source_facebook_pages/schemas/shared/messengerdestinationpagewelcomemessage.json
|
|
62
|
+
source_facebook_pages/schemas/shared/messengerprofile.json
|
|
63
|
+
source_facebook_pages/schemas/shared/nativeoffer.json
|
|
64
|
+
source_facebook_pages/schemas/shared/page.json
|
|
65
|
+
source_facebook_pages/schemas/shared/pageadminnote.json
|
|
66
|
+
source_facebook_pages/schemas/shared/pagecalltoaction.json
|
|
67
|
+
source_facebook_pages/schemas/shared/pagecategory.json
|
|
68
|
+
source_facebook_pages/schemas/shared/pagecommerceeligibility.json
|
|
69
|
+
source_facebook_pages/schemas/shared/pageparking.json
|
|
70
|
+
source_facebook_pages/schemas/shared/pagepaymentoptions.json
|
|
71
|
+
source_facebook_pages/schemas/shared/pagepost.json
|
|
72
|
+
source_facebook_pages/schemas/shared/pagerestaurantservices.json
|
|
73
|
+
source_facebook_pages/schemas/shared/pagerestaurantspecialties.json
|
|
74
|
+
source_facebook_pages/schemas/shared/pagesettings.json
|
|
75
|
+
source_facebook_pages/schemas/shared/pagestartinfo.json
|
|
76
|
+
source_facebook_pages/schemas/shared/pagethreadowner.json
|
|
77
|
+
source_facebook_pages/schemas/shared/pageusermessagethreadlabel.json
|
|
78
|
+
source_facebook_pages/schemas/shared/persona.json
|
|
79
|
+
source_facebook_pages/schemas/shared/photo.json
|
|
80
|
+
source_facebook_pages/schemas/shared/place.json
|
|
81
|
+
source_facebook_pages/schemas/shared/post.json
|
|
82
|
+
source_facebook_pages/schemas/shared/privacy.json
|
|
83
|
+
source_facebook_pages/schemas/shared/productcatalog.json
|
|
84
|
+
source_facebook_pages/schemas/shared/profile.json
|
|
85
|
+
source_facebook_pages/schemas/shared/profilepicturesource.json
|
|
86
|
+
source_facebook_pages/schemas/shared/recommendation.json
|
|
87
|
+
source_facebook_pages/schemas/shared/rtbdynamicpost.json
|
|
88
|
+
source_facebook_pages/schemas/shared/shop.json
|
|
89
|
+
source_facebook_pages/schemas/shared/tab.json
|
|
90
|
+
source_facebook_pages/schemas/shared/targeting.json
|
|
91
|
+
source_facebook_pages/schemas/shared/unifiedthread.json
|
|
92
|
+
source_facebook_pages/schemas/shared/url.json
|
|
93
|
+
source_facebook_pages/schemas/shared/user.json
|
|
94
|
+
source_facebook_pages/schemas/shared/videocopyrightrule.json
|
|
95
|
+
source_facebook_pages/schemas/shared/videolist.json
|
|
96
|
+
source_facebook_pages/schemas/shared/voipinfo.json
|
airbyte-source-facebook-pages-0.3.0/airbyte_source_facebook_pages.egg-info/dependency_links.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
File without changes
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
pytest_plugins = ("connector_acceptance_test.plugin",)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@pytest.fixture(scope="session", autouse=True)
|
|
12
|
+
def connector_setup():
|
|
13
|
+
"""This fixture is a placeholder for external resources that acceptance test might require."""
|
|
14
|
+
# TODO: setup test dependencies
|
|
15
|
+
yield
|
|
16
|
+
# TODO: clean up test dependencies
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"streams": [
|
|
3
|
+
{
|
|
4
|
+
"stream": {
|
|
5
|
+
"name": "page",
|
|
6
|
+
"json_schema": {},
|
|
7
|
+
"supported_sync_modes": ["full_refresh"]
|
|
8
|
+
},
|
|
9
|
+
"sync_mode": "full_refresh",
|
|
10
|
+
"destination_sync_mode": "overwrite"
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"stream": {
|
|
14
|
+
"name": "post",
|
|
15
|
+
"json_schema": {},
|
|
16
|
+
"supported_sync_modes": ["full_refresh"]
|
|
17
|
+
},
|
|
18
|
+
"sync_mode": "full_refresh",
|
|
19
|
+
"destination_sync_mode": "overwrite"
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"stream": {
|
|
23
|
+
"name": "page_insights",
|
|
24
|
+
"json_schema": {},
|
|
25
|
+
"supported_sync_modes": ["full_refresh"]
|
|
26
|
+
},
|
|
27
|
+
"sync_mode": "full_refresh",
|
|
28
|
+
"destination_sync_mode": "overwrite"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"stream": {
|
|
32
|
+
"name": "post_insights",
|
|
33
|
+
"json_schema": {},
|
|
34
|
+
"supported_sync_modes": ["full_refresh"]
|
|
35
|
+
},
|
|
36
|
+
"sync_mode": "full_refresh",
|
|
37
|
+
"destination_sync_mode": "overwrite"
|
|
38
|
+
}
|
|
39
|
+
]
|
|
40
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[metadata]
|
|
2
|
+
name = airbyte-source-facebook-pages
|
|
3
|
+
version = 0.3.0
|
|
4
|
+
author = Airbyte
|
|
5
|
+
author_email = contact@airbyte.io
|
|
6
|
+
long_description = # Facebook Pages Source
|
|
7
|
+
|
|
8
|
+
This is the repository for the Facebook Pages configuration based source connector.
|
|
9
|
+
For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/facebook-pages).
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-pages)
|
|
13
|
+
to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_pages/spec.yaml` file.
|
|
14
|
+
Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
|
|
15
|
+
See `integration_tests/sample_config.json` for a sample config file.
|
|
16
|
+
|
|
17
|
+
**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source facebook-pages test creds`
|
|
18
|
+
and place them into `secrets/config.json`.
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):**
|
|
23
|
+
```bash
|
|
24
|
+
airbyte-ci connectors --name=source-facebook-pages build
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
An image will be built with the tag `airbyte/source-facebook-pages:dev`.
|
|
28
|
+
|
|
29
|
+
**Via `docker build`:**
|
|
30
|
+
```bash
|
|
31
|
+
docker build -t airbyte/source-facebook-pages:dev .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Then run any of the connector commands as follows:
|
|
35
|
+
```
|
|
36
|
+
docker run --rm airbyte/source-facebook-pages:dev spec
|
|
37
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev check --config /secrets/config.json
|
|
38
|
+
docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev discover --config /secrets/config.json
|
|
39
|
+
docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
|
|
43
|
+
```bash
|
|
44
|
+
airbyte-ci connectors --name=source-facebook-pages test
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
|
|
48
|
+
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
|
|
49
|
+
|
|
50
|
+
All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
|
|
51
|
+
We split dependencies between two groups, dependencies that are:
|
|
52
|
+
* required for your connector to work need to go to `MAIN_REQUIREMENTS` list.
|
|
53
|
+
* required for the testing need to go to `TEST_REQUIREMENTS` list
|
|
54
|
+
|
|
55
|
+
You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
|
|
56
|
+
1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-facebook-pages test`
|
|
57
|
+
2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors).
|
|
58
|
+
3. Make sure the `metadata.yaml` content is up to date.
|
|
59
|
+
4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-pages.md`).
|
|
60
|
+
5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention).
|
|
61
|
+
6. Pat yourself on the back for being an awesome contributor.
|
|
62
|
+
7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master.
|
|
63
|
+
long_description_content_type = text/markdown
|
|
64
|
+
|
|
65
|
+
[egg_info]
|
|
66
|
+
tag_build =
|
|
67
|
+
tag_date = 0
|
|
68
|
+
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
from setuptools import find_packages, setup
|
|
7
|
+
|
|
8
|
+
MAIN_REQUIREMENTS = [
|
|
9
|
+
"airbyte-cdk",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock"]
|
|
13
|
+
|
|
14
|
+
setup(
|
|
15
|
+
entry_points={
|
|
16
|
+
"console_scripts": [
|
|
17
|
+
"source-facebook-pages=source_facebook_pages.run:run",
|
|
18
|
+
],
|
|
19
|
+
},
|
|
20
|
+
description="Source implementation for Facebook Pages.",
|
|
21
|
+
packages=find_packages(),
|
|
22
|
+
install_requires=MAIN_REQUIREMENTS,
|
|
23
|
+
package_data={
|
|
24
|
+
"": [
|
|
25
|
+
# Include yaml files in the package (if any)
|
|
26
|
+
"*.yml",
|
|
27
|
+
"*.yaml",
|
|
28
|
+
# Include all json files in the package, up to 4 levels deep
|
|
29
|
+
"*.json",
|
|
30
|
+
"*/*.json",
|
|
31
|
+
"*/*/*.json",
|
|
32
|
+
"*/*/*/*.json",
|
|
33
|
+
"*/*/*/*/*.json",
|
|
34
|
+
]
|
|
35
|
+
},
|
|
36
|
+
extras_require={
|
|
37
|
+
"tests": TEST_REQUIREMENTS,
|
|
38
|
+
},
|
|
39
|
+
)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from dataclasses import InitVar, dataclass
|
|
6
|
+
from http import HTTPStatus
|
|
7
|
+
from typing import Any, Mapping, MutableMapping, Optional, Union
|
|
8
|
+
|
|
9
|
+
import dpath.util
|
|
10
|
+
import pendulum
|
|
11
|
+
import requests
|
|
12
|
+
from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth
|
|
13
|
+
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
|
|
14
|
+
from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader
|
|
15
|
+
from airbyte_cdk.sources.declarative.transformations import RecordTransformation
|
|
16
|
+
from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState
|
|
17
|
+
from requests import HTTPError
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class AuthenticatorFacebookPageAccessToken(NoAuth):
|
|
22
|
+
config: Config
|
|
23
|
+
page_id: Union[InterpolatedString, str]
|
|
24
|
+
access_token: Union[InterpolatedString, str]
|
|
25
|
+
|
|
26
|
+
def __post_init__(self, parameters: Mapping[str, Any]):
|
|
27
|
+
self._page_id = InterpolatedString.create(self.page_id, parameters=parameters).eval(self.config)
|
|
28
|
+
self._access_token = InterpolatedString.create(self.access_token, parameters=parameters).eval(self.config)
|
|
29
|
+
|
|
30
|
+
def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest:
|
|
31
|
+
"""Attach the page access token to params to authenticate on the HTTP request"""
|
|
32
|
+
page_access_token = self.generate_page_access_token()
|
|
33
|
+
request.prepare_url(url=request.url, params={"access_token": page_access_token})
|
|
34
|
+
return request
|
|
35
|
+
|
|
36
|
+
def generate_page_access_token(self) -> str:
|
|
37
|
+
# We are expecting to receive User access token from config. To access
|
|
38
|
+
# Pages API we need to generate Page access token. Page access tokens
|
|
39
|
+
# can be generated from another Page access token (with the same page ID)
|
|
40
|
+
# so if user manually set Page access token instead of User access
|
|
41
|
+
# token it would be no problem unless it has wrong page ID.
|
|
42
|
+
# https://developers.facebook.com/docs/pages/access-tokens#get-a-page-access-token
|
|
43
|
+
try:
|
|
44
|
+
r = requests.get(
|
|
45
|
+
f"https://graph.facebook.com/{self._page_id}", params={"fields": "access_token", "access_token": self._access_token}
|
|
46
|
+
)
|
|
47
|
+
if r.status_code != HTTPStatus.OK:
|
|
48
|
+
raise HTTPError(r.text)
|
|
49
|
+
return r.json().get("access_token")
|
|
50
|
+
except Exception as e:
|
|
51
|
+
raise Exception(f"Error while generating page access token: {e}") from e
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class CustomFieldTransformation(RecordTransformation):
|
|
56
|
+
"""
|
|
57
|
+
Transform all 'date-time' fields from schema (nested included) to rfc3339 format
|
|
58
|
+
Issue: https://github.com/airbytehq/airbyte/issues/23407
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
config: Config
|
|
62
|
+
parameters: InitVar[Mapping[str, Any]]
|
|
63
|
+
|
|
64
|
+
def __post_init__(self, parameters: Mapping[str, Any]):
|
|
65
|
+
self.name = parameters.get("name")
|
|
66
|
+
|
|
67
|
+
def _get_schema_root_properties(self):
|
|
68
|
+
schema_loader = JsonFileSchemaLoader(config=self.config, parameters={"name": self.name})
|
|
69
|
+
schema = schema_loader.get_json_schema()
|
|
70
|
+
return schema["properties"]
|
|
71
|
+
|
|
72
|
+
def _get_date_time_dpath_from_schema(self):
|
|
73
|
+
"""
|
|
74
|
+
Get all dpath in format 'a/b/*/c' from schema with format: 'date-time'
|
|
75
|
+
"""
|
|
76
|
+
schema = self._get_schema_root_properties()
|
|
77
|
+
all_results = dpath.util.search(schema, "**", yielded=True, afilter=lambda x: True if "date-time" in str(x) else False)
|
|
78
|
+
full_dpath = [x[0] for x in all_results if isinstance(x[1], dict) and x[1].get("format") == "date-time"]
|
|
79
|
+
return [path.replace("/properties", "").replace("items", "*") for path in full_dpath]
|
|
80
|
+
|
|
81
|
+
def _date_time_to_rfc3339(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
|
|
82
|
+
"""
|
|
83
|
+
Transform 'date-time' items to RFC3339 format
|
|
84
|
+
"""
|
|
85
|
+
date_time_paths = self._get_date_time_dpath_from_schema()
|
|
86
|
+
for path in date_time_paths:
|
|
87
|
+
if "*" not in path:
|
|
88
|
+
if field_value := dpath.util.get(record, path, default=None):
|
|
89
|
+
dpath.util.set(record, path, pendulum.parse(field_value).to_rfc3339_string())
|
|
90
|
+
else:
|
|
91
|
+
if field_values := dpath.util.values(record, path):
|
|
92
|
+
for i, date_time_value in enumerate(field_values):
|
|
93
|
+
dpath.util.set(record, path.replace("*", str(i)), pendulum.parse(date_time_value).to_rfc3339_string())
|
|
94
|
+
return record
|
|
95
|
+
|
|
96
|
+
def transform(
|
|
97
|
+
self,
|
|
98
|
+
record: Record,
|
|
99
|
+
config: Optional[Config] = None,
|
|
100
|
+
stream_state: Optional[StreamState] = None,
|
|
101
|
+
stream_slice: Optional[StreamSlice] = None,
|
|
102
|
+
) -> Record:
|
|
103
|
+
return self._date_time_to_rfc3339(record)
|