airbyte-source-google-search-console 1.6.0rc1__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airbyte_source_google_search_console-1.6.0rc1.dist-info → airbyte_source_google_search_console-1.7.0.dist-info}/METADATA +1 -1
- {airbyte_source_google_search_console-1.6.0rc1.dist-info → airbyte_source_google_search_console-1.7.0.dist-info}/RECORD +8 -11
- source_google_search_console/manifest.yaml +488 -1
- source_google_search_console/source.py +0 -9
- source_google_search_console/spec.json +7 -0
- source_google_search_console/streams.py +0 -65
- source_google_search_console/schemas/search_analytics_keyword_page_report.json +0 -54
- source_google_search_console/schemas/search_analytics_keyword_site_report_by_page.json +0 -50
- source_google_search_console/schemas/search_analytics_keyword_site_report_by_site.json +0 -50
- {airbyte_source_google_search_console-1.6.0rc1.dist-info → airbyte_source_google_search_console-1.7.0.dist-info}/WHEEL +0 -0
- {airbyte_source_google_search_console-1.6.0rc1.dist-info → airbyte_source_google_search_console-1.7.0.dist-info}/entry_points.txt +0 -0
@@ -2,24 +2,21 @@ source_google_search_console/__init__.py,sha256=HQCPu-CK7XmVDtP9rmTdB2XyraVCc6pv
|
|
2
2
|
source_google_search_console/components.py,sha256=5o8kH2xwYUvk3yjnSd6okJVF2KBUqUIF4V97xRpPAyI,2803
|
3
3
|
source_google_search_console/config_migrations.py,sha256=Cl4SUdJpAf6wMM_vVhqjjU89NfUq9LIGJ9zNrWiBk-A,4235
|
4
4
|
source_google_search_console/exceptions.py,sha256=iD3jYC4WxVCEKGsqQ7Vaj1tbjhJZ4S5mnSDnwFJdsIQ,1097
|
5
|
-
source_google_search_console/manifest.yaml,sha256=
|
5
|
+
source_google_search_console/manifest.yaml,sha256=m6IhF4bRwF9nAivJev2ebB8QPvoKRbYixF-5ZKSBkms,25882
|
6
6
|
source_google_search_console/run.py,sha256=TBkPlseTERarkj6wL8AMEKgm5Xsb2drnltPVH6257-M,2195
|
7
7
|
source_google_search_console/schemas/search_analytics_all_fields.json,sha256=iQxRh_c_yz3uGofqpo1KX571TMmzYjKScb0PtI6SN_Q,1729
|
8
8
|
source_google_search_console/schemas/search_analytics_by_date.json,sha256=meCbWDayc1y0q-Lu-CAdjQVnsM8xZBX3BdF129UC1P8,1388
|
9
9
|
source_google_search_console/schemas/search_analytics_by_device.json,sha256=VtoFjmmv9rx-uhSFaRn0wm4LeSxRIaexrxg2Spvbneo,1525
|
10
10
|
source_google_search_console/schemas/search_analytics_by_page.json,sha256=KyUojZc4Lv3hPswxIJzUL5QDNsbvSugGjl_uHGF7Am4,1473
|
11
11
|
source_google_search_console/schemas/search_analytics_by_query.json,sha256=mYc1Fu1A5TWLZCApZSjjfs_urW5BRUCPQI3rw3yQjt4,1439
|
12
|
-
source_google_search_console/schemas/search_analytics_keyword_page_report.json,sha256=hCQZbpW9UNiCOrnUMkj4oQdzY0k8Hqoh1XkuKHvyvZw,1681
|
13
|
-
source_google_search_console/schemas/search_analytics_keyword_site_report_by_page.json,sha256=lFpbShmTQhFyySc_JCdIOqMP9RfOIA15fpQN9xNXtTI,1626
|
14
|
-
source_google_search_console/schemas/search_analytics_keyword_site_report_by_site.json,sha256=AVUl5x9jLL0FcStbTR-FMy-_A7uM-VlrJ-kdCBXWz_g,1755
|
15
12
|
source_google_search_console/schemas/search_analytics_page_report.json,sha256=-b0Y0LenTchS0q9A2aQ4hIjUjXkYF8erOtyrTMhf6MM,1776
|
16
13
|
source_google_search_console/schemas/search_analytics_site_report_by_page.json,sha256=hWKHkm1reqGGu1dNcWBe6_XkZ5tK-UaiymrYRVgxRxI,1515
|
17
14
|
source_google_search_console/schemas/search_analytics_site_report_by_site.json,sha256=rAh6LuNy7nCrrNM9MTd0qxAVc886ecQaqWRgV63OfyA,1408
|
18
15
|
source_google_search_console/service_account_authenticator.py,sha256=pAWKAXfwfTY3xkXvQJH0EyFphFULdCIcC47YXYTO9X8,1307
|
19
|
-
source_google_search_console/source.py,sha256=
|
20
|
-
source_google_search_console/spec.json,sha256=
|
21
|
-
source_google_search_console/streams.py,sha256=
|
22
|
-
airbyte_source_google_search_console-1.
|
23
|
-
airbyte_source_google_search_console-1.
|
24
|
-
airbyte_source_google_search_console-1.
|
25
|
-
airbyte_source_google_search_console-1.
|
16
|
+
source_google_search_console/source.py,sha256=ddqYq3ZSNrVtGvuQQ5WnHgMLuf92AtumzdDKWGffwRo,8721
|
17
|
+
source_google_search_console/spec.json,sha256=SensIrWvoie2aGwxmgoMYpq2jb3XP3F2myQWCVQ4db8,8980
|
18
|
+
source_google_search_console/streams.py,sha256=yb78hGaKPDCJoFqPSun3lDhr8VhSxcLgXTLtX2ouVpY,15804
|
19
|
+
airbyte_source_google_search_console-1.7.0.dist-info/METADATA,sha256=R-pJQZUfWv2duwzSFWta7vbn8V01TeESsaoRO5728mo,5621
|
20
|
+
airbyte_source_google_search_console-1.7.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
|
21
|
+
airbyte_source_google_search_console-1.7.0.dist-info/entry_points.txt,sha256=DMcgc9bCX-Vt6hm_68pa77qS3eGdeMhg-UdlFc-XKUM,85
|
22
|
+
airbyte_source_google_search_console-1.7.0.dist-info/RECORD,,
|
@@ -199,10 +199,305 @@ definitions:
|
|
199
199
|
schema:
|
200
200
|
$ref: "#/schemas/sitemaps"
|
201
201
|
|
202
|
+
# This stream is only used as a parent stream for search_by_keyword substreams
|
203
|
+
search_appearances_stream:
|
204
|
+
type: DeclarativeStream
|
205
|
+
name: search_appearances
|
206
|
+
retriever:
|
207
|
+
type: SimpleRetriever
|
208
|
+
requester:
|
209
|
+
type: HttpRequester
|
210
|
+
url_base: https://www.googleapis.com/webmasters/v3
|
211
|
+
path: "/sites/{{ sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
|
212
|
+
http_method: POST
|
213
|
+
authenticator: "#/definitions/selective_authenticator"
|
214
|
+
request_headers:
|
215
|
+
Content-Type: "application/json"
|
216
|
+
request_body_json:
|
217
|
+
startDate: "{{ config.get('start_date') }}"
|
218
|
+
endDate: "{{ config.get('end_date') }}"
|
219
|
+
dimensions: ["searchAppearance"]
|
220
|
+
type: "{{ stream_partition.get('search_type') }}"
|
221
|
+
aggregationType: auto
|
222
|
+
dataState: "{{ config.get('data_state', 'final') }}"
|
223
|
+
paginator:
|
224
|
+
type: DefaultPaginator
|
225
|
+
page_token_option:
|
226
|
+
type: RequestOption
|
227
|
+
field_name: startRow
|
228
|
+
inject_into: body_json
|
229
|
+
page_size_option:
|
230
|
+
type: RequestOption
|
231
|
+
field_name: rowLimit
|
232
|
+
inject_into: body_json
|
233
|
+
pagination_strategy:
|
234
|
+
type: OffsetIncrement
|
235
|
+
page_size: 25000
|
236
|
+
inject_on_first_request: true
|
237
|
+
record_selector:
|
238
|
+
type: RecordSelector
|
239
|
+
extractor:
|
240
|
+
type: DpathExtractor
|
241
|
+
field_path:
|
242
|
+
- rows
|
243
|
+
partition_router:
|
244
|
+
- type: ListPartitionRouter
|
245
|
+
values: "{{ config['site_urls'] }}"
|
246
|
+
cursor_field: site_url
|
247
|
+
- type: ListPartitionRouter
|
248
|
+
values:
|
249
|
+
- web
|
250
|
+
- news
|
251
|
+
- image
|
252
|
+
- video
|
253
|
+
cursor_field: search_type
|
254
|
+
transformations:
|
255
|
+
- type: AddFields
|
256
|
+
fields:
|
257
|
+
- path:
|
258
|
+
- searchAppearance
|
259
|
+
value: "{{ record['keys'][0] }}"
|
260
|
+
- type: RemoveFields
|
261
|
+
field_pointers:
|
262
|
+
- - keys
|
263
|
+
|
264
|
+
base_search_analytics_keyword_stream:
|
265
|
+
type: DeclarativeStream
|
266
|
+
retriever:
|
267
|
+
type: SimpleRetriever
|
268
|
+
requester:
|
269
|
+
type: HttpRequester
|
270
|
+
url_base: https://www.googleapis.com/webmasters/v3
|
271
|
+
path: "/sites/{{ sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
|
272
|
+
http_method: POST
|
273
|
+
authenticator: "#/definitions/selective_authenticator"
|
274
|
+
request_headers:
|
275
|
+
Content-Type: "application/json"
|
276
|
+
request_body_json:
|
277
|
+
startDate: "{{ stream_interval.get('start_time') }}"
|
278
|
+
endDate: "{{ stream_interval.get('end_time') }}"
|
279
|
+
dimensions: "{{ parameters.get('dimensions') }}"
|
280
|
+
type: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
|
281
|
+
aggregationType: "{{ 'auto' if config.get('always_use_aggregation_type_auto') else parameters.get('aggregationType') }}"
|
282
|
+
dataState: "{{ config.get('data_state', 'final') }}"
|
283
|
+
dimensionFilterGroups: "{{ [{'groupType': 'and', 'filters': {'dimension': 'searchAppearance', 'operator': 'equals', 'expression': stream_partition.get('search_appearance')}}] }}"
|
284
|
+
error_handler:
|
285
|
+
type: DefaultErrorHandler
|
286
|
+
response_filters:
|
287
|
+
- type: HttpResponseFilter
|
288
|
+
action: FAIL
|
289
|
+
http_codes:
|
290
|
+
- 400
|
291
|
+
error_message: >-
|
292
|
+
Invalid aggregationType '{{ parameters.get('aggregationType') }}' used in the body of the API request. If you see this error, enable the
|
293
|
+
'always_use_aggregation_type_auto' config setting which will automatically use aggregationType=auto
|
294
|
+
paginator:
|
295
|
+
type: DefaultPaginator
|
296
|
+
page_token_option:
|
297
|
+
type: RequestOption
|
298
|
+
field_name: startRow
|
299
|
+
inject_into: body_json
|
300
|
+
page_size_option:
|
301
|
+
type: RequestOption
|
302
|
+
field_name: rowLimit
|
303
|
+
inject_into: body_json
|
304
|
+
pagination_strategy:
|
305
|
+
type: OffsetIncrement
|
306
|
+
page_size: 25000
|
307
|
+
inject_on_first_request: true
|
308
|
+
record_selector:
|
309
|
+
type: RecordSelector
|
310
|
+
extractor:
|
311
|
+
type: DpathExtractor
|
312
|
+
field_path:
|
313
|
+
- rows
|
314
|
+
partition_router:
|
315
|
+
- type: ListPartitionRouter
|
316
|
+
values: "{{ config['site_urls'] }}"
|
317
|
+
cursor_field: site_url
|
318
|
+
- type: SubstreamPartitionRouter
|
319
|
+
parent_stream_configs:
|
320
|
+
- type: ParentStreamConfig
|
321
|
+
parent_key: searchAppearance
|
322
|
+
partition_field: search_appearance
|
323
|
+
stream:
|
324
|
+
$ref: "#/definitions/search_appearances_stream"
|
325
|
+
incremental_sync:
|
326
|
+
type: DatetimeBasedCursor
|
327
|
+
cursor_field: date
|
328
|
+
cursor_datetime_formats:
|
329
|
+
- "%Y-%m-%d"
|
330
|
+
datetime_format: "%Y-%m-%d"
|
331
|
+
start_datetime:
|
332
|
+
type: MinMaxDatetime
|
333
|
+
datetime: "{{ config.get('start_date', '2021-01-01') }}"
|
334
|
+
datetime_format: "%Y-%m-%d"
|
335
|
+
end_datetime:
|
336
|
+
type: MinMaxDatetime
|
337
|
+
datetime: "{{ config.get('end_date', today_utc()) }}"
|
338
|
+
datetime_format: "%Y-%m-%d"
|
339
|
+
step: P3D
|
340
|
+
cursor_granularity: P1D
|
341
|
+
|
342
|
+
search_analytics_keyword_page_report_stream:
|
343
|
+
$ref: "#/definitions/base_search_analytics_keyword_stream"
|
344
|
+
name: search_analytics_keyword_page_report
|
345
|
+
primary_key:
|
346
|
+
- site_url
|
347
|
+
- date
|
348
|
+
- country
|
349
|
+
- device
|
350
|
+
- query
|
351
|
+
- page
|
352
|
+
- search_type
|
353
|
+
transformations:
|
354
|
+
- type: AddFields
|
355
|
+
fields:
|
356
|
+
- path:
|
357
|
+
- site_url
|
358
|
+
value: "{{ stream_partition['site_url'] }}"
|
359
|
+
- path:
|
360
|
+
- search_type
|
361
|
+
value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
|
362
|
+
- type: AddFields
|
363
|
+
fields:
|
364
|
+
- path:
|
365
|
+
- date
|
366
|
+
value: "{{ record['keys'][0] }}"
|
367
|
+
- path:
|
368
|
+
- country
|
369
|
+
value: "{{ record['keys'][1] }}"
|
370
|
+
- path:
|
371
|
+
- device
|
372
|
+
value: "{{ record['keys'][2] }}"
|
373
|
+
- path:
|
374
|
+
- query
|
375
|
+
value: "{{ record['keys'][3] }}"
|
376
|
+
- path:
|
377
|
+
- page
|
378
|
+
value: "{{ record['keys'][4] }}"
|
379
|
+
- type: RemoveFields
|
380
|
+
field_pointers:
|
381
|
+
- - keys
|
382
|
+
schema_loader:
|
383
|
+
type: InlineSchemaLoader
|
384
|
+
schema:
|
385
|
+
$ref: "#/schemas/search_analytics_keyword_page_report"
|
386
|
+
$parameters:
|
387
|
+
aggregationType: auto
|
388
|
+
dimensions:
|
389
|
+
- date
|
390
|
+
- country
|
391
|
+
- device
|
392
|
+
- query
|
393
|
+
- page
|
394
|
+
|
395
|
+
search_analytics_keyword_site_report_by_page_stream:
|
396
|
+
$ref: "#/definitions/base_search_analytics_keyword_stream"
|
397
|
+
name: search_analytics_keyword_site_report_by_page
|
398
|
+
primary_key:
|
399
|
+
- site_url
|
400
|
+
- date
|
401
|
+
- country
|
402
|
+
- device
|
403
|
+
- query
|
404
|
+
- search_type
|
405
|
+
transformations:
|
406
|
+
- type: AddFields
|
407
|
+
fields:
|
408
|
+
- path:
|
409
|
+
- site_url
|
410
|
+
value: "{{ stream_partition['site_url'] }}"
|
411
|
+
- path:
|
412
|
+
- search_type
|
413
|
+
value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
|
414
|
+
- type: AddFields
|
415
|
+
fields:
|
416
|
+
- path:
|
417
|
+
- date
|
418
|
+
value: "{{ record['keys'][0] }}"
|
419
|
+
- path:
|
420
|
+
- country
|
421
|
+
value: "{{ record['keys'][1] }}"
|
422
|
+
- path:
|
423
|
+
- device
|
424
|
+
value: "{{ record['keys'][2] }}"
|
425
|
+
- path:
|
426
|
+
- query
|
427
|
+
value: "{{ record['keys'][3] }}"
|
428
|
+
- type: RemoveFields
|
429
|
+
field_pointers:
|
430
|
+
- - keys
|
431
|
+
schema_loader:
|
432
|
+
type: InlineSchemaLoader
|
433
|
+
schema:
|
434
|
+
$ref: "#/schemas/search_analytics_keyword_site_report_by_page"
|
435
|
+
$parameters:
|
436
|
+
aggregationType: byPage
|
437
|
+
dimensions:
|
438
|
+
- date
|
439
|
+
- country
|
440
|
+
- device
|
441
|
+
- query
|
442
|
+
|
443
|
+
search_analytics_keyword_site_report_by_site_stream:
|
444
|
+
$ref: "#/definitions/base_search_analytics_keyword_stream"
|
445
|
+
name: search_analytics_keyword_site_report_by_site
|
446
|
+
primary_key:
|
447
|
+
- site_url
|
448
|
+
- date
|
449
|
+
- country
|
450
|
+
- device
|
451
|
+
- query
|
452
|
+
- search_type
|
453
|
+
transformations:
|
454
|
+
- type: AddFields
|
455
|
+
fields:
|
456
|
+
- path:
|
457
|
+
- site_url
|
458
|
+
value: "{{ stream_partition['site_url'] }}"
|
459
|
+
- path:
|
460
|
+
- search_type
|
461
|
+
value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
|
462
|
+
- type: AddFields
|
463
|
+
fields:
|
464
|
+
- path:
|
465
|
+
- date
|
466
|
+
value: "{{ record['keys'][0] }}"
|
467
|
+
- path:
|
468
|
+
- country
|
469
|
+
value: "{{ record['keys'][1] }}"
|
470
|
+
- path:
|
471
|
+
- device
|
472
|
+
value: "{{ record['keys'][2] }}"
|
473
|
+
- path:
|
474
|
+
- query
|
475
|
+
value: "{{ record['keys'][3] }}"
|
476
|
+
- type: RemoveFields
|
477
|
+
field_pointers:
|
478
|
+
- - keys
|
479
|
+
schema_loader:
|
480
|
+
type: InlineSchemaLoader
|
481
|
+
schema:
|
482
|
+
$ref: "#/schemas/search_analytics_keyword_site_report_by_site"
|
483
|
+
$parameters:
|
484
|
+
aggregationType: byProperty
|
485
|
+
dimensions:
|
486
|
+
- date
|
487
|
+
- country
|
488
|
+
- device
|
489
|
+
- query
|
490
|
+
|
202
491
|
streams:
|
203
|
-
|
492
|
+
# Regular streams
|
204
493
|
- "#/definitions/sites_stream"
|
205
494
|
- "#/definitions/sitemaps_stream"
|
495
|
+
# Search Analytics streams
|
496
|
+
- "#/definitions/search_analytics_by_country_stream"
|
497
|
+
# Search Analytics Keyword streams
|
498
|
+
- "#/definitions/search_analytics_keyword_page_report_stream"
|
499
|
+
- "#/definitions/search_analytics_keyword_site_report_by_page_stream"
|
500
|
+
- "#/definitions/search_analytics_keyword_site_report_by_site_stream"
|
206
501
|
|
207
502
|
schemas:
|
208
503
|
search_analytics_by_country:
|
@@ -339,3 +634,195 @@ schemas:
|
|
339
634
|
type:
|
340
635
|
- "null"
|
341
636
|
- string
|
637
|
+
|
638
|
+
search_analytics_keyword_page_report:
|
639
|
+
$schema: "https://json-schema.org/draft-07/schema#"
|
640
|
+
type: object
|
641
|
+
additionalProperties: true
|
642
|
+
properties:
|
643
|
+
site_url:
|
644
|
+
description: The URL of the website being monitored.
|
645
|
+
type:
|
646
|
+
- "null"
|
647
|
+
- string
|
648
|
+
search_type:
|
649
|
+
description: "The type of search (e.g., web, image, video)."
|
650
|
+
type:
|
651
|
+
- "null"
|
652
|
+
- string
|
653
|
+
date:
|
654
|
+
description: The date of the search data collected.
|
655
|
+
type:
|
656
|
+
- "null"
|
657
|
+
- string
|
658
|
+
format: date
|
659
|
+
country:
|
660
|
+
description: The country where the search is made.
|
661
|
+
type:
|
662
|
+
- "null"
|
663
|
+
- string
|
664
|
+
device:
|
665
|
+
description: "The device type used for the search (e.g., desktop, mobile)."
|
666
|
+
type:
|
667
|
+
- "null"
|
668
|
+
- string
|
669
|
+
page:
|
670
|
+
description: The page URL on which the keyword appears in search results.
|
671
|
+
type:
|
672
|
+
- "null"
|
673
|
+
- string
|
674
|
+
query:
|
675
|
+
description: The search query used to find the site.
|
676
|
+
type:
|
677
|
+
- "null"
|
678
|
+
- string
|
679
|
+
clicks:
|
680
|
+
description: The number of clicks for the keyword on a specific page.
|
681
|
+
type:
|
682
|
+
- "null"
|
683
|
+
- integer
|
684
|
+
impressions:
|
685
|
+
description: The number of times the keyword appeared in search results.
|
686
|
+
type:
|
687
|
+
- "null"
|
688
|
+
- integer
|
689
|
+
ctr:
|
690
|
+
description: >-
|
691
|
+
Click-through rate which is the percentage of clicks divided by
|
692
|
+
impressions.
|
693
|
+
type:
|
694
|
+
- "null"
|
695
|
+
- number
|
696
|
+
multipleOf: 1.e-25
|
697
|
+
position:
|
698
|
+
description: The average position of the keyword on search results pages.
|
699
|
+
type:
|
700
|
+
- "null"
|
701
|
+
- number
|
702
|
+
multipleOf: 1.e-25
|
703
|
+
|
704
|
+
search_analytics_keyword_site_report_by_page:
|
705
|
+
$schema: "https://json-schema.org/draft-07/schema#"
|
706
|
+
type: object
|
707
|
+
additionalProperties: true
|
708
|
+
properties:
|
709
|
+
site_url:
|
710
|
+
description: The URL of the website for which the search analytics data is retrieved.
|
711
|
+
type:
|
712
|
+
- "null"
|
713
|
+
- string
|
714
|
+
search_type:
|
715
|
+
description: "The type of search conducted (e.g., web, image, video)."
|
716
|
+
type:
|
717
|
+
- "null"
|
718
|
+
- string
|
719
|
+
date:
|
720
|
+
description: The date when the search data was recorded.
|
721
|
+
type:
|
722
|
+
- "null"
|
723
|
+
- string
|
724
|
+
format: date
|
725
|
+
country:
|
726
|
+
description: The country from which the search query originated.
|
727
|
+
type:
|
728
|
+
- "null"
|
729
|
+
- string
|
730
|
+
device:
|
731
|
+
description: "The device type used for the search query (e.g., desktop, mobile)."
|
732
|
+
type:
|
733
|
+
- "null"
|
734
|
+
- string
|
735
|
+
query:
|
736
|
+
description: The search query used by the user.
|
737
|
+
type:
|
738
|
+
- "null"
|
739
|
+
- string
|
740
|
+
clicks:
|
741
|
+
description: The number of times users clicked on your website link in search results.
|
742
|
+
type:
|
743
|
+
- "null"
|
744
|
+
- integer
|
745
|
+
impressions:
|
746
|
+
description: The number of times your website link appeared in search results.
|
747
|
+
type:
|
748
|
+
- "null"
|
749
|
+
- integer
|
750
|
+
ctr:
|
751
|
+
description: "Click-through rate: Number of clicks divided by the number of impressions."
|
752
|
+
type:
|
753
|
+
- "null"
|
754
|
+
- number
|
755
|
+
multipleOf: 1.e-25
|
756
|
+
position:
|
757
|
+
description: The average position of your website link in search results.
|
758
|
+
type:
|
759
|
+
- "null"
|
760
|
+
- number
|
761
|
+
multipleOf: 1.e-25
|
762
|
+
|
763
|
+
search_analytics_keyword_site_report_by_site:
|
764
|
+
$schema: "https://json-schema.org/draft-07/schema#"
|
765
|
+
type: object
|
766
|
+
additionalProperties: true
|
767
|
+
properties:
|
768
|
+
site_url:
|
769
|
+
description: The URL of the site for which the search analytics data is recorded.
|
770
|
+
type:
|
771
|
+
- "null"
|
772
|
+
- string
|
773
|
+
search_type:
|
774
|
+
description: >-
|
775
|
+
The type of search (e.g., web search, image search) that generated the
|
776
|
+
analytics data.
|
777
|
+
type:
|
778
|
+
- "null"
|
779
|
+
- string
|
780
|
+
date:
|
781
|
+
description: The date for which the search analytics data is recorded.
|
782
|
+
type:
|
783
|
+
- "null"
|
784
|
+
- string
|
785
|
+
format: date
|
786
|
+
country:
|
787
|
+
description: The country from which the search originated.
|
788
|
+
type:
|
789
|
+
- "null"
|
790
|
+
- string
|
791
|
+
device:
|
792
|
+
description: >-
|
793
|
+
The type of device used by the user during the search (e.g., desktop,
|
794
|
+
mobile).
|
795
|
+
type:
|
796
|
+
- "null"
|
797
|
+
- string
|
798
|
+
query:
|
799
|
+
description: The search query used by the user to find the site in search results.
|
800
|
+
type:
|
801
|
+
- "null"
|
802
|
+
- string
|
803
|
+
clicks:
|
804
|
+
description: >-
|
805
|
+
The number of times users clicked on the search result linking to the
|
806
|
+
site.
|
807
|
+
type:
|
808
|
+
- "null"
|
809
|
+
- integer
|
810
|
+
impressions:
|
811
|
+
description: The number of times the site was shown in search results to users.
|
812
|
+
type:
|
813
|
+
- "null"
|
814
|
+
- integer
|
815
|
+
ctr:
|
816
|
+
description: >-
|
817
|
+
Click-through rate represents the percentage of users who clicked on the
|
818
|
+
site's link after seeing it in search results.
|
819
|
+
type:
|
820
|
+
- "null"
|
821
|
+
- number
|
822
|
+
multipleOf: 1.e-25
|
823
|
+
position:
|
824
|
+
description: The average ranking position of the site in search results.
|
825
|
+
type:
|
826
|
+
- "null"
|
827
|
+
- number
|
828
|
+
multipleOf: 1.e-25
|
@@ -3,7 +3,6 @@
|
|
3
3
|
#
|
4
4
|
|
5
5
|
import json
|
6
|
-
import logging
|
7
6
|
from typing import Any, List, Mapping, Optional, Tuple, Union
|
8
7
|
from urllib.parse import urlparse
|
9
8
|
|
@@ -20,7 +19,6 @@ from airbyte_cdk.utils import AirbyteTracedException
|
|
20
19
|
from source_google_search_console.exceptions import (
|
21
20
|
InvalidSiteURLValidationError,
|
22
21
|
UnauthorizedOauthError,
|
23
|
-
UnauthorizedServiceAccountError,
|
24
22
|
UnidentifiedError,
|
25
23
|
)
|
26
24
|
from source_google_search_console.service_account_authenticator import ServiceAccountAuthenticator
|
@@ -31,9 +29,6 @@ from source_google_search_console.streams import (
|
|
31
29
|
SearchAnalyticsByDevice,
|
32
30
|
SearchAnalyticsByPage,
|
33
31
|
SearchAnalyticsByQuery,
|
34
|
-
SearchAnalyticsKeywordPageReport,
|
35
|
-
SearchAnalyticsKeywordSiteReportByPage,
|
36
|
-
SearchAnalyticsKeywordSiteReportBySite,
|
37
32
|
SearchAnalyticsPageReport,
|
38
33
|
SearchAnalyticsSiteReportByPage,
|
39
34
|
SearchAnalyticsSiteReportBySite,
|
@@ -159,15 +154,11 @@ class SourceGoogleSearchConsole(YamlDeclarativeSource):
|
|
159
154
|
SearchAnalyticsByQuery(**stream_config),
|
160
155
|
SearchAnalyticsByPage(**stream_config),
|
161
156
|
SearchAnalyticsAllFields(**stream_config),
|
162
|
-
SearchAnalyticsKeywordPageReport(**stream_config),
|
163
157
|
SearchAnalyticsPageReport(**stream_config),
|
164
158
|
SearchAnalyticsSiteReportBySite(**stream_config),
|
165
159
|
SearchAnalyticsSiteReportByPage(**stream_config),
|
166
|
-
SearchAnalyticsKeywordSiteReportByPage(**stream_config),
|
167
|
-
SearchAnalyticsKeywordSiteReportBySite(**stream_config),
|
168
160
|
]
|
169
161
|
)
|
170
|
-
|
171
162
|
streams = streams + self.get_custom_reports(config=config, stream_config=stream_config)
|
172
163
|
|
173
164
|
return streams
|
@@ -157,6 +157,13 @@
|
|
157
157
|
"examples": ["final", "all"],
|
158
158
|
"default": "final",
|
159
159
|
"order": 6
|
160
|
+
},
|
161
|
+
"always_use_aggregation_type_auto": {
|
162
|
+
"type": "boolean",
|
163
|
+
"title": "Always Use Aggregation Type Auto",
|
164
|
+
"description": "Some search analytics streams fail with a 400 error if the specified `aggregationType` is not supported. This is customer implementation dependent and if this error is encountered, enable this setting which will override the existing `aggregationType` to use `auto` which should resolve the stream errors.",
|
165
|
+
"default": false,
|
166
|
+
"order": 7
|
160
167
|
}
|
161
168
|
}
|
162
169
|
},
|
@@ -322,71 +322,6 @@ class SearchAnalyticsAllFields(SearchAnalytics):
|
|
322
322
|
dimensions = ["date", "country", "device", "page", "query"]
|
323
323
|
|
324
324
|
|
325
|
-
class SearchAppearance(SearchAnalytics):
|
326
|
-
"""
|
327
|
-
Dimension searchAppearance can't be used with other dimension.
|
328
|
-
search appearance data (AMP, blue link, rich result, and so on) must be queried using a two-step process.
|
329
|
-
https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data
|
330
|
-
"""
|
331
|
-
|
332
|
-
primary_key = None
|
333
|
-
dimensions = ["searchAppearance"]
|
334
|
-
|
335
|
-
|
336
|
-
class SearchByKeyword(SearchAnalytics):
|
337
|
-
"""
|
338
|
-
Adds searchAppearance value to dimensionFilterGroups in json body
|
339
|
-
https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data
|
340
|
-
|
341
|
-
groupType: "and" - Whether all filters in this group must return true ("and"), or one or more must return true (not yet supported).
|
342
|
-
filters: {"dimension": "searchAppearance", "operator": "equals", "expression": keyword}
|
343
|
-
"""
|
344
|
-
|
345
|
-
def stream_slices(
|
346
|
-
self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
|
347
|
-
) -> Iterable[Optional[Mapping[str, Any]]]:
|
348
|
-
search_appearance_stream = SearchAppearance(self._http_client._session.auth, self._site_urls, self._start_date, self._end_date)
|
349
|
-
|
350
|
-
for stream_slice in super().stream_slices(sync_mode, cursor_field, stream_state):
|
351
|
-
keywords_records = search_appearance_stream.read_records(
|
352
|
-
sync_mode=SyncMode.full_refresh, stream_state=stream_state, stream_slice=stream_slice
|
353
|
-
)
|
354
|
-
keywords = {record["searchAppearance"] for record in keywords_records}
|
355
|
-
|
356
|
-
for keyword in keywords:
|
357
|
-
filters = {"dimension": "searchAppearance", "operator": "equals", "expression": keyword}
|
358
|
-
stream_slice["dimensionFilterGroups"] = [{"groupType": "and", "filters": filters}]
|
359
|
-
|
360
|
-
yield stream_slice
|
361
|
-
|
362
|
-
def request_body_json(
|
363
|
-
self,
|
364
|
-
stream_state: Mapping[str, Any] = None,
|
365
|
-
stream_slice: Mapping[str, Any] = None,
|
366
|
-
next_page_token: Mapping[str, Any] = None,
|
367
|
-
) -> Optional[Union[Dict[str, Any], str]]:
|
368
|
-
data = super().request_body_json(stream_state, stream_slice, next_page_token)
|
369
|
-
data["dimensionFilterGroups"] = stream_slice["dimensionFilterGroups"]
|
370
|
-
return data
|
371
|
-
|
372
|
-
|
373
|
-
class SearchAnalyticsKeywordPageReport(SearchByKeyword):
|
374
|
-
primary_key = ["site_url", "date", "country", "device", "query", "page", "search_type"]
|
375
|
-
dimensions = ["date", "country", "device", "query", "page"]
|
376
|
-
|
377
|
-
|
378
|
-
class SearchAnalyticsKeywordSiteReportByPage(SearchByKeyword):
|
379
|
-
primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
|
380
|
-
dimensions = ["date", "country", "device", "query"]
|
381
|
-
aggregation_type = QueryAggregationType.by_page
|
382
|
-
|
383
|
-
|
384
|
-
class SearchAnalyticsKeywordSiteReportBySite(SearchByKeyword):
|
385
|
-
primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
|
386
|
-
dimensions = ["date", "country", "device", "query"]
|
387
|
-
aggregation_type = QueryAggregationType.by_property
|
388
|
-
|
389
|
-
|
390
325
|
class SearchAnalyticsSiteReportBySite(SearchAnalytics):
|
391
326
|
primary_key = ["site_url", "date", "country", "device", "search_type"]
|
392
327
|
dimensions = ["date", "country", "device"]
|
@@ -1,54 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"$schema": "https://json-schema.org/draft-07/schema#",
|
3
|
-
"type": "object",
|
4
|
-
"additionalProperties": true,
|
5
|
-
"properties": {
|
6
|
-
"site_url": {
|
7
|
-
"description": "The URL of the website being monitored.",
|
8
|
-
"type": ["null", "string"]
|
9
|
-
},
|
10
|
-
"search_type": {
|
11
|
-
"description": "The type of search (e.g., web, image, video).",
|
12
|
-
"type": ["null", "string"]
|
13
|
-
},
|
14
|
-
"date": {
|
15
|
-
"description": "The date of the search data collected.",
|
16
|
-
"type": ["null", "string"],
|
17
|
-
"format": "date"
|
18
|
-
},
|
19
|
-
"country": {
|
20
|
-
"description": "The country where the search is made.",
|
21
|
-
"type": ["null", "string"]
|
22
|
-
},
|
23
|
-
"device": {
|
24
|
-
"description": "The device type used for the search (e.g., desktop, mobile).",
|
25
|
-
"type": ["null", "string"]
|
26
|
-
},
|
27
|
-
"page": {
|
28
|
-
"description": "The page URL on which the keyword appears in search results.",
|
29
|
-
"type": ["null", "string"]
|
30
|
-
},
|
31
|
-
"query": {
|
32
|
-
"description": "The search query used to find the site.",
|
33
|
-
"type": ["null", "string"]
|
34
|
-
},
|
35
|
-
"clicks": {
|
36
|
-
"description": "The number of clicks for the keyword on a specific page.",
|
37
|
-
"type": ["null", "integer"]
|
38
|
-
},
|
39
|
-
"impressions": {
|
40
|
-
"description": "The number of times the keyword appeared in search results.",
|
41
|
-
"type": ["null", "integer"]
|
42
|
-
},
|
43
|
-
"ctr": {
|
44
|
-
"description": "Click-through rate which is the percentage of clicks divided by impressions.",
|
45
|
-
"type": ["null", "number"],
|
46
|
-
"multipleOf": 1e-25
|
47
|
-
},
|
48
|
-
"position": {
|
49
|
-
"description": "The average position of the keyword on search results pages.",
|
50
|
-
"type": ["null", "number"],
|
51
|
-
"multipleOf": 1e-25
|
52
|
-
}
|
53
|
-
}
|
54
|
-
}
|
@@ -1,50 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"$schema": "https://json-schema.org/draft-07/schema#",
|
3
|
-
"type": "object",
|
4
|
-
"additionalProperties": true,
|
5
|
-
"properties": {
|
6
|
-
"site_url": {
|
7
|
-
"description": "The URL of the website for which the search analytics data is retrieved.",
|
8
|
-
"type": ["null", "string"]
|
9
|
-
},
|
10
|
-
"search_type": {
|
11
|
-
"description": "The type of search conducted (e.g., web, image, video).",
|
12
|
-
"type": ["null", "string"]
|
13
|
-
},
|
14
|
-
"date": {
|
15
|
-
"description": "The date when the search data was recorded.",
|
16
|
-
"type": ["null", "string"],
|
17
|
-
"format": "date"
|
18
|
-
},
|
19
|
-
"country": {
|
20
|
-
"description": "The country from which the search query originated.",
|
21
|
-
"type": ["null", "string"]
|
22
|
-
},
|
23
|
-
"device": {
|
24
|
-
"description": "The device type used for the search query (e.g., desktop, mobile).",
|
25
|
-
"type": ["null", "string"]
|
26
|
-
},
|
27
|
-
"query": {
|
28
|
-
"description": "The search query used by the user.",
|
29
|
-
"type": ["null", "string"]
|
30
|
-
},
|
31
|
-
"clicks": {
|
32
|
-
"description": "The number of times users clicked on your website link in search results.",
|
33
|
-
"type": ["null", "integer"]
|
34
|
-
},
|
35
|
-
"impressions": {
|
36
|
-
"description": "The number of times your website link appeared in search results.",
|
37
|
-
"type": ["null", "integer"]
|
38
|
-
},
|
39
|
-
"ctr": {
|
40
|
-
"description": "Click-through rate: Number of clicks divided by the number of impressions.",
|
41
|
-
"type": ["null", "number"],
|
42
|
-
"multipleOf": 1e-25
|
43
|
-
},
|
44
|
-
"position": {
|
45
|
-
"description": "The average position of your website link in search results.",
|
46
|
-
"type": ["null", "number"],
|
47
|
-
"multipleOf": 1e-25
|
48
|
-
}
|
49
|
-
}
|
50
|
-
}
|
@@ -1,50 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"$schema": "https://json-schema.org/draft-07/schema#",
|
3
|
-
"type": "object",
|
4
|
-
"additionalProperties": true,
|
5
|
-
"properties": {
|
6
|
-
"site_url": {
|
7
|
-
"description": "The URL of the site for which the search analytics data is recorded.",
|
8
|
-
"type": ["null", "string"]
|
9
|
-
},
|
10
|
-
"search_type": {
|
11
|
-
"description": "The type of search (e.g., web search, image search) that generated the analytics data.",
|
12
|
-
"type": ["null", "string"]
|
13
|
-
},
|
14
|
-
"date": {
|
15
|
-
"description": "The date for which the search analytics data is recorded.",
|
16
|
-
"type": ["null", "string"],
|
17
|
-
"format": "date"
|
18
|
-
},
|
19
|
-
"country": {
|
20
|
-
"description": "The country from which the search originated.",
|
21
|
-
"type": ["null", "string"]
|
22
|
-
},
|
23
|
-
"device": {
|
24
|
-
"description": "The type of device used by the user during the search (e.g., desktop, mobile).",
|
25
|
-
"type": ["null", "string"]
|
26
|
-
},
|
27
|
-
"query": {
|
28
|
-
"description": "The search query used by the user to find the site in search results.",
|
29
|
-
"type": ["null", "string"]
|
30
|
-
},
|
31
|
-
"clicks": {
|
32
|
-
"description": "The number of times users clicked on the search result linking to the site.",
|
33
|
-
"type": ["null", "integer"]
|
34
|
-
},
|
35
|
-
"impressions": {
|
36
|
-
"description": "The number of times the site was shown in search results to users.",
|
37
|
-
"type": ["null", "integer"]
|
38
|
-
},
|
39
|
-
"ctr": {
|
40
|
-
"description": "Click-through rate represents the percentage of users who clicked on the site's link after seeing it in search results.",
|
41
|
-
"type": ["null", "number"],
|
42
|
-
"multipleOf": 1e-25
|
43
|
-
},
|
44
|
-
"position": {
|
45
|
-
"description": "The average ranking position of the site in search results.",
|
46
|
-
"type": ["null", "number"],
|
47
|
-
"multipleOf": 1e-25
|
48
|
-
}
|
49
|
-
}
|
50
|
-
}
|
File without changes
|