airbyte-source-google-search-console 1.6.0rc1__py3-none-any.whl → 1.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: airbyte-source-google-search-console
3
- Version: 1.6.0rc1
3
+ Version: 1.7.0
4
4
  Summary: Source implementation for Google Search Console.
5
5
  License: Elv2
6
6
  Author: Airbyte
@@ -2,24 +2,21 @@ source_google_search_console/__init__.py,sha256=HQCPu-CK7XmVDtP9rmTdB2XyraVCc6pv
2
2
  source_google_search_console/components.py,sha256=5o8kH2xwYUvk3yjnSd6okJVF2KBUqUIF4V97xRpPAyI,2803
3
3
  source_google_search_console/config_migrations.py,sha256=Cl4SUdJpAf6wMM_vVhqjjU89NfUq9LIGJ9zNrWiBk-A,4235
4
4
  source_google_search_console/exceptions.py,sha256=iD3jYC4WxVCEKGsqQ7Vaj1tbjhJZ4S5mnSDnwFJdsIQ,1097
5
- source_google_search_console/manifest.yaml,sha256=zoPWyJ7gwOOK6yKAk4H92cD-N7Xg3iHtp5hTT_e80QA,10974
5
+ source_google_search_console/manifest.yaml,sha256=m6IhF4bRwF9nAivJev2ebB8QPvoKRbYixF-5ZKSBkms,25882
6
6
  source_google_search_console/run.py,sha256=TBkPlseTERarkj6wL8AMEKgm5Xsb2drnltPVH6257-M,2195
7
7
  source_google_search_console/schemas/search_analytics_all_fields.json,sha256=iQxRh_c_yz3uGofqpo1KX571TMmzYjKScb0PtI6SN_Q,1729
8
8
  source_google_search_console/schemas/search_analytics_by_date.json,sha256=meCbWDayc1y0q-Lu-CAdjQVnsM8xZBX3BdF129UC1P8,1388
9
9
  source_google_search_console/schemas/search_analytics_by_device.json,sha256=VtoFjmmv9rx-uhSFaRn0wm4LeSxRIaexrxg2Spvbneo,1525
10
10
  source_google_search_console/schemas/search_analytics_by_page.json,sha256=KyUojZc4Lv3hPswxIJzUL5QDNsbvSugGjl_uHGF7Am4,1473
11
11
  source_google_search_console/schemas/search_analytics_by_query.json,sha256=mYc1Fu1A5TWLZCApZSjjfs_urW5BRUCPQI3rw3yQjt4,1439
12
- source_google_search_console/schemas/search_analytics_keyword_page_report.json,sha256=hCQZbpW9UNiCOrnUMkj4oQdzY0k8Hqoh1XkuKHvyvZw,1681
13
- source_google_search_console/schemas/search_analytics_keyword_site_report_by_page.json,sha256=lFpbShmTQhFyySc_JCdIOqMP9RfOIA15fpQN9xNXtTI,1626
14
- source_google_search_console/schemas/search_analytics_keyword_site_report_by_site.json,sha256=AVUl5x9jLL0FcStbTR-FMy-_A7uM-VlrJ-kdCBXWz_g,1755
15
12
  source_google_search_console/schemas/search_analytics_page_report.json,sha256=-b0Y0LenTchS0q9A2aQ4hIjUjXkYF8erOtyrTMhf6MM,1776
16
13
  source_google_search_console/schemas/search_analytics_site_report_by_page.json,sha256=hWKHkm1reqGGu1dNcWBe6_XkZ5tK-UaiymrYRVgxRxI,1515
17
14
  source_google_search_console/schemas/search_analytics_site_report_by_site.json,sha256=rAh6LuNy7nCrrNM9MTd0qxAVc886ecQaqWRgV63OfyA,1408
18
15
  source_google_search_console/service_account_authenticator.py,sha256=pAWKAXfwfTY3xkXvQJH0EyFphFULdCIcC47YXYTO9X8,1307
19
- source_google_search_console/source.py,sha256=7FD4ciRrsptU7ZIxAU2xLC37bgjKWzNkflE9ybmgpXM,9113
20
- source_google_search_console/spec.json,sha256=WYtFvaSqWYGm1Dt2yV9G92U78Q94rh9oarbxJe3H7xo,8470
21
- source_google_search_console/streams.py,sha256=T0eqhmxGPDAfFNMXFfG_vM4aYHFaHAHHAoj6s5XnjnI,18760
22
- airbyte_source_google_search_console-1.6.0rc1.dist-info/METADATA,sha256=smSxRuSxL5aXxK-OcjVGdpR_jLJN_-dMEtsKyMu5Oww,5624
23
- airbyte_source_google_search_console-1.6.0rc1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
24
- airbyte_source_google_search_console-1.6.0rc1.dist-info/entry_points.txt,sha256=DMcgc9bCX-Vt6hm_68pa77qS3eGdeMhg-UdlFc-XKUM,85
25
- airbyte_source_google_search_console-1.6.0rc1.dist-info/RECORD,,
16
+ source_google_search_console/source.py,sha256=ddqYq3ZSNrVtGvuQQ5WnHgMLuf92AtumzdDKWGffwRo,8721
17
+ source_google_search_console/spec.json,sha256=SensIrWvoie2aGwxmgoMYpq2jb3XP3F2myQWCVQ4db8,8980
18
+ source_google_search_console/streams.py,sha256=yb78hGaKPDCJoFqPSun3lDhr8VhSxcLgXTLtX2ouVpY,15804
19
+ airbyte_source_google_search_console-1.7.0.dist-info/METADATA,sha256=R-pJQZUfWv2duwzSFWta7vbn8V01TeESsaoRO5728mo,5621
20
+ airbyte_source_google_search_console-1.7.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
21
+ airbyte_source_google_search_console-1.7.0.dist-info/entry_points.txt,sha256=DMcgc9bCX-Vt6hm_68pa77qS3eGdeMhg-UdlFc-XKUM,85
22
+ airbyte_source_google_search_console-1.7.0.dist-info/RECORD,,
@@ -199,10 +199,305 @@ definitions:
199
199
  schema:
200
200
  $ref: "#/schemas/sitemaps"
201
201
 
202
+ # This stream is only used as a parent stream for search_by_keyword substreams
203
+ search_appearances_stream:
204
+ type: DeclarativeStream
205
+ name: search_appearances
206
+ retriever:
207
+ type: SimpleRetriever
208
+ requester:
209
+ type: HttpRequester
210
+ url_base: https://www.googleapis.com/webmasters/v3
211
+ path: "/sites/{{ sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
212
+ http_method: POST
213
+ authenticator: "#/definitions/selective_authenticator"
214
+ request_headers:
215
+ Content-Type: "application/json"
216
+ request_body_json:
217
+ startDate: "{{ config.get('start_date') }}"
218
+ endDate: "{{ config.get('end_date') }}"
219
+ dimensions: ["searchAppearance"]
220
+ type: "{{ stream_partition.get('search_type') }}"
221
+ aggregationType: auto
222
+ dataState: "{{ config.get('data_state', 'final') }}"
223
+ paginator:
224
+ type: DefaultPaginator
225
+ page_token_option:
226
+ type: RequestOption
227
+ field_name: startRow
228
+ inject_into: body_json
229
+ page_size_option:
230
+ type: RequestOption
231
+ field_name: rowLimit
232
+ inject_into: body_json
233
+ pagination_strategy:
234
+ type: OffsetIncrement
235
+ page_size: 25000
236
+ inject_on_first_request: true
237
+ record_selector:
238
+ type: RecordSelector
239
+ extractor:
240
+ type: DpathExtractor
241
+ field_path:
242
+ - rows
243
+ partition_router:
244
+ - type: ListPartitionRouter
245
+ values: "{{ config['site_urls'] }}"
246
+ cursor_field: site_url
247
+ - type: ListPartitionRouter
248
+ values:
249
+ - web
250
+ - news
251
+ - image
252
+ - video
253
+ cursor_field: search_type
254
+ transformations:
255
+ - type: AddFields
256
+ fields:
257
+ - path:
258
+ - searchAppearance
259
+ value: "{{ record['keys'][0] }}"
260
+ - type: RemoveFields
261
+ field_pointers:
262
+ - - keys
263
+
264
+ base_search_analytics_keyword_stream:
265
+ type: DeclarativeStream
266
+ retriever:
267
+ type: SimpleRetriever
268
+ requester:
269
+ type: HttpRequester
270
+ url_base: https://www.googleapis.com/webmasters/v3
271
+ path: "/sites/{{ sanitize_url(stream_partition.get('site_url')) }}/searchAnalytics/query"
272
+ http_method: POST
273
+ authenticator: "#/definitions/selective_authenticator"
274
+ request_headers:
275
+ Content-Type: "application/json"
276
+ request_body_json:
277
+ startDate: "{{ stream_interval.get('start_time') }}"
278
+ endDate: "{{ stream_interval.get('end_time') }}"
279
+ dimensions: "{{ parameters.get('dimensions') }}"
280
+ type: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
281
+ aggregationType: "{{ 'auto' if config.get('always_use_aggregation_type_auto') else parameters.get('aggregationType') }}"
282
+ dataState: "{{ config.get('data_state', 'final') }}"
283
+ dimensionFilterGroups: "{{ [{'groupType': 'and', 'filters': {'dimension': 'searchAppearance', 'operator': 'equals', 'expression': stream_partition.get('search_appearance')}}] }}"
284
+ error_handler:
285
+ type: DefaultErrorHandler
286
+ response_filters:
287
+ - type: HttpResponseFilter
288
+ action: FAIL
289
+ http_codes:
290
+ - 400
291
+ error_message: >-
292
+ Invalid aggregationType '{{ parameters.get('aggregationType') }}' used in the body of the API request. If you see this error, enable the
293
+ 'always_use_aggregation_type_auto' config setting which will automatically use aggregationType=auto
294
+ paginator:
295
+ type: DefaultPaginator
296
+ page_token_option:
297
+ type: RequestOption
298
+ field_name: startRow
299
+ inject_into: body_json
300
+ page_size_option:
301
+ type: RequestOption
302
+ field_name: rowLimit
303
+ inject_into: body_json
304
+ pagination_strategy:
305
+ type: OffsetIncrement
306
+ page_size: 25000
307
+ inject_on_first_request: true
308
+ record_selector:
309
+ type: RecordSelector
310
+ extractor:
311
+ type: DpathExtractor
312
+ field_path:
313
+ - rows
314
+ partition_router:
315
+ - type: ListPartitionRouter
316
+ values: "{{ config['site_urls'] }}"
317
+ cursor_field: site_url
318
+ - type: SubstreamPartitionRouter
319
+ parent_stream_configs:
320
+ - type: ParentStreamConfig
321
+ parent_key: searchAppearance
322
+ partition_field: search_appearance
323
+ stream:
324
+ $ref: "#/definitions/search_appearances_stream"
325
+ incremental_sync:
326
+ type: DatetimeBasedCursor
327
+ cursor_field: date
328
+ cursor_datetime_formats:
329
+ - "%Y-%m-%d"
330
+ datetime_format: "%Y-%m-%d"
331
+ start_datetime:
332
+ type: MinMaxDatetime
333
+ datetime: "{{ config.get('start_date', '2021-01-01') }}"
334
+ datetime_format: "%Y-%m-%d"
335
+ end_datetime:
336
+ type: MinMaxDatetime
337
+ datetime: "{{ config.get('end_date', today_utc()) }}"
338
+ datetime_format: "%Y-%m-%d"
339
+ step: P3D
340
+ cursor_granularity: P1D
341
+
342
+ search_analytics_keyword_page_report_stream:
343
+ $ref: "#/definitions/base_search_analytics_keyword_stream"
344
+ name: search_analytics_keyword_page_report
345
+ primary_key:
346
+ - site_url
347
+ - date
348
+ - country
349
+ - device
350
+ - query
351
+ - page
352
+ - search_type
353
+ transformations:
354
+ - type: AddFields
355
+ fields:
356
+ - path:
357
+ - site_url
358
+ value: "{{ stream_partition['site_url'] }}"
359
+ - path:
360
+ - search_type
361
+ value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
362
+ - type: AddFields
363
+ fields:
364
+ - path:
365
+ - date
366
+ value: "{{ record['keys'][0] }}"
367
+ - path:
368
+ - country
369
+ value: "{{ record['keys'][1] }}"
370
+ - path:
371
+ - device
372
+ value: "{{ record['keys'][2] }}"
373
+ - path:
374
+ - query
375
+ value: "{{ record['keys'][3] }}"
376
+ - path:
377
+ - page
378
+ value: "{{ record['keys'][4] }}"
379
+ - type: RemoveFields
380
+ field_pointers:
381
+ - - keys
382
+ schema_loader:
383
+ type: InlineSchemaLoader
384
+ schema:
385
+ $ref: "#/schemas/search_analytics_keyword_page_report"
386
+ $parameters:
387
+ aggregationType: auto
388
+ dimensions:
389
+ - date
390
+ - country
391
+ - device
392
+ - query
393
+ - page
394
+
395
+ search_analytics_keyword_site_report_by_page_stream:
396
+ $ref: "#/definitions/base_search_analytics_keyword_stream"
397
+ name: search_analytics_keyword_site_report_by_page
398
+ primary_key:
399
+ - site_url
400
+ - date
401
+ - country
402
+ - device
403
+ - query
404
+ - search_type
405
+ transformations:
406
+ - type: AddFields
407
+ fields:
408
+ - path:
409
+ - site_url
410
+ value: "{{ stream_partition['site_url'] }}"
411
+ - path:
412
+ - search_type
413
+ value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
414
+ - type: AddFields
415
+ fields:
416
+ - path:
417
+ - date
418
+ value: "{{ record['keys'][0] }}"
419
+ - path:
420
+ - country
421
+ value: "{{ record['keys'][1] }}"
422
+ - path:
423
+ - device
424
+ value: "{{ record['keys'][2] }}"
425
+ - path:
426
+ - query
427
+ value: "{{ record['keys'][3] }}"
428
+ - type: RemoveFields
429
+ field_pointers:
430
+ - - keys
431
+ schema_loader:
432
+ type: InlineSchemaLoader
433
+ schema:
434
+ $ref: "#/schemas/search_analytics_keyword_site_report_by_page"
435
+ $parameters:
436
+ aggregationType: byPage
437
+ dimensions:
438
+ - date
439
+ - country
440
+ - device
441
+ - query
442
+
443
+ search_analytics_keyword_site_report_by_site_stream:
444
+ $ref: "#/definitions/base_search_analytics_keyword_stream"
445
+ name: search_analytics_keyword_site_report_by_site
446
+ primary_key:
447
+ - site_url
448
+ - date
449
+ - country
450
+ - device
451
+ - query
452
+ - search_type
453
+ transformations:
454
+ - type: AddFields
455
+ fields:
456
+ - path:
457
+ - site_url
458
+ value: "{{ stream_partition['site_url'] }}"
459
+ - path:
460
+ - search_type
461
+ value: "{{ stream_partition.get('parent_slice', {}).get('search_type') }}"
462
+ - type: AddFields
463
+ fields:
464
+ - path:
465
+ - date
466
+ value: "{{ record['keys'][0] }}"
467
+ - path:
468
+ - country
469
+ value: "{{ record['keys'][1] }}"
470
+ - path:
471
+ - device
472
+ value: "{{ record['keys'][2] }}"
473
+ - path:
474
+ - query
475
+ value: "{{ record['keys'][3] }}"
476
+ - type: RemoveFields
477
+ field_pointers:
478
+ - - keys
479
+ schema_loader:
480
+ type: InlineSchemaLoader
481
+ schema:
482
+ $ref: "#/schemas/search_analytics_keyword_site_report_by_site"
483
+ $parameters:
484
+ aggregationType: byProperty
485
+ dimensions:
486
+ - date
487
+ - country
488
+ - device
489
+ - query
490
+
202
491
  streams:
203
- - "#/definitions/search_analytics_by_country_stream"
492
+ # Regular streams
204
493
  - "#/definitions/sites_stream"
205
494
  - "#/definitions/sitemaps_stream"
495
+ # Search Analytics streams
496
+ - "#/definitions/search_analytics_by_country_stream"
497
+ # Search Analytics Keyword streams
498
+ - "#/definitions/search_analytics_keyword_page_report_stream"
499
+ - "#/definitions/search_analytics_keyword_site_report_by_page_stream"
500
+ - "#/definitions/search_analytics_keyword_site_report_by_site_stream"
206
501
 
207
502
  schemas:
208
503
  search_analytics_by_country:
@@ -339,3 +634,195 @@ schemas:
339
634
  type:
340
635
  - "null"
341
636
  - string
637
+
638
+ search_analytics_keyword_page_report:
639
+ $schema: "https://json-schema.org/draft-07/schema#"
640
+ type: object
641
+ additionalProperties: true
642
+ properties:
643
+ site_url:
644
+ description: The URL of the website being monitored.
645
+ type:
646
+ - "null"
647
+ - string
648
+ search_type:
649
+ description: "The type of search (e.g., web, image, video)."
650
+ type:
651
+ - "null"
652
+ - string
653
+ date:
654
+ description: The date of the search data collected.
655
+ type:
656
+ - "null"
657
+ - string
658
+ format: date
659
+ country:
660
+ description: The country where the search is made.
661
+ type:
662
+ - "null"
663
+ - string
664
+ device:
665
+ description: "The device type used for the search (e.g., desktop, mobile)."
666
+ type:
667
+ - "null"
668
+ - string
669
+ page:
670
+ description: The page URL on which the keyword appears in search results.
671
+ type:
672
+ - "null"
673
+ - string
674
+ query:
675
+ description: The search query used to find the site.
676
+ type:
677
+ - "null"
678
+ - string
679
+ clicks:
680
+ description: The number of clicks for the keyword on a specific page.
681
+ type:
682
+ - "null"
683
+ - integer
684
+ impressions:
685
+ description: The number of times the keyword appeared in search results.
686
+ type:
687
+ - "null"
688
+ - integer
689
+ ctr:
690
+ description: >-
691
+ Click-through rate which is the percentage of clicks divided by
692
+ impressions.
693
+ type:
694
+ - "null"
695
+ - number
696
+ multipleOf: 1.e-25
697
+ position:
698
+ description: The average position of the keyword on search results pages.
699
+ type:
700
+ - "null"
701
+ - number
702
+ multipleOf: 1.e-25
703
+
704
+ search_analytics_keyword_site_report_by_page:
705
+ $schema: "https://json-schema.org/draft-07/schema#"
706
+ type: object
707
+ additionalProperties: true
708
+ properties:
709
+ site_url:
710
+ description: The URL of the website for which the search analytics data is retrieved.
711
+ type:
712
+ - "null"
713
+ - string
714
+ search_type:
715
+ description: "The type of search conducted (e.g., web, image, video)."
716
+ type:
717
+ - "null"
718
+ - string
719
+ date:
720
+ description: The date when the search data was recorded.
721
+ type:
722
+ - "null"
723
+ - string
724
+ format: date
725
+ country:
726
+ description: The country from which the search query originated.
727
+ type:
728
+ - "null"
729
+ - string
730
+ device:
731
+ description: "The device type used for the search query (e.g., desktop, mobile)."
732
+ type:
733
+ - "null"
734
+ - string
735
+ query:
736
+ description: The search query used by the user.
737
+ type:
738
+ - "null"
739
+ - string
740
+ clicks:
741
+ description: The number of times users clicked on your website link in search results.
742
+ type:
743
+ - "null"
744
+ - integer
745
+ impressions:
746
+ description: The number of times your website link appeared in search results.
747
+ type:
748
+ - "null"
749
+ - integer
750
+ ctr:
751
+ description: "Click-through rate: Number of clicks divided by the number of impressions."
752
+ type:
753
+ - "null"
754
+ - number
755
+ multipleOf: 1.e-25
756
+ position:
757
+ description: The average position of your website link in search results.
758
+ type:
759
+ - "null"
760
+ - number
761
+ multipleOf: 1.e-25
762
+
763
+ search_analytics_keyword_site_report_by_site:
764
+ $schema: "https://json-schema.org/draft-07/schema#"
765
+ type: object
766
+ additionalProperties: true
767
+ properties:
768
+ site_url:
769
+ description: The URL of the site for which the search analytics data is recorded.
770
+ type:
771
+ - "null"
772
+ - string
773
+ search_type:
774
+ description: >-
775
+ The type of search (e.g., web search, image search) that generated the
776
+ analytics data.
777
+ type:
778
+ - "null"
779
+ - string
780
+ date:
781
+ description: The date for which the search analytics data is recorded.
782
+ type:
783
+ - "null"
784
+ - string
785
+ format: date
786
+ country:
787
+ description: The country from which the search originated.
788
+ type:
789
+ - "null"
790
+ - string
791
+ device:
792
+ description: >-
793
+ The type of device used by the user during the search (e.g., desktop,
794
+ mobile).
795
+ type:
796
+ - "null"
797
+ - string
798
+ query:
799
+ description: The search query used by the user to find the site in search results.
800
+ type:
801
+ - "null"
802
+ - string
803
+ clicks:
804
+ description: >-
805
+ The number of times users clicked on the search result linking to the
806
+ site.
807
+ type:
808
+ - "null"
809
+ - integer
810
+ impressions:
811
+ description: The number of times the site was shown in search results to users.
812
+ type:
813
+ - "null"
814
+ - integer
815
+ ctr:
816
+ description: >-
817
+ Click-through rate represents the percentage of users who clicked on the
818
+ site's link after seeing it in search results.
819
+ type:
820
+ - "null"
821
+ - number
822
+ multipleOf: 1.e-25
823
+ position:
824
+ description: The average ranking position of the site in search results.
825
+ type:
826
+ - "null"
827
+ - number
828
+ multipleOf: 1.e-25
@@ -3,7 +3,6 @@
3
3
  #
4
4
 
5
5
  import json
6
- import logging
7
6
  from typing import Any, List, Mapping, Optional, Tuple, Union
8
7
  from urllib.parse import urlparse
9
8
 
@@ -20,7 +19,6 @@ from airbyte_cdk.utils import AirbyteTracedException
20
19
  from source_google_search_console.exceptions import (
21
20
  InvalidSiteURLValidationError,
22
21
  UnauthorizedOauthError,
23
- UnauthorizedServiceAccountError,
24
22
  UnidentifiedError,
25
23
  )
26
24
  from source_google_search_console.service_account_authenticator import ServiceAccountAuthenticator
@@ -31,9 +29,6 @@ from source_google_search_console.streams import (
31
29
  SearchAnalyticsByDevice,
32
30
  SearchAnalyticsByPage,
33
31
  SearchAnalyticsByQuery,
34
- SearchAnalyticsKeywordPageReport,
35
- SearchAnalyticsKeywordSiteReportByPage,
36
- SearchAnalyticsKeywordSiteReportBySite,
37
32
  SearchAnalyticsPageReport,
38
33
  SearchAnalyticsSiteReportByPage,
39
34
  SearchAnalyticsSiteReportBySite,
@@ -159,15 +154,11 @@ class SourceGoogleSearchConsole(YamlDeclarativeSource):
159
154
  SearchAnalyticsByQuery(**stream_config),
160
155
  SearchAnalyticsByPage(**stream_config),
161
156
  SearchAnalyticsAllFields(**stream_config),
162
- SearchAnalyticsKeywordPageReport(**stream_config),
163
157
  SearchAnalyticsPageReport(**stream_config),
164
158
  SearchAnalyticsSiteReportBySite(**stream_config),
165
159
  SearchAnalyticsSiteReportByPage(**stream_config),
166
- SearchAnalyticsKeywordSiteReportByPage(**stream_config),
167
- SearchAnalyticsKeywordSiteReportBySite(**stream_config),
168
160
  ]
169
161
  )
170
-
171
162
  streams = streams + self.get_custom_reports(config=config, stream_config=stream_config)
172
163
 
173
164
  return streams
@@ -157,6 +157,13 @@
157
157
  "examples": ["final", "all"],
158
158
  "default": "final",
159
159
  "order": 6
160
+ },
161
+ "always_use_aggregation_type_auto": {
162
+ "type": "boolean",
163
+ "title": "Always Use Aggregation Type Auto",
164
+ "description": "Some search analytics streams fail with a 400 error if the specified `aggregationType` is not supported. This is customer implementation dependent and if this error is encountered, enable this setting which will override the existing `aggregationType` to use `auto` which should resolve the stream errors.",
165
+ "default": false,
166
+ "order": 7
160
167
  }
161
168
  }
162
169
  },
@@ -322,71 +322,6 @@ class SearchAnalyticsAllFields(SearchAnalytics):
322
322
  dimensions = ["date", "country", "device", "page", "query"]
323
323
 
324
324
 
325
- class SearchAppearance(SearchAnalytics):
326
- """
327
- Dimension searchAppearance can't be used with other dimension.
328
- search appearance data (AMP, blue link, rich result, and so on) must be queried using a two-step process.
329
- https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data
330
- """
331
-
332
- primary_key = None
333
- dimensions = ["searchAppearance"]
334
-
335
-
336
- class SearchByKeyword(SearchAnalytics):
337
- """
338
- Adds searchAppearance value to dimensionFilterGroups in json body
339
- https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data
340
-
341
- groupType: "and" - Whether all filters in this group must return true ("and"), or one or more must return true (not yet supported).
342
- filters: {"dimension": "searchAppearance", "operator": "equals", "expression": keyword}
343
- """
344
-
345
- def stream_slices(
346
- self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
347
- ) -> Iterable[Optional[Mapping[str, Any]]]:
348
- search_appearance_stream = SearchAppearance(self._http_client._session.auth, self._site_urls, self._start_date, self._end_date)
349
-
350
- for stream_slice in super().stream_slices(sync_mode, cursor_field, stream_state):
351
- keywords_records = search_appearance_stream.read_records(
352
- sync_mode=SyncMode.full_refresh, stream_state=stream_state, stream_slice=stream_slice
353
- )
354
- keywords = {record["searchAppearance"] for record in keywords_records}
355
-
356
- for keyword in keywords:
357
- filters = {"dimension": "searchAppearance", "operator": "equals", "expression": keyword}
358
- stream_slice["dimensionFilterGroups"] = [{"groupType": "and", "filters": filters}]
359
-
360
- yield stream_slice
361
-
362
- def request_body_json(
363
- self,
364
- stream_state: Mapping[str, Any] = None,
365
- stream_slice: Mapping[str, Any] = None,
366
- next_page_token: Mapping[str, Any] = None,
367
- ) -> Optional[Union[Dict[str, Any], str]]:
368
- data = super().request_body_json(stream_state, stream_slice, next_page_token)
369
- data["dimensionFilterGroups"] = stream_slice["dimensionFilterGroups"]
370
- return data
371
-
372
-
373
- class SearchAnalyticsKeywordPageReport(SearchByKeyword):
374
- primary_key = ["site_url", "date", "country", "device", "query", "page", "search_type"]
375
- dimensions = ["date", "country", "device", "query", "page"]
376
-
377
-
378
- class SearchAnalyticsKeywordSiteReportByPage(SearchByKeyword):
379
- primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
380
- dimensions = ["date", "country", "device", "query"]
381
- aggregation_type = QueryAggregationType.by_page
382
-
383
-
384
- class SearchAnalyticsKeywordSiteReportBySite(SearchByKeyword):
385
- primary_key = ["site_url", "date", "country", "device", "query", "search_type"]
386
- dimensions = ["date", "country", "device", "query"]
387
- aggregation_type = QueryAggregationType.by_property
388
-
389
-
390
325
  class SearchAnalyticsSiteReportBySite(SearchAnalytics):
391
326
  primary_key = ["site_url", "date", "country", "device", "search_type"]
392
327
  dimensions = ["date", "country", "device"]
@@ -1,54 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft-07/schema#",
3
- "type": "object",
4
- "additionalProperties": true,
5
- "properties": {
6
- "site_url": {
7
- "description": "The URL of the website being monitored.",
8
- "type": ["null", "string"]
9
- },
10
- "search_type": {
11
- "description": "The type of search (e.g., web, image, video).",
12
- "type": ["null", "string"]
13
- },
14
- "date": {
15
- "description": "The date of the search data collected.",
16
- "type": ["null", "string"],
17
- "format": "date"
18
- },
19
- "country": {
20
- "description": "The country where the search is made.",
21
- "type": ["null", "string"]
22
- },
23
- "device": {
24
- "description": "The device type used for the search (e.g., desktop, mobile).",
25
- "type": ["null", "string"]
26
- },
27
- "page": {
28
- "description": "The page URL on which the keyword appears in search results.",
29
- "type": ["null", "string"]
30
- },
31
- "query": {
32
- "description": "The search query used to find the site.",
33
- "type": ["null", "string"]
34
- },
35
- "clicks": {
36
- "description": "The number of clicks for the keyword on a specific page.",
37
- "type": ["null", "integer"]
38
- },
39
- "impressions": {
40
- "description": "The number of times the keyword appeared in search results.",
41
- "type": ["null", "integer"]
42
- },
43
- "ctr": {
44
- "description": "Click-through rate which is the percentage of clicks divided by impressions.",
45
- "type": ["null", "number"],
46
- "multipleOf": 1e-25
47
- },
48
- "position": {
49
- "description": "The average position of the keyword on search results pages.",
50
- "type": ["null", "number"],
51
- "multipleOf": 1e-25
52
- }
53
- }
54
- }
@@ -1,50 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft-07/schema#",
3
- "type": "object",
4
- "additionalProperties": true,
5
- "properties": {
6
- "site_url": {
7
- "description": "The URL of the website for which the search analytics data is retrieved.",
8
- "type": ["null", "string"]
9
- },
10
- "search_type": {
11
- "description": "The type of search conducted (e.g., web, image, video).",
12
- "type": ["null", "string"]
13
- },
14
- "date": {
15
- "description": "The date when the search data was recorded.",
16
- "type": ["null", "string"],
17
- "format": "date"
18
- },
19
- "country": {
20
- "description": "The country from which the search query originated.",
21
- "type": ["null", "string"]
22
- },
23
- "device": {
24
- "description": "The device type used for the search query (e.g., desktop, mobile).",
25
- "type": ["null", "string"]
26
- },
27
- "query": {
28
- "description": "The search query used by the user.",
29
- "type": ["null", "string"]
30
- },
31
- "clicks": {
32
- "description": "The number of times users clicked on your website link in search results.",
33
- "type": ["null", "integer"]
34
- },
35
- "impressions": {
36
- "description": "The number of times your website link appeared in search results.",
37
- "type": ["null", "integer"]
38
- },
39
- "ctr": {
40
- "description": "Click-through rate: Number of clicks divided by the number of impressions.",
41
- "type": ["null", "number"],
42
- "multipleOf": 1e-25
43
- },
44
- "position": {
45
- "description": "The average position of your website link in search results.",
46
- "type": ["null", "number"],
47
- "multipleOf": 1e-25
48
- }
49
- }
50
- }
@@ -1,50 +0,0 @@
1
- {
2
- "$schema": "https://json-schema.org/draft-07/schema#",
3
- "type": "object",
4
- "additionalProperties": true,
5
- "properties": {
6
- "site_url": {
7
- "description": "The URL of the site for which the search analytics data is recorded.",
8
- "type": ["null", "string"]
9
- },
10
- "search_type": {
11
- "description": "The type of search (e.g., web search, image search) that generated the analytics data.",
12
- "type": ["null", "string"]
13
- },
14
- "date": {
15
- "description": "The date for which the search analytics data is recorded.",
16
- "type": ["null", "string"],
17
- "format": "date"
18
- },
19
- "country": {
20
- "description": "The country from which the search originated.",
21
- "type": ["null", "string"]
22
- },
23
- "device": {
24
- "description": "The type of device used by the user during the search (e.g., desktop, mobile).",
25
- "type": ["null", "string"]
26
- },
27
- "query": {
28
- "description": "The search query used by the user to find the site in search results.",
29
- "type": ["null", "string"]
30
- },
31
- "clicks": {
32
- "description": "The number of times users clicked on the search result linking to the site.",
33
- "type": ["null", "integer"]
34
- },
35
- "impressions": {
36
- "description": "The number of times the site was shown in search results to users.",
37
- "type": ["null", "integer"]
38
- },
39
- "ctr": {
40
- "description": "Click-through rate represents the percentage of users who clicked on the site's link after seeing it in search results.",
41
- "type": ["null", "number"],
42
- "multipleOf": 1e-25
43
- },
44
- "position": {
45
- "description": "The average ranking position of the site in search results.",
46
- "type": ["null", "number"],
47
- "multipleOf": 1e-25
48
- }
49
- }
50
- }