tk-normalizer 1.2.0__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tk-normalizer
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Summary: URL normalization library for consistent URL representation
5
5
  Author-email: Terakeet <engineering@terakeet.com>
6
6
  Maintainer-email: Terakeet <engineering@terakeet.com>
@@ -105,6 +105,7 @@ The following tracking parameters are automatically removed during normalization
105
105
  - `msclkid` (Microsoft Ads)
106
106
  - `aff_id`, `affid` (affiliate tracking)
107
107
  - `referrer`, `adgroupid`, `srsltid`
108
+ - `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
108
109
 
109
110
  ## Advanced Usage
110
111
 
@@ -152,11 +153,11 @@ from tk_normalizer import TkNormalizer
152
153
  normalizer = TkNormalizer("https://blog.example.com/path?a=1")
153
154
 
154
155
  # Dict-like access to individual fields
155
- print(normalizer["normalized_url"]) # blog.example.com/path?a=1
156
- print(normalizer["parent_normalized_url"]) # blog.example.com
157
- print(normalizer["root_normalized_url"]) # example.com
158
- print(normalizer["query_string"]) # a=1
159
- print(normalizer["path"]) # /path
156
+ print(normalizer["normalized_url"]) # blog.example.com/path?a=1
157
+ print(normalizer["parent_normalized_url"]) # blog.example.com
158
+ print(normalizer["root_normalized_url"]) # example.com
159
+ print(normalizer["query_string"]) # a=1
160
+ print(normalizer["path"]) # /path
160
161
 
161
162
  # Iterate over available fields
162
163
  for key in normalizer:
@@ -69,6 +69,7 @@ The following tracking parameters are automatically removed during normalization
69
69
  - `msclkid` (Microsoft Ads)
70
70
  - `aff_id`, `affid` (affiliate tracking)
71
71
  - `referrer`, `adgroupid`, `srsltid`
72
+ - `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
72
73
 
73
74
  ## Advanced Usage
74
75
 
@@ -116,11 +117,11 @@ from tk_normalizer import TkNormalizer
116
117
  normalizer = TkNormalizer("https://blog.example.com/path?a=1")
117
118
 
118
119
  # Dict-like access to individual fields
119
- print(normalizer["normalized_url"]) # blog.example.com/path?a=1
120
- print(normalizer["parent_normalized_url"]) # blog.example.com
121
- print(normalizer["root_normalized_url"]) # example.com
122
- print(normalizer["query_string"]) # a=1
123
- print(normalizer["path"]) # /path
120
+ print(normalizer["normalized_url"]) # blog.example.com/path?a=1
121
+ print(normalizer["parent_normalized_url"]) # blog.example.com
122
+ print(normalizer["root_normalized_url"]) # example.com
123
+ print(normalizer["query_string"]) # a=1
124
+ print(normalizer["path"]) # /path
124
125
 
125
126
  # Iterate over available fields
126
127
  for key in normalizer:
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tk-normalizer"
7
- version = "1.2.0"
7
+ version = "1.3.0"
8
8
  description = "URL normalization library for consistent URL representation"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -78,6 +78,8 @@ class TkNormalizer:
78
78
  "referrer",
79
79
  "adgroupid",
80
80
  "srsltid",
81
+ "gaa_*",
82
+ "eafs_enabled",
81
83
  ]
82
84
 
83
85
  def __init__(self, url: str, log_errors: bool = True) -> None:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tk-normalizer
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Summary: URL normalization library for consistent URL representation
5
5
  Author-email: Terakeet <engineering@terakeet.com>
6
6
  Maintainer-email: Terakeet <engineering@terakeet.com>
@@ -105,6 +105,7 @@ The following tracking parameters are automatically removed during normalization
105
105
  - `msclkid` (Microsoft Ads)
106
106
  - `aff_id`, `affid` (affiliate tracking)
107
107
  - `referrer`, `adgroupid`, `srsltid`
108
+ - `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
108
109
 
109
110
  ## Advanced Usage
110
111
 
@@ -152,11 +153,11 @@ from tk_normalizer import TkNormalizer
152
153
  normalizer = TkNormalizer("https://blog.example.com/path?a=1")
153
154
 
154
155
  # Dict-like access to individual fields
155
- print(normalizer["normalized_url"]) # blog.example.com/path?a=1
156
- print(normalizer["parent_normalized_url"]) # blog.example.com
157
- print(normalizer["root_normalized_url"]) # example.com
158
- print(normalizer["query_string"]) # a=1
159
- print(normalizer["path"]) # /path
156
+ print(normalizer["normalized_url"]) # blog.example.com/path?a=1
157
+ print(normalizer["parent_normalized_url"]) # blog.example.com
158
+ print(normalizer["root_normalized_url"]) # example.com
159
+ print(normalizer["query_string"]) # a=1
160
+ print(normalizer["path"]) # /path
160
161
 
161
162
  # Iterate over available fields
162
163
  for key in normalizer:
@@ -361,6 +361,34 @@ def test_friendly_url_hash_exists() -> None:
361
361
  assert len(hash_val) == 64
362
362
 
363
363
 
364
+ def test_google_extended_access_params_removed() -> None:
365
+ normalizer = TkNormalizer(
366
+ "https://www.wsj.com/market-data/quotes/TSN?gaa_at=eafs&gaa_n=AWEtsqcSfkA2CiTCf4PLiw7D4crACVBHW5gAe0L5ubd"
367
+ "&gaa_ts=691c0b39&gaa_sig=zdv02R-2BLJsEemEuMPdo0DfiS3WAkuRtu4smExyoSHaXWRIZ9HNY2-6hvPV4FM9hTLebiDiOXe8"
368
+ )
369
+ assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
370
+
371
+
372
+ def test_eafs_enabled_removed() -> None:
373
+ normalizer = TkNormalizer("https://www.wsj.com/market-data/quotes/TSN?eafs_enabled=false")
374
+ assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
375
+
376
+
377
+ def test_gaa_wildcard_covers_unseen_params() -> None:
378
+ normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_future_param=x&a=1")
379
+ assert normalizer["normalized_url"] == "wsj.com/articles/some-story?a=1"
380
+
381
+
382
+ def test_gaa_wildcard_requires_the_underscore() -> None:
383
+ normalizer = TkNormalizer("https://example.com/page?gaa=1&gaal=2")
384
+ assert normalizer["normalized_url"] == "example.com/page?gaa=1&gaal=2"
385
+
386
+
387
+ def test_extended_access_removal_keeps_other_params() -> None:
388
+ normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_at=eafs&mod=hp_lead_pos1&page=2")
389
+ assert normalizer["normalized_url"] == "wsj.com/articles/some-story?mod=hp_lead_pos1&page=2"
390
+
391
+
364
392
  # These fixtures are at the bottom for readability of the upper tests
365
393
 
366
394
 
@@ -382,6 +410,10 @@ def happy_normal_fx() -> list[str]:
382
410
  "nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh&srsltid=afmbooo0gkfvz9kwi-1s5atomy-8mtn6nwvrwub1nmxc9z9b8zse6w7h",
383
411
  "nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh",
384
412
  ),
413
+ (
414
+ "https://www.marketwatch.com/investing/stock/aig/company-profile?countrycode=ch&gaa_at=eafs&gaa_ts=687a0c",
415
+ "marketwatch.com/investing/stock/aig/company-profile?countrycode=ch",
416
+ ),
385
417
  (
386
418
  "https://www.libertymutual.com/insurance-resources/auto/main-types-of-car-insurance#:~:text=The%206%20main%20types%[…]that%20you,Uninsured%20Motorist",
387
419
  "libertymutual.com/insurance-resources/auto/main-types-of-car-insurance",
File without changes
File without changes