tk-normalizer 1.2.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tk_normalizer-1.2.0/src/tk_normalizer.egg-info → tk_normalizer-1.3.0}/PKG-INFO +7 -6
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/README.md +6 -5
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/pyproject.toml +1 -1
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/src/tk_normalizer/normalizer.py +2 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0/src/tk_normalizer.egg-info}/PKG-INFO +7 -6
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/tests/test_normalizer.py +32 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/LICENSE +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/setup.cfg +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/src/tk_normalizer/__init__.py +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/SOURCES.txt +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/dependency_links.txt +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/requires.txt +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -105,6 +105,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
105
105
|
- `msclkid` (Microsoft Ads)
|
|
106
106
|
- `aff_id`, `affid` (affiliate tracking)
|
|
107
107
|
- `referrer`, `adgroupid`, `srsltid`
|
|
108
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
108
109
|
|
|
109
110
|
## Advanced Usage
|
|
110
111
|
|
|
@@ -152,11 +153,11 @@ from tk_normalizer import TkNormalizer
|
|
|
152
153
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
153
154
|
|
|
154
155
|
# Dict-like access to individual fields
|
|
155
|
-
print(normalizer["normalized_url"])
|
|
156
|
-
print(normalizer["parent_normalized_url"])
|
|
157
|
-
print(normalizer["root_normalized_url"])
|
|
158
|
-
print(normalizer["query_string"])
|
|
159
|
-
print(normalizer["path"])
|
|
156
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
157
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
158
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
159
|
+
print(normalizer["query_string"]) # a=1
|
|
160
|
+
print(normalizer["path"]) # /path
|
|
160
161
|
|
|
161
162
|
# Iterate over available fields
|
|
162
163
|
for key in normalizer:
|
|
@@ -69,6 +69,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
69
69
|
- `msclkid` (Microsoft Ads)
|
|
70
70
|
- `aff_id`, `affid` (affiliate tracking)
|
|
71
71
|
- `referrer`, `adgroupid`, `srsltid`
|
|
72
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
72
73
|
|
|
73
74
|
## Advanced Usage
|
|
74
75
|
|
|
@@ -116,11 +117,11 @@ from tk_normalizer import TkNormalizer
|
|
|
116
117
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
117
118
|
|
|
118
119
|
# Dict-like access to individual fields
|
|
119
|
-
print(normalizer["normalized_url"])
|
|
120
|
-
print(normalizer["parent_normalized_url"])
|
|
121
|
-
print(normalizer["root_normalized_url"])
|
|
122
|
-
print(normalizer["query_string"])
|
|
123
|
-
print(normalizer["path"])
|
|
120
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
121
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
122
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
123
|
+
print(normalizer["query_string"]) # a=1
|
|
124
|
+
print(normalizer["path"]) # /path
|
|
124
125
|
|
|
125
126
|
# Iterate over available fields
|
|
126
127
|
for key in normalizer:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -105,6 +105,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
105
105
|
- `msclkid` (Microsoft Ads)
|
|
106
106
|
- `aff_id`, `affid` (affiliate tracking)
|
|
107
107
|
- `referrer`, `adgroupid`, `srsltid`
|
|
108
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
108
109
|
|
|
109
110
|
## Advanced Usage
|
|
110
111
|
|
|
@@ -152,11 +153,11 @@ from tk_normalizer import TkNormalizer
|
|
|
152
153
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
153
154
|
|
|
154
155
|
# Dict-like access to individual fields
|
|
155
|
-
print(normalizer["normalized_url"])
|
|
156
|
-
print(normalizer["parent_normalized_url"])
|
|
157
|
-
print(normalizer["root_normalized_url"])
|
|
158
|
-
print(normalizer["query_string"])
|
|
159
|
-
print(normalizer["path"])
|
|
156
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
157
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
158
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
159
|
+
print(normalizer["query_string"]) # a=1
|
|
160
|
+
print(normalizer["path"]) # /path
|
|
160
161
|
|
|
161
162
|
# Iterate over available fields
|
|
162
163
|
for key in normalizer:
|
|
@@ -361,6 +361,34 @@ def test_friendly_url_hash_exists() -> None:
|
|
|
361
361
|
assert len(hash_val) == 64
|
|
362
362
|
|
|
363
363
|
|
|
364
|
+
def test_google_extended_access_params_removed() -> None:
|
|
365
|
+
normalizer = TkNormalizer(
|
|
366
|
+
"https://www.wsj.com/market-data/quotes/TSN?gaa_at=eafs&gaa_n=AWEtsqcSfkA2CiTCf4PLiw7D4crACVBHW5gAe0L5ubd"
|
|
367
|
+
"&gaa_ts=691c0b39&gaa_sig=zdv02R-2BLJsEemEuMPdo0DfiS3WAkuRtu4smExyoSHaXWRIZ9HNY2-6hvPV4FM9hTLebiDiOXe8"
|
|
368
|
+
)
|
|
369
|
+
assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def test_eafs_enabled_removed() -> None:
|
|
373
|
+
normalizer = TkNormalizer("https://www.wsj.com/market-data/quotes/TSN?eafs_enabled=false")
|
|
374
|
+
assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def test_gaa_wildcard_covers_unseen_params() -> None:
|
|
378
|
+
normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_future_param=x&a=1")
|
|
379
|
+
assert normalizer["normalized_url"] == "wsj.com/articles/some-story?a=1"
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def test_gaa_wildcard_requires_the_underscore() -> None:
|
|
383
|
+
normalizer = TkNormalizer("https://example.com/page?gaa=1&gaal=2")
|
|
384
|
+
assert normalizer["normalized_url"] == "example.com/page?gaa=1&gaal=2"
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def test_extended_access_removal_keeps_other_params() -> None:
|
|
388
|
+
normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_at=eafs&mod=hp_lead_pos1&page=2")
|
|
389
|
+
assert normalizer["normalized_url"] == "wsj.com/articles/some-story?mod=hp_lead_pos1&page=2"
|
|
390
|
+
|
|
391
|
+
|
|
364
392
|
# These fixtures are at the bottom for readability of the upper tests
|
|
365
393
|
|
|
366
394
|
|
|
@@ -382,6 +410,10 @@ def happy_normal_fx() -> list[str]:
|
|
|
382
410
|
"nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh&srsltid=afmbooo0gkfvz9kwi-1s5atomy-8mtn6nwvrwub1nmxc9z9b8zse6w7h",
|
|
383
411
|
"nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh",
|
|
384
412
|
),
|
|
413
|
+
(
|
|
414
|
+
"https://www.marketwatch.com/investing/stock/aig/company-profile?countrycode=ch&gaa_at=eafs&gaa_ts=687a0c",
|
|
415
|
+
"marketwatch.com/investing/stock/aig/company-profile?countrycode=ch",
|
|
416
|
+
),
|
|
385
417
|
(
|
|
386
418
|
"https://www.libertymutual.com/insurance-resources/auto/main-types-of-car-insurance#:~:text=The%206%20main%20types%[…]that%20you,Uninsured%20Motorist",
|
|
387
419
|
"libertymutual.com/insurance-resources/auto/main-types-of-car-insurance",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|