tk-normalizer 1.1.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tk_normalizer-1.1.0/src/tk_normalizer.egg-info → tk_normalizer-1.3.0}/PKG-INFO +31 -6
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/README.md +30 -5
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/pyproject.toml +1 -1
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/src/tk_normalizer/__init__.py +1 -1
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/src/tk_normalizer/normalizer.py +24 -4
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0/src/tk_normalizer.egg-info}/PKG-INFO +31 -6
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/tests/test_normalizer.py +87 -1
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/LICENSE +0 -0
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/setup.cfg +0 -0
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/SOURCES.txt +0 -0
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/dependency_links.txt +0 -0
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/requires.txt +0 -0
- {tk_normalizer-1.1.0 → tk_normalizer-1.3.0}/src/tk_normalizer.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -105,6 +105,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
105
105
|
- `msclkid` (Microsoft Ads)
|
|
106
106
|
- `aff_id`, `affid` (affiliate tracking)
|
|
107
107
|
- `referrer`, `adgroupid`, `srsltid`
|
|
108
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
108
109
|
|
|
109
110
|
## Advanced Usage
|
|
110
111
|
|
|
@@ -152,11 +153,11 @@ from tk_normalizer import TkNormalizer
|
|
|
152
153
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
153
154
|
|
|
154
155
|
# Dict-like access to individual fields
|
|
155
|
-
print(normalizer["normalized_url"])
|
|
156
|
-
print(normalizer["parent_normalized_url"])
|
|
157
|
-
print(normalizer["root_normalized_url"])
|
|
158
|
-
print(normalizer["query_string"])
|
|
159
|
-
print(normalizer["path"])
|
|
156
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
157
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
158
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
159
|
+
print(normalizer["query_string"]) # a=1
|
|
160
|
+
print(normalizer["path"]) # /path
|
|
160
161
|
|
|
161
162
|
# Iterate over available fields
|
|
162
163
|
for key in normalizer:
|
|
@@ -229,6 +230,30 @@ pytest --cov=tk_normalizer --cov-report=html
|
|
|
229
230
|
|
|
230
231
|
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
231
232
|
|
|
233
|
+
## Deploying to PYPI
|
|
234
|
+
|
|
235
|
+
We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to our [PYPI package](https://pypi.org/project/tk-normalizer/) when a release is created. Here is how you can do that:
|
|
236
|
+
1. Cut a PR for your change
|
|
237
|
+
2. Make sure you increment your version number in the [pyproject.toml](pyproject.toml) file in your changes
|
|
238
|
+
3. After approval, merge changes to main branch
|
|
239
|
+
4. Cut a new release in GitHub
|
|
240
|
+
- this can be found on the right hand side of the screen when you are at the repo's home page
|
|
241
|
+
- you should see the current release
|
|
242
|
+
5. For consistency in the release:
|
|
243
|
+
- create a new tag that matches the version number you changed earlier
|
|
244
|
+
- add a title with a brief description of the changes
|
|
245
|
+
- add a small description or link to JIRA tickets for updates
|
|
246
|
+
6. After creating the release you should see a workflow get triggered, this will deploy the updated version to pypi
|
|
247
|
+
7. If you want to see check the pypi package page after the workflow completes running
|
|
248
|
+
|
|
249
|
+
> NOTE: DO NOT change the name of the workflow file. If you do the deployment will not work unless we update the configuration in PYPI under trusted publishers
|
|
250
|
+
|
|
251
|
+
If you have questions or concerns reach out.
|
|
252
|
+
|
|
253
|
+
## Deploying to Snowflake -- UDF Update
|
|
254
|
+
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
255
|
+
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
256
|
+
|
|
232
257
|
## License
|
|
233
258
|
|
|
234
259
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -69,6 +69,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
69
69
|
- `msclkid` (Microsoft Ads)
|
|
70
70
|
- `aff_id`, `affid` (affiliate tracking)
|
|
71
71
|
- `referrer`, `adgroupid`, `srsltid`
|
|
72
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
72
73
|
|
|
73
74
|
## Advanced Usage
|
|
74
75
|
|
|
@@ -116,11 +117,11 @@ from tk_normalizer import TkNormalizer
|
|
|
116
117
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
117
118
|
|
|
118
119
|
# Dict-like access to individual fields
|
|
119
|
-
print(normalizer["normalized_url"])
|
|
120
|
-
print(normalizer["parent_normalized_url"])
|
|
121
|
-
print(normalizer["root_normalized_url"])
|
|
122
|
-
print(normalizer["query_string"])
|
|
123
|
-
print(normalizer["path"])
|
|
120
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
121
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
122
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
123
|
+
print(normalizer["query_string"]) # a=1
|
|
124
|
+
print(normalizer["path"]) # /path
|
|
124
125
|
|
|
125
126
|
# Iterate over available fields
|
|
126
127
|
for key in normalizer:
|
|
@@ -193,6 +194,30 @@ pytest --cov=tk_normalizer --cov-report=html
|
|
|
193
194
|
|
|
194
195
|
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
195
196
|
|
|
197
|
+
## Deploying to PYPI
|
|
198
|
+
|
|
199
|
+
We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to our [PYPI package](https://pypi.org/project/tk-normalizer/) when a release is created. Here is how you can do that:
|
|
200
|
+
1. Cut a PR for your change
|
|
201
|
+
2. Make sure you increment your version number in the [pyproject.toml](pyproject.toml) file in your changes
|
|
202
|
+
3. After approval, merge changes to main branch
|
|
203
|
+
4. Cut a new release in GitHub
|
|
204
|
+
- this can be found on the right hand side of the screen when you are at the repo's home page
|
|
205
|
+
- you should see the current release
|
|
206
|
+
5. For consistency in the release:
|
|
207
|
+
- create a new tag that matches the version number you changed earlier
|
|
208
|
+
- add a title with a brief description of the changes
|
|
209
|
+
- add a small description or link to JIRA tickets for updates
|
|
210
|
+
6. After creating the release you should see a workflow get triggered, this will deploy the updated version to pypi
|
|
211
|
+
7. If you want to see check the pypi package page after the workflow completes running
|
|
212
|
+
|
|
213
|
+
> NOTE: DO NOT change the name of the workflow file. If you do the deployment will not work unless we update the configuration in PYPI under trusted publishers
|
|
214
|
+
|
|
215
|
+
If you have questions or concerns reach out.
|
|
216
|
+
|
|
217
|
+
## Deploying to Snowflake -- UDF Update
|
|
218
|
+
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
219
|
+
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
220
|
+
|
|
196
221
|
## License
|
|
197
222
|
|
|
198
223
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -78,6 +78,8 @@ class TkNormalizer:
|
|
|
78
78
|
"referrer",
|
|
79
79
|
"adgroupid",
|
|
80
80
|
"srsltid",
|
|
81
|
+
"gaa_*",
|
|
82
|
+
"eafs_enabled",
|
|
81
83
|
]
|
|
82
84
|
|
|
83
85
|
def __init__(self, url: str, log_errors: bool = True) -> None:
|
|
@@ -91,7 +93,9 @@ class TkNormalizer:
|
|
|
91
93
|
self.root_normalized_url: str
|
|
92
94
|
self.query_string: str
|
|
93
95
|
self.path: str
|
|
96
|
+
self.friendly_url: str
|
|
94
97
|
self.normalized_url, self.parent_normalized_url, self.root_normalized_url = self.process_url(url)
|
|
98
|
+
self.friendly_url = self.build_friendly_url()
|
|
95
99
|
self.url_hashes: dict[str, str] = self.compute_hashes()
|
|
96
100
|
except InvalidUrlException as e:
|
|
97
101
|
if self.log_errors:
|
|
@@ -186,11 +190,11 @@ class TkNormalizer:
|
|
|
186
190
|
def parse_query_params(query: str) -> list[tuple[str, str]]:
|
|
187
191
|
return parse_qsl(query, keep_blank_values=True)
|
|
188
192
|
|
|
189
|
-
def
|
|
190
|
-
|
|
191
|
-
return any(fnmatch.fnmatch(param, pattern) for pattern in self.query_params_to_remove)
|
|
193
|
+
def _is_unwanted_param(self, param_key: str) -> bool:
|
|
194
|
+
return any(fnmatch.fnmatch(param_key.lower(), p) for p in self.query_params_to_remove)
|
|
192
195
|
|
|
193
|
-
|
|
196
|
+
def remove_unwanted_params(self, query_params: list[tuple[str, str]]) -> list[tuple[str, str]]:
|
|
197
|
+
return [(k, v) for k, v in query_params if not self._is_unwanted_param(k)]
|
|
194
198
|
|
|
195
199
|
@staticmethod
|
|
196
200
|
def sort_query_params(query_params: list[tuple[str, str]]) -> list[tuple[str, str]]:
|
|
@@ -219,6 +223,20 @@ class TkNormalizer:
|
|
|
219
223
|
def get_root_normalized_url(netloc: str) -> str:
|
|
220
224
|
return ".".join(netloc.split(".")[-2:])
|
|
221
225
|
|
|
226
|
+
def build_friendly_url(self) -> str:
|
|
227
|
+
parsed = self.parse_url(self.original_url)
|
|
228
|
+
scheme = parsed.scheme
|
|
229
|
+
netloc = parsed.netloc
|
|
230
|
+
path = unquote(parsed.path)
|
|
231
|
+
path = self.remove_trailing_slash(path)
|
|
232
|
+
query_params = self.parse_query_params(parsed.query)
|
|
233
|
+
query_params = self.remove_unwanted_params(query_params)
|
|
234
|
+
query_params = self.sort_query_params(query_params)
|
|
235
|
+
unique_params = self.remove_duplicate_params(query_params)
|
|
236
|
+
query_string = urlencode(unique_params)
|
|
237
|
+
url = f"{scheme}://{netloc}{path}"
|
|
238
|
+
return f"{url}?{query_string}" if query_string else url
|
|
239
|
+
|
|
222
240
|
def compute_hashes(self) -> dict[str, str]:
|
|
223
241
|
def sha256_hash(value: str) -> str:
|
|
224
242
|
return hashlib.sha256(value.encode()).hexdigest()
|
|
@@ -227,11 +245,13 @@ class TkNormalizer:
|
|
|
227
245
|
"normalized_url_hash": sha256_hash(self.normalized_url),
|
|
228
246
|
"parent_normalized_url_hash": sha256_hash(self.parent_normalized_url),
|
|
229
247
|
"root_normalized_url_hash": sha256_hash(self.root_normalized_url),
|
|
248
|
+
"friendly_url_hash": sha256_hash(self.friendly_url),
|
|
230
249
|
}
|
|
231
250
|
|
|
232
251
|
def to_dict(self) -> dict[str, str | list[tuple[str, str]]]:
|
|
233
252
|
return {
|
|
234
253
|
"normalized_url": self.normalized_url,
|
|
254
|
+
"friendly_url": self.friendly_url,
|
|
235
255
|
"parent_normalized_url": self.parent_normalized_url,
|
|
236
256
|
"root_normalized_url": self.root_normalized_url,
|
|
237
257
|
"query_string": self.query_string,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -105,6 +105,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
105
105
|
- `msclkid` (Microsoft Ads)
|
|
106
106
|
- `aff_id`, `affid` (affiliate tracking)
|
|
107
107
|
- `referrer`, `adgroupid`, `srsltid`
|
|
108
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
108
109
|
|
|
109
110
|
## Advanced Usage
|
|
110
111
|
|
|
@@ -152,11 +153,11 @@ from tk_normalizer import TkNormalizer
|
|
|
152
153
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
153
154
|
|
|
154
155
|
# Dict-like access to individual fields
|
|
155
|
-
print(normalizer["normalized_url"])
|
|
156
|
-
print(normalizer["parent_normalized_url"])
|
|
157
|
-
print(normalizer["root_normalized_url"])
|
|
158
|
-
print(normalizer["query_string"])
|
|
159
|
-
print(normalizer["path"])
|
|
156
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
157
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
158
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
159
|
+
print(normalizer["query_string"]) # a=1
|
|
160
|
+
print(normalizer["path"]) # /path
|
|
160
161
|
|
|
161
162
|
# Iterate over available fields
|
|
162
163
|
for key in normalizer:
|
|
@@ -229,6 +230,30 @@ pytest --cov=tk_normalizer --cov-report=html
|
|
|
229
230
|
|
|
230
231
|
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
231
232
|
|
|
233
|
+
## Deploying to PYPI
|
|
234
|
+
|
|
235
|
+
We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to our [PYPI package](https://pypi.org/project/tk-normalizer/) when a release is created. Here is how you can do that:
|
|
236
|
+
1. Cut a PR for your change
|
|
237
|
+
2. Make sure you increment your version number in the [pyproject.toml](pyproject.toml) file in your changes
|
|
238
|
+
3. After approval, merge changes to main branch
|
|
239
|
+
4. Cut a new release in GitHub
|
|
240
|
+
- this can be found on the right hand side of the screen when you are at the repo's home page
|
|
241
|
+
- you should see the current release
|
|
242
|
+
5. For consistency in the release:
|
|
243
|
+
- create a new tag that matches the version number you changed earlier
|
|
244
|
+
- add a title with a brief description of the changes
|
|
245
|
+
- add a small description or link to JIRA tickets for updates
|
|
246
|
+
6. After creating the release you should see a workflow get triggered, this will deploy the updated version to pypi
|
|
247
|
+
7. If you want to see check the pypi package page after the workflow completes running
|
|
248
|
+
|
|
249
|
+
> NOTE: DO NOT change the name of the workflow file. If you do the deployment will not work unless we update the configuration in PYPI under trusted publishers
|
|
250
|
+
|
|
251
|
+
If you have questions or concerns reach out.
|
|
252
|
+
|
|
253
|
+
## Deploying to Snowflake -- UDF Update
|
|
254
|
+
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
255
|
+
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
256
|
+
|
|
232
257
|
## License
|
|
233
258
|
|
|
234
259
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -210,6 +210,7 @@ def test_dict_conversion() -> None:
|
|
|
210
210
|
|
|
211
211
|
# Check all expected fields are present
|
|
212
212
|
assert "normalized_url" in result
|
|
213
|
+
assert "friendly_url" in result
|
|
213
214
|
assert "parent_normalized_url" in result
|
|
214
215
|
assert "root_normalized_url" in result
|
|
215
216
|
assert "query_string" in result
|
|
@@ -217,9 +218,11 @@ def test_dict_conversion() -> None:
|
|
|
217
218
|
assert "normalized_url_hash" in result
|
|
218
219
|
assert "parent_normalized_url_hash" in result
|
|
219
220
|
assert "root_normalized_url_hash" in result
|
|
221
|
+
assert "friendly_url_hash" in result
|
|
220
222
|
|
|
221
223
|
# Check values
|
|
222
224
|
assert result["normalized_url"] == "example.com/path?a=1&b=2"
|
|
225
|
+
assert result["friendly_url"] == "http://www.Example.com/path?a=1&b=2"
|
|
223
226
|
assert result["parent_normalized_url"] == "example.com"
|
|
224
227
|
assert result["root_normalized_url"] == "example.com"
|
|
225
228
|
assert result["query_string"] == "a=1&b=2"
|
|
@@ -265,6 +268,7 @@ def test_iteration() -> None:
|
|
|
265
268
|
# Check expected keys are present
|
|
266
269
|
expected_keys = {
|
|
267
270
|
"normalized_url",
|
|
271
|
+
"friendly_url",
|
|
268
272
|
"parent_normalized_url",
|
|
269
273
|
"root_normalized_url",
|
|
270
274
|
"query_string",
|
|
@@ -272,6 +276,7 @@ def test_iteration() -> None:
|
|
|
272
276
|
"normalized_url_hash",
|
|
273
277
|
"parent_normalized_url_hash",
|
|
274
278
|
"root_normalized_url_hash",
|
|
279
|
+
"friendly_url_hash",
|
|
275
280
|
}
|
|
276
281
|
assert set(keys) == expected_keys
|
|
277
282
|
|
|
@@ -290,6 +295,7 @@ def test_keys_method() -> None:
|
|
|
290
295
|
keys = normalizer.keys()
|
|
291
296
|
expected_keys = {
|
|
292
297
|
"normalized_url",
|
|
298
|
+
"friendly_url",
|
|
293
299
|
"parent_normalized_url",
|
|
294
300
|
"root_normalized_url",
|
|
295
301
|
"query_string",
|
|
@@ -297,6 +303,7 @@ def test_keys_method() -> None:
|
|
|
297
303
|
"normalized_url_hash",
|
|
298
304
|
"parent_normalized_url_hash",
|
|
299
305
|
"root_normalized_url_hash",
|
|
306
|
+
"friendly_url_hash",
|
|
300
307
|
}
|
|
301
308
|
|
|
302
309
|
assert set(keys) == expected_keys
|
|
@@ -304,7 +311,82 @@ def test_keys_method() -> None:
|
|
|
304
311
|
# Keys should be a KeysView object
|
|
305
312
|
assert hasattr(keys, "__iter__")
|
|
306
313
|
assert hasattr(keys, "__len__")
|
|
307
|
-
assert len(keys) ==
|
|
314
|
+
assert len(keys) == 10
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def test_friendly_url_preserves_casing() -> None:
|
|
318
|
+
normalizer = TkNormalizer("https://www.Terakeet.com/SomeUrl?utm_source=something")
|
|
319
|
+
assert normalizer["friendly_url"] == "https://www.Terakeet.com/SomeUrl"
|
|
320
|
+
assert normalizer["normalized_url"] == "terakeet.com/someurl"
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def test_friendly_url_preserves_scheme() -> None:
|
|
324
|
+
normalizer = TkNormalizer("https://example.com/Path")
|
|
325
|
+
assert normalizer["friendly_url"] == "https://example.com/Path"
|
|
326
|
+
|
|
327
|
+
normalizer = TkNormalizer("http://example.com/Path")
|
|
328
|
+
assert normalizer["friendly_url"] == "http://example.com/Path"
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def test_friendly_url_preserves_www() -> None:
|
|
332
|
+
normalizer = TkNormalizer("https://www.Example.com/Path")
|
|
333
|
+
assert normalizer["friendly_url"] == "https://www.Example.com/Path"
|
|
334
|
+
assert normalizer["normalized_url"] == "example.com/path"
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def test_friendly_url_removes_tracking_params() -> None:
|
|
338
|
+
normalizer = TkNormalizer("https://www.Example.com/Page?a=1&utm_source=test&fbclid=abc&b=2")
|
|
339
|
+
assert normalizer["friendly_url"] == "https://www.Example.com/Page?a=1&b=2"
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def test_friendly_url_sorts_and_dedupes_params() -> None:
|
|
343
|
+
normalizer = TkNormalizer("https://Example.com/Page?b=2&a=1&b=2&a=1")
|
|
344
|
+
assert normalizer["friendly_url"] == "https://Example.com/Page?a=1&b=2"
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def test_friendly_url_strips_trailing_slash_and_fragment() -> None:
|
|
348
|
+
normalizer = TkNormalizer("https://www.Example.com/Path/#section")
|
|
349
|
+
assert normalizer["friendly_url"] == "https://www.Example.com/Path"
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def test_friendly_url_no_scheme_input() -> None:
|
|
353
|
+
normalizer = TkNormalizer("example.com/Path?a=1")
|
|
354
|
+
assert normalizer["friendly_url"] == "http://example.com/Path?a=1"
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def test_friendly_url_hash_exists() -> None:
|
|
358
|
+
normalizer = TkNormalizer("https://example.com/path")
|
|
359
|
+
hash_val = normalizer["friendly_url_hash"]
|
|
360
|
+
assert isinstance(hash_val, str)
|
|
361
|
+
assert len(hash_val) == 64
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def test_google_extended_access_params_removed() -> None:
|
|
365
|
+
normalizer = TkNormalizer(
|
|
366
|
+
"https://www.wsj.com/market-data/quotes/TSN?gaa_at=eafs&gaa_n=AWEtsqcSfkA2CiTCf4PLiw7D4crACVBHW5gAe0L5ubd"
|
|
367
|
+
"&gaa_ts=691c0b39&gaa_sig=zdv02R-2BLJsEemEuMPdo0DfiS3WAkuRtu4smExyoSHaXWRIZ9HNY2-6hvPV4FM9hTLebiDiOXe8"
|
|
368
|
+
)
|
|
369
|
+
assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def test_eafs_enabled_removed() -> None:
|
|
373
|
+
normalizer = TkNormalizer("https://www.wsj.com/market-data/quotes/TSN?eafs_enabled=false")
|
|
374
|
+
assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def test_gaa_wildcard_covers_unseen_params() -> None:
|
|
378
|
+
normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_future_param=x&a=1")
|
|
379
|
+
assert normalizer["normalized_url"] == "wsj.com/articles/some-story?a=1"
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def test_gaa_wildcard_requires_the_underscore() -> None:
|
|
383
|
+
normalizer = TkNormalizer("https://example.com/page?gaa=1&gaal=2")
|
|
384
|
+
assert normalizer["normalized_url"] == "example.com/page?gaa=1&gaal=2"
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def test_extended_access_removal_keeps_other_params() -> None:
|
|
388
|
+
normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_at=eafs&mod=hp_lead_pos1&page=2")
|
|
389
|
+
assert normalizer["normalized_url"] == "wsj.com/articles/some-story?mod=hp_lead_pos1&page=2"
|
|
308
390
|
|
|
309
391
|
|
|
310
392
|
# These fixtures are at the bottom for readability of the upper tests
|
|
@@ -328,6 +410,10 @@ def happy_normal_fx() -> list[str]:
|
|
|
328
410
|
"nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh&srsltid=afmbooo0gkfvz9kwi-1s5atomy-8mtn6nwvrwub1nmxc9z9b8zse6w7h",
|
|
329
411
|
"nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh",
|
|
330
412
|
),
|
|
413
|
+
(
|
|
414
|
+
"https://www.marketwatch.com/investing/stock/aig/company-profile?countrycode=ch&gaa_at=eafs&gaa_ts=687a0c",
|
|
415
|
+
"marketwatch.com/investing/stock/aig/company-profile?countrycode=ch",
|
|
416
|
+
),
|
|
331
417
|
(
|
|
332
418
|
"https://www.libertymutual.com/insurance-resources/auto/main-types-of-car-insurance#:~:text=The%206%20main%20types%[…]that%20you,Uninsured%20Motorist",
|
|
333
419
|
"libertymutual.com/insurance-resources/auto/main-types-of-car-insurance",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|