tk-normalizer 1.2.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tk_normalizer-1.2.0/src/tk_normalizer.egg-info → tk_normalizer-1.4.0}/PKG-INFO +36 -13
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/README.md +34 -12
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/pyproject.toml +4 -1
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/src/tk_normalizer/__init__.py +1 -1
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/src/tk_normalizer/normalizer.py +20 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0/src/tk_normalizer.egg-info}/PKG-INFO +36 -13
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/requires.txt +1 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/tests/test_normalizer.py +77 -1
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/LICENSE +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/setup.cfg +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/SOURCES.txt +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/dependency_links.txt +0 -0
- {tk_normalizer-1.2.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -24,6 +24,7 @@ Classifier: Operating System :: OS Independent
|
|
|
24
24
|
Requires-Python: >=3.11
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
|
+
Requires-Dist: tldextract>=5.3.0
|
|
27
28
|
Provides-Extra: dev
|
|
28
29
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
29
30
|
Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
|
|
@@ -105,6 +106,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
105
106
|
- `msclkid` (Microsoft Ads)
|
|
106
107
|
- `aff_id`, `affid` (affiliate tracking)
|
|
107
108
|
- `referrer`, `adgroupid`, `srsltid`
|
|
109
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
108
110
|
|
|
109
111
|
## Advanced Usage
|
|
110
112
|
|
|
@@ -123,13 +125,17 @@ result = dict(normalizer)
|
|
|
123
125
|
print(result)
|
|
124
126
|
# {
|
|
125
127
|
# 'normalized_url': 'blog.example.com/page?a=1&b=2',
|
|
128
|
+
# 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
|
|
126
129
|
# 'parent_normalized_url': 'blog.example.com',
|
|
127
130
|
# 'root_normalized_url': 'example.com',
|
|
131
|
+
# 'registrable_domain': 'example.com',
|
|
128
132
|
# 'query_string': 'a=1&b=2',
|
|
129
133
|
# 'path': '/page',
|
|
130
134
|
# 'normalized_url_hash': '...',
|
|
131
135
|
# 'parent_normalized_url_hash': '...',
|
|
132
|
-
# 'root_normalized_url_hash': '...'
|
|
136
|
+
# 'root_normalized_url_hash': '...',
|
|
137
|
+
# 'registrable_domain_hash': '...',
|
|
138
|
+
# 'friendly_url_hash': '...'
|
|
133
139
|
# }
|
|
134
140
|
```
|
|
135
141
|
|
|
@@ -152,11 +158,12 @@ from tk_normalizer import TkNormalizer
|
|
|
152
158
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
153
159
|
|
|
154
160
|
# Dict-like access to individual fields
|
|
155
|
-
print(normalizer["normalized_url"])
|
|
156
|
-
print(normalizer["parent_normalized_url"])
|
|
157
|
-
print(normalizer["root_normalized_url"])
|
|
158
|
-
print(normalizer["
|
|
159
|
-
print(normalizer["
|
|
161
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
162
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
163
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
164
|
+
print(normalizer["registrable_domain"]) # example.com
|
|
165
|
+
print(normalizer["query_string"]) # a=1
|
|
166
|
+
print(normalizer["path"]) # /path
|
|
160
167
|
|
|
161
168
|
# Iterate over available fields
|
|
162
169
|
for key in normalizer:
|
|
@@ -166,14 +173,29 @@ for key in normalizer:
|
|
|
166
173
|
print(normalizer.keys())
|
|
167
174
|
```
|
|
168
175
|
|
|
176
|
+
## Which URL Form to Use
|
|
177
|
+
|
|
178
|
+
| need | use |
|
|
179
|
+
|------|-----|
|
|
180
|
+
| comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
|
|
181
|
+
| showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
|
|
182
|
+
| linking out, fetching, redriving | the original URL you passed in |
|
|
183
|
+
| grouping by site | `registrable_domain` |
|
|
184
|
+
|
|
185
|
+
`normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
|
|
186
|
+
next to it wherever you need to reach the page again.
|
|
187
|
+
|
|
169
188
|
## Hashing
|
|
170
189
|
|
|
171
190
|
For efficient storage and comparison, SHA-256 hashes are computed for:
|
|
172
191
|
- The normalized URL
|
|
173
192
|
- The parent normal URL (domain without path)
|
|
174
|
-
- The root normal URL (
|
|
193
|
+
- The root normal URL (last two labels of the host)
|
|
194
|
+
- The registrable domain
|
|
195
|
+
- The friendly URL
|
|
175
196
|
|
|
176
|
-
This provides fixed-length representations suitable for database indexing.
|
|
197
|
+
This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
|
|
198
|
+
(`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
|
|
177
199
|
|
|
178
200
|
## Important Caveats
|
|
179
201
|
|
|
@@ -187,6 +209,11 @@ While this normalization process works well for most use cases, there are some l
|
|
|
187
209
|
|
|
188
210
|
4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
|
|
189
211
|
|
|
212
|
+
5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
|
|
213
|
+
multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
|
|
214
|
+
List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
|
|
215
|
+
`root_normalized_url` is unchanged so existing consumers do not shift.
|
|
216
|
+
|
|
190
217
|
## Development
|
|
191
218
|
|
|
192
219
|
### Setting Up Development Environment
|
|
@@ -249,10 +276,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
|
|
|
249
276
|
|
|
250
277
|
If you have questions or concerns reach out.
|
|
251
278
|
|
|
252
|
-
## Deploying to Snowflake -- UDF Update
|
|
253
|
-
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
254
|
-
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
255
|
-
|
|
256
279
|
## License
|
|
257
280
|
|
|
258
281
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -69,6 +69,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
69
69
|
- `msclkid` (Microsoft Ads)
|
|
70
70
|
- `aff_id`, `affid` (affiliate tracking)
|
|
71
71
|
- `referrer`, `adgroupid`, `srsltid`
|
|
72
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
72
73
|
|
|
73
74
|
## Advanced Usage
|
|
74
75
|
|
|
@@ -87,13 +88,17 @@ result = dict(normalizer)
|
|
|
87
88
|
print(result)
|
|
88
89
|
# {
|
|
89
90
|
# 'normalized_url': 'blog.example.com/page?a=1&b=2',
|
|
91
|
+
# 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
|
|
90
92
|
# 'parent_normalized_url': 'blog.example.com',
|
|
91
93
|
# 'root_normalized_url': 'example.com',
|
|
94
|
+
# 'registrable_domain': 'example.com',
|
|
92
95
|
# 'query_string': 'a=1&b=2',
|
|
93
96
|
# 'path': '/page',
|
|
94
97
|
# 'normalized_url_hash': '...',
|
|
95
98
|
# 'parent_normalized_url_hash': '...',
|
|
96
|
-
# 'root_normalized_url_hash': '...'
|
|
99
|
+
# 'root_normalized_url_hash': '...',
|
|
100
|
+
# 'registrable_domain_hash': '...',
|
|
101
|
+
# 'friendly_url_hash': '...'
|
|
97
102
|
# }
|
|
98
103
|
```
|
|
99
104
|
|
|
@@ -116,11 +121,12 @@ from tk_normalizer import TkNormalizer
|
|
|
116
121
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
117
122
|
|
|
118
123
|
# Dict-like access to individual fields
|
|
119
|
-
print(normalizer["normalized_url"])
|
|
120
|
-
print(normalizer["parent_normalized_url"])
|
|
121
|
-
print(normalizer["root_normalized_url"])
|
|
122
|
-
print(normalizer["
|
|
123
|
-
print(normalizer["
|
|
124
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
125
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
126
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
127
|
+
print(normalizer["registrable_domain"]) # example.com
|
|
128
|
+
print(normalizer["query_string"]) # a=1
|
|
129
|
+
print(normalizer["path"]) # /path
|
|
124
130
|
|
|
125
131
|
# Iterate over available fields
|
|
126
132
|
for key in normalizer:
|
|
@@ -130,14 +136,29 @@ for key in normalizer:
|
|
|
130
136
|
print(normalizer.keys())
|
|
131
137
|
```
|
|
132
138
|
|
|
139
|
+
## Which URL Form to Use
|
|
140
|
+
|
|
141
|
+
| need | use |
|
|
142
|
+
|------|-----|
|
|
143
|
+
| comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
|
|
144
|
+
| showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
|
|
145
|
+
| linking out, fetching, redriving | the original URL you passed in |
|
|
146
|
+
| grouping by site | `registrable_domain` |
|
|
147
|
+
|
|
148
|
+
`normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
|
|
149
|
+
next to it wherever you need to reach the page again.
|
|
150
|
+
|
|
133
151
|
## Hashing
|
|
134
152
|
|
|
135
153
|
For efficient storage and comparison, SHA-256 hashes are computed for:
|
|
136
154
|
- The normalized URL
|
|
137
155
|
- The parent normal URL (domain without path)
|
|
138
|
-
- The root normal URL (
|
|
156
|
+
- The root normal URL (last two labels of the host)
|
|
157
|
+
- The registrable domain
|
|
158
|
+
- The friendly URL
|
|
139
159
|
|
|
140
|
-
This provides fixed-length representations suitable for database indexing.
|
|
160
|
+
This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
|
|
161
|
+
(`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
|
|
141
162
|
|
|
142
163
|
## Important Caveats
|
|
143
164
|
|
|
@@ -151,6 +172,11 @@ While this normalization process works well for most use cases, there are some l
|
|
|
151
172
|
|
|
152
173
|
4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
|
|
153
174
|
|
|
175
|
+
5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
|
|
176
|
+
multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
|
|
177
|
+
List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
|
|
178
|
+
`root_normalized_url` is unchanged so existing consumers do not shift.
|
|
179
|
+
|
|
154
180
|
## Development
|
|
155
181
|
|
|
156
182
|
### Setting Up Development Environment
|
|
@@ -213,10 +239,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
|
|
|
213
239
|
|
|
214
240
|
If you have questions or concerns reach out.
|
|
215
241
|
|
|
216
|
-
## Deploying to Snowflake -- UDF Update
|
|
217
|
-
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
218
|
-
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
219
|
-
|
|
220
242
|
## License
|
|
221
243
|
|
|
222
244
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tk-normalizer"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.4.0"
|
|
8
8
|
description = "URL normalization library for consistent URL representation"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -29,6 +29,9 @@ classifiers = [
|
|
|
29
29
|
"Topic :: Text Processing :: Filters",
|
|
30
30
|
"Operating System :: OS Independent",
|
|
31
31
|
]
|
|
32
|
+
dependencies = [
|
|
33
|
+
"tldextract>=5.3.0",
|
|
34
|
+
]
|
|
32
35
|
|
|
33
36
|
[project.urls]
|
|
34
37
|
Homepage = "https://github.com/terakeet/tk-normalizer"
|
|
@@ -5,6 +5,12 @@ import re
|
|
|
5
5
|
from collections.abc import Iterator, KeysView
|
|
6
6
|
from urllib.parse import ParseResult, parse_qsl, unquote, urlencode, urlparse
|
|
7
7
|
|
|
8
|
+
import tldextract
|
|
9
|
+
|
|
10
|
+
# The bundled Public Suffix List snapshot (ICANN section): no network fetch and no cache writes,
|
|
11
|
+
# so it behaves the same in a Snowflake UDF, a worker pool, or a read-only filesystem.
|
|
12
|
+
_PUBLIC_SUFFIXES = tldextract.TLDExtract(suffix_list_urls=(), cache_dir=None)
|
|
13
|
+
|
|
8
14
|
# Uses https://gist.github.com/Integralist/edcfb88c925658a13fc3e51f581fe4bc as a starting point
|
|
9
15
|
# Modified for more current rules regarding host/domain/tld naming.
|
|
10
16
|
|
|
@@ -78,6 +84,8 @@ class TkNormalizer:
|
|
|
78
84
|
"referrer",
|
|
79
85
|
"adgroupid",
|
|
80
86
|
"srsltid",
|
|
87
|
+
"gaa_*",
|
|
88
|
+
"eafs_enabled",
|
|
81
89
|
]
|
|
82
90
|
|
|
83
91
|
def __init__(self, url: str, log_errors: bool = True) -> None:
|
|
@@ -89,10 +97,12 @@ class TkNormalizer:
|
|
|
89
97
|
self.normalized_url: str
|
|
90
98
|
self.parent_normalized_url: str
|
|
91
99
|
self.root_normalized_url: str
|
|
100
|
+
self.registrable_domain: str
|
|
92
101
|
self.query_string: str
|
|
93
102
|
self.path: str
|
|
94
103
|
self.friendly_url: str
|
|
95
104
|
self.normalized_url, self.parent_normalized_url, self.root_normalized_url = self.process_url(url)
|
|
105
|
+
self.registrable_domain = self.get_registrable_domain(self.parent_normalized_url)
|
|
96
106
|
self.friendly_url = self.build_friendly_url()
|
|
97
107
|
self.url_hashes: dict[str, str] = self.compute_hashes()
|
|
98
108
|
except InvalidUrlException as e:
|
|
@@ -217,6 +227,14 @@ class TkNormalizer:
|
|
|
217
227
|
def get_parent_normalized_url(netloc: str) -> str:
|
|
218
228
|
return netloc
|
|
219
229
|
|
|
230
|
+
@staticmethod
|
|
231
|
+
def get_registrable_domain(netloc: str) -> str:
|
|
232
|
+
"""The host one label below its public suffix: bbc.co.uk for news.bbc.co.uk, where the root is co.uk.
|
|
233
|
+
|
|
234
|
+
Falls back to the host when there is no such label (an IP address, or a bare suffix like gov.uk).
|
|
235
|
+
"""
|
|
236
|
+
return _PUBLIC_SUFFIXES(netloc).top_domain_under_public_suffix or netloc
|
|
237
|
+
|
|
220
238
|
@staticmethod
|
|
221
239
|
def get_root_normalized_url(netloc: str) -> str:
|
|
222
240
|
return ".".join(netloc.split(".")[-2:])
|
|
@@ -243,6 +261,7 @@ class TkNormalizer:
|
|
|
243
261
|
"normalized_url_hash": sha256_hash(self.normalized_url),
|
|
244
262
|
"parent_normalized_url_hash": sha256_hash(self.parent_normalized_url),
|
|
245
263
|
"root_normalized_url_hash": sha256_hash(self.root_normalized_url),
|
|
264
|
+
"registrable_domain_hash": sha256_hash(self.registrable_domain),
|
|
246
265
|
"friendly_url_hash": sha256_hash(self.friendly_url),
|
|
247
266
|
}
|
|
248
267
|
|
|
@@ -252,6 +271,7 @@ class TkNormalizer:
|
|
|
252
271
|
"friendly_url": self.friendly_url,
|
|
253
272
|
"parent_normalized_url": self.parent_normalized_url,
|
|
254
273
|
"root_normalized_url": self.root_normalized_url,
|
|
274
|
+
"registrable_domain": self.registrable_domain,
|
|
255
275
|
"query_string": self.query_string,
|
|
256
276
|
"path": self.path,
|
|
257
277
|
**self.url_hashes,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -24,6 +24,7 @@ Classifier: Operating System :: OS Independent
|
|
|
24
24
|
Requires-Python: >=3.11
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
|
+
Requires-Dist: tldextract>=5.3.0
|
|
27
28
|
Provides-Extra: dev
|
|
28
29
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
29
30
|
Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
|
|
@@ -105,6 +106,7 @@ The following tracking parameters are automatically removed during normalization
|
|
|
105
106
|
- `msclkid` (Microsoft Ads)
|
|
106
107
|
- `aff_id`, `affid` (affiliate tracking)
|
|
107
108
|
- `referrer`, `adgroupid`, `srsltid`
|
|
109
|
+
- `gaa_*` (all Google Extended Access parameters), `eafs_enabled` (its companion flag)
|
|
108
110
|
|
|
109
111
|
## Advanced Usage
|
|
110
112
|
|
|
@@ -123,13 +125,17 @@ result = dict(normalizer)
|
|
|
123
125
|
print(result)
|
|
124
126
|
# {
|
|
125
127
|
# 'normalized_url': 'blog.example.com/page?a=1&b=2',
|
|
128
|
+
# 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
|
|
126
129
|
# 'parent_normalized_url': 'blog.example.com',
|
|
127
130
|
# 'root_normalized_url': 'example.com',
|
|
131
|
+
# 'registrable_domain': 'example.com',
|
|
128
132
|
# 'query_string': 'a=1&b=2',
|
|
129
133
|
# 'path': '/page',
|
|
130
134
|
# 'normalized_url_hash': '...',
|
|
131
135
|
# 'parent_normalized_url_hash': '...',
|
|
132
|
-
# 'root_normalized_url_hash': '...'
|
|
136
|
+
# 'root_normalized_url_hash': '...',
|
|
137
|
+
# 'registrable_domain_hash': '...',
|
|
138
|
+
# 'friendly_url_hash': '...'
|
|
133
139
|
# }
|
|
134
140
|
```
|
|
135
141
|
|
|
@@ -152,11 +158,12 @@ from tk_normalizer import TkNormalizer
|
|
|
152
158
|
normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
153
159
|
|
|
154
160
|
# Dict-like access to individual fields
|
|
155
|
-
print(normalizer["normalized_url"])
|
|
156
|
-
print(normalizer["parent_normalized_url"])
|
|
157
|
-
print(normalizer["root_normalized_url"])
|
|
158
|
-
print(normalizer["
|
|
159
|
-
print(normalizer["
|
|
161
|
+
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
162
|
+
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
163
|
+
print(normalizer["root_normalized_url"]) # example.com
|
|
164
|
+
print(normalizer["registrable_domain"]) # example.com
|
|
165
|
+
print(normalizer["query_string"]) # a=1
|
|
166
|
+
print(normalizer["path"]) # /path
|
|
160
167
|
|
|
161
168
|
# Iterate over available fields
|
|
162
169
|
for key in normalizer:
|
|
@@ -166,14 +173,29 @@ for key in normalizer:
|
|
|
166
173
|
print(normalizer.keys())
|
|
167
174
|
```
|
|
168
175
|
|
|
176
|
+
## Which URL Form to Use
|
|
177
|
+
|
|
178
|
+
| need | use |
|
|
179
|
+
|------|-----|
|
|
180
|
+
| comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
|
|
181
|
+
| showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
|
|
182
|
+
| linking out, fetching, redriving | the original URL you passed in |
|
|
183
|
+
| grouping by site | `registrable_domain` |
|
|
184
|
+
|
|
185
|
+
`normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
|
|
186
|
+
next to it wherever you need to reach the page again.
|
|
187
|
+
|
|
169
188
|
## Hashing
|
|
170
189
|
|
|
171
190
|
For efficient storage and comparison, SHA-256 hashes are computed for:
|
|
172
191
|
- The normalized URL
|
|
173
192
|
- The parent normal URL (domain without path)
|
|
174
|
-
- The root normal URL (
|
|
193
|
+
- The root normal URL (last two labels of the host)
|
|
194
|
+
- The registrable domain
|
|
195
|
+
- The friendly URL
|
|
175
196
|
|
|
176
|
-
This provides fixed-length representations suitable for database indexing.
|
|
197
|
+
This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
|
|
198
|
+
(`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
|
|
177
199
|
|
|
178
200
|
## Important Caveats
|
|
179
201
|
|
|
@@ -187,6 +209,11 @@ While this normalization process works well for most use cases, there are some l
|
|
|
187
209
|
|
|
188
210
|
4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
|
|
189
211
|
|
|
212
|
+
5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
|
|
213
|
+
multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
|
|
214
|
+
List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
|
|
215
|
+
`root_normalized_url` is unchanged so existing consumers do not shift.
|
|
216
|
+
|
|
190
217
|
## Development
|
|
191
218
|
|
|
192
219
|
### Setting Up Development Environment
|
|
@@ -249,10 +276,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
|
|
|
249
276
|
|
|
250
277
|
If you have questions or concerns reach out.
|
|
251
278
|
|
|
252
|
-
## Deploying to Snowflake -- UDF Update
|
|
253
|
-
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
254
|
-
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
255
|
-
|
|
256
279
|
## License
|
|
257
280
|
|
|
258
281
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import hashlib
|
|
1
2
|
import multiprocessing
|
|
2
3
|
import time
|
|
3
4
|
from multiprocessing import Process
|
|
@@ -271,11 +272,13 @@ def test_iteration() -> None:
|
|
|
271
272
|
"friendly_url",
|
|
272
273
|
"parent_normalized_url",
|
|
273
274
|
"root_normalized_url",
|
|
275
|
+
"registrable_domain",
|
|
274
276
|
"query_string",
|
|
275
277
|
"path",
|
|
276
278
|
"normalized_url_hash",
|
|
277
279
|
"parent_normalized_url_hash",
|
|
278
280
|
"root_normalized_url_hash",
|
|
281
|
+
"registrable_domain_hash",
|
|
279
282
|
"friendly_url_hash",
|
|
280
283
|
}
|
|
281
284
|
assert set(keys) == expected_keys
|
|
@@ -298,11 +301,13 @@ def test_keys_method() -> None:
|
|
|
298
301
|
"friendly_url",
|
|
299
302
|
"parent_normalized_url",
|
|
300
303
|
"root_normalized_url",
|
|
304
|
+
"registrable_domain",
|
|
301
305
|
"query_string",
|
|
302
306
|
"path",
|
|
303
307
|
"normalized_url_hash",
|
|
304
308
|
"parent_normalized_url_hash",
|
|
305
309
|
"root_normalized_url_hash",
|
|
310
|
+
"registrable_domain_hash",
|
|
306
311
|
"friendly_url_hash",
|
|
307
312
|
}
|
|
308
313
|
|
|
@@ -311,7 +316,7 @@ def test_keys_method() -> None:
|
|
|
311
316
|
# Keys should be a KeysView object
|
|
312
317
|
assert hasattr(keys, "__iter__")
|
|
313
318
|
assert hasattr(keys, "__len__")
|
|
314
|
-
assert len(keys) ==
|
|
319
|
+
assert len(keys) == 12
|
|
315
320
|
|
|
316
321
|
|
|
317
322
|
def test_friendly_url_preserves_casing() -> None:
|
|
@@ -361,6 +366,73 @@ def test_friendly_url_hash_exists() -> None:
|
|
|
361
366
|
assert len(hash_val) == 64
|
|
362
367
|
|
|
363
368
|
|
|
369
|
+
def test_google_extended_access_params_removed() -> None:
|
|
370
|
+
normalizer = TkNormalizer(
|
|
371
|
+
"https://www.wsj.com/market-data/quotes/TSN?gaa_at=eafs&gaa_n=AWEtsqcSfkA2CiTCf4PLiw7D4crACVBHW5gAe0L5ubd"
|
|
372
|
+
"&gaa_ts=691c0b39&gaa_sig=zdv02R-2BLJsEemEuMPdo0DfiS3WAkuRtu4smExyoSHaXWRIZ9HNY2-6hvPV4FM9hTLebiDiOXe8"
|
|
373
|
+
)
|
|
374
|
+
assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def test_eafs_enabled_removed() -> None:
|
|
378
|
+
normalizer = TkNormalizer("https://www.wsj.com/market-data/quotes/TSN?eafs_enabled=false")
|
|
379
|
+
assert normalizer["normalized_url"] == "wsj.com/market-data/quotes/tsn"
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def test_gaa_wildcard_covers_unseen_params() -> None:
|
|
383
|
+
normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_future_param=x&a=1")
|
|
384
|
+
assert normalizer["normalized_url"] == "wsj.com/articles/some-story?a=1"
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def test_gaa_wildcard_requires_the_underscore() -> None:
|
|
388
|
+
normalizer = TkNormalizer("https://example.com/page?gaa=1&gaal=2")
|
|
389
|
+
assert normalizer["normalized_url"] == "example.com/page?gaa=1&gaal=2"
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def test_extended_access_removal_keeps_other_params() -> None:
|
|
393
|
+
normalizer = TkNormalizer("https://www.wsj.com/articles/some-story?gaa_at=eafs&mod=hp_lead_pos1&page=2")
|
|
394
|
+
assert normalizer["normalized_url"] == "wsj.com/articles/some-story?mod=hp_lead_pos1&page=2"
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@pytest.mark.parametrize(
|
|
398
|
+
("url", "registrable", "root"),
|
|
399
|
+
[
|
|
400
|
+
# Multi-label public suffixes: the root is the suffix itself, the registrable domain is the site.
|
|
401
|
+
("https://www.abbvie.co.uk/our-science.html", "abbvie.co.uk", "co.uk"),
|
|
402
|
+
("https://news.bbc.co.uk/sport", "bbc.co.uk", "co.uk"),
|
|
403
|
+
("https://www.adelaidenow.com.au/news", "adelaidenow.com.au", "com.au"),
|
|
404
|
+
("https://www.forbes.com.mx/negocios", "forbes.com.mx", "com.mx"),
|
|
405
|
+
# Single-label suffixes agree with the root.
|
|
406
|
+
("https://finance.yahoo.com/quote/AAPL", "yahoo.com", "yahoo.com"),
|
|
407
|
+
("https://example.com/path", "example.com", "example.com"),
|
|
408
|
+
# Ports are not part of the domain.
|
|
409
|
+
("https://shop.example.co.uk:8080/cart", "example.co.uk", "co.uk:8080"),
|
|
410
|
+
],
|
|
411
|
+
)
|
|
412
|
+
def test_registrable_domain(url: str, registrable: str, root: str) -> None:
|
|
413
|
+
normalizer = TkNormalizer(url)
|
|
414
|
+
assert normalizer["registrable_domain"] == registrable
|
|
415
|
+
# root_normalized_url keeps its existing value so current consumers do not shift.
|
|
416
|
+
assert normalizer["root_normalized_url"] == root
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
@pytest.mark.parametrize(
|
|
420
|
+
("url", "registrable"),
|
|
421
|
+
[
|
|
422
|
+
# A bare public suffix has no label under it: fall back to the host.
|
|
423
|
+
("https://www.gov.uk/browse/tax", "gov.uk"),
|
|
424
|
+
("http://93.184.216.34/path", "93.184.216.34"),
|
|
425
|
+
],
|
|
426
|
+
)
|
|
427
|
+
def test_registrable_domain_falls_back_to_host(url: str, registrable: str) -> None:
|
|
428
|
+
assert TkNormalizer(url)["registrable_domain"] == registrable
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def test_registrable_domain_hash() -> None:
|
|
432
|
+
normalizer = TkNormalizer("https://news.bbc.co.uk/sport")
|
|
433
|
+
assert normalizer["registrable_domain_hash"] == hashlib.sha256(b"bbc.co.uk").hexdigest()
|
|
434
|
+
|
|
435
|
+
|
|
364
436
|
# These fixtures are at the bottom for readability of the upper tests
|
|
365
437
|
|
|
366
438
|
|
|
@@ -382,6 +454,10 @@ def happy_normal_fx() -> list[str]:
|
|
|
382
454
|
"nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh&srsltid=afmbooo0gkfvz9kwi-1s5atomy-8mtn6nwvrwub1nmxc9z9b8zse6w7h",
|
|
383
455
|
"nordstrom.com/browse/men/clothing/pants?filterbymaterial=mesh",
|
|
384
456
|
),
|
|
457
|
+
(
|
|
458
|
+
"https://www.marketwatch.com/investing/stock/aig/company-profile?countrycode=ch&gaa_at=eafs&gaa_ts=687a0c",
|
|
459
|
+
"marketwatch.com/investing/stock/aig/company-profile?countrycode=ch",
|
|
460
|
+
),
|
|
385
461
|
(
|
|
386
462
|
"https://www.libertymutual.com/insurance-resources/auto/main-types-of-car-insurance#:~:text=The%206%20main%20types%[…]that%20you,Uninsured%20Motorist",
|
|
387
463
|
"libertymutual.com/insurance-resources/auto/main-types-of-car-insurance",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|