tk-normalizer 1.3.0__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tk-normalizer
3
- Version: 1.3.0
3
+ Version: 1.4.0
4
4
  Summary: URL normalization library for consistent URL representation
5
5
  Author-email: Terakeet <engineering@terakeet.com>
6
6
  Maintainer-email: Terakeet <engineering@terakeet.com>
@@ -24,6 +24,7 @@ Classifier: Operating System :: OS Independent
24
24
  Requires-Python: >=3.11
25
25
  Description-Content-Type: text/markdown
26
26
  License-File: LICENSE
27
+ Requires-Dist: tldextract>=5.3.0
27
28
  Provides-Extra: dev
28
29
  Requires-Dist: pytest>=7.0.0; extra == "dev"
29
30
  Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
@@ -124,13 +125,17 @@ result = dict(normalizer)
124
125
  print(result)
125
126
  # {
126
127
  # 'normalized_url': 'blog.example.com/page?a=1&b=2',
128
+ # 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
127
129
  # 'parent_normalized_url': 'blog.example.com',
128
130
  # 'root_normalized_url': 'example.com',
131
+ # 'registrable_domain': 'example.com',
129
132
  # 'query_string': 'a=1&b=2',
130
133
  # 'path': '/page',
131
134
  # 'normalized_url_hash': '...',
132
135
  # 'parent_normalized_url_hash': '...',
133
- # 'root_normalized_url_hash': '...'
136
+ # 'root_normalized_url_hash': '...',
137
+ # 'registrable_domain_hash': '...',
138
+ # 'friendly_url_hash': '...'
134
139
  # }
135
140
  ```
136
141
 
@@ -156,6 +161,7 @@ normalizer = TkNormalizer("https://blog.example.com/path?a=1")
156
161
  print(normalizer["normalized_url"]) # blog.example.com/path?a=1
157
162
  print(normalizer["parent_normalized_url"]) # blog.example.com
158
163
  print(normalizer["root_normalized_url"]) # example.com
164
+ print(normalizer["registrable_domain"]) # example.com
159
165
  print(normalizer["query_string"]) # a=1
160
166
  print(normalizer["path"]) # /path
161
167
 
@@ -167,14 +173,29 @@ for key in normalizer:
167
173
  print(normalizer.keys())
168
174
  ```
169
175
 
176
+ ## Which URL Form to Use
177
+
178
+ | need | use |
179
+ |------|-----|
180
+ | comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
181
+ | showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
182
+ | linking out, fetching, redriving | the original URL you passed in |
183
+ | grouping by site | `registrable_domain` |
184
+
185
+ `normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
186
+ next to it wherever you need to reach the page again.
187
+
170
188
  ## Hashing
171
189
 
172
190
  For efficient storage and comparison, SHA-256 hashes are computed for:
173
191
  - The normalized URL
174
192
  - The parent normal URL (domain without path)
175
- - The root normal URL (root domain without subdomains)
193
+ - The root normal URL (last two labels of the host)
194
+ - The registrable domain
195
+ - The friendly URL
176
196
 
177
- This provides fixed-length representations suitable for database indexing.
197
+ This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
198
+ (`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
178
199
 
179
200
  ## Important Caveats
180
201
 
@@ -188,6 +209,11 @@ While this normalization process works well for most use cases, there are some l
188
209
 
189
210
  4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
190
211
 
212
+ 5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
213
+ multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
214
+ List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
215
+ `root_normalized_url` is unchanged so existing consumers do not shift.
216
+
191
217
  ## Development
192
218
 
193
219
  ### Setting Up Development Environment
@@ -250,10 +276,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
250
276
 
251
277
  If you have questions or concerns reach out.
252
278
 
253
- ## Deploying to Snowflake -- UDF Update
254
- The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
255
- Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
256
-
257
279
  ## License
258
280
 
259
281
  This project is licensed under the MIT License - see the LICENSE file for details.
@@ -88,13 +88,17 @@ result = dict(normalizer)
88
88
  print(result)
89
89
  # {
90
90
  # 'normalized_url': 'blog.example.com/page?a=1&b=2',
91
+ # 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
91
92
  # 'parent_normalized_url': 'blog.example.com',
92
93
  # 'root_normalized_url': 'example.com',
94
+ # 'registrable_domain': 'example.com',
93
95
  # 'query_string': 'a=1&b=2',
94
96
  # 'path': '/page',
95
97
  # 'normalized_url_hash': '...',
96
98
  # 'parent_normalized_url_hash': '...',
97
- # 'root_normalized_url_hash': '...'
99
+ # 'root_normalized_url_hash': '...',
100
+ # 'registrable_domain_hash': '...',
101
+ # 'friendly_url_hash': '...'
98
102
  # }
99
103
  ```
100
104
 
@@ -120,6 +124,7 @@ normalizer = TkNormalizer("https://blog.example.com/path?a=1")
120
124
  print(normalizer["normalized_url"]) # blog.example.com/path?a=1
121
125
  print(normalizer["parent_normalized_url"]) # blog.example.com
122
126
  print(normalizer["root_normalized_url"]) # example.com
127
+ print(normalizer["registrable_domain"]) # example.com
123
128
  print(normalizer["query_string"]) # a=1
124
129
  print(normalizer["path"]) # /path
125
130
 
@@ -131,14 +136,29 @@ for key in normalizer:
131
136
  print(normalizer.keys())
132
137
  ```
133
138
 
139
+ ## Which URL Form to Use
140
+
141
+ | need | use |
142
+ |------|-----|
143
+ | comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
144
+ | showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
145
+ | linking out, fetching, redriving | the original URL you passed in |
146
+ | grouping by site | `registrable_domain` |
147
+
148
+ `normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
149
+ next to it wherever you need to reach the page again.
150
+
134
151
  ## Hashing
135
152
 
136
153
  For efficient storage and comparison, SHA-256 hashes are computed for:
137
154
  - The normalized URL
138
155
  - The parent normal URL (domain without path)
139
- - The root normal URL (root domain without subdomains)
156
+ - The root normal URL (last two labels of the host)
157
+ - The registrable domain
158
+ - The friendly URL
140
159
 
141
- This provides fixed-length representations suitable for database indexing.
160
+ This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
161
+ (`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
142
162
 
143
163
  ## Important Caveats
144
164
 
@@ -152,6 +172,11 @@ While this normalization process works well for most use cases, there are some l
152
172
 
153
173
  4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
154
174
 
175
+ 5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
176
+ multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
177
+ List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
178
+ `root_normalized_url` is unchanged so existing consumers do not shift.
179
+
155
180
  ## Development
156
181
 
157
182
  ### Setting Up Development Environment
@@ -214,10 +239,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
214
239
 
215
240
  If you have questions or concerns reach out.
216
241
 
217
- ## Deploying to Snowflake -- UDF Update
218
- The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
219
- Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
220
-
221
242
  ## License
222
243
 
223
244
  This project is licensed under the MIT License - see the LICENSE file for details.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tk-normalizer"
7
- version = "1.3.0"
7
+ version = "1.4.0"
8
8
  description = "URL normalization library for consistent URL representation"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -29,6 +29,9 @@ classifiers = [
29
29
  "Topic :: Text Processing :: Filters",
30
30
  "Operating System :: OS Independent",
31
31
  ]
32
+ dependencies = [
33
+ "tldextract>=5.3.0",
34
+ ]
32
35
 
33
36
  [project.urls]
34
37
  Homepage = "https://github.com/terakeet/tk-normalizer"
@@ -8,5 +8,5 @@ query parameters, and more.
8
8
 
9
9
  from .normalizer import InvalidUrlException, TkNormalizer
10
10
 
11
- __version__ = "1.2.0"
11
+ __version__ = "1.4.0"
12
12
  __all__ = ["TkNormalizer", "InvalidUrlException"]
@@ -5,6 +5,12 @@ import re
5
5
  from collections.abc import Iterator, KeysView
6
6
  from urllib.parse import ParseResult, parse_qsl, unquote, urlencode, urlparse
7
7
 
8
+ import tldextract
9
+
10
+ # The bundled Public Suffix List snapshot (ICANN section): no network fetch and no cache writes,
11
+ # so it behaves the same in a Snowflake UDF, a worker pool, or a read-only filesystem.
12
+ _PUBLIC_SUFFIXES = tldextract.TLDExtract(suffix_list_urls=(), cache_dir=None)
13
+
8
14
  # Uses https://gist.github.com/Integralist/edcfb88c925658a13fc3e51f581fe4bc as a starting point
9
15
  # Modified for more current rules regarding host/domain/tld naming.
10
16
 
@@ -91,10 +97,12 @@ class TkNormalizer:
91
97
  self.normalized_url: str
92
98
  self.parent_normalized_url: str
93
99
  self.root_normalized_url: str
100
+ self.registrable_domain: str
94
101
  self.query_string: str
95
102
  self.path: str
96
103
  self.friendly_url: str
97
104
  self.normalized_url, self.parent_normalized_url, self.root_normalized_url = self.process_url(url)
105
+ self.registrable_domain = self.get_registrable_domain(self.parent_normalized_url)
98
106
  self.friendly_url = self.build_friendly_url()
99
107
  self.url_hashes: dict[str, str] = self.compute_hashes()
100
108
  except InvalidUrlException as e:
@@ -219,6 +227,14 @@ class TkNormalizer:
219
227
  def get_parent_normalized_url(netloc: str) -> str:
220
228
  return netloc
221
229
 
230
+ @staticmethod
231
+ def get_registrable_domain(netloc: str) -> str:
232
+ """The host one label below its public suffix: bbc.co.uk for news.bbc.co.uk, where the root is co.uk.
233
+
234
+ Falls back to the host when there is no such label (an IP address, or a bare suffix like gov.uk).
235
+ """
236
+ return _PUBLIC_SUFFIXES(netloc).top_domain_under_public_suffix or netloc
237
+
222
238
  @staticmethod
223
239
  def get_root_normalized_url(netloc: str) -> str:
224
240
  return ".".join(netloc.split(".")[-2:])
@@ -245,6 +261,7 @@ class TkNormalizer:
245
261
  "normalized_url_hash": sha256_hash(self.normalized_url),
246
262
  "parent_normalized_url_hash": sha256_hash(self.parent_normalized_url),
247
263
  "root_normalized_url_hash": sha256_hash(self.root_normalized_url),
264
+ "registrable_domain_hash": sha256_hash(self.registrable_domain),
248
265
  "friendly_url_hash": sha256_hash(self.friendly_url),
249
266
  }
250
267
 
@@ -254,6 +271,7 @@ class TkNormalizer:
254
271
  "friendly_url": self.friendly_url,
255
272
  "parent_normalized_url": self.parent_normalized_url,
256
273
  "root_normalized_url": self.root_normalized_url,
274
+ "registrable_domain": self.registrable_domain,
257
275
  "query_string": self.query_string,
258
276
  "path": self.path,
259
277
  **self.url_hashes,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tk-normalizer
3
- Version: 1.3.0
3
+ Version: 1.4.0
4
4
  Summary: URL normalization library for consistent URL representation
5
5
  Author-email: Terakeet <engineering@terakeet.com>
6
6
  Maintainer-email: Terakeet <engineering@terakeet.com>
@@ -24,6 +24,7 @@ Classifier: Operating System :: OS Independent
24
24
  Requires-Python: >=3.11
25
25
  Description-Content-Type: text/markdown
26
26
  License-File: LICENSE
27
+ Requires-Dist: tldextract>=5.3.0
27
28
  Provides-Extra: dev
28
29
  Requires-Dist: pytest>=7.0.0; extra == "dev"
29
30
  Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
@@ -124,13 +125,17 @@ result = dict(normalizer)
124
125
  print(result)
125
126
  # {
126
127
  # 'normalized_url': 'blog.example.com/page?a=1&b=2',
128
+ # 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
127
129
  # 'parent_normalized_url': 'blog.example.com',
128
130
  # 'root_normalized_url': 'example.com',
131
+ # 'registrable_domain': 'example.com',
129
132
  # 'query_string': 'a=1&b=2',
130
133
  # 'path': '/page',
131
134
  # 'normalized_url_hash': '...',
132
135
  # 'parent_normalized_url_hash': '...',
133
- # 'root_normalized_url_hash': '...'
136
+ # 'root_normalized_url_hash': '...',
137
+ # 'registrable_domain_hash': '...',
138
+ # 'friendly_url_hash': '...'
134
139
  # }
135
140
  ```
136
141
 
@@ -156,6 +161,7 @@ normalizer = TkNormalizer("https://blog.example.com/path?a=1")
156
161
  print(normalizer["normalized_url"]) # blog.example.com/path?a=1
157
162
  print(normalizer["parent_normalized_url"]) # blog.example.com
158
163
  print(normalizer["root_normalized_url"]) # example.com
164
+ print(normalizer["registrable_domain"]) # example.com
159
165
  print(normalizer["query_string"]) # a=1
160
166
  print(normalizer["path"]) # /path
161
167
 
@@ -167,14 +173,29 @@ for key in normalizer:
167
173
  print(normalizer.keys())
168
174
  ```
169
175
 
176
+ ## Which URL Form to Use
177
+
178
+ | need | use |
179
+ |------|-----|
180
+ | comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
181
+ | showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
182
+ | linking out, fetching, redriving | the original URL you passed in |
183
+ | grouping by site | `registrable_domain` |
184
+
185
+ `normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
186
+ next to it wherever you need to reach the page again.
187
+
170
188
  ## Hashing
171
189
 
172
190
  For efficient storage and comparison, SHA-256 hashes are computed for:
173
191
  - The normalized URL
174
192
  - The parent normal URL (domain without path)
175
- - The root normal URL (root domain without subdomains)
193
+ - The root normal URL (last two labels of the host)
194
+ - The registrable domain
195
+ - The friendly URL
176
196
 
177
- This provides fixed-length representations suitable for database indexing.
197
+ This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
198
+ (`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
178
199
 
179
200
  ## Important Caveats
180
201
 
@@ -188,6 +209,11 @@ While this normalization process works well for most use cases, there are some l
188
209
 
189
210
  4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
190
211
 
212
+ 5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
213
+ multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
214
+ List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
215
+ `root_normalized_url` is unchanged so existing consumers do not shift.
216
+
191
217
  ## Development
192
218
 
193
219
  ### Setting Up Development Environment
@@ -250,10 +276,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
250
276
 
251
277
  If you have questions or concerns reach out.
252
278
 
253
- ## Deploying to Snowflake -- UDF Update
254
- The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
255
- Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
256
-
257
279
  ## License
258
280
 
259
281
  This project is licensed under the MIT License - see the LICENSE file for details.
@@ -1,3 +1,4 @@
1
+ tldextract>=5.3.0
1
2
 
2
3
  [dev]
3
4
  pytest>=7.0.0
@@ -1,3 +1,4 @@
1
+ import hashlib
1
2
  import multiprocessing
2
3
  import time
3
4
  from multiprocessing import Process
@@ -271,11 +272,13 @@ def test_iteration() -> None:
271
272
  "friendly_url",
272
273
  "parent_normalized_url",
273
274
  "root_normalized_url",
275
+ "registrable_domain",
274
276
  "query_string",
275
277
  "path",
276
278
  "normalized_url_hash",
277
279
  "parent_normalized_url_hash",
278
280
  "root_normalized_url_hash",
281
+ "registrable_domain_hash",
279
282
  "friendly_url_hash",
280
283
  }
281
284
  assert set(keys) == expected_keys
@@ -298,11 +301,13 @@ def test_keys_method() -> None:
298
301
  "friendly_url",
299
302
  "parent_normalized_url",
300
303
  "root_normalized_url",
304
+ "registrable_domain",
301
305
  "query_string",
302
306
  "path",
303
307
  "normalized_url_hash",
304
308
  "parent_normalized_url_hash",
305
309
  "root_normalized_url_hash",
310
+ "registrable_domain_hash",
306
311
  "friendly_url_hash",
307
312
  }
308
313
 
@@ -311,7 +316,7 @@ def test_keys_method() -> None:
311
316
  # Keys should be a KeysView object
312
317
  assert hasattr(keys, "__iter__")
313
318
  assert hasattr(keys, "__len__")
314
- assert len(keys) == 10
319
+ assert len(keys) == 12
315
320
 
316
321
 
317
322
  def test_friendly_url_preserves_casing() -> None:
@@ -389,6 +394,45 @@ def test_extended_access_removal_keeps_other_params() -> None:
389
394
  assert normalizer["normalized_url"] == "wsj.com/articles/some-story?mod=hp_lead_pos1&page=2"
390
395
 
391
396
 
397
+ @pytest.mark.parametrize(
398
+ ("url", "registrable", "root"),
399
+ [
400
+ # Multi-label public suffixes: the root is the suffix itself, the registrable domain is the site.
401
+ ("https://www.abbvie.co.uk/our-science.html", "abbvie.co.uk", "co.uk"),
402
+ ("https://news.bbc.co.uk/sport", "bbc.co.uk", "co.uk"),
403
+ ("https://www.adelaidenow.com.au/news", "adelaidenow.com.au", "com.au"),
404
+ ("https://www.forbes.com.mx/negocios", "forbes.com.mx", "com.mx"),
405
+ # Single-label suffixes agree with the root.
406
+ ("https://finance.yahoo.com/quote/AAPL", "yahoo.com", "yahoo.com"),
407
+ ("https://example.com/path", "example.com", "example.com"),
408
+ # Ports are not part of the domain.
409
+ ("https://shop.example.co.uk:8080/cart", "example.co.uk", "co.uk:8080"),
410
+ ],
411
+ )
412
+ def test_registrable_domain(url: str, registrable: str, root: str) -> None:
413
+ normalizer = TkNormalizer(url)
414
+ assert normalizer["registrable_domain"] == registrable
415
+ # root_normalized_url keeps its existing value so current consumers do not shift.
416
+ assert normalizer["root_normalized_url"] == root
417
+
418
+
419
+ @pytest.mark.parametrize(
420
+ ("url", "registrable"),
421
+ [
422
+ # A bare public suffix has no label under it: fall back to the host.
423
+ ("https://www.gov.uk/browse/tax", "gov.uk"),
424
+ ("http://93.184.216.34/path", "93.184.216.34"),
425
+ ],
426
+ )
427
+ def test_registrable_domain_falls_back_to_host(url: str, registrable: str) -> None:
428
+ assert TkNormalizer(url)["registrable_domain"] == registrable
429
+
430
+
431
+ def test_registrable_domain_hash() -> None:
432
+ normalizer = TkNormalizer("https://news.bbc.co.uk/sport")
433
+ assert normalizer["registrable_domain_hash"] == hashlib.sha256(b"bbc.co.uk").hexdigest()
434
+
435
+
392
436
  # These fixtures are at the bottom for readability of the upper tests
393
437
 
394
438
 
File without changes
File without changes