tk-normalizer 1.3.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tk_normalizer-1.3.0/src/tk_normalizer.egg-info → tk_normalizer-1.4.0}/PKG-INFO +30 -8
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/README.md +28 -7
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/pyproject.toml +4 -1
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/src/tk_normalizer/__init__.py +1 -1
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/src/tk_normalizer/normalizer.py +18 -0
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0/src/tk_normalizer.egg-info}/PKG-INFO +30 -8
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/requires.txt +1 -0
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/tests/test_normalizer.py +45 -1
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/LICENSE +0 -0
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/setup.cfg +0 -0
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/SOURCES.txt +0 -0
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/dependency_links.txt +0 -0
- {tk_normalizer-1.3.0 → tk_normalizer-1.4.0}/src/tk_normalizer.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -24,6 +24,7 @@ Classifier: Operating System :: OS Independent
|
|
|
24
24
|
Requires-Python: >=3.11
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
|
+
Requires-Dist: tldextract>=5.3.0
|
|
27
28
|
Provides-Extra: dev
|
|
28
29
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
29
30
|
Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
|
|
@@ -124,13 +125,17 @@ result = dict(normalizer)
|
|
|
124
125
|
print(result)
|
|
125
126
|
# {
|
|
126
127
|
# 'normalized_url': 'blog.example.com/page?a=1&b=2',
|
|
128
|
+
# 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
|
|
127
129
|
# 'parent_normalized_url': 'blog.example.com',
|
|
128
130
|
# 'root_normalized_url': 'example.com',
|
|
131
|
+
# 'registrable_domain': 'example.com',
|
|
129
132
|
# 'query_string': 'a=1&b=2',
|
|
130
133
|
# 'path': '/page',
|
|
131
134
|
# 'normalized_url_hash': '...',
|
|
132
135
|
# 'parent_normalized_url_hash': '...',
|
|
133
|
-
# 'root_normalized_url_hash': '...'
|
|
136
|
+
# 'root_normalized_url_hash': '...',
|
|
137
|
+
# 'registrable_domain_hash': '...',
|
|
138
|
+
# 'friendly_url_hash': '...'
|
|
134
139
|
# }
|
|
135
140
|
```
|
|
136
141
|
|
|
@@ -156,6 +161,7 @@ normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
|
156
161
|
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
157
162
|
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
158
163
|
print(normalizer["root_normalized_url"]) # example.com
|
|
164
|
+
print(normalizer["registrable_domain"]) # example.com
|
|
159
165
|
print(normalizer["query_string"]) # a=1
|
|
160
166
|
print(normalizer["path"]) # /path
|
|
161
167
|
|
|
@@ -167,14 +173,29 @@ for key in normalizer:
|
|
|
167
173
|
print(normalizer.keys())
|
|
168
174
|
```
|
|
169
175
|
|
|
176
|
+
## Which URL Form to Use
|
|
177
|
+
|
|
178
|
+
| need | use |
|
|
179
|
+
|------|-----|
|
|
180
|
+
| comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
|
|
181
|
+
| showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
|
|
182
|
+
| linking out, fetching, redriving | the original URL you passed in |
|
|
183
|
+
| grouping by site | `registrable_domain` |
|
|
184
|
+
|
|
185
|
+
`normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
|
|
186
|
+
next to it wherever you need to reach the page again.
|
|
187
|
+
|
|
170
188
|
## Hashing
|
|
171
189
|
|
|
172
190
|
For efficient storage and comparison, SHA-256 hashes are computed for:
|
|
173
191
|
- The normalized URL
|
|
174
192
|
- The parent normal URL (domain without path)
|
|
175
|
-
- The root normal URL (
|
|
193
|
+
- The root normal URL (last two labels of the host)
|
|
194
|
+
- The registrable domain
|
|
195
|
+
- The friendly URL
|
|
176
196
|
|
|
177
|
-
This provides fixed-length representations suitable for database indexing.
|
|
197
|
+
This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
|
|
198
|
+
(`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
|
|
178
199
|
|
|
179
200
|
## Important Caveats
|
|
180
201
|
|
|
@@ -188,6 +209,11 @@ While this normalization process works well for most use cases, there are some l
|
|
|
188
209
|
|
|
189
210
|
4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
|
|
190
211
|
|
|
212
|
+
5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
|
|
213
|
+
multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
|
|
214
|
+
List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
|
|
215
|
+
`root_normalized_url` is unchanged so existing consumers do not shift.
|
|
216
|
+
|
|
191
217
|
## Development
|
|
192
218
|
|
|
193
219
|
### Setting Up Development Environment
|
|
@@ -250,10 +276,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
|
|
|
250
276
|
|
|
251
277
|
If you have questions or concerns reach out.
|
|
252
278
|
|
|
253
|
-
## Deploying to Snowflake -- UDF Update
|
|
254
|
-
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
255
|
-
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
256
|
-
|
|
257
279
|
## License
|
|
258
280
|
|
|
259
281
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -88,13 +88,17 @@ result = dict(normalizer)
|
|
|
88
88
|
print(result)
|
|
89
89
|
# {
|
|
90
90
|
# 'normalized_url': 'blog.example.com/page?a=1&b=2',
|
|
91
|
+
# 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
|
|
91
92
|
# 'parent_normalized_url': 'blog.example.com',
|
|
92
93
|
# 'root_normalized_url': 'example.com',
|
|
94
|
+
# 'registrable_domain': 'example.com',
|
|
93
95
|
# 'query_string': 'a=1&b=2',
|
|
94
96
|
# 'path': '/page',
|
|
95
97
|
# 'normalized_url_hash': '...',
|
|
96
98
|
# 'parent_normalized_url_hash': '...',
|
|
97
|
-
# 'root_normalized_url_hash': '...'
|
|
99
|
+
# 'root_normalized_url_hash': '...',
|
|
100
|
+
# 'registrable_domain_hash': '...',
|
|
101
|
+
# 'friendly_url_hash': '...'
|
|
98
102
|
# }
|
|
99
103
|
```
|
|
100
104
|
|
|
@@ -120,6 +124,7 @@ normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
|
120
124
|
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
121
125
|
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
122
126
|
print(normalizer["root_normalized_url"]) # example.com
|
|
127
|
+
print(normalizer["registrable_domain"]) # example.com
|
|
123
128
|
print(normalizer["query_string"]) # a=1
|
|
124
129
|
print(normalizer["path"]) # /path
|
|
125
130
|
|
|
@@ -131,14 +136,29 @@ for key in normalizer:
|
|
|
131
136
|
print(normalizer.keys())
|
|
132
137
|
```
|
|
133
138
|
|
|
139
|
+
## Which URL Form to Use
|
|
140
|
+
|
|
141
|
+
| need | use |
|
|
142
|
+
|------|-----|
|
|
143
|
+
| comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
|
|
144
|
+
| showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
|
|
145
|
+
| linking out, fetching, redriving | the original URL you passed in |
|
|
146
|
+
| grouping by site | `registrable_domain` |
|
|
147
|
+
|
|
148
|
+
`normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
|
|
149
|
+
next to it wherever you need to reach the page again.
|
|
150
|
+
|
|
134
151
|
## Hashing
|
|
135
152
|
|
|
136
153
|
For efficient storage and comparison, SHA-256 hashes are computed for:
|
|
137
154
|
- The normalized URL
|
|
138
155
|
- The parent normal URL (domain without path)
|
|
139
|
-
- The root normal URL (
|
|
156
|
+
- The root normal URL (last two labels of the host)
|
|
157
|
+
- The registrable domain
|
|
158
|
+
- The friendly URL
|
|
140
159
|
|
|
141
|
-
This provides fixed-length representations suitable for database indexing.
|
|
160
|
+
This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
|
|
161
|
+
(`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
|
|
142
162
|
|
|
143
163
|
## Important Caveats
|
|
144
164
|
|
|
@@ -152,6 +172,11 @@ While this normalization process works well for most use cases, there are some l
|
|
|
152
172
|
|
|
153
173
|
4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
|
|
154
174
|
|
|
175
|
+
5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
|
|
176
|
+
multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
|
|
177
|
+
List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
|
|
178
|
+
`root_normalized_url` is unchanged so existing consumers do not shift.
|
|
179
|
+
|
|
155
180
|
## Development
|
|
156
181
|
|
|
157
182
|
### Setting Up Development Environment
|
|
@@ -214,10 +239,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
|
|
|
214
239
|
|
|
215
240
|
If you have questions or concerns reach out.
|
|
216
241
|
|
|
217
|
-
## Deploying to Snowflake -- UDF Update
|
|
218
|
-
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
219
|
-
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
220
|
-
|
|
221
242
|
## License
|
|
222
243
|
|
|
223
244
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tk-normalizer"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.4.0"
|
|
8
8
|
description = "URL normalization library for consistent URL representation"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -29,6 +29,9 @@ classifiers = [
|
|
|
29
29
|
"Topic :: Text Processing :: Filters",
|
|
30
30
|
"Operating System :: OS Independent",
|
|
31
31
|
]
|
|
32
|
+
dependencies = [
|
|
33
|
+
"tldextract>=5.3.0",
|
|
34
|
+
]
|
|
32
35
|
|
|
33
36
|
[project.urls]
|
|
34
37
|
Homepage = "https://github.com/terakeet/tk-normalizer"
|
|
@@ -5,6 +5,12 @@ import re
|
|
|
5
5
|
from collections.abc import Iterator, KeysView
|
|
6
6
|
from urllib.parse import ParseResult, parse_qsl, unquote, urlencode, urlparse
|
|
7
7
|
|
|
8
|
+
import tldextract
|
|
9
|
+
|
|
10
|
+
# The bundled Public Suffix List snapshot (ICANN section): no network fetch and no cache writes,
|
|
11
|
+
# so it behaves the same in a Snowflake UDF, a worker pool, or a read-only filesystem.
|
|
12
|
+
_PUBLIC_SUFFIXES = tldextract.TLDExtract(suffix_list_urls=(), cache_dir=None)
|
|
13
|
+
|
|
8
14
|
# Uses https://gist.github.com/Integralist/edcfb88c925658a13fc3e51f581fe4bc as a starting point
|
|
9
15
|
# Modified for more current rules regarding host/domain/tld naming.
|
|
10
16
|
|
|
@@ -91,10 +97,12 @@ class TkNormalizer:
|
|
|
91
97
|
self.normalized_url: str
|
|
92
98
|
self.parent_normalized_url: str
|
|
93
99
|
self.root_normalized_url: str
|
|
100
|
+
self.registrable_domain: str
|
|
94
101
|
self.query_string: str
|
|
95
102
|
self.path: str
|
|
96
103
|
self.friendly_url: str
|
|
97
104
|
self.normalized_url, self.parent_normalized_url, self.root_normalized_url = self.process_url(url)
|
|
105
|
+
self.registrable_domain = self.get_registrable_domain(self.parent_normalized_url)
|
|
98
106
|
self.friendly_url = self.build_friendly_url()
|
|
99
107
|
self.url_hashes: dict[str, str] = self.compute_hashes()
|
|
100
108
|
except InvalidUrlException as e:
|
|
@@ -219,6 +227,14 @@ class TkNormalizer:
|
|
|
219
227
|
def get_parent_normalized_url(netloc: str) -> str:
|
|
220
228
|
return netloc
|
|
221
229
|
|
|
230
|
+
@staticmethod
|
|
231
|
+
def get_registrable_domain(netloc: str) -> str:
|
|
232
|
+
"""The host one label below its public suffix: bbc.co.uk for news.bbc.co.uk, where the root is co.uk.
|
|
233
|
+
|
|
234
|
+
Falls back to the host when there is no such label (an IP address, or a bare suffix like gov.uk).
|
|
235
|
+
"""
|
|
236
|
+
return _PUBLIC_SUFFIXES(netloc).top_domain_under_public_suffix or netloc
|
|
237
|
+
|
|
222
238
|
@staticmethod
|
|
223
239
|
def get_root_normalized_url(netloc: str) -> str:
|
|
224
240
|
return ".".join(netloc.split(".")[-2:])
|
|
@@ -245,6 +261,7 @@ class TkNormalizer:
|
|
|
245
261
|
"normalized_url_hash": sha256_hash(self.normalized_url),
|
|
246
262
|
"parent_normalized_url_hash": sha256_hash(self.parent_normalized_url),
|
|
247
263
|
"root_normalized_url_hash": sha256_hash(self.root_normalized_url),
|
|
264
|
+
"registrable_domain_hash": sha256_hash(self.registrable_domain),
|
|
248
265
|
"friendly_url_hash": sha256_hash(self.friendly_url),
|
|
249
266
|
}
|
|
250
267
|
|
|
@@ -254,6 +271,7 @@ class TkNormalizer:
|
|
|
254
271
|
"friendly_url": self.friendly_url,
|
|
255
272
|
"parent_normalized_url": self.parent_normalized_url,
|
|
256
273
|
"root_normalized_url": self.root_normalized_url,
|
|
274
|
+
"registrable_domain": self.registrable_domain,
|
|
257
275
|
"query_string": self.query_string,
|
|
258
276
|
"path": self.path,
|
|
259
277
|
**self.url_hashes,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tk-normalizer
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: URL normalization library for consistent URL representation
|
|
5
5
|
Author-email: Terakeet <engineering@terakeet.com>
|
|
6
6
|
Maintainer-email: Terakeet <engineering@terakeet.com>
|
|
@@ -24,6 +24,7 @@ Classifier: Operating System :: OS Independent
|
|
|
24
24
|
Requires-Python: >=3.11
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
|
+
Requires-Dist: tldextract>=5.3.0
|
|
27
28
|
Provides-Extra: dev
|
|
28
29
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
29
30
|
Requires-Dist: pytest-mock>=3.0.0; extra == "dev"
|
|
@@ -124,13 +125,17 @@ result = dict(normalizer)
|
|
|
124
125
|
print(result)
|
|
125
126
|
# {
|
|
126
127
|
# 'normalized_url': 'blog.example.com/page?a=1&b=2',
|
|
128
|
+
# 'friendly_url': 'http://blog.example.com/page?a=1&b=2',
|
|
127
129
|
# 'parent_normalized_url': 'blog.example.com',
|
|
128
130
|
# 'root_normalized_url': 'example.com',
|
|
131
|
+
# 'registrable_domain': 'example.com',
|
|
129
132
|
# 'query_string': 'a=1&b=2',
|
|
130
133
|
# 'path': '/page',
|
|
131
134
|
# 'normalized_url_hash': '...',
|
|
132
135
|
# 'parent_normalized_url_hash': '...',
|
|
133
|
-
# 'root_normalized_url_hash': '...'
|
|
136
|
+
# 'root_normalized_url_hash': '...',
|
|
137
|
+
# 'registrable_domain_hash': '...',
|
|
138
|
+
# 'friendly_url_hash': '...'
|
|
134
139
|
# }
|
|
135
140
|
```
|
|
136
141
|
|
|
@@ -156,6 +161,7 @@ normalizer = TkNormalizer("https://blog.example.com/path?a=1")
|
|
|
156
161
|
print(normalizer["normalized_url"]) # blog.example.com/path?a=1
|
|
157
162
|
print(normalizer["parent_normalized_url"]) # blog.example.com
|
|
158
163
|
print(normalizer["root_normalized_url"]) # example.com
|
|
164
|
+
print(normalizer["registrable_domain"]) # example.com
|
|
159
165
|
print(normalizer["query_string"]) # a=1
|
|
160
166
|
print(normalizer["path"]) # /path
|
|
161
167
|
|
|
@@ -167,14 +173,29 @@ for key in normalizer:
|
|
|
167
173
|
print(normalizer.keys())
|
|
168
174
|
```
|
|
169
175
|
|
|
176
|
+
## Which URL Form to Use
|
|
177
|
+
|
|
178
|
+
| need | use |
|
|
179
|
+
|------|-----|
|
|
180
|
+
| comparing, joining, deduping, counting | `normalized_url` (or `normalized_url_hash`) |
|
|
181
|
+
| showing a URL to a person | `friendly_url` (original casing, tracking params removed) |
|
|
182
|
+
| linking out, fetching, redriving | the original URL you passed in |
|
|
183
|
+
| grouping by site | `registrable_domain` |
|
|
184
|
+
|
|
185
|
+
`normalized_url` is lowercased on purpose, so case variants of the same page compare equal. Keep the original URL
|
|
186
|
+
next to it wherever you need to reach the page again.
|
|
187
|
+
|
|
170
188
|
## Hashing
|
|
171
189
|
|
|
172
190
|
For efficient storage and comparison, SHA-256 hashes are computed for:
|
|
173
191
|
- The normalized URL
|
|
174
192
|
- The parent normal URL (domain without path)
|
|
175
|
-
- The root normal URL (
|
|
193
|
+
- The root normal URL (last two labels of the host)
|
|
194
|
+
- The registrable domain
|
|
195
|
+
- The friendly URL
|
|
176
196
|
|
|
177
|
-
This provides fixed-length representations suitable for database indexing.
|
|
197
|
+
This provides fixed-length representations suitable for database indexing. Storing the 32-byte digest
|
|
198
|
+
(`bytes.fromhex(...)`) instead of the 64-character hex string halves the key size.
|
|
178
199
|
|
|
179
200
|
## Important Caveats
|
|
180
201
|
|
|
@@ -188,6 +209,11 @@ While this normalization process works well for most use cases, there are some l
|
|
|
188
209
|
|
|
189
210
|
4. **Fragment removal**: URL fragments (#anchors) are removed, which may affect single-page applications.
|
|
190
211
|
|
|
212
|
+
5. **Root vs registrable domain**: `root_normalized_url` is the last two labels of the host, so it is wrong under
|
|
213
|
+
multi-label public suffixes: `news.bbc.co.uk` gets the root `co.uk`. `registrable_domain` uses the Public Suffix
|
|
214
|
+
List (ICANN section, bundled with `tldextract`) and gives `bbc.co.uk`. Group sites by `registrable_domain`.
|
|
215
|
+
`root_normalized_url` is unchanged so existing consumers do not shift.
|
|
216
|
+
|
|
191
217
|
## Development
|
|
192
218
|
|
|
193
219
|
### Setting Up Development Environment
|
|
@@ -250,10 +276,6 @@ We have a [workflow](.github/workflows/deploy_to_pypi.yml) set up to deploy to o
|
|
|
250
276
|
|
|
251
277
|
If you have questions or concerns reach out.
|
|
252
278
|
|
|
253
|
-
## Deploying to Snowflake -- UDF Update
|
|
254
|
-
The normalizer implementaiton in snowflake is defined at [TERAKEET.COMMON.NORMALIZE_URL](https://app.snowflake.com/rbvgkpe/fhb30323/#/data/databases/TERAKEET/schemas/COMMON/user-function/NORMALIZE_URL(VARCHAR)).
|
|
255
|
-
Once PYPI has been deployed, another [workflow](.github/workflows/deploy_to_snowflake.yml) will run and update the UDF in Snowflake. That UDF needs the new version of PYPI and will just override the current function. This happens instantly after PYPI has been deployed.
|
|
256
|
-
|
|
257
279
|
## License
|
|
258
280
|
|
|
259
281
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import hashlib
|
|
1
2
|
import multiprocessing
|
|
2
3
|
import time
|
|
3
4
|
from multiprocessing import Process
|
|
@@ -271,11 +272,13 @@ def test_iteration() -> None:
|
|
|
271
272
|
"friendly_url",
|
|
272
273
|
"parent_normalized_url",
|
|
273
274
|
"root_normalized_url",
|
|
275
|
+
"registrable_domain",
|
|
274
276
|
"query_string",
|
|
275
277
|
"path",
|
|
276
278
|
"normalized_url_hash",
|
|
277
279
|
"parent_normalized_url_hash",
|
|
278
280
|
"root_normalized_url_hash",
|
|
281
|
+
"registrable_domain_hash",
|
|
279
282
|
"friendly_url_hash",
|
|
280
283
|
}
|
|
281
284
|
assert set(keys) == expected_keys
|
|
@@ -298,11 +301,13 @@ def test_keys_method() -> None:
|
|
|
298
301
|
"friendly_url",
|
|
299
302
|
"parent_normalized_url",
|
|
300
303
|
"root_normalized_url",
|
|
304
|
+
"registrable_domain",
|
|
301
305
|
"query_string",
|
|
302
306
|
"path",
|
|
303
307
|
"normalized_url_hash",
|
|
304
308
|
"parent_normalized_url_hash",
|
|
305
309
|
"root_normalized_url_hash",
|
|
310
|
+
"registrable_domain_hash",
|
|
306
311
|
"friendly_url_hash",
|
|
307
312
|
}
|
|
308
313
|
|
|
@@ -311,7 +316,7 @@ def test_keys_method() -> None:
|
|
|
311
316
|
# Keys should be a KeysView object
|
|
312
317
|
assert hasattr(keys, "__iter__")
|
|
313
318
|
assert hasattr(keys, "__len__")
|
|
314
|
-
assert len(keys) ==
|
|
319
|
+
assert len(keys) == 12
|
|
315
320
|
|
|
316
321
|
|
|
317
322
|
def test_friendly_url_preserves_casing() -> None:
|
|
@@ -389,6 +394,45 @@ def test_extended_access_removal_keeps_other_params() -> None:
|
|
|
389
394
|
assert normalizer["normalized_url"] == "wsj.com/articles/some-story?mod=hp_lead_pos1&page=2"
|
|
390
395
|
|
|
391
396
|
|
|
397
|
+
@pytest.mark.parametrize(
|
|
398
|
+
("url", "registrable", "root"),
|
|
399
|
+
[
|
|
400
|
+
# Multi-label public suffixes: the root is the suffix itself, the registrable domain is the site.
|
|
401
|
+
("https://www.abbvie.co.uk/our-science.html", "abbvie.co.uk", "co.uk"),
|
|
402
|
+
("https://news.bbc.co.uk/sport", "bbc.co.uk", "co.uk"),
|
|
403
|
+
("https://www.adelaidenow.com.au/news", "adelaidenow.com.au", "com.au"),
|
|
404
|
+
("https://www.forbes.com.mx/negocios", "forbes.com.mx", "com.mx"),
|
|
405
|
+
# Single-label suffixes agree with the root.
|
|
406
|
+
("https://finance.yahoo.com/quote/AAPL", "yahoo.com", "yahoo.com"),
|
|
407
|
+
("https://example.com/path", "example.com", "example.com"),
|
|
408
|
+
# Ports are not part of the domain.
|
|
409
|
+
("https://shop.example.co.uk:8080/cart", "example.co.uk", "co.uk:8080"),
|
|
410
|
+
],
|
|
411
|
+
)
|
|
412
|
+
def test_registrable_domain(url: str, registrable: str, root: str) -> None:
|
|
413
|
+
normalizer = TkNormalizer(url)
|
|
414
|
+
assert normalizer["registrable_domain"] == registrable
|
|
415
|
+
# root_normalized_url keeps its existing value so current consumers do not shift.
|
|
416
|
+
assert normalizer["root_normalized_url"] == root
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
@pytest.mark.parametrize(
|
|
420
|
+
("url", "registrable"),
|
|
421
|
+
[
|
|
422
|
+
# A bare public suffix has no label under it: fall back to the host.
|
|
423
|
+
("https://www.gov.uk/browse/tax", "gov.uk"),
|
|
424
|
+
("http://93.184.216.34/path", "93.184.216.34"),
|
|
425
|
+
],
|
|
426
|
+
)
|
|
427
|
+
def test_registrable_domain_falls_back_to_host(url: str, registrable: str) -> None:
|
|
428
|
+
assert TkNormalizer(url)["registrable_domain"] == registrable
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def test_registrable_domain_hash() -> None:
|
|
432
|
+
normalizer = TkNormalizer("https://news.bbc.co.uk/sport")
|
|
433
|
+
assert normalizer["registrable_domain_hash"] == hashlib.sha256(b"bbc.co.uk").hexdigest()
|
|
434
|
+
|
|
435
|
+
|
|
392
436
|
# These fixtures are at the bottom for readability of the upper tests
|
|
393
437
|
|
|
394
438
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|