liburlparser 1.4.4__tar.gz → 1.4.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {liburlparser-1.4.4 → liburlparser-1.4.6}/.github/workflows/wheels.yml +15 -16
- liburlparser-1.4.6/.ruff_cache/0.2.1/17168405204037786600 +0 -0
- liburlparser-1.4.4/.ruff_cache/0.1.6/9292255169009148407 → liburlparser-1.4.6/.ruff_cache/0.2.1/17509219516083451397 +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/CMakeLists.txt +34 -31
- {liburlparser-1.4.4 → liburlparser-1.4.6}/PKG-INFO +107 -55
- {liburlparser-1.4.4 → liburlparser-1.4.6}/README.md +106 -54
- liburlparser-1.4.6/cmake/modules/PyProject.cmake +28 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/docs/Doxyfile.in +1 -1
- {liburlparser-1.4.4 → liburlparser-1.4.6}/examples/main.cpp +1 -2
- liburlparser-1.4.6/include/urlparser.h +209 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/pyproject.toml +1 -1
- liburlparser-1.4.4/.ruff_cache/0.1.6/11868774027860405427 +0 -0
- liburlparser-1.4.4/.ruff_cache/0.1.6/3317816331866568629 +0 -0
- liburlparser-1.4.4/include/urlparser.h +0 -85
- liburlparser-1.4.4/noxfile.py +0 -24
- {liburlparser-1.4.4 → liburlparser-1.4.6}/.clang-format +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/.github/workflows/cmake.yml +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/.gitignore +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/.gitmodules +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/.ruff_cache/.gitignore +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/.ruff_cache/CACHEDIR.TAG +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/LICENSE +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/conda.recipe/meta.yaml +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/docs/images/logo/liburlparser-logo-1.svg +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/docs/images/logo/liburlparser-logo-2.svg +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/examples/common.h +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/binding/main.cpp +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/liburlparser/__init__.py +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/liburlparser/__main__.py +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/liburlparser/core.py +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/psl.cpp +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/psl.h +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/url.cpp +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/url.h +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/urlparser_host.cpp +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/src/urlparser_url.cpp +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/cpp/common.h +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/cpp/test_host.cpp +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/cpp/test_url.cpp +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/data/host_data.csv +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/data/url_data.csv +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/python/test_extra.py +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/python/test_host.py +0 -0
- {liburlparser-1.4.4 → liburlparser-1.4.6}/tests/python/test_url.py +0 -0
|
@@ -5,16 +5,15 @@ on:
|
|
|
5
5
|
pull_request:
|
|
6
6
|
push:
|
|
7
7
|
branches:
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
- published
|
|
8
|
+
- master
|
|
9
|
+
tags:
|
|
10
|
+
- "v*"
|
|
12
11
|
|
|
13
12
|
env:
|
|
14
13
|
FORCE_COLOR: 3
|
|
15
14
|
|
|
16
15
|
concurrency:
|
|
17
|
-
group:
|
|
16
|
+
group: github.workflow−{{ github.workflow }}-{{ github.ref }}
|
|
18
17
|
cancel-in-progress: true
|
|
19
18
|
|
|
20
19
|
jobs:
|
|
@@ -54,7 +53,7 @@ jobs:
|
|
|
54
53
|
submodules: true
|
|
55
54
|
|
|
56
55
|
- name: Build wheels
|
|
57
|
-
uses: pypa/cibuildwheel@v2.
|
|
56
|
+
uses: pypa/cibuildwheel@v2.16.5
|
|
58
57
|
env:
|
|
59
58
|
# CIBW_ARCHS_MACOS: universal2
|
|
60
59
|
# CIBW_ARCHS_WINDOWS: auto ARM64
|
|
@@ -71,11 +70,11 @@ jobs:
|
|
|
71
70
|
path: wheelhouse/*.whl
|
|
72
71
|
|
|
73
72
|
|
|
74
|
-
|
|
75
|
-
name: Upload
|
|
73
|
+
release:
|
|
74
|
+
name: Release Assets and Upload to PyPI
|
|
76
75
|
needs: [build_wheels, build_sdist]
|
|
77
76
|
runs-on: ubuntu-latest
|
|
78
|
-
if: github.
|
|
77
|
+
if: startsWith(github.ref, 'refs/tags/')
|
|
79
78
|
|
|
80
79
|
steps:
|
|
81
80
|
- uses: actions/setup-python@v4
|
|
@@ -87,13 +86,13 @@ jobs:
|
|
|
87
86
|
name: artifact
|
|
88
87
|
path: dist
|
|
89
88
|
|
|
90
|
-
-
|
|
89
|
+
- name: Release with assets
|
|
90
|
+
uses: softprops/action-gh-release@v1
|
|
91
|
+
with:
|
|
92
|
+
files: dist/*
|
|
93
|
+
|
|
94
|
+
- name: Publishing to PyPi
|
|
95
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
91
96
|
with:
|
|
92
97
|
password: ${{ secrets.PYPI_PASSWORD }}
|
|
93
98
|
user: ${{ secrets.PYPI_USERNAME }}
|
|
94
|
-
|
|
95
|
-
# - name: Release
|
|
96
|
-
# uses: softprops/action-gh-release@v1
|
|
97
|
-
# if: startsWith(github.ref, 'refs/tags/')
|
|
98
|
-
# with:
|
|
99
|
-
# files: dist/*
|
|
Binary file
|
|
Binary file
|
|
@@ -5,6 +5,12 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
|
|
5
5
|
|
|
6
6
|
set(CMAKE_CXX_EXTENSIONS OFF)
|
|
7
7
|
|
|
8
|
+
|
|
9
|
+
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")
|
|
10
|
+
include(PyProject)
|
|
11
|
+
message(STATUS "Project: ${PyProject_NAME}@v${PyProject_VERSION}")
|
|
12
|
+
|
|
13
|
+
|
|
8
14
|
if(NOT CMAKE_BUILD_TYPE)
|
|
9
15
|
set(CMAKE_BUILD_TYPE Release)
|
|
10
16
|
endif()
|
|
@@ -14,27 +20,28 @@ endif()
|
|
|
14
20
|
#set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g")
|
|
15
21
|
|
|
16
22
|
|
|
17
|
-
if(MSVC)
|
|
18
|
-
# Microsoft Visual Studio compiler
|
|
19
|
-
set(MSVC_FLAGS "/O2")
|
|
20
|
-
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${MSVC_FLAGS}")
|
|
21
|
-
elseif(CMAKE_COMPILER_IS_GNUCXX)
|
|
22
|
-
# GCC or Clang compiler
|
|
23
|
-
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
24
|
-
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
|
25
|
-
# Linux
|
|
26
|
-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
|
|
27
|
-
else()
|
|
28
|
-
# Other OS
|
|
29
|
-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=generic")
|
|
30
|
-
endif()
|
|
31
|
-
endif()
|
|
23
|
+
#if(MSVC)
|
|
24
|
+
# # Microsoft Visual Studio compiler
|
|
25
|
+
# set(MSVC_FLAGS "/O2")
|
|
26
|
+
# set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${MSVC_FLAGS}")
|
|
27
|
+
#elseif(CMAKE_COMPILER_IS_GNUCXX)
|
|
28
|
+
# # GCC or Clang compiler
|
|
29
|
+
# set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
30
|
+
# if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
|
31
|
+
# # Linux
|
|
32
|
+
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
|
|
33
|
+
# else()
|
|
34
|
+
# # Other OS
|
|
35
|
+
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=generic")
|
|
36
|
+
# endif()
|
|
37
|
+
#endif()
|
|
32
38
|
|
|
33
39
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "" FORCE)
|
|
34
40
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING "" FORCE)
|
|
35
41
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "" FORCE)
|
|
36
42
|
|
|
37
|
-
project(
|
|
43
|
+
project(${PyProject_NAME} LANGUAGES CXX)
|
|
44
|
+
|
|
38
45
|
|
|
39
46
|
# Download the public suffix list file from the internet
|
|
40
47
|
set(PUBLIC_SUFFIX_LIST_URL "https://publicsuffix.org/list/public_suffix_list.dat")
|
|
@@ -43,12 +50,17 @@ if(NOT DEFINED SKBUILD)
|
|
|
43
50
|
set(PUBLIC_SUFFIX_LIST_DAT "${CMAKE_CURRENT_BINARY_DIR}/${PUBLIC_SUFFIX_LIST_DAT}")
|
|
44
51
|
endif()
|
|
45
52
|
|
|
53
|
+
# Download the public suffix list file from the internet if not already exists
|
|
46
54
|
if(EXISTS "${PUBLIC_SUFFIX_LIST_DAT}")
|
|
47
|
-
|
|
55
|
+
message(STATUS "[DATA] ${PUBLIC_SUFFIX_LIST_DAT} is found.")
|
|
48
56
|
else()
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
57
|
+
message(WARNING "[DATA] ${PUBLIC_SUFFIX_LIST_DAT} is not found. Trying to download it.")
|
|
58
|
+
file(DOWNLOAD "${PUBLIC_SUFFIX_LIST_URL}" ${PUBLIC_SUFFIX_LIST_DAT})
|
|
59
|
+
if(EXISTS "${PUBLIC_SUFFIX_LIST_DAT}")
|
|
60
|
+
message(STATUS "[DATA] ${PUBLIC_SUFFIX_LIST_DAT} is downloaded.")
|
|
61
|
+
else()
|
|
62
|
+
message(FATAL_ERROR "[DATA] Failed to download ${PUBLIC_SUFFIX_LIST_DAT}.")
|
|
63
|
+
endif()
|
|
52
64
|
endif()
|
|
53
65
|
|
|
54
66
|
file(GLOB SOURCES ${PROJECT_SOURCE_DIR}/src/*.cpp)
|
|
@@ -85,9 +97,6 @@ if(DEFINED SKBUILD)
|
|
|
85
97
|
src/binding/main.cpp
|
|
86
98
|
)
|
|
87
99
|
|
|
88
|
-
# set(PYBIND11_NEWPYTHON ON)
|
|
89
|
-
# find_package(pybind11 CONFIG REQUIRED)
|
|
90
|
-
# pybind11_add_module(_core src/${SKBUILD_PROJECT_NAME}/main.cpp)
|
|
91
100
|
target_sources(_core PRIVATE ${SOURCES})
|
|
92
101
|
target_include_directories(_core PRIVATE include src)
|
|
93
102
|
target_compile_definitions(_core PRIVATE VERSION_INFO=${SKBUILD_PROJECT_VERSION})
|
|
@@ -133,7 +142,7 @@ install(TARGETS urlparser
|
|
|
133
142
|
|
|
134
143
|
|
|
135
144
|
enable_testing()
|
|
136
|
-
add_subdirectory(third_party/googletest)
|
|
145
|
+
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/googletest)
|
|
137
146
|
file(GLOB TEST_SOURCES ${PROJECT_SOURCE_DIR}/tests/cpp/*.cpp)
|
|
138
147
|
|
|
139
148
|
add_executable(test_liburlparser ${TEST_SOURCES})
|
|
@@ -150,13 +159,7 @@ message(STATUS "Doxygen_FOUND : ${Doxygen_FOUND}")
|
|
|
150
159
|
# Find Doxygen package
|
|
151
160
|
if (Doxygen_FOUND)
|
|
152
161
|
# Configure Doxyfile.in template
|
|
153
|
-
configure_file(docs/Doxyfile.in ${
|
|
154
|
-
|
|
155
|
-
# Add target to generate Doxyfile
|
|
156
|
-
add_custom_target(generate_doxyfile
|
|
157
|
-
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/Doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
|
|
158
|
-
COMMENT "Generating Doxyfile"
|
|
159
|
-
)
|
|
162
|
+
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
|
|
160
163
|
|
|
161
164
|
# Add target to generate documentation
|
|
162
165
|
add_custom_target(docs
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: liburlparser
|
|
3
|
-
Version: 1.4.
|
|
3
|
+
Version: 1.4.6
|
|
4
4
|
Summary: Fastest Url parser in the world
|
|
5
5
|
Author-Email: Mohammad Raziei <mohammadraziei1375@gmail.com>
|
|
6
6
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -37,9 +37,29 @@ Description-Content-Type: text/markdown
|
|
|
37
37
|
</h4>
|
|
38
38
|
</p>
|
|
39
39
|
|
|
40
|
-
](https://github.com/mohammadraziei/liburlparser "Go to GitHub repo")
|
|
41
|
+
[](https://github.com/mohammadraziei/liburlparser)
|
|
42
|
+
[](https://github.com/mohammadraziei/liburlparser)
|
|
43
|
+
|
|
44
|
+
[](https://pypi.org/project/liburlparser/)
|
|
45
|
+

|
|
46
|
+

|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
[](https://github.com/mohammadraziei/liburlparser/releases/)
|
|
50
|
+
[](#license)
|
|
51
|
+
[](https://github.com/mohammadraziei/liburlparser/issues)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
[](https://sonarcloud.io/summary/new_code?id=MohammadRaziei_liburlparser)
|
|
55
|
+
|
|
56
|
+
[](https://sonarcloud.io/summary/new_code?id=MohammadRaziei_liburlparser)
|
|
57
|
+
[](https://snyk.io/advisor/python/liburlparser)
|
|
58
|
+
|
|
59
|
+
[//]: # ([](https://mohammadraziei.github.io/liburlparser/))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
43
63
|
<!--
|
|
44
64
|

|
|
45
65
|
[](https://pypi.org/project/PyDomainExtractor/)
|
|
@@ -65,13 +85,42 @@ Description-Content-Type: text/markdown
|
|
|
65
85
|
|
|
66
86
|
## About The Project
|
|
67
87
|
|
|
88
|
+
**liburlparser** is a powerful domain extractor library written in C++ with Python bindings. It provides efficient URL parsing capabilities for both C++ and Python, making it a valuable tool for projects that involve working with web addresses.
|
|
89
|
+
|
|
68
90
|
### Features
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
Here are some key features of **liburlparser**:
|
|
94
|
+
|
|
95
|
+
1. **Multiple Language Support**:
|
|
96
|
+
- liburlparser can be used in multiple programming languages, including `Python`, `C++`, and `Shell`.
|
|
97
|
+
- It offers an intuitive interface that remains consistent across both C++ and Python.
|
|
98
|
+
|
|
99
|
+
2. **Clean Code Design**:
|
|
100
|
+
- The library provides two separate classes: `Url` and `Host`.
|
|
101
|
+
- This separation allows for cleaner and more organized code when dealing with URLs.
|
|
102
|
+
|
|
103
|
+
3. **Public Suffix List Support**:
|
|
104
|
+
- liburlparser supports known combinatorial suffixes (e.g., "ac.ir") using the public_suffix_list.
|
|
105
|
+
- It can also handle unknown suffixes (e.g., "comm" in "google.comm").
|
|
106
|
+
|
|
107
|
+
4. **Automatic Public Suffix List Updates**:
|
|
108
|
+
- Before each build and deployment, liburlparser updates the public_suffix_list automatically.
|
|
109
|
+
|
|
110
|
+
5. **Host Properties**:
|
|
111
|
+
- The `Host` class includes properties such as subdomain, domain, domain name, and suffix.
|
|
112
|
+
|
|
113
|
+
6. **URL Properties**:
|
|
114
|
+
- The `Url` class provides properties like protocol, userinfo, host (and all host properties), port, path, query parameters, and fragment.
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
<!--
|
|
69
118
|
* Multiple programming language supported such as `Python`, `C++` and `Shell`
|
|
70
119
|
* Intuitive interface and identical in C++ and Python
|
|
71
120
|
* Provide two seperated class Url and Host for the purpose of clean code
|
|
72
121
|
* Also support [public_suffix_list](https://publicsuffix.org/list/public_suffix_list.dat) for known combinatorial suffix such as "ac.ir"
|
|
73
122
|
* Support unknown suffix like "google.comm" (it detect "comm" as suffix)
|
|
74
|
-
* Update public_suffix_list automatically before each build and deploy
|
|
123
|
+
* Update public_suffix_list automatically before each build and deploy
|
|
75
124
|
* Host properties:
|
|
76
125
|
* subdomain
|
|
77
126
|
* domain
|
|
@@ -86,54 +135,7 @@ Description-Content-Type: text/markdown
|
|
|
86
135
|
* query
|
|
87
136
|
* params
|
|
88
137
|
* fragment
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
## Setup
|
|
92
|
-
### C++:
|
|
93
|
-
|
|
94
|
-
#### build steps:
|
|
95
|
-
```sh
|
|
96
|
-
git clone https://github.com/mohammadraziei/liburlparser
|
|
97
|
-
mkdir -p build; cd build
|
|
98
|
-
cmake ..
|
|
99
|
-
# Build the project:
|
|
100
|
-
make
|
|
101
|
-
# [Optional] run tests:
|
|
102
|
-
make test
|
|
103
|
-
# [Optional] make documents:
|
|
104
|
-
make docs
|
|
105
|
-
# [Optional] Run examples:
|
|
106
|
-
./example
|
|
107
|
-
# Make install
|
|
108
|
-
sudo make install
|
|
109
|
-
```
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
### Python and Command Line:
|
|
114
|
-
Be aware that it required `python>=3.8`
|
|
115
|
-
#### Installation
|
|
116
|
-
###### pip by [pypi](https://pypi.org/project/liburlparser/)
|
|
117
|
-
```sh
|
|
118
|
-
pip install liburlparser
|
|
119
|
-
```
|
|
120
|
-
if you want to use psl.update to update the public suffix list, you must install the `online` version
|
|
121
|
-
```sh
|
|
122
|
-
pip install "liburlparser[online]"
|
|
123
|
-
```
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
Or
|
|
127
|
-
###### pip by [git](https://github.com/mohammadraziei/liburlparser)
|
|
128
|
-
```sh
|
|
129
|
-
pip install git+https://github.com/mohammadraziei/liburlparser
|
|
130
|
-
```
|
|
131
|
-
Or
|
|
132
|
-
###### manually
|
|
133
|
-
```sh
|
|
134
|
-
git clone https://github.com/mohammadraziei/liburlparser
|
|
135
|
-
pip install ./liburlparser
|
|
136
|
-
```
|
|
138
|
+
-->
|
|
137
139
|
|
|
138
140
|
## Usage
|
|
139
141
|
|
|
@@ -215,6 +217,56 @@ TLD::Host host = TLD::Host::fromUrl("https://ee.aut.ac.ir/about");
|
|
|
215
217
|
you can see all methods in python we can use in c++ very easily
|
|
216
218
|
|
|
217
219
|
|
|
220
|
+
|
|
221
|
+
## Installation
|
|
222
|
+
### C++:
|
|
223
|
+
|
|
224
|
+
#### build steps:
|
|
225
|
+
```sh
|
|
226
|
+
git clone https://github.com/mohammadraziei/liburlparser
|
|
227
|
+
mkdir -p build; cd build
|
|
228
|
+
cmake ..
|
|
229
|
+
# Build the project:
|
|
230
|
+
make
|
|
231
|
+
# [Optional] run tests:
|
|
232
|
+
make test
|
|
233
|
+
# [Optional] make documents:
|
|
234
|
+
make docs
|
|
235
|
+
# [Optional] Run examples:
|
|
236
|
+
./example
|
|
237
|
+
# Make install
|
|
238
|
+
sudo make install
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
### Python and Command Line:
|
|
244
|
+
Be aware that it required `python>=3.8`
|
|
245
|
+
#### Installation
|
|
246
|
+
###### pip by [pypi](https://pypi.org/project/liburlparser/)
|
|
247
|
+
```sh
|
|
248
|
+
pip install liburlparser
|
|
249
|
+
```
|
|
250
|
+
if you want to use psl.update to update the public suffix list, you must install the `online` version
|
|
251
|
+
```sh
|
|
252
|
+
pip install "liburlparser[online]"
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
Or
|
|
257
|
+
###### pip by [git](https://github.com/mohammadraziei/liburlparser)
|
|
258
|
+
```sh
|
|
259
|
+
pip install git+https://github.com/mohammadraziei/liburlparser
|
|
260
|
+
```
|
|
261
|
+
Or
|
|
262
|
+
###### manually
|
|
263
|
+
```sh
|
|
264
|
+
git clone https://github.com/mohammadraziei/liburlparser
|
|
265
|
+
pip install ./liburlparser
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
|
|
218
270
|
### Performance
|
|
219
271
|
|
|
220
272
|
|
|
@@ -254,10 +306,10 @@ Distributed under the MIT License. See [LICENSE](LICENSE) for more information.
|
|
|
254
306
|
|
|
255
307
|
<!-- Gal Ben David - gal@intsights.com -->
|
|
256
308
|
|
|
257
|
-
Project Link:
|
|
309
|
+
Project Link:
|
|
258
310
|
- [https://github.com/mohammadraziei/liburlparser](https://github.com/mohammadraziei/liburlparser)
|
|
259
311
|
- [https://pypi.org/project/liburlparser](https://pypi.org/project/liburlparser)
|
|
260
312
|
|
|
261
313
|
|
|
262
314
|
|
|
263
|
-
[license-shield]: https://img.shields.io/github/license/othneildrew/Best-README-Template.svg?style=flat-square
|
|
315
|
+
[license-shield]: https://img.shields.io/github/license/othneildrew/Best-README-Template.svg?style=flat-square
|
|
@@ -10,9 +10,29 @@
|
|
|
10
10
|
</h4>
|
|
11
11
|
</p>
|
|
12
12
|
|
|
13
|
-
](https://github.com/mohammadraziei/liburlparser "Go to GitHub repo")
|
|
14
|
+
[](https://github.com/mohammadraziei/liburlparser)
|
|
15
|
+
[](https://github.com/mohammadraziei/liburlparser)
|
|
16
|
+
|
|
17
|
+
[](https://pypi.org/project/liburlparser/)
|
|
18
|
+

|
|
19
|
+

|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
[](https://github.com/mohammadraziei/liburlparser/releases/)
|
|
23
|
+
[](#license)
|
|
24
|
+
[](https://github.com/mohammadraziei/liburlparser/issues)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
[](https://sonarcloud.io/summary/new_code?id=MohammadRaziei_liburlparser)
|
|
28
|
+
|
|
29
|
+
[](https://sonarcloud.io/summary/new_code?id=MohammadRaziei_liburlparser)
|
|
30
|
+
[](https://snyk.io/advisor/python/liburlparser)
|
|
31
|
+
|
|
32
|
+
[//]: # ([](https://mohammadraziei.github.io/liburlparser/))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
16
36
|
<!--
|
|
17
37
|

|
|
18
38
|
[](https://pypi.org/project/PyDomainExtractor/)
|
|
@@ -38,13 +58,42 @@
|
|
|
38
58
|
|
|
39
59
|
## About The Project
|
|
40
60
|
|
|
61
|
+
**liburlparser** is a powerful domain extractor library written in C++ with Python bindings. It provides efficient URL parsing capabilities for both C++ and Python, making it a valuable tool for projects that involve working with web addresses.
|
|
62
|
+
|
|
41
63
|
### Features
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
Here are some key features of **liburlparser**:
|
|
67
|
+
|
|
68
|
+
1. **Multiple Language Support**:
|
|
69
|
+
- liburlparser can be used in multiple programming languages, including `Python`, `C++`, and `Shell`.
|
|
70
|
+
- It offers an intuitive interface that remains consistent across both C++ and Python.
|
|
71
|
+
|
|
72
|
+
2. **Clean Code Design**:
|
|
73
|
+
- The library provides two separate classes: `Url` and `Host`.
|
|
74
|
+
- This separation allows for cleaner and more organized code when dealing with URLs.
|
|
75
|
+
|
|
76
|
+
3. **Public Suffix List Support**:
|
|
77
|
+
- liburlparser supports known combinatorial suffixes (e.g., "ac.ir") using the public_suffix_list.
|
|
78
|
+
- It can also handle unknown suffixes (e.g., "comm" in "google.comm").
|
|
79
|
+
|
|
80
|
+
4. **Automatic Public Suffix List Updates**:
|
|
81
|
+
- Before each build and deployment, liburlparser updates the public_suffix_list automatically.
|
|
82
|
+
|
|
83
|
+
5. **Host Properties**:
|
|
84
|
+
- The `Host` class includes properties such as subdomain, domain, domain name, and suffix.
|
|
85
|
+
|
|
86
|
+
6. **URL Properties**:
|
|
87
|
+
- The `Url` class provides properties like protocol, userinfo, host (and all host properties), port, path, query parameters, and fragment.
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
<!--
|
|
42
91
|
* Multiple programming language supported such as `Python`, `C++` and `Shell`
|
|
43
92
|
* Intuitive interface and identical in C++ and Python
|
|
44
93
|
* Provide two seperated class Url and Host for the purpose of clean code
|
|
45
94
|
* Also support [public_suffix_list](https://publicsuffix.org/list/public_suffix_list.dat) for known combinatorial suffix such as "ac.ir"
|
|
46
95
|
* Support unknown suffix like "google.comm" (it detect "comm" as suffix)
|
|
47
|
-
* Update public_suffix_list automatically before each build and deploy
|
|
96
|
+
* Update public_suffix_list automatically before each build and deploy
|
|
48
97
|
* Host properties:
|
|
49
98
|
* subdomain
|
|
50
99
|
* domain
|
|
@@ -59,54 +108,7 @@
|
|
|
59
108
|
* query
|
|
60
109
|
* params
|
|
61
110
|
* fragment
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
## Setup
|
|
65
|
-
### C++:
|
|
66
|
-
|
|
67
|
-
#### build steps:
|
|
68
|
-
```sh
|
|
69
|
-
git clone https://github.com/mohammadraziei/liburlparser
|
|
70
|
-
mkdir -p build; cd build
|
|
71
|
-
cmake ..
|
|
72
|
-
# Build the project:
|
|
73
|
-
make
|
|
74
|
-
# [Optional] run tests:
|
|
75
|
-
make test
|
|
76
|
-
# [Optional] make documents:
|
|
77
|
-
make docs
|
|
78
|
-
# [Optional] Run examples:
|
|
79
|
-
./example
|
|
80
|
-
# Make install
|
|
81
|
-
sudo make install
|
|
82
|
-
```
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
### Python and Command Line:
|
|
87
|
-
Be aware that it required `python>=3.8`
|
|
88
|
-
#### Installation
|
|
89
|
-
###### pip by [pypi](https://pypi.org/project/liburlparser/)
|
|
90
|
-
```sh
|
|
91
|
-
pip install liburlparser
|
|
92
|
-
```
|
|
93
|
-
if you want to use psl.update to update the public suffix list, you must install the `online` version
|
|
94
|
-
```sh
|
|
95
|
-
pip install "liburlparser[online]"
|
|
96
|
-
```
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
Or
|
|
100
|
-
###### pip by [git](https://github.com/mohammadraziei/liburlparser)
|
|
101
|
-
```sh
|
|
102
|
-
pip install git+https://github.com/mohammadraziei/liburlparser
|
|
103
|
-
```
|
|
104
|
-
Or
|
|
105
|
-
###### manually
|
|
106
|
-
```sh
|
|
107
|
-
git clone https://github.com/mohammadraziei/liburlparser
|
|
108
|
-
pip install ./liburlparser
|
|
109
|
-
```
|
|
111
|
+
-->
|
|
110
112
|
|
|
111
113
|
## Usage
|
|
112
114
|
|
|
@@ -188,6 +190,56 @@ TLD::Host host = TLD::Host::fromUrl("https://ee.aut.ac.ir/about");
|
|
|
188
190
|
you can see all methods in python we can use in c++ very easily
|
|
189
191
|
|
|
190
192
|
|
|
193
|
+
|
|
194
|
+
## Installation
|
|
195
|
+
### C++:
|
|
196
|
+
|
|
197
|
+
#### build steps:
|
|
198
|
+
```sh
|
|
199
|
+
git clone https://github.com/mohammadraziei/liburlparser
|
|
200
|
+
mkdir -p build; cd build
|
|
201
|
+
cmake ..
|
|
202
|
+
# Build the project:
|
|
203
|
+
make
|
|
204
|
+
# [Optional] run tests:
|
|
205
|
+
make test
|
|
206
|
+
# [Optional] make documents:
|
|
207
|
+
make docs
|
|
208
|
+
# [Optional] Run examples:
|
|
209
|
+
./example
|
|
210
|
+
# Make install
|
|
211
|
+
sudo make install
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
### Python and Command Line:
|
|
217
|
+
Be aware that it required `python>=3.8`
|
|
218
|
+
#### Installation
|
|
219
|
+
###### pip by [pypi](https://pypi.org/project/liburlparser/)
|
|
220
|
+
```sh
|
|
221
|
+
pip install liburlparser
|
|
222
|
+
```
|
|
223
|
+
if you want to use psl.update to update the public suffix list, you must install the `online` version
|
|
224
|
+
```sh
|
|
225
|
+
pip install "liburlparser[online]"
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
Or
|
|
230
|
+
###### pip by [git](https://github.com/mohammadraziei/liburlparser)
|
|
231
|
+
```sh
|
|
232
|
+
pip install git+https://github.com/mohammadraziei/liburlparser
|
|
233
|
+
```
|
|
234
|
+
Or
|
|
235
|
+
###### manually
|
|
236
|
+
```sh
|
|
237
|
+
git clone https://github.com/mohammadraziei/liburlparser
|
|
238
|
+
pip install ./liburlparser
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
|
|
191
243
|
### Performance
|
|
192
244
|
|
|
193
245
|
|
|
@@ -227,10 +279,10 @@ Distributed under the MIT License. See [LICENSE](LICENSE) for more information.
|
|
|
227
279
|
|
|
228
280
|
<!-- Gal Ben David - gal@intsights.com -->
|
|
229
281
|
|
|
230
|
-
Project Link:
|
|
282
|
+
Project Link:
|
|
231
283
|
- [https://github.com/mohammadraziei/liburlparser](https://github.com/mohammadraziei/liburlparser)
|
|
232
284
|
- [https://pypi.org/project/liburlparser](https://pypi.org/project/liburlparser)
|
|
233
285
|
|
|
234
286
|
|
|
235
287
|
|
|
236
|
-
[license-shield]: https://img.shields.io/github/license/othneildrew/Best-README-Template.svg?style=flat-square
|
|
288
|
+
[license-shield]: https://img.shields.io/github/license/othneildrew/Best-README-Template.svg?style=flat-square
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# cmake/modules/PyProject.cmake
|
|
2
|
+
function(extract_pyproject_info)
|
|
3
|
+
set(filename "${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml")
|
|
4
|
+
set(PyProject_TOML_PATH ${filename} PARENT_SCOPE)
|
|
5
|
+
|
|
6
|
+
# Read lines from the file into a list
|
|
7
|
+
file(STRINGS ${filename} Lines)
|
|
8
|
+
|
|
9
|
+
# Iterate over the list to find the version line
|
|
10
|
+
foreach(Line ${Lines})
|
|
11
|
+
# Use a regular expression to extract the version number
|
|
12
|
+
if(Line MATCHES "version = \"([0-9]+\\.[0-9]+\\.[0-9]+)\"")
|
|
13
|
+
set(PyProject_VERSION "${CMAKE_MATCH_1}" PARENT_SCOPE)
|
|
14
|
+
endif()
|
|
15
|
+
|
|
16
|
+
# Extract the project name
|
|
17
|
+
if(Line MATCHES "name = \"([A-Za-z0-9_-]+)\"")
|
|
18
|
+
set(PyProject_NAME "${CMAKE_MATCH_1}" PARENT_SCOPE)
|
|
19
|
+
endif()
|
|
20
|
+
|
|
21
|
+
# Extract the project description
|
|
22
|
+
if(Line MATCHES "description = \"([A-Za-z0-9_ -]+)\"")
|
|
23
|
+
set(PyProject_DESCRIPTION "${CMAKE_MATCH_1}" PARENT_SCOPE)
|
|
24
|
+
endif()
|
|
25
|
+
endforeach()
|
|
26
|
+
endfunction()
|
|
27
|
+
|
|
28
|
+
extract_pyproject_info()
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file urlparser.h
|
|
3
|
+
* @brief Defines classes for parsing and handling URLs and hosts.
|
|
4
|
+
*
|
|
5
|
+
* This file contains the declaration of the Url and Host classes, which provide
|
|
6
|
+
* functionality for parsing and managing URLs and hosts within them. The Url class
|
|
7
|
+
* allows extraction and manipulation of various components of a URL, while the Host
|
|
8
|
+
* class focuses on handling and extracting domain-related information from a host.
|
|
9
|
+
*
|
|
10
|
+
* The Url class encapsulates a URL and provides methods to access its components
|
|
11
|
+
* such as protocol, subdomain, domain, suffix, query, fragment, userinfo, port,
|
|
12
|
+
* and parameters. Additionally, it offers methods for constructing a URL string and
|
|
13
|
+
* obtaining specific parts of the URL.
|
|
14
|
+
*
|
|
15
|
+
* The Host class represents a host part of a URL and offers functionalities for
|
|
16
|
+
* extracting domain-related details such as suffix, domain, subdomain, and the full
|
|
17
|
+
* domain. It also supports the removal of 'www' from the host and provides methods
|
|
18
|
+
* to obtain domain-specific information.
|
|
19
|
+
*
|
|
20
|
+
* Example Usage:
|
|
21
|
+
* @code
|
|
22
|
+
* // Creating a URL object and accessing its methods
|
|
23
|
+
* TLD::Url url("https://www.example.com/path/to/resource");
|
|
24
|
+
* std::cout << "URL Protocol: " << url.protocol() << std::endl;
|
|
25
|
+
* std::cout << "URL Domain: " << url.domain() << std::endl;
|
|
26
|
+
* std::cout << "URL Suffix: " << url.suffix() << std::endl;
|
|
27
|
+
* std::cout << "URL Query: " << url.query() << std::endl;
|
|
28
|
+
*
|
|
29
|
+
* // Creating a Host object and calling its methods
|
|
30
|
+
* TLD::Host host("www.example.com");
|
|
31
|
+
* std::cout << "Host Domain: " << host.domain() << std::endl;
|
|
32
|
+
* std::cout << "Host Suffix: " << host.suffix() << std::endl;
|
|
33
|
+
* std::cout << "Host Subdomain: " << host.subdomain() << std::endl;
|
|
34
|
+
* @endcode
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
#ifndef TLD_URLPARSER_H
|
|
38
|
+
#define TLD_URLPARSER_H
|
|
39
|
+
|
|
40
|
+
#include <iostream>
|
|
41
|
+
#include <memory>
|
|
42
|
+
#include <string>
|
|
43
|
+
#include <vector>
|
|
44
|
+
|
|
45
|
+
#ifndef PUBLIC_SUFFIX_LIST_DAT
|
|
46
|
+
#define PUBLIC_SUFFIX_LIST_DAT "public_suffix_list.dat"
|
|
47
|
+
#endif
|
|
48
|
+
|
|
49
|
+
namespace TLD {
|
|
50
|
+
constexpr bool DEFAULT_IGNORE_WWW = false;
|
|
51
|
+
|
|
52
|
+
using QueryParams = std::vector<std::string>;
|
|
53
|
+
|
|
54
|
+
class Host;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* @brief Represents a URL.
|
|
58
|
+
*
|
|
59
|
+
* The Url class provides functionalities for parsing and managing URLs. It
|
|
60
|
+
* allows access to various components of a URL such as protocol, subdomain,
|
|
61
|
+
* domain, suffix, query, fragment, userinfo, port, and parameters.
|
|
62
|
+
*/
|
|
63
|
+
class Url {
|
|
64
|
+
public:
|
|
65
|
+
/**
|
|
66
|
+
* @brief Check if the Public Suffix List (PSL) is loaded.
|
|
67
|
+
* @return true if the PSL is loaded, false otherwise.
|
|
68
|
+
*/
|
|
69
|
+
static bool isPslLoaded() noexcept;
|
|
70
|
+
/**
|
|
71
|
+
* @brief Extract the host from a given URL.
|
|
72
|
+
* @param url The URL from which to extract the host.
|
|
73
|
+
* @return The extracted host.
|
|
74
|
+
*/
|
|
75
|
+
static std::string extractHost(const std::string& url) noexcept;
|
|
76
|
+
|
|
77
|
+
public:
|
|
78
|
+
/**
|
|
79
|
+
* @brief Construct a Url object from a given URL string.
|
|
80
|
+
* @param url The URL string.
|
|
81
|
+
* @param ignore_www Whether to ignore the "www" subdomain.
|
|
82
|
+
*/
|
|
83
|
+
Url(const std::string& url, const bool ignore_www = DEFAULT_IGNORE_WWW);
|
|
84
|
+
/**
|
|
85
|
+
* @brief Default constructor for the Url class.
|
|
86
|
+
*/
|
|
87
|
+
Url() noexcept = default;
|
|
88
|
+
|
|
89
|
+
bool operator==(const Url&) const;
|
|
90
|
+
|
|
91
|
+
std::string str() const noexcept;
|
|
92
|
+
/**
|
|
93
|
+
* @brief Get the protocol of the URL.
|
|
94
|
+
* @return str Protocol of the URL.
|
|
95
|
+
*/
|
|
96
|
+
const std::string& protocol() const noexcept;
|
|
97
|
+
/**
|
|
98
|
+
* @brief Get the Subdomain of the URL.
|
|
99
|
+
* @return str Subdomain of the URL.
|
|
100
|
+
*/
|
|
101
|
+
const std::string& subdomain() const noexcept;
|
|
102
|
+
/**
|
|
103
|
+
* @brief Get the Domain of the URL.
|
|
104
|
+
* @return str Domain of the URL.
|
|
105
|
+
*/
|
|
106
|
+
const std::string& domain() const noexcept;
|
|
107
|
+
/**
|
|
108
|
+
* @brief Get the Suffix of the URL.
|
|
109
|
+
* @return str Suffix of the URL.
|
|
110
|
+
*/
|
|
111
|
+
const std::string& suffix() const noexcept;
|
|
112
|
+
/**
|
|
113
|
+
* @brief Get the Query of the URL.
|
|
114
|
+
* @return str Query of the URL.
|
|
115
|
+
*/
|
|
116
|
+
const std::string& query() const noexcept;
|
|
117
|
+
/**
|
|
118
|
+
* @brief Get the Fragment of the URL.
|
|
119
|
+
* @return str Fragment of the URL.
|
|
120
|
+
*/
|
|
121
|
+
const std::string& fragment() const noexcept;
|
|
122
|
+
/**
|
|
123
|
+
* @brief Get the Userinfo of the URL.
|
|
124
|
+
* @return str Userinfo of the URL.
|
|
125
|
+
*/
|
|
126
|
+
const std::string& userinfo() const noexcept;
|
|
127
|
+
/**
|
|
128
|
+
* @brief Get the abspath of the URL.
|
|
129
|
+
* @return str abspath of the URL.
|
|
130
|
+
*/
|
|
131
|
+
std::string abspath() const noexcept;
|
|
132
|
+
/**
|
|
133
|
+
* @brief Get the domain name of the URL.
|
|
134
|
+
* it is the same as domain() + "." + subdomain().
|
|
135
|
+
* @return str port of the URL.
|
|
136
|
+
*/
|
|
137
|
+
std::string domainName() const noexcept;
|
|
138
|
+
/**
|
|
139
|
+
* @brief Get the full domain of the URL.
|
|
140
|
+
* @return str full domain of the URL.
|
|
141
|
+
*/
|
|
142
|
+
const std::string& fulldomain() const noexcept;
|
|
143
|
+
/**
|
|
144
|
+
* @brief Get the port of the URL.
|
|
145
|
+
* @return int port of the URL.
|
|
146
|
+
*/
|
|
147
|
+
const int port() const noexcept;
|
|
148
|
+
/**
|
|
149
|
+
* @brief Get the parameters of the URL.
|
|
150
|
+
* @return QueryParams parameters of the URL.
|
|
151
|
+
*/
|
|
152
|
+
QueryParams params() const noexcept;
|
|
153
|
+
/**
|
|
154
|
+
* @brief Get the host of the URL.
|
|
155
|
+
* @return TLD::Host object of the URL.
|
|
156
|
+
*/
|
|
157
|
+
const Host& host() const;
|
|
158
|
+
|
|
159
|
+
private:
|
|
160
|
+
class Impl;
|
|
161
|
+
std::shared_ptr<Impl> impl; // since all methods are constants
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* @brief Represents a Host part of a URL.
|
|
166
|
+
*
|
|
167
|
+
* The Host class encapsulates functionalities for handling the host component
|
|
168
|
+
* of a URL. It provides methods to extract domain-specific details such as
|
|
169
|
+
* suffix, domain, subdomain, and the full domain.
|
|
170
|
+
*/
|
|
171
|
+
class Host {
|
|
172
|
+
public:
|
|
173
|
+
static Host fromUrl(const std::string& url,
|
|
174
|
+
const bool ignore_www = DEFAULT_IGNORE_WWW);
|
|
175
|
+
static void loadPslFromPath(const std::string& filepath);
|
|
176
|
+
static void loadPslFromString(const std::string& filestr);
|
|
177
|
+
/**
|
|
178
|
+
* @brief Check if the Public Suffix List (PSL) is loaded.
|
|
179
|
+
* @return true if the PSL is loaded, false otherwise.
|
|
180
|
+
*/
|
|
181
|
+
static bool isPslLoaded() noexcept;
|
|
182
|
+
static std::string_view removeWWW(const std::string_view&) noexcept;
|
|
183
|
+
|
|
184
|
+
public:
|
|
185
|
+
Host(const std::string& host, const bool ignore_www = DEFAULT_IGNORE_WWW);
|
|
186
|
+
Host() = default;
|
|
187
|
+
|
|
188
|
+
bool operator==(const Host&) const;
|
|
189
|
+
bool operator==(const std::string&) const;
|
|
190
|
+
|
|
191
|
+
const std::string& suffix() const noexcept;
|
|
192
|
+
const std::string& domain() const noexcept;
|
|
193
|
+
std::string domainName() const noexcept;
|
|
194
|
+
const std::string& subdomain() const noexcept;
|
|
195
|
+
const std::string& fulldomain() const noexcept;
|
|
196
|
+
const std::string& str() const noexcept;
|
|
197
|
+
|
|
198
|
+
private:
|
|
199
|
+
class Impl;
|
|
200
|
+
std::shared_ptr<Impl> impl; // since all methods are constants
|
|
201
|
+
};
|
|
202
|
+
} // namespace TLD
|
|
203
|
+
|
|
204
|
+
std::ostream& operator<<(std::ostream& os, const TLD::QueryParams& dt);
|
|
205
|
+
|
|
206
|
+
std::ostream& operator<<(std::ostream& os, const TLD::Url& dt);
|
|
207
|
+
|
|
208
|
+
std::ostream& operator<<(std::ostream& os, const TLD::Host& dt);
|
|
209
|
+
#endif // TLD_URLPARSER_H
|
|
Binary file
|
|
Binary file
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
#ifndef DOMAINEXTRACTOR_URLPARSER_H
|
|
2
|
-
#define DOMAINEXTRACTOR_URLPARSER_H
|
|
3
|
-
|
|
4
|
-
#include <iostream>
|
|
5
|
-
#include <memory>
|
|
6
|
-
#include <string>
|
|
7
|
-
#include <vector>
|
|
8
|
-
|
|
9
|
-
#ifndef PUBLIC_SUFFIX_LIST_DAT
|
|
10
|
-
#define PUBLIC_SUFFIX_LIST_DAT "public_suffix_list.dat"
|
|
11
|
-
#endif
|
|
12
|
-
|
|
13
|
-
namespace TLD {
|
|
14
|
-
constexpr bool DEFAULT_IGNORE_WWW = false;
|
|
15
|
-
|
|
16
|
-
using QueryParams = std::vector<std::string>;
|
|
17
|
-
|
|
18
|
-
class Host;
|
|
19
|
-
|
|
20
|
-
class Url {
|
|
21
|
-
public:
|
|
22
|
-
static bool isPslLoaded() noexcept;
|
|
23
|
-
static std::string extractHost(const std::string& url) noexcept;
|
|
24
|
-
|
|
25
|
-
public:
|
|
26
|
-
Url(const std::string& url, const bool ignore_www = DEFAULT_IGNORE_WWW);
|
|
27
|
-
Url() noexcept = default;
|
|
28
|
-
|
|
29
|
-
bool operator==(const Url&) const;
|
|
30
|
-
|
|
31
|
-
std::string str() const noexcept;
|
|
32
|
-
const std::string& protocol() const noexcept;
|
|
33
|
-
const std::string& subdomain() const noexcept;
|
|
34
|
-
const std::string& domain() const noexcept;
|
|
35
|
-
const std::string& suffix() const noexcept;
|
|
36
|
-
const std::string& query() const noexcept;
|
|
37
|
-
const std::string& fragment() const noexcept;
|
|
38
|
-
const std::string& userinfo() const noexcept;
|
|
39
|
-
std::string abspath() const noexcept;
|
|
40
|
-
std::string domainName() const noexcept;
|
|
41
|
-
const std::string& fulldomain() const noexcept;
|
|
42
|
-
const int port() const noexcept;
|
|
43
|
-
QueryParams params() const noexcept;
|
|
44
|
-
const Host& host() const;
|
|
45
|
-
|
|
46
|
-
private:
|
|
47
|
-
class Impl;
|
|
48
|
-
std::shared_ptr<Impl> impl; // since all methods are constants
|
|
49
|
-
};
|
|
50
|
-
|
|
51
|
-
class Host {
|
|
52
|
-
public:
|
|
53
|
-
static Host fromUrl(const std::string& url,
|
|
54
|
-
const bool ignore_www = DEFAULT_IGNORE_WWW);
|
|
55
|
-
static void loadPslFromPath(const std::string& filepath);
|
|
56
|
-
static void loadPslFromString(const std::string& filestr);
|
|
57
|
-
static bool isPslLoaded() noexcept;
|
|
58
|
-
static std::string_view removeWWW(const std::string_view&) noexcept;
|
|
59
|
-
|
|
60
|
-
public:
|
|
61
|
-
Host(const std::string& host, const bool ignore_www = DEFAULT_IGNORE_WWW);
|
|
62
|
-
Host() = default;
|
|
63
|
-
|
|
64
|
-
bool operator==(const Host&) const;
|
|
65
|
-
bool operator==(const std::string&) const;
|
|
66
|
-
|
|
67
|
-
const std::string& suffix() const noexcept;
|
|
68
|
-
const std::string& domain() const noexcept;
|
|
69
|
-
std::string domainName() const noexcept;
|
|
70
|
-
const std::string& subdomain() const noexcept;
|
|
71
|
-
const std::string& fulldomain() const noexcept;
|
|
72
|
-
const std::string& str() const noexcept;
|
|
73
|
-
|
|
74
|
-
private:
|
|
75
|
-
class Impl;
|
|
76
|
-
std::shared_ptr<Impl> impl; // since all methods are constants
|
|
77
|
-
};
|
|
78
|
-
} // namespace TLD
|
|
79
|
-
|
|
80
|
-
std::ostream& operator<<(std::ostream& os, const TLD::QueryParams& dt);
|
|
81
|
-
|
|
82
|
-
std::ostream& operator<<(std::ostream& os, const TLD::Url& dt);
|
|
83
|
-
|
|
84
|
-
std::ostream& operator<<(std::ostream& os, const TLD::Host& dt);
|
|
85
|
-
#endif // DOMAINEXTRACTOR_URLPARSER_H
|
liburlparser-1.4.4/noxfile.py
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
#!/bin/python3
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import nox
|
|
5
|
-
|
|
6
|
-
nox.options.sessions = ["lint", "tests"]
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
@nox.session
|
|
10
|
-
def lint(session: nox.Session) -> None:
|
|
11
|
-
"""
|
|
12
|
-
Run the linter.
|
|
13
|
-
"""
|
|
14
|
-
session.install("pre-commit")
|
|
15
|
-
session.run("pre-commit", "run", "--all-files", *session.posargs)
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@nox.session
|
|
19
|
-
def tests(session: nox.Session) -> None:
|
|
20
|
-
"""
|
|
21
|
-
Run the unit and regular tests.
|
|
22
|
-
"""
|
|
23
|
-
session.install(".[test]")
|
|
24
|
-
session.run("pytest", *session.posargs)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|