fast-url-parser 1.0.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -0,0 +1,23 @@
1
+ Metadata-Version: 2.4
2
+ Name: fast-url-parser
3
+ Version: 1.0.14
4
+ Summary: fast url parser for zsodata
5
+ Home-page: http://www.zsodata.com
6
+ Author: zsodata
7
+ Author-email: team@zso.io
8
+ License: BSD License
9
+ Platform: all
10
+ Requires-Python: >=3.7
11
+ Dynamic: author
12
+ Dynamic: author-email
13
+ Dynamic: description
14
+ Dynamic: home-page
15
+ Dynamic: license
16
+ Dynamic: platform
17
+ Dynamic: requires-python
18
+ Dynamic: summary
19
+
20
+ pip uninstall fast-url-parser
21
+
22
+ pip install --upgrade fast-url-parser -i https://pypi.python.org/simple
23
+ pip show fast-url-parser
@@ -0,0 +1,4 @@
1
+ pip uninstall fast-url-parser
2
+
3
+ pip install --upgrade fast-url-parser -i https://pypi.python.org/simple
4
+ pip show fast-url-parser
@@ -0,0 +1 @@
1
+ from __future__ import absolute_import
@@ -0,0 +1,35 @@
1
+ """
2
+ 配置管理模块
3
+ 从环境变量中读取配置,如果未设置则使用默认值
4
+ """
5
+
6
+ import os
7
+
8
+
9
+ # 默认的meta服务基础地址
10
+ DEFAULT_BASE_URL = "https://meta-api.zcbot.cn/api/v1/meta"
11
+
12
+ # 默认的更新周期1小时
13
+ DEFAULT_UPDATE_DELAY = 3600
14
+
15
+
16
+ def get_base_url():
17
+ """
18
+ 获取meta服务基础地址
19
+ 优先从环境变量 META_API_BASE_URL 读取
20
+ 如果未设置则使用默认值
21
+ """
22
+ return os.environ.get("META_API_BASE_URL", DEFAULT_BASE_URL)
23
+
24
+
25
+ def get_update_delay():
26
+ """
27
+ 获取规则更新周期(秒)
28
+ 优先从环境变量 META_UPDATE_DELAY 读取
29
+ 如果未设置则使用默认值 3600秒(1小时)
30
+ """
31
+ try:
32
+ delay = int(os.environ.get("META_UPDATE_DELAY", DEFAULT_UPDATE_DELAY))
33
+ return delay
34
+ except (ValueError, TypeError):
35
+ return DEFAULT_UPDATE_DELAY
@@ -0,0 +1,47 @@
1
+ import logging
2
+ from typing import Union
3
+
4
+ from .config import get_base_url
5
+ from .utils import singleton, http_get_list, TimerRefreshMixin
6
+
7
+
8
+ @singleton
9
+ class FileNameFormatter(TimerRefreshMixin, object):
10
+ """
11
+ 文件命名格式配置
12
+ """
13
+ logger = logging.getLogger(__name__)
14
+
15
+ def __init__(self):
16
+ self.config_map = dict()
17
+ self._timer = None
18
+ self._load_configs()
19
+ self._start_timer()
20
+
21
+ def format_name(self, config_key: str, sn: str, row_id: str, index: str, sort: str, padding_index: str, padding_sort: str, image_type: str) -> Union[str, tuple]:
22
+ _format = None
23
+ if config_key:
24
+ file_name_config = self.config_map.get(config_key) or {}
25
+ # 1、优先按index读取mapper的配置
26
+ mapper = file_name_config.get(f'{image_type}_mapper', {}) or file_name_config.get('mapper', {}) or {}
27
+ _format = mapper.get(str(index), None) or mapper.get('ALL') or None
28
+ # 2、如果mapper无对应index的配置(或无mapper),读取default字段
29
+ if not _format:
30
+ _format = file_name_config.get('default', None) or None
31
+ # 3、如以上配置均无,使用固定默认格式
32
+ if not _format:
33
+ _format = '%(row_id)s-%(sort)s'
34
+
35
+ return _format % {'sn': sn, 'row_id': row_id, 'index': index, 'sort': sort, 'padding_index': padding_index, 'padding_sort': padding_sort}
36
+
37
+ def _refresh_data(self):
38
+ self._load_configs()
39
+
40
+ def _load_configs(self):
41
+ url = f'{get_base_url()}/file-name-configs'
42
+ rows = http_get_list(url)
43
+ new_map = {}
44
+ for row in rows:
45
+ new_map[row.get('_id')] = row
46
+ self.config_map = new_map
47
+ self.logger.info(f'有效文件名规则: {len(self.config_map)}条')
@@ -0,0 +1,36 @@
1
+ import logging
2
+ from typing import Dict
3
+
4
+ from .config import get_base_url
5
+ from .utils import singleton, http_get_list, TimerRefreshMixin
6
+
7
+
8
+ @singleton
9
+ class ImageThumbConfig(TimerRefreshMixin, object):
10
+ """
11
+ 缩略图尺寸配置
12
+ """
13
+ logger = logging.getLogger(__name__)
14
+
15
+ def __init__(self):
16
+ self.config_map = dict()
17
+ self._timer = None
18
+ self._load_configs()
19
+ self._start_timer()
20
+
21
+ def get_thumbs(self, config_key: str) -> Dict:
22
+ if config_key and config_key in self.config_map:
23
+ return self.config_map.get(config_key).get('thumbs', {}) or {}
24
+ return {}
25
+
26
+ def _refresh_data(self):
27
+ self._load_configs()
28
+
29
+ def _load_configs(self):
30
+ url = f'{get_base_url()}/image-thumb-configs'
31
+ rows = http_get_list(url)
32
+ new_map = {}
33
+ for row in rows:
34
+ new_map[row.get('_id')] = row
35
+ self.config_map = new_map
36
+ self.logger.info(f'缩略图尺寸配置规则: {len(self.config_map)}条')
@@ -0,0 +1,129 @@
1
+ import logging
2
+ import urllib.parse as parser
3
+ from typing import Union
4
+ from pydantic import BaseModel
5
+ from .rule import RuleHolder
6
+ from .utils import clean_url
7
+
8
+ LOGGER = logging.getLogger(__name__)
9
+ rule_holder = RuleHolder()
10
+
11
+
12
+ class UrlModel(BaseModel):
13
+ """
14
+ 通用基础数据模型
15
+ """
16
+ # 链接序列号(全局唯一) 如:jd:5129155、tmall:576748721316,3985068128611
17
+ link_sn: str = None
18
+ # 网站编码 如:jd
19
+ plat_code: str = None
20
+ # 网站名称 如:京东
21
+ plat_name: str = None
22
+ # 链接商品编码(多编码链接,编码之间逗号分隔) 如:5129155、576748721316,3985068128611
23
+ ec_sku_id: Union[str, tuple] = None
24
+
25
+
26
+ def parse_url(url) -> Union[UrlModel, None]:
27
+ """
28
+ 解析url链接
29
+ 有效链接返回UrlModel,无效链接返回None
30
+ """
31
+ plat_code, plat_name, ec_sku_id = _match_url(url)
32
+ if plat_code and ec_sku_id:
33
+ link_sn = _build_link_sn(plat_code, ec_sku_id)
34
+ return UrlModel(link_sn=link_sn, plat_code=plat_code, plat_name=plat_name, ec_sku_id=ec_sku_id)
35
+
36
+ return None
37
+
38
+
39
+ def _build_link_sn(plat_code, ec_sku_id):
40
+ """
41
+ 构建link_sn编码规则
42
+ """
43
+ if plat_code and ec_sku_id:
44
+ return f'{plat_code}:{ec_sku_id}'
45
+
46
+ return None
47
+
48
+
49
+ def _match_url(url):
50
+ plat_code = ''
51
+ plat_name = ''
52
+ ec_sku_id = ''
53
+ _url = clean_url(url)
54
+ if _url:
55
+ host = parser.urlparse(_url).hostname
56
+ rule = rule_holder.get_rule(host)
57
+ if host and rule:
58
+ plat_code = rule.get('plat_code')
59
+ plat_name = rule.get('plat_name')
60
+ sku_param = rule.get('sku_param', [])
61
+ if sku_param:
62
+ ec_sku_id = _parse_sku_by_param(_url, sku_param)
63
+ patterns = rule.get('patterns', [])
64
+ if patterns and not ec_sku_id:
65
+ ec_sku_id = _parse_sku_by_path(_url, patterns)
66
+
67
+ return plat_code, plat_name, ec_sku_id
68
+
69
+
70
+ def _match_plat_code(url, default='*'):
71
+ plat_code = default
72
+ if url:
73
+ host = parser.urlparse(url).hostname
74
+ rule = rule_holder.get_rule(host)
75
+ if host and rule:
76
+ plat_code = rule.get('plat_code')
77
+
78
+ return plat_code
79
+
80
+
81
+ def _parse_sku_by_path(url, patterns, token=','):
82
+ """
83
+ 根据url中path部分解析skuId
84
+ :param url:
85
+ :param patterns:
86
+ :return:
87
+ """
88
+ try:
89
+ for ptn in patterns:
90
+ arr = ptn.findall(url.strip())
91
+ if len(arr):
92
+ rs = arr[0]
93
+ if token and isinstance(rs, tuple):
94
+ return token.join(rs)
95
+ else:
96
+ return rs
97
+ except Exception as e:
98
+ LOGGER.error('match sku error[parse_sku_by_path]: url=%s, ex=%s' % (url, e))
99
+
100
+ return ''
101
+
102
+
103
+ def _parse_sku_by_param(url, sku_param, token=','):
104
+ """
105
+ 根据url中param部分解析skuId
106
+ :param url:
107
+ :param sku_param:
108
+ :return:
109
+ """
110
+ try:
111
+ params = parser.parse_qs(parser.urlparse(url).query)
112
+
113
+ rs_list = []
114
+ if len(params) and sku_param:
115
+ for param in sku_param:
116
+ if param in params and params.get(param)[0].strip():
117
+ rs_list.append(str(params.get(param)[0].strip()))
118
+
119
+ if token:
120
+ return token.join(rs_list)
121
+ else:
122
+ if len(rs_list) == 1:
123
+ return rs_list[0]
124
+ else:
125
+ return tuple(rs_list)
126
+ except Exception as e:
127
+ LOGGER.error('match sku error[parse_sku_by_param]: url=%s, ex=%s' % (url, e))
128
+
129
+ return ''
@@ -0,0 +1,64 @@
1
+ import re
2
+ import logging
3
+ from .config import get_base_url
4
+ from .utils import singleton, http_get_list, TimerRefreshMixin
5
+
6
+
7
+ @singleton
8
+ class RuleHolder(TimerRefreshMixin, object):
9
+ """
10
+ 常见电商平台商品编码解析规则
11
+ """
12
+ logger = logging.getLogger(__name__)
13
+
14
+ def __init__(self):
15
+ self.rule_map = {}
16
+ self._timer = None
17
+ self.reload()
18
+ self._start_timer()
19
+
20
+ def get_rule(self, host):
21
+ return self.rule_map.get(host)
22
+
23
+ def _refresh_data(self):
24
+ self.reload()
25
+
26
+ def reload(self):
27
+ """
28
+ 加载最新规则,可挂到定时任务上定期更新
29
+ :return:
30
+ """
31
+ plats = self._fetch_platforms()
32
+ plat_map = {}
33
+ for plat in plats:
34
+ plat_map[plat.get('_id')] = plat.get('name')
35
+
36
+ rows = self._fetch_url_rules()
37
+ new_map = {}
38
+ for row in rows:
39
+ regex_list = row.get('regex', [])
40
+ patterns = []
41
+ for regex in regex_list:
42
+ try:
43
+ patterns.append(re.compile(regex))
44
+ except re.error:
45
+ self.logger.error(f'错误规则: regex={regex}, row={row}')
46
+
47
+ plat_code = row.get('plat_code')
48
+ plat_name = row.get('plat_name', None) or plat_map.get(plat_code) or plat_code
49
+ new_map[row.get('_id')] = {
50
+ 'plat_code': plat_code,
51
+ 'plat_name': plat_name,
52
+ 'sku_param': row.get('params', []),
53
+ 'patterns': patterns,
54
+ }
55
+ self.rule_map = new_map
56
+ self.logger.info(f'更新链接分拣规则: {len(self.rule_map)}条')
57
+
58
+ def _fetch_platforms(self):
59
+ url = f'{get_base_url()}/platforms'
60
+ return http_get_list(url)
61
+
62
+ def _fetch_url_rules(self):
63
+ url = f'{get_base_url()}/url-parser-rule'
64
+ return http_get_list(url)
@@ -0,0 +1,65 @@
1
+ import re
2
+ import logging
3
+ import urllib.parse as parser
4
+ from typing import List, Union
5
+ from .config import get_base_url
6
+ from .parser import _parse_sku_by_param, _parse_sku_by_path
7
+ from .utils import singleton, clean_url, http_get_list, TimerRefreshMixin
8
+
9
+ LOGGER = logging.getLogger(__name__)
10
+
11
+
12
+ @singleton
13
+ class SpiderUrlParser(TimerRefreshMixin, object):
14
+ """
15
+ 常见电商平台商品编码解析规则
16
+ """
17
+ logger = logging.getLogger(__name__)
18
+
19
+ def __init__(self, spider_id: Union[str, List]):
20
+ self.spider_id = spider_id
21
+ self.rule_map = dict()
22
+ self._timer = None
23
+ self._load_rules()
24
+ self._start_timer()
25
+
26
+ def parse(self, url) -> Union[str, tuple]:
27
+ ec_sku_id = ''
28
+ _url = clean_url(url)
29
+ if _url:
30
+ host = parser.urlparse(_url).hostname
31
+ rule = self.rule_map.get(host)
32
+ if host and rule:
33
+ sku_param = rule.get('sku_param', [])
34
+ if sku_param:
35
+ ec_sku_id = _parse_sku_by_param(_url, sku_param, None)
36
+ patterns = rule.get('patterns', [])
37
+ if patterns and not ec_sku_id:
38
+ ec_sku_id = _parse_sku_by_path(_url, patterns, None)
39
+
40
+ return ec_sku_id
41
+
42
+ def _refresh_data(self):
43
+ self._load_rules()
44
+
45
+ def _load_rules(self):
46
+ url = f'{get_base_url()}/spiders/rules?spiderId={self.spider_id}'
47
+ rows = http_get_list(url)
48
+ new_map = {}
49
+ for row in rows:
50
+ regex_list = row.get('regex', [])
51
+ patterns = []
52
+ for regex in regex_list:
53
+ try:
54
+ patterns.append(re.compile(regex))
55
+ except re.error:
56
+ LOGGER.error(f'错误规则: regex={regex}, row={row}')
57
+
58
+ plat_code = row.get('plat_code')
59
+ new_map[row.get('_id')] = {
60
+ 'plat_code': plat_code,
61
+ 'sku_param': row.get('params', []),
62
+ 'patterns': patterns,
63
+ }
64
+ self.rule_map = new_map
65
+ self.logger.info(f'有效链接解析规则: {len(self.rule_map)}条')
@@ -0,0 +1,97 @@
1
+ import json
2
+ import logging
3
+ import threading
4
+ from functools import wraps
5
+ from urllib.error import URLError
6
+ from urllib.request import Request, urlopen
7
+ from .config import get_update_delay
8
+
9
+ LOGGER = logging.getLogger(__name__)
10
+
11
+
12
+ class TimerRefreshMixin:
13
+ """
14
+ 定时刷新混入类,提供同步首次加载 + 异步定时刷新能力
15
+ 子类需实现 _refresh_data() 方法,并在 __init__ 中按以下顺序调用:
16
+ self._timer = None
17
+ self._refresh_data() # 同步首次加载
18
+ self._start_timer() # 启动定时刷新
19
+ """
20
+
21
+ def _refresh_data(self):
22
+ """子类覆盖此方法实现数据加载逻辑"""
23
+ pass
24
+
25
+ def _start_timer(self):
26
+ """启动定时更新定时器"""
27
+ if self._timer:
28
+ self._timer.cancel()
29
+ delay = get_update_delay()
30
+ self._timer = threading.Timer(delay, self._timer_callback)
31
+ self._timer.daemon = True
32
+ self._timer.start()
33
+
34
+ def _timer_callback(self):
35
+ """定时器回调:刷新数据并重启定时器"""
36
+ self._refresh_data()
37
+ self._start_timer()
38
+
39
+ def stop(self):
40
+ """停止定时更新"""
41
+ if self._timer:
42
+ self._timer.cancel()
43
+ self._timer = None
44
+
45
+
46
+ def singleton(cls):
47
+ _instance = {}
48
+
49
+ @wraps(cls)
50
+ def _singlenton(*args, **kargs):
51
+ if cls not in _instance:
52
+ _instance[cls] = cls(*args, **kargs)
53
+ return _instance[cls]
54
+
55
+ return _singlenton
56
+
57
+
58
+ def clean_url(url):
59
+ """
60
+ 链接非法字符清理
61
+ """
62
+ fix_err = {
63
+ '&': '&',
64
+ 'amp;': '',
65
+ '&': '&',
66
+ ' ': ''
67
+ }
68
+ _url = url
69
+ if _url:
70
+ for key in fix_err:
71
+ if key in _url:
72
+ _url = _url.replace(key, fix_err.get(key))
73
+
74
+ return _url
75
+
76
+
77
+ def http_get_list(url, timeout=5):
78
+ """
79
+ 调HTTP GET接口,返回data列表
80
+ 请求失败返回空列表
81
+ """
82
+ try:
83
+ req = Request(url)
84
+ req.add_header('User-Agent', 'UrlParser/1.0')
85
+ req.add_header('Accept', 'application/json')
86
+
87
+ with urlopen(req, timeout=timeout) as response:
88
+ if response.status == 200:
89
+ data = json.loads(response.read().decode('utf-8'))
90
+ if data.get("success") and data.get("data"):
91
+ return data["data"]
92
+ except URLError as e:
93
+ LOGGER.error(f'请求失败: url={url}, error={e}')
94
+ except Exception as e:
95
+ LOGGER.error(f'请求异常: url={url}, error={e}')
96
+
97
+ return []
@@ -0,0 +1,23 @@
1
+ Metadata-Version: 2.4
2
+ Name: fast-url-parser
3
+ Version: 1.0.14
4
+ Summary: fast url parser for zsodata
5
+ Home-page: http://www.zsodata.com
6
+ Author: zsodata
7
+ Author-email: team@zso.io
8
+ License: BSD License
9
+ Platform: all
10
+ Requires-Python: >=3.7
11
+ Dynamic: author
12
+ Dynamic: author-email
13
+ Dynamic: description
14
+ Dynamic: home-page
15
+ Dynamic: license
16
+ Dynamic: platform
17
+ Dynamic: requires-python
18
+ Dynamic: summary
19
+
20
+ pip uninstall fast-url-parser
21
+
22
+ pip install --upgrade fast-url-parser -i https://pypi.python.org/simple
23
+ pip show fast-url-parser
@@ -0,0 +1,16 @@
1
+ MANIFEST.in
2
+ README.rst
3
+ setup.py
4
+ fast_url_parser/__init__.py
5
+ fast_url_parser/config.py
6
+ fast_url_parser/file_name_config.py
7
+ fast_url_parser/image_thumb_config.py
8
+ fast_url_parser/parser.py
9
+ fast_url_parser/rule.py
10
+ fast_url_parser/spider.py
11
+ fast_url_parser/utils.py
12
+ fast_url_parser.egg-info/PKG-INFO
13
+ fast_url_parser.egg-info/SOURCES.txt
14
+ fast_url_parser.egg-info/dependency_links.txt
15
+ fast_url_parser.egg-info/not-zip-safe
16
+ fast_url_parser.egg-info/top_level.txt
@@ -0,0 +1 @@
1
+ fast_url_parser
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,21 @@
1
+ from distutils.core import setup
2
+ from setuptools import find_packages
3
+
4
+ with open('README.rst', 'r') as f:
5
+ long_description = f.read()
6
+
7
+ setup(name='fast-url-parser',
8
+ version='1.0.14',
9
+ description='fast url parser for zsodata',
10
+ long_description=long_description,
11
+ author='zsodata',
12
+ author_email='team@zso.io',
13
+ url='http://www.zsodata.com',
14
+ python_requires='>=3.7',
15
+ license='BSD License',
16
+ packages=find_packages(),
17
+ platforms=['all'],
18
+ include_package_data=True,
19
+ zip_safe=False,
20
+ )
21
+ # zso@2022