igntui 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- igntui/__init__.py +57 -0
- igntui/__main__.py +10 -0
- igntui/app.py +37 -0
- igntui/cli/__init__.py +31 -0
- igntui/cli/base.py +104 -0
- igntui/cli/commands/__init__.py +17 -0
- igntui/cli/commands/cache_cmd.py +95 -0
- igntui/cli/commands/generate_cmd.py +85 -0
- igntui/cli/commands/list_cmd.py +66 -0
- igntui/cli/commands/test_cmd.py +64 -0
- igntui/cli/commands/tui_cmd.py +34 -0
- igntui/cli/parser.py +162 -0
- igntui/cli/setup.py +71 -0
- igntui/core/__init__.py +8 -0
- igntui/core/api/__init__.py +16 -0
- igntui/core/api/client.py +222 -0
- igntui/core/api/errors.py +29 -0
- igntui/core/api/rate_limiter.py +30 -0
- igntui/core/api/request_handler.py +124 -0
- igntui/core/api/response.py +16 -0
- igntui/core/cache.py +284 -0
- igntui/core/config.py +174 -0
- igntui/core/search.py +354 -0
- igntui/main.py +74 -0
- igntui/tui/__init__.py +22 -0
- igntui/tui/actions.py +254 -0
- igntui/tui/app.py +246 -0
- igntui/tui/curses_setup.py +88 -0
- igntui/tui/event_handler.py +310 -0
- igntui/tui/lifecycle.py +102 -0
- igntui/tui/renderer.py +108 -0
- igntui/tui/state.py +77 -0
- igntui/ui/__init__.py +33 -0
- igntui/ui/components/__init__.py +31 -0
- igntui/ui/components/base_dialog.py +75 -0
- igntui/ui/components/base_panel.py +85 -0
- igntui/ui/components/confirm_dialog.py +100 -0
- igntui/ui/components/content_panel.py +77 -0
- igntui/ui/components/export_dialog.py +139 -0
- igntui/ui/components/help_dialog.py +161 -0
- igntui/ui/components/save_dialog.py +109 -0
- igntui/ui/components/search_panel.py +52 -0
- igntui/ui/components/selected_panel.py +107 -0
- igntui/ui/components/splash.py +236 -0
- igntui/ui/components/status_bar.py +78 -0
- igntui/ui/components/templates_panel.py +153 -0
- igntui/utils/__init__.py +19 -0
- igntui/utils/logging.py +288 -0
- igntui-0.0.0.dist-info/METADATA +425 -0
- igntui-0.0.0.dist-info/RECORD +52 -0
- igntui-0.0.0.dist-info/WHEEL +4 -0
- igntui-0.0.0.dist-info/entry_points.txt +4 -0
igntui/cli/parser.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-\
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
from .. import __description__, __version__, get_version_string
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def create_base_parser() -> argparse.ArgumentParser:
|
|
13
|
+
parser = argparse.ArgumentParser(
|
|
14
|
+
prog="igntui",
|
|
15
|
+
description=__description__,
|
|
16
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
17
|
+
epilog=f"""
|
|
18
|
+
igntui v{__version__} - Interactive GitIgnore Template User Interface
|
|
19
|
+
|
|
20
|
+
Examples:
|
|
21
|
+
igntui # Launch interactive TUI
|
|
22
|
+
igntui list # List available templates
|
|
23
|
+
igntui generate python # Generate .gitignore for Python
|
|
24
|
+
igntui tui # Explicitly launch TUI mode
|
|
25
|
+
|
|
26
|
+
For more information, visit: https://github.com/toptal/gitignore.io
|
|
27
|
+
""",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
parser.add_argument(
|
|
31
|
+
"--version", "-v", action="version", version=get_version_string()
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"--verbose", action="store_true", help="Enable verbose/debug output"
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
parser.add_argument(
|
|
39
|
+
"--log-level",
|
|
40
|
+
"-l",
|
|
41
|
+
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
|
42
|
+
help="Set specific logging level",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
parser.add_argument(
|
|
46
|
+
"--config", "-c", type=Path, help="Path to custom configuration file"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
parser.add_argument(
|
|
50
|
+
"--no-cache", action="store_true", help="Disable caching for this session"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
return parser
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def create_command_parser(
|
|
57
|
+
base_parser: argparse.ArgumentParser,
|
|
58
|
+
) -> argparse.ArgumentParser:
|
|
59
|
+
subparsers = base_parser.add_subparsers(
|
|
60
|
+
dest="command",
|
|
61
|
+
title="commands",
|
|
62
|
+
description="Available commands",
|
|
63
|
+
help="Command to execute",
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
tui_parser = subparsers.add_parser(
|
|
67
|
+
"tui",
|
|
68
|
+
help="Launch interactive TUI (default)",
|
|
69
|
+
description="Launch the interactive Terminal User Interface",
|
|
70
|
+
)
|
|
71
|
+
tui_parser.add_argument(
|
|
72
|
+
"--no-splash", action="store_true", help="Skip the splash screen"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
list_parser = subparsers.add_parser(
|
|
76
|
+
"list",
|
|
77
|
+
help="List available templates",
|
|
78
|
+
description="List all available .gitignore templates",
|
|
79
|
+
)
|
|
80
|
+
list_parser.add_argument(
|
|
81
|
+
"--filter", "-f", metavar="PATTERN", help="Filter templates by pattern"
|
|
82
|
+
)
|
|
83
|
+
list_parser.add_argument(
|
|
84
|
+
"--count", "-c", action="store_true", help="Show count only"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
generate_parser = subparsers.add_parser(
|
|
88
|
+
"generate",
|
|
89
|
+
help="Generate .gitignore content",
|
|
90
|
+
description="Generate .gitignore content for specified templates",
|
|
91
|
+
)
|
|
92
|
+
generate_parser.add_argument(
|
|
93
|
+
"templates", nargs="+", help="Template names to generate (space-separated)"
|
|
94
|
+
)
|
|
95
|
+
generate_parser.add_argument(
|
|
96
|
+
"--output",
|
|
97
|
+
"-o",
|
|
98
|
+
type=Path,
|
|
99
|
+
metavar="FILE",
|
|
100
|
+
help="Output file path (default: stdout)",
|
|
101
|
+
)
|
|
102
|
+
generate_parser.add_argument(
|
|
103
|
+
"--append",
|
|
104
|
+
"-a",
|
|
105
|
+
action="store_true",
|
|
106
|
+
help="Append to existing file instead of overwriting",
|
|
107
|
+
)
|
|
108
|
+
generate_parser.add_argument(
|
|
109
|
+
"--force",
|
|
110
|
+
"-f",
|
|
111
|
+
action="store_true",
|
|
112
|
+
help="Force overwrite without confirmation",
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
cache_parser = subparsers.add_parser(
|
|
116
|
+
"cache", help="Manage cache", description="Manage template and API cache"
|
|
117
|
+
)
|
|
118
|
+
cache_subparsers = cache_parser.add_subparsers(
|
|
119
|
+
dest="cache_action", title="cache actions", help="Cache action to perform"
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
cache_subparsers.add_parser("clear", help="Clear all cache")
|
|
123
|
+
cache_subparsers.add_parser("stats", help="Show cache statistics")
|
|
124
|
+
cache_subparsers.add_parser("info", help="Show cache information")
|
|
125
|
+
|
|
126
|
+
test_parser = subparsers.add_parser(
|
|
127
|
+
"test",
|
|
128
|
+
help="Test API connection",
|
|
129
|
+
description="Test connection to gitignore.io API",
|
|
130
|
+
)
|
|
131
|
+
test_parser.add_argument(
|
|
132
|
+
"--timeout",
|
|
133
|
+
type=int,
|
|
134
|
+
default=10,
|
|
135
|
+
help="Connection timeout in seconds (default: 10)",
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
return base_parser
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def get_command_instance(command_name: str, cli_instance):
|
|
142
|
+
from .commands import (
|
|
143
|
+
CacheCommand,
|
|
144
|
+
GenerateCommand,
|
|
145
|
+
ListCommand,
|
|
146
|
+
TestCommand,
|
|
147
|
+
TUICommand,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
commands = {
|
|
151
|
+
"list": ListCommand,
|
|
152
|
+
"generate": GenerateCommand,
|
|
153
|
+
"tui": TUICommand,
|
|
154
|
+
"cache": CacheCommand,
|
|
155
|
+
"test": TestCommand,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
command_class = commands.get(command_name)
|
|
159
|
+
if command_class:
|
|
160
|
+
return command_class(cli_instance)
|
|
161
|
+
|
|
162
|
+
return None
|
igntui/cli/setup.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from logging.handlers import RotatingFileHandler
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
from ..core.config import config
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def setup_logging(verbose: bool = False, log_level: Optional[str] = None) -> None:
|
|
13
|
+
if log_level:
|
|
14
|
+
level = getattr(logging, log_level.upper(), logging.INFO)
|
|
15
|
+
elif verbose:
|
|
16
|
+
level = logging.DEBUG
|
|
17
|
+
else:
|
|
18
|
+
log_config = config.logging_config
|
|
19
|
+
level = getattr(logging, log_config["level"].upper(), logging.INFO)
|
|
20
|
+
|
|
21
|
+
formatter = logging.Formatter(
|
|
22
|
+
"%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
|
23
|
+
datefmt="%Y-%m-%d %H:%M:%S",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
handlers = []
|
|
27
|
+
|
|
28
|
+
log_config = config.logging_config
|
|
29
|
+
if log_config["file_enabled"]:
|
|
30
|
+
try:
|
|
31
|
+
log_file = config.get_log_file()
|
|
32
|
+
log_file.parent.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
|
|
34
|
+
file_handler = RotatingFileHandler(
|
|
35
|
+
log_file,
|
|
36
|
+
maxBytes=log_config["max_file_size"],
|
|
37
|
+
backupCount=log_config["backup_count"],
|
|
38
|
+
encoding="utf-8",
|
|
39
|
+
)
|
|
40
|
+
file_handler.setFormatter(formatter)
|
|
41
|
+
file_handler.setLevel(level)
|
|
42
|
+
handlers.append(file_handler)
|
|
43
|
+
|
|
44
|
+
except (OSError, PermissionError) as e:
|
|
45
|
+
print(f"Warning: Could not set up file logging: {e}")
|
|
46
|
+
|
|
47
|
+
if log_config["console_enabled"] or not handlers:
|
|
48
|
+
console_handler = logging.StreamHandler()
|
|
49
|
+
console_handler.setFormatter(formatter)
|
|
50
|
+
console_handler.setLevel(level)
|
|
51
|
+
handlers.append(console_handler)
|
|
52
|
+
|
|
53
|
+
logging.basicConfig(level=level, handlers=handlers, force=True)
|
|
54
|
+
|
|
55
|
+
logger = logging.getLogger(__name__)
|
|
56
|
+
logger.debug(f"Logging configured at {logging.getLevelName(level)} level")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def check_curses_availability() -> bool:
|
|
60
|
+
try:
|
|
61
|
+
import curses
|
|
62
|
+
|
|
63
|
+
return True
|
|
64
|
+
except ImportError:
|
|
65
|
+
return False
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def print_curses_error() -> None:
|
|
69
|
+
print("Error: curses module not available.")
|
|
70
|
+
print("On Windows, install: pip install windows-curses")
|
|
71
|
+
print("On Linux/Mac, curses should be included with Python.")
|
igntui/core/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
from .client import GitIgnoreAPI
|
|
6
|
+
from .response import APIResponse
|
|
7
|
+
from .errors import APIError, NetworkError, RateLimitError, ServiceUnavailableError
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"GitIgnoreAPI",
|
|
11
|
+
"APIResponse",
|
|
12
|
+
"APIError",
|
|
13
|
+
"NetworkError",
|
|
14
|
+
"RateLimitError",
|
|
15
|
+
"ServiceUnavailableError",
|
|
16
|
+
]
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
from typing import Any, Dict, List, Optional
|
|
7
|
+
|
|
8
|
+
from ..cache import CacheManager, TemplateCache
|
|
9
|
+
from ..config import config
|
|
10
|
+
from .request_handler import RequestHandler
|
|
11
|
+
from .response import APIResponse
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class GitIgnoreAPI:
|
|
17
|
+
def __init__(self, cache_manager: Optional[CacheManager] = None):
|
|
18
|
+
self.base_url = config.get("api", "base_url")
|
|
19
|
+
self.timeout = config.get("api", "timeout")
|
|
20
|
+
self.user_agent = config.get("api", "user_agent")
|
|
21
|
+
self.retry_attempts = config.get("api", "retry_attempts")
|
|
22
|
+
self.cache_ttl = config.get("api", "cache_ttl")
|
|
23
|
+
self.request_handler = RequestHandler(
|
|
24
|
+
user_agent=self.user_agent,
|
|
25
|
+
timeout=self.timeout,
|
|
26
|
+
retry_attempts=self.retry_attempts,
|
|
27
|
+
)
|
|
28
|
+
self.cache_manager = cache_manager or CacheManager(
|
|
29
|
+
cache_dir=config.get_cache_dir(), default_ttl=self.cache_ttl
|
|
30
|
+
)
|
|
31
|
+
self.template_cache = TemplateCache(self.cache_manager)
|
|
32
|
+
self.stats = {"cache_hits": 0, "cache_misses": 0}
|
|
33
|
+
|
|
34
|
+
def list_templates(self, force_refresh: bool = False) -> APIResponse:
|
|
35
|
+
if not force_refresh:
|
|
36
|
+
cached_templates = self.template_cache.get_template_list()
|
|
37
|
+
if cached_templates is not None:
|
|
38
|
+
logger.debug("Using cached template list")
|
|
39
|
+
self.stats["cache_hits"] += 1
|
|
40
|
+
return APIResponse(success=True, data=cached_templates, from_cache=True)
|
|
41
|
+
|
|
42
|
+
self.stats["cache_misses"] += 1
|
|
43
|
+
url = f"{self.base_url}/list"
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
response = self.request_handler.make_request_with_retry(url)
|
|
47
|
+
|
|
48
|
+
if response.success:
|
|
49
|
+
templates = self._parse_template_list(response.data)
|
|
50
|
+
|
|
51
|
+
self.template_cache.set_template_list(templates)
|
|
52
|
+
|
|
53
|
+
response.data = templates
|
|
54
|
+
logger.info(f"Fetched {len(templates)} templates from API")
|
|
55
|
+
|
|
56
|
+
return response
|
|
57
|
+
|
|
58
|
+
except Exception as e:
|
|
59
|
+
logger.error(f"Failed to fetch template list: {e}")
|
|
60
|
+
return APIResponse(success=False, data=[], error_message=str(e))
|
|
61
|
+
|
|
62
|
+
def get_templates(
|
|
63
|
+
self, technologies: List[str], force_refresh: bool = False
|
|
64
|
+
) -> APIResponse:
|
|
65
|
+
if not technologies:
|
|
66
|
+
return APIResponse(
|
|
67
|
+
success=True,
|
|
68
|
+
data="# No templates selected\n# Select templates from the Available Templates panel",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
clean_techs = self._clean_technology_names(technologies)
|
|
72
|
+
if not clean_techs:
|
|
73
|
+
return APIResponse(
|
|
74
|
+
success=False, data="", error_message="No valid templates provided"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
if not force_refresh:
|
|
78
|
+
cached_content = self.template_cache.get_template_content(clean_techs)
|
|
79
|
+
if cached_content is not None:
|
|
80
|
+
logger.debug(f"Using cached content for {len(clean_techs)} templates")
|
|
81
|
+
self.stats["cache_hits"] += 1
|
|
82
|
+
return APIResponse(success=True, data=cached_content, from_cache=True)
|
|
83
|
+
|
|
84
|
+
self.stats["cache_misses"] += 1
|
|
85
|
+
tech_string = ",".join(clean_techs).lower()
|
|
86
|
+
url = f"{self.base_url}/{tech_string}"
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
response = self.request_handler.make_request_with_retry(url)
|
|
90
|
+
|
|
91
|
+
if response.success:
|
|
92
|
+
self.template_cache.set_template_content(clean_techs, response.data)
|
|
93
|
+
logger.info(f"Fetched content for templates: {', '.join(clean_techs)}")
|
|
94
|
+
|
|
95
|
+
return response
|
|
96
|
+
|
|
97
|
+
except Exception as e:
|
|
98
|
+
logger.error(f"Failed to fetch template content: {e}")
|
|
99
|
+
|
|
100
|
+
fallback_content = f"""# Error generating content: {str(e)}
|
|
101
|
+
# Selected templates: {', '.join(clean_techs)}
|
|
102
|
+
#
|
|
103
|
+
# This error typically means:
|
|
104
|
+
# - Network connectivity issues
|
|
105
|
+
# - Invalid template names
|
|
106
|
+
# - API service temporarily unavailable
|
|
107
|
+
#
|
|
108
|
+
# Try refreshing the template list or check your internet connection."""
|
|
109
|
+
|
|
110
|
+
return APIResponse(
|
|
111
|
+
success=False, data=fallback_content, error_message=str(e)
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
def test_connection(self) -> APIResponse:
|
|
115
|
+
try:
|
|
116
|
+
logger.info("Testing API connectivity...")
|
|
117
|
+
|
|
118
|
+
response = self.request_handler.make_request(
|
|
119
|
+
f"{self.base_url}/list?limit=1"
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
if response.success:
|
|
123
|
+
test_results = {
|
|
124
|
+
"status": "connected",
|
|
125
|
+
"response_time": response.response_time,
|
|
126
|
+
"api_url": self.base_url,
|
|
127
|
+
"cache_stats": self.cache_manager.get_stats(),
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return APIResponse(success=True, data=test_results)
|
|
131
|
+
else:
|
|
132
|
+
return APIResponse(
|
|
133
|
+
success=False,
|
|
134
|
+
data={"status": "failed"},
|
|
135
|
+
error_message="API test request failed",
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
except Exception as e:
|
|
139
|
+
logger.error(f"API connection test failed: {e}")
|
|
140
|
+
return APIResponse(
|
|
141
|
+
success=False, data={"status": "error"}, error_message=str(e)
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
145
|
+
request_stats = self.request_handler.get_stats()
|
|
146
|
+
total_requests = request_stats["requests_made"]
|
|
147
|
+
avg_response_time = (
|
|
148
|
+
request_stats["total_response_time"] / total_requests
|
|
149
|
+
if total_requests > 0
|
|
150
|
+
else 0.0
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
cache_stats = self.cache_manager.get_stats()
|
|
154
|
+
|
|
155
|
+
return {
|
|
156
|
+
"api_stats": {
|
|
157
|
+
"requests_made": total_requests,
|
|
158
|
+
"avg_response_time": avg_response_time,
|
|
159
|
+
"error_rate": request_stats["errors"] / max(1, total_requests),
|
|
160
|
+
"base_url": self.base_url,
|
|
161
|
+
"timeout": self.timeout,
|
|
162
|
+
"retry_attempts": self.retry_attempts,
|
|
163
|
+
},
|
|
164
|
+
"cache_stats": cache_stats,
|
|
165
|
+
"performance_stats": {
|
|
166
|
+
"cache_hit_rate": (
|
|
167
|
+
self.stats["cache_hits"]
|
|
168
|
+
/ max(1, self.stats["cache_hits"] + self.stats["cache_misses"])
|
|
169
|
+
),
|
|
170
|
+
"total_cache_operations": self.stats["cache_hits"]
|
|
171
|
+
+ self.stats["cache_misses"],
|
|
172
|
+
},
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
def clear_cache(self) -> None:
|
|
176
|
+
self.cache_manager.clear()
|
|
177
|
+
logger.info("Cleared all API cache data")
|
|
178
|
+
|
|
179
|
+
def invalidate_template(self, template_name: str) -> int:
|
|
180
|
+
return self.template_cache.invalidate_template_content(template_name)
|
|
181
|
+
|
|
182
|
+
def _parse_template_list(self, response_text: str) -> List[str]:
|
|
183
|
+
all_templates = []
|
|
184
|
+
|
|
185
|
+
for line in response_text.strip().split("\n"):
|
|
186
|
+
for template in line.split(","):
|
|
187
|
+
clean_template = template.strip()
|
|
188
|
+
if clean_template and self._is_valid_template_name(clean_template):
|
|
189
|
+
all_templates.append(clean_template)
|
|
190
|
+
|
|
191
|
+
unique_templates = sorted(set(all_templates), key=str.lower)
|
|
192
|
+
|
|
193
|
+
logger.debug(f"Parsed {len(unique_templates)} unique templates")
|
|
194
|
+
return unique_templates
|
|
195
|
+
|
|
196
|
+
def _clean_technology_names(self, technologies: List[str]) -> List[str]:
|
|
197
|
+
clean_techs = []
|
|
198
|
+
|
|
199
|
+
for tech in technologies:
|
|
200
|
+
clean_tech = str(tech).strip()
|
|
201
|
+
|
|
202
|
+
clean_tech = "".join(c for c in clean_tech if c.isalnum() or c in "-_+.")
|
|
203
|
+
|
|
204
|
+
if clean_tech and self._is_valid_template_name(clean_tech):
|
|
205
|
+
clean_techs.append(clean_tech)
|
|
206
|
+
else:
|
|
207
|
+
logger.warning(f"Skipping invalid template name: {tech}")
|
|
208
|
+
|
|
209
|
+
return clean_techs
|
|
210
|
+
|
|
211
|
+
def _is_valid_template_name(self, name: str) -> bool:
|
|
212
|
+
if not name or len(name) > 100:
|
|
213
|
+
return False
|
|
214
|
+
|
|
215
|
+
if not any(c.isalnum() for c in name):
|
|
216
|
+
return False
|
|
217
|
+
|
|
218
|
+
suspicious_patterns = ["..", "//", "\\\\", "<", ">", "|"]
|
|
219
|
+
if any(pattern in name for pattern in suspicious_patterns):
|
|
220
|
+
return False
|
|
221
|
+
|
|
222
|
+
return True
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class APIError(Exception):
|
|
9
|
+
def __init__(
|
|
10
|
+
self,
|
|
11
|
+
message: str,
|
|
12
|
+
status_code: Optional[int] = None,
|
|
13
|
+
retry_after: Optional[int] = None,
|
|
14
|
+
):
|
|
15
|
+
super().__init__(message)
|
|
16
|
+
self.status_code = status_code
|
|
17
|
+
self.retry_after = retry_after
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class NetworkError(APIError):
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RateLimitError(APIError):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ServiceUnavailableError(APIError):
|
|
29
|
+
pass
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import time
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class RateLimiter:
|
|
13
|
+
def __init__(self, min_interval: float = 0.1):
|
|
14
|
+
self.min_interval = min_interval
|
|
15
|
+
self._last_request_time = 0
|
|
16
|
+
|
|
17
|
+
def wait_if_needed(self) -> None:
|
|
18
|
+
current_time = time.time()
|
|
19
|
+
time_since_last = current_time - self._last_request_time
|
|
20
|
+
|
|
21
|
+
if time_since_last < self.min_interval:
|
|
22
|
+
sleep_time = self.min_interval - time_since_last
|
|
23
|
+
logger.debug(f"Rate limiting: sleeping for {sleep_time:.3f}s")
|
|
24
|
+
time.sleep(sleep_time)
|
|
25
|
+
|
|
26
|
+
def mark_request(self) -> None:
|
|
27
|
+
self._last_request_time = time.time()
|
|
28
|
+
|
|
29
|
+
def reset(self) -> None:
|
|
30
|
+
self._last_request_time = 0
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import time
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.request
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
from .errors import APIError, NetworkError, RateLimitError, ServiceUnavailableError
|
|
12
|
+
from .rate_limiter import RateLimiter
|
|
13
|
+
from .response import APIResponse
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class RequestHandler:
|
|
19
|
+
def __init__(self, user_agent: str, timeout: float = 30.0, retry_attempts: int = 3):
|
|
20
|
+
self.user_agent = user_agent
|
|
21
|
+
self.timeout = timeout
|
|
22
|
+
self.retry_attempts = retry_attempts
|
|
23
|
+
self.rate_limiter = RateLimiter(min_interval=0.1)
|
|
24
|
+
self.stats = {"requests_made": 0, "errors": 0, "total_response_time": 0.0}
|
|
25
|
+
|
|
26
|
+
def make_request(self, url: str, timeout: Optional[float] = None) -> APIResponse:
|
|
27
|
+
self.rate_limiter.wait_if_needed()
|
|
28
|
+
|
|
29
|
+
start_time = time.time()
|
|
30
|
+
timeout = timeout or self.timeout
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
req = urllib.request.Request(
|
|
34
|
+
url,
|
|
35
|
+
headers={
|
|
36
|
+
"User-Agent": self.user_agent,
|
|
37
|
+
"Accept": "text/plain",
|
|
38
|
+
"Accept-Encoding": "identity",
|
|
39
|
+
},
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
43
|
+
content = response.read().decode("utf-8")
|
|
44
|
+
response_time = time.time() - start_time
|
|
45
|
+
|
|
46
|
+
self.rate_limiter.mark_request()
|
|
47
|
+
self.stats["requests_made"] += 1
|
|
48
|
+
self.stats["total_response_time"] += response_time
|
|
49
|
+
|
|
50
|
+
logger.debug(f"API request to {url} took {response_time:.3f}s")
|
|
51
|
+
|
|
52
|
+
return APIResponse(
|
|
53
|
+
success=True,
|
|
54
|
+
data=content,
|
|
55
|
+
status_code=response.getcode(),
|
|
56
|
+
response_time=response_time,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
except urllib.error.HTTPError as e:
|
|
60
|
+
response_time = time.time() - start_time
|
|
61
|
+
self.stats["errors"] += 1
|
|
62
|
+
|
|
63
|
+
error_msg = f"HTTP {e.code}: {e.reason}"
|
|
64
|
+
|
|
65
|
+
if e.code == 429:
|
|
66
|
+
retry_after = int(e.headers.get("Retry-After", 60))
|
|
67
|
+
raise RateLimitError(error_msg, e.code, retry_after)
|
|
68
|
+
elif e.code >= 500:
|
|
69
|
+
raise ServiceUnavailableError(error_msg, e.code)
|
|
70
|
+
else:
|
|
71
|
+
raise APIError(error_msg, e.code)
|
|
72
|
+
|
|
73
|
+
except urllib.error.URLError as e:
|
|
74
|
+
response_time = time.time() - start_time
|
|
75
|
+
self.stats["errors"] += 1
|
|
76
|
+
|
|
77
|
+
if "timeout" in str(e.reason).lower():
|
|
78
|
+
raise NetworkError(f"Request timeout after {timeout}s")
|
|
79
|
+
else:
|
|
80
|
+
raise NetworkError(f"Network error: {e.reason}")
|
|
81
|
+
|
|
82
|
+
except Exception as e:
|
|
83
|
+
response_time = time.time() - start_time
|
|
84
|
+
self.stats["errors"] += 1
|
|
85
|
+
raise APIError(f"Unexpected error: {str(e)}")
|
|
86
|
+
|
|
87
|
+
def make_request_with_retry(self, url: str) -> APIResponse:
|
|
88
|
+
last_exception = None
|
|
89
|
+
|
|
90
|
+
for attempt in range(self.retry_attempts):
|
|
91
|
+
try:
|
|
92
|
+
return self.make_request(url)
|
|
93
|
+
|
|
94
|
+
except RateLimitError as e:
|
|
95
|
+
if attempt < self.retry_attempts - 1:
|
|
96
|
+
wait_time = e.retry_after or (2**attempt)
|
|
97
|
+
logger.warning(
|
|
98
|
+
f"Rate limited, waiting {wait_time}s before retry {attempt + 1}"
|
|
99
|
+
)
|
|
100
|
+
time.sleep(wait_time)
|
|
101
|
+
last_exception = e
|
|
102
|
+
else:
|
|
103
|
+
raise
|
|
104
|
+
|
|
105
|
+
except (NetworkError, ServiceUnavailableError) as e:
|
|
106
|
+
if attempt < self.retry_attempts - 1:
|
|
107
|
+
wait_time = 2**attempt
|
|
108
|
+
logger.warning(
|
|
109
|
+
f"Request failed, retrying in {wait_time}s (attempt {attempt + 1})"
|
|
110
|
+
)
|
|
111
|
+
time.sleep(wait_time)
|
|
112
|
+
last_exception = e
|
|
113
|
+
else:
|
|
114
|
+
raise
|
|
115
|
+
|
|
116
|
+
except APIError:
|
|
117
|
+
raise
|
|
118
|
+
|
|
119
|
+
if last_exception:
|
|
120
|
+
raise last_exception
|
|
121
|
+
raise APIError("All retry attempts failed")
|
|
122
|
+
|
|
123
|
+
def get_stats(self) -> dict:
|
|
124
|
+
return self.stats.copy()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any, Optional
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class APIResponse:
|
|
11
|
+
success: bool
|
|
12
|
+
data: Any
|
|
13
|
+
error_message: Optional[str] = None
|
|
14
|
+
status_code: Optional[int] = None
|
|
15
|
+
response_time: Optional[float] = None
|
|
16
|
+
from_cache: bool = False
|