django-robots-manager 6.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_robots_manager-6.2.1/LICENSE.txt +29 -0
- django_robots_manager-6.2.1/PKG-INFO +175 -0
- django_robots_manager-6.2.1/README.md +143 -0
- django_robots_manager-6.2.1/pyproject.toml +57 -0
- django_robots_manager-6.2.1/robots/__init__.py +3 -0
- django_robots_manager-6.2.1/robots/admin.py +36 -0
- django_robots_manager-6.2.1/robots/apps.py +12 -0
- django_robots_manager-6.2.1/robots/checks.py +39 -0
- django_robots_manager-6.2.1/robots/forms.py +64 -0
- django_robots_manager-6.2.1/robots/locale/ru/LC_MESSAGES/django.mo +0 -0
- django_robots_manager-6.2.1/robots/locale/ru/LC_MESSAGES/django.po +188 -0
- django_robots_manager-6.2.1/robots/migrations/0001_initial.py +103 -0
- django_robots_manager-6.2.1/robots/migrations/0002_alter_id_fields.py +26 -0
- django_robots_manager-6.2.1/robots/migrations/0003_rule_comment.py +25 -0
- django_robots_manager-6.2.1/robots/migrations/0004_cleanparam.py +56 -0
- django_robots_manager-6.2.1/robots/migrations/0005_alter_rule_disallowed.py +23 -0
- django_robots_manager-6.2.1/robots/migrations/__init__.py +0 -0
- django_robots_manager-6.2.1/robots/models.py +216 -0
- django_robots_manager-6.2.1/robots/settings.py +24 -0
- django_robots_manager-6.2.1/robots/templates/robots/rule_list.html +13 -0
- django_robots_manager-6.2.1/robots/urls.py +7 -0
- django_robots_manager-6.2.1/robots/validators.py +49 -0
- django_robots_manager-6.2.1/robots/views.py +112 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Copyright (c) 2008-, Jannis Leidel
|
|
2
|
+
Copyright (c) 2026-, Artem Fabrikov
|
|
3
|
+
All rights reserved.
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are
|
|
7
|
+
met:
|
|
8
|
+
|
|
9
|
+
* Redistributions of source code must retain the above copyright
|
|
10
|
+
notice, this list of conditions and the following disclaimer.
|
|
11
|
+
* Redistributions in binary form must reproduce the above
|
|
12
|
+
copyright notice, this list of conditions and the following
|
|
13
|
+
disclaimer in the documentation and/or other materials provided
|
|
14
|
+
with the distribution.
|
|
15
|
+
* Neither the name of the author nor the names of other
|
|
16
|
+
contributors may be used to endorse or promote products derived
|
|
17
|
+
from this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
20
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
21
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
22
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
23
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
24
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
25
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
26
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
27
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
28
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-robots-manager
|
|
3
|
+
Version: 6.2.1
|
|
4
|
+
Summary: Robots exclusion application for Django, complementing Sitemaps. Continuation of django-robots 6.x after a long pause in upstream releases.
|
|
5
|
+
License-Expression: BSD-3-Clause
|
|
6
|
+
License-File: LICENSE.txt
|
|
7
|
+
Author: Artem Fabrikov
|
|
8
|
+
Author-email: a.fabrikov1406@yandex.ru
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Classifier: Environment :: Web Environment
|
|
11
|
+
Classifier: Framework :: Django
|
|
12
|
+
Classifier: Framework :: Django :: 4.2
|
|
13
|
+
Classifier: Framework :: Django :: 5.2
|
|
14
|
+
Classifier: Framework :: Django :: 6.0
|
|
15
|
+
Classifier: Framework :: Django :: 6.1
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
25
|
+
Requires-Dist: Django (>=4.2)
|
|
26
|
+
Project-URL: Homepage, https://github.com/KitKat-ru/django-robots-manager
|
|
27
|
+
Project-URL: Issues, https://github.com/KitKat-ru/django-robots-manager/issues
|
|
28
|
+
Project-URL: Repository, https://github.com/KitKat-ru/django-robots-manager
|
|
29
|
+
Project-URL: Upstream, https://github.com/jazzband/django-robots/
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# django-robots-manager
|
|
33
|
+
|
|
34
|
+
[](https://github.com/KitKat-ru/django-robots-manager/actions/workflows/tests.yml)
|
|
35
|
+
[](https://github.com/KitKat-ru/django-robots-manager/tags)
|
|
36
|
+
[](https://github.com/KitKat-ru/django-robots-manager/actions/workflows/tests.yml)
|
|
37
|
+
[](https://github.com/KitKat-ru/django-robots-manager/actions/workflows/tests.yml)
|
|
38
|
+
[](https://github.com/KitKat-ru/django-robots-manager/blob/main/LICENSE.txt)
|
|
39
|
+
[](https://github.com/astral-sh/ruff)
|
|
40
|
+
[](https://github.com/pre-commit/pre-commit)
|
|
41
|
+
|
|
42
|
+
A continuation of [django-robots](https://github.com/jazzband/django-robots/),
|
|
43
|
+
picking up from the 6.x series (based on release 6.1), after a long pause in releases
|
|
44
|
+
of the original library.
|
|
45
|
+
|
|
46
|
+
The import path and app label stay `robots`, so it is a drop-in replacement for
|
|
47
|
+
django-robots 5.0 and 6.x: existing tables and migration history are reused.
|
|
48
|
+
|
|
49
|
+
## Changes from upstream 6.1
|
|
50
|
+
|
|
51
|
+
- `__version__` is read via `importlib.metadata` only; the `pkg_resources` fallback
|
|
52
|
+
and `default_app_config` (Django < 3.2) are removed.
|
|
53
|
+
- The South guard in `robots.migrations` is removed.
|
|
54
|
+
- `RuleAdminForm` rejects a rule whose allowed and disallowed URLs share a pattern.
|
|
55
|
+
- `RuleAdminForm` rejects a second rule for the same robot (case-insensitive) on the
|
|
56
|
+
same site.
|
|
57
|
+
- `ROBOTS_USE_HOST` now defaults to `False`: Yandex
|
|
58
|
+
[stopped using](https://webmaster.yandex.ru/blog/301-y-redirekt-polnostyu-zamenil-direktivu-host)
|
|
59
|
+
the `Host` directive in 2018, and Google
|
|
60
|
+
[never supported it](https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt)
|
|
61
|
+
(it is not part of [RFC 9309](https://www.rfc-editor.org/rfc/rfc9309) either). Set it
|
|
62
|
+
to `True` to keep the old output.
|
|
63
|
+
- `ROBOTS_SITE_BY_REQUEST` looks the site up the same way as Django's sites framework:
|
|
64
|
+
case-insensitively, retrying without the port (`example.com:8000` matches
|
|
65
|
+
`example.com`).
|
|
66
|
+
- `robots.txt` is served as `text/plain; charset=utf-8`
|
|
67
|
+
([RFC 9309](https://www.rfc-editor.org/rfc/rfc9309) requires UTF-8), with rules sorted
|
|
68
|
+
by robot and URLs by pattern, in a fixed number of queries.
|
|
69
|
+
- `Rule.comment`: an optional single-line note rendered as a `# ...` line above the
|
|
70
|
+
rule's group in `robots.txt`.
|
|
71
|
+
- URL patterns are percent-encoded on output: non-ASCII characters (e.g. Cyrillic),
|
|
72
|
+
whitespace and `#` become `%XX`, as
|
|
73
|
+
[Yandex requires](https://yandex.ru/support/webmaster/ru/controlling-robot/robots-txt).
|
|
74
|
+
Patterns are stored as entered, and raw and encoded forms of the same path count as
|
|
75
|
+
the same pattern when checking allowed/disallowed conflicts.
|
|
76
|
+
- [Clean-param](https://yandex.ru/support/webmaster/ru/robot-workings/clean-param)
|
|
77
|
+
directives (Yandex): see below.
|
|
78
|
+
- System checks for `ROBOTS_SITEMAP_URLS`: `robots.E001` if it is a string instead of
|
|
79
|
+
a list, `robots.W001` for URLs that are not absolute, `robots.W002` for non-ASCII
|
|
80
|
+
domains.
|
|
81
|
+
- Only the `ru` locale is shipped.
|
|
82
|
+
|
|
83
|
+
## Installation
|
|
84
|
+
|
|
85
|
+
Requires Python 3.10+ and Django 4.2+ (tested with Django 4.2, 5.2, 6.0 and 6.1).
|
|
86
|
+
Projects on older Python or Django versions can stay on django-robots 6.1.
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
INSTALLED_APPS = [
|
|
90
|
+
"django.contrib.sites",
|
|
91
|
+
...
|
|
92
|
+
"robots",
|
|
93
|
+
]
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
urlpatterns = [
|
|
98
|
+
re_path(r"^robots\.txt", include("robots.urls")),
|
|
99
|
+
]
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Settings (`ROBOTS_SITEMAP_URLS`, `ROBOTS_USE_SITEMAP`, `ROBOTS_USE_HOST`,
|
|
103
|
+
`ROBOTS_CACHE_TIMEOUT`, `ROBOTS_SITE_BY_REQUEST`, `ROBOTS_USE_SCHEME_IN_HOST`,
|
|
104
|
+
`ROBOTS_SITEMAP_VIEW_NAME`) are the same as upstream, except for the changes listed
|
|
105
|
+
above.
|
|
106
|
+
|
|
107
|
+
### Internationalized domains
|
|
108
|
+
|
|
109
|
+
Store non-ASCII domains in `Site.domain` (and in `ROBOTS_SITEMAP_URLS`) in Punycode,
|
|
110
|
+
e.g. `xn--d1aqf.xn--p1ai` instead of `дом.рф`:
|
|
111
|
+
|
|
112
|
+
- the domain is written to `Sitemap:` (and `Host:`) as stored, and
|
|
113
|
+
[Yandex requires](https://yandex.ru/support/webmaster/ru/controlling-robot/robots-txt)
|
|
114
|
+
Punycode there;
|
|
115
|
+
- with `ROBOTS_SITE_BY_REQUEST = True` the site is looked up by the request `Host`
|
|
116
|
+
header, which is always Punycode, so a site stored as `дом.рф` is not found and
|
|
117
|
+
`robots.txt` responds with an error.
|
|
118
|
+
|
|
119
|
+
## Clean-param
|
|
120
|
+
|
|
121
|
+
`Clean-param` tells Yandex which URL parameters do not change the page content, so
|
|
122
|
+
`/catalog/?ref=vk` and `/catalog/?sid=1` are crawled and indexed as `/catalog/`. Other
|
|
123
|
+
search engines ignore it.
|
|
124
|
+
|
|
125
|
+
Add directives in the admin under *Clean-param directives* and attach them to sites:
|
|
126
|
+
|
|
127
|
+
| Parameters | Path | Output |
|
|
128
|
+
|---|---|---|
|
|
129
|
+
| `ref&sid` | `/catalog/` | `Clean-param: ref&sid /catalog/` |
|
|
130
|
+
| `sort` | *(empty)* | `Clean-param: sort` (whole site) |
|
|
131
|
+
|
|
132
|
+
The directive is cross-sectional, so it is rendered once per file, next to `Sitemap`,
|
|
133
|
+
not inside a `User-agent` group. Following the Yandex rules, parameter names are
|
|
134
|
+
case-sensitive, the path may contain only `A-Za-z0-9.-/*_` (a leading `/` is added
|
|
135
|
+
if missing, as for URL patterns), and the whole line is limited to 500 characters.
|
|
136
|
+
Unlike `Allow`/`Disallow`, the path is not percent-encoded, so pages with Cyrillic
|
|
137
|
+
paths (e.g. `/о-компании/`) can only be covered by a directive without a path.
|
|
138
|
+
|
|
139
|
+
## Development
|
|
140
|
+
|
|
141
|
+
`tests/` holds a minimal Django project (SQLite) used both for the test suite and for
|
|
142
|
+
trying the app locally. It is not part of the distributed package. Run the commands
|
|
143
|
+
from the repository root.
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
python -m venv .venv
|
|
147
|
+
. .venv/bin/activate
|
|
148
|
+
pip install -e .
|
|
149
|
+
export DJANGO_SETTINGS_MODULE=tests.settings
|
|
150
|
+
|
|
151
|
+
python -m django test tests # run the test suite
|
|
152
|
+
|
|
153
|
+
pip install "coverage[toml]" # test coverage, as in CI
|
|
154
|
+
python -m coverage run -m django test tests
|
|
155
|
+
python -m coverage report
|
|
156
|
+
|
|
157
|
+
python -m django migrate
|
|
158
|
+
python -m django createsuperuser
|
|
159
|
+
python -m django runserver # http://localhost:8000/admin/, http://localhost:8000/robots.txt
|
|
160
|
+
|
|
161
|
+
python -m django makemigrations robots --check --dry-run # after model changes
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
The demo project uses `SITE_ID = 1` (`example.com`), so rules must be attached to that
|
|
165
|
+
site to appear in `/robots.txt`.
|
|
166
|
+
|
|
167
|
+
Linting and formatting use [ruff](https://docs.astral.sh/ruff/) via
|
|
168
|
+
[pre-commit](https://pre-commit.com/); neither is a dependency of the package.
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
pip install pre-commit
|
|
172
|
+
pre-commit install # run the hooks on every commit
|
|
173
|
+
pre-commit run --all-files # run them on the whole repository
|
|
174
|
+
```
|
|
175
|
+
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# django-robots-manager
|
|
2
|
+
|
|
3
|
+
[](https://github.com/KitKat-ru/django-robots-manager/actions/workflows/tests.yml)
|
|
4
|
+
[](https://github.com/KitKat-ru/django-robots-manager/tags)
|
|
5
|
+
[](https://github.com/KitKat-ru/django-robots-manager/actions/workflows/tests.yml)
|
|
6
|
+
[](https://github.com/KitKat-ru/django-robots-manager/actions/workflows/tests.yml)
|
|
7
|
+
[](https://github.com/KitKat-ru/django-robots-manager/blob/main/LICENSE.txt)
|
|
8
|
+
[](https://github.com/astral-sh/ruff)
|
|
9
|
+
[](https://github.com/pre-commit/pre-commit)
|
|
10
|
+
|
|
11
|
+
A continuation of [django-robots](https://github.com/jazzband/django-robots/),
|
|
12
|
+
picking up from the 6.x series (based on release 6.1), after a long pause in releases
|
|
13
|
+
of the original library.
|
|
14
|
+
|
|
15
|
+
The import path and app label stay `robots`, so it is a drop-in replacement for
|
|
16
|
+
django-robots 5.0 and 6.x: existing tables and migration history are reused.
|
|
17
|
+
|
|
18
|
+
## Changes from upstream 6.1
|
|
19
|
+
|
|
20
|
+
- `__version__` is read via `importlib.metadata` only; the `pkg_resources` fallback
|
|
21
|
+
and `default_app_config` (Django < 3.2) are removed.
|
|
22
|
+
- The South guard in `robots.migrations` is removed.
|
|
23
|
+
- `RuleAdminForm` rejects a rule whose allowed and disallowed URLs share a pattern.
|
|
24
|
+
- `RuleAdminForm` rejects a second rule for the same robot (case-insensitive) on the
|
|
25
|
+
same site.
|
|
26
|
+
- `ROBOTS_USE_HOST` now defaults to `False`: Yandex
|
|
27
|
+
[stopped using](https://webmaster.yandex.ru/blog/301-y-redirekt-polnostyu-zamenil-direktivu-host)
|
|
28
|
+
the `Host` directive in 2018, and Google
|
|
29
|
+
[never supported it](https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt)
|
|
30
|
+
(it is not part of [RFC 9309](https://www.rfc-editor.org/rfc/rfc9309) either). Set it
|
|
31
|
+
to `True` to keep the old output.
|
|
32
|
+
- `ROBOTS_SITE_BY_REQUEST` looks the site up the same way as Django's sites framework:
|
|
33
|
+
case-insensitively, retrying without the port (`example.com:8000` matches
|
|
34
|
+
`example.com`).
|
|
35
|
+
- `robots.txt` is served as `text/plain; charset=utf-8`
|
|
36
|
+
([RFC 9309](https://www.rfc-editor.org/rfc/rfc9309) requires UTF-8), with rules sorted
|
|
37
|
+
by robot and URLs by pattern, in a fixed number of queries.
|
|
38
|
+
- `Rule.comment`: an optional single-line note rendered as a `# ...` line above the
|
|
39
|
+
rule's group in `robots.txt`.
|
|
40
|
+
- URL patterns are percent-encoded on output: non-ASCII characters (e.g. Cyrillic),
|
|
41
|
+
whitespace and `#` become `%XX`, as
|
|
42
|
+
[Yandex requires](https://yandex.ru/support/webmaster/ru/controlling-robot/robots-txt).
|
|
43
|
+
Patterns are stored as entered, and raw and encoded forms of the same path count as
|
|
44
|
+
the same pattern when checking allowed/disallowed conflicts.
|
|
45
|
+
- [Clean-param](https://yandex.ru/support/webmaster/ru/robot-workings/clean-param)
|
|
46
|
+
directives (Yandex): see below.
|
|
47
|
+
- System checks for `ROBOTS_SITEMAP_URLS`: `robots.E001` if it is a string instead of
|
|
48
|
+
a list, `robots.W001` for URLs that are not absolute, `robots.W002` for non-ASCII
|
|
49
|
+
domains.
|
|
50
|
+
- Only the `ru` locale is shipped.
|
|
51
|
+
|
|
52
|
+
## Installation
|
|
53
|
+
|
|
54
|
+
Requires Python 3.10+ and Django 4.2+ (tested with Django 4.2, 5.2, 6.0 and 6.1).
|
|
55
|
+
Projects on older Python or Django versions can stay on django-robots 6.1.
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
INSTALLED_APPS = [
|
|
59
|
+
"django.contrib.sites",
|
|
60
|
+
...
|
|
61
|
+
"robots",
|
|
62
|
+
]
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
urlpatterns = [
|
|
67
|
+
re_path(r"^robots\.txt", include("robots.urls")),
|
|
68
|
+
]
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Settings (`ROBOTS_SITEMAP_URLS`, `ROBOTS_USE_SITEMAP`, `ROBOTS_USE_HOST`,
|
|
72
|
+
`ROBOTS_CACHE_TIMEOUT`, `ROBOTS_SITE_BY_REQUEST`, `ROBOTS_USE_SCHEME_IN_HOST`,
|
|
73
|
+
`ROBOTS_SITEMAP_VIEW_NAME`) are the same as upstream, except for the changes listed
|
|
74
|
+
above.
|
|
75
|
+
|
|
76
|
+
### Internationalized domains
|
|
77
|
+
|
|
78
|
+
Store non-ASCII domains in `Site.domain` (and in `ROBOTS_SITEMAP_URLS`) in Punycode,
|
|
79
|
+
e.g. `xn--d1aqf.xn--p1ai` instead of `дом.рф`:
|
|
80
|
+
|
|
81
|
+
- the domain is written to `Sitemap:` (and `Host:`) as stored, and
|
|
82
|
+
[Yandex requires](https://yandex.ru/support/webmaster/ru/controlling-robot/robots-txt)
|
|
83
|
+
Punycode there;
|
|
84
|
+
- with `ROBOTS_SITE_BY_REQUEST = True` the site is looked up by the request `Host`
|
|
85
|
+
header, which is always Punycode, so a site stored as `дом.рф` is not found and
|
|
86
|
+
`robots.txt` responds with an error.
|
|
87
|
+
|
|
88
|
+
## Clean-param
|
|
89
|
+
|
|
90
|
+
`Clean-param` tells Yandex which URL parameters do not change the page content, so
|
|
91
|
+
`/catalog/?ref=vk` and `/catalog/?sid=1` are crawled and indexed as `/catalog/`. Other
|
|
92
|
+
search engines ignore it.
|
|
93
|
+
|
|
94
|
+
Add directives in the admin under *Clean-param directives* and attach them to sites:
|
|
95
|
+
|
|
96
|
+
| Parameters | Path | Output |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| `ref&sid` | `/catalog/` | `Clean-param: ref&sid /catalog/` |
|
|
99
|
+
| `sort` | *(empty)* | `Clean-param: sort` (whole site) |
|
|
100
|
+
|
|
101
|
+
The directive is cross-sectional, so it is rendered once per file, next to `Sitemap`,
|
|
102
|
+
not inside a `User-agent` group. Following the Yandex rules, parameter names are
|
|
103
|
+
case-sensitive, the path may contain only `A-Za-z0-9.-/*_` (a leading `/` is added
|
|
104
|
+
if missing, as for URL patterns), and the whole line is limited to 500 characters.
|
|
105
|
+
Unlike `Allow`/`Disallow`, the path is not percent-encoded, so pages with Cyrillic
|
|
106
|
+
paths (e.g. `/о-компании/`) can only be covered by a directive without a path.
|
|
107
|
+
|
|
108
|
+
## Development
|
|
109
|
+
|
|
110
|
+
`tests/` holds a minimal Django project (SQLite) used both for the test suite and for
|
|
111
|
+
trying the app locally. It is not part of the distributed package. Run the commands
|
|
112
|
+
from the repository root.
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
python -m venv .venv
|
|
116
|
+
. .venv/bin/activate
|
|
117
|
+
pip install -e .
|
|
118
|
+
export DJANGO_SETTINGS_MODULE=tests.settings
|
|
119
|
+
|
|
120
|
+
python -m django test tests # run the test suite
|
|
121
|
+
|
|
122
|
+
pip install "coverage[toml]" # test coverage, as in CI
|
|
123
|
+
python -m coverage run -m django test tests
|
|
124
|
+
python -m coverage report
|
|
125
|
+
|
|
126
|
+
python -m django migrate
|
|
127
|
+
python -m django createsuperuser
|
|
128
|
+
python -m django runserver # http://localhost:8000/admin/, http://localhost:8000/robots.txt
|
|
129
|
+
|
|
130
|
+
python -m django makemigrations robots --check --dry-run # after model changes
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
The demo project uses `SITE_ID = 1` (`example.com`), so rules must be attached to that
|
|
134
|
+
site to appear in `/robots.txt`.
|
|
135
|
+
|
|
136
|
+
Linting and formatting use [ruff](https://docs.astral.sh/ruff/) via
|
|
137
|
+
[pre-commit](https://pre-commit.com/); neither is a dependency of the package.
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
pip install pre-commit
|
|
141
|
+
pre-commit install # run the hooks on every commit
|
|
142
|
+
pre-commit run --all-files # run them on the whole repository
|
|
143
|
+
```
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "django-robots-manager"
|
|
3
|
+
version = "6.2.1"
|
|
4
|
+
description = "Robots exclusion application for Django, complementing Sitemaps. Continuation of django-robots 6.x after a long pause in upstream releases."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [{ name = "Artem Fabrikov", email = "a.fabrikov1406@yandex.ru" }]
|
|
7
|
+
license = "BSD-3-Clause"
|
|
8
|
+
license-files = ["LICENSE.txt"]
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = ["Django>=4.2"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Environment :: Web Environment",
|
|
13
|
+
"Framework :: Django",
|
|
14
|
+
"Framework :: Django :: 4.2",
|
|
15
|
+
"Framework :: Django :: 5.2",
|
|
16
|
+
"Framework :: Django :: 6.0",
|
|
17
|
+
"Framework :: Django :: 6.1",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Programming Language :: Python :: 3.14",
|
|
26
|
+
"Topic :: Internet :: WWW/HTTP :: Dynamic Content",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/KitKat-ru/django-robots-manager"
|
|
31
|
+
Repository = "https://github.com/KitKat-ru/django-robots-manager"
|
|
32
|
+
Issues = "https://github.com/KitKat-ru/django-robots-manager/issues"
|
|
33
|
+
Upstream = "https://github.com/jazzband/django-robots/"
|
|
34
|
+
|
|
35
|
+
[tool.poetry]
|
|
36
|
+
packages = [{ include = "robots" }]
|
|
37
|
+
|
|
38
|
+
[build-system]
|
|
39
|
+
requires = ["poetry-core>=2.0"]
|
|
40
|
+
build-backend = "poetry.core.masonry.api"
|
|
41
|
+
|
|
42
|
+
[tool.ruff.lint]
|
|
43
|
+
extend-select = ["W", "I", "B", "C4"]
|
|
44
|
+
|
|
45
|
+
[tool.ruff.lint.per-file-ignores]
|
|
46
|
+
# Django generates `dependencies` and `operations` as plain class-level lists.
|
|
47
|
+
"robots/migrations/*" = ["RUF012"]
|
|
48
|
+
|
|
49
|
+
[tool.ruff.lint.isort]
|
|
50
|
+
known-first-party = ["robots"]
|
|
51
|
+
|
|
52
|
+
[tool.coverage.run]
|
|
53
|
+
source = ["robots"]
|
|
54
|
+
omit = ["robots/migrations/*"]
|
|
55
|
+
|
|
56
|
+
[tool.coverage.report]
|
|
57
|
+
show_missing = true
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from django.contrib import admin
|
|
2
|
+
from django.http import HttpRequest
|
|
3
|
+
from django.utils.translation import gettext_lazy as _
|
|
4
|
+
|
|
5
|
+
from robots.forms import RuleAdminForm
|
|
6
|
+
from robots.models import CleanParam, Rule, Url
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class RuleAdmin(admin.ModelAdmin):
|
|
10
|
+
form = RuleAdminForm
|
|
11
|
+
fieldsets = (
|
|
12
|
+
(None, {"fields": ("robot", "sites", "comment")}),
|
|
13
|
+
(_("URL patterns"), {"fields": ("allowed", "disallowed")}),
|
|
14
|
+
(
|
|
15
|
+
_("Advanced options"),
|
|
16
|
+
{"classes": ("collapse",), "fields": ("crawl_delay",)},
|
|
17
|
+
),
|
|
18
|
+
)
|
|
19
|
+
list_filter = ("sites",)
|
|
20
|
+
list_display = ("robot", "allowed_urls", "disallowed_urls")
|
|
21
|
+
search_fields = ("robot", "allowed__pattern", "disallowed__pattern")
|
|
22
|
+
filter_horizontal = ("sites", "allowed", "disallowed")
|
|
23
|
+
|
|
24
|
+
def get_queryset(self, request: HttpRequest):
|
|
25
|
+
return super().get_queryset(request).prefetch_related("allowed", "disallowed")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CleanParamAdmin(admin.ModelAdmin):
|
|
29
|
+
list_display = ("parameters", "path")
|
|
30
|
+
list_filter = ("sites",)
|
|
31
|
+
filter_horizontal = ("sites",)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
admin.site.register(Url)
|
|
35
|
+
admin.site.register(Rule, RuleAdmin)
|
|
36
|
+
admin.site.register(CleanParam, CleanParamAdmin)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from django.apps import AppConfig
|
|
2
|
+
from django.core import checks
|
|
3
|
+
|
|
4
|
+
from robots.checks import check_sitemap_urls
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class RobotsConfig(AppConfig):
|
|
8
|
+
default_auto_field = "django.db.models.BigAutoField"
|
|
9
|
+
name = "robots"
|
|
10
|
+
|
|
11
|
+
def ready(self):
|
|
12
|
+
checks.register(check_sitemap_urls)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from urllib.parse import SplitResult, urlsplit
|
|
2
|
+
|
|
3
|
+
from django.core import checks
|
|
4
|
+
|
|
5
|
+
from robots import settings
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def check_sitemap_urls(app_configs, **kwargs):
|
|
9
|
+
"""Validate that ROBOTS_SITEMAP_URLS is a list of absolute ASCII URLs."""
|
|
10
|
+
sitemap_urls = settings.SITEMAP_URLS
|
|
11
|
+
if isinstance(sitemap_urls, str):
|
|
12
|
+
return [
|
|
13
|
+
checks.Error(
|
|
14
|
+
"ROBOTS_SITEMAP_URLS must be a list or tuple of URLs, not a string.",
|
|
15
|
+
hint="Wrap the URL in a list: ['https://example.com/sitemap.xml'].",
|
|
16
|
+
id="robots.E001",
|
|
17
|
+
)
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
messages = []
|
|
21
|
+
for url in sitemap_urls:
|
|
22
|
+
parts: SplitResult = urlsplit(url)
|
|
23
|
+
if parts.scheme not in ("http", "https") or not parts.netloc:
|
|
24
|
+
messages.append(
|
|
25
|
+
checks.Warning(
|
|
26
|
+
f"ROBOTS_SITEMAP_URLS has a URL that is not absolute: {url!r}.",
|
|
27
|
+
hint="Use an absolute URL, e.g. 'https://example.com/sitemap.xml'.",
|
|
28
|
+
id="robots.W001",
|
|
29
|
+
)
|
|
30
|
+
)
|
|
31
|
+
elif not parts.netloc.isascii():
|
|
32
|
+
messages.append(
|
|
33
|
+
checks.Warning(
|
|
34
|
+
f"ROBOTS_SITEMAP_URLS contains a non-ASCII domain: {url!r}.",
|
|
35
|
+
hint="Write the domain in Punycode, e.g. 'xn--d1aqf.xn--p1ai'.",
|
|
36
|
+
id="robots.W002",
|
|
37
|
+
)
|
|
38
|
+
)
|
|
39
|
+
return messages
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from django import forms
|
|
2
|
+
from django.contrib.sites.models import Site
|
|
3
|
+
from django.utils.translation import gettext_lazy as _
|
|
4
|
+
|
|
5
|
+
from robots.models import Rule, encode_pattern
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class RuleAdminForm(forms.ModelForm):
|
|
9
|
+
class Meta:
|
|
10
|
+
model = Rule
|
|
11
|
+
fields = "__all__"
|
|
12
|
+
|
|
13
|
+
def clean(self):
|
|
14
|
+
if not self.cleaned_data.get("disallowed", False) and not self.cleaned_data.get(
|
|
15
|
+
"allowed", False
|
|
16
|
+
):
|
|
17
|
+
raise forms.ValidationError(
|
|
18
|
+
_("Please specify at least one allowed or disallowed URL.")
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
allowed = self.cleaned_data.get("allowed")
|
|
22
|
+
disallowed = self.cleaned_data.get("disallowed")
|
|
23
|
+
if allowed and disallowed:
|
|
24
|
+
allowed_patterns = {
|
|
25
|
+
encode_pattern(pattern): pattern
|
|
26
|
+
for pattern in allowed.values_list("pattern", flat=True)
|
|
27
|
+
}
|
|
28
|
+
conflicts = allowed_patterns.keys() & {
|
|
29
|
+
encode_pattern(pattern)
|
|
30
|
+
for pattern in disallowed.values_list("pattern", flat=True)
|
|
31
|
+
}
|
|
32
|
+
if conflicts:
|
|
33
|
+
raise forms.ValidationError(
|
|
34
|
+
_(
|
|
35
|
+
"URL patterns cannot be both allowed and disallowed: "
|
|
36
|
+
"%(patterns)s."
|
|
37
|
+
),
|
|
38
|
+
params={
|
|
39
|
+
"patterns": ", ".join(
|
|
40
|
+
sorted(allowed_patterns[key] for key in conflicts)
|
|
41
|
+
)
|
|
42
|
+
},
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
robot = self.cleaned_data.get("robot")
|
|
46
|
+
sites = self.cleaned_data.get("sites")
|
|
47
|
+
if robot and sites:
|
|
48
|
+
other_rules = Rule.objects.filter(robot__iexact=robot).exclude(
|
|
49
|
+
pk=self.instance.pk
|
|
50
|
+
)
|
|
51
|
+
duplicate_domains = (
|
|
52
|
+
Site.objects.filter(pk__in=sites, rule__in=other_rules)
|
|
53
|
+
.values_list("domain", flat=True)
|
|
54
|
+
.distinct()
|
|
55
|
+
)
|
|
56
|
+
if duplicate_domains:
|
|
57
|
+
raise forms.ValidationError(
|
|
58
|
+
_("A rule for robot %(robot)s already exists on sites: %(sites)s."),
|
|
59
|
+
params={
|
|
60
|
+
"robot": robot,
|
|
61
|
+
"sites": ", ".join(sorted(duplicate_domains)),
|
|
62
|
+
},
|
|
63
|
+
)
|
|
64
|
+
return self.cleaned_data
|
|
Binary file
|