flru-parser 0.3.1__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flru_parser-0.3.1 → flru_parser-0.3.2}/CHANGELOG.md +11 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/PKG-INFO +7 -13
- {flru_parser-0.3.1 → flru_parser-0.3.2}/README.md +5 -12
- {flru_parser-0.3.1 → flru_parser-0.3.2}/docs/PUBLISHING.md +3 -3
- {flru_parser-0.3.1 → flru_parser-0.3.2}/pyproject.toml +2 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/__init__.py +1 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/client.py +25 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/easy.py +7 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/models.py +1 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/parsers/projects.py +11 -4
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/security.py +1 -3
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru_parser.egg-info/PKG-INFO +7 -13
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru_parser.egg-info/SOURCES.txt +1 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru_parser.egg-info/requires.txt +1 -0
- flru_parser-0.3.2/tests/test_client_easy_edges.py +147 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_easy_api.py +4 -4
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_freelancers.py +1 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_parsers.py +32 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_resilience.py +1 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_security_transport.py +1 -1
- {flru_parser-0.3.1 → flru_parser-0.3.2}/CODE_OF_CONDUCT.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/CONTRIBUTING.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/LICENSE +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/MANIFEST.in +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/SECURITY.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/SUPPORT.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/docs/ARCHITECTURE.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/docs/SELECTOR_MAINTENANCE.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/docs/SIMPLE_API.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/docs/ru/README.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/examples/README.md +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/examples/basic.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/examples/incremental_sqlite.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/examples/production_config.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/examples/simple_one_shot.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/examples/sync_usage.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/setup.cfg +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/batch.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/canary.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/config.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/cookies.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/exceptions.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/filters.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/integrations/__init__.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/integrations/opentelemetry.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/integrations/prometheus.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/observability.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/parsers/__init__.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/parsers/common.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/parsers/freelancers.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/parsers/users.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/proxy.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/py.typed +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/resilience.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/robots.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/state.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/sync.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru/transport.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru_parser.egg-info/dependency_links.txt +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru_parser.egg-info/entry_points.txt +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/src/flru_parser.egg-info/top_level.txt +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/fixtures/freelancers.html +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/fixtures/project.html +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/fixtures/projects.html +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/fixtures/user.html +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_api_extensions.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_client.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_diagnostics_and_common.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_generic_page.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_properties.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_robots_cookies.py +0 -0
- {flru_parser-0.3.1 → flru_parser-0.3.2}/tests/test_transport_retries.py +0 -0
|
@@ -4,6 +4,17 @@ All notable changes are documented here. The project follows [Semantic Versionin
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.3.2] - 2026-07-26
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
|
|
11
|
+
- Parse current FL.ru catalog cards as complete cards instead of mistaking title elements
|
|
12
|
+
for card containers, restoring descriptions, budgets, kinds, and other card fields.
|
|
13
|
+
- Follow FL.ru's `/projects/page-N/` pagination so `pages="all"` traverses the catalog.
|
|
14
|
+
- Default the high-level projects APIs to orders (`kind=1`), preventing vacancies from
|
|
15
|
+
appearing among orders unless explicitly requested through `types`.
|
|
16
|
+
- Keep package and parser metadata versions synchronized.
|
|
17
|
+
|
|
7
18
|
## [0.3.1] - 2026-07-26
|
|
8
19
|
|
|
9
20
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flru-parser
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Simple, typed and resilient async/sync parser client for public FL.ru pages
|
|
5
5
|
Author: Andrey
|
|
6
6
|
License-Expression: MIT
|
|
@@ -27,6 +27,7 @@ Requires-Dist: beautifulsoup4<5,>=4.12
|
|
|
27
27
|
Requires-Dist: httpx[http2]<1,>=0.28
|
|
28
28
|
Requires-Dist: lxml<7,>=5
|
|
29
29
|
Requires-Dist: pydantic<3,>=2.10
|
|
30
|
+
Requires-Dist: tzdata>=2025.2
|
|
30
31
|
Provides-Extra: postgres
|
|
31
32
|
Requires-Dist: asyncpg<1,>=0.30; extra == "postgres"
|
|
32
33
|
Provides-Extra: redis
|
|
@@ -43,9 +44,8 @@ Dynamic: license-file
|
|
|
43
44
|
[](https://pypi.org/project/flru-parser/)
|
|
44
45
|
[](LICENSE)
|
|
45
46
|
[](src/flru/py.typed)
|
|
46
|
-
[](https://github.com/N3ffus/flru-parser/actions/workflows/ci.yml)
|
|
47
|
+
[](https://github.com/N3ffus/flru-parser/actions/workflows/ci.yml?query=branch%3Amain)
|
|
48
|
+
[](https://codecov.io/gh/N3ffus/flru-parser/branch/main)
|
|
49
49
|
|
|
50
50
|
A typed, resilient, read-only Python client for public pages on [FL.ru](https://www.fl.ru/).
|
|
51
51
|
The default API is intentionally small; production controls remain available when needed.
|
|
@@ -370,13 +370,7 @@ Install `flru-parser[observability]` before using these adapters.
|
|
|
370
370
|
|
|
371
371
|
## Quality
|
|
372
372
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
```text
|
|
376
|
-
47 passed, 1 optional Hypothesis suite skipped locally
|
|
377
|
-
Branch coverage: 85.76%
|
|
378
|
-
Coverage gate: 85%
|
|
379
|
-
```
|
|
373
|
+
The CI and Codecov badges above always show the current `main` result.
|
|
380
374
|
|
|
381
375
|
Run the complete local checks:
|
|
382
376
|
|
|
@@ -404,8 +398,8 @@ The release workflow uses PyPI Trusted Publishing and checks that the Git tag ma
|
|
|
404
398
|
|
|
405
399
|
```bash
|
|
406
400
|
uv run python scripts/configure_project.py YOUR_GITHUB_USERNAME
|
|
407
|
-
git tag -a v0.3.
|
|
408
|
-
git push origin main v0.3.
|
|
401
|
+
git tag -a v0.3.2 -m "flru-parser 0.3.2"
|
|
402
|
+
git push origin main v0.3.2
|
|
409
403
|
```
|
|
410
404
|
|
|
411
405
|
See [docs/PUBLISHING.md](docs/PUBLISHING.md) for the one-time PyPI and GitHub setup.
|
|
@@ -4,9 +4,8 @@
|
|
|
4
4
|
[](https://pypi.org/project/flru-parser/)
|
|
5
5
|
[](LICENSE)
|
|
6
6
|
[](src/flru/py.typed)
|
|
7
|
-
[](https://github.com/N3ffus/flru-parser/actions/workflows/ci.yml)
|
|
7
|
+
[](https://github.com/N3ffus/flru-parser/actions/workflows/ci.yml?query=branch%3Amain)
|
|
8
|
+
[](https://codecov.io/gh/N3ffus/flru-parser/branch/main)
|
|
10
9
|
|
|
11
10
|
A typed, resilient, read-only Python client for public pages on [FL.ru](https://www.fl.ru/).
|
|
12
11
|
The default API is intentionally small; production controls remain available when needed.
|
|
@@ -331,13 +330,7 @@ Install `flru-parser[observability]` before using these adapters.
|
|
|
331
330
|
|
|
332
331
|
## Quality
|
|
333
332
|
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
```text
|
|
337
|
-
47 passed, 1 optional Hypothesis suite skipped locally
|
|
338
|
-
Branch coverage: 85.76%
|
|
339
|
-
Coverage gate: 85%
|
|
340
|
-
```
|
|
333
|
+
The CI and Codecov badges above always show the current `main` result.
|
|
341
334
|
|
|
342
335
|
Run the complete local checks:
|
|
343
336
|
|
|
@@ -365,8 +358,8 @@ The release workflow uses PyPI Trusted Publishing and checks that the Git tag ma
|
|
|
365
358
|
|
|
366
359
|
```bash
|
|
367
360
|
uv run python scripts/configure_project.py YOUR_GITHUB_USERNAME
|
|
368
|
-
git tag -a v0.3.
|
|
369
|
-
git push origin main v0.3.
|
|
361
|
+
git tag -a v0.3.2 -m "flru-parser 0.3.2"
|
|
362
|
+
git push origin main v0.3.2
|
|
370
363
|
```
|
|
371
364
|
|
|
372
365
|
See [docs/PUBLISHING.md](docs/PUBLISHING.md) for the one-time PyPI and GitHub setup.
|
|
@@ -34,7 +34,7 @@ uv pip install \
|
|
|
34
34
|
--python /tmp/flru-test/bin/python \
|
|
35
35
|
--index-url https://test.pypi.org/simple/ \
|
|
36
36
|
--extra-index-url https://pypi.org/simple/ \
|
|
37
|
-
flru-parser==0.3.
|
|
37
|
+
flru-parser==0.3.2
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
On Windows, use `/tmp/flru-test/Scripts/python.exe` instead.
|
|
@@ -47,8 +47,8 @@ On Windows, use `/tmp/flru-test/Scripts/python.exe` instead.
|
|
|
47
47
|
4. Commit, create an annotated tag, and push it:
|
|
48
48
|
|
|
49
49
|
```bash
|
|
50
|
-
git tag -a v0.3.
|
|
51
|
-
git push origin main v0.3.
|
|
50
|
+
git tag -a v0.3.2 -m "flru-parser 0.3.2"
|
|
51
|
+
git push origin main v0.3.2
|
|
52
52
|
```
|
|
53
53
|
|
|
54
54
|
The workflow builds distributions once, validates them, and publishes the exact same artifacts through PyPI Trusted Publishing.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "flru-parser"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "Simple, typed and resilient async/sync parser client for public FL.ru pages"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -17,6 +17,7 @@ dependencies = [
|
|
|
17
17
|
"httpx[http2]>=0.28,<1",
|
|
18
18
|
"lxml>=5,<7",
|
|
19
19
|
"pydantic>=2.10,<3",
|
|
20
|
+
"tzdata>=2025.2",
|
|
20
21
|
]
|
|
21
22
|
classifiers = [
|
|
22
23
|
"Development Status :: 4 - Beta",
|
|
@@ -22,7 +22,7 @@ from .exceptions import (
|
|
|
22
22
|
RobotsDeniedError,
|
|
23
23
|
SelectorDriftError,
|
|
24
24
|
)
|
|
25
|
-
from .filters import ProjectFilters
|
|
25
|
+
from .filters import ProjectFilters, ProjectType
|
|
26
26
|
from .models import (
|
|
27
27
|
Category,
|
|
28
28
|
CrawlCheckpoint,
|
|
@@ -31,6 +31,7 @@ from .models import (
|
|
|
31
31
|
PageData,
|
|
32
32
|
PortfolioItem,
|
|
33
33
|
ProjectDetail,
|
|
34
|
+
ProjectKind,
|
|
34
35
|
ProjectPage,
|
|
35
36
|
ProjectSummary,
|
|
36
37
|
RequestMetrics,
|
|
@@ -199,6 +200,14 @@ class FLClient:
|
|
|
199
200
|
timezone_name=self.config.parser_timezone,
|
|
200
201
|
)
|
|
201
202
|
self._validate_project_page(result)
|
|
203
|
+
if filters and filters.project_types:
|
|
204
|
+
kind_map = {
|
|
205
|
+
ProjectType.ORDER: ProjectKind.ORDER,
|
|
206
|
+
ProjectType.VACANCY: ProjectKind.VACANCY,
|
|
207
|
+
ProjectType.CONTEST: ProjectKind.CONTEST,
|
|
208
|
+
}
|
|
209
|
+
requested = {kind_map[item] for item in filters.project_types}
|
|
210
|
+
result.items = [item for item in result.items if item.kind in requested]
|
|
202
211
|
return result
|
|
203
212
|
except ParseError as exc:
|
|
204
213
|
await self._record_parse_failure(effective_url, html, exc)
|
|
@@ -335,6 +344,16 @@ class FLClient:
|
|
|
335
344
|
fail_fast: bool = True,
|
|
336
345
|
) -> AsyncIterator[ProjectSummary]:
|
|
337
346
|
seen: set[int] = set()
|
|
347
|
+
allowed_kinds = {
|
|
348
|
+
ProjectType.ORDER: ProjectKind.ORDER,
|
|
349
|
+
ProjectType.VACANCY: ProjectKind.VACANCY,
|
|
350
|
+
ProjectType.CONTEST: ProjectKind.CONTEST,
|
|
351
|
+
}
|
|
352
|
+
requested_kinds = (
|
|
353
|
+
{allowed_kinds[item] for item in filters.project_types}
|
|
354
|
+
if filters and filters.project_types
|
|
355
|
+
else set()
|
|
356
|
+
)
|
|
338
357
|
async for page in self.iter_project_pages(
|
|
339
358
|
start_page=start_page,
|
|
340
359
|
max_pages=max_pages,
|
|
@@ -345,6 +364,11 @@ class FLClient:
|
|
|
345
364
|
fail_fast=fail_fast,
|
|
346
365
|
):
|
|
347
366
|
for item in page.items:
|
|
367
|
+
# FL.ru can include pinned cards of another kind even when a
|
|
368
|
+
# `kind` filter is present, so enforce the requested kinds
|
|
369
|
+
# locally as well.
|
|
370
|
+
if requested_kinds and item.kind not in requested_kinds:
|
|
371
|
+
continue
|
|
348
372
|
if deduplicate and item.id in seen:
|
|
349
373
|
continue
|
|
350
374
|
seen.add(item.id)
|
|
@@ -444,12 +444,18 @@ def _project_filters(
|
|
|
444
444
|
types: ProjectTypeInput,
|
|
445
445
|
with_budget: bool | None,
|
|
446
446
|
) -> ProjectFilters:
|
|
447
|
+
# The public "projects" API represents FL.ru orders. FL.ru's unfiltered
|
|
448
|
+
# catalog also contains vacancies and contests, so request orders unless
|
|
449
|
+
# callers explicitly select one or more kinds.
|
|
450
|
+
project_types = _normalize_project_types(types)
|
|
451
|
+
if types is None:
|
|
452
|
+
project_types = frozenset({ProjectType.ORDER})
|
|
447
453
|
return ProjectFilters(
|
|
448
454
|
query=query,
|
|
449
455
|
category=category,
|
|
450
456
|
budget_from=min_budget,
|
|
451
457
|
budget_to=max_budget,
|
|
452
|
-
project_types=
|
|
458
|
+
project_types=project_types,
|
|
453
459
|
only_with_budget=with_budget,
|
|
454
460
|
)
|
|
455
461
|
|
|
@@ -39,7 +39,7 @@ from .common import (
|
|
|
39
39
|
|
|
40
40
|
TITLE_SELECTORS = ("h1", '[itemprop="name"]', ".b-page__title", ".project-title")
|
|
41
41
|
DESCRIPTION_SELECTORS = (
|
|
42
|
-
'[itemprop="description"]', ".b-layout__txt_padbot_20", ".b-post__txt",
|
|
42
|
+
'[itemprop="description"]', ".b-layout__txt_padbot_20", ".b-post__txt.text-5",
|
|
43
43
|
".project-description", '[class*="description"]',
|
|
44
44
|
)
|
|
45
45
|
|
|
@@ -48,8 +48,12 @@ def _find_card(anchor: Tag) -> Tag:
|
|
|
48
48
|
for parent in anchor.parents:
|
|
49
49
|
if not isinstance(parent, Tag):
|
|
50
50
|
continue
|
|
51
|
-
classes =
|
|
52
|
-
if
|
|
51
|
+
classes = set(parent.get("class", []))
|
|
52
|
+
if (
|
|
53
|
+
parent.name == "article"
|
|
54
|
+
or classes.intersection({"b-post", "project-card", "project", "catalog-item"})
|
|
55
|
+
or str(parent.get("data-id", "")).startswith("qa-lenta-")
|
|
56
|
+
):
|
|
53
57
|
return parent
|
|
54
58
|
if parent.name in {"main", "body"}:
|
|
55
59
|
break
|
|
@@ -124,7 +128,10 @@ def _next_page_url(soup: BeautifulSoup, url: str, page: int) -> str | None:
|
|
|
124
128
|
return absolute_url(url, str(explicit.get("href")))
|
|
125
129
|
for anchor in soup.select("a[href]"):
|
|
126
130
|
href = absolute_url(url, str(anchor.get("href")))
|
|
127
|
-
if href and
|
|
131
|
+
if href and (
|
|
132
|
+
parse_qs(urlsplit(href).query).get("page") == [str(page + 1)]
|
|
133
|
+
or re.search(rf"/page-{page + 1}/?$", urlsplit(href).path)
|
|
134
|
+
):
|
|
128
135
|
return href
|
|
129
136
|
return None
|
|
130
137
|
|
|
@@ -33,9 +33,7 @@ def redact_url(url: str | None) -> str | None:
|
|
|
33
33
|
parts = urlsplit(url)
|
|
34
34
|
hostname = parts.hostname or ""
|
|
35
35
|
port = f":{parts.port}" if parts.port else ""
|
|
36
|
-
userinfo = ""
|
|
37
|
-
if parts.username:
|
|
38
|
-
userinfo = f"{parts.username}:***@"
|
|
36
|
+
userinfo = "***@" if parts.username else ""
|
|
39
37
|
netloc = f"{userinfo}{hostname}{port}"
|
|
40
38
|
query = _SECRET_QUERY_RE.sub(lambda match: f"{match.group(1)}=***", parts.query)
|
|
41
39
|
return urlunsplit(SplitResult(parts.scheme, netloc, parts.path, query, parts.fragment))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flru-parser
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Simple, typed and resilient async/sync parser client for public FL.ru pages
|
|
5
5
|
Author: Andrey
|
|
6
6
|
License-Expression: MIT
|
|
@@ -27,6 +27,7 @@ Requires-Dist: beautifulsoup4<5,>=4.12
|
|
|
27
27
|
Requires-Dist: httpx[http2]<1,>=0.28
|
|
28
28
|
Requires-Dist: lxml<7,>=5
|
|
29
29
|
Requires-Dist: pydantic<3,>=2.10
|
|
30
|
+
Requires-Dist: tzdata>=2025.2
|
|
30
31
|
Provides-Extra: postgres
|
|
31
32
|
Requires-Dist: asyncpg<1,>=0.30; extra == "postgres"
|
|
32
33
|
Provides-Extra: redis
|
|
@@ -43,9 +44,8 @@ Dynamic: license-file
|
|
|
43
44
|
[](https://pypi.org/project/flru-parser/)
|
|
44
45
|
[](LICENSE)
|
|
45
46
|
[](src/flru/py.typed)
|
|
46
|
-
[](https://github.com/N3ffus/flru-parser/actions/workflows/ci.yml)
|
|
47
|
+
[](https://github.com/N3ffus/flru-parser/actions/workflows/ci.yml?query=branch%3Amain)
|
|
48
|
+
[](https://codecov.io/gh/N3ffus/flru-parser/branch/main)
|
|
49
49
|
|
|
50
50
|
A typed, resilient, read-only Python client for public pages on [FL.ru](https://www.fl.ru/).
|
|
51
51
|
The default API is intentionally small; production controls remain available when needed.
|
|
@@ -370,13 +370,7 @@ Install `flru-parser[observability]` before using these adapters.
|
|
|
370
370
|
|
|
371
371
|
## Quality
|
|
372
372
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
```text
|
|
376
|
-
47 passed, 1 optional Hypothesis suite skipped locally
|
|
377
|
-
Branch coverage: 85.76%
|
|
378
|
-
Coverage gate: 85%
|
|
379
|
-
```
|
|
373
|
+
The CI and Codecov badges above always show the current `main` result.
|
|
380
374
|
|
|
381
375
|
Run the complete local checks:
|
|
382
376
|
|
|
@@ -404,8 +398,8 @@ The release workflow uses PyPI Trusted Publishing and checks that the Git tag ma
|
|
|
404
398
|
|
|
405
399
|
```bash
|
|
406
400
|
uv run python scripts/configure_project.py YOUR_GITHUB_USERNAME
|
|
407
|
-
git tag -a v0.3.
|
|
408
|
-
git push origin main v0.3.
|
|
401
|
+
git tag -a v0.3.2 -m "flru-parser 0.3.2"
|
|
402
|
+
git push origin main v0.3.2
|
|
409
403
|
```
|
|
410
404
|
|
|
411
405
|
See [docs/PUBLISHING.md](docs/PUBLISHING.md) for the one-time PyPI and GitHub setup.
|
|
@@ -53,6 +53,7 @@ src/flru_parser.egg-info/requires.txt
|
|
|
53
53
|
src/flru_parser.egg-info/top_level.txt
|
|
54
54
|
tests/test_api_extensions.py
|
|
55
55
|
tests/test_client.py
|
|
56
|
+
tests/test_client_easy_edges.py
|
|
56
57
|
tests/test_diagnostics_and_common.py
|
|
57
58
|
tests/test_easy_api.py
|
|
58
59
|
tests/test_freelancers.py
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from unittest.mock import AsyncMock
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from flru import ClientConfig, FLClient, ProjectFilters, ProjectKind, ProjectSummary, ProjectType
|
|
11
|
+
from flru.easy import (
|
|
12
|
+
Client,
|
|
13
|
+
_normalize_cookies,
|
|
14
|
+
_normalize_page_count,
|
|
15
|
+
_normalize_project_types,
|
|
16
|
+
_normalize_proxies,
|
|
17
|
+
_project_filters,
|
|
18
|
+
_state_store,
|
|
19
|
+
_validate_start_page,
|
|
20
|
+
)
|
|
21
|
+
from flru.exceptions import BlockedError, EmptyPageError, SelectorDriftError
|
|
22
|
+
from flru.models import ParseDiagnostics, ProjectPage
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _page(*, page: int = 1, has_next: bool = False, items: list[ProjectSummary] | None = None) -> ProjectPage:
|
|
26
|
+
return ProjectPage(
|
|
27
|
+
page=page,
|
|
28
|
+
url="https://www.fl.ru/projects/",
|
|
29
|
+
items=items or [],
|
|
30
|
+
has_next=has_next,
|
|
31
|
+
diagnostics=ParseDiagnostics(page_fingerprint="test"),
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_easy_normalizers_and_validation(tmp_path: Path) -> None:
|
|
36
|
+
assert _normalize_proxies(None) == ()
|
|
37
|
+
assert _normalize_proxies("http://proxy") == ("http://proxy",)
|
|
38
|
+
assert _normalize_proxies(["a", "b"]) == ("a", "b")
|
|
39
|
+
assert _normalize_cookies(None) == {}
|
|
40
|
+
assert _normalize_cookies({"session": "value"}) == {"session": "value"}
|
|
41
|
+
assert _normalize_page_count("all") is None
|
|
42
|
+
assert _normalize_page_count(2) == 2
|
|
43
|
+
for value in (0, True, "two"):
|
|
44
|
+
with pytest.raises(ValueError):
|
|
45
|
+
_normalize_page_count(value) # type: ignore[arg-type]
|
|
46
|
+
for value in (0, True):
|
|
47
|
+
with pytest.raises(ValueError):
|
|
48
|
+
_validate_start_page(value)
|
|
49
|
+
assert _normalize_project_types(["order", "ВАКАНСИЯ"])
|
|
50
|
+
default_filters = _project_filters(
|
|
51
|
+
query=None,
|
|
52
|
+
category=None,
|
|
53
|
+
min_budget=None,
|
|
54
|
+
max_budget=None,
|
|
55
|
+
types=None,
|
|
56
|
+
with_budget=None,
|
|
57
|
+
)
|
|
58
|
+
assert default_filters.to_params()["kind"] == ["1"]
|
|
59
|
+
with pytest.raises(ValueError, match="unknown project type"):
|
|
60
|
+
_normalize_project_types("other")
|
|
61
|
+
store, owned = _state_store(tmp_path / "state.db")
|
|
62
|
+
assert owned
|
|
63
|
+
assert store is not None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_client_url_and_response_helpers() -> None:
|
|
67
|
+
client = FLClient(ClientConfig())
|
|
68
|
+
assert client._absolute("projects/") == "https://www.fl.ru/projects/"
|
|
69
|
+
assert client._projects_url(None).endswith("/projects/")
|
|
70
|
+
assert client._projects_url("projects/category/dev").endswith("/projects/category/dev/")
|
|
71
|
+
assert client._projects_url("https://example.test/x") == "https://example.test/x"
|
|
72
|
+
assert client._freelancers_url(None, 2).endswith("/freelancers/page-2/")
|
|
73
|
+
assert client._freelancers_url("https://example.test/team", 2).endswith("/team/page-2/")
|
|
74
|
+
assert client._project_url(12).endswith("/projects/12/")
|
|
75
|
+
assert client._username("https://www.fl.ru/users/name/") == "name"
|
|
76
|
+
with pytest.raises(ValueError):
|
|
77
|
+
client._username("https://www.fl.ru/no-user/")
|
|
78
|
+
assert client._page_number("https://x/?page=4") == 4
|
|
79
|
+
assert client._page_number("https://x/page-3/") == 3
|
|
80
|
+
assert client._page_number(None) is None
|
|
81
|
+
assert client._endpoint("https://x/projects/1") == "projects"
|
|
82
|
+
assert client._endpoint("https://x/") == "root"
|
|
83
|
+
assert client._url_with_params("https://x", {"a": [1, 2]}) == "https://x?a=1&a=2"
|
|
84
|
+
response = httpx.Response(200, headers={"Date": "invalid"})
|
|
85
|
+
assert FLClient._response_datetime(response).tzinfo is not None
|
|
86
|
+
dated = httpx.Response(200, headers={"Date": "Sat, 25 Jul 2026 12:00:00 GMT"})
|
|
87
|
+
assert FLClient._response_datetime(dated) == datetime(2026, 7, 25, 12, tzinfo=timezone.utc)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@pytest.mark.asyncio
|
|
91
|
+
async def test_client_iteration_errors_and_validation() -> None:
|
|
92
|
+
client = FLClient(ClientConfig(respect_robots_txt=False))
|
|
93
|
+
with pytest.raises(ValueError, match="batch_size"):
|
|
94
|
+
_ = [page async for page in client.iter_project_pages(batch_size=0)]
|
|
95
|
+
|
|
96
|
+
client.get_projects_batch_result = AsyncMock(return_value=type("Batch", (), {"failed": [], "successful": []})()) # type: ignore[method-assign]
|
|
97
|
+
assert [page async for page in client.iter_project_pages(max_pages=1)] == []
|
|
98
|
+
|
|
99
|
+
order = ProjectSummary(
|
|
100
|
+
id=1,
|
|
101
|
+
title="order",
|
|
102
|
+
url="https://www.fl.ru/projects/1/",
|
|
103
|
+
kind=ProjectKind.ORDER,
|
|
104
|
+
)
|
|
105
|
+
vacancy = ProjectSummary(
|
|
106
|
+
id=2,
|
|
107
|
+
title="vacancy",
|
|
108
|
+
url="https://www.fl.ru/projects/2/",
|
|
109
|
+
kind=ProjectKind.VACANCY,
|
|
110
|
+
)
|
|
111
|
+
client.iter_project_pages = lambda **_: _async_pages(_page(items=[order, vacancy])) # type: ignore[method-assign]
|
|
112
|
+
filtered = [
|
|
113
|
+
item
|
|
114
|
+
async for item in client.iter_projects(
|
|
115
|
+
filters=ProjectFilters(project_types=frozenset({ProjectType.ORDER}))
|
|
116
|
+
)
|
|
117
|
+
]
|
|
118
|
+
assert filtered == [order]
|
|
119
|
+
|
|
120
|
+
empty = _page()
|
|
121
|
+
with pytest.raises(EmptyPageError):
|
|
122
|
+
client._validate_project_page(empty)
|
|
123
|
+
drift = _page()
|
|
124
|
+
drift.diagnostics.candidate_links_found = 1
|
|
125
|
+
with pytest.raises(SelectorDriftError):
|
|
126
|
+
client._validate_project_page(drift)
|
|
127
|
+
accepted = _page(items=[ProjectSummary(id=1, title="x", url="https://www.fl.ru/projects/1/")])
|
|
128
|
+
client._validate_project_page(accepted)
|
|
129
|
+
await client.close()
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
async def _async_pages(*pages: ProjectPage): # type: ignore[no-untyped-def]
|
|
133
|
+
for page in pages:
|
|
134
|
+
yield page
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@pytest.mark.asyncio
|
|
138
|
+
async def test_simple_client_delegates_single_page_and_details() -> None:
|
|
139
|
+
api = Client(respect_robots_txt=False)
|
|
140
|
+
item = ProjectSummary(id=1, title="x", url="https://www.fl.ru/projects/1/")
|
|
141
|
+
api.get_projects = AsyncMock(return_value=[item]) # type: ignore[method-assign]
|
|
142
|
+
api.get_project_details_result = AsyncMock(return_value=type("Result", (), {"successful": [item], "raise_for_errors": lambda self: None})()) # type: ignore[method-assign]
|
|
143
|
+
assert await api.projects() == [item]
|
|
144
|
+
assert await api.projects(details=True) == [item]
|
|
145
|
+
with pytest.raises(ValueError, match="pages"):
|
|
146
|
+
await api.user("name", pages=0)
|
|
147
|
+
await api.close()
|
|
@@ -84,7 +84,7 @@ async def test_simple_details_stream_user_and_freelancers() -> None:
|
|
|
84
84
|
freelancers = await api.freelancers()
|
|
85
85
|
|
|
86
86
|
assert details and details[0].full_description
|
|
87
|
-
assert len(streamed) ==
|
|
87
|
+
assert len(streamed) == 1
|
|
88
88
|
assert project.id == 5500001
|
|
89
89
|
assert user.projects and user.reviews and user.portfolio
|
|
90
90
|
assert freelancers[0].username == "dev-one"
|
|
@@ -97,7 +97,7 @@ async def test_simple_incremental_and_one_shot() -> None:
|
|
|
97
97
|
first = await api.new_projects(state, pages=2)
|
|
98
98
|
second = await api.new_projects(state, pages=2, stop_after_known=1)
|
|
99
99
|
|
|
100
|
-
assert len(first) ==
|
|
100
|
+
assert len(first) == 1
|
|
101
101
|
assert second == []
|
|
102
102
|
|
|
103
103
|
projects = await fetch_projects(
|
|
@@ -108,7 +108,7 @@ async def test_simple_incremental_and_one_shot() -> None:
|
|
|
108
108
|
"transport": httpx.MockTransport(router),
|
|
109
109
|
}
|
|
110
110
|
)
|
|
111
|
-
assert len(projects) ==
|
|
111
|
+
assert len(projects) == 1
|
|
112
112
|
|
|
113
113
|
|
|
114
114
|
|
|
@@ -154,7 +154,7 @@ def test_sync_simple_api() -> None:
|
|
|
154
154
|
projects = api.projects()
|
|
155
155
|
project = api.project(5500001)
|
|
156
156
|
|
|
157
|
-
assert len(projects) ==
|
|
157
|
+
assert len(projects) == 1
|
|
158
158
|
assert project.id == 5500001
|
|
159
159
|
|
|
160
160
|
|
|
@@ -4,7 +4,7 @@ from flru.parsers import parse_freelancer_list
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
def test_freelancer_list() -> None:
|
|
7
|
-
html = (Path(__file__).parent / "fixtures" / "freelancers.html").read_text()
|
|
7
|
+
html = (Path(__file__).parent / "fixtures" / "freelancers.html").read_text(encoding="utf-8")
|
|
8
8
|
page = parse_freelancer_list(html, "https://www.fl.ru/freelancers/")
|
|
9
9
|
assert len(page.items) == 1
|
|
10
10
|
assert page.has_next is True
|
|
@@ -35,6 +35,38 @@ def test_project_list() -> None:
|
|
|
35
35
|
assert second.budget.amount_max == Decimal("180000")
|
|
36
36
|
|
|
37
37
|
|
|
38
|
+
def test_current_catalog_card_and_path_pagination() -> None:
|
|
39
|
+
html = """
|
|
40
|
+
<html><body><main>
|
|
41
|
+
<div class="b-page__lenta">
|
|
42
|
+
<div class="pt-24 b-page__lenta_item b-post" data-id="qa-lenta-1">
|
|
43
|
+
<div class="b-post__grid">
|
|
44
|
+
<h2 class="text-h5 b-post__title b-post__grid_title">
|
|
45
|
+
<a href="/projects/5515261/video.html">Заменить лицо и голос на видео</a>
|
|
46
|
+
</h2>
|
|
47
|
+
<div class="b-post__price b-post__grid_price">от 30 000 ₽</div>
|
|
48
|
+
<div class="b-post__txt text-5">Полное описание актуальной карточки заказа.</div>
|
|
49
|
+
</div>
|
|
50
|
+
<div class="b-post__foot">Заказ 1 час назад 7 ответов</div>
|
|
51
|
+
</div>
|
|
52
|
+
</div>
|
|
53
|
+
<a href="/projects/page-2/">Следующая →</a>
|
|
54
|
+
</main></body></html>
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
page = parse_project_list(html, "https://www.fl.ru/projects/", page=1)
|
|
58
|
+
|
|
59
|
+
assert len(page.items) == 1
|
|
60
|
+
assert page.next_url == "https://www.fl.ru/projects/page-2/"
|
|
61
|
+
assert page.has_next is True
|
|
62
|
+
project = page.items[0]
|
|
63
|
+
assert project.kind is ProjectKind.ORDER
|
|
64
|
+
assert project.description == "Полное описание актуальной карточки заказа."
|
|
65
|
+
assert project.budget is not None
|
|
66
|
+
assert project.budget.amount_min == Decimal("30000")
|
|
67
|
+
assert project.budget.currency == "RUB"
|
|
68
|
+
|
|
69
|
+
|
|
38
70
|
def test_project_detail() -> None:
|
|
39
71
|
project = parse_project_detail(
|
|
40
72
|
read("project.html"),
|
|
@@ -89,5 +89,5 @@ async def test_proxy_snapshot_redacts_credentials() -> None:
|
|
|
89
89
|
|
|
90
90
|
snapshot = await pool.snapshot()
|
|
91
91
|
|
|
92
|
-
assert snapshot[0].url == "http
|
|
92
|
+
assert snapshot[0].url == "http://***@proxy.example:8080"
|
|
93
93
|
assert "password" not in (snapshot[0].url or "")
|
|
@@ -21,7 +21,7 @@ def config(**kwargs):
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def test_url_security_and_redaction() -> None:
|
|
24
|
-
assert redact_url("http://user:pass@proxy.local:8080/x?token=abc") == "http
|
|
24
|
+
assert redact_url("http://user:pass@proxy.local:8080/x?token=abc") == "http://***@proxy.local:8080/x?token=***"
|
|
25
25
|
assert validate_url("https://st.fl.ru/x", frozenset({"fl.ru"}), True).endswith("/x")
|
|
26
26
|
with pytest.raises(SecurityError):
|
|
27
27
|
validate_url("ftp://www.fl.ru/x", frozenset({"fl.ru"}), True)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|